diff --git a/SharpHDiffPatch.Core/Binary/BinaryExtensions.cs b/SharpHDiffPatch.Core/Binary/BinaryExtensions.cs index bd37813..ed0f3ec 100644 --- a/SharpHDiffPatch.Core/Binary/BinaryExtensions.cs +++ b/SharpHDiffPatch.Core/Binary/BinaryExtensions.cs @@ -1,10 +1,10 @@ -using SharpHDiffPatch.Core.Patch; -using System; +using System; using System.Buffers; using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; +using SharpHDiffPatch.Core.Patch; namespace SharpHDiffPatch.Core.Binary { @@ -178,10 +178,9 @@ public static void GetPathsFromStream(this Stream reader, out string[] outputPat public static unsafe void GetPathsFromBuffer(this Span buffer, out string[] outputPaths, int count) { outputPaths = new string[count]; - int inLen = buffer.Length; int idx = 0, strIdx = 0; -#if (NETSTANDARD2_0 || NET461_OR_GREATER) +#if NETSTANDARD2_0 || NET461_OR_GREATER int len = 0; #endif fixed (byte* inputPtr = &MemoryMarshal.GetReference(buffer)) diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Constants.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Constants.cs index 8989730..6e7a929 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Constants.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Constants.cs @@ -1,561 +1,159 @@ -/* - * Copyright 2001,2004-2005 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +using System; -/* - * This package is based on the work done by Keiron Liddle, Aftex Software - * to whom the Ant project is very grateful for his - * great code. - */ +namespace SharpHDiffPatch.Core.Binary.Compression.BZip2; -namespace SharpCompress.Compressors.BZip2 +/// +/// Defines internal values for both compression and decompression +/// +internal static class BZip2Constants { - /** - * Base class for both the compress and decompress classes. - * Holds common arrays, and static data. - * - * @author Keiron Liddle -*/ - - internal sealed class BZip2Constants + internal enum DecodeState : byte { - public const int baseBlockSize = 100000; - public const int MAX_ALPHA_SIZE = 258; - public const int MAX_CODE_LEN = 23; - public const int RUNA = 0; - public const int RUNB = 1; - public const int N_GROUPS = 6; - public const int G_SIZE = 50; - public const int N_ITERS = 4; - public const int MAX_SELECTORS = (2 + (900000 / G_SIZE)); - public const int NUM_OVERSHOOT_BYTES = 20; - - public static int[] rNums = - { - 619, - 720, - 127, - 481, - 931, - 816, - 813, - 233, - 566, - 247, - 985, - 724, - 205, - 454, - 863, - 491, - 741, - 242, - 949, - 214, - 733, - 859, - 335, - 708, - 621, - 574, - 73, - 654, - 730, - 472, - 419, - 436, - 278, - 496, - 867, - 210, - 399, - 680, - 480, - 51, - 878, - 465, - 811, - 169, - 869, - 675, - 611, - 697, - 867, - 561, - 862, - 687, - 507, - 283, - 482, - 129, - 807, - 591, - 733, - 623, - 150, - 238, - 59, - 379, - 684, - 877, - 625, - 169, - 643, - 105, - 170, - 607, - 520, - 932, - 727, - 476, - 693, - 425, - 174, - 647, - 73, - 122, - 335, - 530, - 442, - 853, - 695, - 249, - 445, - 515, - 909, - 545, - 703, - 919, - 874, - 474, - 882, - 500, - 594, - 612, - 641, - 801, - 220, - 162, - 819, - 984, - 589, - 513, - 495, - 799, - 161, - 604, - 958, - 533, - 221, - 400, - 386, - 867, - 600, - 782, - 382, - 596, - 414, - 171, - 516, - 375, - 682, - 485, - 911, - 276, - 98, - 553, - 163, - 354, - 666, - 933, - 424, - 341, - 533, - 870, - 227, - 730, - 475, - 186, - 263, - 647, - 537, - 686, - 600, - 224, - 469, - 68, - 770, - 919, - 190, - 373, - 294, - 822, - 808, - 206, - 184, - 943, - 795, - 384, - 383, - 461, - 404, - 758, - 839, - 887, - 715, - 67, - 618, - 276, - 204, - 918, - 873, - 777, - 604, - 560, - 951, - 160, - 578, - 722, - 79, - 804, - 96, - 409, - 713, - 940, - 652, - 934, - 970, - 447, - 318, - 353, - 859, - 672, - 112, - 785, - 645, - 863, - 803, - 350, - 139, - 93, - 354, - 99, - 820, - 908, - 609, - 772, - 154, - 274, - 580, - 184, - 79, - 626, - 630, - 742, - 653, - 282, - 762, - 623, - 680, - 81, - 927, - 626, - 789, - 125, - 411, - 521, - 938, - 300, - 821, - 78, - 343, - 175, - 128, - 250, - 170, - 774, - 972, - 275, - 999, - 639, - 495, - 78, - 352, - 126, - 857, - 956, - 358, - 619, - 580, - 124, - 737, - 594, - 701, - 612, - 669, - 112, - 134, - 694, - 363, - 992, - 809, - 743, - 168, - 974, - 944, - 375, - 748, - 52, - 600, - 747, - 642, - 182, - 862, - 81, - 344, - 805, - 988, - 739, - 511, - 655, - 814, - 334, - 249, - 515, - 897, - 955, - 664, - 981, - 649, - 113, - 974, - 459, - 893, - 228, - 433, - 837, - 553, - 268, - 926, - 240, - 102, - 654, - 459, - 51, - 686, - 754, - 806, - 760, - 493, - 403, - 415, - 394, - 687, - 700, - 946, - 670, - 656, - 610, - 738, - 392, - 760, - 799, - 887, - 653, - 978, - 321, - 576, - 617, - 626, - 502, - 894, - 679, - 243, - 440, - 680, - 879, - 194, - 572, - 640, - 724, - 926, - 56, - 204, - 700, - 707, - 151, - 457, - 449, - 797, - 195, - 791, - 558, - 945, - 679, - 297, - 59, - 87, - 824, - 713, - 663, - 412, - 693, - 342, - 606, - 134, - 108, - 571, - 364, - 631, - 212, - 174, - 643, - 304, - 329, - 343, - 97, - 430, - 751, - 497, - 314, - 983, - 374, - 822, - 928, - 140, - 206, - 73, - 263, - 980, - 736, - 876, - 478, - 430, - 305, - 170, - 514, - 364, - 692, - 829, - 82, - 855, - 953, - 676, - 246, - 369, - 970, - 294, - 750, - 807, - 827, - 150, - 790, - 288, - 923, - 804, - 378, - 215, - 828, - 592, - 281, - 565, - 555, - 710, - 82, - 896, - 831, - 547, - 261, - 524, - 462, - 293, - 465, - 502, - 56, - 661, - 821, - 976, - 991, - 658, - 869, - 905, - 758, - 745, - 193, - 768, - 550, - 608, - 933, - 378, - 286, - 215, - 979, - 792, - 961, - 61, - 688, - 793, - 644, - 986, - 403, - 106, - 366, - 905, - 644, - 372, - 567, - 466, - 434, - 645, - 210, - 389, - 550, - 919, - 135, - 780, - 773, - 635, - 389, - 707, - 100, - 626, - 958, - 165, - 504, - 920, - 176, - 193, - 713, - 857, - 265, - 203, - 50, - 668, - 108, - 645, - 990, - 626, - 197, - 510, - 357, - 358, - 850, - 858, - 364, - 936, - 638 - }; + StartBlock, + RandPartA, + RandPartB, + RandPartC, + NoRandPartA, + NoRandPartB, + NoRandPartC } + + /// + /// Random numbers used to randomise repetitive blocks + /// + public static readonly ushort[] RandomNumbers = + [ + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 + ]; + + /// + /// When multiplied by compression parameter (1-9) gives the block size for compression + /// 9 gives the best compression but uses the most memory. + /// + public const int BaseBlockSize = 100000; + + /// + /// Backend constant + /// + public const int MaximumAlphaSize = 258; + + /// + /// Backend constant + /// + public const int MaximumCodeLength = 23; + + /// + /// Backend constant + /// + public const int RunA = 0; + + /// + /// Backend constant + /// + public const int RunB = 1; + + /// + /// Backend constant + /// + public const int GroupCount = 6; + + /// + /// Backend constant + /// + public const int GroupSize = 50; + + /// + /// Backend constant + /// + public const int MaximumSelectors = 2 + 900000 / GroupSize; + + /// + /// Initial Identity MTF constant + /// + public static ReadOnlySpan IdentityMtf => + [ + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255 + ]; } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Crc32.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Crc32.cs new file mode 100644 index 0000000..8517680 --- /dev/null +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Crc32.cs @@ -0,0 +1,421 @@ +using System; +using System.Buffers.Binary; +using System.IO.Hashing; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +#if NET6_0_OR_GREATER +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +// ReSharper disable IdentifierTypo +// ReSharper disable InconsistentNaming +#endif + +namespace SharpHDiffPatch.Core.Binary.Compression.BZip2; + +file static class BZip2Crc32Premul +{ + internal const uint InitialState = uint.MaxValue; + internal const int HashSize = sizeof(uint); + private const uint Polynomial = 0x04C11DB7u; + + // Eight concatenated 256-entry tables. + private static readonly uint[] SCrcLookup = CreateSlicingBy8Lookup(); + +#if NET6_0_OR_GREATER + internal const int PclmulThreshold = 128; + + /* + * P(x) = x^32 + 0x04C11DB7 + * + * Folding constants: + * + * k1 = x^576 mod P = 0x8833794C + * k2 = x^512 mod P = 0xE6228B11 + * k3 = x^192 mod P = 0xC5B9CD4C + * k4 = x^128 mod P = 0xE8A45605 + * k5 = x^96 mod P = 0xF200AA66 + * k6 = x^64 mod P = 0x490D678D + * + * mu = floor(x^64 / P) over GF(2) = 0x104D101DF + * + * Vector128.Create() specifies the low 64-bit lane first. + */ + private static readonly Vector128 SK1K2 = + Vector128.Create(0xE6228B11UL, // k2 + 0x8833794CUL // k1 + ); + + private static readonly Vector128 SK3K4 = + Vector128.Create(0xE8A45605UL, // k4 + 0xC5B9CD4CUL // k3 + ); + + private static readonly Vector128 SFoldConstants = + Vector128.Create(0x04C11DB7UL, // x^32 mod P + 0xF200AA66UL // k5 + ); + + private static readonly Vector128 SReverseBytesMask = + Vector128.Create((byte)15, 14, 13, 12, + 11, 10, 9, 8, + 7, 6, 5, 4, + 3, 2, 1, 0); + + private static readonly Vector128 SLower64Mask = + Vector128.Create(ulong.MaxValue, 0UL); + + private static readonly Vector128 SLow32PerLaneMask = + Vector128.Create(0xFFFFFFFFUL, 0xFFFFFFFFUL); + + private const ulong K6 = 0x490D678DUL; + private const ulong Mu = 0x104D101DFUL; + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static uint UpdatePclmul( + uint crc, + ReadOnlySpan source) + { + // The vector path processes complete 16-byte blocks. + int vectorLength = source.Length & -Vector128.Count; + + ref byte sourceRef = ref MemoryMarshal.GetReference(source); + + Vector128 x1; + int offset; + + if (vectorLength >= 4 * Vector128.Count) + { + // Start four independent 128-bit folding streams. + x1 = LoadReversed(ref sourceRef); + Vector128 x2 = LoadReversed(ref Unsafe.Add(ref sourceRef, 16)); + Vector128 x3 = LoadReversed(ref Unsafe.Add(ref sourceRef, 32)); + Vector128 x4 = LoadReversed(ref Unsafe.Add(ref sourceRef, 48)); + + offset = 64; + + /* + * The CRC occupies the most-significant 32 bits of the + * first reversed 128-bit input block. + */ + Vector128 initialCrc = Sse2.ShiftLeftLogical128BitLane(Vector128.CreateScalar((ulong)crc << 32), 8); + + x1 = Sse2.Xor(x1, initialCrc); + + while (vectorLength - offset >= 64) + { + Vector128 y1 = LoadReversed(ref Unsafe.Add(ref sourceRef, offset)); + Vector128 y2 = LoadReversed(ref Unsafe.Add(ref sourceRef, offset + 16)); + Vector128 y3 = LoadReversed(ref Unsafe.Add(ref sourceRef, offset + 32)); + Vector128 y4 = LoadReversed(ref Unsafe.Add(ref sourceRef, offset + 48)); + + x1 = FoldPolynomialPair(y1, x1, SK1K2); + x2 = FoldPolynomialPair(y2, x2, SK1K2); + x3 = FoldPolynomialPair(y3, x3, SK1K2); + x4 = FoldPolynomialPair(y4, x4, SK1K2); + + offset += 64; + } + + // Merge the four parallel streams. + x1 = FoldPolynomialPair(x2, x1, SK3K4); + x1 = FoldPolynomialPair(x3, x1, SK3K4); + x1 = FoldPolynomialPair(x4, x1, SK3K4); + } + else + { + x1 = LoadReversed(ref sourceRef); + Vector128 initialCrc = Sse2.ShiftLeftLogical128BitLane(Vector128.CreateScalar((ulong)crc << 32), 8); + x1 = Sse2.Xor(x1, initialCrc); + + offset = 16; + } + + // Fold remaining complete 16-byte blocks. + while (vectorLength - offset >= 16) + { + Vector128 next = LoadReversed(ref Unsafe.Add(ref sourceRef, offset)); + + x1 = FoldPolynomialPair(next, x1, SK3K4); + offset += 16; + } + + /* + * Fold 128 bits down toward 64 bits. + * + * SFoldConstants contains: + * low lane: x^32 mod P + * high lane: x^96 mod P + */ + x1 = FoldPolynomialPair(Vector128.Zero, x1, SFoldConstants); + + /* + * Fold 64 bits toward 32 bits: + * + * upper64(x1) * k6 XOR lower64(x1) + */ + Vector128 lower64 = Sse2.And(x1, SLower64Mask); + Vector128 folded64 = Pclmulqdq.CarrylessMultiply(x1, Vector128.CreateScalar(K6), 0x01); + + x1 = Sse2.Xor(folded64, lower64); + + // Barrett reduction from the remaining polynomial to 32 bits. + Vector128 temporary = x1; + + x1 = Sse2.ShiftRightLogical128BitLane(x1, 4); + x1 = Sse2.And(x1, SLow32PerLaneMask); + x1 = Pclmulqdq.CarrylessMultiply(x1, Vector128.CreateScalar(Mu), 0x00); + x1 = Sse2.ShiftRightLogical128BitLane(x1, 4); + x1 = Sse2.And(x1, SLow32PerLaneMask); + x1 = Pclmulqdq.CarrylessMultiply(x1, Vector128.CreateScalar((ulong)Polynomial), 0x00); + x1 = Sse2.Xor(x1, temporary); + uint reducedCrc = x1.AsUInt32().GetElement(0); + + // Process the final 0–15 bytes using the existing implementation. + return UpdateSlicingBy8(reducedCrc, source[vectorLength..]); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadReversed(ref byte source) + { + var value = Unsafe.ReadUnaligned>(ref source); + return Ssse3.Shuffle(value, SReverseBytesMask).AsUInt64(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 FoldPolynomialPair( + Vector128 target, + Vector128 source, + Vector128 constants) + { + /* + * target XOR= + * CLMUL(source.low, constants.low) XOR + * CLMUL(source.high, constants.high) + */ + Vector128 lowerProduct = Pclmulqdq.CarrylessMultiply(source, constants, 0x00); + Vector128 upperProduct = Pclmulqdq.CarrylessMultiply(source, constants, 0x11); + Vector128 products = Sse2.Xor(lowerProduct, upperProduct); + return Sse2.Xor(target, products); + } +#endif + + internal static uint UpdateSlicingBy8( + uint crc, + ReadOnlySpan source) + // The JIT treats this as a platform constant. + => BitConverter.IsLittleEndian + ? UpdateSlicingBy8LittleEndian(crc, source) + : UpdateSlicingBy8Portable(crc, source); + + private static uint UpdateSlicingBy8LittleEndian( + uint crc, + ReadOnlySpan source) + { + ref byte sourceRef = ref MemoryMarshal.GetReference(source); + ref uint tableRef = ref SCrcLookup[0]; + + int offset = 0; + int length = source.Length; + + /* + * Two-block unrolling reduces loop bookkeeping. + * + * The two CRC operations are still dependent, so larger + * unrolling generally provides diminishing returns. + */ + while (length >= 16) + { + ulong block0 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref sourceRef, offset)); + ulong block1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref sourceRef, offset + 8)); + crc = UpdateSlicingBy8Block(crc, block0, ref tableRef); + crc = UpdateSlicingBy8Block(crc, block1, ref tableRef); + + offset += 16; + length -= 16; + } + + if (length >= 8) + { + ulong block = Unsafe.ReadUnaligned(ref Unsafe.Add(ref sourceRef, offset)); + crc = UpdateSlicingBy8Block(crc, block, ref tableRef); + offset += 8; + length -= 8; + } + + // Process the remaining 0–7 bytes. + while (length != 0) + { + byte value = Unsafe.Add(ref sourceRef, offset); + byte index = (byte)((crc >> 24) ^ value); + crc = (crc << 8) ^ Unsafe.Add(ref tableRef, index); + offset++; + length--; + } + + return crc; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint UpdateSlicingBy8Block( + uint crc, + ulong block, + ref uint table) + { + /* + * A little-endian ulong load gives: + * + * bits 0–7 = source[0] + * bits 8–15 = source[1] + * ... + * bits 56–63 = source[7] + * + * ReverseEndianness converts source[0..3] into the + * big-endian uint required by the non-reflected CRC. + */ + uint first = crc ^ BinaryPrimitives.ReverseEndianness((uint)block); + return Unsafe.Add(ref table, 0 * 256 + (byte)(block >> 56)) ^ + Unsafe.Add(ref table, 1 * 256 + (byte)(block >> 48)) ^ + Unsafe.Add(ref table, 2 * 256 + (byte)(block >> 40)) ^ + Unsafe.Add(ref table, 3 * 256 + (byte)(block >> 32)) ^ + Unsafe.Add(ref table, 4 * 256 + (byte)first) ^ + Unsafe.Add(ref table, 5 * 256 + (byte)(first >> 8)) ^ + Unsafe.Add(ref table, 6 * 256 + (byte)(first >> 16)) ^ + Unsafe.Add(ref table, 7 * 256 + (byte)(first >> 24)); + } + + private static uint UpdateSlicingBy8Portable( + uint crc, + ReadOnlySpan source) + { + ref byte sourceRef = ref MemoryMarshal.GetReference(source); + ref uint tableRef = ref SCrcLookup[0]; + + int offset = 0; + int length = source.Length; + + while (length >= 8) + { + uint first = crc ^ + ((uint)Unsafe.Add(ref sourceRef, offset) << 24) ^ + ((uint)Unsafe.Add(ref sourceRef, offset + 1) << 16) ^ + ((uint)Unsafe.Add(ref sourceRef, offset + 2) << 8) ^ + Unsafe.Add(ref sourceRef, offset + 3); + + crc = Unsafe.Add(ref tableRef, 0 * 256 + Unsafe.Add(ref sourceRef, offset + 7)) ^ + Unsafe.Add(ref tableRef, 1 * 256 + Unsafe.Add(ref sourceRef, offset + 6)) ^ + Unsafe.Add(ref tableRef, 2 * 256 + Unsafe.Add(ref sourceRef, offset + 5)) ^ + Unsafe.Add(ref tableRef, 3 * 256 + Unsafe.Add(ref sourceRef, offset + 4)) ^ + Unsafe.Add(ref tableRef, 4 * 256 + (byte)first) ^ + Unsafe.Add(ref tableRef, 5 * 256 + (byte)(first >> 8)) ^ + Unsafe.Add(ref tableRef, 6 * 256 + (byte)(first >> 16)) ^ + Unsafe.Add(ref tableRef, 7 * 256 + (byte)(first >> 24)); + + offset += 8; + length -= 8; + } + + while (length != 0) + { + byte index = (byte)((crc >> 24) ^ Unsafe.Add(ref sourceRef, offset)); + crc = (crc << 8) ^ Unsafe.Add(ref tableRef, index); + offset++; + length--; + } + + return crc; + } + + private static uint[] CreateSlicingBy8Lookup() + { + uint[] table = new uint[8 * 256]; + + // T0: CRC of one byte in the most-significant position. + for (uint value = 0; value < 256; value++) + { + uint crc = value << 24; + for (int bit = 0; bit < 8; bit++) + { + crc = (crc & 0x80000000u) != 0 ? (crc << 1) ^ Polynomial : crc << 1; + } + + table[value] = crc; + } + + /* + * For a non-reflected, left-shifting CRC: + * + * T[n][x] = advance T[n - 1][x] by one zero byte. + */ + for (int slice = 1; slice < 8; slice++) + { + int previousOffset = (slice - 1) * 256; + int currentOffset = slice * 256; + + for (int value = 0; value < 256; value++) + { + uint crc = table[previousOffset + value]; + table[currentOffset + value] = (crc << 8) ^ table[(byte)(crc >> 24)]; + } + } + + return table; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint UpdateByte(uint crc, byte value) + { + ref uint table = ref SCrcLookup[0]; + + byte index = (byte)((crc >> 24) ^ value); + return (crc << 8) ^ Unsafe.Add(ref table, index); + } +} + +public sealed class BZip2Crc32 : NonCryptographicHashAlgorithm +{ + private uint _crc = BZip2Crc32Premul.InitialState; + + public BZip2Crc32() + : base(BZip2Crc32Premul.HashSize) + { + } + + private BZip2Crc32(uint crc) + : base(BZip2Crc32Premul.HashSize) + { + _crc = crc; + } + + public BZip2Crc32 Clone() => new(_crc); + + public void AppendByte(byte value) => _crc = BZip2Crc32Premul.UpdateByte(_crc, value); + + public override void Append(ReadOnlySpan source) => _crc = Update(_crc, source); + + public override void Reset() => _crc = BZip2Crc32Premul.InitialState; + + protected override void GetCurrentHashCore(Span destination) => BinaryPrimitives.WriteUInt32BigEndian(destination, ~_crc); + + protected override void GetHashAndResetCore(Span destination) + { + BinaryPrimitives.WriteUInt32BigEndian(destination, ~_crc); + _crc = BZip2Crc32Premul.InitialState; + } + + public uint GetCurrentHashAsUInt32() => ~_crc; + + private static uint Update(uint crc, ReadOnlySpan source) + { +#if NET6_0_OR_GREATER + if (BitConverter.IsLittleEndian && + Pclmulqdq.IsSupported && + Ssse3.IsSupported && + source.Length >= BZip2Crc32Premul.PclmulThreshold) + { + return BZip2Crc32Premul.UpdatePclmul(crc, source); + } +#endif + + return BZip2Crc32Premul.UpdateSlicingBy8(crc, source); + } +} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Exception.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Exception.cs new file mode 100644 index 0000000..f90bb15 --- /dev/null +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2Exception.cs @@ -0,0 +1,53 @@ +using System; +using System.Runtime.Serialization; + +namespace SharpHDiffPatch.Core.Binary.Compression.BZip2; + +/// +/// BZip2Exception represents exceptions specific to BZip2 classes and code. +/// +[Serializable] +public class BZip2Exception : Exception +{ + /// + /// Initialise a new instance of . + /// + public BZip2Exception() + { + } + + /// + /// Initialise a new instance of with its message string. + /// + /// A that describes the error. + public BZip2Exception(string message) + : base(message) + { + } + + /// + /// Initialise a new instance of . + /// + /// A that describes the error. + /// The that caused this exception. + public BZip2Exception(string message, Exception innerException) + : base(message, innerException) + { + } + + /// + /// Initializes a new instance of the BZip2Exception class with serialized data. + /// + /// + /// The System.Runtime.Serialization.SerializationInfo that holds the serialized + /// object data about the exception being thrown. + /// + /// + /// The System.Runtime.Serialization.StreamingContext that contains contextual information + /// about the source or destination. + /// + protected BZip2Exception(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } +} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2InputStream.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2InputStream.cs new file mode 100644 index 0000000..c4a94ce --- /dev/null +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/BZip2InputStream.cs @@ -0,0 +1,1317 @@ +using System; +using System.Buffers; +using System.IO; +using System.Numerics; +using System.Runtime.CompilerServices; + +// ReSharper disable CommentTypo + +namespace SharpHDiffPatch.Core.Binary.Compression.BZip2; + +/// +/// An input stream that decompresses files in the BZip2 format. +/// +public sealed class BZip2InputStream : Stream +{ + private const int InputBufferSize = 32 << 10; + private const int MaximumHuffmanLength = 20; + private const int AlphaTableStride = BZip2Constants.MaximumAlphaSize; + private const int CodeTableStride = BZip2Constants.MaximumCodeLength; + + private readonly bool _decompressConcatenated; + private readonly bool _leaveOpen; + private readonly Stream _baseStream; + + // Buffer compressed input. The original implementation called Stream.ReadByte() + // for every compressed byte, which is particularly expensive for unbuffered streams. + private readonly byte[] _inputBuffer; + private int _inputOffset; + private int _inputCount; + + private int _last; + private int _blockSize100K; + private bool _blockRandomised; + + private uint _bsBuff; + private int _bsLive; + + private readonly BZip2Crc32 _nCrc32 = new(); + + private readonly byte[] _seqToUnseq; + private byte[] _selector; + + // Reused per block. This removes the allocations previously made by + // ReceiveDecodingTables() and GetAndMoveToFrontDecode(). + private byte[] _yy; + private readonly byte[] _codeLengths; + + private int[] _tt = []; + private byte[] _ll8 = []; + private ushort[] _perm; + + private readonly int[] _unZfTab; + private readonly int[] _limit = new int[BZip2Constants.GroupCount * CodeTableStride]; + private readonly int[] _baseArray = new int[BZip2Constants.GroupCount * CodeTableStride]; + private readonly byte[] _minLens = new byte[BZip2Constants.GroupCount]; + + private bool _streamEnd; + private bool _blockEndPending; + private int _currentChar = -1; + + private BZip2Constants.DecodeState _currentState = BZip2Constants.DecodeState.StartBlock; + + private uint _storedBlockCrc; + private uint _computedCombinedCrc; + + private byte _count; + private ushort _chPrev; + private ushort _ch2; + private int _tPos; + private ushort _rNToGo; + private ushort _rTPos; + private int _i2; + private byte _j2; + private byte _z; + + private readonly int _inputBufferCapacity; + private bool _disposed; + + private static long TryGetStreamLength(Stream stream) + { + try + { + return stream.Length; + } + catch + { + return -1; + } + } + + public BZip2InputStream(Stream stream, bool decompressConcatenated, bool leaveOpen = false) + { + if (stream == null!) + { + throw new ArgumentNullException(nameof(stream)); + } + + _baseStream = stream; + _decompressConcatenated = decompressConcatenated; + _leaveOpen = leaveOpen; + + try + { + long streamLength = TryGetStreamLength(stream); + _inputBufferCapacity = + streamLength is > 0 and < InputBufferSize + ? (int)streamLength + : InputBufferSize; + + _yy = ArrayPool.Shared.Rent(256); + _seqToUnseq = ArrayPool.Shared.Rent(256); + _unZfTab = ArrayPool.Shared.Rent(256); + _inputBuffer = ArrayPool.Shared.Rent(_inputBufferCapacity); + _selector = ArrayPool.Shared.Rent(BZip2Constants.MaximumSelectors); + _codeLengths = ArrayPool.Shared.Rent(BZip2Constants.GroupCount * AlphaTableStride); + _perm = ArrayPool.Shared.Rent(BZip2Constants.GroupCount * AlphaTableStride); + + Initialize(); + + if (_streamEnd) + { + return; + } + + int origPtr = InitBlock(); + if (!_streamEnd) + { + SetupBlock(origPtr); + } + } + catch + { + ReturnPooledBuffers(); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (_disposed) + { + base.Dispose(disposing); + return; + } + + _disposed = true; + + ReturnPooledBuffers(); + + if (disposing && IsStreamOwner && !_leaveOpen) + { + _baseStream.Dispose(); + } + + base.Dispose(disposing); + } + + private void ReturnPooledBuffers() + { + if (_yy.Length != 0) ArrayPool.Shared.Return(_yy); + if (_inputBuffer.Length != 0) ArrayPool.Shared.Return(_inputBuffer); + if (_selector.Length != 0) ArrayPool.Shared.Return(_selector); + if (_codeLengths.Length != 0) ArrayPool.Shared.Return(_codeLengths); + if (_perm.Length != 0) ArrayPool.Shared.Return(_perm); + if (_ll8.Length != 0) ArrayPool.Shared.Return(_ll8); + if (_tt.Length != 0) ArrayPool.Shared.Return(_tt); + if (_unZfTab.Length != 0) ArrayPool.Shared.Return(_unZfTab); + if (_seqToUnseq.Length != 0) ArrayPool.Shared.Return(_seqToUnseq); + } + + public bool IsStreamOwner { get; set; } = true; + + public override bool CanRead => _baseStream.CanRead; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => _baseStream.Length; + + public override long Position + { + get => throw new NotSupportedException("Cannot get the position of the compressed data"); + set => throw new NotSupportedException("BZip2InputStream position cannot be set"); + } + + public override void Flush() => _baseStream.Flush(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException("BZip2InputStream Seek not supported"); + + public override void SetLength(long value) => throw new NotSupportedException("BZip2InputStream SetLength not supported"); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException("BZip2InputStream Write not supported"); + + public override void WriteByte(byte value) => throw new NotSupportedException("BZip2InputStream WriteByte not supported"); + + public override int Read(byte[] buffer, int offset, int count) + { + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if ((uint)offset > (uint)buffer.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + return (uint)count > (uint)(buffer.Length - offset) + ? throw new ArgumentOutOfRangeException(nameof(count)) + : ReadCore(buffer.AsSpan(offset, count)); + } + +#if NET6_0_OR_GREATER + public override int Read(Span buffer) => ReadCore(buffer); +#endif + + private int ReadCore(Span destination) + { + int written = 0; + int crcStart = 0; + + // Huffman decoding and inverse-BWT traversal are inherently serial, but the + // second RLE stage frequently exposes byte runs. Emit those runs in bulk and + // feed the CRC one span at a time instead of one byte at a time. + while (written < destination.Length && !_streamEnd) + { + if (_blockEndPending) + { + AppendCrc(destination.Slice(crcStart, written - crcStart)); + crcStart = written; + FinishBlock(); + continue; + } + + if (_currentState is BZip2Constants.DecodeState.NoRandPartC + or BZip2Constants.DecodeState.RandPartC) + { + int available = _z - _j2 + 1; // Includes the already prepared current byte. + int toWrite = Math.Min(available, destination.Length - written); + + destination.Slice(written, toWrite).Fill((byte)_currentChar); + written += toWrite; + + if (toWrite < available) + { + // Leave the next repeated byte prepared for the following Read(). + _j2 += (byte)toWrite; + break; + } + + if (_currentState == BZip2Constants.DecodeState.RandPartC) + { + CompleteRandPartC(); + } + else + { + CompleteNoRandPartC(); + } + + continue; + } + + destination[written++] = (byte)_currentChar; + AdvanceState(); + } + + AppendCrc(destination.Slice(crcStart, written - crcStart)); + + // Match the old implementation's timing: consuming the final byte of a block + // also validates that block and prepares the next one before Read() returns. + if (_blockEndPending) + { + FinishBlock(); + } + + return written; + } + + public override int ReadByte() + { + if (_streamEnd) + { + return -1; + } + + int value = _currentChar; + _nCrc32.AppendByte((byte)value); + AdvanceState(); + + if (_blockEndPending) + { + FinishBlock(); + } + + return value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AppendCrc(ReadOnlySpan source) + { + if (!source.IsEmpty) + { + _nCrc32.Append(source); + } + } + + private void FinishBlock() + { + _blockEndPending = false; + EndBlock(); + + int origPtr = InitBlock(); + if (!_streamEnd) + { + SetupBlock(origPtr); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AdvanceState() + { + switch (_currentState) + { + case BZip2Constants.DecodeState.RandPartB: + SetupRandPartB(); + break; + + case BZip2Constants.DecodeState.RandPartC: + SetupRandPartC(); + break; + + case BZip2Constants.DecodeState.NoRandPartB: + SetupNoRandPartB(); + break; + + case BZip2Constants.DecodeState.NoRandPartC: + SetupNoRandPartC(); + break; + + case BZip2Constants.DecodeState.StartBlock: + case BZip2Constants.DecodeState.RandPartA: + case BZip2Constants.DecodeState.NoRandPartA: + break; + } + } + + private void Initialize() + { + int magic1 = BsR(8); + int magic2 = BsR(8); + int magic3 = BsR(8); + int magic4 = BsR(8); + + if (magic1 != 'B' || + magic2 != 'Z' || + magic3 != 'h' || + magic4 < '1' || + magic4 > '9') + { + _streamEnd = true; + return; + } + + SetDecompressStructureSizes(magic4 - '0'); + _computedCombinedCrc = 0; + } + + private int InitBlock() + { + int origPtr = 0; + while (true) + { + int magic1 = BsR(8); + int magic2 = BsR(8); + int magic3 = BsR(8); + int magic4 = BsR(8); + int magic5 = BsR(8); + int magic6 = BsR(8); + + if (magic1 == 0x17 && magic2 == 0x72 && magic3 == 0x45 && + magic4 == 0x38 && magic5 == 0x50 && magic6 == 0x90) + { + if (Complete()) + { + return origPtr; + } + + // Complete() consumed the next member's BZh header. Continue by + // reading that member's first block marker. + continue; + } + + if (magic1 != 0x31 || magic2 != 0x41 || magic3 != 0x59 || + magic4 != 0x26 || magic5 != 0x53 || magic6 != 0x59) + { + BadBlockHeader(); + } + + _storedBlockCrc = unchecked((uint)BsGetInt32()); + _blockRandomised = BsR1() != 0; + + origPtr = GetAndMoveToFrontDecode(); + + _nCrc32.Reset(); + _currentState = BZip2Constants.DecodeState.StartBlock; + return origPtr; + } + } + + private void EndBlock() + { + uint computedBlockCrc = _nCrc32.GetCurrentHashAsUInt32(); + if (_storedBlockCrc != computedBlockCrc) + { + CrcError(); + } + + _computedCombinedCrc = + BitOperations.RotateLeft(_computedCombinedCrc, 1) ^ + computedBlockCrc; + } + + private bool Complete() + { + if (unchecked((uint)BsGetInt32()) != _computedCombinedCrc) + { + CrcError(); + } + + if (!_decompressConcatenated) + { + // Bulk input may have prefetched data following the first member. Restore + // the underlying position when possible so callers can continue reading it. + RewindUnreadInput(); + _streamEnd = true; + return true; + } + + // Each bzip2 member is padded to the next byte boundary. At this point the + // remaining bits in the cache are only that padding. + _bsBuff = 0; + _bsLive = 0; + + if (TryInitializeNextMember()) return false; + + _streamEnd = true; + return true; + } + + private bool TryInitializeNextMember() + { + if (!TryReadCompressedByte(out int magic1)) + { + return false; + } + + int magic2 = ReadCompressedByte(); + int magic3 = ReadCompressedByte(); + int magic4 = ReadCompressedByte(); + + if (magic1 != 'B' || + magic2 != 'Z' || + magic3 != 'h') + { + throw new BZip2Exception("Unexpected data after a valid BZip2 stream"); + } + + if (magic4 is < '1' or > '9') + { + throw new BZip2Exception("Invalid BZip2 block size"); + } + + SetDecompressStructureSizes(magic4 - '0'); + _computedCombinedCrc = 0; + return true; + } + + private void RewindUnreadInput() + { + int unread = _inputCount - _inputOffset; + if (unread != 0 && _baseStream.CanSeek) + { + _baseStream.Seek(-unread, SeekOrigin.Current); + } + + _inputOffset = 0; + _inputCount = 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void FillBuffer() + { + _bsBuff = (_bsBuff << 8) | (uint)ReadCompressedByte(); + _bsLive += 8; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int ReadCompressedByte() + { + if (!TryReadCompressedByte(out int value)) + { + CompressedStreamEof(); + } + + return value; + } + + private bool TryReadCompressedByte(out int value) + { + if (_inputOffset >= _inputCount) + { + // A non-seekable stream must not be read past the end of the first member + // when concatenation is disabled. Seekable streams are rewound in Complete(). + int requested = _blockSize100K == 0 || + (!_decompressConcatenated && !_baseStream.CanSeek) + ? 1 + : _inputBufferCapacity; + + _inputCount = _baseStream.Read(_inputBuffer, 0, requested); + _inputOffset = 0; + + if (_inputCount <= 0) + { + value = -1; + return false; + } + } + + value = _inputBuffer[_inputOffset++]; + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int BsR1() + { + if (_bsLive == 0) FillBuffer(); + return (int)((_bsBuff >> --_bsLive) & 1u); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int BsR(int bitCount) + { + while (_bsLive < bitCount) + { + FillBuffer(); + } + + int shift = _bsLive - bitCount; + uint mask = (1u << bitCount) - 1u; + int value = (int)((_bsBuff >> shift) & mask); + _bsLive = shift; + return value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int BsGetInt32() => (BsR(16) << 16) | BsR(16); + +#if NET6_0_OR_GREATER + [SkipLocalsInit] +#endif + private void ReceiveDecodingTables(out int nInUse, + out int alphaSize, + out int selectorCount) + { + nInUse = 0; + + // The format stores sixteen group-presence bits followed by sixteen symbol + // bits for every present group. Reading them in 16-bit chunks removes up to + // 272 calls to BsR1() per block. + int inUse16 = BsR(16); + for (int group = 0; group < 16; group++) + { + if ((inUse16 & (0x8000 >> group)) == 0) + { + continue; + } + + int inUseGroup = BsR(16); + int symbolBase = group << 4; + + while (inUseGroup != 0) + { + int bit = BitOperations.LeadingZeroCount((uint)inUseGroup) - 16; + _seqToUnseq[nInUse++] = (byte)(symbolBase + bit); + + inUseGroup &= ~(0x8000 >> bit); + } + } + + int nGroups = BsR(3); + alphaSize = nInUse + 2; + selectorCount = BsR(15); + + if (nGroups < 2 || nGroups > BZip2Constants.GroupCount || + selectorCount < 1 || selectorCount > BZip2Constants.MaximumSelectors) + { + DataError(); + } + + Span selectorPositionInitial = [0, 1, 2, 3, 4, 5]; + + // Decode selector MTF values immediately, avoiding selectorMtf[] entirely. + for (int i = 0; i < selectorCount; i++) + { + int selectorMtf = 0; + while (BsR1() != 0) + { + selectorMtf++; + if (selectorMtf >= nGroups) + { + DataError(); + } + } + + ref byte selectorPosition = ref selectorPositionInitial[0]; + byte selected = Unsafe.Add(ref selectorPosition, selectorMtf); + + while (selectorMtf != 0) + { + Unsafe.Add(ref selectorPosition, selectorMtf) = + Unsafe.Add(ref selectorPosition, selectorMtf - 1); + + selectorMtf--; + } + + selectorPosition = selected; + _selector[i] = selected; + } + + for (int table = 0; table < nGroups; table++) + { + int alphaOffset = table * AlphaTableStride; + int currentLength = BsR(5); + + for (int symbol = 0; symbol < alphaSize; symbol++) + { + while (BsR1() != 0) + { + currentLength += BsR1() == 0 ? 1 : -1; + if ((uint)(currentLength - 1) >= MaximumHuffmanLength) + { + DataError(); + } + } + + if ((uint)(currentLength - 1) >= MaximumHuffmanLength) + { + DataError(); + } + + _codeLengths[alphaOffset + symbol] = (byte)currentLength; + } + } + + for (int table = 0; table < nGroups; table++) + { + int alphaOffset = table * AlphaTableStride; + int minLength = MaximumHuffmanLength; + int maxLength = 0; + + ref byte codeLength = + ref Unsafe.Add(ref _codeLengths[0], alphaOffset); + + int remaining = alphaSize; + + while (remaining != 0) + { + int length = codeLength; + + minLength = Math.Min(minLength, length); + maxLength = Math.Max(maxLength, length); + + codeLength = ref Unsafe.Add(ref codeLength, 1); + remaining--; + } + + HbCreateDecodeTables(table, minLength, maxLength, alphaSize); + + _minLens[table] = (byte)minLength; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int DecodeNextSymbol( + ref int groupNumber, + ref int groupPosition, + int selectorCount, + int alphaSize) + { + if (groupPosition == 0) + { + int nextGroup = groupNumber + 1; + if ((uint)nextGroup >= (uint)selectorCount) + { + DataError(); + } + + groupNumber = nextGroup; + groupPosition = BZip2Constants.GroupSize; + } + + groupPosition--; + + ref byte selectorRef = ref _selector[0]; + int table = Unsafe.Add(ref selectorRef, groupNumber); + + int codeOffset = table * CodeTableStride; + int alphaOffset = table * AlphaTableStride; + + ref int limitRef = ref _limit[0]; + ref int baseRef = ref _baseArray[0]; + ref ushort permRef = ref _perm[0]; + ref byte minRef = ref _minLens[0]; + + int codeLength = Unsafe.Add(ref minRef, table); + int codeBits = BsR(codeLength); + + while (codeBits > Unsafe.Add(ref limitRef, codeOffset + codeLength)) + { + if (codeLength >= MaximumHuffmanLength) + { + DataError(); + } + + codeLength++; + codeBits = (codeBits << 1) | BsR1(); + } + + int permIndex = codeBits - Unsafe.Add(ref baseRef, codeOffset + codeLength); + if ((uint)permIndex >= (uint)alphaSize) + { + DataError(); + } + + return Unsafe.Add(ref permRef, alphaOffset + permIndex); + } + + private int GetAndMoveToFrontDecode() + { + int blockCapacity = BZip2Constants.BaseBlockSize * _blockSize100K; + int origPtr = BsR(24); + + ReceiveDecodingTables(out int nInUse, + out int alphaSize, + out int selectorCount); + + int endOfBlock = nInUse + 1; + int groupNumber = -1; + int groupPosition = 0; + + BZip2Constants.IdentityMtf.CopyTo(_yy); + + _last = -1; + int nextSymbol = DecodeNextSymbol(ref groupNumber, ref groupPosition, selectorCount, alphaSize); + + while (nextSymbol != endOfBlock) + { + if (nextSymbol is BZip2Constants.RunA or BZip2Constants.RunB) + { + int runLength = -1; + int runPower = 1; + + do + { + if (runPower > blockCapacity) + { + BlockOverrun(); + } + + runLength += nextSymbol == BZip2Constants.RunA + ? runPower + : runPower << 1; + + runPower <<= 1; + nextSymbol = DecodeNextSymbol(ref groupNumber, ref groupPosition, selectorCount, alphaSize); + } + while (nextSymbol is BZip2Constants.RunA or BZip2Constants.RunB); + + runLength++; + int runStart = _last + 1; + + if (nInUse == 0 || runLength < 0 || runLength > blockCapacity - runStart) + { + BlockOverrun(); + } + + byte value = _seqToUnseq[_yy[0]]; + _unZfTab[value] += runLength; + + // Span.Fill is lowered to an optimized bulk fill by modern runtimes + // and uses SIMD when profitable. System.Memory supplies it on netstandard2.0. + _ll8.AsSpan(runStart, runLength).Fill(value); + _last += runLength; + continue; + } + + int mtfIndex = nextSymbol - 1; + if ((uint)mtfIndex >= (uint)nInUse) + { + DataError(); + } + + int nextIndex = _last + 1; + if (nextIndex >= blockCapacity) + { + BlockOverrun(); + } + + byte mtfValue = _yy[mtfIndex]; + byte valueAtIndex = _seqToUnseq[mtfValue]; + + _last = nextIndex; + _unZfTab[valueAtIndex]++; + _ll8[nextIndex] = valueAtIndex; + + if (mtfIndex != 0) + { + // Small MTF indexes dominate normal data, and a short scalar move avoids + // the fixed cost of entering the runtime copy helper. Larger moves use + // overlap-safe memmove, whose implementation selects the best CPU path. + if (mtfIndex <= 16) + { + ref byte yy = ref _yy[0]; + + int moveIndex = mtfIndex; + do + { + Unsafe.Add(ref yy, moveIndex) = Unsafe.Add(ref yy, moveIndex - 1); + } + while (--moveIndex != 0); + } + else + { + _yy.AsSpan(0, mtfIndex).CopyTo(_yy.AsSpan(1)); + } + + _yy[0] = mtfValue; + } + + nextSymbol = DecodeNextSymbol(ref groupNumber, ref groupPosition, selectorCount, alphaSize); + } + + return origPtr; + } + +#if NET6_0_OR_GREATER + [SkipLocalsInit] +#endif + private void SetupBlock(int origPtr) + { + int last = _last; + + if (last < 0 || (uint)origPtr > (uint)last) + { + DataError(); + } + + Span cumulativeFrequency = stackalloc int[257]; + + ref int frequency = ref cumulativeFrequency[0]; + ref int counts = ref _unZfTab[0]; + + frequency = 0; + int cumulative = 0; + + for (int symbol = 0; symbol < 256; symbol++) + { + ref int count = ref Unsafe.Add(ref counts, symbol); + + cumulative += count; + Unsafe.Add(ref frequency, symbol + 1) = cumulative; + + count = 0; + } + + int length = last + 1; + if (length >= 256 * 1024) + { + BuildTransformationTable4Way(cumulativeFrequency, length); + } + else + { + BuildTransformationTableScalar(cumulativeFrequency, length); + } + + _tPos = _tt[origPtr]; + _count = 0; + _i2 = 0; + _ch2 = 256; + + if (_blockRandomised) + { + _rNToGo = 0; + _rTPos = 0; + SetupRandPartA(); + } + else + { + SetupNoRandPartA(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void BuildTransformationTableScalar( + Span cumulativeFrequency, + int length) + { + ref byte source = ref _ll8[0]; + ref int destination = ref _tt[0]; + ref int cursors = ref cumulativeFrequency[0]; + + ref byte current = ref source; + + int index = 0; + int remaining = length; + + while (remaining != 0) + { + byte value = current; + ref int cursor = ref Unsafe.Add(ref cursors, value); + + Unsafe.Add(ref destination, cursor) = index; + cursor++; + + current = ref Unsafe.Add(ref current, 1); + index++; + remaining--; + } + } + +#if NET6_0_OR_GREATER + [SkipLocalsInit] +#endif + private void BuildTransformationTable4Way( + Span cumulativeFrequency, + int length) + { + const int symbolCount = 256; + const int chunkCount = 4; + + // 4 × 256 local histograms and 4 × 256 independent cursors. + Span chunkCounts = stackalloc int[chunkCount * symbolCount]; + Span chunkCursors = stackalloc int[chunkCount * symbolCount]; + chunkCounts.Clear(); + + int chunkSize = length / chunkCount; + + const int start0 = 0; + int start2 = chunkSize * 2; + int start3 = chunkSize * 3; + + ref byte ll8 = ref _ll8[0]; + ref int counts = ref chunkCounts[0]; + + // First pass: calculate one histogram per contiguous chunk. + CountChunk(ref ll8, start0, chunkSize, ref Unsafe.Add(ref counts, 0 * symbolCount)); + CountChunk(ref ll8, chunkSize, start2, ref Unsafe.Add(ref counts, 1 * symbolCount)); + CountChunk(ref ll8, start2, start3, ref Unsafe.Add(ref counts, 2 * symbolCount)); + CountChunk(ref ll8, start3, length, ref Unsafe.Add(ref counts, 3 * symbolCount)); + + ref int globalStart = ref cumulativeFrequency[0]; + ref int cursors = ref chunkCursors[0]; + + // Assign a disjoint output range to each chunk for every symbol. + for (int symbol = 0; symbol < symbolCount; symbol++) + { + int position = Unsafe.Add(ref globalStart, symbol); + Unsafe.Add(ref cursors, 0 * symbolCount + symbol) = position; + position += Unsafe.Add(ref counts, 0 * symbolCount + symbol); + Unsafe.Add(ref cursors, 1 * symbolCount + symbol) = position; + position += Unsafe.Add(ref counts, 1 * symbolCount + symbol); + Unsafe.Add(ref cursors, 2 * symbolCount + symbol) = position; + position += Unsafe.Add(ref counts, 2 * symbolCount + symbol); + Unsafe.Add(ref cursors, 3 * symbolCount + symbol) = position; + } + + ref int tt = ref _tt[0]; + + // The four scatter loops no longer share cursor state. + ScatterChunk(ref ll8, ref tt, start0, chunkSize, ref Unsafe.Add(ref cursors, 0 * symbolCount)); + ScatterChunk(ref ll8, ref tt, chunkSize, start2, ref Unsafe.Add(ref cursors, 1 * symbolCount)); + ScatterChunk(ref ll8, ref tt, start2, start3, ref Unsafe.Add(ref cursors, 2 * symbolCount)); + ScatterChunk(ref ll8, ref tt, start3, length, ref Unsafe.Add(ref cursors, 3 * symbolCount)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CountChunk( + ref byte source, + int start, + int end, + ref int counts) + { + ref byte current = ref Unsafe.Add(ref source, start); + int remaining = end - start; + + while (remaining >= 4) + { + byte value0 = current; + byte value1 = Unsafe.Add(ref current, 1); + byte value2 = Unsafe.Add(ref current, 2); + byte value3 = Unsafe.Add(ref current, 3); + + Unsafe.Add(ref counts, value0)++; + Unsafe.Add(ref counts, value1)++; + Unsafe.Add(ref counts, value2)++; + Unsafe.Add(ref counts, value3)++; + + current = ref Unsafe.Add(ref current, 4); + remaining -= 4; + } + + while (remaining != 0) + { + Unsafe.Add(ref counts, current)++; + current = ref Unsafe.Add(ref current, 1); + remaining--; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ScatterChunk( + ref byte source, + ref int destination, + int start, + int end, + ref int cursors) + { + ref byte current = ref Unsafe.Add(ref source, start); + + int index = start; + int remaining = end - start; + + while (remaining != 0) + { + byte value = current; + ref int cursor = ref Unsafe.Add(ref cursors, value); + + Unsafe.Add(ref destination, cursor) = index; + cursor++; + + current = ref Unsafe.Add(ref current, 1); + index++; + remaining--; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupRandPartA() + { + if (_i2 <= _last) + { + _chPrev = _ch2; + _ch2 = _ll8[_tPos]; + _tPos = _tt[_tPos]; + + if (_rNToGo == 0) + { + _rNToGo = BZip2Constants.RandomNumbers[_rTPos++]; + if (_rTPos == 512) + { + _rTPos = 0; + } + } + + _rNToGo--; + _ch2 ^= (ushort)(_rNToGo == 1 ? 1 : 0); + _i2++; + + _currentChar = _ch2; + _currentState = BZip2Constants.DecodeState.RandPartB; + + return; + } + + _currentState = BZip2Constants.DecodeState.StartBlock; + _blockEndPending = true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupNoRandPartA() + { + if (_i2 <= _last) + { + _chPrev = _ch2; + _ch2 = _ll8[_tPos]; + _tPos = _tt[_tPos]; + _i2++; + + _currentChar = _ch2; + _currentState = BZip2Constants.DecodeState.NoRandPartB; + return; + } + + _currentState = BZip2Constants.DecodeState.StartBlock; + _blockEndPending = true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupRandPartB() + { + if (_ch2 != _chPrev) + { + _currentState = BZip2Constants.DecodeState.RandPartA; + _count = 1; + SetupRandPartA(); + return; + } + + _count++; + if (_count < 4) + { + _currentState = BZip2Constants.DecodeState.RandPartA; + SetupRandPartA(); + return; + } + + _z = _ll8[_tPos]; + _tPos = _tt[_tPos]; + + if (_rNToGo == 0) + { + _rNToGo = BZip2Constants.RandomNumbers[_rTPos++]; + if (_rTPos == 512) + { + _rTPos = 0; + } + } + + _rNToGo--; + _z ^= (byte)(_rNToGo == 1 ? 1 : 0); + _j2 = 0; + _currentState = BZip2Constants.DecodeState.RandPartC; + SetupRandPartC(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupRandPartC() + { + if (_j2 < _z) + { + _currentChar = _ch2; + _j2++; + return; + } + + CompleteRandPartC(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CompleteRandPartC() + { + _currentState = BZip2Constants.DecodeState.RandPartA; + _i2++; + _count = 0; + SetupRandPartA(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupNoRandPartB() + { + if (_ch2 != _chPrev) + { + _currentState = BZip2Constants.DecodeState.NoRandPartA; + _count = 1; + SetupNoRandPartA(); + return; + } + + _count++; + if (_count < 4) + { + _currentState = BZip2Constants.DecodeState.NoRandPartA; + SetupNoRandPartA(); + return; + } + + _z = _ll8[_tPos]; + _tPos = _tt[_tPos]; + _currentState = BZip2Constants.DecodeState.NoRandPartC; + _j2 = 0; + SetupNoRandPartC(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SetupNoRandPartC() + { + if (_j2 < _z) + { + _currentChar = _ch2; + _j2++; + return; + } + + CompleteNoRandPartC(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CompleteNoRandPartC() + { + _i2++; + _currentState = BZip2Constants.DecodeState.NoRandPartA; + _count = 0; + SetupNoRandPartA(); + } + + private void SetDecompressStructureSizes(int newSize100K) + { + if ((uint)(newSize100K - 1) >= 9) + { + throw new BZip2Exception("Invalid block size"); + } + + _blockSize100K = newSize100K; + int requiredLength = BZip2Constants.BaseBlockSize * newSize100K; + + if (_ll8.Length >= requiredLength && _tt.Length >= requiredLength) + { + return; + } + + byte[] newLl8 = ArrayPool.Shared.Rent(requiredLength); + int[] newTt = ArrayPool.Shared.Rent(requiredLength); + + byte[] oldLl8 = _ll8; + int[] oldTt = _tt; + + _ll8 = newLl8; + _tt = newTt; + + if (oldLl8.Length != 0) ArrayPool.Shared.Return(oldLl8); + if (oldTt.Length != 0) ArrayPool.Shared.Return(oldTt); + } + +#if NET6_0_OR_GREATER + [SkipLocalsInit] +#endif + private void HbCreateDecodeTables( + int table, + int minLength, + int maxLength, + int alphaSize) + { + int codeOffset = table * CodeTableStride; + int alphaOffset = table * AlphaTableStride; + + Span lengthCounts = stackalloc int[CodeTableStride]; + lengthCounts.Clear(); + + ref int countRef = ref lengthCounts[0]; + ref byte lengthsRef = ref Unsafe.Add(ref _codeLengths[0], alphaOffset); + + for (int symbol = 0; symbol < alphaSize; symbol++) + { + int length = Unsafe.Add(ref lengthsRef, symbol); + Unsafe.Add(ref countRef, length)++; + } + + Span positions = stackalloc int[CodeTableStride]; + + ref int positionRef = ref positions[0]; + int permutationIndex = alphaOffset; + + for (int length = minLength; length <= maxLength; length++) + { + Unsafe.Add(ref positionRef, length) = permutationIndex; + permutationIndex += Unsafe.Add(ref countRef, length); + } + + ref ushort permRef = ref _perm[0]; + + // Stable scatter preserves ascending symbol order within each length. + for (int symbol = 0; symbol < alphaSize; symbol++) + { + int length = Unsafe.Add(ref lengthsRef, symbol); + ref int position = ref Unsafe.Add(ref positionRef, length); + + Unsafe.Add(ref permRef, position++) = (ushort)symbol; + } + + // Build baseArray directly from the already available counts. + ref int baseRef = ref Unsafe.Add(ref _baseArray[0], codeOffset); + + baseRef = 0; + + for (int length = 1; length < CodeTableStride; length++) + { + Unsafe.Add(ref baseRef, length) = Unsafe.Add(ref baseRef, length - 1) + Unsafe.Add(ref countRef, length - 1); + } + + ref int limitRef = ref Unsafe.Add(ref _limit[0], codeOffset); + int code = 0; + for (int length = minLength; length <= maxLength; length++) + { + code += Unsafe.Add(ref baseRef, length + 1) - Unsafe.Add(ref baseRef, length); + Unsafe.Add(ref limitRef, length) = code - 1; + code <<= 1; + } + + for (int length = minLength + 1; length <= maxLength; length++) + { + Unsafe.Add(ref baseRef, length) = ((Unsafe.Add(ref limitRef, length - 1) + 1) << 1) - Unsafe.Add(ref baseRef, length); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CompressedStreamEof() => throw new EndOfStreamException("BZip2 input stream end of compressed stream"); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BlockOverrun() => throw new BZip2Exception("BZip2 input stream block overrun"); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void BadBlockHeader() => throw new BZip2Exception("BZip2 input stream bad block header"); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void CrcError() => throw new BZip2Exception("BZip2 input stream crc error"); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void DataError() => throw new BZip2Exception("Bzip data error"); +} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/CBZip2InputStream.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/CBZip2InputStream.cs deleted file mode 100644 index a79fef3..0000000 --- a/SharpHDiffPatch.Core/Binary/Compression/BZip2/CBZip2InputStream.cs +++ /dev/null @@ -1,1120 +0,0 @@ -using System; -using System.IO; - -/* - * Copyright 2001,2004-2005 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This package is based on the work done by Keiron Liddle, Aftex Software - * to whom the Ant project is very grateful for his - * great code. - */ - -namespace SharpCompress.Compressors.BZip2 -{ - /** - * An input stream that decompresses from the BZip2 format (with the file - * header chars) to be read as any other stream. - * - * @author Keiron Liddle - * - * NB: note this class has been modified to read the leading BZ from the - * start of the BZIP2 stream to make it compatible with other PGP programs. - */ - - internal sealed class CBZip2InputStream : Stream - { - private static void Cadvise() - { - //System.out.Println("CRC Error"); - //throw new CCoruptionError(); - } - - private static void BadBGLengths() => Cadvise(); - - private static void BitStreamEOF() => Cadvise(); - - private static void CompressedStreamEOF() => Cadvise(); - - private void MakeMaps() - { - int i; - nInUse = 0; - for (i = 0; i < 256; i++) - { - if (inUse[i]) - { - seqToUnseq[nInUse] = (char)i; - unseqToSeq[i] = (char)nInUse; - nInUse++; - } - } - } - - /* - index of the last char in the block, so - the block size == last + 1. - */ - private int last; - - /* - index in zptr[] of original string after sorting. - */ - private int origPtr; - - /* - always: in the range 0 .. 9. - The current block size is 100000 * this number. - */ - private int blockSize100k; - - private bool blockRandomised; - - private int bsBuff; - private int bsLive; - private readonly CRC mCrc = new CRC(); - - private readonly bool[] inUse = new bool[256]; - private int nInUse; - - private readonly char[] seqToUnseq = new char[256]; - private readonly char[] unseqToSeq = new char[256]; - - private readonly char[] selector = new char[BZip2Constants.MAX_SELECTORS]; - private readonly char[] selectorMtf = new char[BZip2Constants.MAX_SELECTORS]; - - private int[] tt; - private char[] ll8; - - /* - freq table collected to save a pass over the data - during decompression. - */ - private readonly int[] unzftab = new int[256]; - - private readonly int[][] limit = InitIntArray( - BZip2Constants.N_GROUPS, - BZip2Constants.MAX_ALPHA_SIZE - ); - private readonly int[][] basev = InitIntArray( - BZip2Constants.N_GROUPS, - BZip2Constants.MAX_ALPHA_SIZE - ); - private readonly int[][] perm = InitIntArray( - BZip2Constants.N_GROUPS, - BZip2Constants.MAX_ALPHA_SIZE - ); - private readonly int[] minLens = new int[BZip2Constants.N_GROUPS]; - - private Stream bsStream; - - private bool streamEnd; - - private int currentChar = -1; - - private const int START_BLOCK_STATE = 1; - private const int RAND_PART_A_STATE = 2; - private const int RAND_PART_B_STATE = 3; - private const int RAND_PART_C_STATE = 4; - private const int NO_RAND_PART_A_STATE = 5; - private const int NO_RAND_PART_B_STATE = 6; - private const int NO_RAND_PART_C_STATE = 7; - - private int currentState = START_BLOCK_STATE; - - private int storedBlockCRC, - storedCombinedCRC; - private int computedBlockCRC, - computedCombinedCRC; - private readonly bool decompressConcatenated; - - private int i2, - count, - chPrev, - ch2; - private int i, - tPos; - private int rNToGo; - private int rTPos; - private int j2; - private char z; - private bool isDisposed; - private bool isLeaveOpen; - - internal CBZip2InputStream(Stream zStream, bool decompressConcatenated, bool leaveOpen = false) - { - this.decompressConcatenated = decompressConcatenated; - ll8 = null; - tt = null; - isLeaveOpen = leaveOpen; - BsSetStream(zStream); - Initialize(true); - InitBlock(); - SetupBlock(); - } - - protected override void Dispose(bool disposing) - { - if (isDisposed) - { - return; - } - isDisposed = true; - base.Dispose(disposing); - if (!isLeaveOpen) bsStream?.Dispose(); - } - - internal static int[][] InitIntArray(int n1, int n2) - { - var a = new int[n1][]; - for (var k = 0; k < n1; ++k) - { - a[k] = new int[n2]; - } - return a; - } - - internal static char[][] InitCharArray(int n1, int n2) - { - var a = new char[n1][]; - for (var k = 0; k < n1; ++k) - { - a[k] = new char[n2]; - } - return a; - } - - public override int ReadByte() - { - if (streamEnd) - { - return -1; - } - var retChar = currentChar; - switch (currentState) - { - case START_BLOCK_STATE: - break; - case RAND_PART_A_STATE: - break; - case RAND_PART_B_STATE: - SetupRandPartB(); - break; - case RAND_PART_C_STATE: - SetupRandPartC(); - break; - case NO_RAND_PART_A_STATE: - break; - case NO_RAND_PART_B_STATE: - SetupNoRandPartB(); - break; - case NO_RAND_PART_C_STATE: - SetupNoRandPartC(); - break; - default: - break; - } - return retChar; - } - - private bool Initialize(bool isFirstStream) - { - var magic0 = bsStream.ReadByte(); - var magic1 = bsStream.ReadByte(); - var magic2 = bsStream.ReadByte(); - if (magic0 == -1 && !isFirstStream) - { - return false; - } - if (magic0 != 'B' || magic1 != 'Z' || magic2 != 'h') - { - throw new IOException("Not a BZIP2 marked stream"); - } - var magic3 = bsStream.ReadByte(); - if (magic3 < '1' || magic3 > '9') - { - BsFinishedWithStream(); - streamEnd = true; - return false; - } - - SetDecompressStructureSizes(magic3 - '0'); - bsLive = 0; - computedCombinedCRC = 0; - return true; - } - - private void InitBlock() - { - char magic1, - magic2, - magic3, - magic4; - char magic5, - magic6; - - while (true) - { - magic1 = BsGetUChar(); - magic2 = BsGetUChar(); - magic3 = BsGetUChar(); - magic4 = BsGetUChar(); - magic5 = BsGetUChar(); - magic6 = BsGetUChar(); - if ( - magic1 != 0x17 - || magic2 != 0x72 - || magic3 != 0x45 - || magic4 != 0x38 - || magic5 != 0x50 - || magic6 != 0x90 - ) - { - break; - } - - if (Complete()) - { - return; - } - } - - if ( - magic1 != 0x31 - || magic2 != 0x41 - || magic3 != 0x59 - || magic4 != 0x26 - || magic5 != 0x53 - || magic6 != 0x59 - ) - { - BadBlockHeader(); - streamEnd = true; - return; - } - - storedBlockCRC = BsGetInt32(); - - if (BsR(1) == 1) - { - blockRandomised = true; - } - else - { - blockRandomised = false; - } - - // currBlockNo++; - GetAndMoveToFrontDecode(); - - mCrc.InitialiseCRC(); - currentState = START_BLOCK_STATE; - } - - private void EndBlock() - { - computedBlockCRC = mCrc.GetFinalCRC(); - /* A bad CRC is considered a fatal error. */ - if (storedBlockCRC != computedBlockCRC) - { - CrcError(); - } - - computedCombinedCRC = (computedCombinedCRC << 1) | (int)(((uint)computedCombinedCRC) >> 31); - computedCombinedCRC ^= computedBlockCRC; - } - - private bool Complete() - { - storedCombinedCRC = BsGetInt32(); - if (storedCombinedCRC != computedCombinedCRC) - { - CrcError(); - } - - var complete = !decompressConcatenated || !Initialize(false); - if (complete) - { - BsFinishedWithStream(); - streamEnd = true; - } - - // Look for the next .bz2 stream if decompressing - // concatenated files. - return complete; - } - - private static void BlockOverrun() => Cadvise(); - - private static void BadBlockHeader() => Cadvise(); - - private static void CrcError() => Cadvise(); - - private void BsFinishedWithStream() - { - if (!isLeaveOpen) - { - bsStream?.Dispose(); - bsStream = null; - } - } - - private void BsSetStream(Stream f) - { - bsStream = f; - bsLive = 0; - bsBuff = 0; - } - - private int BsR(int n) - { - int v; - while (bsLive < n) - { - int zzi; - int thech = '\0'; - try - { - thech = (char)bsStream.ReadByte(); - } - catch (IOException) - { - CompressedStreamEOF(); - } - if (thech == '\uffff') - { - CompressedStreamEOF(); - } - zzi = thech; - bsBuff = (bsBuff << 8) | (zzi & 0xff); - bsLive += 8; - } - - v = (bsBuff >> (bsLive - n)) & ((1 << n) - 1); - bsLive -= n; - return v; - } - - private char BsGetUChar() => (char)BsR(8); - - private int BsGetint() - { - var u = 0; - u = (u << 8) | BsR(8); - u = (u << 8) | BsR(8); - u = (u << 8) | BsR(8); - u = (u << 8) | BsR(8); - return u; - } - - private int BsGetIntVS(int numBits) => BsR(numBits); - - private int BsGetInt32() => BsGetint(); - - private void HbCreateDecodeTables( - int[] limit, - int[] basev, - int[] perm, - char[] length, - int minLen, - int maxLen, - int alphaSize - ) - { - int pp, - i, - j, - vec; - - pp = 0; - for (i = minLen; i <= maxLen; i++) - { - for (j = 0; j < alphaSize; j++) - { - if (length[j] == i) - { - perm[pp] = j; - pp++; - } - } - } - - for (i = 0; i < BZip2Constants.MAX_CODE_LEN; i++) - { - basev[i] = 0; - } - for (i = 0; i < alphaSize; i++) - { - basev[length[i] + 1]++; - } - - for (i = 1; i < BZip2Constants.MAX_CODE_LEN; i++) - { - basev[i] += basev[i - 1]; - } - - for (i = 0; i < BZip2Constants.MAX_CODE_LEN; i++) - { - limit[i] = 0; - } - vec = 0; - - for (i = minLen; i <= maxLen; i++) - { - vec += (basev[i + 1] - basev[i]); - limit[i] = vec - 1; - vec <<= 1; - } - for (i = minLen + 1; i <= maxLen; i++) - { - basev[i] = ((limit[i - 1] + 1) << 1) - basev[i]; - } - } - - private void RecvDecodingTables() - { - var len = InitCharArray(BZip2Constants.N_GROUPS, BZip2Constants.MAX_ALPHA_SIZE); - int i, - j, - t, - nGroups, - nSelectors, - alphaSize; - int minLen, - maxLen; - var inUse16 = new bool[16]; - - /* Receive the mapping table */ - for (i = 0; i < 16; i++) - { - if (BsR(1) == 1) - { - inUse16[i] = true; - } - else - { - inUse16[i] = false; - } - } - - for (i = 0; i < 256; i++) - { - inUse[i] = false; - } - - for (i = 0; i < 16; i++) - { - if (inUse16[i]) - { - for (j = 0; j < 16; j++) - { - if (BsR(1) == 1) - { - inUse[(i * 16) + j] = true; - } - } - } - } - - MakeMaps(); - alphaSize = nInUse + 2; - - /* Now the selectors */ - nGroups = BsR(3); - nSelectors = BsR(15); - for (i = 0; i < nSelectors; i++) - { - j = 0; - while (BsR(1) == 1) - { - j++; - } - selectorMtf[i] = (char)j; - } - - /* Undo the MTF values for the selectors. */ - { - var pos = new char[BZip2Constants.N_GROUPS]; - char tmp, - v; - for (v = '\0'; v < nGroups; v++) - { - pos[v] = v; - } - - for (i = 0; i < nSelectors; i++) - { - v = selectorMtf[i]; - tmp = pos[v]; - while (v > 0) - { - pos[v] = pos[v - 1]; - v--; - } - pos[0] = tmp; - selector[i] = tmp; - } - } - - /* Now the coding tables */ - for (t = 0; t < nGroups; t++) - { - var curr = BsR(5); - for (i = 0; i < alphaSize; i++) - { - while (BsR(1) == 1) - { - if (BsR(1) == 0) - { - curr++; - } - else - { - curr--; - } - } - len[t][i] = (char)curr; - } - } - - /* Create the Huffman decoding tables */ - for (t = 0; t < nGroups; t++) - { - minLen = 32; - maxLen = 0; - for (i = 0; i < alphaSize; i++) - { - if (len[t][i] > maxLen) - { - maxLen = len[t][i]; - } - if (len[t][i] < minLen) - { - minLen = len[t][i]; - } - } - HbCreateDecodeTables(limit[t], basev[t], perm[t], len[t], minLen, maxLen, alphaSize); - minLens[t] = minLen; - } - } - - private void GetAndMoveToFrontDecode() - { - var yy = new char[256]; - int i, - j, - nextSym, - limitLast; - int EOB, - groupNo, - groupPos; - - limitLast = BZip2Constants.baseBlockSize * blockSize100k; - origPtr = BsGetIntVS(24); - - RecvDecodingTables(); - EOB = nInUse + 1; - groupNo = -1; - groupPos = 0; - - /* - Setting up the unzftab entries here is not strictly - necessary, but it does save having to do it later - in a separate pass, and so saves a block's worth of - cache misses. - */ - for (i = 0; i <= 255; i++) - { - unzftab[i] = 0; - } - - for (i = 0; i <= 255; i++) - { - yy[i] = (char)i; - } - - last = -1; - - { - int zt, - zn, - zvec, - zj; - if (groupPos == 0) - { - groupNo++; - groupPos = BZip2Constants.G_SIZE; - } - groupPos--; - zt = selector[groupNo]; - zn = minLens[zt]; - zvec = BsR(zn); - while (zvec > limit[zt][zn]) - { - zn++; - { - { - while (bsLive < 1) - { - int zzi; - var thech = '\0'; - try - { - thech = (char)bsStream.ReadByte(); - } - catch (IOException) - { - CompressedStreamEOF(); - } - if (thech == '\uffff') - { - CompressedStreamEOF(); - } - zzi = thech; - bsBuff = (bsBuff << 8) | (zzi & 0xff); - bsLive += 8; - } - } - zj = (bsBuff >> (bsLive - 1)) & 1; - bsLive--; - } - zvec = (zvec << 1) | zj; - } - nextSym = perm[zt][zvec - basev[zt][zn]]; - } - - while (true) - { - if (nextSym == EOB) - { - break; - } - - if (nextSym == BZip2Constants.RUNA || nextSym == BZip2Constants.RUNB) - { - char ch; - var s = -1; - var N = 1; - do - { - if (nextSym == BZip2Constants.RUNA) - { - s += (0 + 1) * N; - } - else if (nextSym == BZip2Constants.RUNB) - { - s += (1 + 1) * N; - } - N *= 2; - { - int zt, - zn, - zvec, - zj; - if (groupPos == 0) - { - groupNo++; - groupPos = BZip2Constants.G_SIZE; - } - groupPos--; - zt = selector[groupNo]; - zn = minLens[zt]; - zvec = BsR(zn); - while (zvec > limit[zt][zn]) - { - zn++; - { - { - while (bsLive < 1) - { - int zzi; - var thech = '\0'; - try - { - thech = (char)bsStream.ReadByte(); - } - catch (IOException) - { - CompressedStreamEOF(); - } - if (thech == '\uffff') - { - CompressedStreamEOF(); - } - zzi = thech; - bsBuff = (bsBuff << 8) | (zzi & 0xff); - bsLive += 8; - } - } - zj = (bsBuff >> (bsLive - 1)) & 1; - bsLive--; - } - zvec = (zvec << 1) | zj; - } - nextSym = perm[zt][zvec - basev[zt][zn]]; - } - } while (nextSym == BZip2Constants.RUNA || nextSym == BZip2Constants.RUNB); - - s++; - ch = seqToUnseq[yy[0]]; - unzftab[ch] += s; - - while (s > 0) - { - last++; - ll8[last] = ch; - s--; - } - - if (last >= limitLast) - { - BlockOverrun(); - } - } - else - { - char tmp; - last++; - if (last >= limitLast) - { - BlockOverrun(); - } - - tmp = yy[nextSym - 1]; - unzftab[seqToUnseq[tmp]]++; - ll8[last] = seqToUnseq[tmp]; - - /* - This loop is hammered during decompression, - hence the unrolling. - - for (j = nextSym-1; j > 0; j--) yy[j] = yy[j-1]; - */ - - j = nextSym - 1; - for (; j > 3; j -= 4) - { - yy[j] = yy[j - 1]; - yy[j - 1] = yy[j - 2]; - yy[j - 2] = yy[j - 3]; - yy[j - 3] = yy[j - 4]; - } - for (; j > 0; j--) - { - yy[j] = yy[j - 1]; - } - - yy[0] = tmp; - { - int zt, - zn, - zvec, - zj; - if (groupPos == 0) - { - groupNo++; - groupPos = BZip2Constants.G_SIZE; - } - groupPos--; - zt = selector[groupNo]; - zn = minLens[zt]; - zvec = BsR(zn); - while (zvec > limit[zt][zn]) - { - zn++; - { - { - while (bsLive < 1) - { - int zzi; - var thech = '\0'; - try - { - thech = (char)bsStream.ReadByte(); - } - catch (IOException) - { - CompressedStreamEOF(); - } - zzi = thech; - bsBuff = (bsBuff << 8) | (zzi & 0xff); - bsLive += 8; - } - } - zj = (bsBuff >> (bsLive - 1)) & 1; - bsLive--; - } - zvec = (zvec << 1) | zj; - } - nextSym = perm[zt][zvec - basev[zt][zn]]; - } - } - } - } - - private void SetupBlock() - { - Span cftab = stackalloc int[257]; - char ch; - - cftab[0] = 0; - for (i = 1; i <= 256; i++) - { - cftab[i] = unzftab[i - 1]; - } - for (i = 1; i <= 256; i++) - { - cftab[i] += cftab[i - 1]; - } - - for (i = 0; i <= last; i++) - { - ch = ll8[i]; - tt[cftab[ch]] = i; - cftab[ch]++; - } - - tPos = tt[origPtr]; - - count = 0; - i2 = 0; - ch2 = 256; /* not a char and not EOF */ - - if (blockRandomised) - { - rNToGo = 0; - rTPos = 0; - SetupRandPartA(); - } - else - { - SetupNoRandPartA(); - } - } - - private void SetupRandPartA() - { - if (i2 <= last) - { - chPrev = ch2; - ch2 = ll8[tPos]; - tPos = tt[tPos]; - if (rNToGo == 0) - { - rNToGo = BZip2Constants.rNums[rTPos]; - rTPos++; - if (rTPos == 512) - { - rTPos = 0; - } - } - rNToGo--; - ch2 ^= (rNToGo == 1) ? 1 : 0; - i2++; - - currentChar = ch2; - currentState = RAND_PART_B_STATE; - mCrc.UpdateCRC(ch2); - } - else - { - EndBlock(); - InitBlock(); - SetupBlock(); - } - } - - private void SetupNoRandPartA() - { - if (i2 <= last) - { - chPrev = ch2; - ch2 = ll8[tPos]; - tPos = tt[tPos]; - i2++; - - currentChar = ch2; - currentState = NO_RAND_PART_B_STATE; - mCrc.UpdateCRC(ch2); - } - else - { - EndBlock(); - InitBlock(); - SetupBlock(); - } - } - - private void SetupRandPartB() - { - if (ch2 != chPrev) - { - currentState = RAND_PART_A_STATE; - count = 1; - SetupRandPartA(); - } - else - { - count++; - if (count >= 4) - { - z = ll8[tPos]; - tPos = tt[tPos]; - if (rNToGo == 0) - { - rNToGo = BZip2Constants.rNums[rTPos]; - rTPos++; - if (rTPos == 512) - { - rTPos = 0; - } - } - rNToGo--; - z ^= (char)((rNToGo == 1) ? 1 : 0); - j2 = 0; - currentState = RAND_PART_C_STATE; - SetupRandPartC(); - } - else - { - currentState = RAND_PART_A_STATE; - SetupRandPartA(); - } - } - } - - private void SetupRandPartC() - { - if (j2 < z) - { - currentChar = ch2; - mCrc.UpdateCRC(ch2); - j2++; - } - else - { - currentState = RAND_PART_A_STATE; - i2++; - count = 0; - SetupRandPartA(); - } - } - - private void SetupNoRandPartB() - { - if (ch2 != chPrev) - { - currentState = NO_RAND_PART_A_STATE; - count = 1; - SetupNoRandPartA(); - } - else - { - count++; - if (count >= 4) - { - z = ll8[tPos]; - tPos = tt[tPos]; - currentState = NO_RAND_PART_C_STATE; - j2 = 0; - SetupNoRandPartC(); - } - else - { - currentState = NO_RAND_PART_A_STATE; - SetupNoRandPartA(); - } - } - } - - private void SetupNoRandPartC() - { - if (j2 < z) - { - currentChar = ch2; - mCrc.UpdateCRC(ch2); - j2++; - } - else - { - currentState = NO_RAND_PART_A_STATE; - i2++; - count = 0; - SetupNoRandPartA(); - } - } - - private void SetDecompressStructureSizes(int newSize100k) - { - if (!(0 <= newSize100k && newSize100k <= 9 && 0 <= blockSize100k && blockSize100k <= 9)) - { - // throw new IOException("Invalid block size"); - } - - blockSize100k = newSize100k; - - if (newSize100k == 0) - { - return; - } - - var n = BZip2Constants.baseBlockSize * newSize100k; - ll8 = new char[n]; - tt = new int[n]; - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) - { - var c = -1; - int k; - for (k = 0; k < count; ++k) - { - c = ReadByte(); - if (c == -1) - { - break; - } - buffer[k + offset] = (byte)c; - } - return k; - } - - public override long Seek(long offset, SeekOrigin origin) => 0; - - public override void SetLength(long value) { } - - public override void Write(byte[] buffer, int offset, int count) { } - - public override void WriteByte(byte value) { } - - public override bool CanRead => true; - - public override bool CanSeek => false; - - public override bool CanWrite => false; - - public override long Length => 0; - - public override long Position - { - get => 0; - set { } - } - } - -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/CRC.cs b/SharpHDiffPatch.Core/Binary/Compression/BZip2/CRC.cs deleted file mode 100644 index d187150..0000000 --- a/SharpHDiffPatch.Core/Binary/Compression/BZip2/CRC.cs +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Copyright 2001,2004-2005 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This package is based on the work done by Keiron Liddle), Aftex Software - * to whom the Ant project is very grateful for his - * great code. - */ - -namespace SharpCompress.Compressors.BZip2 -{ - /** - * A simple class the hold and calculate the CRC for sanity checking - * of the data. - * - * @author Keiron Liddle - */ - - internal sealed class CRC - { - internal static int[] crc32Table = - { - 0x00000000, - 0x04c11db7, - 0x09823b6e, - 0x0d4326d9, - 0x130476dc, - 0x17c56b6b, - 0x1a864db2, - 0x1e475005, - 0x2608edb8, - 0x22c9f00f, - 0x2f8ad6d6, - 0x2b4bcb61, - 0x350c9b64, - 0x31cd86d3, - 0x3c8ea00a, - 0x384fbdbd, - 0x4c11db70, - 0x48d0c6c7, - 0x4593e01e, - 0x4152fda9, - 0x5f15adac, - 0x5bd4b01b, - 0x569796c2, - 0x52568b75, - 0x6a1936c8, - 0x6ed82b7f, - 0x639b0da6, - 0x675a1011, - 0x791d4014, - 0x7ddc5da3, - 0x709f7b7a, - 0x745e66cd, - unchecked((int)0x9823b6e0), - unchecked((int)0x9ce2ab57), - unchecked((int)0x91a18d8e), - unchecked((int)0x95609039), - unchecked((int)0x8b27c03c), - unchecked((int)0x8fe6dd8b), - unchecked((int)0x82a5fb52), - unchecked((int)0x8664e6e5), - unchecked((int)0xbe2b5b58), - unchecked((int)0xbaea46ef), - unchecked((int)0xb7a96036), - unchecked((int)0xb3687d81), - unchecked((int)0xad2f2d84), - unchecked((int)0xa9ee3033), - unchecked((int)0xa4ad16ea), - unchecked((int)0xa06c0b5d), - unchecked((int)0xd4326d90), - unchecked((int)0xd0f37027), - unchecked((int)0xddb056fe), - unchecked((int)0xd9714b49), - unchecked((int)0xc7361b4c), - unchecked((int)0xc3f706fb), - unchecked((int)0xceb42022), - unchecked((int)0xca753d95), - unchecked((int)0xf23a8028), - unchecked((int)0xf6fb9d9f), - unchecked((int)0xfbb8bb46), - unchecked((int)0xff79a6f1), - unchecked((int)0xe13ef6f4), - unchecked((int)0xe5ffeb43), - unchecked((int)0xe8bccd9a), - unchecked((int)0xec7dd02d), - 0x34867077, - 0x30476dc0, - 0x3d044b19, - 0x39c556ae, - 0x278206ab, - 0x23431b1c, - 0x2e003dc5, - 0x2ac12072, - 0x128e9dcf, - 0x164f8078, - 0x1b0ca6a1, - 0x1fcdbb16, - 0x018aeb13, - 0x054bf6a4, - 0x0808d07d, - 0x0cc9cdca, - 0x7897ab07, - 0x7c56b6b0, - 0x71159069, - 0x75d48dde, - 0x6b93dddb, - 0x6f52c06c, - 0x6211e6b5, - 0x66d0fb02, - 0x5e9f46bf, - 0x5a5e5b08, - 0x571d7dd1, - 0x53dc6066, - 0x4d9b3063, - 0x495a2dd4, - 0x44190b0d, - 0x40d816ba, - unchecked((int)0xaca5c697), - unchecked((int)0xa864db20), - unchecked((int)0xa527fdf9), - unchecked((int)0xa1e6e04e), - unchecked((int)0xbfa1b04b), - unchecked((int)0xbb60adfc), - unchecked((int)0xb6238b25), - unchecked((int)0xb2e29692), - unchecked((int)0x8aad2b2f), - unchecked((int)0x8e6c3698), - unchecked((int)0x832f1041), - unchecked((int)0x87ee0df6), - unchecked((int)0x99a95df3), - unchecked((int)0x9d684044), - unchecked((int)0x902b669d), - unchecked((int)0x94ea7b2a), - unchecked((int)0xe0b41de7), - unchecked((int)0xe4750050), - unchecked((int)0xe9362689), - unchecked((int)0xedf73b3e), - unchecked((int)0xf3b06b3b), - unchecked((int)0xf771768c), - unchecked((int)0xfa325055), - unchecked((int)0xfef34de2), - unchecked((int)0xc6bcf05f), - unchecked((int)0xc27dede8), - unchecked((int)0xcf3ecb31), - unchecked((int)0xcbffd686), - unchecked((int)0xd5b88683), - unchecked((int)0xd1799b34), - unchecked((int)0xdc3abded), - unchecked((int)0xd8fba05a), - 0x690ce0ee, - 0x6dcdfd59, - 0x608edb80, - 0x644fc637, - 0x7a089632, - 0x7ec98b85, - 0x738aad5c, - 0x774bb0eb, - 0x4f040d56, - 0x4bc510e1, - 0x46863638, - 0x42472b8f, - 0x5c007b8a, - 0x58c1663d, - 0x558240e4, - 0x51435d53, - 0x251d3b9e, - 0x21dc2629, - 0x2c9f00f0, - 0x285e1d47, - 0x36194d42, - 0x32d850f5, - 0x3f9b762c, - 0x3b5a6b9b, - 0x0315d626, - 0x07d4cb91, - 0x0a97ed48, - 0x0e56f0ff, - 0x1011a0fa, - 0x14d0bd4d, - 0x19939b94, - 0x1d528623, - unchecked((int)0xf12f560e), - unchecked((int)0xf5ee4bb9), - unchecked((int)0xf8ad6d60), - unchecked((int)0xfc6c70d7), - unchecked((int)0xe22b20d2), - unchecked((int)0xe6ea3d65), - unchecked((int)0xeba91bbc), - unchecked((int)0xef68060b), - unchecked((int)0xd727bbb6), - unchecked((int)0xd3e6a601), - unchecked((int)0xdea580d8), - unchecked((int)0xda649d6f), - unchecked((int)0xc423cd6a), - unchecked((int)0xc0e2d0dd), - unchecked((int)0xcda1f604), - unchecked((int)0xc960ebb3), - unchecked((int)0xbd3e8d7e), - unchecked((int)0xb9ff90c9), - unchecked((int)0xb4bcb610), - unchecked((int)0xb07daba7), - unchecked((int)0xae3afba2), - unchecked((int)0xaafbe615), - unchecked((int)0xa7b8c0cc), - unchecked((int)0xa379dd7b), - unchecked((int)0x9b3660c6), - unchecked((int)0x9ff77d71), - unchecked((int)0x92b45ba8), - unchecked((int)0x9675461f), - unchecked((int)0x8832161a), - unchecked((int)0x8cf30bad), - unchecked((int)0x81b02d74), - unchecked((int)0x857130c3), - 0x5d8a9099, - 0x594b8d2e, - 0x5408abf7, - 0x50c9b640, - 0x4e8ee645, - 0x4a4ffbf2, - 0x470cdd2b, - 0x43cdc09c, - 0x7b827d21, - 0x7f436096, - 0x7200464f, - 0x76c15bf8, - 0x68860bfd, - 0x6c47164a, - 0x61043093, - 0x65c52d24, - 0x119b4be9, - 0x155a565e, - 0x18197087, - 0x1cd86d30, - 0x029f3d35, - 0x065e2082, - 0x0b1d065b, - 0x0fdc1bec, - 0x3793a651, - 0x3352bbe6, - 0x3e119d3f, - 0x3ad08088, - 0x2497d08d, - 0x2056cd3a, - 0x2d15ebe3, - 0x29d4f654, - unchecked((int)0xc5a92679), - unchecked((int)0xc1683bce), - unchecked((int)0xcc2b1d17), - unchecked((int)0xc8ea00a0), - unchecked((int)0xd6ad50a5), - unchecked((int)0xd26c4d12), - unchecked((int)0xdf2f6bcb), - unchecked((int)0xdbee767c), - unchecked((int)0xe3a1cbc1), - unchecked((int)0xe760d676), - unchecked((int)0xea23f0af), - unchecked((int)0xeee2ed18), - unchecked((int)0xf0a5bd1d), - unchecked((int)0xf464a0aa), - unchecked((int)0xf9278673), - unchecked((int)0xfde69bc4), - unchecked((int)0x89b8fd09), - unchecked((int)0x8d79e0be), - unchecked((int)0x803ac667), - unchecked((int)0x84fbdbd0), - unchecked((int)0x9abc8bd5), - unchecked((int)0x9e7d9662), - unchecked((int)0x933eb0bb), - unchecked((int)0x97ffad0c), - unchecked((int)0xafb010b1), - unchecked((int)0xab710d06), - unchecked((int)0xa6322bdf), - unchecked((int)0xa2f33668), - unchecked((int)0xbcb4666d), - unchecked((int)0xb8757bda), - unchecked((int)0xb5365d03), - unchecked((int)0xb1f740b4) - }; - - - internal CRC() => InitialiseCRC(); - - internal void InitialiseCRC() => globalCrc = unchecked((int)0xffffffff); - - internal int GetFinalCRC() => ~globalCrc; - - internal int GetGlobalCRC() => globalCrc; - - internal void SetGlobalCRC(int newCrc) => globalCrc = newCrc; - - internal void UpdateCRC(int inCh) - { - var temp = (globalCrc >> 24) ^ inCh; - if (temp < 0) - { - temp = 256 + temp; - } - globalCrc = (globalCrc << 8) ^ crc32Table[temp]; - } - - internal int globalCrc; - } -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/LICENSE b/SharpHDiffPatch.Core/Binary/Compression/BZip2/LICENSE index 94e2f5b..2021420 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/BZip2/LICENSE +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/LICENSE @@ -1,21 +1,17 @@ -The MIT License (MIT) +Copyright © 2000-2018 SharpZipLib Contributors -Copyright (c) 2014 Adam Hathcock +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/BZip2/README.md b/SharpHDiffPatch.Core/Binary/Compression/BZip2/README.md index 777b383..402803f 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/BZip2/README.md +++ b/SharpHDiffPatch.Core/Binary/Compression/BZip2/README.md @@ -1,196 +1,23 @@ -# SharpCompress - -SharpCompress is a compression library in pure C# for .NET Standard 2.0, 2.1, .NET Core 3.1 and .NET 5.0 that can unrar, un7zip, unzip, untar unbzip2, ungzip, unlzip with forward-only reading and file random access APIs. Write support for zip/tar/bzip2/gzip/lzip are implemented. - -The major feature is support for non-seekable streams so large files can be processed on the fly (i.e. download stream). - -GitHub Actions Build - -[![SharpCompress](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml/badge.svg)](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml) - -## Need Help? - -Post Issues on Github! - -Check the [Supported Formats](FORMATS.md) and [Basic Usage.](USAGE.md) - -## Recommended Formats - -In general, I recommend GZip (Deflate)/BZip2 (BZip)/LZip (LZMA) as the simplicity of the formats lend to better long term archival as well as the streamability. Tar is often used in conjunction for multiple files in a single archive (e.g. `.tar.gz`) - -Zip is okay, but it's a very hap-hazard format and the variation in headers and implementations makes it hard to get correct. Uses Deflate by default but supports a lot of compression methods. - -RAR is not recommended as it's a propriatory format and the compression is closed source. Use Tar/LZip for LZMA - -7Zip and XZ both are overly complicated. 7Zip does not support streamable formats. XZ has known holes explained here: (http://www.nongnu.org/lzip/xz_inadequate.html) Use Tar/LZip for LZMA compression instead. - -## A Simple Request - -Hi everyone. I hope you're using SharpCompress and finding it useful. Please give me feedback on what you'd like to see changed especially as far as usability goes. New feature suggestions are always welcome as well. I would also like to know what projects SharpCompress is being used in. I like seeing how it is used to give me ideas for future versions. Thanks! - -Please do not email me directly to ask for help. If you think there is a real issue, please report it here. - -## Want to contribute? - -I'm always looking for help or ideas. Please submit code or email with ideas. Unfortunately, just letting me know you'd like to help is not enough because I really have no overall plan of what needs to be done. I'll definitely accept code submissions and add you as a member of the project! - -## TODOs (always lots) - -* RAR 5 decryption support -* 7Zip writing -* Zip64 (Need writing and extend Reading) -* Multi-volume Zip support. - -## Version Log - -### Version 0.18 - -* [Now on Github releases](https://github.com/adamhathcock/sharpcompress/releases/tag/0.18) - -### Version 0.17.1 - -* Fix - [Bug Fix for .NET Core on Windows](https://github.com/adamhathcock/sharpcompress/pull/257) - -### Version 0.17.0 - -* New - Full LZip support! Can read and write LZip files and Tars inside LZip files. [Make LZip a first class citizen. #241](https://github.com/adamhathcock/sharpcompress/issues/241) -* New - XZ read support! Can read XZ files and Tars inside XZ files. [XZ in SharpCompress #91](https://github.com/adamhathcock/sharpcompress/issues/94) -* Fix - [Regression - zip file writing on seekable streams always assumed stream start was 0. Introduced with Zip64 writing.](https://github.com/adamhathcock/sharpcompress/issues/244) -* Fix - [Zip files with post-data descriptors can be properly skipped via decompression](https://github.com/adamhathcock/sharpcompress/issues/162) - -### Version 0.16.2 - -* Fix [.NET 3.5 should support files and cryptography (was a regression from 0.16.0)](https://github.com/adamhathcock/sharpcompress/pull/251) -* Fix [Zip per entry compression customization wrote the wrong method into the zip archive](https://github.com/adamhathcock/sharpcompress/pull/249) - -### Version 0.16.1 - -* Fix [Preserve compression method when getting a compressed stream](https://github.com/adamhathcock/sharpcompress/pull/235) -* Fix [RAR entry key normalization fix](https://github.com/adamhathcock/sharpcompress/issues/201) - -### Version 0.16.0 - -* Breaking - [Progress Event Tracking rethink](https://github.com/adamhathcock/sharpcompress/pull/226) -* Update to VS2017 - [VS2017](https://github.com/adamhathcock/sharpcompress/pull/231) - Framework targets have been changed. -* New - [Add Zip64 writing](https://github.com/adamhathcock/sharpcompress/pull/211) -* [Fix invalid/mismatching Zip version flags.](https://github.com/adamhathcock/sharpcompress/issues/164) - This allows nuget/System.IO.Packaging to read zip files generated by SharpCompress -* [Fix 7Zip directory hiding](https://github.com/adamhathcock/sharpcompress/pull/215/files) -* [Verify RAR CRC headers](https://github.com/adamhathcock/sharpcompress/pull/220) - -### Version 0.15.2 - -* [Fix invalid headers](https://github.com/adamhathcock/sharpcompress/pull/210) - fixes an issue creating large-ish zip archives that was introduced with zip64 reading. - -### Version 0.15.1 - -* [Zip64 extending information and ZipReader](https://github.com/adamhathcock/sharpcompress/pull/206) - -### Version 0.15.0 - -* [Add zip64 support for ZipArchive extraction](https://github.com/adamhathcock/sharpcompress/pull/205) - -### Version 0.14.1 - -* [.NET Assemblies aren't strong named](https://github.com/adamhathcock/sharpcompress/issues/158) -* [Pkware encryption for Zip files didn't allow for multiple reads of an entry](https://github.com/adamhathcock/sharpcompress/issues/197) -* [GZip Entry couldn't be read multiple times](https://github.com/adamhathcock/sharpcompress/issues/198) - -### Version 0.14.0 - -* [Support for LZip reading in for Tars](https://github.com/adamhathcock/sharpcompress/pull/191) - -### Version 0.13.1 - -* [Fix null password on ReaderFactory. Fix null options on SevenZipArchive](https://github.com/adamhathcock/sharpcompress/pull/188) -* [Make PpmdProperties lazy to avoid unnecessary allocations.](https://github.com/adamhathcock/sharpcompress/pull/185) - -### Version 0.13.0 - -* Breaking change: Big refactor of Options on API. -* 7Zip supports Deflate - -### Version 0.12.4 - -* Forward only zip issue fix https://github.com/adamhathcock/sharpcompress/issues/160 -* Try to fix frameworks again by copying targets from JSON.NET - -### Version 0.12.3 - -* 7Zip fixes https://github.com/adamhathcock/sharpcompress/issues/73 -* Maybe all profiles will work with project.json now - -### Version 0.12.2 - -* Support Profile 259 again - -### Version 0.12.1 - -* Support Silverlight 5 - -### Version 0.12.0 - -* .NET Core RTM! -* Bug fix for Tar long paths - -### Version 0.11.6 - -* Bug fix for global header in Tar -* Writers now have a leaveOpen `bool` overload. They won't close streams if not-requested to. - -### Version 0.11.5 - -* Bug fix in Skip method - -### Version 0.11.4 - -* SharpCompress is now endian neutral (matters for Mono platforms) -* Fix for Inflate (need to change implementation) -* Fixes for RAR detection - -### Version 0.11.1 - -* Added Cancel on IReader -* Removed .NET 2.0 support and LinqBridge dependency - -### Version 0.11 - -* Been over a year, contains mainly fixes from contributors! -* Possible breaking change: ArchiveEncoding is UTF8 by default now. -* TAR supports writing long names using longlink -* RAR Protect Header added - -### Version 0.10.3 - -* Finally fixed Disposal issue when creating a new archive with the Archive API - -### Version 0.10.2 - -* Fixed Rar Header reading for invalid extended time headers. -* Windows Store assembly is now strong named -* Known issues with Long Tar names being worked on -* Updated to VS2013 -* Portable targets SL5 and Windows Phone 8 (up from SL4 and WP7) - -### Version 0.10.1 - -* Fixed 7Zip extraction performance problem - -### Version 0.10: - -* Added support for RAR Decryption (thanks to https://github.com/hrasyid) -* Embedded some BouncyCastle crypto classes to allow RAR Decryption and Winzip AES Decryption in Portable and Windows Store DLLs -* Built in Release (I think) - -XZ implementation based on: https://github.com/sambott/XZ.NET by @sambott - -XZ BCJ filters support contributed by Louis-Michel Bergeron, on behalf of aDolus Technology Inc. - 2022 - -7Zip implementation based on: https://code.google.com/p/managed-lzma/ - -LICENSE -Copyright (c) 2000 - 2011 The Legion Of The Bouncy Castle (http://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# Attention +This implementation of BZip2InputStream has been heavily modified from the original SharpZipLib implementation and is more optimized. +The original implementation can be found [here](https://github.com/icsharpcode/SharpZipLib/tree/master/src/ICSharpCode.SharpZipLib/BZip2) + +## Benchmark result +File: UnityPlayer.dll.bz2 (The UnityPlayer.dll file was originated from Honkai Impact 3rd (Mainland China server) v8.9.0, compressed with BZip2) +Size: 12,869,917 bytes compressed (30,144,408 bytes uncompressed) +Uncompressed File Hash: 7D0B8178 (CRC32) + +| Method | Mean | Error | StdDev | Allocated | +|------------------------- |---------:|--------:|--------:|-----------:| +| BZip2InputStreamModified | 661.7 ms | 3.29 ms | 2.91 ms | 1.38 KB | +| BZip2InputStreamOld | 738.1 ms | 2.29 ms | 2.15 ms | 5481.11 KB | + +Hardware Bench: +- CPU: AMD Ryzen 7 9800X3D +- Memory: G.Skill Ripjaws M5 64GB (32x2) DDR5-6000 CL36-36-36-96 + +Big credits to the original authors of SharpZipLib for their work on BZip2 decompression. +- [Mike Krüger](http://www.icsharpcode.net/pub/relations/krueger.aspx) +- John Reilly +- David Pierson +- Neil McNeight \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/CompressionStreamHelper.cs b/SharpHDiffPatch.Core/Binary/Compression/CompressionStreamHelper.cs index 69e5591..41b88bb 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/CompressionStreamHelper.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/CompressionStreamHelper.cs @@ -3,21 +3,25 @@ // ReSharper disable CommentTypo // ReSharper disable InconsistentNaming -using SharpCompress.Compressors.BZip2; -using SharpCompress.Compressors.LZMA; -using SharpHDiffPatch.Core.Binary.Streams; using System; -#if NET6_0_OR_GREATER -using System.Collections.Generic; -#endif using System.IO; using System.IO.Compression; using System.Runtime.InteropServices; +using SharpHDiffPatch.Core.Binary.Compression.BZip2; +using SharpHDiffPatch.Core.Binary.Compression.Lzma; +using SharpHDiffPatch.Core.Binary.Streams; + +#if NET6_0_OR_GREATER +using System.Collections.Generic; +using ZstdNet; +#endif + #if NETSTANDARD2_0_OR_GREATER || NET6_0_OR_GREATER using ZstdManagedDecompressor = ZstdSharp.Decompressor; using ZstdManagedDecompressorParameter = ZstdSharp.Unsafe.ZSTD_dParameter; using ZstdManagedStream = ZstdSharp.DecompressionStream; #endif + #if !NETSTANDARD2_0_OR_GREATER using ZstdNativeDecompressor = ZstdNet.DecompressionOptions; using ZstdNativeDecompressorParameter = ZstdNet.ZSTD_dParameter; @@ -66,24 +70,16 @@ internal static void GetDecompressStreamPlugin(CompressionMode type, Stream sour return; } - switch (type) + decompStream = type switch { - case CompressionMode.nocomp: - decompStream = rawStream; break; - case CompressionMode.zstd: - decompStream = CreateZstdStream(rawStream); break; - case CompressionMode.zlib: - decompStream = new DeflateStream(rawStream, System.IO.Compression.CompressionMode.Decompress, true); break; - case CompressionMode.bz2: - decompStream = new CBZip2InputStream(rawStream, false, true); break; - case CompressionMode.pbz2: - decompStream = new CBZip2InputStream(rawStream, true, true); break; - case CompressionMode.lzma: - case CompressionMode.lzma2: - decompStream = CreateLzmaStream(rawStream); break; - default: - throw new NotSupportedException($"[PatchCore::GetDecompressStreamPlugin] Compression Type: {type} is not supported"); - } + CompressionMode.nocomp => rawStream, + CompressionMode.zstd => CreateZstdStream(rawStream), + CompressionMode.zlib => new DeflateStream(rawStream, System.IO.Compression.CompressionMode.Decompress, true), + CompressionMode.bz2 => new BZip2InputStream(rawStream, false, true), + CompressionMode.pbz2 => new BZip2InputStream(rawStream, true, true), + CompressionMode.lzma or CompressionMode.lzma2 => CreateLzmaStream(rawStream), + _ => throw new NotSupportedException($"[PatchCore::GetDecompressStreamPlugin] Compression Type: {type} is not supported") + }; } private static Stream CreateZstdStream(Stream rawStream) @@ -91,7 +87,7 @@ private static Stream CreateZstdStream(Stream rawStream) if (_createZstdStreamFallback != null) return _createZstdStreamFallback(rawStream); #if !(NETSTANDARD2_0_OR_GREATER || NET461_OR_GREATER) - if (ZstdNet.DllUtils.IsLibraryExist(ZstdNet.DllUtils.DllName)) + if (DllUtils.IsLibraryExist(DllUtils.DllName)) _createZstdStreamFallback = CreateZstdNativeStream; else _createZstdStreamFallback = CreateZstdManagedStream; @@ -109,7 +105,7 @@ private static Stream CreateZstdStream(Stream rawStream) */ #if !NETSTANDARD2_0_OR_GREATER private static Stream CreateZstdNativeStream(Stream rawStream) => - new ZstdNativeStream(rawStream, new ZstdNativeDecompressor(null, new Dictionary() + new ZstdNativeStream(rawStream, new ZstdNativeDecompressor(null, new Dictionary { { ZstdNativeDecompressorParameter.ZSTD_d_windowLogMax, ZstdWindowLogMax } })); @@ -117,7 +113,7 @@ private static Stream CreateZstdNativeStream(Stream rawStream) => private static Stream CreateZstdManagedStream(Stream rawStream) { - ZstdManagedDecompressor decompressor = new ZstdManagedDecompressor(); + ZstdManagedDecompressor decompressor = new(); decompressor.SetParameter(ZstdManagedDecompressorParameter.ZSTD_d_windowLogMax, ZstdWindowLogMax); return new ZstdManagedStream(rawStream, decompressor, 16 << 10); } @@ -125,14 +121,14 @@ private static Stream CreateZstdManagedStream(Stream rawStream) private static Stream CreateLzmaStream(Stream rawStream) { int propLen = rawStream.ReadByte(); - if (propLen != 5) return new LzmaStream([(byte)propLen], rawStream); // Get LZMA2 if propLen != 5 + if (propLen != 5) return new LzmaInputStream([(byte)propLen], rawStream, true); // Get LZMA2 if propLen != 5 // Get LZMA if propLen == 5 byte[] props = new byte[propLen]; _ = rawStream.Read(props, 0, propLen); int dicSize = MemoryMarshal.Read(props.AsSpan(1)); HDiffPatch.Event.PushLog($"[PatchCore::CreateLzmaStream] Assigning LZMA stream with dictionary size: {dicSize}", Verbosity.Verbose); - return new LzmaStream(props, rawStream, -1, -1, rawStream, false); + return new LzmaInputStream(props, rawStream, -1, -1, rawStream, false, true); } } } diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/BitVector.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/BitVector.cs deleted file mode 100644 index 312d282..0000000 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/BitVector.cs +++ /dev/null @@ -1,102 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace SharpCompress.Compressors.LZMA -{ - internal class BitVector - { - private readonly uint[] _mBits; - - public BitVector(int length) - { - Length = length; - _mBits = new uint[(length + 31) >> 5]; - } - - public BitVector(int length, bool initValue) - { - Length = length; - _mBits = new uint[(length + 31) >> 5]; - - if (initValue) - { - for (var i = 0; i < _mBits.Length; i++) - { - _mBits[i] = ~0u; - } - } - } - - public BitVector(List bits) - : this(bits.Count) - { - for (var i = 0; i < bits.Count; i++) - { - if (bits[i]) - { - SetBit(i); - } - } - } - - public bool[] ToArray() - { - var bits = new bool[Length]; - for (var i = 0; i < bits.Length; i++) - { - bits[i] = this[i]; - } - return bits; - } - - public int Length { get; } - - public bool this[int index] - { - get - { - if (index < 0 || index >= Length) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - return (_mBits[index >> 5] & (1u << (index & 31))) != 0; - } - } - - public void SetBit(int index) - { - if (index < 0 || index >= Length) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - _mBits[index >> 5] |= 1u << (index & 31); - } - - internal bool GetAndSet(int index) - { - if (index < 0 || index >= Length) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - var bits = _mBits[index >> 5]; - var mask = 1u << (index & 31); - _mBits[index >> 5] |= mask; - return (bits & mask) != 0; - } - - public override string ToString() - { - var sb = new StringBuilder(Length); - for (var i = 0; i < Length; i++) - { - sb.Append(this[i] ? 'x' : '.'); - } - return sb.ToString(); - } - } - -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/ICoder.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/ICoder.cs deleted file mode 100644 index ad24be7..0000000 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/ICoder.cs +++ /dev/null @@ -1,167 +0,0 @@ -using System; -using System.IO; - -namespace SharpCompress.Compressors.LZMA -{ - /// - /// The exception that is thrown when an error in input stream occurs during decoding. - /// - internal class DataErrorException : Exception - { - public DataErrorException() - : base("Data Error") { } - } - - /// - /// The exception that is thrown when the value of an argument is outside the allowable range. - /// - internal class InvalidParamException : Exception - { - public InvalidParamException() - : base("Invalid Parameter") { } - } - - internal interface ICodeProgress - { - /// - /// Callback progress. - /// - /// - /// input size. -1 if unknown. - /// - /// - /// output size. -1 if unknown. - /// - void SetProgress(long inSize, long outSize); - } - - internal interface ICoder - { - /// - /// Codes streams. - /// - /// - /// input Stream. - /// - /// - /// output Stream. - /// - /// - /// input Size. -1 if unknown. - /// - /// - /// output Size. -1 if unknown. - /// - /// - /// callback progress reference. - /// - void Code(Stream inStream, Stream outStream, long inSize, long outSize, ICodeProgress progress); - } - - /* - public interface ICoder2 - { - void Code(ISequentialInStream []inStreams, - const UInt64 []inSizes, - ISequentialOutStream []outStreams, - UInt64 []outSizes, - ICodeProgress progress); - }; - */ - - /// - /// Provides the fields that represent properties idenitifiers for compressing. - /// - internal enum CoderPropId - { - /// - /// Specifies default property. - /// - DefaultProp = 0, - - /// - /// Specifies size of dictionary. - /// - DictionarySize, - - /// - /// Specifies size of memory for PPM*. - /// - UsedMemorySize, - - /// - /// Specifies order for PPM methods. - /// - Order, - - /// - /// Specifies Block Size. - /// - BlockSize, - - /// - /// Specifies number of postion state bits for LZMA (0 - x - 4). - /// - PosStateBits, - - /// - /// Specifies number of literal context bits for LZMA (0 - x - 8). - /// - LitContextBits, - - /// - /// Specifies number of literal position bits for LZMA (0 - x - 4). - /// - LitPosBits, - - /// - /// Specifies number of fast bytes for LZ*. - /// - NumFastBytes, - - /// - /// Specifies match finder. LZMA: "BT2", "BT4" or "BT4B". - /// - MatchFinder, - - /// - /// Specifies the number of match finder cyckes. - /// - MatchFinderCycles, - - /// - /// Specifies number of passes. - /// - NumPasses, - - /// - /// Specifies number of algorithm. - /// - Algorithm, - - /// - /// Specifies the number of threads. - /// - NumThreads, - - /// - /// Specifies mode with end marker. - /// - EndMarker - } - - internal interface ISetCoderProperties - { - void SetCoderProperties(ReadOnlySpan propIDs, ReadOnlySpan properties); - } - - internal interface IWriteCoderProperties - { - void WriteCoderProperties(Stream outStream); - } - - internal interface ISetDecoderProperties - { - void SetDecoderProperties(byte[] properties); - } -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LZ/LzOutWindow.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LZ/LzOutWindow.cs index 818b61c..913c8d9 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LZ/LzOutWindow.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LZ/LzOutWindow.cs @@ -1,200 +1,269 @@ using System; +using System.Buffers; using System.IO; +using System.Runtime.CompilerServices; -namespace SharpCompress.Compressors.LZMA.LZ +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma.LZ; + +internal class OutWindow : IDisposable { - internal class OutWindow + private byte[] _buffer = []; + private int _windowSize; + private int _pos; + private int _streamPos; + private int _pendingLen; + private int _pendingDist; + private Stream _stream; + + public long Total; + public long Limit; + + public void Create(int windowSize) + { + if (_buffer.Length < windowSize) + { + ReturnPooledBuffer(); + _buffer = ArrayPool.Shared.Rent(windowSize); + } + + _buffer[windowSize - 1] = 0; + _windowSize = windowSize; + _pos = 0; + _streamPos = 0; + _pendingLen = 0; + Total = 0; + Limit = 0; + } + + public void Reset() => Create(_windowSize); + + public void Init(Stream stream) { - private byte[] _buffer; - private int _windowSize; - private int _pos; - private int _streamPos; - private int _pendingLen; - private int _pendingDist; - private Stream _stream; + ReleaseStream(); + _stream = stream; + } - public long _total; - public long _limit; + public void Train(Stream stream) + { + long len = stream.Length; + int size = len < _windowSize ? (int)len : _windowSize; + stream.Position = len - size; - public void Create(int windowSize) + Total = 0; + Limit = size; + _pos = _windowSize - size; + CopyStream(stream, size); + + if (_pos == _windowSize) { - if (_windowSize != windowSize) - { - _buffer = new byte[windowSize]; - } - else - { - _buffer[windowSize - 1] = 0; - } - _windowSize = windowSize; _pos = 0; - _streamPos = 0; - _pendingLen = 0; - _total = 0; - _limit = 0; } + _streamPos = _pos; + } - public void Reset() => Create(_windowSize); + public void ReleaseStream() + { + Flush(); + _stream = null; + } - public void Init(Stream stream) + public void Flush() + { + if (_stream is null) { - ReleaseStream(); - _stream = stream; + return; } - - public void Train(Stream stream) + int size = _pos - _streamPos; + if (size == 0) { - var len = stream.Length; - var size = (len < _windowSize) ? (int)len : _windowSize; - stream.Position = len - size; - _total = 0; - _limit = size; - _pos = _windowSize - size; - CopyStream(stream, size); - if (_pos == _windowSize) - { - _pos = 0; - } - _streamPos = _pos; + return; } - public void ReleaseStream() + _stream.Write(_buffer, _streamPos, size); + if (_pos >= _windowSize) { - Flush(); - _stream = null; + _pos = 0; } - public void Flush() + _streamPos = _pos; + } + + public void CopyBlock(int distance, int len) + { + int remaining = len; + int source = _pos - distance - 1; + if (source < 0) { - if (_stream is null) - { - return; - } - var size = _pos - _streamPos; - if (size == 0) - { - return; - } - _stream.Write(_buffer, _streamPos, size); - if (_pos >= _windowSize) - { - _pos = 0; - } - _streamPos = _pos; + source += _windowSize; } - public void CopyBlock(int distance, int len) + while (remaining > 0 && _pos < _windowSize && Total < Limit) { - var size = len; - var pos = _pos - distance - 1; - if (pos < 0) + int copySize = Math.Min(remaining, _windowSize - _pos); + long available = Limit - Total; + if (copySize > available) { - pos += _windowSize; + copySize = (int)available; } - for (; size > 0 && _pos < _windowSize && _total < _limit; size--) + + ref byte buffer = ref _buffer[0]; + int beforeWrap = Math.Min(copySize, _windowSize - source); + CopyBytes(ref buffer, source, _pos, beforeWrap); + source += beforeWrap; + _pos += beforeWrap; + + int afterWrap = copySize - beforeWrap; + if (afterWrap > 0) { - if (pos >= _windowSize) - { - pos = 0; - } - _buffer[_pos++] = _buffer[pos++]; - _total++; - if (_pos >= _windowSize) - { - Flush(); - } + source = 0; + CopyBytes(ref buffer, source, _pos, afterWrap); + source += afterWrap; + _pos += afterWrap; } - _pendingLen = size; - _pendingDist = distance; - } - public void PutByte(byte b) - { - _buffer[_pos++] = b; - _total++; + remaining -= copySize; + Total += copySize; if (_pos >= _windowSize) { Flush(); } } - public byte GetByte(int distance) + _pendingLen = remaining; + _pendingDist = distance; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void CopyBytes(ref byte buffer, int sourceOffset, int destinationOffset, int count) + { + ref byte source = ref Unsafe.Add(ref buffer, sourceOffset); + ref byte destination = ref Unsafe.Add(ref buffer, destinationOffset); + + bool isNonOverlapping = sourceOffset + count <= destinationOffset || destinationOffset + count <= sourceOffset; + + if (count >= 16 && isNonOverlapping) { - var pos = _pos - distance - 1; - if (pos < 0) - { - pos += _windowSize; - } - return _buffer[pos]; + ReadOnlySpan sourceSpan = new(Unsafe.AsPointer(ref source), count); + Span destinationSpan = new(Unsafe.AsPointer(ref destination), count); + + sourceSpan.CopyTo(destinationSpan); + return; } - public int CopyStream(Stream stream, int len) + ref byte destinationEnd = ref Unsafe.Add(ref destination, count); + while (Unsafe.IsAddressLessThan(ref destination, ref destinationEnd)) { - var size = len; - while (size > 0 && _pos < _windowSize && _total < _limit) - { - var curSize = _windowSize - _pos; - if (curSize > _limit - _total) - { - curSize = (int)(_limit - _total); - } - if (curSize > size) - { - curSize = size; - } - var numReadBytes = stream.Read(_buffer, _pos, curSize); - if (numReadBytes == 0) - { - throw new DataErrorException(); - } - size -= numReadBytes; - _pos += numReadBytes; - _total += numReadBytes; - if (_pos >= _windowSize) - { - Flush(); - } - } - return len - size; + destination = source; + source = ref Unsafe.Add(ref source, 1); + destination = ref Unsafe.Add(ref destination, 1); } + } - public void SetLimit(long size) => _limit = _total + size; + public void PutByte(byte b) + { + _buffer[_pos++] = b; + Total++; - public bool HasSpace => _pos < _windowSize && _total < _limit; + if (_pos >= _windowSize) + { + Flush(); + } + } - public bool HasPending => _pendingLen > 0; + public byte GetByte(int distance) + { + int pos = _pos - distance - 1; + if (pos < 0) + { + pos += _windowSize; + } + return _buffer[pos]; + } - public int Read(byte[] buffer, int offset, int count) + public int CopyStream(Stream stream, int len) + { + int size = len; + while (size > 0 && _pos < _windowSize && Total < Limit) { - if (_streamPos >= _pos) + int curSize = _windowSize - _pos; + if (curSize > Limit - Total) { - return 0; + curSize = (int)(Limit - Total); } - var size = _pos - _streamPos; - if (size > count) + if (curSize > size) { - size = count; + curSize = size; } - Buffer.BlockCopy(_buffer, _streamPos, buffer, offset, size); - _streamPos += size; - if (_streamPos >= _windowSize) + + int numReadBytes = stream.Read(_buffer, _pos, curSize); + if (numReadBytes == 0) { - _pos = 0; - _streamPos = 0; + throw new LzmaDataErrorException(); } - return size; - } - public void CopyPending() - { - if (_pendingLen > 0) + size -= numReadBytes; + _pos += numReadBytes; + Total += numReadBytes; + if (_pos >= _windowSize) { - CopyBlock(_pendingDist, _pendingLen); + Flush(); } } + return len - size; + } + + public void SetLimit(long size) => Limit = Total + size; + + public bool HasSpace => _pos < _windowSize && Total < Limit; - public int AvailableBytes => _pos - _streamPos; + public bool HasPending => _pendingLen > 0; + + public int Read(byte[] buffer, int offset, int count) + { + if (_streamPos >= _pos) + { + return 0; + } + + int size = _pos - _streamPos; + if (size > count) + { + size = count; + } + + _buffer.AsSpan(_streamPos, size).CopyTo(buffer.AsSpan(offset, size)); + _streamPos += size; + if (_streamPos < _windowSize) return size; + + _pos = 0; + _streamPos = 0; + return size; + } + + public void CopyPending() + { + if (_pendingLen > 0) + { + CopyBlock(_pendingDist, _pendingLen); + } } + public int AvailableBytes => _pos - _streamPos; + + public void Dispose() + { + ReleaseStream(); + ReturnPooledBuffer(); + } + + private void ReturnPooledBuffer() + { + byte[] buffer = _buffer; + _buffer = []; + if (buffer.Length != 0) + { + ArrayPool.Shared.Return(buffer); + } + } } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaBase.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaBase.cs index 02950af..72f1650 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaBase.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaBase.cs @@ -1,94 +1,70 @@ -namespace SharpCompress.Compressors.LZMA +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma; + +internal abstract class Base { - internal abstract class Base + public const uint KNumStates = 12; + + public struct State { - public const uint K_NUM_REP_DISTANCES = 4; - public const uint K_NUM_STATES = 12; + public uint Index; - // static byte []kLiteralNextStates = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; - // static byte []kMatchNextStates = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; - // static byte []kRepNextStates = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; - // static byte []kShortRepNextStates = {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + public void Init() => Index = 0; - public struct State + public void UpdateChar() { - public uint _index; - - public void Init() => _index = 0; - - public void UpdateChar() + switch (Index) { - if (_index < 4) - { - _index = 0; - } - else if (_index < 10) - { - _index -= 3; - } - else - { - _index -= 6; - } + case < 4: + Index = 0; + break; + case < 10: + Index -= 3; + break; + default: + Index -= 6; + break; } + } - public void UpdateMatch() => _index = (uint)(_index < 7 ? 7 : 10); - - public void UpdateRep() => _index = (uint)(_index < 7 ? 8 : 11); + public void UpdateMatch() => Index = (uint)(Index < 7 ? 7 : 10); - public void UpdateShortRep() => _index = (uint)(_index < 7 ? 9 : 11); + public void UpdateRep() => Index = (uint)(Index < 7 ? 8 : 11); - public bool IsCharState() => _index < 7; - } + public void UpdateShortRep() => Index = (uint)(Index < 7 ? 9 : 11); - public const int K_NUM_POS_SLOT_BITS = 6; - public const int K_DIC_LOG_SIZE_MIN = 0; + public bool IsCharState() => Index < 7; + } - // public const int kDicLogSizeMax = 30; - // public const uint kDistTableSizeMax = kDicLogSizeMax * 2; + public const int KNumPosSlotBits = 6; - public const int K_NUM_LEN_TO_POS_STATES_BITS = 2; // it's for speed optimization - public const uint K_NUM_LEN_TO_POS_STATES = 1 << K_NUM_LEN_TO_POS_STATES_BITS; + public const int KNumLenToPosStatesBits = 2; // it's for speed optimization + public const uint KNumLenToPosStates = 1 << KNumLenToPosStatesBits; - public const uint K_MATCH_MIN_LEN = 2; + public const uint KMatchMinLen = 2; - public static uint GetLenToPosState(uint len) + public static uint GetLenToPosState(uint len) + { + len -= KMatchMinLen; + if (len < KNumLenToPosStates) { - len -= K_MATCH_MIN_LEN; - if (len < K_NUM_LEN_TO_POS_STATES) - { - return len; - } - return K_NUM_LEN_TO_POS_STATES - 1; + return len; } + return KNumLenToPosStates - 1; + } - public const int K_NUM_ALIGN_BITS = 4; - public const uint K_ALIGN_TABLE_SIZE = 1 << K_NUM_ALIGN_BITS; - public const uint K_ALIGN_MASK = (K_ALIGN_TABLE_SIZE - 1); - - public const uint K_START_POS_MODEL_INDEX = 4; - public const uint K_END_POS_MODEL_INDEX = 14; - public const uint K_NUM_POS_MODELS = K_END_POS_MODEL_INDEX - K_START_POS_MODEL_INDEX; - - public const uint K_NUM_FULL_DISTANCES = 1 << ((int)K_END_POS_MODEL_INDEX / 2); - - public const uint K_NUM_LIT_POS_STATES_BITS_ENCODING_MAX = 4; - public const uint K_NUM_LIT_CONTEXT_BITS_MAX = 8; + public const int KNumAlignBits = 4; - public const int K_NUM_POS_STATES_BITS_MAX = 4; - public const uint K_NUM_POS_STATES_MAX = (1 << K_NUM_POS_STATES_BITS_MAX); - public const int K_NUM_POS_STATES_BITS_ENCODING_MAX = 4; - public const uint K_NUM_POS_STATES_ENCODING_MAX = (1 << K_NUM_POS_STATES_BITS_ENCODING_MAX); + public const uint KStartPosModelIndex = 4; + public const uint KEndPosModelIndex = 14; - public const int K_NUM_LOW_LEN_BITS = 3; - public const int K_NUM_MID_LEN_BITS = 3; - public const int K_NUM_HIGH_LEN_BITS = 8; - public const uint K_NUM_LOW_LEN_SYMBOLS = 1 << K_NUM_LOW_LEN_BITS; - public const uint K_NUM_MID_LEN_SYMBOLS = 1 << K_NUM_MID_LEN_BITS; + public const uint KNumFullDistances = 1 << ((int)KEndPosModelIndex / 2); - public const uint K_NUM_LEN_SYMBOLS = - K_NUM_LOW_LEN_SYMBOLS + K_NUM_MID_LEN_SYMBOLS + (1 << K_NUM_HIGH_LEN_BITS); + public const int KNumPosStatesBitsMax = 4; + public const uint KNumPosStatesMax = 1 << KNumPosStatesBitsMax; - public const uint K_MATCH_MAX_LEN = K_MATCH_MIN_LEN + K_NUM_LEN_SYMBOLS - 1; - } + public const int KNumLowLenBits = 3; + public const int KNumMidLenBits = 3; + public const int KNumHighLenBits = 8; + public const uint KNumLowLenSymbols = 1 << KNumLowLenBits; + public const uint KNumMidLenSymbols = 1 << KNumMidLenBits; } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaDecoder.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaDecoder.cs index 856c2fe..d89e4d7 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaDecoder.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaDecoder.cs @@ -1,290 +1,271 @@ using System; +using System.Buffers; using System.IO; -using SharpCompress.Compressors.LZMA.LZ; -using SharpCompress.Compressors.LZMA.RangeCoder; +using System.Runtime.CompilerServices; +using SharpHDiffPatch.Core.Binary.Compression.Lzma.LZ; +using SharpHDiffPatch.Core.Binary.Compression.Lzma.RangeCoder; -namespace SharpCompress.Compressors.LZMA +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma; + +internal class Decoder : IDisposable { - internal class Decoder : ICoder, ISetDecoderProperties // ,System.IO.Stream + private class LenDecoder : IDisposable { - private class LenDecoder + private const int KLowModelsCount = (int)Base.KNumPosStatesMax << Base.KNumLowLenBits; + private const int KMidModelsOffset = KLowModelsCount; + private const int KHighModelsOffset = KMidModelsOffset + ((int)Base.KNumPosStatesMax << Base.KNumMidLenBits); + private const int KModelsCount = KHighModelsOffset + (1 << Base.KNumHighLenBits); + + private BitDecoder _choice; + private BitDecoder _choice2; + private BitDecoder[] _models = []; + private uint _numPosStates; + + public void Create(uint numPosStates) { - private BitDecoder _choice = new BitDecoder(); - private BitDecoder _choice2 = new BitDecoder(); - private readonly BitTreeDecoder[] _lowCoder = new BitTreeDecoder[Base.K_NUM_POS_STATES_MAX]; - private readonly BitTreeDecoder[] _midCoder = new BitTreeDecoder[Base.K_NUM_POS_STATES_MAX]; - private BitTreeDecoder _highCoder = new BitTreeDecoder(Base.K_NUM_HIGH_LEN_BITS); - private uint _numPosStates; - - public void Create(uint numPosStates) + if (_models.Length == 0) { - for (var posState = _numPosStates; posState < numPosStates; posState++) - { - _lowCoder[posState] = new BitTreeDecoder(Base.K_NUM_LOW_LEN_BITS); - _midCoder[posState] = new BitTreeDecoder(Base.K_NUM_MID_LEN_BITS); - } - _numPosStates = numPosStates; + _models = ArrayPool.Shared.Rent(KModelsCount); } + _numPosStates = numPosStates; + } - public void Init() + public void Init() + { + _choice.Init(); + _choice2.Init(); + + int activeLowModels = (int)_numPosStates << Base.KNumLowLenBits; + int activeMidModels = (int)_numPosStates << Base.KNumMidLenBits; + BitDecoder.Init(_models, 0, activeLowModels); + BitDecoder.Init(_models, KMidModelsOffset, activeMidModels); + BitDecoder.Init(_models, KHighModelsOffset, 1 << Base.KNumHighLenBits); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint Decode(RangeDecoder rangeDecoder, uint posState) + { + if (_choice.Decode(rangeDecoder) == 0) { - _choice.Init(); - for (uint posState = 0; posState < _numPosStates; posState++) - { - _lowCoder[posState].Init(); - _midCoder[posState].Init(); - } - _choice2.Init(); - _highCoder.Init(); + int modelOffset = (int)posState << Base.KNumLowLenBits; + return BitTreeDecoder.Decode(_models, modelOffset, rangeDecoder, Base.KNumLowLenBits); } - public uint Decode(RangeCoder.Decoder rangeDecoder, uint posState) + uint symbol = Base.KNumLowLenSymbols; + if (_choice2.Decode(rangeDecoder) == 0) { - if (_choice.Decode(rangeDecoder) == 0) - { - return _lowCoder[posState].Decode(rangeDecoder); - } - var symbol = Base.K_NUM_LOW_LEN_SYMBOLS; - if (_choice2.Decode(rangeDecoder) == 0) - { - symbol += _midCoder[posState].Decode(rangeDecoder); - } - else - { - symbol += Base.K_NUM_MID_LEN_SYMBOLS; - symbol += _highCoder.Decode(rangeDecoder); - } - return symbol; + int modelOffset = KMidModelsOffset + ((int)posState << Base.KNumMidLenBits); + symbol += BitTreeDecoder.Decode(_models, modelOffset, rangeDecoder, Base.KNumMidLenBits); + } + else + { + symbol += Base.KNumMidLenSymbols; + symbol += BitTreeDecoder.Decode(_models, KHighModelsOffset, rangeDecoder, Base.KNumHighLenBits); } + return symbol; } - private class LiteralDecoder + public void Dispose() { - private struct Decoder2 + BitDecoder[] models = _models; + _models = []; + _numPosStates = 0; + + if (models.Length != 0) { - private BitDecoder[] _decoders; + ArrayPool.Shared.Return(models); + } + } + } - public void Create() => _decoders = new BitDecoder[0x300]; + private class LiteralDecoder : IDisposable + { + private const int KNumModelsPerState = 0x300; - public void Init() - { - for (var i = 0; i < 0x300; i++) - { - _decoders[i].Init(); - } - } + private BitDecoder[] _coders; + private int _modelCount; + private int _numPrevBits; + private int _numPosBits; + private uint _posMask; - public byte DecodeNormal(RangeCoder.Decoder rangeDecoder) - { - uint symbol = 1; - do - { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); - } while (symbol < 0x100); - return (byte)symbol; - } - - public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, byte matchByte) - { - uint symbol = 1; - do - { - var matchBit = (uint)(matchByte >> 7) & 1; - matchByte <<= 1; - var bit = _decoders[((1 + matchBit) << 8) + symbol].Decode(rangeDecoder); - symbol = (symbol << 1) | bit; - if (matchBit != bit) - { - while (symbol < 0x100) - { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); - } - break; - } - } while (symbol < 0x100); - return (byte)symbol; - } + public void Create(int numPosBits, int numPrevBits) + { + if (_coders != null && _numPrevBits == numPrevBits && _numPosBits == numPosBits) + { + return; } - private Decoder2[] _coders; - private int _numPrevBits; - private int _numPosBits; - private uint _posMask; + _numPosBits = numPosBits; + _posMask = ((uint)1 << numPosBits) - 1; + _numPrevBits = numPrevBits; + + int numStates = 1 << (_numPrevBits + _numPosBits); + int modelCount = checked(numStates * KNumModelsPerState); + + ReturnModels(); + _coders = ArrayPool.Shared.Rent(modelCount); + _modelCount = modelCount; + } + + public void Init() => BitDecoder.Init(_coders, 0, _modelCount); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private uint GetState(uint pos, byte prevByte) => ((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits)); - public void Create(int numPosBits, int numPrevBits) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte DecodeNormal(RangeDecoder rangeDecoder, uint pos, byte prevByte) + { + ref BitDecoder decoders = ref Unsafe.Add( + ref _coders[0], + (int)GetState(pos, prevByte) * KNumModelsPerState); + uint symbol = 1; + do { - if (_coders != null && _numPrevBits == numPrevBits && _numPosBits == numPosBits) - { - return; - } - _numPosBits = numPosBits; - _posMask = ((uint)1 << numPosBits) - 1; - _numPrevBits = numPrevBits; - var numStates = (uint)1 << (_numPrevBits + _numPosBits); - _coders = new Decoder2[numStates]; - for (uint i = 0; i < numStates; i++) - { - _coders[i].Create(); - } - } + symbol = (symbol << 1) | Unsafe.Add(ref decoders, (int)symbol).Decode(rangeDecoder); + } while (symbol < 0x100); + return unchecked((byte)symbol); + } - public void Init() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public byte DecodeWithMatchByte( + RangeDecoder rangeDecoder, + uint pos, + byte prevByte, + byte matchByte) + { + ref BitDecoder decoders = ref Unsafe.Add( + ref _coders[0], + (int)GetState(pos, prevByte) * KNumModelsPerState); + uint symbol = 1; + do { - var numStates = (uint)1 << (_numPrevBits + _numPosBits); - for (uint i = 0; i < numStates; i++) - { - _coders[i].Init(); - } - } + uint matchBit = (uint)(matchByte >> 7) & 1; + matchByte <<= 1; - private uint GetState(uint pos, byte prevByte) => - ((pos & _posMask) << _numPrevBits) + (uint)(prevByte >> (8 - _numPrevBits)); + int modelIndex = (int)(((1 + matchBit) << 8) + symbol); + uint bit = Unsafe.Add(ref decoders, modelIndex).Decode(rangeDecoder); + symbol = (symbol << 1) | bit; - public byte DecodeNormal(RangeCoder.Decoder rangeDecoder, uint pos, byte prevByte) => - _coders[GetState(pos, prevByte)].DecodeNormal(rangeDecoder); + if (matchBit == bit) continue; - public byte DecodeWithMatchByte( - RangeCoder.Decoder rangeDecoder, - uint pos, - byte prevByte, - byte matchByte - ) => _coders[GetState(pos, prevByte)].DecodeWithMatchByte(rangeDecoder, matchByte); + while (symbol < 0x100) + { + symbol = (symbol << 1) | Unsafe.Add(ref decoders, (int)symbol).Decode(rangeDecoder); + } + break; + } while (symbol < 0x100); + return unchecked((byte)symbol); } - private OutWindow _outWindow; + public void Dispose() => ReturnModels(); + + private void ReturnModels() + { + if (_coders == null) return; - private readonly BitDecoder[] _isMatchDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; - private readonly BitDecoder[] _isRepDecoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG0Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG1Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRepG2Decoders = new BitDecoder[Base.K_NUM_STATES]; - private readonly BitDecoder[] _isRep0LongDecoders = new BitDecoder[ - Base.K_NUM_STATES << Base.K_NUM_POS_STATES_BITS_MAX - ]; + ArrayPool.Shared.Return(_coders); + _coders = null; + _modelCount = 0; + } + } - private readonly BitTreeDecoder[] _posSlotDecoder = new BitTreeDecoder[ - Base.K_NUM_LEN_TO_POS_STATES - ]; - private readonly BitDecoder[] _posDecoders = new BitDecoder[ - Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX - ]; + private OutWindow _outWindow; - private BitTreeDecoder _posAlignDecoder = new BitTreeDecoder(Base.K_NUM_ALIGN_BITS); + private const int KStatePosModelsCount = (int)Base.KNumStates << Base.KNumPosStatesBitsMax; + private const int KStateModelsCount = (int)Base.KNumStates; + private const int KIsMatchOffset = 0; + private const int KIsRepOffset = KIsMatchOffset + KStatePosModelsCount; + private const int KIsRepG0Offset = KIsRepOffset + KStateModelsCount; + private const int KIsRepG1Offset = KIsRepG0Offset + KStateModelsCount; + private const int KIsRepG2Offset = KIsRepG1Offset + KStateModelsCount; + private const int KIsRep0LongOffset = KIsRepG2Offset + KStateModelsCount; + private const int KPosDecodersOffset = KIsRep0LongOffset + KStatePosModelsCount; + private const int KPosDecodersCount = (int)(Base.KNumFullDistances - Base.KEndPosModelIndex); + private const int KPosSlotModelsOffset = KPosDecodersOffset + KPosDecodersCount; + private const int KPosSlotModelsCount = (int)Base.KNumLenToPosStates << Base.KNumPosSlotBits; + private const int KPosAlignModelsOffset = KPosSlotModelsOffset + KPosSlotModelsCount; + private const int KModelsCount = KPosAlignModelsOffset + (1 << Base.KNumAlignBits); - private readonly LenDecoder _lenDecoder = new LenDecoder(); - private readonly LenDecoder _repLenDecoder = new LenDecoder(); + private BitDecoder[] _models; + private readonly LenDecoder _lenDecoder = new(); + private readonly LenDecoder _repLenDecoder = new(); + private readonly LiteralDecoder _literalDecoder = new(); - private readonly LiteralDecoder _literalDecoder = new LiteralDecoder(); + private int _dictionarySize; - private int _dictionarySize; + private uint _posStateMask; - private uint _posStateMask; + private Base.State _state; + private uint _rep0, _rep1, _rep2, _rep3; - private Base.State _state = new Base.State(); - private uint _rep0, - _rep1, - _rep2, - _rep3; + public Decoder() + { + _dictionarySize = -1; + _models = ArrayPool.Shared.Rent(KModelsCount); + } - public Decoder() + private void CreateDictionary() + { + if (_dictionarySize < 0) { - _dictionarySize = -1; - for (var i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++) - { - _posSlotDecoder[i] = new BitTreeDecoder(Base.K_NUM_POS_SLOT_BITS); - } + throw new LzmaInvalidParamException(); } + _outWindow = new OutWindow(); + int blockSize = Math.Max(_dictionarySize, 1 << 12); + _outWindow.Create(blockSize); + } - private void CreateDictionary() + private void SetLiteralProperties(int lp, int lc) + { + if (lp > 8 || lc > 8) { - if (_dictionarySize < 0) - { - throw new InvalidParamException(); - } - _outWindow = new OutWindow(); - var blockSize = Math.Max(_dictionarySize, (1 << 12)); - _outWindow.Create(blockSize); + throw new LzmaInvalidParamException(); } - private void SetLiteralProperties(int lp, int lc) - { - if (lp > 8) - { - throw new InvalidParamException(); - } - if (lc > 8) - { - throw new InvalidParamException(); - } - _literalDecoder.Create(lp, lc); - } + _literalDecoder.Create(lp, lc); + } - private void SetPosBitsProperties(int pb) + private void SetPosBitsProperties(int pb) + { + if (pb > Base.KNumPosStatesBitsMax) { - if (pb > Base.K_NUM_POS_STATES_BITS_MAX) - { - throw new InvalidParamException(); - } - var numPosStates = (uint)1 << pb; - _lenDecoder.Create(numPosStates); - _repLenDecoder.Create(numPosStates); - _posStateMask = numPosStates - 1; + throw new LzmaInvalidParamException(); } - private void Init() - { - uint i; - for (i = 0; i < Base.K_NUM_STATES; i++) - { - for (uint j = 0; j <= _posStateMask; j++) - { - var index = (i << Base.K_NUM_POS_STATES_BITS_MAX) + j; - _isMatchDecoders[index].Init(); - _isRep0LongDecoders[index].Init(); - } - _isRepDecoders[i].Init(); - _isRepG0Decoders[i].Init(); - _isRepG1Decoders[i].Init(); - _isRepG2Decoders[i].Init(); - } + uint numPosStates = (uint)1 << pb; + _lenDecoder.Create(numPosStates); + _repLenDecoder.Create(numPosStates); + _posStateMask = numPosStates - 1; + } - _literalDecoder.Init(); - for (i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++) - { - _posSlotDecoder[i].Init(); - } + private void Init() + { + BitDecoder.Init(_models, 0, KModelsCount); - // _PosSpecDecoder.Init(); - for (i = 0; i < Base.K_NUM_FULL_DISTANCES - Base.K_END_POS_MODEL_INDEX; i++) - { - _posDecoders[i].Init(); - } + _literalDecoder.Init(); + _lenDecoder.Init(); + _repLenDecoder.Init(); - _lenDecoder.Init(); - _repLenDecoder.Init(); - _posAlignDecoder.Init(); + _state.Init(); + _rep0 = 0; + _rep1 = 0; + _rep2 = 0; + _rep3 = 0; + } - _state.Init(); - _rep0 = 0; - _rep1 = 0; - _rep2 = 0; - _rep3 = 0; + public void Code( + Stream inStream, + Stream outStream, + long inSize, + long outSize) + { + if (_outWindow is null) + { + CreateDictionary(); } - public void Code( - Stream inStream, - Stream outStream, - long inSize, - long outSize, - ICodeProgress progress - ) + if (_outWindow != null) { - if (_outWindow is null) - { - CreateDictionary(); - } _outWindow.Init(outStream); if (outSize > 0) { @@ -292,184 +273,194 @@ ICodeProgress progress } else { - _outWindow.SetLimit(long.MaxValue - _outWindow._total); + _outWindow.SetLimit(long.MaxValue - _outWindow.Total); } - var rangeDecoder = new RangeCoder.Decoder(); - rangeDecoder.Init(inStream); + RangeDecoder rangeDecoder = new(); + rangeDecoder.Init(inStream, inSize); Code(_dictionarySize, _outWindow, rangeDecoder); - _outWindow.ReleaseStream(); - rangeDecoder.ReleaseStream(); - - _outWindow = null; + _outWindow.Dispose(); + rangeDecoder.Dispose(); } - internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder rangeDecoder) + _outWindow = null; + } + + internal bool Code(int dictionarySize, OutWindow outWindow, RangeDecoder rangeDecoder) + { + int dictionarySizeCheck = Math.Max(dictionarySize, 1); + + outWindow.CopyPending(); + + ref BitDecoder models = ref _models[0]; + ref BitDecoder isMatchDecoders = ref Unsafe.Add(ref models, KIsMatchOffset); + ref BitDecoder isRepDecoders = ref Unsafe.Add(ref models, KIsRepOffset); + ref BitDecoder isRepG0Decoders = ref Unsafe.Add(ref models, KIsRepG0Offset); + ref BitDecoder isRepG1Decoders = ref Unsafe.Add(ref models, KIsRepG1Offset); + ref BitDecoder isRepG2Decoders = ref Unsafe.Add(ref models, KIsRepG2Offset); + ref BitDecoder isRep0LongDecoders = ref Unsafe.Add(ref models, KIsRep0LongOffset); + + while (outWindow.HasSpace) { - var dictionarySizeCheck = Math.Max(dictionarySize, 1); + uint posState = (uint)outWindow.Total & _posStateMask; + int stateIndex = (int)_state.Index; + int statePosIndex = (stateIndex << Base.KNumPosStatesBitsMax) + (int)posState; - outWindow.CopyPending(); + if (Unsafe.Add(ref isMatchDecoders, statePosIndex).Decode(rangeDecoder) == 0) + { + byte prevByte = outWindow.GetByte(0); + byte b = !_state.IsCharState() + ? _literalDecoder.DecodeWithMatchByte(rangeDecoder, (uint)outWindow.Total, prevByte, outWindow.GetByte((int)_rep0)) + : _literalDecoder.DecodeNormal(rangeDecoder, (uint)outWindow.Total, prevByte); - while (outWindow.HasSpace) + outWindow.PutByte(b); + _state.UpdateChar(); + } + else { - var posState = (uint)outWindow._total & _posStateMask; - if ( - _isMatchDecoders[ - (_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState - ].Decode(rangeDecoder) == 0 - ) + uint len; + if (Unsafe.Add(ref isRepDecoders, stateIndex).Decode(rangeDecoder) == 1) { - byte b; - var prevByte = outWindow.GetByte(0); - if (!_state.IsCharState()) + if (Unsafe.Add(ref isRepG0Decoders, stateIndex).Decode(rangeDecoder) == 0) { - b = _literalDecoder.DecodeWithMatchByte( - rangeDecoder, - (uint)outWindow._total, - prevByte, - outWindow.GetByte((int)_rep0) - ); + if (Unsafe.Add(ref isRep0LongDecoders, statePosIndex).Decode(rangeDecoder) == 0) + { + _state.UpdateShortRep(); + outWindow.PutByte(outWindow.GetByte((int)_rep0)); + continue; + } } else { - b = _literalDecoder.DecodeNormal( - rangeDecoder, - (uint)outWindow._total, - prevByte - ); - } - outWindow.PutByte(b); - _state.UpdateChar(); - } - else - { - uint len; - if (_isRepDecoders[_state._index].Decode(rangeDecoder) == 1) - { - if (_isRepG0Decoders[_state._index].Decode(rangeDecoder) == 0) + uint distance; + if (Unsafe.Add(ref isRepG1Decoders, stateIndex).Decode(rangeDecoder) == 0) { - if ( - _isRep0LongDecoders[ - (_state._index << Base.K_NUM_POS_STATES_BITS_MAX) + posState - ].Decode(rangeDecoder) == 0 - ) - { - _state.UpdateShortRep(); - outWindow.PutByte(outWindow.GetByte((int)_rep0)); - continue; - } + distance = _rep1; } else { - uint distance; - if (_isRepG1Decoders[_state._index].Decode(rangeDecoder) == 0) + if (Unsafe.Add(ref isRepG2Decoders, stateIndex).Decode(rangeDecoder) == 0) { - distance = _rep1; + distance = _rep2; } else { - if (_isRepG2Decoders[_state._index].Decode(rangeDecoder) == 0) - { - distance = _rep2; - } - else - { - distance = _rep3; - _rep3 = _rep2; - } - _rep2 = _rep1; + distance = _rep3; + _rep3 = _rep2; } - _rep1 = _rep0; - _rep0 = distance; + _rep2 = _rep1; } - len = _repLenDecoder.Decode(rangeDecoder, posState) + Base.K_MATCH_MIN_LEN; - _state.UpdateRep(); + _rep1 = _rep0; + _rep0 = distance; } - else + len = _repLenDecoder.Decode(rangeDecoder, posState) + Base.KMatchMinLen; + _state.UpdateRep(); + } + else + { + _rep3 = _rep2; + _rep2 = _rep1; + _rep1 = _rep0; + + len = Base.KMatchMinLen + _lenDecoder.Decode(rangeDecoder, posState); + _state.UpdateMatch(); + + int posSlotOffset = KPosSlotModelsOffset + ((int)Base.GetLenToPosState(len) << Base.KNumPosSlotBits); + uint posSlot = BitTreeDecoder.Decode(_models, posSlotOffset, rangeDecoder, Base.KNumPosSlotBits); + if (posSlot >= Base.KStartPosModelIndex) { - _rep3 = _rep2; - _rep2 = _rep1; - _rep1 = _rep0; - len = Base.K_MATCH_MIN_LEN + _lenDecoder.Decode(rangeDecoder, posState); - _state.UpdateMatch(); - var posSlot = _posSlotDecoder[Base.GetLenToPosState(len)].Decode(rangeDecoder); - if (posSlot >= Base.K_START_POS_MODEL_INDEX) + int numDirectBits = (int)((posSlot >> 1) - 1); + _rep0 = (2 | (posSlot & 1)) << numDirectBits; + + if (posSlot < Base.KEndPosModelIndex) { - var numDirectBits = (int)((posSlot >> 1) - 1); - _rep0 = ((2 | (posSlot & 1)) << numDirectBits); - if (posSlot < Base.K_END_POS_MODEL_INDEX) - { - _rep0 += BitTreeDecoder.ReverseDecode( - _posDecoders, - _rep0 - posSlot - 1, - rangeDecoder, - numDirectBits - ); - } - else - { - _rep0 += ( - rangeDecoder.DecodeDirectBits(numDirectBits - Base.K_NUM_ALIGN_BITS) - << Base.K_NUM_ALIGN_BITS - ); - _rep0 += _posAlignDecoder.ReverseDecode(rangeDecoder); - } + _rep0 += BitTreeDecoder.ReverseDecode( + _models, + (uint)KPosDecodersOffset + _rep0 - posSlot - 1, + rangeDecoder, + numDirectBits + ); } else { - _rep0 = posSlot; + _rep0 += rangeDecoder.DecodeDirectBits(numDirectBits - Base.KNumAlignBits) << Base.KNumAlignBits; + _rep0 += BitTreeDecoder.ReverseDecode( + _models, + KPosAlignModelsOffset, + rangeDecoder, + Base.KNumAlignBits); } } - if (_rep0 >= outWindow._total || _rep0 >= dictionarySizeCheck) + else { - if (_rep0 == 0xFFFFFFFF) - { - return true; - } - throw new DataErrorException(); + _rep0 = posSlot; } - outWindow.CopyBlock((int)_rep0, (int)len); } + if (_rep0 >= outWindow.Total || _rep0 >= dictionarySizeCheck) + { + return _rep0 == 0xFFFFFFFF + ? true + : throw new LzmaDataErrorException(); + } + outWindow.CopyBlock((int)_rep0, (int)len); } - return false; } + return false; + } - public void SetDecoderProperties(byte[] properties) + public void SetDecoderProperties(byte[] properties) + { + if (properties.Length < 1) { - if (properties.Length < 1) - { - throw new InvalidParamException(); - } - var lc = properties[0] % 9; - var remainder = properties[0] / 9; - var lp = remainder % 5; - var pb = remainder / 5; - if (pb > Base.K_NUM_POS_STATES_BITS_MAX) - { - throw new InvalidParamException(); - } - SetLiteralProperties(lp, lc); - SetPosBitsProperties(pb); - Init(); - if (properties.Length >= 5) - { - _dictionarySize = 0; - for (var i = 0; i < 4; i++) - { - _dictionarySize += properties[1 + i] << (i * 8); - } - } + throw new LzmaInvalidParamException(); } - public void Train(Stream stream) + int lc = properties[0] % 9; + int remainder = properties[0] / 9; + int lp = remainder % 5; + int pb = remainder / 5; + if (pb > Base.KNumPosStatesBitsMax) { - if (_outWindow is null) - { - CreateDictionary(); - } - _outWindow.Train(stream); + throw new LzmaInvalidParamException(); + } + + SetLiteralProperties(lp, lc); + SetPosBitsProperties(pb); + Init(); + + if (properties.Length < 5) return; + + _dictionarySize = 0; + for (int i = 0; i < 4; i++) + { + _dictionarySize += properties[1 + i] << (i * 8); + } + } + + public void Train(Stream stream) + { + if (_outWindow is null) + { + CreateDictionary(); + } + + _outWindow?.Train(stream); + } + + public void Dispose() + { + BitDecoder[] models = _models; + _models = []; + + _lenDecoder.Dispose(); + _repLenDecoder.Dispose(); + _literalDecoder.Dispose(); + + if (models.Length != 0) + { + ArrayPool.Shared.Return(models); } } -} \ No newline at end of file +} diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaExceptions.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaExceptions.cs new file mode 100644 index 0000000..9f02974 --- /dev/null +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaExceptions.cs @@ -0,0 +1,13 @@ +using System; + +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma; + +/// +/// The exception that is thrown when an error in input stream occurs during decoding. +/// +internal class LzmaDataErrorException() : Exception("Data Error"); + +/// +/// The exception that is thrown when the value of an argument is outside the allowable range. +/// +internal class LzmaInvalidParamException() : Exception("Invalid Parameter"); \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaInputStream.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaInputStream.cs new file mode 100644 index 0000000..7b1b5c2 --- /dev/null +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaInputStream.cs @@ -0,0 +1,287 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using SharpHDiffPatch.Core.Binary.Compression.Lzma.LZ; +using SharpHDiffPatch.Core.Binary.Compression.Lzma.RangeCoder; + +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma; + +public sealed class LzmaInputStream : Stream +{ + private readonly Stream _inputStream; + private readonly long _inputSize; + private readonly long _outputSize; + + private readonly int _dictionarySize; + private readonly OutWindow _outWindow = new(); + private readonly RangeDecoder _rangeDecoder = new(); + private Decoder _decoder; + + private long _position; + private bool _endReached; + private long _availableBytes; + private long _rangeDecoderLimit; + private long _inputPosition; + + // LZMA2 + private readonly bool _isLzma2; + private readonly bool _leaveOpen; + + private bool _uncompressedChunk; + private bool _needDictReset = true; + private bool _needProps = true; + private bool _isDisposed; + + public LzmaInputStream(byte[] properties, Stream inputStream, bool leaveOpen = false) + : this(properties, inputStream, -1, -1, null, properties.Length < 5, leaveOpen) { } + + public LzmaInputStream(byte[] properties, Stream inputStream, long inputSize, bool leaveOpen = false) + : this(properties, inputStream, inputSize, -1, null, properties.Length < 5, leaveOpen) { } + + public LzmaInputStream(byte[] properties, Stream inputStream, long inputSize, long outputSize, bool leaveOpen = false) + : this(properties, inputStream, inputSize, outputSize, null, properties.Length < 5, leaveOpen) { } + + public LzmaInputStream( + byte[] properties, + Stream inputStream, + long inputSize, + long outputSize, + Stream presetDictionary, + bool isLzma2, + bool leaveOpen = false) + { + _inputStream = inputStream; + _inputSize = inputSize; + _outputSize = outputSize; + _isLzma2 = isLzma2; + _leaveOpen = leaveOpen; + + if (!isLzma2) + { + _dictionarySize = BinaryPrimitives.ReadInt32LittleEndian(properties.AsSpan(1)); + _outWindow.Create(_dictionarySize); + if (presetDictionary != null) + { + _outWindow.Train(presetDictionary); + } + + _rangeDecoder.Init(inputStream, inputSize); + + _decoder = new Decoder(); + _decoder.SetDecoderProperties(properties); + Properties = properties; + + _availableBytes = outputSize < 0 ? long.MaxValue : outputSize; + _rangeDecoderLimit = inputSize; + } + else + { + _dictionarySize = 2 | (properties[0] & 1); + _dictionarySize <<= (properties[0] >> 1) + 11; + + _outWindow.Create(_dictionarySize); + if (presetDictionary != null) + { + _outWindow.Train(presetDictionary); + _needDictReset = false; + } + + Properties = new byte[1]; + _availableBytes = 0; + } + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override void Flush() { } + + protected override void Dispose(bool disposing) + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + _decoder?.Dispose(); + _rangeDecoder.Dispose(); + _outWindow.Dispose(); + + if (disposing && !_leaveOpen) + { + _inputStream?.Dispose(); + } + base.Dispose(disposing); + } + + public override long Length => _position + _availableBytes; + + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (_endReached) + { + return 0; + } + + int total = 0; + while (total < count) + { + if (_availableBytes == 0) + { + if (_isLzma2) + { + DecodeChunkHeader(); + } + else + { + _endReached = true; + } + if (_endReached) + { + break; + } + } + + int toProcess = count - total; + if (toProcess > _availableBytes) + { + toProcess = (int)_availableBytes; + } + + _outWindow.SetLimit(toProcess); + if (_uncompressedChunk) + { + _inputPosition += _outWindow.CopyStream(_inputStream, toProcess); + } + else if (_decoder.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) + { + _availableBytes = _outWindow.AvailableBytes; + } + + int read = _outWindow.Read(buffer, offset, toProcess); + total += read; + offset += read; + _position += read; + _availableBytes -= read; + + if (_availableBytes != 0 || _uncompressedChunk) continue; + + // Check range corruption scenario + if (!_rangeDecoder.IsFinished || (_rangeDecoderLimit >= 0 && _rangeDecoder.Total != _rangeDecoderLimit)) + { + // Stream might have End Of Stream marker + _outWindow.SetLimit(toProcess + 1); + if (!_decoder.Code(_dictionarySize, _outWindow, _rangeDecoder)) + { + _rangeDecoder.ReleaseStream(); + throw new LzmaDataErrorException(); + } + } + + _rangeDecoder.ReleaseStream(); + _inputPosition += _rangeDecoder.Total; + if (_outWindow.HasPending) + { + throw new LzmaDataErrorException(); + } + } + + if (!_endReached) return total; + if ((_inputSize >= 0 && _inputPosition != _inputSize) || (_outputSize >= 0 && _position != _outputSize)) + { + throw new LzmaDataErrorException(); + } + + return total; + } + + private void DecodeChunkHeader() + { + int control = _inputStream.ReadByte(); + _inputPosition++; + + switch (control) + { + case 0x00: + _endReached = true; + return; + case >= 0xE0: + case 0x01: + _needProps = true; + _needDictReset = false; + _outWindow.Reset(); + break; + default: + { + if (_needDictReset) + { + throw new LzmaDataErrorException(); + } + + break; + } + } + + switch (control) + { + case >= 0x80: + { + _uncompressedChunk = false; + + _availableBytes = (control & 0x1F) << 16; + _availableBytes += (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; + _inputPosition += 2; + + _rangeDecoderLimit = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; + _inputPosition += 2; + + if (control >= 0xC0) + { + _needProps = false; + Properties[0] = (byte)_inputStream.ReadByte(); + _inputPosition++; + + _decoder ??= new Decoder(); + _decoder.SetDecoderProperties(Properties); + } + else if (_needProps) + { + throw new LzmaDataErrorException(); + } + else if (control >= 0xA0) + { + _decoder ??= new Decoder(); + _decoder.SetDecoderProperties(Properties); + } + + _rangeDecoder.Init(_inputStream, _rangeDecoderLimit); + break; + } + case > 0x02: + throw new LzmaDataErrorException(); + default: + _uncompressedChunk = true; + _availableBytes = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; + _inputPosition += 2; + break; + } + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public byte[] Properties { get; } +} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaStream.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaStream.cs deleted file mode 100644 index 14952ab..0000000 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/LzmaStream.cs +++ /dev/null @@ -1,285 +0,0 @@ -using System; -using System.Buffers.Binary; -using System.IO; -using SharpCompress.Compressors.LZMA.LZ; - -namespace SharpCompress.Compressors.LZMA -{ - internal class LzmaStream : Stream - { - private readonly Stream _inputStream; - private readonly long _inputSize; - private readonly long _outputSize; - - private readonly int _dictionarySize; - private readonly OutWindow _outWindow = new OutWindow(); - private readonly RangeCoder.Decoder _rangeDecoder = new RangeCoder.Decoder(); - private Decoder _decoder; - - private long _position; - private bool _endReached; - private long _availableBytes; - private long _rangeDecoderLimit; - private long _inputPosition; - - // LZMA2 - private readonly bool _isLzma2; - private bool _uncompressedChunk; - private bool _needDictReset = true; - private bool _needProps = true; - - private bool _isDisposed; - - internal LzmaStream(byte[] properties, Stream inputStream) - : this(properties, inputStream, -1, -1, null, properties.Length < 5) { } - - internal LzmaStream(byte[] properties, Stream inputStream, long inputSize) - : this(properties, inputStream, inputSize, -1, null, properties.Length < 5) { } - - internal LzmaStream(byte[] properties, Stream inputStream, long inputSize, long outputSize) - : this(properties, inputStream, inputSize, outputSize, null, properties.Length < 5) { } - - internal LzmaStream( - byte[] properties, - Stream inputStream, - long inputSize, - long outputSize, - Stream presetDictionary, - bool isLzma2 - ) - { - _inputStream = inputStream; - _inputSize = inputSize; - _outputSize = outputSize; - _isLzma2 = isLzma2; - - if (!isLzma2) - { - _dictionarySize = BinaryPrimitives.ReadInt32LittleEndian(properties.AsSpan(1)); - _outWindow.Create(_dictionarySize); - if (presetDictionary != null) - { - _outWindow.Train(presetDictionary); - } - - _rangeDecoder.Init(inputStream); - - _decoder = new Decoder(); - _decoder.SetDecoderProperties(properties); - Properties = properties; - - _availableBytes = outputSize < 0 ? long.MaxValue : outputSize; - _rangeDecoderLimit = inputSize; - } - else - { - _dictionarySize = 2 | (properties[0] & 1); - _dictionarySize <<= (properties[0] >> 1) + 11; - - _outWindow.Create(_dictionarySize); - if (presetDictionary != null) - { - _outWindow.Train(presetDictionary); - _needDictReset = false; - } - - Properties = new byte[1]; - _availableBytes = 0; - } - } - - public override bool CanRead => true; - - public override bool CanSeek => false; - - public override bool CanWrite => false; - - public override void Flush() { } - - protected override void Dispose(bool disposing) - { - if (_isDisposed) - { - return; - } - _isDisposed = true; - if (disposing) - { - _inputStream?.Dispose(); - } - base.Dispose(disposing); - } - - public override long Length => _position + _availableBytes; - - public override long Position - { - get => _position; - set => throw new NotSupportedException(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - if (_endReached) - { - return 0; - } - - var total = 0; - while (total < count) - { - if (_availableBytes == 0) - { - if (_isLzma2) - { - DecodeChunkHeader(); - } - else - { - _endReached = true; - } - if (_endReached) - { - break; - } - } - - var toProcess = count - total; - if (toProcess > _availableBytes) - { - toProcess = (int)_availableBytes; - } - - _outWindow.SetLimit(toProcess); - if (_uncompressedChunk) - { - _inputPosition += _outWindow.CopyStream(_inputStream, toProcess); - } - else if (_decoder.Code(_dictionarySize, _outWindow, _rangeDecoder) && _outputSize < 0) - { - _availableBytes = _outWindow.AvailableBytes; - } - - var read = _outWindow.Read(buffer, offset, toProcess); - total += read; - offset += read; - _position += read; - _availableBytes -= read; - - if (_availableBytes == 0 && !_uncompressedChunk) - { - // Check range corruption scenario - if ( - !_rangeDecoder.IsFinished - || (_rangeDecoderLimit >= 0 && _rangeDecoder._total != _rangeDecoderLimit) - ) - { - // Stream might have End Of Stream marker - _outWindow.SetLimit(toProcess + 1); - if (!_decoder.Code(_dictionarySize, _outWindow, _rangeDecoder)) - { - _rangeDecoder.ReleaseStream(); - throw new DataErrorException(); - } - } - - _rangeDecoder.ReleaseStream(); - - _inputPosition += _rangeDecoder._total; - if (_outWindow.HasPending) - { - throw new DataErrorException(); - } - } - } - - if (_endReached) - { - if (_inputSize >= 0 && _inputPosition != _inputSize) - { - throw new DataErrorException(); - } - if (_outputSize >= 0 && _position != _outputSize) - { - throw new DataErrorException(); - } - } - - return total; - } - - private void DecodeChunkHeader() - { - var control = _inputStream.ReadByte(); - _inputPosition++; - - if (control == 0x00) - { - _endReached = true; - return; - } - - if (control >= 0xE0 || control == 0x01) - { - _needProps = true; - _needDictReset = false; - _outWindow.Reset(); - } - else if (_needDictReset) - { - throw new DataErrorException(); - } - - if (control >= 0x80) - { - _uncompressedChunk = false; - - _availableBytes = (control & 0x1F) << 16; - _availableBytes += (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; - - _rangeDecoderLimit = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; - - if (control >= 0xC0) - { - _needProps = false; - Properties[0] = (byte)_inputStream.ReadByte(); - _inputPosition++; - - _decoder = new Decoder(); - _decoder.SetDecoderProperties(Properties); - } - else if (_needProps) - { - throw new DataErrorException(); - } - else if (control >= 0xA0) - { - _decoder = new Decoder(); - _decoder.SetDecoderProperties(Properties); - } - - _rangeDecoder.Init(_inputStream); - } - else if (control > 0x02) - { - throw new DataErrorException(); - } - else - { - _uncompressedChunk = true; - _availableBytes = (_inputStream.ReadByte() << 8) + _inputStream.ReadByte() + 1; - _inputPosition += 2; - } - } - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public byte[] Properties { get; } = new byte[5]; - } -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/README.md b/SharpHDiffPatch.Core/Binary/Compression/Lzma/README.md index 2068110..d0609ca 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/README.md +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/README.md @@ -1,199 +1,18 @@ -# SharpCompress - -SharpCompress is a compression library in pure C# for .NET Standard 2.0, 2.1, .NET Core 3.1 and .NET 5.0 that can unrar, un7zip, unzip, untar unbzip2, ungzip, unlzip with forward-only reading and file random access APIs. Write support for zip/tar/bzip2/gzip/lzip are implemented. - -The major feature is support for non-seekable streams so large files can be processed on the fly (i.e. download stream). - -GitHub Actions Build - -[![SharpCompress](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml/badge.svg)](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml) -[![Static Badge](https://img.shields.io/badge/API%20Documentation-RobiniaDocs-43bc00?logo=readme&logoColor=white)](https://www.robiniadocs.com/d/sharpcompress/api/SharpCompress.html) - -## Need Help? - -Post Issues on Github! - -Check the [Supported Formats](FORMATS.md) and [Basic Usage.](USAGE.md) - -## Recommended Formats - -In general, I recommend GZip (Deflate)/BZip2 (BZip)/LZip (LZMA) as the simplicity of the formats lend to better long term archival as well as the streamability. Tar is often used in conjunction for multiple files in a single archive (e.g. `.tar.gz`) - -Zip is okay, but it's a very hap-hazard format and the variation in headers and implementations makes it hard to get correct. Uses Deflate by default but supports a lot of compression methods. - -RAR is not recommended as it's a propriatory format and the compression is closed source. Use Tar/LZip for LZMA - -7Zip and XZ both are overly complicated. 7Zip does not support streamable formats. XZ has known holes explained here: (http://www.nongnu.org/lzip/xz_inadequate.html) Use Tar/LZip for LZMA compression instead. - -## A Simple Request - -Hi everyone. I hope you're using SharpCompress and finding it useful. Please give me feedback on what you'd like to see changed especially as far as usability goes. New feature suggestions are always welcome as well. I would also like to know what projects SharpCompress is being used in. I like seeing how it is used to give me ideas for future versions. Thanks! - -Please do not email me directly to ask for help. If you think there is a real issue, please report it here. - -## Want to contribute? - -I'm always looking for help or ideas. Please submit code or email with ideas. Unfortunately, just letting me know you'd like to help is not enough because I really have no overall plan of what needs to be done. I'll definitely accept code submissions and add you as a member of the project! - -## TODOs (always lots) - -* RAR 5 decryption support -* 7Zip writing -* Zip64 (Need writing and extend Reading) -* Multi-volume Zip support. - -## Version Log - -* [Releases](https://github.com/adamhathcock/sharpcompress/releases) - -### Version 0.18 - -* [Now on Github releases](https://github.com/adamhathcock/sharpcompress/releases/tag/0.18) - -### Version 0.17.1 - -* Fix - [Bug Fix for .NET Core on Windows](https://github.com/adamhathcock/sharpcompress/pull/257) - -### Version 0.17.0 - -* New - Full LZip support! Can read and write LZip files and Tars inside LZip files. [Make LZip a first class citizen. #241](https://github.com/adamhathcock/sharpcompress/issues/241) -* New - XZ read support! Can read XZ files and Tars inside XZ files. [XZ in SharpCompress #91](https://github.com/adamhathcock/sharpcompress/issues/94) -* Fix - [Regression - zip file writing on seekable streams always assumed stream start was 0. Introduced with Zip64 writing.](https://github.com/adamhathcock/sharpcompress/issues/244) -* Fix - [Zip files with post-data descriptors can be properly skipped via decompression](https://github.com/adamhathcock/sharpcompress/issues/162) - -### Version 0.16.2 - -* Fix [.NET 3.5 should support files and cryptography (was a regression from 0.16.0)](https://github.com/adamhathcock/sharpcompress/pull/251) -* Fix [Zip per entry compression customization wrote the wrong method into the zip archive](https://github.com/adamhathcock/sharpcompress/pull/249) - -### Version 0.16.1 - -* Fix [Preserve compression method when getting a compressed stream](https://github.com/adamhathcock/sharpcompress/pull/235) -* Fix [RAR entry key normalization fix](https://github.com/adamhathcock/sharpcompress/issues/201) - -### Version 0.16.0 - -* Breaking - [Progress Event Tracking rethink](https://github.com/adamhathcock/sharpcompress/pull/226) -* Update to VS2017 - [VS2017](https://github.com/adamhathcock/sharpcompress/pull/231) - Framework targets have been changed. -* New - [Add Zip64 writing](https://github.com/adamhathcock/sharpcompress/pull/211) -* [Fix invalid/mismatching Zip version flags.](https://github.com/adamhathcock/sharpcompress/issues/164) - This allows nuget/System.IO.Packaging to read zip files generated by SharpCompress -* [Fix 7Zip directory hiding](https://github.com/adamhathcock/sharpcompress/pull/215/files) -* [Verify RAR CRC headers](https://github.com/adamhathcock/sharpcompress/pull/220) - -### Version 0.15.2 - -* [Fix invalid headers](https://github.com/adamhathcock/sharpcompress/pull/210) - fixes an issue creating large-ish zip archives that was introduced with zip64 reading. - -### Version 0.15.1 - -* [Zip64 extending information and ZipReader](https://github.com/adamhathcock/sharpcompress/pull/206) - -### Version 0.15.0 - -* [Add zip64 support for ZipArchive extraction](https://github.com/adamhathcock/sharpcompress/pull/205) - -### Version 0.14.1 - -* [.NET Assemblies aren't strong named](https://github.com/adamhathcock/sharpcompress/issues/158) -* [Pkware encryption for Zip files didn't allow for multiple reads of an entry](https://github.com/adamhathcock/sharpcompress/issues/197) -* [GZip Entry couldn't be read multiple times](https://github.com/adamhathcock/sharpcompress/issues/198) - -### Version 0.14.0 - -* [Support for LZip reading in for Tars](https://github.com/adamhathcock/sharpcompress/pull/191) - -### Version 0.13.1 - -* [Fix null password on ReaderFactory. Fix null options on SevenZipArchive](https://github.com/adamhathcock/sharpcompress/pull/188) -* [Make PpmdProperties lazy to avoid unnecessary allocations.](https://github.com/adamhathcock/sharpcompress/pull/185) - -### Version 0.13.0 - -* Breaking change: Big refactor of Options on API. -* 7Zip supports Deflate - -### Version 0.12.4 - -* Forward only zip issue fix https://github.com/adamhathcock/sharpcompress/issues/160 -* Try to fix frameworks again by copying targets from JSON.NET - -### Version 0.12.3 - -* 7Zip fixes https://github.com/adamhathcock/sharpcompress/issues/73 -* Maybe all profiles will work with project.json now - -### Version 0.12.2 - -* Support Profile 259 again - -### Version 0.12.1 - -* Support Silverlight 5 - -### Version 0.12.0 - -* .NET Core RTM! -* Bug fix for Tar long paths - -### Version 0.11.6 - -* Bug fix for global header in Tar -* Writers now have a leaveOpen `bool` overload. They won't close streams if not-requested to. - -### Version 0.11.5 - -* Bug fix in Skip method - -### Version 0.11.4 - -* SharpCompress is now endian neutral (matters for Mono platforms) -* Fix for Inflate (need to change implementation) -* Fixes for RAR detection - -### Version 0.11.1 - -* Added Cancel on IReader -* Removed .NET 2.0 support and LinqBridge dependency - -### Version 0.11 - -* Been over a year, contains mainly fixes from contributors! -* Possible breaking change: ArchiveEncoding is UTF8 by default now. -* TAR supports writing long names using longlink -* RAR Protect Header added - -### Version 0.10.3 - -* Finally fixed Disposal issue when creating a new archive with the Archive API - -### Version 0.10.2 - -* Fixed Rar Header reading for invalid extended time headers. -* Windows Store assembly is now strong named -* Known issues with Long Tar names being worked on -* Updated to VS2013 -* Portable targets SL5 and Windows Phone 8 (up from SL4 and WP7) - -### Version 0.10.1 - -* Fixed 7Zip extraction performance problem - -### Version 0.10: - -* Added support for RAR Decryption (thanks to https://github.com/hrasyid) -* Embedded some BouncyCastle crypto classes to allow RAR Decryption and Winzip AES Decryption in Portable and Windows Store DLLs -* Built in Release (I think) - -XZ implementation based on: https://github.com/sambott/XZ.NET by @sambott - -XZ BCJ filters support contributed by Louis-Michel Bergeron, on behalf of aDolus Technology Inc. - 2022 - -7Zip implementation based on: https://code.google.com/p/managed-lzma/ - -LICENSE -Copyright (c) 2000 - 2011 The Legion Of The Bouncy Castle (http://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# Attention +This implementation of LzmaStream has been heavily modified from the original managed-lzma and sharpcompress implementation and is more optimized. +The original implementation can be found [here](https://github.com/adamhathcock/sharpcompress/tree/master/src/SharpCompress/Compressors/LZMA) + +## Benchmark result +| Method | Mean | Error | StdDev | Ratio | Gen0 | Gen1 | Allocated | Alloc Ratio | +|--------- |---------:|---------:|---------:|------:|-------:|-------:|----------:|------------:| +| Lzma2New | 23.75 us | 0.197 us | 0.165 us | 0.72 | 0.1526 | - | 8.23 KB | 0.17 | +| Lzma2Old | 33.17 us | 0.061 us | 0.054 us | 1.00 | 0.9766 | 0.1221 | 48.52 KB | 1.00 | + +Hardware Bench: +- CPU: AMD Ryzen 7 9800X3D +- Memory: G.Skill Ripjaws M5 64GB (32x2) DDR5-6000 CL36-36-36-96 + +Big credits to the original authors of LZMA compression, managed-lzma and sharpcompress for their work. +- [Adam Hathcock (sharpcompress)](https://github.com/adamhathcock) +- [Igor Pavlov (LZMA compression)](https://sourceforge.net/u/ipavlov/profile/) +- [Tobias Käs (managed-lzma)](https://github.com/weltkante/managed-lzma) \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoder.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoder.cs index e2e06a3..784fc12 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoder.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoder.cs @@ -1,120 +1,136 @@ +using System; +using System.Buffers; using System.IO; +using System.Runtime.CompilerServices; -namespace SharpCompress.Compressors.LZMA.RangeCoder +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma.RangeCoder; + +internal class RangeDecoder : IDisposable { - internal class Decoder + public const uint KTopValue = 1 << 24; + private const int InputBufferSize = 32 << 10; + public uint Range; + public uint Code; + + public Stream Stream; + public long Total; + + private byte[] _inputBuffer = []; + private int _inputOffset; + private int _inputCount; + private long _inputLimit; + private bool _useInputBuffer; + + public void Init(Stream stream, long inputLimit = -1) { - public const uint K_TOP_VALUE = (1 << 24); - public uint _range; - public uint _code; + Stream = stream; + _inputOffset = 0; + _inputCount = 0; + _inputLimit = inputLimit; + _useInputBuffer = inputLimit >= 0; + + Code = 0; + Range = 0xFFFFFFFF; + Total = 0; + for (int i = 0; i < 5; i++) + { + Code = (Code << 8) | ReadByte(); + } + } - // public Buffer.InBuffer Stream = new Buffer.InBuffer(1 << 16); - public Stream _stream; - public long _total; + public void ReleaseStream() => Stream = null; - public void Init(Stream stream) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Normalize() + { + while (Range < KTopValue) { - // Stream.Init(stream); - _stream = stream; - - _code = 0; - _range = 0xFFFFFFFF; - for (var i = 0; i < 5; i++) - { - _code = (_code << 8) | (byte)_stream.ReadByte(); - } - _total = 5; + Code = (Code << 8) | ReadByte(); + Range <<= 8; } + } - public void ReleaseStream() => - // Stream.ReleaseStream(); - _stream = null; + public void Decode(uint start, uint size) + { + Code -= start * Range; + Range *= size; + Normalize(); + } - public void CloseStream() => _stream.Dispose(); + public uint DecodeDirectBits(int numTotalBits) + { + uint range = Range; + uint code = Code; + uint result = 0; + for (int i = numTotalBits; i > 0; i--) + { + range >>= 1; + uint t = (code - range) >> 31; + code -= range & (t - 1); + result = (result << 1) | (1 - t); + + if (range >= KTopValue) continue; + code = (code << 8) | ReadByte(); + range <<= 8; + } + + Range = range; + Code = code; + return result; + } - public void Normalize() + public bool IsFinished => Code == 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal uint ReadByte() + { + if (!_useInputBuffer) { - while (_range < K_TOP_VALUE) - { - _code = (_code << 8) | (byte)_stream.ReadByte(); - _range <<= 8; - _total++; - } + Total++; + return (byte)Stream.ReadByte(); } - public void Normalize2() + if (_inputOffset >= _inputCount) { - if (_range < K_TOP_VALUE) - { - _code = (_code << 8) | (byte)_stream.ReadByte(); - _range <<= 8; - _total++; - } + FillInputBuffer(); } - public uint GetThreshold(uint total) => _code / (_range /= total); + Total++; + return _inputBuffer[_inputOffset++]; + } - public void Decode(uint start, uint size) + [MethodImpl(MethodImplOptions.NoInlining)] + private void FillInputBuffer() + { + long remaining = _inputLimit - Total; + if (remaining <= 0) { - _code -= start * _range; - _range *= size; - Normalize(); + throw new LzmaDataErrorException(); } - public uint DecodeDirectBits(int numTotalBits) + if (_inputBuffer.Length == 0) { - var range = _range; - var code = _code; - uint result = 0; - for (var i = numTotalBits; i > 0; i--) - { - range >>= 1; - /* - result <<= 1; - if (code >= range) - { - code -= range; - result |= 1; - } - */ - var t = (code - range) >> 31; - code -= range & (t - 1); - result = (result << 1) | (1 - t); - - if (range < K_TOP_VALUE) - { - code = (code << 8) | (byte)_stream.ReadByte(); - range <<= 8; - _total++; - } - } - _range = range; - _code = code; - return result; + _inputBuffer = ArrayPool.Shared.Rent(InputBufferSize); } - public uint DecodeBit(uint size0, int numTotalBits) + int requested = (int)Math.Min(_inputBuffer.Length, remaining); + _inputCount = Stream.Read(_inputBuffer, 0, requested); + _inputOffset = 0; + if (_inputCount <= 0) { - var newBound = (_range >> numTotalBits) * size0; - uint symbol; - if (_code < newBound) - { - symbol = 0; - _range = newBound; - } - else - { - symbol = 1; - _code -= newBound; - _range -= newBound; - } - Normalize(); - return symbol; + throw new LzmaDataErrorException(); } + } - public bool IsFinished => _code == 0; + public void Dispose() + { + ReleaseStream(); + byte[] inputBuffer = _inputBuffer; + _inputBuffer = []; - // ulong GetProcessedSize() {return Stream.GetProcessedSize(); } + if (inputBuffer.Length != 0) + { + ArrayPool.Shared.Return(inputBuffer); + } } - } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBit.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBit.cs index 6bf8c14..d643876 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBit.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBit.cs @@ -1,42 +1,55 @@ -namespace SharpCompress.Compressors.LZMA.RangeCoder +using System; +using System.Runtime.CompilerServices; + +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma.RangeCoder; + +internal struct BitDecoder { - internal struct BitDecoder - { - public const int K_NUM_BIT_MODEL_TOTAL_BITS = 11; - public const uint K_BIT_MODEL_TOTAL = (1 << K_NUM_BIT_MODEL_TOTAL_BITS); - private const int K_NUM_MOVE_BITS = 5; + public const int KNumBitModelTotalBits = 11; + public const uint KBitModelTotal = 1 << KNumBitModelTotalBits; + private const int KNumMoveBits = 5; + + private uint _prob; - private uint _prob; + public void Init() => _prob = KBitModelTotal >> 1; - public void Init() => _prob = K_BIT_MODEL_TOTAL >> 1; + public static void Init(BitDecoder[] decoders) => Init(decoders, 0, decoders.Length); + + public static void Init(BitDecoder[] decoders, int start, int length) => decoders.AsSpan(start, length).Fill(new BitDecoder { _prob = KBitModelTotal >> 1 }); + + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint Decode(RangeDecoder rangeDecoder) + { + uint range = rangeDecoder.Range; + uint code = rangeDecoder.Code; + uint probability = _prob; + uint newBound = (range >> KNumBitModelTotalBits) * probability; + uint symbol; - public uint Decode(Decoder rangeDecoder) + if (code < newBound) { - var newBound = (rangeDecoder._range >> K_NUM_BIT_MODEL_TOTAL_BITS) * _prob; - if (rangeDecoder._code < newBound) - { - rangeDecoder._range = newBound; - _prob += (K_BIT_MODEL_TOTAL - _prob) >> K_NUM_MOVE_BITS; - if (rangeDecoder._range < Decoder.K_TOP_VALUE) - { - rangeDecoder._code = - (rangeDecoder._code << 8) | (byte)rangeDecoder._stream.ReadByte(); - rangeDecoder._range <<= 8; - rangeDecoder._total++; - } - return 0; - } - rangeDecoder._range -= newBound; - rangeDecoder._code -= newBound; - _prob -= (_prob) >> K_NUM_MOVE_BITS; - if (rangeDecoder._range < Decoder.K_TOP_VALUE) - { - rangeDecoder._code = (rangeDecoder._code << 8) | (byte)rangeDecoder._stream.ReadByte(); - rangeDecoder._range <<= 8; - rangeDecoder._total++; - } - return 1; + range = newBound; + probability += (KBitModelTotal - probability) >> KNumMoveBits; + symbol = 0; + } + else + { + range -= newBound; + code -= newBound; + probability -= probability >> KNumMoveBits; + symbol = 1; } - } + if (range < RangeDecoder.KTopValue) + { + code = (code << 8) | rangeDecoder.ReadByte(); + range <<= 8; + } + + rangeDecoder.Range = range; + rangeDecoder.Code = code; + _prob = probability; + return symbol; + } } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBitTree.cs b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBitTree.cs index 23eb5c2..e383c4f 100644 --- a/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBitTree.cs +++ b/SharpHDiffPatch.Core/Binary/Compression/Lzma/RangeCoder/RangeCoderBitTree.cs @@ -1,65 +1,76 @@ -namespace SharpCompress.Compressors.LZMA.RangeCoder +using System.Runtime.CompilerServices; + +namespace SharpHDiffPatch.Core.Binary.Compression.Lzma.RangeCoder; + +internal readonly struct BitTreeDecoder(int numBitLevels) { - internal readonly struct BitTreeDecoder + private readonly BitDecoder[] _models = new BitDecoder[1 << numBitLevels]; + + public void Init() => BitDecoder.Init(_models, 1, _models.Length - 1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint Decode(RangeDecoder rangeDecoder) { - private readonly BitDecoder[] _models; - private readonly int _numBitLevels; + ref BitDecoder models = ref _models[0]; + uint m = 1; - public BitTreeDecoder(int numBitLevels) + for (int bitIndex = numBitLevels; bitIndex > 0; bitIndex--) { - _numBitLevels = numBitLevels; - _models = new BitDecoder[1 << numBitLevels]; + m = (m << 1) + Unsafe.Add(ref models, (int)m).Decode(rangeDecoder); } + return m - ((uint)1 << numBitLevels); + } - public void Init() - { - for (uint i = 1; i < (1 << _numBitLevels); i++) - { - _models[i].Init(); - } - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint Decode( + BitDecoder[] models, + int startIndex, + RangeDecoder rangeDecoder, + int numBitLevels) + { + ref BitDecoder modelBase = ref Unsafe.Add(ref models[0], startIndex); + uint m = 1; - public uint Decode(Decoder rangeDecoder) + for (int bitIndex = numBitLevels; bitIndex > 0; bitIndex--) { - uint m = 1; - for (var bitIndex = _numBitLevels; bitIndex > 0; bitIndex--) - { - m = (m << 1) + _models[m].Decode(rangeDecoder); - } - return m - ((uint)1 << _numBitLevels); + m = (m << 1) + Unsafe.Add(ref modelBase, (int)m).Decode(rangeDecoder); } + return m - ((uint)1 << numBitLevels); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint ReverseDecode(RangeDecoder rangeDecoder) + { + ref BitDecoder models = ref _models[0]; + uint m = 1; + uint symbol = 0; - public uint ReverseDecode(Decoder rangeDecoder) + for (int bitIndex = 0; bitIndex < numBitLevels; bitIndex++) { - uint m = 1; - uint symbol = 0; - for (var bitIndex = 0; bitIndex < _numBitLevels; bitIndex++) - { - var bit = _models[m].Decode(rangeDecoder); - m <<= 1; - m += bit; - symbol |= (bit << bitIndex); - } - return symbol; + uint bit = Unsafe.Add(ref models, (int)m).Decode(rangeDecoder); + m = (m << 1) + bit; + symbol |= bit << bitIndex; } + return symbol; + } + + public static uint ReverseDecode( + BitDecoder[] models, + uint startIndex, + RangeDecoder rangeDecoder, + int numBitLevels) + { + ref BitDecoder modelBase = ref models[0]; + uint m = 1; + uint symbol = 0; - public static uint ReverseDecode( - BitDecoder[] models, - uint startIndex, - Decoder rangeDecoder, - int numBitLevels - ) + for (int bitIndex = 0; bitIndex < numBitLevels; bitIndex++) { - uint m = 1; - uint symbol = 0; - for (var bitIndex = 0; bitIndex < numBitLevels; bitIndex++) - { - var bit = models[startIndex + m].Decode(rangeDecoder); - m <<= 1; - m += bit; - symbol |= (bit << bitIndex); - } - return symbol; + int modelIndex = unchecked((int)(startIndex + m)); + uint bit = Unsafe.Add(ref modelBase, modelIndex).Decode(rangeDecoder); + m = (m << 1) + bit; + symbol |= bit << bitIndex; } + return symbol; } } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Binary/Streams/ChunkStream.cs b/SharpHDiffPatch.Core/Binary/Streams/ChunkStream.cs index 1eb6c12..45f8549 100644 --- a/SharpHDiffPatch.Core/Binary/Streams/ChunkStream.cs +++ b/SharpHDiffPatch.Core/Binary/Streams/ChunkStream.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.IO; namespace SharpHDiffPatch.Core.Binary.Streams @@ -14,7 +15,6 @@ public sealed class ChunkStream : Stream private bool IsDisposing { get; } public ChunkStream(Stream stream, long start, long end, bool isDisposing = false) - : base() { _stream = stream; @@ -85,7 +85,21 @@ public override void Write(byte[] buffer, int offset, int count) #if !(NETSTANDARD2_0 || NET461_OR_GREATER) public override void CopyTo(Stream destination, int bufferSize) { - throw new NotSupportedException(); + if (bufferSize <= 0) bufferSize = 4 << 10; + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + int read; + while ((read = Read(buffer.AsSpan(0, bufferSize))) > 0) + { + destination.Write(buffer.AsSpan(0, read)); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } } #endif diff --git a/SharpHDiffPatch.Core/Binary/Streams/CombinedStream.cs b/SharpHDiffPatch.Core/Binary/Streams/CombinedStream.cs index a6d1619..7fc1e95 100644 --- a/SharpHDiffPatch.Core/Binary/Streams/CombinedStream.cs +++ b/SharpHDiffPatch.Core/Binary/Streams/CombinedStream.cs @@ -1,6 +1,4 @@ // ReSharper disable CommentTypo -// ReSharper disable SwitchStatementHandlesSomeKnownEnumValuesWithDefault -// ReSharper disable UseIndexFromEndExpression /* * Original Code by lassevk @@ -9,406 +7,666 @@ using System; using System.IO; -using System.Linq; +using System.Threading; +using System.Threading.Tasks; -#if !NETCOREAPP -#pragma warning disable IDE0056 -#endif +namespace SharpHDiffPatch.Core.Binary.Streams; -namespace SharpHDiffPatch.Core.Binary.Streams +public class CombinedStreamSegment + where T : Stream { - public class NewFileCombinedStream - { - public FileStream Stream { get; set; } - public long Size { get; set; } - } + public T Stream { get; set; } + public long Length { get; set; } +} + +/// +/// This class is a descendant that manages multiple underlying +/// streams which are considered to be chained together to one large stream. Only reading +/// and seeking is allowed, writing will throw exceptions. +/// +public sealed class CombinedStream : Stream + where T : Stream +{ + private readonly T[] _underlyingStreams; + private readonly long[] _streamEnds; + + private readonly bool _leaveOpen; + private readonly bool _canWrite; + + private long _position; + private int _index; + private bool _disposed; /// - /// This class is a descendant that manages multiple underlying - /// streams which are considered to be chained together to one large stream. Only reading - /// and seeking is allowed, writing will throw exceptions. + /// Constructs a new on top of the specified array + /// of streams. /// - public sealed class CombinedStream : Stream + /// + /// An array of objects that will be chained together and + /// considered to be one big stream. + /// + /// Keep all underlying streams opened while disposing this current stream. + public CombinedStream(T[] underlyingStreams, + bool leaveOpen = false) { - private readonly FileStream[] _underlyingStreams; - private readonly long[] _underlyingStartingPositions; - - private long _position; - private int _index; - - /// - /// Constructs a new on top of the specified array - /// of streams. - /// - /// - /// An array of objects that will be chained together and - /// considered to be one big stream. - /// - public CombinedStream(params FileStream[] underlyingStreams) + if (underlyingStreams == null) + throw new ArgumentNullException(nameof(underlyingStreams), $"[{nameof(CombinedStream)}()] underlyingStreams"); + + if (underlyingStreams.Length == 0) { - if (underlyingStreams == null) - throw new ArgumentNullException(nameof(underlyingStreams), "[CombinedStream::ctor()] underlyingStreams"); + throw new ArgumentException($"[{nameof(CombinedStream)}()] At least one stream is required.", + nameof(underlyingStreams)); + } + + _underlyingStreams = underlyingStreams; + _streamEnds = new long[underlyingStreams.Length]; + _leaveOpen = leaveOpen; + + bool canWrite = true; + long totalLength = 0; + + for (int i = 0; i < _underlyingStreams.Length; i++) + { + Stream stream = _underlyingStreams[i]; + + if (stream == null) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] The array contains a null stream.", + nameof(underlyingStreams)); + } - foreach (FileStream stream in underlyingStreams) + if (!stream.CanRead) { - if (stream == null) - throw new ArgumentException("[CombinedStream::ctor()] underlyingStreams contains a null stream reference", nameof(underlyingStreams)); - if (!stream.CanRead) - throw new InvalidOperationException("[CombinedStream::ctor()] CanRead not true for all streams"); - if (!stream.CanSeek) - throw new InvalidOperationException("[CombinedStream::ctor()] CanSeek not true for all streams"); + throw new ArgumentException($"[{nameof(CombinedStream)}()] Every underlying stream must be readable.", + nameof(underlyingStreams)); } - _underlyingStreams = underlyingStreams; - _underlyingStartingPositions = new long[underlyingStreams.Length]; + if (!stream.CanSeek) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] Every underlying stream must be seekable.", + nameof(underlyingStreams)); + } - _position = 0; - _index = 0; + canWrite &= stream.CanWrite; - _underlyingStartingPositions[0] = 0; - for (int index = 1; index < _underlyingStartingPositions.Length; index++) - _underlyingStartingPositions[index] = _underlyingStartingPositions[index - 1] + _underlyingStreams[index - 1].Length; + totalLength = checked(totalLength + stream.Length); + _streamEnds[i] = totalLength; + } - Length = _underlyingStartingPositions[_underlyingStartingPositions.Length - 1] + _underlyingStreams[_underlyingStreams.Length - 1].Length; + Length = totalLength; + _canWrite = canWrite; #if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::ctor()] Total length of the CombinedStream: {_TotalLength} bytes with total of {underlyingStreams.Length} streams", Verbosity.Debug); + HDiffPatch.Event.PushLog($"[{nameof(CombinedStream)}()] Total length of the CombinedStream: {totalLength} bytes with total of {underlyingStreams.Length} streams", Verbosity.Debug); #endif + } + + /// + /// Constructs a new on top of the specified array + /// of streams. + /// + /// + /// An array of objects that will be chained together and + /// considered to be one big stream. + /// + /// Keep all underlying streams opened while disposing this current stream. + public CombinedStream(CombinedStreamSegment[] underlyingStreams, + bool leaveOpen = false) + { + if (underlyingStreams == null) + throw new ArgumentNullException(nameof(underlyingStreams), $"[{nameof(CombinedStream)}()] underlyingStreams"); + + if (underlyingStreams.Length == 0) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] At least one stream is required.", + nameof(underlyingStreams)); } - /// - /// Constructs a new on top of the specified array - /// of streams. - /// - /// - /// An array of objects that will be chained together and - /// considered to be one big stream. - /// - public CombinedStream(params NewFileCombinedStream[] underlyingStreams) + _underlyingStreams = new T[underlyingStreams.Length]; + _streamEnds = new long[underlyingStreams.Length]; + _leaveOpen = leaveOpen; + + bool canWrite = true; + long totalLength = 0; + + for (int i = 0; i < underlyingStreams.Length; i++) { - if (underlyingStreams == null) - throw new ArgumentNullException(nameof(underlyingStreams), "[CombinedStream::ctor()] underlyingStreams"); + CombinedStreamSegment segment = underlyingStreams[i]; + + if (segment == null) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] The array contains a null segment.", nameof(underlyingStreams)); + } - _underlyingStreams = new FileStream[underlyingStreams.Length]; - _underlyingStartingPositions = new long[underlyingStreams.Length]; + T stream = segment.Stream; - foreach (NewFileCombinedStream stream in underlyingStreams) + if (stream == null) { - if (stream.Stream == null) - throw new ArgumentException("[CombinedStream::ctor()] underlyingStreams contains a null stream reference", nameof(underlyingStreams)); - if (!stream.Stream.CanRead) - throw new InvalidOperationException("[CombinedStream::ctor()] CanRead not true for all streams"); - if (!stream.Stream.CanSeek) - throw new InvalidOperationException("[CombinedStream::ctor()] CanSeek not true for all streams"); - - stream.Stream.SetLength(stream.Size); -#if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::ctor()] Initializing file with length {stream.size} bytes: {stream.stream.Name}", Verbosity.Debug); -#endif + throw new ArgumentException($"[{nameof(CombinedStream)}()] A segment contains a null stream.", nameof(underlyingStreams)); + } + + if (segment.Length < 0) + { + throw new ArgumentOutOfRangeException(nameof(underlyingStreams), $"[{nameof(CombinedStream)}()] Segment length cannot be negative."); } - Array.Copy(underlyingStreams.Select(x => x.Stream).ToArray(), _underlyingStreams, underlyingStreams.Length); + if (!stream.CanRead) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] Every underlying stream must be readable.", nameof(underlyingStreams)); + } - _position = 0; - _index = 0; + if (!stream.CanSeek) + { + throw new ArgumentException($"[{nameof(CombinedStream)}()] Every underlying stream must be seekable.", nameof(underlyingStreams)); + } - _underlyingStartingPositions[0] = 0; - for (int index = 1; index < _underlyingStartingPositions.Length; index++) - _underlyingStartingPositions[index] = _underlyingStartingPositions[index - 1] + underlyingStreams[index - 1].Size; + canWrite &= stream.CanWrite; + + _underlyingStreams[i] = stream; + + totalLength = checked(totalLength + segment.Length); + _streamEnds[i] = totalLength; + } - Length = _underlyingStartingPositions[_underlyingStartingPositions.Length - 1] + underlyingStreams[_underlyingStreams.Length - 1].Size; + Length = totalLength; + _canWrite = canWrite; #if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::ctor()] Total length of the CombinedStream: {_TotalLength} bytes with total of {underlyingStreams.Length} streams", Verbosity.Debug); + HDiffPatch.Event.PushLog($"[{nameof(CombinedStream)}()] Total length of the CombinedStream: {totalLength} bytes with total of {underlyingStreams.Length} streams", Verbosity.Debug); #endif - } + } + + /// + public override bool CanRead => !_disposed; + + /// + public override bool CanSeek => !_disposed; + + /// + public override bool CanWrite => !_disposed && _canWrite; + + /// + public override void Flush() + { + foreach (T stream in _underlyingStreams) + stream.Flush(); + } + + /// + public override async Task FlushAsync(CancellationToken cancellationToken) + { + ThrowIfDisposed(); + + if (!_canWrite) + return; - /// - /// Gets a value indicating whether the current stream supports reading. - /// - /// - /// true. - /// - /// - /// Always true for . - /// - public override bool CanRead => true; - - /// - /// Gets a value indicating whether the current stream supports seeking. - /// - /// - /// true. - /// - /// - /// Always true for . - /// - public override bool CanSeek => true; - - /// - /// Gets a value indicating whether the current stream supports writing. - /// - /// - /// false. - /// - /// - /// Always false for . - /// - public override bool CanWrite => true; - - /// - /// When overridden in a derived class, clears all buffers for this stream and causes any buffered data to be written to the underlying device. - /// - /// An I/O error occurs. - public override void Flush() + foreach (T stream in _underlyingStreams) + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + /// + protected override void Dispose(bool disposing) + { + if (!_disposed) { - foreach (FileStream stream in _underlyingStreams) - stream.Flush(); + _disposed = true; + + if (disposing && !_leaveOpen) + { + foreach (T stream in _underlyingStreams) + stream.Dispose(); + } } - protected override void Dispose(bool disposing) + base.Dispose(disposing); + } + + /// + public override long Length { get; } + + /// + public override long Position + { + get { - base.Dispose(disposing); - if (_underlyingStreams == null) return; - foreach (FileStream stream in _underlyingStreams) - stream.Dispose(); + ThrowIfDisposed(); + return _position; } + set + { + ThrowIfDisposed(); + + if ((ulong)value > (ulong)Length) + throw new ArgumentOutOfRangeException(nameof(value)); + + _position = value; + _index = FindStreamIndex(_streamEnds, value, Length, _index); + } + } + + /// + public override void SetLength(long value) => throw new NotSupportedException($"[{nameof(CombinedStream)}::SetLength] The method or operation is not supported by CombinedStream."); + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP2_1_OR_GREATER + /// + public override int Read(Span buffer) + { + ThrowIfDisposed(); + + if (buffer.IsEmpty || _position == Length) + return 0; - /// - /// Gets the total length in bytes of the underlying streams. - /// - /// - /// The total length of the underlying streams. - /// - /// - /// A long value representing the total length of the underlying streams in bytes. - /// - /// A class derived from Stream does not support seeking. - /// Methods were called after the stream was closed. - public override long Length { get; } - - /// - /// Gets or sets the position within the current stream. - /// - /// - /// The current position within the stream. - /// An I/O error occurs. - /// The stream does not support seeking. - /// Methods were called after the stream was closed. - public override long Position + int totalRead = 0; + + while (!buffer.IsEmpty && _position < Length) { - get => _position; + SkipExhaustedStreams(); + + Stream stream = _underlyingStreams[_index]; + long streamStart = GetStreamStart(_streamEnds, _index); + long localPosition = _position - streamStart; + long available = _streamEnds[_index] - _position; - set + if (available <= 0) { - if (value < 0 || value > Length) - throw new ArgumentOutOfRangeException(nameof(value)); + if (!MoveNextStream()) + break; - _position = value; - if (value == Length) - _index = _underlyingStreams.Length - 1; - else - { - while (_index > 0 && _position < _underlyingStartingPositions[_index]) - _index--; + continue; + } - while (_index < _underlyingStreams.Length - 1 && _position >= _underlyingStartingPositions[_index] + _underlyingStreams[_index].Length) - _index++; - } + int requested = (int)Math.Min(buffer.Length, available); + + int read; +#if NET6_0_OR_GREATER + if (stream is FileStream { SafeFileHandle: not null } fileStream) + { + read = RandomAccess.Read(fileStream.SafeFileHandle, + buffer[..requested], + localPosition); + goto Advance; } - } +#endif -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - public override int Read(Span buffer) - { - int result = 0; - int count = buffer.Length; - int offset = 0; + if (stream.Position != localPosition) + stream.Position = localPosition; + + read = stream.Read(buffer[..requested]); - while (count > 0) + if (read == 0) { - _underlyingStreams[_index].Position = _position - _underlyingStartingPositions[_index]; - int bytesRead = _underlyingStreams[_index].Read(buffer[offset..]); - result += bytesRead; - offset += bytesRead; - count -= bytesRead; - _position += bytesRead; - - if (count <= 0) continue; - if (_index < _underlyingStreams.Length - 1) + if (_position < _streamEnds[_index]) { - _index++; -#if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::Read] Moving the stream to Index: {_Index}", Verbosity.Debug); -#endif + throw new EndOfStreamException($"[{nameof(CombinedStream)}::ReadCore] An underlying stream ended before its expected length."); } - else + + if (!MoveNextStream()) break; + + continue; } - return result; + Advance: + totalRead += read; + _position += read; + buffer = buffer[read..]; } + + return totalRead; + } #endif - /// - /// Reads a sequence of bytes from the current stream and advances the position within the stream by the number of bytes read. - /// - /// An array of bytes. When this method returns, the buffer contains the specified byte array with the values between offset and (offset + count - 1) replaced by the bytes read from the current source. - /// The zero-based byte offset in buffer at which to begin storing the data read from the current stream. - /// The maximum number of bytes to be read from the current stream. - /// - /// The total number of bytes read into the buffer. This can be less than the number of bytes requested if that many bytes are not currently available, or zero (0) if the end of the stream has been reached. - /// - /// The sum of offset and count is larger than the buffer length. - /// Methods were called after the stream was closed. - /// The stream does not support reading. - /// buffer is null. - /// An I/O error occurs. - /// offset or count is negative. - public override int Read(byte[] buffer, int offset, int count) + /// + public override int Read(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + ThrowIfDisposed(); + + return ReadCore(buffer, offset, count); + } + + private int ReadCore(byte[] buffer, int offset, int count) + { + if (count == 0 || _position == Length) + return 0; + + int totalRead = 0; + + while (count != 0 && _position < Length) { - int result = 0; - while (count > 0) + SkipExhaustedStreams(); + + Stream stream = _underlyingStreams[_index]; + long streamStart = GetStreamStart(_streamEnds, _index); + long localPosition = _position - streamStart; + long available = _streamEnds[_index] - _position; + + if (available <= 0) { - _underlyingStreams[_index].Position = _position - _underlyingStartingPositions[_index]; - int bytesRead = _underlyingStreams[_index].Read(buffer, offset, count); - result += bytesRead; - offset += bytesRead; - count -= bytesRead; - _position += bytesRead; - - if (count <= 0) continue; - if (_index < _underlyingStreams.Length - 1) - { - _index++; -#if SHOWMOREDEBUGING - HDiffPatch.Event.PushLog($"[CombinedStream::Read] Moving the stream to Index: {_Index}", Verbosity.Debug); -#endif - } - else + if (!MoveNextStream()) break; + + continue; } - return result; - } + int requested = (int)Math.Min(count, available); - /// - /// Sets the position within the current stream. - /// - /// A byte offset relative to the origin parameter. - /// A value of type indicating the reference point used to obtain the new position. - /// - /// The new position within the current stream. - /// - /// An I/O error occurs. - /// The stream does not support seeking, such as if the stream is constructed from a pipe or console output. - /// Methods were called after the stream was closed. - public override long Seek(long offset, SeekOrigin origin) - { - switch (origin) + int read; +#if NET6_0_OR_GREATER + if (stream is FileStream { SafeFileHandle: not null } fileStream) { - case SeekOrigin.Begin: - Position = offset; - break; + read = RandomAccess.Read(fileStream.SafeFileHandle, + buffer.AsSpan(offset, requested), + localPosition); + goto Advance; + } +#endif + if (stream.Position != localPosition) + stream.Position = localPosition; - case SeekOrigin.Current: - Position += offset; - break; + read = stream.Read(buffer, offset, requested); + + if (read == 0) + { + // The FileStream became shorter than the length captured + // by this CombinedStream, or another owner changed it. + if (_position < _streamEnds[_index]) + { + throw new EndOfStreamException($"[{nameof(CombinedStream)}::ReadCore] An underlying stream ended before its expected length."); + } - case SeekOrigin.End: - Position = Length + offset; + if (!MoveNextStream()) break; + + continue; } - return Position; + Advance: + totalRead += read; + offset += read; + count -= read; + _position += read; } - /// - /// Throws since the - /// class does not supports changing the length. - /// - /// The desired length of the current stream in bytes. - /// - /// does not support this operation. - /// - public override void SetLength(long value) + return totalRead; + } + + /// + public override long Seek(long offset, SeekOrigin origin) + { + ThrowIfDisposed(); + long position = origin switch { - throw new NotSupportedException("[CombinedStream::SetLength] The method or operation is not supported by CombinedStream."); - } + SeekOrigin.Begin => offset, + SeekOrigin.Current => checked(_position + offset), + SeekOrigin.End => checked(Length + offset), + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + + Position = position; + return position; + } -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - public override void Write(ReadOnlySpan buffer) +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP2_1_OR_GREATER + /// + public override void Write(ReadOnlySpan buffer) + { + ThrowIfDisposed(); + EnsureWritable(buffer.Length); + + while (!buffer.IsEmpty) { - long count = buffer.Length; - long offset = 0; - while (count > 0) + SkipExhaustedStreams(); + + Stream stream = _underlyingStreams[_index]; + long streamStart = GetStreamStart(_streamEnds, _index); + long localPosition = _position - streamStart; + long available = _streamEnds[_index] - _position; + + if (available <= 0) { - _underlyingStreams[_index].Position = _position - _underlyingStartingPositions[_index]; - long bytesWrite = count; - long remainedMaxLength = Math.Min(_underlyingStreams[_index].Length - _underlyingStreams[_index].Position, - long.MaxValue); - if (remainedMaxLength < count) + if (!MoveNextStream()) { - bytesWrite = remainedMaxLength; + throw new EndOfStreamException($"[{nameof(CombinedStream)}::Write] The write exceeds the combined stream length."); } - _underlyingStreams[_index].Write(buffer.Slice((int)offset, (int)bytesWrite)); - offset += bytesWrite; - count -= bytesWrite; - _position += bytesWrite; - if (count <= 0) continue; - if (_index < _underlyingStreams.Length - 1) - { - _index++; -#if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::Write] Moving the stream to Index: {_Index}", Verbosity.Debug); -#endif - } - else - break; + continue; } + + int writable = (int)Math.Min(buffer.Length, available); + +#if NET6_0_OR_GREATER + if (stream is FileStream { SafeFileHandle: not null } fileStream) + { + RandomAccess.Write(fileStream.SafeFileHandle, + buffer[..writable], + localPosition); + goto Advance; + } +#endif + + if (stream.Position != localPosition) + stream.Position = localPosition; + + stream.Write(buffer[..writable]); + + Advance: + _position += writable; + buffer = buffer[writable..]; } + } #endif - /// - /// Throws since the - /// class does not supports writing to the underlying streams. - /// - /// An array of bytes. This method copies count bytes from buffer to the current stream. - /// The zero-based byte offset in buffer at which to begin copying bytes to the current stream. - /// The number of bytes to be written to the current stream. - /// - /// does not support this operation. - /// - public override void Write(byte[] buffer, int offset, int count) + /// + public override void Write(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + ThrowIfDisposed(); + EnsureWritable(count); + + WriteCore(buffer, offset, count); + } + + private void WriteCore(byte[] buffer, int offset, int count) + { + while (count != 0) { - while (count > 0) + SkipExhaustedStreams(); + + Stream stream = _underlyingStreams[_index]; + long streamStart = GetStreamStart(_streamEnds, _index); + long localPosition = _position - streamStart; + long available = _streamEnds[_index] - _position; + + if (available <= 0) { - _underlyingStreams[_index].Position = _position - _underlyingStartingPositions[_index]; - int bytesWrite = count; - int remainedMaxLength = (int)Math.Min( - _underlyingStreams[_index].Length - _underlyingStreams[_index].Position, - int.MaxValue); - if (remainedMaxLength < count) + if (!MoveNextStream()) { - bytesWrite = remainedMaxLength; + throw new EndOfStreamException($"[{nameof(CombinedStream)}::WriteCore] The write exceeds the combined stream length."); } - _underlyingStreams[_index].Write(buffer, offset, bytesWrite); - offset += bytesWrite; - count -= bytesWrite; - _position += bytesWrite; - if (count <= 0) continue; - if (_index < _underlyingStreams.Length - 1) - { - _index++; -#if SHOWMOREDEBUGINFO - HDiffPatch.Event.PushLog($"[CombinedStream::Write] Moving the stream to Index: {_Index}", Verbosity.Debug); -#endif - } - else - break; + continue; } + + int writable = (int)Math.Min(count, available); + +#if NET6_0_OR_GREATER + if (stream is FileStream { SafeFileHandle: not null } fileStream) + { + RandomAccess.Write(fileStream.SafeFileHandle, + buffer.AsSpan(offset, writable), + localPosition); + goto Advance; + } +#endif + + if (stream.Position != localPosition) + stream.Position = localPosition; + + stream.Write(buffer, offset, writable); + + Advance: + offset += writable; + count -= writable; + _position += writable; + } + } + + private void ThrowIfDisposed() + { + if (_disposed) + throw new ObjectDisposedException(nameof(CombinedStream)); + } + +#if !NETSTANDARD2_1_OR_GREATER && !NETCOREAPP2_1_OR_GREATER + private static void ValidateBufferArguments( + byte[] buffer, + int offset, + int count) + { + if (buffer == null) + throw new ArgumentNullException(nameof(buffer)); + + if (offset < 0) + throw new ArgumentOutOfRangeException(nameof(offset)); + + if (count < 0) + throw new ArgumentOutOfRangeException(nameof(count)); + + if (buffer.Length - offset < count) + throw new ArgumentException($"[{nameof(CombinedStream)}::ValidateBufferArguments] Offset and count exceed the buffer length."); + } +#endif + + private void SkipExhaustedStreams() + { + while (_index < _underlyingStreams.Length - 1 && + _position >= _streamEnds[_index]) + { + _index++; + } + } + + private bool MoveNextStream() + { + if (_index >= _underlyingStreams.Length - 1) + return false; + + _index++; + return true; + } + + private void EnsureWritable(int count) + { + if (!_canWrite) + throw new NotSupportedException($"[{nameof(CombinedStream)}::EnsureWritable] The stream is not writable."); + + if (count > Length - _position) + { + throw new EndOfStreamException($"[{nameof(CombinedStream)}::EnsureWritable] The write exceeds the combined stream's fixed length."); + } + } + + private static long GetStreamStart(long[] streamEnds, int index) => index == 0 ? 0 : streamEnds[index - 1]; + + private static int FindStreamIndex(long[] streamEnds, long position, long length, int index) + { + int lastIndex = streamEnds.Length - 1; + + // Position == Length represents EOF. Keep the final stream selected, + // including when it is a zero-length trailing stream. + if (position == length) + return lastIndex; + + long currentStart = index == 0 ? 0 : streamEnds[index - 1]; + long currentEnd = streamEnds[index]; + + // Most common case: the new position remains in the current segment. + if (position >= currentStart && position < currentEnd) + return index; + + return position >= currentEnd + ? FindForward(streamEnds, position, index, lastIndex) + : FindBackward(streamEnds, position, index); + } + + private static int FindForward(long[] streamEnds, long position, int currentIndex, int lastIndex) + { + int nextIndex = currentIndex + 1; + + // Common when reading or seeking across one boundary. + if (nextIndex <= lastIndex && + position < streamEnds[nextIndex]) + { + return nextIndex; } + + int low = nextIndex; + int high = nextIndex; + int step = 1; + + // Find an upper bound exponentially rather than searching the + // entire remaining range immediately. + while (high < lastIndex && position >= streamEnds[high]) + { + low = high + 1; + + int remaining = lastIndex - high; + int increment = step < remaining ? step : remaining; + + high += increment; + + if (step <= int.MaxValue / 2) + step <<= 1; + } + + return FindFirstEndGreaterThan(streamEnds, position, low, high); + } + + private static int FindBackward(long[] streamEnds, long position, int currentIndex) + { + int high = currentIndex - 1; + + // Adjacent segment fast path. + if (high >= 0) + { + long start = high == 0 ? 0 : streamEnds[high - 1]; + + if (position >= start && position < streamEnds[high]) + return high; + } + + int low = high; + int step = 1; + + while (low > 0 && position < streamEnds[low - 1]) + { + int decrement = Math.Min(step, low); + low -= decrement; + + if (step <= int.MaxValue / 2) + step <<= 1; + } + + return FindFirstEndGreaterThan(streamEnds, position, low, high); + } + + private static int FindFirstEndGreaterThan( + long[] streamEnds, + long position, + int low, + int high) + { + while (low < high) + { + int middle = low + ((high - low) >> 1); + + if (streamEnds[middle] > position) + high = middle; + else + low = middle + 1; + } + + return low; } } \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Event/PatchEvent.cs b/SharpHDiffPatch.Core/Event/PatchEvent.cs index ec43ff2..325d583 100644 --- a/SharpHDiffPatch.Core/Event/PatchEvent.cs +++ b/SharpHDiffPatch.Core/Event/PatchEvent.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; namespace SharpHDiffPatch.Core.Event { @@ -10,28 +11,57 @@ public class LoggerEvent(string message, Verbosity logLevel) public sealed class PatchEvent { - public PatchEvent() + private const double ScOneSecond = 1000; + private long _scLastTick = Environment.TickCount; + private long _scLastReceivedBytes; + private double _scLastSpeed; + + public void UpdateEvent(long currentSizePatched, long totalSizeToBePatched, long read, double totalSecond) { - Speed = 0; - CurrentSizePatched = 0; - TotalSizeToBePatched = 0; - Read = 0; + Speed = (long)(currentSizePatched / totalSecond); + CurrentSizePatched = currentSizePatched; + TotalSizeToBePatched = totalSizeToBePatched; + Read = read; } - public void UpdateEvent(long CurrentSizePatched, long TotalSizeToBePatched, long Read, double TotalSecond) + private long _currentSizePatched; + public long CurrentSizePatched { - Speed = (long)(CurrentSizePatched / TotalSecond); - this.CurrentSizePatched = CurrentSizePatched; - this.TotalSizeToBePatched = TotalSizeToBePatched; - this.Read = Read; + get => _currentSizePatched; + private set + { + double speed = CalculateSpeed(value - _currentSizePatched); + Speed = (long)speed; + TimeLeft = TimeSpan.FromSeconds((TotalSizeToBePatched - value) / (double.IsInfinity(speed) || speed <= 0 ? 1 : speed)); + ProgressPercentage = Math.Round(value / (double)TotalSizeToBePatched * 100, 2); + _currentSizePatched = value; + } } - public long CurrentSizePatched { get; private set; } - public long TotalSizeToBePatched { get; private set; } - public double ProgressPercentage => Math.Round((CurrentSizePatched / (double)TotalSizeToBePatched) * 100, 2); - public long Read { get; private set; } - public long Speed { get; private set; } - public TimeSpan TimeLeft => checked(TimeSpan.FromSeconds((TotalSizeToBePatched - CurrentSizePatched) / UnZeroed(Speed))); - private long UnZeroed(long Input) => Math.Max(Input, 1); + public long TotalSizeToBePatched { get; private set; } + public double ProgressPercentage { get; private set; } + public long Read { get; private set; } + public long Speed { get; private set; } + public TimeSpan TimeLeft { get; private set; } + + private double CalculateSpeed(long receivedBytes) => CalculateSpeed(receivedBytes, ref _scLastSpeed, ref _scLastReceivedBytes, ref _scLastTick); + + private static double CalculateSpeed(long receivedBytes, ref double lastSpeedToUse, ref long lastReceivedBytesToUse, ref long lastTickToUse) + { + long currentTick = Environment.TickCount - lastTickToUse + 1; + long totalReceivedInSecond = Interlocked.Add(ref lastReceivedBytesToUse, receivedBytes); + double speed = totalReceivedInSecond * ScOneSecond / currentTick; + + if (!(currentTick > ScOneSecond)) + { + return lastSpeedToUse; + } + + lastSpeedToUse = speed; + _ = Interlocked.Exchange(ref lastSpeedToUse, speed); + _ = Interlocked.Exchange(ref lastReceivedBytesToUse, 0); + _ = Interlocked.Exchange(ref lastTickToUse, Environment.TickCount); + return lastSpeedToUse; + } } } diff --git a/SharpHDiffPatch.Core/HDiffPatch.cs b/SharpHDiffPatch.Core/HDiffPatch.cs index c21ceaf..85bc464 100644 --- a/SharpHDiffPatch.Core/HDiffPatch.cs +++ b/SharpHDiffPatch.Core/HDiffPatch.cs @@ -117,8 +117,8 @@ public sealed class HDiffPatch private Stream diffStream { get; set; } private bool isPatchDir { get; set; } - internal static PatchEvent PatchEvent = new PatchEvent(); - public static EventListener Event = new EventListener(); + internal static PatchEvent PatchEvent = new(); + public static EventListener Event = new(); public static Verbosity LogVerbosity { get; set; } = Verbosity.Quiet; @@ -234,7 +234,7 @@ public static long GetHDiffOldSize(Stream diffStream) public static HeaderInfoExt GetHDiffHeaderInfo(string diffFilePath) { - using FileStream fs = new FileStream(diffFilePath, FileMode.Open, FileAccess.Read); + using FileStream fs = new(diffFilePath, FileMode.Open, FileAccess.Read); _ = Header.TryParseHeaderInfo(fs, diffFilePath, out HeaderInfo headerInfo, out DataReferenceInfo headerInfoReference); return new HeaderInfoExt { headerInfo = headerInfo, dataReferenceInfo = headerInfoReference }; } diff --git a/SharpHDiffPatch.Core/Hash/Adler64/Adler64.cs b/SharpHDiffPatch.Core/Hash/Adler64/Adler64.cs deleted file mode 100644 index 609d53a..0000000 --- a/SharpHDiffPatch.Core/Hash/Adler64/Adler64.cs +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Adler64-SIMD implementation by Anarh2404 under MIT License - * https://github.com/Anarh2404/AdlerSimd - * - * And original Adler (Adler32) implementation by Jean-Loup Gailly and Mark Adler from ZLib - * https://www.zlib.net | https://github.com/madler/zlib - */ - -using System; -using System.Runtime.InteropServices; -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -#endif - -namespace SharpHDiffPatch.Core.Hash.Adler64 -{ - public static class Adler64 - { - private const ulong MOD64 = 4294967291; - private const uint NMAX64 = 363898415; - private const int MAXPART = 363898400; - private const int BLOCK_SIZE = 32; - - public static ulong GetAdler64(ReadOnlySpan buffer, ulong adler = 1) - { - ulong s1 = adler & 0xffffffff; - ulong s2 = adler >> 32; -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - if (Ssse3.IsSupported) - { - return GetSse(buffer, s1, s2); - } -#endif - return GetSimpleOptimized(buffer, s1, s2); - } - - internal static ulong GetSimple(ReadOnlySpan buffer, ulong s1, ulong s2) - { - foreach (var n in buffer) - { - s1 = (s1 + n) % MOD64; - s2 = (s2 + s1) % MOD64; - } - - return (s2 << 32) | s1; - } - - - internal static ulong GetSimpleOptimized(ReadOnlySpan buf, ulong adler, ulong sum2) - { - // Workaroud - // TODO: try to find the problem - - if (buf.Length > MAXPART) - { - int parts = (buf.Length / MAXPART) + 1; - ulong result = 0; - for (int i = 0; i < parts; i++) - { - var start = MAXPART * i; - var count = Math.Min(buf.Length - start, MAXPART); - var slice = buf.Slice(start, count); - result = GetSimpleOptimizedInternal(slice, adler, sum2); - adler = result & 0xffffffff; - sum2 = result >> 32; - } - - return result; - } - - return GetSimpleOptimizedInternal(buf, adler, sum2); - } - - internal static ulong GetSimpleOptimizedInternal(ReadOnlySpan buf, ulong adler, ulong sum2) - { - ulong n; - ulong len = (ulong)buf.Length; - if (len == 1) - { - adler += buf[0]; - if (adler >= MOD64) - adler -= MOD64; - sum2 += adler; - if (sum2 >= MOD64) - sum2 -= MOD64; - return adler | (sum2 << 32); - } - var idx = 0; - if (len < 16) - { - while (len-- != 0) - { - adler += buf[idx++]; - sum2 += adler; - } - if (adler >= MOD64) - adler -= MOD64; - sum2 %= MOD64; /* only added so many BASE's */ - return adler | (sum2 << 32); - } - - /* do length NMAX blocks -- requires just one modulo operation */ - - while (len >= NMAX64) - { - len -= NMAX64; - n = NMAX64 / 16; /* NMAX is divisible by 16 */ - do - { - /* 16 sums unrolled */ - adler += buf[idx + 0]; - sum2 += adler; - adler += buf[idx + 1]; - sum2 += adler; - adler += buf[idx + 2]; - sum2 += adler; - adler += buf[idx + 3]; - sum2 += adler; - adler += buf[idx + 4]; - sum2 += adler; - adler += buf[idx + 5]; - sum2 += adler; - adler += buf[idx + 6]; - sum2 += adler; - adler += buf[idx + 7]; - sum2 += adler; - adler += buf[idx + 8]; - sum2 += adler; - adler += buf[idx + 9]; - sum2 += adler; - adler += buf[idx + 10]; - sum2 += adler; - adler += buf[idx + 11]; - sum2 += adler; - adler += buf[idx + 12]; - sum2 += adler; - adler += buf[idx + 13]; - sum2 += adler; - adler += buf[idx + 14]; - sum2 += adler; - adler += buf[idx + 15]; - sum2 += adler; - - idx += 16; - } while (--n != 0); - adler %= MOD64; - sum2 %= MOD64; - } - - /* do remaining bytes (less than NMAX, still just one modulo) */ - if (len > 0) - { /* avoid modulos if none remaining */ - while (len >= 16) - { - len -= 16; - /* 16 sums unrolled */ - adler += buf[idx + 0]; - sum2 += adler; - adler += buf[idx + 1]; - sum2 += adler; - adler += buf[idx + 2]; - sum2 += adler; - adler += buf[idx + 3]; - sum2 += adler; - adler += buf[idx + 4]; - sum2 += adler; - adler += buf[idx + 5]; - sum2 += adler; - adler += buf[idx + 6]; - sum2 += adler; - adler += buf[idx + 7]; - sum2 += adler; - adler += buf[idx + 8]; - sum2 += adler; - adler += buf[idx + 9]; - sum2 += adler; - adler += buf[idx + 10]; - sum2 += adler; - adler += buf[idx + 11]; - sum2 += adler; - adler += buf[idx + 12]; - sum2 += adler; - adler += buf[idx + 13]; - sum2 += adler; - adler += buf[idx + 14]; - sum2 += adler; - adler += buf[idx + 15]; - sum2 += adler; - idx += 16; - } - while (len-- != 0) - { - adler += buf[idx++]; - sum2 += adler; - } - adler %= MOD64; - sum2 %= MOD64; - } - - /* return recombined sums */ - return adler | (sum2 << 32); - } - -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - internal unsafe static ulong GetSse(ReadOnlySpan buffer, ulong s1, ulong s2) - { - uint len = (uint)buffer.Length; - - uint blocks = len / BLOCK_SIZE; - len = len - blocks * BLOCK_SIZE; - - Vector128 tap1 = Vector128.Create(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17); - Vector128 tap2 = Vector128.Create(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); - Vector128 zero = Vector128.Zero; - Vector128 onesShort = Vector128.Create(1, 1, 1, 1, 1, 1, 1, 1); - Vector128 onesInt = Vector128.Create(1, 1, 1, 1); - Vector128 shuffleMask2301 = Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11); - Vector128 shuffleMask1032 = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); - Vector128 shuffleMaskTrim = Vector128.Create(0, 1, 2, 3, 255, 255, 255, 255, 8, 9, 10, 11, 255, 255, 255, 255); - // A B C D -> B A D C - const int S2301 = 2 << 6 | 3 << 4 | 0 << 2 | 1; - - - fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) - { - var buf = bufPtr; - - while (blocks != 0) - { - uint n = NMAX64 / BLOCK_SIZE; - if (n > blocks) - { - n = blocks; - } - - blocks -= n; - - // Process n blocks of data. At most NMAX data bytes can be - // processed before s2 must be reduced modulo BASE. - Vector128 v_ps = Vector128.Create(0, s1 * n); - Vector128 v_s2 = Vector128.Create(0, s2); - Vector128 v_s1 = Vector128.Create(0ul, 0); - - do - { - // Load 32 input bytes. - Vector128 bytes1 = Sse2.LoadVector128(&buf[0]); - Vector128 bytes2 = Sse2.LoadVector128(&buf[16]); - - - // Add previous block byte sum to v_ps. - v_ps = Sse2.Add(v_ps, v_s1); - - - - // Horizontally add the bytes for s1, multiply-adds the - // bytes by [ 32, 31, 30, ... ] for s2. - Vector128 sad1 = Sse2.SumAbsoluteDifferences(bytes1, zero); - v_s1 = Sse2.Add(v_s1, sad1.AsUInt64()); - Vector128 mad11 = Ssse3.MultiplyAddAdjacent(bytes1, tap1); - Vector128 mad12 = Sse2.MultiplyAddAdjacent(mad11, onesShort); - var mad121 = Sse2.Add(mad12, Sse2.Shuffle(mad12, S2301)); - var madTrimmed1 = Ssse3.Shuffle(mad121.AsByte(), shuffleMaskTrim); - var madTimmed1ULong = madTrimmed1.AsUInt64(); - v_s2 = Sse2.Add(v_s2, madTimmed1ULong); - - - - Vector128 sad2 = Sse2.SumAbsoluteDifferences(bytes2, zero); - v_s1 = Sse2.Add(v_s1, sad2.AsUInt64()); - Vector128 mad2 = Ssse3.MultiplyAddAdjacent(bytes2, tap2); - Vector128 mad22 = Sse2.MultiplyAddAdjacent(mad2, onesShort); - var mad221 = Sse2.Add(mad22, Sse2.Shuffle(mad22, S2301)); - var madTrimmed2 = Ssse3.Shuffle(mad221.AsByte(), shuffleMaskTrim); - var madTimmed2ULong = madTrimmed2.AsUInt64(); - v_s2 = Sse2.Add(v_s2, madTimmed2ULong); - - - buf += BLOCK_SIZE; - - n--; - } while (n != 0); - - - var shifted = Sse2.ShiftLeftLogical(v_ps, 5); - v_s2 = Sse2.Add(v_s2, shifted); - - s1 += v_s1.GetElement(0); - s1 += v_s1.GetElement(1); - - - s2 = v_s2.GetElement(0); - s2 += v_s2.GetElement(1); - - s1 %= MOD64; - s2 %= MOD64; - } - - if (len > 0) - { - if (len >= 16) - { - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - s2 += (s1 += *buf++); - len -= 16; - } - - while (len-- > 0) - { - s2 += (s1 += *buf++); - } - if (s1 >= MOD64) - { - s1 -= MOD64; - } - - s2 %= MOD64; - } - - return s1 | (s2 << 32); - } - } -#endif - } -} \ No newline at end of file diff --git a/SharpHDiffPatch.Core/Hash/Crc32/Crc32Algorithm.cs b/SharpHDiffPatch.Core/Hash/Crc32/Crc32Algorithm.cs deleted file mode 100644 index 945098d..0000000 --- a/SharpHDiffPatch.Core/Hash/Crc32/Crc32Algorithm.cs +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Fast Crc32-NET implementation by Force.NET Team under MIT License - * https://github.com/force-net/Crc32.NET - */ - -using System; -using System.Buffers.Binary; -using System.Security.Cryptography; - -namespace SharpHDiffPatch.Core.Hash.Crc32 -{ - /// - /// Implementation of CRC-32. - /// This class supports several convenient static methods returning the CRC as UInt32. - /// - public class Crc32Algorithm : HashAlgorithm - { - private uint _currentCrc; - - private readonly bool _isBigEndian = true; - - private readonly SafeProxy _proxy = new SafeProxy(); - - /// - /// Initializes a new instance of the class. - /// - public Crc32Algorithm() - { - HashSizeValue = 32; - } - - /// - /// Initializes a new instance of the class. - /// - /// Should return bytes result as big endian or little endian - // Crc32 by dariogriffo uses big endian, so, we need to be compatible and return big endian as default - public Crc32Algorithm(bool isBigEndian = true) - : this() - { - _isBigEndian = isBigEndian; - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// Input buffer containing data to be checksummed. - public void Append(ReadOnlySpan input) - { - _currentCrc = AppendInternal(_currentCrc, input); - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// - /// Initial CRC value for the algorithm. It is zero for the first buffer. - /// Subsequent buffers should have their initial value set to CRC value returned by previous call to this method. - /// - /// Input buffer containing data to be checksummed. - public void Append(byte[] input) - { - _currentCrc = AppendInternal(_currentCrc, input); - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// - /// Initial CRC value for the algorithm. It is zero for the first buffer. - /// Subsequent buffers should have their initial value set to CRC value returned by previous call to this method. - /// - /// Input buffer with data to be checksummed. - /// Offset of the input data within the buffer. - /// Length of the input data in the buffer. - /// Accumulated CRC-32 of all buffers processed so far. - public void Append(byte[] input, int offset, int length) - { - if (input == null) - throw new ArgumentNullException("input"); - if (offset < 0 || length < 0 || offset + length > input.Length) - throw new ArgumentOutOfRangeException("length"); - _currentCrc = AppendInternal(_currentCrc, input, offset, length); - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// - /// Initial CRC value for the algorithm. It is zero for the first buffer. - /// Subsequent buffers should have their initial value set to CRC value returned by previous call to this method. - /// - /// Input buffer with data to be checksummed. - /// Offset of the input data within the buffer. - /// Length of the input data in the buffer. - /// Accumulated CRC-32 of all buffers processed so far. - public uint Append(uint initial, byte[] input, int offset, int length) - { - if (input == null) - throw new ArgumentNullException("input"); - if (offset < 0 || length < 0 || offset + length > input.Length) - throw new ArgumentOutOfRangeException("length"); - return AppendInternal(initial, input, offset, length); - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// - /// Initial CRC value for the algorithm. It is zero for the first buffer. - /// Subsequent buffers should have their initial value set to CRC value returned by previous call to this method. - /// - /// Input buffer containing data to be checksummed. - /// Accumulated CRC-32 of all buffers processed so far. - public uint Append(uint initial, byte[] input) - { - if (input == null) - throw new ArgumentNullException(); - return AppendInternal(initial, input, 0, input.Length); - } - - /// - /// Computes CRC-32 from multiple buffers. - /// Call this method multiple times to chain multiple buffers. - /// - /// - /// Initial CRC value for the algorithm. It is zero for the first buffer. - /// Subsequent buffers should have their initial value set to CRC value returned by previous call to this method. - /// - /// Input buffer containing data to be checksummed. - /// Accumulated CRC-32 of all buffers processed so far. - public uint Append(uint initial, ReadOnlySpan input) - { - return AppendInternal(initial, input); - } - - /// - /// Computes CRC-32 from input buffer. - /// - /// Input buffer containing data to be checksummed. - /// CRC-32 buffer of the input buffer. - public byte[] ComputeHashByte(byte[] input) - { - _currentCrc = AppendInternal(0, input); - return HashFinal(); - } - - /// - /// Computes CRC-32 from input buffer. - /// - /// Input buffer containing data to be checksummed. - /// CRC-32 buffer of the input buffer. - public byte[] ComputeHashByte(ReadOnlySpan input) - { - _currentCrc = AppendInternal(0, input); - return HashFinal(); - } - - /// - /// Computes CRC-32 from input buffer. - /// - /// Input buffer with data to be checksummed. - /// Offset of the input data within the buffer. - /// Length of the input data in the buffer. - /// CRC-32 of the data in the buffer. - public uint Compute(byte[] input, int offset, int length) - { - return Append(0, input, offset, length); - } - - /// - /// Computes CRC-32 from input buffer. - /// - /// Input buffer containing data to be checksummed. - /// CRC-32 of the data in the buffer. - public uint Compute(byte[] input) - { - return Append(0, input); - } - - /// - /// Computes CRC-32 from input buffer. - /// - /// Input buffer with data to be checksummed. - /// CRC-32 of the data in the buffer. - public uint Compute(ReadOnlySpan input) - { - return Append(0, input); - } - - /// - /// Computes CRC-32 from input buffer and writes it after end of data (buffer should have 4 bytes reserved space for it). Can be used in conjunction with - /// - /// Input buffer with data to be checksummed. - /// Offset of the input data within the buffer. - /// Length of the input data in the buffer. - /// CRC-32 of the data in the buffer. - public uint ComputeAndWriteToEnd(byte[] input, int offset, int length) - { - if (length + 4 > input.Length) - throw new ArgumentOutOfRangeException("length", "Length of data should be less than array length - 4 bytes of CRC data"); - var crc = Append(0, input, offset, length); - var r = offset + length; - input[r] = (byte)crc; - input[r + 1] = (byte)(crc >> 8); - input[r + 2] = (byte)(crc >> 16); - input[r + 3] = (byte)(crc >> 24); - return crc; - } - - /// - /// Computes CRC-32 from input buffer - 4 bytes and writes it as last 4 bytes of buffer. Can be used in conjunction with - /// - /// Input buffer with data to be checksummed. - /// CRC-32 of the data in the buffer. - public uint ComputeAndWriteToEnd(byte[] input) - { - if (input.Length < 4) - throw new ArgumentOutOfRangeException("input", "Input array should be 4 bytes at least"); - return ComputeAndWriteToEnd(input, 0, input.Length - 4); - } - - /// - /// Validates correctness of CRC-32 data in source buffer with assumption that CRC-32 data located at end of buffer in reverse bytes order. Can be used in conjunction with - /// - /// Input buffer with data to be checksummed. - /// Offset of the input data within the buffer. - /// Length of the input data in the buffer with CRC-32 bytes. - /// Is checksum valid. - public bool IsValidWithCrcAtEnd(byte[] input, int offset, int lengthWithCrc) - { - return Append(0, input, offset, lengthWithCrc) == 0x2144DF1C; - } - - /// - /// Validates correctness of CRC-32 data in source buffer with assumption that CRC-32 data located at end of buffer in reverse bytes order. Can be used in conjunction with - /// - /// Input buffer with data to be checksummed. - /// Is checksum valid. - public bool IsValidWithCrcAtEnd(byte[] input) - { - if (input.Length < 4) - throw new ArgumentOutOfRangeException("input", "Input array should be 4 bytes at least"); - return Append(0, input, 0, input.Length) == 0x2144DF1C; - } - - /// - /// Resets internal state of the algorithm. Used internally. - /// - public override void Initialize() - { - _currentCrc = 0; - } - - /// - /// Appends CRC-32 from given buffer - /// - protected override void HashCore(byte[] input, int offset, int length) - { - _currentCrc = AppendInternal(_currentCrc, input, offset, length); - } - -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - /// - /// Appends CRC-32 from given buffer - /// - protected override void HashCore(ReadOnlySpan source) - { - _currentCrc = AppendInternal(_currentCrc, source); - } -#endif - - /// - /// Computes CRC-32 from - /// - protected override byte[] HashFinal() - { - if (_isBigEndian) - return new[] { (byte)(_currentCrc >> 24), (byte)(_currentCrc >> 16), (byte)(_currentCrc >> 8), (byte)_currentCrc }; - else - return new[] { (byte)_currentCrc, (byte)(_currentCrc >> 8), (byte)(_currentCrc >> 16), (byte)(_currentCrc >> 24) }; - } - -#if !(NETSTANDARD2_0 || NET461_OR_GREATER) - /// - /// Computes CRC-32 from - /// - protected override bool TryHashFinal(Span destination, out int bytesWritten) - { - if (destination.Length < 4) - { - bytesWritten = 0; - return false; - } - - if (_isBigEndian) - { - BinaryPrimitives.WriteUInt32BigEndian(destination, _currentCrc); - } - else - { - BinaryPrimitives.WriteUInt32LittleEndian(destination, _currentCrc); - } - - bytesWritten = 4; - return true; - } -#endif - - /// - /// Get final hash from processed buffer - /// - public override byte[] Hash - { - get - { - return HashFinal(); - } - } - - private uint AppendInternal(uint initial, byte[] input, int offset, int length) - { - if (length > 0) - { - return _proxy.Append(initial, input, offset, length); - } - else - return initial; - } - - private uint AppendInternal(uint initial, ReadOnlySpan input) - { - if (input.Length > 0) - { - return _proxy.Append(initial, input); - } - else - return initial; - } - } -} diff --git a/SharpHDiffPatch.Core/Hash/Crc32/SafeProxy.cs b/SharpHDiffPatch.Core/Hash/Crc32/SafeProxy.cs deleted file mode 100644 index 34ebd56..0000000 --- a/SharpHDiffPatch.Core/Hash/Crc32/SafeProxy.cs +++ /dev/null @@ -1,83 +0,0 @@ -/* This is .NET safe implementation of Crc32 algorithm. - * This implementation was investigated as fastest from different variants. It based on Robert Vazan native implementations of Crc32C - * Also, it is good for x64 and for x86, so, it seems, there is no sense to do 2 different realizations. - * - * Addition: some speed increase was found with splitting xor to 4 independent blocks. Also, some attempts to optimize unaligned tails was unsuccessfull (JIT limitations?). - * - * - * Max Vysokikh, 2016-2017 - */ - -using System; - -namespace SharpHDiffPatch.Core.Hash.Crc32 -{ - internal class SafeProxy - { - private const uint Poly = 0xedb88320u; - - private readonly uint[] _table = new uint[16 * 256]; - - internal SafeProxy() - { - Init(Poly); - } - - protected void Init(uint poly) - { - var table = _table; - for (uint i = 0; i < 256; i++) - { - uint res = i; - for (int t = 0; t < 16; t++) - { - for (int k = 0; k < 8; k++) res = (res & 1) == 1 ? poly ^ (res >> 1) : (res >> 1); - table[(t * 256) + i] = res; - } - } - } - - public uint Append(uint crc, byte[] input, int offset, int length) - { - return Append(crc, input.AsSpan(offset, length)); - } - - public uint Append(uint crc, ReadOnlySpan input) - { - uint crcLocal = uint.MaxValue ^ crc; - - uint[] table = _table; - while (input.Length >= 16) - { - var a = table[(3 * 256) + input[12]] - ^ table[(2 * 256) + input[13]] - ^ table[(1 * 256) + input[14]] - ^ table[(0 * 256) + input[15]]; - - var b = table[(7 * 256) + input[8]] - ^ table[(6 * 256) + input[9]] - ^ table[(5 * 256) + input[10]] - ^ table[(4 * 256) + input[11]]; - - var c = table[(11 * 256) + input[4]] - ^ table[(10 * 256) + input[5]] - ^ table[(9 * 256) + input[6]] - ^ table[(8 * 256) + input[7]]; - - var d = table[(15 * 256) + ((byte)crcLocal ^ input[0])] - ^ table[(14 * 256) + ((byte)(crcLocal >> 8) ^ input[1])] - ^ table[(13 * 256) + ((byte)(crcLocal >> 16) ^ input[2])] - ^ table[(12 * 256) + ((crcLocal >> 24) ^ input[3])]; - - crcLocal = d ^ c ^ b ^ a; - input = input.Slice(16); - } - - var i = 0; - while (i < input.Length) - crcLocal = table[(byte)(crcLocal ^ input[i++])] ^ crcLocal >> 8; - - return crcLocal ^ uint.MaxValue; - } - } -} diff --git a/SharpHDiffPatch.Core/Patch/Header.cs b/SharpHDiffPatch.Core/Patch/Header.cs index 67baa97..6252143 100644 --- a/SharpHDiffPatch.Core/Patch/Header.cs +++ b/SharpHDiffPatch.Core/Patch/Header.cs @@ -1,5 +1,4 @@ using SharpHDiffPatch.Core.Binary; -using SharpHDiffPatch.Core.Binary.Compression; using System; using System.IO; @@ -8,7 +7,7 @@ namespace SharpHDiffPatch.Core.Patch internal sealed class Header { #if !(NETSTANDARD2_0 || NET461_OR_GREATER) - private static readonly char[] HDIFF_HEAD = new char[] { 'H', 'D', 'I', 'F', 'F' }; + private static readonly char[] HDIFF_HEAD = ['H', 'D', 'I', 'F', 'F']; #else private const string HDIFF_HEAD = "HDIFF"; #endif @@ -71,7 +70,7 @@ private static void TryReadExternReferenceInfo(Stream sr, string diffPath, ref H long curPos = sr.Position; referenceInfo.headDataOffset = curPos; - curPos += (referenceInfo.headDataCompressedSize > 0 ? referenceInfo.headDataCompressedSize : referenceInfo.headDataSize); + curPos += referenceInfo.headDataCompressedSize > 0 ? referenceInfo.headDataCompressedSize : referenceInfo.headDataSize; referenceInfo.privateExternDataOffset = curPos; curPos += referenceInfo.privateExternDataSize; @@ -145,10 +144,10 @@ private static void TryReadNonSingleFileHeaderInfo(Stream sr, string diffPath, r GetDiffChunkInfo(sr, out headerInfo.chunkInfo, typeEndPos); - headerInfo.compressedCount = ((headerInfo.chunkInfo.compress_cover_buf_size > 1) ? 1 : 0) - + ((headerInfo.chunkInfo.compress_rle_ctrlBuf_size > 1) ? 1 : 0) - + ((headerInfo.chunkInfo.compress_rle_codeBuf_size > 1) ? 1 : 0) - + ((headerInfo.chunkInfo.compress_newDataDiff_size > 1) ? 1 : 0); + headerInfo.compressedCount = (headerInfo.chunkInfo.compress_cover_buf_size > 1 ? 1 : 0) + + (headerInfo.chunkInfo.compress_rle_ctrlBuf_size > 1 ? 1 : 0) + + (headerInfo.chunkInfo.compress_rle_codeBuf_size > 1 ? 1 : 0) + + (headerInfo.chunkInfo.compress_newDataDiff_size > 1 ? 1 : 0); HDiffPatch.Event.PushLog($"[Header::TryReadNonSingleFileHeaderInfo] compressedCount: {headerInfo.compressedCount}", Verbosity.Debug); } diff --git a/SharpHDiffPatch.Core/Patch/PatchCore.cs b/SharpHDiffPatch.Core/Patch/PatchCore.cs index 3cc3ed3..fb6abbc 100644 --- a/SharpHDiffPatch.Core/Patch/PatchCore.cs +++ b/SharpHDiffPatch.Core/Patch/PatchCore.cs @@ -135,8 +135,8 @@ public Stream GetBufferStreamFromOffset(CompressionMode compMode, Stream sourceS private MemoryStream CreateAndCopyToMemoryStream(Stream source) { - MemoryStream returnStream = new MemoryStream(); - byte[] buffer = ArrayPool.Shared.Rent(MaxArrayPoolLen); + MemoryStream returnStream = new(); + byte[] buffer = ArrayPool.Shared.Rent(MaxArrayPoolLen); try { @@ -189,7 +189,7 @@ internal static IEnumerable EnumerateCoverHeaders(Stream coverReade oldPosBack = oldPos; newPosBack += copyLength; - oldPosBack += true ? coverLength : 0; + oldPosBack += coverLength; yield return new CoverHeader(oldPos, newPosBack, coverLength, coverCount); newPosBack += coverLength; @@ -217,7 +217,7 @@ internal static IEnumerable EnumerateCoverHeaders(Stream coverReade newPosBack += copyLength; oldPosBack = oldPos; - oldPosBack += true ? coverLength : 0; + oldPosBack += coverLength; yield return new CoverHeader(oldPos, newPosBack, coverLength, coverCount); newPosBack += coverLength; @@ -243,15 +243,15 @@ internal void RunCopySimilarFilesRoutine() private void WriteCoverStreamToOutput(Stream[] clips, Stream inputStream, Stream outputStream, long coverCount, long coverSize, long newDataSize) { byte[] sharedBuffer = ArrayPool.Shared.Rent(MaxArrayPoolLen); - MemoryStream cacheOutputStream = new MemoryStream(); + MemoryStream cacheOutputStream = new(); try { RunCopySimilarFilesRoutine(); - long newPosBack = 0; - RleRefClipStruct rleStruct = new RleRefClipStruct(); - CoverHeader[] headers = EnumerateCoverHeaders(clips[0], coverSize, coverCount).ToArray(); + long newPosBack = 0; + RleRefClipStruct rleStruct = new(); + CoverHeader[] headers = [.. EnumerateCoverHeaders(clips[0], coverSize, coverCount)]; for (int i = 0; i < headers.Length; i++) { @@ -559,8 +559,8 @@ private void CopyOldSimilarToNewFiles(DirectoryReferencePair dirData) while (curPathIndex < pathCount) { - if ((curNewRefIndex < newRefCount) - && (curPathIndex == (dirData.NewRefList.Length > 0 ? (int)dirData.NewRefList[curNewRefIndex] : curNewRefIndex))) + if (curNewRefIndex < newRefCount + && curPathIndex == (dirData.NewRefList.Length > 0 ? (int)dirData.NewRefList[curNewRefIndex] : curNewRefIndex)) { bool isPathADir = IsPathADir(dirData.NewUtf8PathList[(int)dirData.NewRefList[curNewRefIndex]]); @@ -568,7 +568,7 @@ private void CopyOldSimilarToNewFiles(DirectoryReferencePair dirData) ++curNewRefIndex; } else if (curSamePairIndex < samePairCount - && (curPathIndex == (int)dirData.DataSamePairList[curSamePairIndex].NewIndex)) + && curPathIndex == (int)dirData.DataSamePairList[curSamePairIndex].NewIndex) { ++curSamePairIndex; ++curPathIndex; @@ -620,9 +620,9 @@ private void CopyFile(string inputPath, string outputPath) #else byte[] buffer = new byte[MaxArrayCopyLen]; #endif - using FileStream ifs = new FileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read); - using FileStream ofs = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.Write); - int read; + using FileStream ifs = new(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read); + using FileStream ofs = new(outputPath, FileMode.Create, FileAccess.Write, FileShare.Write); + int read; #if !(NETSTANDARD2_0 || NET461_OR_GREATER) while ((read = ifs.Read(buffer)) > 0) #else diff --git a/SharpHDiffPatch.Core/Patch/PatchCoreFastBuffer.cs b/SharpHDiffPatch.Core/Patch/PatchCoreFastBuffer.cs index 5df7a7c..1baf9e6 100644 --- a/SharpHDiffPatch.Core/Patch/PatchCoreFastBuffer.cs +++ b/SharpHDiffPatch.Core/Patch/PatchCoreFastBuffer.cs @@ -38,7 +38,7 @@ public void UncoverBufferClipsStream(Stream[] clips, Stream inputStream, Stream { if (!PatchSizeHelper.CanUseFastBuffer(headerInfo)) { - HDiffPatch.Event.PushLog("[PatchCoreFastBuffer::UncoverBufferClipsStream] Fast buffer requirements exceeded; delegating to streaming patch core.", Verbosity.Info); + HDiffPatch.Event.PushLog("[PatchCoreFastBuffer::UncoverBufferClipsStream] Fast buffer requirements exceeded; delegating to streaming patch core."); _core.UncoverBufferClipsStream(clips, inputStream, outputStream, headerInfo); return; } @@ -189,8 +189,8 @@ private unsafe void WriteCoverStreamToOutputFast(Stream[] clips, Stream inputStr : new byte[rleCodeBufSize]; #endif - RleRefClipStruct rleStruct = new RleRefClipStruct(); - int coverBufferLen = checked(sizeof(CoverHeader) * PatchSizeHelper.ToCheckedInt32(headerInfo.chunkInfo.coverCount, nameof(headerInfo.chunkInfo.coverCount))); + RleRefClipStruct rleStruct = new(); + int coverBufferLen = checked(sizeof(CoverHeader) * PatchSizeHelper.ToCheckedInt32(headerInfo.chunkInfo.coverCount, nameof(headerInfo.chunkInfo.coverCount))); byte[] coverBuffer = #if NET6_0_OR_GREATER GC.AllocateUninitializedArray(coverBufferLen); diff --git a/SharpHDiffPatch.Core/Patch/PatchDir.cs b/SharpHDiffPatch.Core/Patch/PatchDir.cs index 210c381..91343cf 100644 --- a/SharpHDiffPatch.Core/Patch/PatchDir.cs +++ b/SharpHDiffPatch.Core/Patch/PatchDir.cs @@ -93,8 +93,8 @@ public void Patch(string input, string output, Action writeBytesDelegate, HDiffPatch.Event.PushLog($"[PatchDir::Patch] Total new size: {totalSizePatched} bytes ({newPatchSize} (new data) + {samePathSize} (same data))", Verbosity.Verbose); - FileStream[] mergedOldStream = GetRefOldStreams(dirData); - NewFileCombinedStream[] mergedNewStream = GetRefNewStreams(dirData); + FileStream[] mergedOldStream = GetRefOldStreams(dirData); + CombinedStreamSegment[] mergedNewStream = GetRefNewStreams(dirData); HDiffPatch.Event.PushLog($"[PatchDir::Patch] Initialized {mergedOldStream.Length} old files and {mergedNewStream.Length} new files into combined stream", Verbosity.Verbose); HDiffPatch.Event.PushLog($"[PatchDir::Patch] Seek the patch stream to: {_referenceInfo.hdiffDataOffset}. Jump to read header for clip streams!", Verbosity.Verbose); @@ -110,8 +110,8 @@ public void Patch(string input, string output, Action writeBytesDelegate, patchCore.SetDirectoryReferencePair(dirData); patchCore.SetSizeToBePatched(totalSizePatched); - using (Stream newStream = new CombinedStream(mergedNewStream)) - using (Stream oldStream = new CombinedStream(mergedOldStream)) + using (Stream newStream = new CombinedStream(mergedNewStream)) + using (Stream oldStream = new CombinedStream(mergedOldStream)) { long oldFileSize = GetOldFileSize(dirData); if (oldStream.Length != _headerInfo.oldDataSize) @@ -172,18 +172,21 @@ private long GetSameFileSize(DirectoryReferencePair dirData) private IPatchCore CreatePatchCore(Action writeBytesDelegate, long totalSizePatched) { bool wantFastBuffer = _useFastBuffer && _useBufferedPatch && !_headerInfo.isSingleCompressedDiff; - if (wantFastBuffer && PatchSizeHelper.CanUseFastBuffer(_headerInfo)) - return new PatchCoreFastBuffer(totalSizePatched, Stopwatch.StartNew(), _basePathInput, _basePathOutput, writeBytesDelegate, _token); - - if (wantFastBuffer) - HDiffPatch.Event.PushLog("[PatchDir::CreatePatchCore] Fast buffer disabled: patch chunk sizes exceed int32-safe limits; using streaming patch core.", Verbosity.Info); + switch (wantFastBuffer) + { + case true when PatchSizeHelper.CanUseFastBuffer(_headerInfo): + return new PatchCoreFastBuffer(totalSizePatched, Stopwatch.StartNew(), _basePathInput, _basePathOutput, writeBytesDelegate, _token); + case true: + HDiffPatch.Event.PushLog("[PatchDir::CreatePatchCore] Fast buffer disabled: patch chunk sizes exceed int32-safe limits; using streaming patch core."); + break; + } return new PatchCore(totalSizePatched, Stopwatch.StartNew(), _basePathInput, _basePathOutput, writeBytesDelegate, _token); } private void StartPatchRoutine(Stream inputStream, Stream outputStream, long newDataSize, long offset, IPatchCore patchCore) { - Stream[] clips = new Stream[_headerInfo.isSingleCompressedDiff ? 1 : 4]; + var clips = new Stream[_headerInfo.isSingleCompressedDiff ? 1 : 4]; Stream[] sourceClips = _headerInfo.isSingleCompressedDiff ? [ _spawnPatchStream() ] : [ @@ -202,7 +205,7 @@ [ _spawnPatchStream() ] : offset += _headerInfo.singleChunkInfo.diffDataPos; clips[0] = patchCore.GetBufferStreamFromOffset(_headerInfo.compMode, sourceClips[0], offset + coverPadding, - _headerInfo.singleChunkInfo.uncompressedSize, _headerInfo.singleChunkInfo.compressedSize, out long nextLength, + _headerInfo.singleChunkInfo.uncompressedSize, _headerInfo.singleChunkInfo.compressedSize, out long _, _useBufferedPatch, false); } else @@ -252,9 +255,9 @@ private FileStream[] GetRefOldStreams(DirectoryReferencePair dirData) return streams; } - private NewFileCombinedStream[] GetRefNewStreams(DirectoryReferencePair dirData) + private CombinedStreamSegment[] GetRefNewStreams(DirectoryReferencePair dirData) { - NewFileCombinedStream[] streams = new NewFileCombinedStream[dirData.NewRefList.Length]; + var streams = new CombinedStreamSegment[dirData.NewRefList.Length]; for (int i = 0; i < dirData.NewRefList.Length; i++) { ref string newPathByIndex = ref PatchCore.NewPathByIndex(dirData.NewUtf8PathList, dirData.NewRefList[i]); @@ -266,12 +269,12 @@ private NewFileCombinedStream[] GetRefNewStreams(DirectoryReferencePair dirData) HDiffPatch.Event.PushLog($"[PatchDir::GetRefNewStreams] Assigning stream to the new path: {combinedNewPath}", Verbosity.Debug); - NewFileCombinedStream @new = new NewFileCombinedStream + var stream = new CombinedStreamSegment { Stream = new FileStream(combinedNewPath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite), - Size = dirData.NewRefSizeList[i] + Length = dirData.NewRefSizeList[i] }; - streams[i] = @new; + streams[i] = stream; } return streams; @@ -280,7 +283,7 @@ private NewFileCombinedStream[] GetRefNewStreams(DirectoryReferencePair dirData) private DirectoryReferencePair InitializeDirPatcher(Stream reader) { HDiffPatch.Event.PushLog("[PatchDir::InitializeDirPatcher] Reading PatchDir header...", Verbosity.Verbose); - DirectoryReferencePair returnValue = new DirectoryReferencePair(); + DirectoryReferencePair returnValue = new(); HDiffPatch.Event.PushLog($"[PatchDir::InitializeDirPatcher] Reading path string buffers -> OldPath: {(int)_referenceInfo.inputSumSize}", Verbosity.Verbose); reader.GetPathsFromStream(out returnValue.OldUtf8PathList, (int)_referenceInfo.inputSumSize, (int)_referenceInfo.inputDirCount); diff --git a/SharpHDiffPatch.Core/Patch/PatchSingle.cs b/SharpHDiffPatch.Core/Patch/PatchSingle.cs index 3f4b380..6e943bb 100644 --- a/SharpHDiffPatch.Core/Patch/PatchSingle.cs +++ b/SharpHDiffPatch.Core/Patch/PatchSingle.cs @@ -21,8 +21,8 @@ public void Patch(string input, string output, Action writeBytesDelegate, _isUseFullBuffer = useFullBuffer; _isUseFastBuffer = useFastBuffer; - using FileStream inputStream = new FileStream(input, FileMode.Open, FileAccess.Read, FileShare.Read, headerInfo.oldDataSize.GetFileStreamBufferSize()); - using FileStream outputStream = new FileStream(output, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, headerInfo.newDataSize.GetFileStreamBufferSize()); + using FileStream inputStream = new(input, FileMode.Open, FileAccess.Read, FileShare.Read, headerInfo.oldDataSize.GetFileStreamBufferSize()); + using FileStream outputStream = new(output, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite, headerInfo.newDataSize.GetFileStreamBufferSize()); if (inputStream.Length != headerInfo.oldDataSize) throw new InvalidDataException($"[PatchSingle::Patch] The patch directory is expecting old size to be equivalent as: {headerInfo.oldDataSize} bytes, but the input file has unmatched size: {inputStream.Length} bytes!"); @@ -41,7 +41,7 @@ private IPatchCore CreatePatchCore(string input, string output, Action wri return new PatchCoreFastBuffer(headerInfo.newDataSize, Stopwatch.StartNew(), input, output, writeBytesDelegate, token); if (wantFastBuffer) - HDiffPatch.Event.PushLog("[PatchSingle::CreatePatchCore] Fast buffer disabled: patch chunk sizes exceed int32-safe limits; using streaming patch core.", Verbosity.Info); + HDiffPatch.Event.PushLog("[PatchSingle::CreatePatchCore] Fast buffer disabled: patch chunk sizes exceed int32-safe limits; using streaming patch core."); return new PatchCore(headerInfo.newDataSize, Stopwatch.StartNew(), input, output, writeBytesDelegate, token); } diff --git a/SharpHDiffPatch.Core/SharpHDiffPatch.Core.csproj b/SharpHDiffPatch.Core/SharpHDiffPatch.Core.csproj index 47502f7..cc16717 100644 --- a/SharpHDiffPatch.Core/SharpHDiffPatch.Core.csproj +++ b/SharpHDiffPatch.Core/SharpHDiffPatch.Core.csproj @@ -24,6 +24,7 @@ + diff --git a/SharpHDiffPatch.sln b/SharpHDiffPatch.sln index fbf471d..5dd2029 100644 --- a/SharpHDiffPatch.sln +++ b/SharpHDiffPatch.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.6.33617.297 +# Visual Studio Version 18 +VisualStudioVersion = 18.7.11925.98 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharpHDiffPatch.Core", "SharpHDiffPatch.Core\SharpHDiffPatch.Core.csproj", "{57CB7A6D-2474-4A01-BE1A-5D1488F81390}" EndProject diff --git a/SharpHDiffPatch/Program.cs b/SharpHDiffPatch/Program.cs index db70cfa..1d8b68d 100644 --- a/SharpHDiffPatch/Program.cs +++ b/SharpHDiffPatch/Program.cs @@ -158,9 +158,9 @@ private static void PrintLog(LoggerEvent e) { string label = e.LogLevel switch { - Verbosity.Info => $"[Info] ", - Verbosity.Verbose => $"[Verbose] ", - Verbosity.Debug => $"[Debug] ", + Verbosity.Info => "[Info] ", + Verbosity.Verbose => "[Verbose] ", + Verbosity.Debug => "[Debug] ", _ => "" }; @@ -174,8 +174,14 @@ private static async void EventListener_PatchEvent(object? sender, PatchEvent e) if (await CheckIfNeedRefreshStopwatch()) { Console.Write( - $"Patching: {e.ProgressPercentage}% | {SummarizeSizeSimple(e.CurrentSizePatched)}/{SummarizeSizeSimple(e.TotalSizeToBePatched)} @{SummarizeSizeSimple(e.Speed)}/s \r"); + $"Patching: {e.ProgressPercentage}% | " + + $"{SummarizeSizeSimple(e.CurrentSizePatched)}/{SummarizeSizeSimple(e.TotalSizeToBePatched)} " + + $"@{SummarizeSizeSimple(e.Speed)}/s " + + $"[{string.Format("{0:hh}h:{0:mm}m:{0:ss}s remaining", TimeSpan.FromSeconds((e.TotalSizeToBePatched - e.CurrentSizePatched) / UnZeroed(e.Speed)))}] \r"); } + + return; + static double UnZeroed(double input) => Math.Max(input, 1); } catch { diff --git a/SharpHDiffPatch/Properties/launchSettings.json b/SharpHDiffPatch/Properties/launchSettings.json index 981bbd5..365f845 100644 --- a/SharpHDiffPatch/Properties/launchSettings.json +++ b/SharpHDiffPatch/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "SharpHDiffPatch-bin": { "commandName": "Project", - "commandLineArgs": "\"G:\\CollapseData\\Hi3SEA\\BH3_v8.0.0_647b2f09ce4e\" \"G:\\CollapseData\\Hi3SEA\\delta-m0.diff\" \"G:\\CollapseData\\Hi3SEA\\BH3_v8.1.0_d950a68ed449_dotnet\" -l Verbose -B -b partial" + "commandLineArgs": "\"G:\\Hi3SEA\" \"G:\\Hi3CNRecipe.cookbook.bz2\" \"K:\\Hi3CN-dotnet\" -l Verbose -B -b partial" } } } \ No newline at end of file