zstd-ruby/spec/zstd-ruby-streaming-decompress_spec.rb at 3109328dccbab6eef9d85174dcb681597c2bf37c · SpringMT/zstd-ruby · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
require "spec_helper"
require 'zstd-ruby'
require 'securerandom'

RSpec.describe Zstd::StreamingDecompress do
  describe 'streaming decompress' do
    it 'shoud work' do
      # str = SecureRandom.hex(150)
      str = "foo bar buzz" * 100
      cstr = Zstd.compress(str)
      stream = Zstd::StreamingDecompress.new
      result = ''
      result << stream.decompress(cstr[0, 5])
      result << stream.decompress(cstr[5, 5])
      result << stream.decompress(cstr[10..-1])
      expect(result).to eq(str)
    end
  end

  describe 'decompress_with_pos' do
    it 'should return decompressed data and consumed input position' do
      str = "hello world test data"
      cstr = Zstd.compress(str)
      stream = Zstd::StreamingDecompress.new

      # Test with partial input
      result_array = stream.decompress_with_pos(cstr[0, 10])
      expect(result_array).to be_an(Array)
      expect(result_array.length).to eq(2)

      decompressed_data = result_array[0]
      consumed_bytes = result_array[1]

      expect(decompressed_data).to be_a(String)
      expect(consumed_bytes).to be_a(Integer)
      expect(consumed_bytes).to be > 0
      expect(consumed_bytes).to be <= 10
    end

    it 'should work with complete compressed data' do
      str = "foo bar buzz"
      cstr = Zstd.compress(str)
      stream = Zstd::StreamingDecompress.new

      result_array = stream.decompress_with_pos(cstr)
      decompressed_data = result_array[0]
      consumed_bytes = result_array[1]

      expect(decompressed_data).to eq(str)
      expect(consumed_bytes).to eq(cstr.length)
    end

    it 'should work with multiple calls' do
      str = "test data for multiple calls"
      cstr = Zstd.compress(str)
      stream = Zstd::StreamingDecompress.new

      result = ''
      total_consumed = 0
      chunk_size = 5

      while total_consumed < cstr.length
        remaining_data = cstr[total_consumed..-1]
        chunk = remaining_data[0, chunk_size]

        result_array = stream.decompress_with_pos(chunk)
        decompressed_chunk = result_array[0]
        consumed_bytes = result_array[1]

        result << decompressed_chunk
        total_consumed += consumed_bytes

        expect(consumed_bytes).to be > 0
        expect(consumed_bytes).to be <= chunk.length

        # If we consumed less than the chunk size, we might be done or need more data
        break if consumed_bytes < chunk.length && total_consumed == cstr.length
      end

      expect(result).to eq(str)
      expect(total_consumed).to eq(cstr.length)
    end
  end

  describe 'streaming decompress + GC.compact' do
    it 'shoud work' do
      # str = SecureRandom.hex(150)
      str = "foo bar buzz" * 100
      cstr = Zstd.compress(str)
      stream = Zstd::StreamingDecompress.new
      result = ''
      result << stream.decompress(cstr[0, 5])
      result << stream.decompress(cstr[5, 5])
      GC.compact
      result << stream.decompress(cstr[10..-1])
      expect(result).to eq(str)
    end
  end

  describe 'String dictionary streaming decompress + GC.compact' do
    let(:dictionary) do
      File.read("#{__dir__}/dictionary")
    end
    let(:user_json) do
      File.read("#{__dir__}/user_springmt.json")
    end
    it 'shoud work' do
      compressed_json = Zstd.compress(user_json, dict: dictionary)
      stream = Zstd::StreamingDecompress.new(dict: dictionary)
      result = ''
      result << stream.decompress(compressed_json[0, 5])
      result << stream.decompress(compressed_json[5, 5])
      GC.compact
      result << stream.decompress(compressed_json[10..-1])
      expect(result).to eq(user_json)
    end
  end

  describe 'Zstd::DDict dictionary streaming decompress + GC.compact' do
    let(:dictionary) do
      File.read("#{__dir__}/dictionary")
    end
    let(:ddict) do
      Zstd::DDict.new(dictionary)
    end
    let(:user_json) do
      File.read("#{__dir__}/user_springmt.json")
    end
    it 'shoud work' do
      compressed_json = Zstd.compress(user_json, dict: dictionary)
      stream = Zstd::StreamingDecompress.new(dict: ddict)
      result = ''
      result << stream.decompress(compressed_json[0, 5])
      result << stream.decompress(compressed_json[5, 5])
      GC.compact
      result << stream.decompress(compressed_json[10..-1])
      expect(result).to eq(user_json)
    end
  end

  describe 'nil dictionary streaming decompress + GC.compact' do
    let(:dictionary) do
      File.read("#{__dir__}/dictionary")
    end
    let(:user_json) do
      File.read("#{__dir__}/user_springmt.json")
    end
    it 'shoud work' do
      compressed_json = Zstd.compress(user_json)
      stream = Zstd::StreamingDecompress.new(dict: nil)
      result = ''
      result << stream.decompress(compressed_json[0, 5])
      result << stream.decompress(compressed_json[5, 5])
      GC.compact
      result << stream.decompress(compressed_json[10..-1])
      expect(result).to eq(user_json)
    end
  end

  if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0.0')
    describe 'Ractor' do
      it 'should be supported' do
        r = Ractor.new {
          cstr = Zstd.compress('foo bar buzz')
          stream = Zstd::StreamingDecompress.new
          result = ''
          result << stream.decompress(cstr[0, 5])
          result << stream.decompress(cstr[5, 5])
          result << stream.decompress(cstr[10..-1])
          result
        }
        expect(r.take).to eq('foo bar buzz')
      end
    end
  end
end