Skip to content

Commit 12234f8

Browse files
fferrandisfferrandis
committed
HD-4220: adapt get range
1 parent 98a5347 commit 12234f8

2 files changed

Lines changed: 150 additions & 41 deletions

File tree

‎backend.go‎

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,6 @@ func (backend *Backend) ChunkInfo(fragRangeLen int, pieceSize int) ChunkInfo {
372372
if nrChunks*chunkSize != fragRangeLen {
373373
nrChunks++
374374
}
375-
376375
return ChunkInfo{
377376
ChunkSize: chunkSize,
378377
NrChunk: nrChunks,
@@ -392,7 +391,7 @@ func (backend *Backend) LinearizeMatrix(frags []ValidatedFragment, pieceSize int
392391
/* Fragments are sorted beforehand with the index of the first chunk.
393392
All chunks of a fragments share the same index. */
394393
fragsIndex := make([]int, len(frags))
395-
for i := 0; i < len(frags); i++ {
394+
for i := range frags {
396395
fragsIndex[i] = i
397396
}
398397

@@ -591,41 +590,65 @@ type RangeMatrix struct {
591590
* p4 [-[*]- -]
592591
*
593592
*/
594-
func (backend *Backend) GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize int) *RangeMatrix {
595-
chunkSize := pieceSize + backend.headerSize
596-
groupSize := pieceSize * backend.K
593+
func (backend *Backend) GetRangeMatrix(startIncl, endIncl, cellDataSize, fragSize int) *RangeMatrix {
594+
nrColumns := backend.K
595+
cellSize := cellDataSize + backend.headerSize
596+
lineSize := cellDataSize * nrColumns
597597

598598
/* At this point we don't know what is the true payload size, but we
599599
can at least check that it doesn't exceed the maximum payload that
600600
this configuration can handle. */
601-
nrChunkByFrag := fragSize / chunkSize
602-
dataLenPerFrag := fragSize - nrChunkByFrag*backend.headerSize
603-
maxDataLen := dataLenPerFrag * backend.K
604-
if startIncl >= maxDataLen || endIncl >= maxDataLen || startIncl > endIncl {
601+
nrLines := fragSize / cellSize
602+
603+
/* Inside a fragment (ie, inside a column), what is the
604+
stored amount of data? */
605+
fragDataSize := fragSize - nrLines*backend.headerSize
606+
607+
maxDataSize := fragDataSize * nrColumns
608+
609+
if startIncl >= maxDataSize || endIncl >= maxDataSize || startIncl > endIncl {
605610
return nil
606611
}
607612

608-
pieceStartIncl := startIncl / pieceSize
609-
pieceEndIncl := endIncl / pieceSize
613+
/* convert cells indices to (x,y) indices */
614+
615+
/* Considering our (x,y) matrix as a single row, what
616+
are the indices of the start and end of the range we are interested in? */
617+
idxStart := startIncl / cellDataSize
618+
idxEnd := endIncl / cellDataSize
619+
620+
/* as we have the indices of the first cell and the last cell,
621+
* we can derive the indices of the first line and the last line
622+
* Based on this first line and last line, we can deduce
623+
* the amount of data to read in each fragment.
624+
*/
625+
lineStart := idxStart / nrColumns
626+
lineEnd := idxEnd / nrColumns
627+
628+
/*
629+
* as we have the indices of the first cell and the last cell,
630+
* we can compute the first column (e.g the first fragment)
631+
* where to start the read
632+
*/
633+
columnStart := idxStart % nrColumns
610634

611-
groupStartIncl := pieceStartIncl / backend.K
612-
groupEndIncl := pieceEndIncl / backend.K
635+
nrCellsToRead := (idxEnd + 1 - idxStart)
636+
dataOffset := idxStart * cellDataSize
613637

614-
fragFirstIncl := pieceStartIncl % backend.K
615-
fragCount := (pieceEndIncl + 1 - pieceStartIncl)
616-
dataOffset := pieceStartIncl * pieceSize
638+
totalLines := (maxDataSize + lineSize - 1) / lineSize
639+
isLastStripe := (lineEnd == totalLines-1)
617640

618641
/* When wrapping around, we read the full groups. */
619-
if fragFirstIncl+fragCount > backend.K {
620-
fragFirstIncl = 0
621-
fragCount = backend.K
622-
dataOffset = groupStartIncl * groupSize
642+
if columnStart+nrCellsToRead > nrColumns || isLastStripe {
643+
columnStart = 0
644+
nrCellsToRead = nrColumns
645+
dataOffset = lineStart * lineSize
623646
}
624647

625648
/* For each fragment, this is the minimum range to read -- including
626649
the header -- to decode or repair the data. */
627-
inFragRangeStartIncl := groupStartIncl * chunkSize
628-
inFragRangeEndExcl := (groupEndIncl + 1) * chunkSize
650+
inFragRangeStartIncl := lineStart * cellSize
651+
inFragRangeEndExcl := (lineEnd + 1) * cellSize
629652

630653
/* The output buffer only contains the data necessary to read the range,
631654
and the requested range must be adjusted to be relative
@@ -637,13 +660,13 @@ func (backend *Backend) GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize i
637660
linearizedRangeStartIncl := startIncl - dataOffset
638661

639662
/* Decoding always works on a group boundary. */
640-
decodedRangeStartIncl := startIncl - groupStartIncl*groupSize
663+
decodedRangeStartIncl := startIncl - lineStart*lineSize
641664

642665
return &RangeMatrix{
643666
ReqStartIncl: startIncl,
644667
ReqEndIncl: endIncl,
645-
FragFirstIncl: fragFirstIncl,
646-
FragCount: fragCount,
668+
FragFirstIncl: columnStart,
669+
FragCount: nrCellsToRead,
647670
InFragRangeStartIncl: inFragRangeStartIncl,
648671
InFragRangeEndExcl: inFragRangeEndExcl,
649672
DecodedRangeStartIncl: decodedRangeStartIncl,

‎backend_test.go‎

Lines changed: 102 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -201,14 +201,13 @@ func TestEncodeDecode(t *testing.T) {
201201
}
202202
}
203203

204-
decode := func(frags [][]byte, description string) bool {
204+
decode := func(frags [][]byte, description string) {
205205
decoded, err := backend.DecodeMatrix(frags, DefaultChunkSize)
206206
require.NoError(t, err)
207207
defer decoded.Free()
208208
require.True(t, bytes.Equal(decoded.Data, pattern),
209209
"%v:%d(%v) pattern: %v, got: %q",
210210
description, patternIndex, backend, pattern, decoded.Data)
211-
return true
212211
}
213212

214213
decode(frags, "all frags")
@@ -253,12 +252,11 @@ func TestReconstruct(t *testing.T) {
253252
defer data.Free()
254253
frags := data.Data
255254

256-
reconstruct := func(recon_frags [][]byte, frag_index int, description string) bool {
255+
reconstruct := func(recon_frags [][]byte, frag_index int, description string) {
257256
data, err := backend.ReconstructMatrix(recon_frags, frag_index, DefaultChunkSize)
258257
require.NoError(t, err, "%v: %v: %q for pattern %d", description, backend, err, patternIndex)
259258
defer data.Free()
260259
require.True(t, bytes.Equal(data.Data, frags[frag_index]), "%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, data.Data)
261-
return true
262260
}
263261
reconstruct(shuf(frags[:params.K]), params.K+params.M-1, "last frag from data frags")
264262
reconstruct(shuf(frags[params.M:]), 0, "first frag with parity frags")
@@ -802,17 +800,107 @@ func TestEncodeM(t *testing.T) {
802800
_ = backend.Close()
803801
}
804802

805-
func TestLinearizeMatrix(t *testing.T) {
806-
assert := assert.New(t)
803+
func TestTwoLinearizeMatrix(t *testing.T) {
804+
backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: 2, M: 1, W: 8, HD: 5})
805+
require.NoError(t, err)
806+
defer func() {
807+
_ = backend.Close()
808+
}()
809+
currentChunkSize := 512
810+
dataSize := currentChunkSize*2 + 10
811+
startIncl := dataSize - 3
812+
endIncl := dataSize - 1
813+
814+
data := make([]byte, dataSize)
815+
for i := range dataSize {
816+
data[i] = byte('A' + i%26)
817+
}
818+
bm := NewBufferMatrix(currentChunkSize, len(data), backend.K)
819+
_, err = io.Copy(bm, bytes.NewReader(data))
820+
require.NoError(t, err)
821+
bm.Finish()
822+
encoded, err := backend.EncodeMatrixWithBufferMatrix(bm, currentChunkSize)
823+
require.NoError(t, err)
824+
defer encoded.Free()
807825

826+
rangeM := backend.GetRangeMatrix(startIncl, endIncl, currentChunkSize, len(encoded.Data[0]))
827+
require.NotNil(t, rangeM)
828+
829+
/* Decode the matrix as if it was requested and
830+
checks that the result matches the payload on the requested range. */
831+
frags := make([][]byte, 0)
832+
for i := 0; i < rangeM.FragCount; i++ {
833+
fragIdx := (rangeM.FragFirstIncl + i) % backend.K
834+
buffer := encoded.Data[fragIdx][rangeM.InFragRangeStartIncl:rangeM.InFragRangeEndExcl]
835+
frags = append(frags, buffer)
836+
}
837+
838+
decoded, err := backend.LinearizeMatrix(frags, currentChunkSize)
839+
require.NoError(t, err)
840+
defer decoded.Free()
841+
842+
expected := data[startIncl:endIncl]
843+
844+
linearizedRangeEndExcl := rangeM.LinearizedRangeStartIncl + (endIncl - startIncl)
845+
found := decoded.Data[rangeM.LinearizedRangeStartIncl:linearizedRangeEndExcl]
846+
847+
require.True(t, bytes.Equal(expected, found))
848+
}
849+
850+
func TestOneLinearizeMatrix(t *testing.T) {
851+
backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: 4, M: 2, W: 8, HD: 5})
852+
require.NoError(t, err)
853+
defer func() {
854+
_ = backend.Close()
855+
}()
856+
dataSize := 105623
857+
startIncl := 59441
858+
endIncl := 64149
859+
data := make([]byte, dataSize)
860+
for i := range dataSize {
861+
data[i] = byte('A' + i%26)
862+
}
863+
bm := NewBufferMatrix(DefaultChunkSize, len(data), backend.K)
864+
_, err = io.Copy(bm, bytes.NewReader(data))
865+
require.NoError(t, err)
866+
bm.Finish()
867+
encoded, err := backend.EncodeMatrixWithBufferMatrix(bm, DefaultChunkSize)
868+
require.NoError(t, err)
869+
defer encoded.Free()
870+
871+
rangeM := backend.GetRangeMatrix(startIncl, endIncl, DefaultChunkSize, len(encoded.Data[0]))
872+
require.NotNil(t, rangeM)
873+
874+
/* Decode the matrix as if it was requested and
875+
checks that the result matches the payload on the requested range. */
876+
frags := make([][]byte, 0)
877+
for i := 0; i < rangeM.FragCount; i++ {
878+
fragIdx := (rangeM.FragFirstIncl + i) % backend.K
879+
buffer := encoded.Data[fragIdx][rangeM.InFragRangeStartIncl:rangeM.InFragRangeEndExcl]
880+
frags = append(frags, buffer)
881+
}
882+
883+
decoded, err := backend.LinearizeMatrix(frags, DefaultChunkSize)
884+
require.NoError(t, err)
885+
defer decoded.Free()
886+
887+
expected := data[startIncl : endIncl+1]
888+
889+
linearizedRangeEndExcl := rangeM.LinearizedRangeStartIncl + (endIncl - startIncl) + 1
890+
found := decoded.Data[rangeM.LinearizedRangeStartIncl:linearizedRangeEndExcl]
891+
require.True(t, bytes.Equal(expected, found))
892+
}
893+
894+
func TestLinearizeMatrix(t *testing.T) {
808895
pieceSize := DefaultChunkSize
809896
k := 4
810897
m := 1
811898

812899
backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: k, M: m, W: 8, HD: m})
813-
if err != nil {
814-
t.Fatalf("cannot init backend: (%v)", err)
815-
}
900+
require.NoError(t, err)
901+
defer func() {
902+
_ = backend.Close()
903+
}()
816904

817905
rangeValues := func(values []reflect.Value, rng *rand.Rand) {
818906
dataSize := 1 + rng.Intn(7*1024*1024)
@@ -848,7 +936,7 @@ func TestLinearizeMatrix(t *testing.T) {
848936

849937
fragSize := len(encoded.Data[0])
850938
rangeM := backend.GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize)
851-
assert.NotNil(rangeM)
939+
require.NotNil(t, rangeM)
852940

853941
/* Decode the matrix as if it was requested and
854942
checks that the result matches the payload on the requested range. */
@@ -860,7 +948,7 @@ func TestLinearizeMatrix(t *testing.T) {
860948
}
861949

862950
decoded, err := backend.LinearizeMatrix(frags, pieceSize)
863-
assert.Nil(err)
951+
require.NoError(t, err)
864952
defer decoded.Free()
865953

866954
expected := data[startIncl : endIncl+1]
@@ -874,9 +962,8 @@ func TestLinearizeMatrix(t *testing.T) {
874962
Values: rangeValues,
875963
}
876964

877-
if err := quick.Check(checkRange, &config); err != nil {
878-
t.Error(err)
879-
}
965+
require.NoError(t, quick.Check(checkRange, &config))
966+
880967
}
881968

882969
func TestDecodeMatrix(t *testing.T) {
@@ -1109,12 +1196,11 @@ func TestEncodeDecodeMatrix(t *testing.T) {
11091196
defer data.Free()
11101197

11111198
frags := data.Data
1112-
decode := func(frags [][]byte, description string) bool {
1199+
decode := func(frags [][]byte, description string) {
11131200
decoded, err := backend.DecodeMatrix(frags, DefaultChunkSize)
11141201
require.NoError(t, err)
11151202
require.True(t, bytes.Equal(decoded.Data, pattern), "%v: Expected %v to roundtrip pattern %d, got %q", description, backend, patternIndex, decoded.Data)
11161203
defer decoded.Free()
1117-
return true
11181204
}
11191205

11201206
decode(frags, "all frags")

0 commit comments

Comments
 (0)