HD-4220: pack slice to their real size

Author: fferrandis

backend.go

@@ -213,6 +213,15 @@ type EncodeData struct {
 	Free func()   // cleanup closure (to free C allocated data once it becomes useless)
 }
 
+func (e *EncodeData) String() string {
+	str := "EncodeData{Data: ["
+	for i, data := range e.Data {
+		str += fmt.Sprintf("len(data[%d]) = %+v, ", i, len(data))
+	}
+	str += "]}"
+	return str
+}
+
 // Encode is the general purpose encoding function. It encodes data according
 // backend params and returns an EncodeData structure containing the Fragments
 func (backend *Backend) Encode(data []byte) (*EncodeData, error) {
@@ -262,6 +271,7 @@ func (backend *Backend) EncodeMatrixWithBufferMatrix(bm *BufferMatrix, chunkSize
 	C.encode_chunk_buffermatrix_prepare(backend.libecDesc, pData, pDataLen,
 		cChunkSize, C.int(bm.FragLen()), nbFrags, &ctx)
 
+	var totlen int64 = 0
 	var errCounter uint32
 
 	wg.Add(int(ctx.number_of_subgroup))
@@ -273,6 +283,8 @@ func (backend *Backend) EncodeMatrixWithBufferMatrix(bm *BufferMatrix, chunkSize
 			if i == int(ctx.number_of_subgroup)-1 {
 				fragLen = C.size_t(bm.FragLenLastSubGroup())
 			}
+			// totlen should be atomicall updated
+			atomic.AddInt64(&totlen, int64(fragLen)+int64(backend.headerSize))
 			r := C.encode_chunk_buffermatrix(backend.libecDesc, pData, pDataLen,
 				nbFrags, &ctx, C.int(nth), fragLen)
 
@@ -294,13 +306,14 @@ func (backend *Backend) EncodeMatrixWithBufferMatrix(bm *BufferMatrix, chunkSize
 	result := make([][]byte, backend.K+backend.M)
 	fragLen := ctx.frags_len
 	flen := bm.FragLen()
+
 	for i := 0; i < backend.K; i++ {
-		result[i] = data[i*flen : (i+1)*flen]
+		result[i] = data[i*flen : (i)*flen+int(totlen)]
 	}
 
 	for i := 0; i < backend.M; i++ {
 		str := cGetArrayItem(ctx.codings, i)
-		result[i+backend.K] = (*[1 << 30]byte)(str)[:int(C.int(fragLen)):int(C.int(fragLen))]
+		result[i+backend.K] = (*[1 << 30]byte)(str)[:int(C.int(totlen)):int(C.int(fragLen))]
 	}
 
 	return &EncodeData{result, func() {
@@ -599,6 +612,9 @@ func (backend *Backend) GetRangeMatrix(startIncl, endIncl, cellDataSize, fragSiz
 	   can at least check that it doesn't exceed the maximum payload that
 	   this configuration can handle. */
 	nrLines := fragSize / cellSize
+	if nrLines*cellSize < fragSize {
+		nrLines++
+	}
 
 	/* Inside a fragment (ie, inside a column), what is the
 	   stored amount of data? */
@@ -763,7 +779,8 @@ func (backend *Backend) ReconstructMatrix(frags [][]byte, fragIndex int, pieceSi
 	dataB, data := backend.pool.New(dlen)
 
 	var errCounter uint32
-	// TODO use goroutines here to leverage multicore computation
+
+	totLenPerFrag := int64(0)
 	wg.Add(chunkNr)
 	for i := 0; i < chunkNr; i++ {
 		go func(chunkIdx int) {
@@ -774,14 +791,17 @@ func (backend *Backend) ReconstructMatrix(frags [][]byte, fragIndex int, pieceSi
 			if err := backend.reconstruct(vect, fragIndex, data[chunkIdx*chunkSize:]); err != nil {
 				atomic.AddUint32(&errCounter, 1)
 			}
+			info := GetFragmentInfo(vect[0])
+			// atomicly add to totLenPerFrag
+			atomic.AddInt64(&totLenPerFrag, int64(info.Size)+int64(backend.headerSize))
 			wg.Done()
 		}(i)
 	}
 	wg.Wait()
 	if errCounter != 0 {
 		return nil, errors.New("sub reconstruction failed")
 	}
-	return &DecodeData{data[:dlen:dlen], func() {
+	return &DecodeData{data[:totLenPerFrag:dlen], func() {
 		backend.pool.Release(dataB)
 	}}, nil
 }

backend_test.go

@@ -488,7 +488,7 @@ func TestGC(t *testing.T) {
 		},
 	}
 
-	nbRoutines := 500
+	nbRoutines := 1
 
 	for _, test := range tests {
 		t.Run(test.name, func(_ *testing.T) {
@@ -815,16 +815,25 @@ func TestLinearizeMatrixAndReconstruct(t *testing.T) {
 		useOldFormat bool
 	}{
 		{
-			chunkSize: 512,
-			dataSize:  512*2 + 10,
-			startIncl: 512*2 - 3,
-			endIncl:   512*2 - 1,
+			chunkSize:    DefaultChunkSize,
+			dataSize:     DefaultChunkSize*2 + 10,
+			startIncl:    3,
+			endIncl:      9,
+			useOldFormat: false,
 		},
 		{
-			chunkSize: DefaultChunkSize,
-			dataSize:  105623,
-			startIncl: 59441,
-			endIncl:   64149,
+			chunkSize:    DefaultChunkSize,
+			dataSize:     DefaultChunkSize*2 + 10,
+			startIncl:    3,
+			endIncl:      9,
+			useOldFormat: true,
+		},
+		{
+			chunkSize:    512,
+			dataSize:     512*2 + 10,
+			startIncl:    512*2 - 3,
+			endIncl:      512*2 - 1,
+			useOldFormat: false,
 		},
 		{
 			chunkSize:    512,
@@ -833,13 +842,34 @@ func TestLinearizeMatrixAndReconstruct(t *testing.T) {
 			endIncl:      512*2 - 1,
 			useOldFormat: true,
 		},
+		{
+			chunkSize:    DefaultChunkSize,
+			dataSize:     105623,
+			startIncl:    59441,
+			endIncl:      64149,
+			useOldFormat: false,
+		},
 		{
 			chunkSize:    DefaultChunkSize,
 			dataSize:     105623,
 			startIncl:    59441,
 			endIncl:      64149,
 			useOldFormat: true,
 		},
+		{
+			chunkSize:    DefaultChunkSize,
+			dataSize:     105623,
+			startIncl:    105610,
+			endIncl:      105622,
+			useOldFormat: true,
+		},
+		{
+			chunkSize:    DefaultChunkSize,
+			dataSize:     105623,
+			startIncl:    105610,
+			endIncl:      105622,
+			useOldFormat: false,
+		},
 	}
 
 	for _, param := range testParams {
@@ -854,9 +884,7 @@ func TestLinearizeMatrixAndReconstruct(t *testing.T) {
 			endIncl := p.endIncl
 
 			data := make([]byte, dataSize)
-			for i := range dataSize {
-				data[i] = byte('A' + i%26)
-			}
+			_, _ = cryptorand.Read(data)
 			bm := NewBufferMatrix(currentChunkSize, len(data), backend.K)
 			if p.useOldFormat {
 				bm.UseOldFormat()
@@ -867,9 +895,11 @@ func TestLinearizeMatrixAndReconstruct(t *testing.T) {
 			encoded, err := backend.EncodeMatrixWithBufferMatrix(bm, currentChunkSize)
 			require.NoError(t, err)
 			defer encoded.Free()
+			fmt.Printf("%+v\n", encoded)
 
 			rangeM := backend.GetRangeMatrix(startIncl, endIncl, currentChunkSize, len(encoded.Data[0]))
 			require.NotNil(t, rangeM)
+			fmt.Printf("%+v\n", rangeM)
 
 			/* Decode the matrix as if it was requested and
 			   checks that the result matches the payload on the requested range. */
@@ -901,8 +931,12 @@ func TestLinearizeMatrixAndReconstruct(t *testing.T) {
 
 			require.NoError(t, err)
 			defer reconstructed.Free()
-
-			require.True(t, bytes.Equal(frags2[0], reconstructed.Data))
+			expected2 := frags2[0]
+			if !p.useOldFormat {
+				lenToCompare := len(reconstructed.Data)
+				expected2 = expected2[:lenToCompare]
+			}
+			require.True(t, bytes.Equal(expected2, reconstructed.Data))
 		})
 	}
 }

buffer.go

@@ -2,6 +2,7 @@ package erasurecode
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 )
 
@@ -15,6 +16,11 @@ type BufferInfo struct {
 	newStyle           bool
 }
 
+// Implement String interface
+func (b BufferInfo) String() string {
+	return fmt.Sprintf("BufferInfo{hdrSize: %d, bufSize: %d, len: %d, k: %d, curBlock: %d, leftInBlock: %d, sizeOfLastSubGroup: %d, newStyle: %t}", b.hdrSize, b.bufSize, b.len, b.k, b.curBlock, b.leftInBlock, b.sizeOfLastSubGroup, b.newStyle)
+}
+
 // FragLen returns the size of a "fragment" aligned to a block size (data + header)
 func (b BufferInfo) FragLen() int {
 	return b.SubGroups() * (b.bufSize + b.hdrSize)
@@ -82,6 +88,10 @@ type BufferMatrix struct {
 	BufferInfo
 }
 
+func (b *BufferMatrix) String() string {
+	return fmt.Sprintf("BufferMatrix{len(b): %+v,finished: %+v, BufferInfo: %+v}", len(b.b), b.finished, b.BufferInfo)
+}
+
 // NewBufferMatrix returns a new buffer suitable for <len> data and organized
 // such as it can be injected into EncodeMatrixWithBuffer without allocation/copying
 // the data into shards

buffer_test.go

@@ -98,10 +98,6 @@ func TestEncodeBufferMatrix(t *testing.T) {
 
 			frags := encoded.Data
 
-			for i := 0; i < k+m; i++ {
-				assert.Equal(t, b.FragLen(), len(frags[i]))
-			}
-
 			decoded, err := backend.DecodeMatrix(frags, blockSize)
 			assert.NoError(t, err)
 			if err != nil {