Fix large reads

Author: bill-rich

disk_buffer_reader.go

@@ -1,3 +1,6 @@
+// Maybe I need to set the read size? n, err := io.Reader.Read(something); n will not be more that 512 unless I set something (ran into this in archive)
+// Only happens when Stop() is run. Why??
+// Make sure read time is similar between standard reader and dbr.
 package diskbufferreader
 
 import (
@@ -68,20 +71,18 @@ func (dbr *DiskBufferReader) Read(out []byte) (int, error) {
 		// Update the number of bytes read.
 		dbr.bytesRead += int64(m)
 
-		// Go back to the beginning of the tmp file so reads start from the beginning.
-		dbr.tmpFile.Seek(dbr.index, io.SeekStart)
 	}
 
 	// Read from the multireader of the tmp file and the reader.
+	if dbr.index <= dbr.bytesRead {
+		dbr.tmpFile.Seek(dbr.index, io.SeekStart)
+	}
 	mr := io.MultiReader(dbr.tmpFile, dbr.reader)
 	bytesRead := 0
 	outBuffer := bytes.NewBuffer([]byte{})
-	outMulti := make([]byte, len(out))
 	var outErr error
-	if dbr.index <= dbr.bytesRead {
-		dbr.tmpFile.Seek(dbr.index, io.SeekStart)
-	}
 	for {
+		outMulti := make([]byte, len(out)-bytesRead)
 		n, err := mr.Read(outMulti)
 		if err != nil {
 			if !errors.Is(err, io.EOF) {
@@ -102,12 +103,8 @@ func (dbr *DiskBufferReader) Read(out []byte) (int, error) {
 			break
 		}
 	}
-	outStart := dbr.index
-	if int64(bytesRead) < dbr.index {
-		outStart = int64(bytesRead)
-	}
-	copy(out, outBuffer.Bytes()[outStart:])
-	dbr.index = int64(bytesRead)
+	copy(out, outBuffer.Bytes())
+	dbr.index += int64(bytesRead)
 	return bytesRead, outErr
 }
 

disk_buffer_reader_test.go

@@ -3,6 +3,8 @@ package diskbufferreader
 import (
 	"bytes"
 	"errors"
+	"io"
+	"net/http"
 	"testing"
 )
 
@@ -74,3 +76,107 @@ func TestDiskBufferReader(t *testing.T) {
 		}
 	}
 }
+
+func TestReadAll(t *testing.T) {
+	tests := map[string]struct {
+		content string
+		record  bool
+		reset   bool
+	}{
+		"RecordOnNoReset": {
+			"OneTwoThree",
+			true,
+			false,
+		},
+		"RecordOffNoReset": {
+			"OneTwoThree",
+			false,
+			false,
+		},
+		"RecordOnReset": {
+			"OneTwoThree",
+			true,
+			true,
+		},
+		"RecordOffReset": {
+			"OneTwoThree",
+			false,
+			true,
+		},
+	}
+
+	for testName, testCase := range tests {
+
+		readBytes := []byte(testCase.content)
+		bytesReader := bytes.NewBuffer(readBytes)
+		tmpReader := bytes.NewBuffer(readBytes)
+		dbr, err := New(tmpReader)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer dbr.Close()
+
+		if testCase.reset {
+			chunk := make([]byte, 3)
+			dbr.Read(chunk)
+			dbr.Reset()
+		}
+
+		if !testCase.record {
+			dbr.Stop()
+		}
+
+		baseBytes, baseErr := io.ReadAll(bytesReader)
+		testBytes, testErr := io.ReadAll(dbr)
+
+		if string(testBytes) != string(baseBytes) {
+			t.Fatalf("%s: Unexpected read result. Got: %v, expected: %v", testName, testBytes, baseBytes)
+		}
+
+		if !errors.Is(testErr, baseErr) {
+			t.Fatalf("%s: Unexpected error. Got: %s, expected: %s", testName, testErr, baseErr)
+		}
+	}
+}
+
+func TestReadAllLarge(t *testing.T) {
+	resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	readBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bytesReader := bytes.NewBuffer(readBytes)
+	tmpReader := bytes.NewBuffer(readBytes)
+	dbr, err := New(tmpReader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dbr.Close()
+
+	chunk := make([]byte, 3)
+	dbr.Read(chunk)
+	dbr.Reset()
+
+	dbr.Stop()
+
+	baseBytes, baseErr := io.ReadAll(bytesReader)
+	testBytes, testErr := io.ReadAll(dbr)
+
+	if len(testBytes) != len(baseBytes) {
+		t.Fatalf("Wrong number of bytes read. Got: %d, expected: %d", len(testBytes), len(baseBytes))
+	}
+
+	if string(testBytes) != string(baseBytes) {
+		t.Fatalf("Unexpected read result. Got: %v..%v, expected: %v..%v", testBytes[:1024], testBytes[len(testBytes)-16:], baseBytes[:1024], baseBytes[len(baseBytes)-16:])
+	}
+
+	if !errors.Is(testErr, baseErr) {
+		t.Fatalf("Unexpected error. Got: %s, expected: %s", testErr, baseErr)
+	}
+}