func TestMultiHasher(t *testing.T) {
for _, test := range hashTestSet {
mh := fs.NewMultiHasher()
n, err := io.Copy(mh, bytes.NewBuffer(test.input))
if err != nil {
t.Fatal(err)
}
if int(n) != len(test.input) {
t.Fatalf("copy mismatch: %d != %d", n, len(test.input))
}
sums := mh.Sums()
for k, v := range sums {
expect, ok := test.output[k]
if !ok {
t.Errorf("Unknown hash type %v, sum: %q", k, v)
}
if expect != v {
t.Errorf("hash %v mismatch %q != %q", k, v, expect)
}
}
// Test that all are present
for k, v := range test.output {
expect, ok := sums[k]
if !ok {
t.Errorf("did not calculate hash type %v, sum: %q", k, v)
}
if expect != v {
t.Errorf("hash %d mismatch %q != %q", k, v, expect)
}
}
}
}
func testPut(t *testing.T, file *fstest.Item) {
again:
buf := bytes.NewBufferString(fstest.RandomString(100))
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
tries := 1
const maxTries = 10
file.Size = int64(buf.Len())
obji := fs.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
obj, err := remote.Put(in, obji)
if err != nil {
// Retry if err returned a retry error
if fs.IsRetryError(err) && tries < maxTries {
t.Logf("Put error: %v - low level retry %d/%d", err, tries, maxTries)
time.Sleep(2 * time.Second)
tries++
goto again
}
require.NoError(t, err, "Put error")
}
file.Hashes = hash.Sums()
file.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file.Path)
file.Check(t, obj, remote.Precision())
}
// Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
err := o.mkdirAll()
if err != nil {
return err
}
out, err := os.Create(o.path)
if err != nil {
return err
}
// Calculate the md5sum of the object we are reading as we go along
hash := fs.NewMultiHasher()
in = io.TeeReader(in, hash)
_, err = io.Copy(out, in)
outErr := out.Close()
if err != nil {
return err
}
if outErr != nil {
return outErr
}
// All successful so update the hashes
o.hashes = hash.Sums()
// Set the mtime
o.SetModTime(modTime)
// ReRead info now that we have finished
return o.lstat()
}
// Open an object for read
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
var offset int64
for _, option := range options {
switch x := option.(type) {
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Log(o, "Unsupported mandatory option: %v", option)
}
}
}
fd, err := os.Open(o.path)
if err != nil {
return
}
if offset != 0 {
// seek the object
_, err = fd.Seek(offset, 0)
// don't attempt to make checksums
return fd, err
}
// Update the md5sum as we go along
in = &localOpenFile{
o: o,
in: fd,
hash: fs.NewMultiHasher(),
}
return in, nil
}
// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
in, err := obj.Open()
if err != nil {
t.Fatalf("Open() return error: %v", err)
}
hasher := fs.NewMultiHasher()
n, err := io.Copy(hasher, in)
if err != nil {
t.Fatalf("io.Copy() return error: %v", err)
}
if n != file1.Size {
t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
}
err = in.Close()
if err != nil {
t.Fatalf("in.Close() return error: %v", err)
}
// Check content of file by comparing the calculated hashes
for hashType, got := range hasher.Sums() {
want := file1.Hashes[hashType]
if want != got {
t.Errorf("%v is wrong %v != %v", hashType, want, got)
}
}
}
// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t)
contents := fstest.RandomString(200)
buf := bytes.NewBufferString(contents)
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path)
obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
err := obj.Update(in, obji)
require.NoError(t, err)
file1.Hashes = hash.Sums()
// check the object has been updated
file1.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision())
// check contents correct
assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ")
file1Contents = contents
}
// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
in, err = os.Open(o.path)
if err != nil {
return
}
// Update the md5sum as we go along
in = &localOpenFile{
o: o,
in: in,
hash: fs.NewMultiHasher(),
}
return
}
// NewItem creates an item from a string content
func NewItem(Path, Content string, modTime time.Time) Item {
i := Item{
Path: Path,
ModTime: modTime,
Size: int64(len(Content)),
}
hash := fs.NewMultiHasher()
buf := bytes.NewBufferString(Content)
_, err := io.Copy(hash, buf)
if err != nil {
log.Fatalf("Failed to create item: %v", err)
}
i.Hashes = hash.Sums()
return i
}
func testPut(t *testing.T, file *fstest.Item) {
buf := bytes.NewBufferString(fstest.RandomString(100))
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
file.Size = int64(buf.Len())
obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
if err != nil {
t.Fatal("Put error", err)
}
file.Hashes = hash.Sums()
file.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file.Path)
file.Check(t, obj, remote.Precision())
}
// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t)
buf := bytes.NewBufferString(fstest.RandomString(200))
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path)
obji := fs.NewStaticObjectInfo("", file1.ModTime, file1.Size, true, nil, obj.Fs())
err := obj.Update(in, obji)
require.NoError(t, err)
file1.Hashes = hash.Sums()
file1.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision())
}
// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) {
skipIfNotOk(t)
obj := findObject(t, file1.Path)
in, err := obj.Open()
require.NoError(t, err)
hasher := fs.NewMultiHasher()
n, err := io.Copy(hasher, in)
require.NoError(t, err)
require.Equal(t, file1.Size, n, "Read wrong number of bytes")
err = in.Close()
require.NoError(t, err)
// Check content of file by comparing the calculated hashes
for hashType, got := range hasher.Sums() {
assert.Equal(t, file1.Hashes[hashType], got)
}
}
// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
skipIfNotOk(t)
buf := bytes.NewBufferString(fstest.RandomString(200))
hash := fs.NewMultiHasher()
in := io.TeeReader(buf, hash)
file1.Size = int64(buf.Len())
obj := findObject(t, file1.Path)
err := obj.Update(in, file1.ModTime, file1.Size)
if err != nil {
t.Fatal("Update error", err)
}
file1.Hashes = hash.Sums()
file1.Check(t, obj, remote.Precision())
// Re-read the object and check again
obj = findObject(t, file1.Path)
file1.Check(t, obj, remote.Precision())
}
func TestMultiHasher(t *testing.T) {
for _, test := range hashTestSet {
mh := fs.NewMultiHasher()
n, err := io.Copy(mh, bytes.NewBuffer(test.input))
require.NoError(t, err)
assert.Len(t, test.input, int(n))
sums := mh.Sums()
for k, v := range sums {
expect, ok := test.output[k]
require.True(t, ok)
assert.Equal(t, v, expect)
}
// Test that all are present
for k, v := range test.output {
expect, ok := sums[k]
require.True(t, ok)
assert.Equal(t, v, expect)
}
}
}
// Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
err := o.mkdirAll()
if err != nil {
return err
}
out, err := os.Create(o.path)
if err != nil {
return err
}
// Calculate the hash of the object we are reading as we go along
hash := fs.NewMultiHasher()
in = io.TeeReader(in, hash)
_, err = io.Copy(out, in)
closeErr := out.Close()
if err == nil {
err = closeErr
}
if err != nil {
fs.Debug(o, "Removing partially written file on error: %v", err)
if removeErr := os.Remove(o.path); removeErr != nil {
fs.ErrorLog(o, "Failed to remove partially written file: %v", removeErr)
}
return err
}
// All successful so update the hashes
o.hashes = hash.Sums()
// Set the mtime
err = o.SetModTime(src.ModTime())
if err != nil {
return err
}
// ReRead info now that we have finished
return o.lstat()
}