// downloadChunk downloads the chunk froom s3 func (d *downloader) downloadChunk(chunk dlchunk) { if d.getErr() != nil { return } // Get the next byte range of data in := &s3.GetObjectInput{} awsutil.Copy(in, d.in) rng := fmt.Sprintf("bytes=%d-%d", chunk.start, chunk.start+chunk.size-1) in.Range = &rng req, resp := d.ctx.S3.GetObjectRequest(in) req.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler("S3Manager")) err := req.Send() if err != nil { d.setErr(err) } else { d.setTotalBytes(resp) // Set total if not yet set. n, err := io.Copy(&chunk, resp.Body) resp.Body.Close() if err != nil { d.setErr(err) } d.incrWritten(n) } }
func TestCopyDifferentStructs(t *testing.T) { type SrcFoo struct { A int B []*string C map[string]*int SrcUnique string SameNameDiffType int unexportedPtr *int ExportedPtr *int } type DstFoo struct { A int B []*string C map[string]*int DstUnique int SameNameDiffType string unexportedPtr *int ExportedPtr *int } // Create the initial value str1 := "hello" str2 := "bye bye" int1 := 1 int2 := 2 f1 := &SrcFoo{ A: 1, B: []*string{&str1, &str2}, C: map[string]*int{ "A": &int1, "B": &int2, }, SrcUnique: "unique", SameNameDiffType: 1, unexportedPtr: &int1, ExportedPtr: &int2, } // Do the copy var f2 DstFoo awsutil.Copy(&f2, f1) // Values are equal assert.Equal(t, f2.A, f1.A) assert.Equal(t, f2.B, f1.B) assert.Equal(t, f2.C, f1.C) assert.Equal(t, "unique", f1.SrcUnique) assert.Equal(t, 1, f1.SameNameDiffType) assert.Equal(t, 0, f2.DstUnique) assert.Equal(t, "", f2.SameNameDiffType) assert.Equal(t, int1, *f1.unexportedPtr) assert.Nil(t, f2.unexportedPtr) assert.Equal(t, int2, *f1.ExportedPtr) assert.Equal(t, int2, *f2.ExportedPtr) }
func TestCopyReader(t *testing.T) { var buf io.Reader = bytes.NewReader([]byte("hello world")) var r io.Reader awsutil.Copy(&r, buf) b, err := ioutil.ReadAll(r) assert.NoError(t, err) assert.Equal(t, []byte("hello world"), b) // empty bytes because this is not a deep copy b, err = ioutil.ReadAll(buf) assert.NoError(t, err) assert.Equal(t, []byte(""), b) }
// singlePart contains upload logic for uploading a single chunk via // a regular PutObject request. Multipart requests require at least two // parts, or at least 5MB of data. func (u *uploader) singlePart(buf io.ReadSeeker) (*UploadOutput, error) { params := &s3.PutObjectInput{} awsutil.Copy(params, u.in) params.Body = buf req, out := u.ctx.S3.PutObjectRequest(params) req.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler("S3Manager")) if err := req.Send(); err != nil { return nil, err } url := req.HTTPRequest.URL.String() return &UploadOutput{ Location: url, VersionID: out.VersionId, }, nil }
func TestCopyNestedWithUnexported(t *testing.T) { type Bar struct { a int B int } type Foo struct { A string B Bar } f1 := &Foo{A: "string", B: Bar{a: 1, B: 2}} var f2 Foo awsutil.Copy(&f2, f1) // Values match assert.Equal(t, f2.A, f1.A) assert.NotEqual(t, f2.B, f1.B) assert.NotEqual(t, f2.B.a, f1.B.a) assert.Equal(t, f2.B.B, f2.B.B) }
func TestCopy(t *testing.T) { type Foo struct { A int B []*string C map[string]*int } // Create the initial value str1 := "hello" str2 := "bye bye" int1 := 1 int2 := 2 f1 := &Foo{ A: 1, B: []*string{&str1, &str2}, C: map[string]*int{ "A": &int1, "B": &int2, }, } // Do the copy var f2 Foo awsutil.Copy(&f2, f1) // Values are equal assert.Equal(t, f2.A, f1.A) assert.Equal(t, f2.B, f1.B) assert.Equal(t, f2.C, f1.C) // But pointers are not! str3 := "nothello" int3 := 57 f2.A = 100 f2.B[0] = &str3 f2.C["B"] = &int3 assert.NotEqual(t, f2.A, f1.A) assert.NotEqual(t, f2.B, f1.B) assert.NotEqual(t, f2.C, f1.C) }
func ExampleCopy() { type Foo struct { A int B []*string } // Create the initial value str1 := "hello" str2 := "bye bye" f1 := &Foo{A: 1, B: []*string{&str1, &str2}} // Do the copy var f2 Foo awsutil.Copy(&f2, f1) // Print the result fmt.Println(awsutil.Prettify(f2)) // Output: // { // A: 1, // B: ["hello","bye bye"] // } }
func TestCopyIgnoreNilMembers(t *testing.T) { type Foo struct { A *string B []string C map[string]string } f := &Foo{} assert.Nil(t, f.A) assert.Nil(t, f.B) assert.Nil(t, f.C) var f2 Foo awsutil.Copy(&f2, f) assert.Nil(t, f2.A) assert.Nil(t, f2.B) assert.Nil(t, f2.C) fcopy := awsutil.CopyOf(f) f3 := fcopy.(*Foo) assert.Nil(t, f3.A) assert.Nil(t, f3.B) assert.Nil(t, f3.C) }
func TestCopyNil(t *testing.T) { var s string awsutil.Copy(&s, nil) assert.Equal(t, "", s) }
func TestCopyPrimitive(t *testing.T) { str := "hello" var s string awsutil.Copy(&s, &str) assert.Equal(t, "hello", s) }
// upload will perform a multipart upload using the firstBuf buffer containing // the first chunk of data. func (u *multiuploader) upload(firstBuf io.ReadSeeker) (*UploadOutput, error) { params := &s3.CreateMultipartUploadInput{} awsutil.Copy(params, u.in) // Create the multipart req, resp := u.ctx.S3.CreateMultipartUploadRequest(params) req.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler("S3Manager")) if err := req.Send(); err != nil { return nil, err } u.uploadID = *resp.UploadId // Create the workers ch := make(chan chunk, u.ctx.Concurrency) for i := 0; i < u.ctx.Concurrency; i++ { u.wg.Add(1) go u.readChunk(ch) } // Send part 1 to the workers var num int64 = 1 ch <- chunk{buf: firstBuf, num: num} // Read and queue the rest of the parts for u.geterr() == nil { // This upload exceeded maximum number of supported parts, error now. if num > int64(u.ctx.MaxUploadParts) || num > int64(MaxUploadParts) { var msg string if num > int64(u.ctx.MaxUploadParts) { msg = fmt.Sprintf("exceeded total allowed configured MaxUploadParts (%d). Adjust PartSize to fit in this limit", u.ctx.MaxUploadParts) } else { msg = fmt.Sprintf("exceeded total allowed S3 limit MaxUploadParts (%d). Adjust PartSize to fit in this limit", MaxUploadParts) } u.seterr(awserr.New("TotalPartsExceeded", msg, nil)) break } num++ buf, err := u.nextReader() if err == io.EOF { break } ch <- chunk{buf: buf, num: num} if err != nil && err != io.ErrUnexpectedEOF { u.seterr(awserr.New( "ReadRequestBody", "read multipart upload data failed", err)) break } } // Close the channel, wait for workers, and complete upload close(ch) u.wg.Wait() complete := u.complete() if err := u.geterr(); err != nil { return nil, &multiUploadError{ awsError: awserr.New( "MultipartUpload", "upload multipart failed", err), uploadID: u.uploadID, } } return &UploadOutput{ Location: *complete.Location, VersionID: complete.VersionId, UploadID: u.uploadID, }, nil }