func (objClient *ObjectClient) addObject(reader io.Reader, length uint64, expectedHash *hash.Hash) (hash.Hash, bool, error) { var request objectserver.AddObjectRequest var reply objectserver.AddObjectResponse if length < 1 { return reply.Hash, false, errors.New( "zero length object cannot be added") } srpcClient, err := objClient.getClient() if err != nil { return reply.Hash, false, err } conn, err := srpcClient.Call("ObjectServer.AddObjects") if err != nil { return reply.Hash, false, err } defer conn.Close() request.Length = length request.ExpectedHash = expectedHash encoder := gob.NewEncoder(conn) encoder.Encode(request) nCopied, err := io.Copy(conn, reader) if err != nil { return reply.Hash, false, err } if uint64(nCopied) != length { return reply.Hash, false, errors.New(fmt.Sprintf( "failed to copy, wanted: %d, got: %d bytes", length, nCopied)) } // Send end-of-stream marker. request = objectserver.AddObjectRequest{} encoder.Encode(request) conn.Flush() decoder := gob.NewDecoder(conn) if err := decoder.Decode(&reply); err != nil { return reply.Hash, false, err } if reply.Error != nil { return reply.Hash, false, err } if expectedHash != nil && *expectedHash != reply.Hash { return reply.Hash, false, errors.New(fmt.Sprintf( "received hash: %x != expected: %x", reply.Hash, *expectedHash)) } return reply.Hash, reply.Added, nil }
func (objQ *ObjectAdderQueue) add(reader io.Reader, length uint64) ( hash.Hash, error) { var hash hash.Hash if err := objQ.consumeErrors(); err != nil { return hash, err } data := make([]byte, length) nRead, err := io.ReadFull(reader, data) if err != nil { return hash, err } if uint64(nRead) != length { return hash, errors.New(fmt.Sprintf( "failed to read file data, wanted: %d, got: %d bytes", length, nRead)) } hasher := sha512.New() if _, err := hasher.Write(data); err != nil { return hash, err } copy(hash[:], hasher.Sum(nil)) // Send in a goroutine to increase concurrency. A small win. objQ.sendSemaphore <- true go func() { defer func() { <-objQ.sendSemaphore }() var request objectserver.AddObjectRequest request.Length = uint64(len(data)) request.ExpectedHash = &hash objQ.encoder.Encode(request) objQ.conn.Write(data) objQ.getResponseChan <- true }() return hash, nil }