func main() {
flag.Parse()
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatal(err)
}
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
log.Fatal(err)
}
kc.Want_replicas = *Replicas
kc.Client.Timeout = 10 * time.Minute
nextBuf := make(chan []byte, *WriteThreads)
nextLocator := make(chan string, *ReadThreads+*WriteThreads)
go countBeans(nextLocator)
for i := 0; i < *WriteThreads; i++ {
go makeBufs(nextBuf, i)
go doWrites(kc, nextBuf, nextLocator)
}
for i := 0; i < *ReadThreads; i++ {
go doReads(kc, nextLocator)
}
<-make(chan struct{})
}
func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
testdata := make([]byte, blocksize)
for i := 0; i < blocksize; i++ {
testdata[i] = byte(' ')
}
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
kc, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, check.Equals, nil)
loc, _, err := kc.PutB(testdata[:])
c.Assert(err, check.Equals, nil)
mtext := "."
for i := 0; i < 100; i++ {
mtext = mtext + " " + loc
}
mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
coll := map[string]interface{}{}
err = arv.Create("collections",
map[string]interface{}{
"collection": map[string]interface{}{
"name": fmt.Sprintf("testdata blocksize=%d", blocksize),
"manifest_text": mtext,
},
}, &coll)
c.Assert(err, check.Equals, nil)
uuid := coll["uuid"].(string)
hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
c.Check(size, check.Equals, int64(blocksize)*100)
}
// Since keepstore does not come into picture in tests,
// we need to explicitly start the goroutine in tests.
func RunTestPullWorker(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
keepClient, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, Equals, nil)
pullq = NewWorkQueue()
go RunPullWorker(pullq, keepClient)
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
kc, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, check.Equals, nil)
kc.PutB([]byte("Hello world\n"))
kc.PutB([]byte("foo"))
kc.PutB([]byte("foobar"))
}
// setup keepclient using the config provided
func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) {
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
ApiInsecure: config.APIHostInsecure,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
External: config.ExternalClient,
}
// if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
if keepServicesJSON == "" {
kc, err = keepclient.MakeKeepClient(&arv)
if err != nil {
return nil, 0, err
}
} else {
kc = keepclient.New(&arv)
err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
if err != nil {
return kc, 0, err
}
}
if isDst {
// Get default replications value from destination, if it is not already provided
if replications == 0 {
value, err := arv.Discovery("defaultCollectionReplication")
if err == nil {
replications = int(value.(float64))
} else {
return nil, 0, err
}
}
kc.Want_replicas = replications
}
// If srcBlobSignatureTTL is not provided, get it from API server discovery doc
blobSignatureTTL = srcBlobSignatureTTL
if !isDst && srcBlobSignatureTTL == 0 {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
} else {
return nil, 0, err
}
}
return kc, blobSignatureTTL, nil
}
func main() {
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatal(err)
}
jobUuid := os.Getenv("JOB_UUID")
taskUuid := os.Getenv("TASK_UUID")
tmpdir := os.Getenv("TASK_WORK")
keepmount := os.Getenv("TASK_KEEPMOUNT")
var jobStruct Job
var taskStruct Task
err = api.Get("jobs", jobUuid, nil, &jobStruct)
if err != nil {
log.Fatal(err)
}
err = api.Get("job_tasks", taskUuid, nil, &taskStruct)
if err != nil {
log.Fatal(err)
}
var kc IKeepClient
kc, err = keepclient.MakeKeepClient(&api)
if err != nil {
log.Fatal(err)
}
syscall.Umask(0022)
err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)
if err == nil {
os.Exit(0)
} else if _, ok := err.(TempFail); ok {
log.Print(err)
os.Exit(TASK_TEMPFAIL)
} else if _, ok := err.(PermFail); ok {
os.Exit(1)
} else {
log.Fatal(err)
}
}
// setup keepclient using the config provided
func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
ApiInsecure: config.APIHostInsecure,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
External: config.ExternalClient,
}
// if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
if keepServicesJSON == "" {
kc, err = keepclient.MakeKeepClient(&arv)
if err != nil {
return
}
} else {
kc = keepclient.New(&arv)
err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
if err != nil {
return
}
}
// Get if blobSignatureTTL is not provided
ttl = blobSignatureTTL
if blobSignatureTTL == 0 {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
ttl = time.Duration(int(value.(float64))) * time.Second
} else {
return nil, 0, err
}
}
return
}
func main() {
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatal(err)
}
// Container may not have certificates installed, so need to look for
// /etc/arvados/ca-certificates.crt in addition to normal system certs.
var certFiles = []string{
"/etc/ssl/certs/ca-certificates.crt", // Debian
"/etc/pki/tls/certs/ca-bundle.crt", // Red Hat
"/etc/arvados/ca-certificates.crt",
}
certs := x509.NewCertPool()
for _, file := range certFiles {
data, err := ioutil.ReadFile(file)
if err == nil {
log.Printf("Using TLS certificates at %v", file)
certs.AppendCertsFromPEM(data)
}
}
api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
jobUuid := os.Getenv("JOB_UUID")
taskUuid := os.Getenv("TASK_UUID")
tmpdir := os.Getenv("TASK_WORK")
keepmount := os.Getenv("TASK_KEEPMOUNT")
var jobStruct Job
var taskStruct Task
err = api.Get("jobs", jobUuid, nil, &jobStruct)
if err != nil {
log.Fatal(err)
}
err = api.Get("job_tasks", taskUuid, nil, &taskStruct)
if err != nil {
log.Fatal(err)
}
var kc IKeepClient
kc, err = keepclient.MakeKeepClient(&api)
if err != nil {
log.Fatal(err)
}
syscall.Umask(0022)
err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)
if err == nil {
os.Exit(0)
} else if _, ok := err.(TempFail); ok {
log.Print(err)
os.Exit(TASK_TEMPFAIL)
} else if _, ok := err.(PermFail); ok {
os.Exit(1)
} else {
log.Fatal(err)
}
}
func main() {
var (
listen string
no_get bool
no_put bool
default_replicas int
timeout int64
pidfile string
)
flagset := flag.NewFlagSet("keepproxy", flag.ExitOnError)
flagset.StringVar(
&listen,
"listen",
DefaultAddr,
"Interface on which to listen for requests, in the format "+
"ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port "+
"to listen on all network interfaces.")
flagset.BoolVar(
&no_get,
"no-get",
false,
"If set, disable GET operations")
flagset.BoolVar(
&no_put,
"no-put",
false,
"If set, disable PUT operations")
flagset.IntVar(
&default_replicas,
"default-replicas",
2,
"Default number of replicas to write if not specified by the client.")
flagset.Int64Var(
&timeout,
"timeout",
15,
"Timeout on requests to internal Keep services (default 15 seconds)")
flagset.StringVar(
&pidfile,
"pid",
"",
"Path to write pid file")
flagset.Parse(os.Args[1:])
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatalf("Error setting up arvados client %s", err.Error())
}
if os.Getenv("ARVADOS_DEBUG") != "" {
keepclient.DebugPrintf = log.Printf
}
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
log.Fatalf("Error setting up keep client %s", err.Error())
}
if pidfile != "" {
f, err := os.Create(pidfile)
if err != nil {
log.Fatalf("Error writing pid file (%s): %s", pidfile, err.Error())
}
fmt.Fprint(f, os.Getpid())
f.Close()
defer os.Remove(pidfile)
}
kc.Want_replicas = default_replicas
kc.Client.Timeout = time.Duration(timeout) * time.Second
go kc.RefreshServices(5*time.Minute, 3*time.Second)
listener, err = net.Listen("tcp", listen)
if err != nil {
log.Fatalf("Could not listen on %v", listen)
}
log.Printf("Arvados Keep proxy started listening on %v", listener.Addr())
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
term := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
s := <-sig
log.Println("caught signal:", s)
listener.Close()
}(term)
signal.Notify(term, syscall.SIGTERM)
signal.Notify(term, syscall.SIGINT)
// Start serving requests.
http.Serve(listener, MakeRESTRouter(!no_get, !no_put, kc))
log.Println("shutting down")
}
func singlerun(arv arvadosclient.ArvadosClient) error {
var err error
if isAdmin, err := util.UserIsAdmin(arv); err != nil {
return errors.New("Error verifying admin token: " + err.Error())
} else if !isAdmin {
return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
}
if logEventTypePrefix != "" {
arvLogger, err = logger.NewLogger(logger.LoggerParams{
Client: arv,
EventTypePrefix: logEventTypePrefix,
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
loggerutil.LogRunInfo(arvLogger)
if arvLogger != nil {
arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
}
var (
dataFetcher summary.DataFetcher
readCollections collection.ReadCollections
keepServerInfo keep.ReadServers
)
if summary.ShouldReadData() {
dataFetcher = summary.ReadData
} else {
dataFetcher = BuildDataFetcher(arv)
}
err = dataFetcher(arvLogger, &readCollections, &keepServerInfo)
if err != nil {
return err
}
err = summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
if err != nil {
return err
}
buckets := summary.BucketReplication(readCollections, keepServerInfo)
bucketCounts := buckets.Counts()
replicationSummary := buckets.SummarizeBuckets(readCollections)
replicationCounts := replicationSummary.ComputeCounts()
log.Printf("Blocks In Collections: %d, "+
"\nBlocks In Keep: %d.",
len(readCollections.BlockToDesiredReplication),
len(keepServerInfo.BlockToServers))
log.Println(replicationCounts.PrettyPrint())
log.Printf("Blocks Histogram:")
for _, rlbss := range bucketCounts {
log.Printf("%+v: %10d",
rlbss.Levels,
rlbss.Count)
}
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
return fmt.Errorf("Error setting up keep client %v", err.Error())
}
// Log that we're finished. We force the recording, since go will
// not wait for the write timer before exiting.
if arvLogger != nil {
defer arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
summaryInfo := logger.GetOrCreateMap(p, "summary_info")
summaryInfo["block_replication_counts"] = bucketCounts
summaryInfo["replication_summary"] = replicationCounts
p["summary_info"] = summaryInfo
p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
})
}
pullServers := summary.ComputePullServers(kc,
&keepServerInfo,
readCollections.BlockToDesiredReplication,
replicationSummary.UnderReplicatedBlocks)
pullLists := summary.BuildPullLists(pullServers)
trashLists, trashErr := summary.BuildTrashLists(kc,
&keepServerInfo,
replicationSummary.KeepBlocksNotInCollections)
err = summary.WritePullLists(arvLogger, pullLists, dryRun)
if err != nil {
return err
}
if trashErr != nil {
return err
}
keep.SendTrashLists(arvLogger, kc, trashLists, dryRun)
return nil
}
func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
var statusCode = 0
var statusText string
remoteAddr := r.RemoteAddr
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
remoteAddr = xff + "," + remoteAddr
}
w := httpserver.WrapResponseWriter(wOrig)
defer func() {
if statusCode == 0 {
statusCode = w.WroteStatus()
} else if w.WroteStatus() == 0 {
w.WriteHeader(statusCode)
} else if w.WroteStatus() != statusCode {
httpserver.Log(r.RemoteAddr, "WARNING",
fmt.Sprintf("Our status changed from %d to %d after we sent headers", w.WroteStatus(), statusCode))
}
if statusText == "" {
statusText = http.StatusText(statusCode)
}
httpserver.Log(remoteAddr, statusCode, statusText, w.WroteBodyBytes(), r.Method, r.Host, r.URL.Path, r.URL.RawQuery)
}()
if r.Method != "GET" && r.Method != "POST" {
statusCode, statusText = http.StatusMethodNotAllowed, r.Method
return
}
if r.Header.Get("Origin") != "" {
// Allow simple cross-origin requests without user
// credentials ("user credentials" as defined by CORS,
// i.e., cookies, HTTP authentication, and client-side
// SSL certificates. See
// http://www.w3.org/TR/cors/#user-credentials).
w.Header().Set("Access-Control-Allow-Origin", "*")
}
arv := clientPool.Get()
if arv == nil {
statusCode, statusText = http.StatusInternalServerError, "Pool failed: "+clientPool.Err().Error()
return
}
defer clientPool.Put(arv)
pathParts := strings.Split(r.URL.Path[1:], "/")
var targetID string
var targetPath []string
var tokens []string
var reqTokens []string
var pathToken bool
var attachment bool
credentialsOK := trustAllContent
if r.Host != "" && r.Host == attachmentOnlyHost {
credentialsOK = true
attachment = true
} else if r.FormValue("disposition") == "attachment" {
attachment = true
}
if targetID = parseCollectionIDFromDNSName(r.Host); targetID != "" {
// http://ID.collections.example/PATH...
credentialsOK = true
targetPath = pathParts
} else if len(pathParts) >= 2 && strings.HasPrefix(pathParts[0], "c=") {
// /c=ID/PATH...
targetID = parseCollectionIDFromURL(pathParts[0][2:])
targetPath = pathParts[1:]
} else if len(pathParts) >= 3 && pathParts[0] == "collections" {
if len(pathParts) >= 5 && pathParts[1] == "download" {
// /collections/download/ID/TOKEN/PATH...
targetID = pathParts[2]
tokens = []string{pathParts[3]}
targetPath = pathParts[4:]
pathToken = true
} else {
// /collections/ID/PATH...
targetID = pathParts[1]
tokens = anonymousTokens
targetPath = pathParts[2:]
}
} else {
statusCode = http.StatusNotFound
return
}
formToken := r.FormValue("api_token")
if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
// The client provided an explicit token in the POST
// body. The Origin header indicates this *might* be
// an AJAX request, in which case redirect-with-cookie
// won't work: we should just serve the content in the
// POST response. This is safe because:
//
// * We're supplying an attachment, not inline
// content, so we don't need to convert the POST to
// a GET and avoid the "really resubmit form?"
// problem.
//
// * The token isn't embedded in the URL, so we don't
// need to worry about bookmarks and copy/paste.
tokens = append(tokens, formToken)
} else if formToken != "" {
// The client provided an explicit token in the query
// string, or a form in POST body. We must put the
// token in an HttpOnly cookie, and redirect to the
// same URL with the query param redacted and method =
// GET.
if !credentialsOK {
// It is not safe to copy the provided token
// into a cookie unless the current vhost
// (origin) serves only a single collection or
// we are in trustAllContent mode.
statusCode = http.StatusBadRequest
return
}
// The HttpOnly flag is necessary to prevent
// JavaScript code (included in, or loaded by, a page
// in the collection being served) from employing the
// user's token beyond reading other files in the same
// domain, i.e., same collection.
//
// The 303 redirect is necessary in the case of a GET
// request to avoid exposing the token in the Location
// bar, and in the case of a POST request to avoid
// raising warnings when the user refreshes the
// resulting page.
http.SetCookie(w, &http.Cookie{
Name: "arvados_api_token",
Value: auth.EncodeTokenCookie([]byte(formToken)),
Path: "/",
HttpOnly: true,
})
// Propagate query parameters (except api_token) from
// the original request.
redirQuery := r.URL.Query()
redirQuery.Del("api_token")
redir := (&url.URL{
Host: r.Host,
Path: r.URL.Path,
RawQuery: redirQuery.Encode(),
}).String()
w.Header().Add("Location", redir)
statusCode, statusText = http.StatusSeeOther, redir
w.WriteHeader(statusCode)
io.WriteString(w, `<A href="`)
io.WriteString(w, html.EscapeString(redir))
io.WriteString(w, `">Continue</A>`)
return
}
if tokens == nil && strings.HasPrefix(targetPath[0], "t=") {
// http://ID.example/t=TOKEN/PATH...
// /c=ID/t=TOKEN/PATH...
//
// This form must only be used to pass scoped tokens
// that give permission for a single collection. See
// FormValue case above.
tokens = []string{targetPath[0][2:]}
pathToken = true
targetPath = targetPath[1:]
}
if tokens == nil {
if credentialsOK {
reqTokens = auth.NewCredentialsFromHTTPRequest(r).Tokens
}
tokens = append(reqTokens, anonymousTokens...)
}
if len(targetPath) > 0 && targetPath[0] == "_" {
// If a collection has a directory called "t=foo" or
// "_", it can be served at
// //collections.example/_/t=foo/ or
// //collections.example/_/_/ respectively:
// //collections.example/t=foo/ won't work because
// t=foo will be interpreted as a token "foo".
targetPath = targetPath[1:]
}
tokenResult := make(map[string]int)
collection := make(map[string]interface{})
found := false
for _, arv.ApiToken = range tokens {
err := arv.Get("collections", targetID, nil, &collection)
if err == nil {
// Success
found = true
break
}
if srvErr, ok := err.(arvadosclient.APIServerError); ok {
switch srvErr.HttpStatusCode {
case 404, 401:
// Token broken or insufficient to
// retrieve collection
tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
continue
}
}
// Something more serious is wrong
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
if !found {
if pathToken || !credentialsOK {
// Either the URL is a "secret sharing link"
// that didn't work out (and asking the client
// for additional credentials would just be
// confusing), or we don't even accept
// credentials at this path.
statusCode = http.StatusNotFound
return
}
for _, t := range reqTokens {
if tokenResult[t] == 404 {
// The client provided valid token(s), but the
// collection was not found.
statusCode = http.StatusNotFound
return
}
}
// The client's token was invalid (e.g., expired), or
// the client didn't even provide one. Propagate the
// 401 to encourage the client to use a [different]
// token.
//
// TODO(TC): This response would be confusing to
// someone trying (anonymously) to download public
// data that has been deleted. Allow a referrer to
// provide this context somehow?
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
statusCode = http.StatusUnauthorized
return
}
filename := strings.Join(targetPath, "/")
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
if kc.Client != nil && kc.Client.Transport != nil {
// Workaround for https://dev.arvados.org/issues/9005
if t, ok := kc.Client.Transport.(*http.Transport); ok {
defer t.CloseIdleConnections()
}
}
rdr, err := kc.CollectionFileReader(collection, filename)
if os.IsNotExist(err) {
statusCode = http.StatusNotFound
return
} else if err != nil {
statusCode, statusText = http.StatusBadGateway, err.Error()
return
}
defer rdr.Close()
basenamePos := strings.LastIndex(filename, "/")
if basenamePos < 0 {
basenamePos = 0
}
extPos := strings.LastIndex(filename, ".")
if extPos > basenamePos {
// Now extPos is safely >= 0.
if t := mime.TypeByExtension(filename[extPos:]); t != "" {
w.Header().Set("Content-Type", t)
}
}
if rdr, ok := rdr.(keepclient.ReadCloserWithLen); ok {
w.Header().Set("Content-Length", fmt.Sprintf("%d", rdr.Len()))
}
applyContentDispositionHdr(w, r, filename[basenamePos:], attachment)
rangeRdr, statusCode := applyRangeHdr(w, r, rdr)
w.WriteHeader(statusCode)
_, err = io.Copy(w, rangeRdr)
if err != nil {
statusCode, statusText = http.StatusBadGateway, err.Error()
}
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
log.Print("TestPutAndGet start")
os.Args = []string{"keepproxy", "-listen=:29950"}
listener = nil
go main()
time.Sleep(100 * time.Millisecond)
setupProxyService()
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
kc, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, Equals, nil)
c.Check(kc.Arvados.External, Equals, true)
c.Check(kc.Using_proxy, Equals, true)
c.Check(len(kc.LocalRoots()), Equals, 1)
for _, root := range kc.LocalRoots() {
c.Check(root, Equals, "http://localhost:29950")
}
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
waitForListener()
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
var hash2 string
{
_, _, err := kc.Ask(hash)
c.Check(err, Equals, keepclient.BlockNotFound)
log.Print("Finished Ask (expected BlockNotFound)")
}
{
reader, _, _, err := kc.Get(hash)
c.Check(reader, Equals, nil)
c.Check(err, Equals, keepclient.BlockNotFound)
log.Print("Finished Get (expected BlockNotFound)")
}
// Note in bug #5309 among other errors keepproxy would set
// Content-Length incorrectly on the 404 BlockNotFound response, this
// would result in a protocol violation that would prevent reuse of the
// connection, which would manifest by the next attempt to use the
// connection (in this case the PutB below) failing. So to test for
// that bug it's necessary to trigger an error response (such as
// BlockNotFound) and then do something else with the same httpClient
// connection.
{
var rep int
var err error
hash2, rep, err = kc.PutB([]byte("foo"))
c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
c.Check(rep, Equals, 2)
c.Check(err, Equals, nil)
log.Print("Finished PutB (expected success)")
}
{
blocklen, _, err := kc.Ask(hash2)
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(3))
log.Print("Finished Ask (expected success)")
}
{
reader, blocklen, _, err := kc.Get(hash2)
c.Assert(err, Equals, nil)
all, err := ioutil.ReadAll(reader)
c.Check(all, DeepEquals, []byte("foo"))
c.Check(blocklen, Equals, int64(3))
log.Print("Finished Get (expected success)")
}
{
var rep int
var err error
hash2, rep, err = kc.PutB([]byte(""))
c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
c.Check(rep, Equals, 2)
c.Check(err, Equals, nil)
log.Print("Finished PutB zero block")
}
{
reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
c.Assert(err, Equals, nil)
all, err := ioutil.ReadAll(reader)
c.Check(all, DeepEquals, []byte(""))
c.Check(blocklen, Equals, int64(0))
log.Print("Finished Get zero block")
}
log.Print("TestPutAndGet done")
}