func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) { tokens := strings.Fields(indexLine) if len(tokens) != 2 { err = fmt.Errorf("Expected 2 tokens per line but received a "+ "line containing %v instead.", tokens) } var locator blockdigest.BlockLocator if locator, err = blockdigest.ParseBlockLocator(tokens[0]); err != nil { err = fmt.Errorf("%v Received error while parsing line \"%s\"", err, indexLine) return } if len(locator.Hints) > 0 { err = fmt.Errorf("Block locator in index line should not contain hints "+ "but it does: %v", locator) return } blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64) if err != nil { return } blockInfo.Digest = blockdigest.DigestWithSize{Digest: locator.Digest, Size: uint32(locator.Size)} return }
// Blocks may appear mulitple times within the same manifest if they // are used by multiple files. In that case this Iterator will output // the same block multiple times. func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator { blockChannel := make(chan blockdigest.BlockLocator) go func(streamChannel <-chan ManifestStream) { for m := range streamChannel { for _, block := range m.Blocks { if b, err := blockdigest.ParseBlockLocator(block); err == nil { blockChannel <- b } else { log.Printf("ERROR: Failed to parse block: %v", err) } } } close(blockChannel) }(m.StreamIter()) return blockChannel }
// Blocks may appear multiple times within the same manifest if they // are used by multiple files. In that case this Iterator will output // the same block multiple times. // // In order to detect parse errors, caller must check m.Err after the returned channel closes. func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator { blockChannel := make(chan blockdigest.BlockLocator) go func(streamChannel <-chan ManifestStream) { for ms := range streamChannel { if ms.Err != nil { m.Err = ms.Err continue } for _, block := range ms.Blocks { if b, err := blockdigest.ParseBlockLocator(block); err == nil { blockChannel <- b } else { m.Err = err } } } close(blockChannel) }(m.StreamIter()) return blockChannel }
func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) { ksStubResponses := make(map[string]arvadostest.StubResponse) ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)} ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody} ksStub := arvadostest.ServerStub{ksStubResponses} ks := httptest.NewServer(&ksStub) defer ks.Close() ksURL, err := url.Parse(ks.URL) c.Check(err, IsNil) ksHost, port, err := net.SplitHostPort(ksURL.Host) ksPort, err := strconv.Atoi(port) c.Check(err, IsNil) servers_list := ServiceList{ ItemsAvailable: 1, KeepServers: []ServerAddress{{ SSL: false, Host: ksHost, Port: ksPort, UUID: "abcdefg", ServiceType: "disk", }}, } ksJSON, _ := json.Marshal(servers_list) apiStubResponses := make(map[string]arvadostest.StubResponse) apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)} apiStub := arvadostest.ServerStub{apiStubResponses} api := httptest.NewServer(&apiStub) defer api.Close() arv := arvadosclient.ArvadosClient{ Scheme: "http", ApiServer: api.URL[7:], ApiToken: "abc123", Client: &http.Client{Transport: &http.Transport{}}, } kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}} kc.SetServiceRoots(map[string]string{"xxxx": ks.URL}, map[string]string{"xxxx": ks.URL}, map[string]string{}) params := GetKeepServersParams{ Client: arv, Logger: nil, Limit: 10, } // GetKeepServersAndSummarize results, err := GetKeepServersAndSummarize(params) if testData.expectedError == "" { c.Assert(err, IsNil) c.Assert(results, NotNil) blockToServers := results.BlockToServers blockLocators := strings.Split(testData.indexResponseBody, "\n") for _, loc := range blockLocators { locator := strings.Split(loc, " ")[0] if locator != "" { blockLocator, err := blockdigest.ParseBlockLocator(locator) c.Assert(err, IsNil) blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)} blockServerInfo := blockToServers[blockDigestWithSize] c.Assert(blockServerInfo[0].Mtime, NotNil) } } } else { c.Assert(err, ErrorMatches, testData.expectedError) } }