func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) {
tokens := strings.Fields(indexLine)
if len(tokens) != 2 {
err = fmt.Errorf("Expected 2 tokens per line but received a "+
"line containing %v instead.",
tokens)
}
var locator blockdigest.BlockLocator
if locator, err = blockdigest.ParseBlockLocator(tokens[0]); err != nil {
err = fmt.Errorf("%v Received error while parsing line \"%s\"",
err, indexLine)
return
}
if len(locator.Hints) > 0 {
err = fmt.Errorf("Block locator in index line should not contain hints "+
"but it does: %v",
locator)
return
}
blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
if err != nil {
return
}
blockInfo.Digest =
blockdigest.DigestWithSize{Digest: locator.Digest,
Size: uint32(locator.Size)}
return
}
// Blocks may appear mulitple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator {
blockChannel := make(chan blockdigest.BlockLocator)
go func(streamChannel <-chan ManifestStream) {
for m := range streamChannel {
for _, block := range m.Blocks {
if b, err := blockdigest.ParseBlockLocator(block); err == nil {
blockChannel <- b
} else {
log.Printf("ERROR: Failed to parse block: %v", err)
}
}
}
close(blockChannel)
}(m.StreamIter())
return blockChannel
}
// Blocks may appear multiple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
//
// In order to detect parse errors, caller must check m.Err after the returned channel closes.
func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator {
blockChannel := make(chan blockdigest.BlockLocator)
go func(streamChannel <-chan ManifestStream) {
for ms := range streamChannel {
if ms.Err != nil {
m.Err = ms.Err
continue
}
for _, block := range ms.Blocks {
if b, err := blockdigest.ParseBlockLocator(block); err == nil {
blockChannel <- b
} else {
m.Err = err
}
}
}
close(blockChannel)
}(m.StreamIter())
return blockChannel
}
func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
ksStubResponses := make(map[string]arvadostest.StubResponse)
ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)}
ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody}
ksStub := arvadostest.ServerStub{ksStubResponses}
ks := httptest.NewServer(&ksStub)
defer ks.Close()
ksURL, err := url.Parse(ks.URL)
c.Check(err, IsNil)
ksHost, port, err := net.SplitHostPort(ksURL.Host)
ksPort, err := strconv.Atoi(port)
c.Check(err, IsNil)
servers_list := ServiceList{
ItemsAvailable: 1,
KeepServers: []ServerAddress{{
SSL: false,
Host: ksHost,
Port: ksPort,
UUID: "abcdefg",
ServiceType: "disk",
}},
}
ksJSON, _ := json.Marshal(servers_list)
apiStubResponses := make(map[string]arvadostest.StubResponse)
apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)}
apiStub := arvadostest.ServerStub{apiStubResponses}
api := httptest.NewServer(&apiStub)
defer api.Close()
arv := arvadosclient.ArvadosClient{
Scheme: "http",
ApiServer: api.URL[7:],
ApiToken: "abc123",
Client: &http.Client{Transport: &http.Transport{}},
}
kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
kc.SetServiceRoots(map[string]string{"xxxx": ks.URL},
map[string]string{"xxxx": ks.URL},
map[string]string{})
params := GetKeepServersParams{
Client: arv,
Logger: nil,
Limit: 10,
}
// GetKeepServersAndSummarize
results, err := GetKeepServersAndSummarize(params)
if testData.expectedError == "" {
c.Assert(err, IsNil)
c.Assert(results, NotNil)
blockToServers := results.BlockToServers
blockLocators := strings.Split(testData.indexResponseBody, "\n")
for _, loc := range blockLocators {
locator := strings.Split(loc, " ")[0]
if locator != "" {
blockLocator, err := blockdigest.ParseBlockLocator(locator)
c.Assert(err, IsNil)
blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)}
blockServerInfo := blockToServers[blockDigestWithSize]
c.Assert(blockServerInfo[0].Mtime, NotNil)
}
}
} else {
c.Assert(err, ErrorMatches, testData.expectedError)
}
}