Esempio n. 1
0
func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err error) {
	tokens := strings.Fields(indexLine)
	if len(tokens) != 2 {
		err = fmt.Errorf("Expected 2 tokens per line but received a "+
			"line containing %v instead.",
			tokens)
	}

	var locator blockdigest.BlockLocator
	if locator, err = blockdigest.ParseBlockLocator(tokens[0]); err != nil {
		err = fmt.Errorf("%v Received error while parsing line \"%s\"",
			err, indexLine)
		return
	}
	if len(locator.Hints) > 0 {
		err = fmt.Errorf("Block locator in index line should not contain hints "+
			"but it does: %v",
			locator)
		return
	}

	blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
	if err != nil {
		return
	}
	blockInfo.Digest =
		blockdigest.DigestWithSize{Digest: locator.Digest,
			Size: uint32(locator.Size)}
	return
}
Esempio n. 2
0
// Blocks may appear mulitple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator {
	blockChannel := make(chan blockdigest.BlockLocator)
	go func(streamChannel <-chan ManifestStream) {
		for m := range streamChannel {
			for _, block := range m.Blocks {
				if b, err := blockdigest.ParseBlockLocator(block); err == nil {
					blockChannel <- b
				} else {
					log.Printf("ERROR: Failed to parse block: %v", err)
				}
			}
		}
		close(blockChannel)
	}(m.StreamIter())
	return blockChannel
}
Esempio n. 3
0
// Blocks may appear multiple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
//
// In order to detect parse errors, caller must check m.Err after the returned channel closes.
func (m *Manifest) BlockIterWithDuplicates() <-chan blockdigest.BlockLocator {
	blockChannel := make(chan blockdigest.BlockLocator)
	go func(streamChannel <-chan ManifestStream) {
		for ms := range streamChannel {
			if ms.Err != nil {
				m.Err = ms.Err
				continue
			}
			for _, block := range ms.Blocks {
				if b, err := blockdigest.ParseBlockLocator(block); err == nil {
					blockChannel <- b
				} else {
					m.Err = err
				}
			}
		}
		close(blockChannel)
	}(m.StreamIter())
	return blockChannel
}
Esempio n. 4
0
func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
	ksStubResponses := make(map[string]arvadostest.StubResponse)
	ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)}
	ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody}
	ksStub := arvadostest.ServerStub{ksStubResponses}
	ks := httptest.NewServer(&ksStub)
	defer ks.Close()

	ksURL, err := url.Parse(ks.URL)
	c.Check(err, IsNil)
	ksHost, port, err := net.SplitHostPort(ksURL.Host)
	ksPort, err := strconv.Atoi(port)
	c.Check(err, IsNil)

	servers_list := ServiceList{
		ItemsAvailable: 1,
		KeepServers: []ServerAddress{{
			SSL:         false,
			Host:        ksHost,
			Port:        ksPort,
			UUID:        "abcdefg",
			ServiceType: "disk",
		}},
	}
	ksJSON, _ := json.Marshal(servers_list)
	apiStubResponses := make(map[string]arvadostest.StubResponse)
	apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)}
	apiStub := arvadostest.ServerStub{apiStubResponses}

	api := httptest.NewServer(&apiStub)
	defer api.Close()

	arv := arvadosclient.ArvadosClient{
		Scheme:    "http",
		ApiServer: api.URL[7:],
		ApiToken:  "abc123",
		Client:    &http.Client{Transport: &http.Transport{}},
	}

	kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
	kc.SetServiceRoots(map[string]string{"xxxx": ks.URL},
		map[string]string{"xxxx": ks.URL},
		map[string]string{})

	params := GetKeepServersParams{
		Client: arv,
		Logger: nil,
		Limit:  10,
	}

	// GetKeepServersAndSummarize
	results, err := GetKeepServersAndSummarize(params)

	if testData.expectedError == "" {
		c.Assert(err, IsNil)
		c.Assert(results, NotNil)

		blockToServers := results.BlockToServers

		blockLocators := strings.Split(testData.indexResponseBody, "\n")
		for _, loc := range blockLocators {
			locator := strings.Split(loc, " ")[0]
			if locator != "" {
				blockLocator, err := blockdigest.ParseBlockLocator(locator)
				c.Assert(err, IsNil)

				blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)}
				blockServerInfo := blockToServers[blockDigestWithSize]
				c.Assert(blockServerInfo[0].Mtime, NotNil)
			}
		}
	} else {
		c.Assert(err, ErrorMatches, testData.expectedError)
	}
}