func SetupDataManagerTest(t *testing.T) {
os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
// start api and keep servers
arvadostest.ResetEnv()
arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
var err error
arv, err = arvadosclient.MakeArvadosClient()
if err != nil {
t.Fatalf("Error making arvados client: %s", err)
}
arv.ApiToken = arvadostest.DataManagerToken
// keep client
keepClient = &keepclient.KeepClient{
Arvados: &arv,
Want_replicas: 2,
Client: &http.Client{},
}
// discover keep services
if err = keepClient.DiscoverKeepServers(); err != nil {
t.Fatalf("Error discovering keep services: %s", err)
}
keepServers = []string{}
for _, host := range keepClient.LocalRoots() {
keepServers = append(keepServers, host)
}
}
func (s *NoKeepServerSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
// We need API to have some keep services listed, but the
// services themselves should be unresponsive.
arvadostest.StartKeep(2, false)
arvadostest.StopKeep(2)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
if *no_server {
c.Skip("Skipping tests that require server")
return
}
arvadostest.StartAPI()
arvadostest.StartKeep()
}
func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) {
var srcConfig apiConfig
srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
srcConfig.APIToken = arvadostest.DataManagerToken
srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
arvadostest.StartKeep(2, false)
_, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0)
c.Check(err, IsNil)
c.Assert(ttl, Equals, blobSignatureTTL)
}
func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) {
config := setupConfigFile(c, "config")
defer os.Remove(config)
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
args := []string{"-config", config, "-block-hash-file", "no-such-file"}
err := doMain(args)
c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}
func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) {
config := setupConfigFile(c, "config")
defer os.Remove(config)
// Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
args := []string{"-config", config}
err := doMain(args)
c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
arvadostest.StartKeep(2, true)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
kc, err := keepclient.MakeKeepClient(&arv)
c.Assert(err, check.Equals, nil)
kc.PutB([]byte("Hello world\n"))
kc.PutB([]byte("foo"))
kc.PutB([]byte("foobar"))
}
func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
srcConfig := setupConfigFile(c, "src")
dstConfig := setupConfigFile(c, "dst")
args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
os.Args = append(os.Args, args...)
// Start keepservers. Since we are not doing any tweaking as
// in setupRsync func, kcSrc and kcDst will be the same and no
// actual copying to dst will happen, but that's ok.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
err := doMain()
c.Check(err, IsNil)
}
func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) {
var config apiConfig
config.APIHost = os.Getenv("ARVADOS_API_HOST")
config.APIToken = arvadostest.DataManagerToken
config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
// Start Keep servers
arvadostest.StartKeep(2, enforcePermissions)
// setup keepclients
var err error
kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl)
c.Assert(ttl, Equals, blobSignatureTTL)
c.Check(err, IsNil)
}
func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
// start api and keep servers
arvadostest.StartAPI()
arvadostest.StartKeep()
// make arvadosclient
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
t.Error("Error creating arv")
}
// keep client
keepClient = &keepclient.KeepClient{
Arvados: &arv,
Want_replicas: 1,
Using_proxy: true,
Client: &http.Client{},
}
// discover keep services
var servers []string
if err := keepClient.DiscoverKeepServers(); err != nil {
t.Error("Error discovering keep services")
}
for _, host := range keepClient.LocalRoots() {
servers = append(servers, host)
}
// Put content if the test needs it
if wantData {
locator, _, err := keepClient.PutB([]byte(testData.Content))
if err != nil {
t.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
}
if locator == "" {
t.Errorf("No locator found after putting test data")
}
}
// Create pullRequest for the test
pullRequest := PullRequest{
Locator: testData.Locator,
Servers: servers,
}
return pullRequest
}
func (s *DoMainTestSuite) Test_doMain(c *C) {
// Start keepservers.
arvadostest.StartKeep(2, false)
defer arvadostest.StopKeep(2)
config := setupConfigFile(c, "config")
defer os.Remove(config)
locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2})
defer os.Remove(locatorFile)
args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
err := doMain(args)
c.Check(err, NotNil)
c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
}
func (s *integrationSuite) SetUpSuite(c *check.C) {
if testing.Short() {
c.Skip("-short")
}
arvadostest.ResetEnv()
arvadostest.StartAPI()
arvadostest.StartKeep(4, true)
arv, err := arvadosclient.MakeArvadosClient()
arv.ApiToken = arvadostest.DataManagerToken
c.Assert(err, check.IsNil)
s.keepClient = &keepclient.KeepClient{
Arvados: &arv,
Client: &http.Client{},
}
c.Assert(s.keepClient.DiscoverKeepServers(), check.IsNil)
s.putReplicas(c, "foo", 4)
s.putReplicas(c, "bar", 1)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
// Testing keep-rsync needs two sets of keep services: src and dst.
// The test setup hence creates 3 servers instead of the default 2,
// and uses the first 2 as src and the 3rd as dst keep servers.
func setupRsync(c *C, enforcePermissions bool, replications int) {
// srcConfig
var srcConfig apiConfig
srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
srcConfig.APIToken = arvadostest.DataManagerToken
srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
// dstConfig
var dstConfig apiConfig
dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
dstConfig.APIToken = arvadostest.DataManagerToken
dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
if enforcePermissions {
blobSigningKey = arvadostest.BlobSigningKey
}
// Start Keep servers
arvadostest.StartKeep(3, enforcePermissions)
// setup keepclients
var err error
kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
c.Check(err, IsNil)
kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
c.Check(err, IsNil)
for uuid := range kcSrc.LocalRoots() {
if strings.HasSuffix(uuid, "02") {
delete(kcSrc.LocalRoots(), uuid)
}
}
for uuid := range kcSrc.GatewayRoots() {
if strings.HasSuffix(uuid, "02") {
delete(kcSrc.GatewayRoots(), uuid)
}
}
for uuid := range kcSrc.WritableLocalRoots() {
if strings.HasSuffix(uuid, "02") {
delete(kcSrc.WritableLocalRoots(), uuid)
}
}
for uuid := range kcDst.LocalRoots() {
if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
delete(kcDst.LocalRoots(), uuid)
}
}
for uuid := range kcDst.GatewayRoots() {
if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
delete(kcDst.GatewayRoots(), uuid)
}
}
for uuid := range kcDst.WritableLocalRoots() {
if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
delete(kcDst.WritableLocalRoots(), uuid)
}
}
if replications == 0 {
// Must have got default replications value of 2 from dst discovery document
c.Assert(kcDst.Want_replicas, Equals, 2)
} else {
// Since replications value is provided, it is used
c.Assert(kcDst.Want_replicas, Equals, replications)
}
}