Пример #1
0
func chunkServer() {
	loadConfigOrDie()
	debug.SetGCPercent(config.GCPercent)

	stores := make([]store.Store, len(config.Chunk.Dirs))
	for i := range config.Chunk.Dirs {
		dir := config.Chunk.Dirs[i]
		construct := func() store.Store {
			log.Printf("Trying to open store at %v", dir)

			start := time.Now()

			ds, err := storedir.OpenDirectory(
				dir,
				config.Chunk.Scrubber.SleepPerFile.Duration,
				config.Chunk.Scrubber.SleepPerByte.Duration,
			)

			if err != nil {
				log.Printf("Couldn't open store at %v: %v", dir, err)
				return nil
			}

			dur := time.Now().Sub(start)

			log.Printf("Store at %v opened with UUID %v in %v",
				dir, uuid.Fmt(ds.UUID()), dur)

			return ds
		}
		stores[i] = store.NewRetryStore(construct, time.Second*15)
	}

	var h http.Handler
	var err error
	h, err = chunkserver.New(stores)
	if err != nil {
		log.Fatalf("Couldn't initialize handler: %v", err)
	}

	h = httputil.NewLimitParallelism(config.Chunk.ParallelRequests, h)
	h = httputil.AddDebugHandlers(h, config.Chunk.Debug)
	if !config.Chunk.DisableHTTPLogging {
		h = httputil.LogHTTPRequests(h)
	}
	serveOrDie(config.Chunk.Listen, h)
}
Пример #2
0
func prepareMultiTest(t testing.TB, need, total, serverCount int) ([]*killHandler, *Multi, []*storetests.MockStore, func()) {
	var killers []*killHandler
	var mockstores []*storetests.MockStore

	var servers []*httptest.Server
	var chunkServers []*chunkserver.Handler
	var finder *Finder
	var multi *Multi

	done := func() {
		if multi != nil {
			multi.Close()
		}
		if finder != nil {
			finder.Stop()
		}
		for _, srv := range servers {
			srv.Close()
		}
		for _, cs := range chunkServers {
			cs.Close()
		}
		for _, mock := range mockstores {
			mock.Close()
		}
	}

	for i := 0; i < serverCount; i++ {
		mock := storetests.NewMockStore(0)
		mockstores = append(mockstores, mock)

		cs, err := chunkserver.New([]store.Store{mock})
		if err != nil {
			done()
			t.Fatalf("Couldn't create chunkserver: %v", err)
		}
		chunkServers = append(chunkServers, cs)

		killer := newKillHandler(cs)
		srv := httptest.NewServer(killer)

		killers = append(killers, killer)
		servers = append(servers, srv)
	}

	db := ram.New()

	finder, err := NewFinder(db)
	if err != nil {
		done()
		t.Fatalf("Couldn't create new finder: %v", err)
	}

	for _, srv := range servers {
		err = finder.Scan(srv.URL)
		if err != nil {
			done()
			t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
		}
	}

	if len(finder.Stores()) != len(servers) {
		done()
		t.Fatalf("Finder did not find all stores")
	}

	multi, err = NewMulti(db, finder, 0)
	if err != nil {
		done()
		t.Fatalf("Couldn't create multi: %v", err)
	}

	err = multi.SetRedundancy(need, total)
	if err != nil {
		done()
		t.Fatalf("Couldn't set redundancy levels: %v", err)
	}

	return killers, multi, mockstores, done
}
Пример #3
0
func TestFinderScan(t *testing.T) {
	db := ram.New()

	mock := storetests.NewMockStore(0)

	cs, err := chunkserver.New([]store.Store{mock})
	if err != nil {
		t.Fatalf("Couldn't create chunkserver: %v", err)
	}
	defer cs.Close()

	cs.WaitAllAvailable()

	killer := newKillHandler(cs)
	srv := httptest.NewServer(killer)
	defer srv.Close()

	f, err := NewFinder(db)
	if err != nil {
		t.Fatalf("Couldn't create new finder: %v", err)
	}
	defer f.Stop()

	err = f.Scan(srv.URL)
	if err != nil {
		t.Fatalf("Couldn't scan %v: %v", srv.URL, err)
	}

	// newly scanned store should be in the Finder
	stores := f.Stores()
	if _, ok := stores[mock.UUID()]; !ok {
		t.Fatalf("Finder did not find uuid of directory store")
	}

	// kill the store and update the Finder
	killer.setKilled(true)
	f.test(0)

	// should have been removed
	stores = f.Stores()
	if len(stores) > 0 {
		t.Fatalf("Finder did not remove dead store")
	}

	// but should stay in the DB
	err = db.RunReadTx(func(ctx kvl.Ctx) error {
		layer, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		loc, err := layer.GetLocation(mock.UUID())
		if err != nil {
			return err
		}

		if loc == nil {
			return fmt.Errorf("No location in database")
		}

		return nil
	})
	if err != nil {
		t.Fatalf("Couldn't verify locations: %v", err)
	}

	// when the store comes back
	killer.setKilled(false)
	err = f.Rescan()
	if err != nil {
		t.Fatalf("Couldn't Rescan: %v", err)
	}

	// it should be there again
	stores = f.Stores()
	if _, ok := stores[mock.UUID()]; !ok {
		t.Fatalf("Finder did not find uuid of directory store after resurrection")
	}
}