func chunkServer() { loadConfigOrDie() debug.SetGCPercent(config.GCPercent) stores := make([]store.Store, len(config.Chunk.Dirs)) for i := range config.Chunk.Dirs { dir := config.Chunk.Dirs[i] construct := func() store.Store { log.Printf("Trying to open store at %v", dir) start := time.Now() ds, err := storedir.OpenDirectory( dir, config.Chunk.Scrubber.SleepPerFile.Duration, config.Chunk.Scrubber.SleepPerByte.Duration, ) if err != nil { log.Printf("Couldn't open store at %v: %v", dir, err) return nil } dur := time.Now().Sub(start) log.Printf("Store at %v opened with UUID %v in %v", dir, uuid.Fmt(ds.UUID()), dur) return ds } stores[i] = store.NewRetryStore(construct, time.Second*15) } var h http.Handler var err error h, err = chunkserver.New(stores) if err != nil { log.Fatalf("Couldn't initialize handler: %v", err) } h = httputil.NewLimitParallelism(config.Chunk.ParallelRequests, h) h = httputil.AddDebugHandlers(h, config.Chunk.Debug) if !config.Chunk.DisableHTTPLogging { h = httputil.LogHTTPRequests(h) } serveOrDie(config.Chunk.Listen, h) }
func prepareMultiTest(t testing.TB, need, total, serverCount int) ([]*killHandler, *Multi, []*storetests.MockStore, func()) { var killers []*killHandler var mockstores []*storetests.MockStore var servers []*httptest.Server var chunkServers []*chunkserver.Handler var finder *Finder var multi *Multi done := func() { if multi != nil { multi.Close() } if finder != nil { finder.Stop() } for _, srv := range servers { srv.Close() } for _, cs := range chunkServers { cs.Close() } for _, mock := range mockstores { mock.Close() } } for i := 0; i < serverCount; i++ { mock := storetests.NewMockStore(0) mockstores = append(mockstores, mock) cs, err := chunkserver.New([]store.Store{mock}) if err != nil { done() t.Fatalf("Couldn't create chunkserver: %v", err) } chunkServers = append(chunkServers, cs) killer := newKillHandler(cs) srv := httptest.NewServer(killer) killers = append(killers, killer) servers = append(servers, srv) } db := ram.New() finder, err := NewFinder(db) if err != nil { done() t.Fatalf("Couldn't create new finder: %v", err) } for _, srv := range servers { err = finder.Scan(srv.URL) if err != nil { done() t.Fatalf("Couldn't scan %v: %v", srv.URL, err) } } if len(finder.Stores()) != len(servers) { done() t.Fatalf("Finder did not find all stores") } multi, err = NewMulti(db, finder, 0) if err != nil { done() t.Fatalf("Couldn't create multi: %v", err) } err = multi.SetRedundancy(need, total) if err != nil { done() t.Fatalf("Couldn't set redundancy levels: %v", err) } return killers, multi, mockstores, done }
func TestFinderScan(t *testing.T) { db := ram.New() mock := storetests.NewMockStore(0) cs, err := chunkserver.New([]store.Store{mock}) if err != nil { t.Fatalf("Couldn't create chunkserver: %v", err) } defer cs.Close() cs.WaitAllAvailable() killer := newKillHandler(cs) srv := httptest.NewServer(killer) defer srv.Close() f, err := NewFinder(db) if err != nil { t.Fatalf("Couldn't create new finder: %v", err) } defer f.Stop() err = f.Scan(srv.URL) if err != nil { t.Fatalf("Couldn't scan %v: %v", srv.URL, err) } // newly scanned store should be in the Finder stores := f.Stores() if _, ok := stores[mock.UUID()]; !ok { t.Fatalf("Finder did not find uuid of directory store") } // kill the store and update the Finder killer.setKilled(true) f.test(0) // should have been removed stores = f.Stores() if len(stores) > 0 { t.Fatalf("Finder did not remove dead store") } // but should stay in the DB err = db.RunReadTx(func(ctx kvl.Ctx) error { layer, err := meta.Open(ctx) if err != nil { return err } loc, err := layer.GetLocation(mock.UUID()) if err != nil { return err } if loc == nil { return fmt.Errorf("No location in database") } return nil }) if err != nil { t.Fatalf("Couldn't verify locations: %v", err) } // when the store comes back killer.setKilled(false) err = f.Rescan() if err != nil { t.Fatalf("Couldn't Rescan: %v", err) } // it should be there again stores = f.Stores() if _, ok := stores[mock.UUID()]; !ok { t.Fatalf("Finder did not find uuid of directory store after resurrection") } }