// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
names := packageNames()
indexFiles := make([]string, 0, len(names))
for _, name := range names {
if strings.HasSuffix(name, ".idx") && name != "full.idx" {
indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
}
}
filesInIndex.Set(float64(len(indexFiles)))
log.Printf("Got %d index files\n", len(indexFiles))
if len(indexFiles) < 2 {
return
}
tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
if err != nil {
log.Fatal(err)
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
t0 := time.Now()
index.ConcatN(tmpIndexPath.Name(), indexFiles...)
t1 := time.Now()
log.Printf("merged in %v\n", t1.Sub(t0))
//for i := 1; i < len(indexFiles); i++ {
// log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
// t0 := time.Now()
// index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
// t1 := time.Now()
// log.Printf("merged in %v\n", t1.Sub(t0))
//}
log.Printf("merged into shard %s\n", tmpIndexPath.Name())
// If full.idx does not exist (i.e. on initial deployment), just move the
// new index to full.idx, the dcs-index-backend will not be running anyway.
fullIdxPath := filepath.Join(*unpackedPath, "full.idx")
if _, err := os.Stat(fullIdxPath); os.IsNotExist(err) {
if err := os.Rename(tmpIndexPath.Name(), fullIdxPath); err != nil {
log.Fatal(err)
}
return
}
successfulMerges.Inc()
// Replace the current index with the newly created index.
if _, err := indexBackend.ReplaceIndex(context.Background(), &proto.ReplaceIndexRequest{ReplacementPath: filepath.Base(tmpIndexPath.Name())}); err != nil {
log.Fatalf("dcs-index-backend ReplaceIndex failed: %v", err)
}
}
// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
file, err := os.Open(*unpackedPath)
if err != nil {
log.Fatal(err)
}
defer file.Close()
names, err := file.Readdirnames(-1)
if err != nil {
log.Fatal(err)
}
indexFiles := make([]string, 0, len(names))
for _, name := range names {
if strings.HasSuffix(name, ".idx") {
indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
}
}
log.Printf("Got %d index files\n", len(indexFiles))
if len(indexFiles) == 1 {
return
}
tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
if err != nil {
log.Fatal(err)
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
t0 := time.Now()
index.ConcatN(tmpIndexPath.Name(), indexFiles...)
t1 := time.Now()
log.Printf("merged in %v\n", t1.Sub(t0))
//for i := 1; i < len(indexFiles); i++ {
// log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
// t0 := time.Now()
// index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
// t1 := time.Now()
// log.Printf("merged in %v\n", t1.Sub(t0))
//}
log.Printf("merged into shard %s\n", tmpIndexPath.Name())
}
// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
names := packageNames()
indexFiles := make([]string, 0, len(names))
for _, name := range names {
if strings.HasSuffix(name, ".idx") && name != "full.idx" {
indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
}
}
filesInIndex.Set(float64(len(indexFiles)))
log.Printf("Got %d index files\n", len(indexFiles))
if len(indexFiles) == 1 {
return
}
tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
if err != nil {
log.Fatal(err)
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
t0 := time.Now()
index.ConcatN(tmpIndexPath.Name(), indexFiles...)
t1 := time.Now()
log.Printf("merged in %v\n", t1.Sub(t0))
//for i := 1; i < len(indexFiles); i++ {
// log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
// t0 := time.Now()
// index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
// t1 := time.Now()
// log.Printf("merged in %v\n", t1.Sub(t0))
//}
log.Printf("merged into shard %s\n", tmpIndexPath.Name())
// If full.idx does not exist (i.e. on initial deployment), just move the
// new index to full.idx, the dcs-index-backend will not be running anyway.
fullIdxPath := filepath.Join(*unpackedPath, "full.idx")
if _, err := os.Stat(fullIdxPath); os.IsNotExist(err) {
if err := os.Rename(tmpIndexPath.Name(), fullIdxPath); err != nil {
log.Fatal(err)
}
return
}
successfulMerges.Inc()
// Replace the current index with the newly created index.
resp, err := http.Get(fmt.Sprintf("http://localhost:28081/replace?shard=%s", filepath.Base(tmpIndexPath.Name())))
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
body, _ := ioutil.ReadAll(resp.Body)
log.Fatalf("dcs-index-backend /replace response: %+v (body: %s)\n", resp, body)
}
}