Beispiel #1
0
// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
	names := packageNames()
	indexFiles := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".idx") && name != "full.idx" {
			indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
		}
	}

	filesInIndex.Set(float64(len(indexFiles)))

	log.Printf("Got %d index files\n", len(indexFiles))
	if len(indexFiles) < 2 {
		return
	}
	tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
	if err != nil {
		log.Fatal(err)
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	t0 := time.Now()
	index.ConcatN(tmpIndexPath.Name(), indexFiles...)
	t1 := time.Now()
	log.Printf("merged in %v\n", t1.Sub(t0))
	//for i := 1; i < len(indexFiles); i++ {
	//	log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
	//	t0 := time.Now()
	//	index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
	//	t1 := time.Now()
	//	log.Printf("merged in %v\n", t1.Sub(t0))
	//}
	log.Printf("merged into shard %s\n", tmpIndexPath.Name())

	// If full.idx does not exist (i.e. on initial deployment), just move the
	// new index to full.idx, the dcs-index-backend will not be running anyway.
	fullIdxPath := filepath.Join(*unpackedPath, "full.idx")
	if _, err := os.Stat(fullIdxPath); os.IsNotExist(err) {
		if err := os.Rename(tmpIndexPath.Name(), fullIdxPath); err != nil {
			log.Fatal(err)
		}
		return
	}

	successfulMerges.Inc()

	// Replace the current index with the newly created index.
	if _, err := indexBackend.ReplaceIndex(context.Background(), &proto.ReplaceIndexRequest{ReplacementPath: filepath.Base(tmpIndexPath.Name())}); err != nil {
		log.Fatalf("dcs-index-backend ReplaceIndex failed: %v", err)
	}
}
Beispiel #2
0
// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
	file, err := os.Open(*unpackedPath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	names, err := file.Readdirnames(-1)
	if err != nil {
		log.Fatal(err)
	}
	indexFiles := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".idx") {
			indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
		}
	}

	log.Printf("Got %d index files\n", len(indexFiles))
	if len(indexFiles) == 1 {
		return
	}
	tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
	if err != nil {
		log.Fatal(err)
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	t0 := time.Now()
	index.ConcatN(tmpIndexPath.Name(), indexFiles...)
	t1 := time.Now()
	log.Printf("merged in %v\n", t1.Sub(t0))
	//for i := 1; i < len(indexFiles); i++ {
	//	log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
	//	t0 := time.Now()
	//	index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
	//	t1 := time.Now()
	//	log.Printf("merged in %v\n", t1.Sub(t0))
	//}
	log.Printf("merged into shard %s\n", tmpIndexPath.Name())
}
Beispiel #3
0
// Merges all packages in *unpackedPath into a big index shard.
func mergeToShard() {
	names := packageNames()
	indexFiles := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".idx") && name != "full.idx" {
			indexFiles = append(indexFiles, filepath.Join(*unpackedPath, name))
		}
	}

	filesInIndex.Set(float64(len(indexFiles)))

	log.Printf("Got %d index files\n", len(indexFiles))
	if len(indexFiles) == 1 {
		return
	}
	tmpIndexPath, err := ioutil.TempFile(*unpackedPath, "newshard")
	if err != nil {
		log.Fatal(err)
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	t0 := time.Now()
	index.ConcatN(tmpIndexPath.Name(), indexFiles...)
	t1 := time.Now()
	log.Printf("merged in %v\n", t1.Sub(t0))
	//for i := 1; i < len(indexFiles); i++ {
	//	log.Printf("merging %s with %s\n", indexFiles[i-1], indexFiles[i])
	//	t0 := time.Now()
	//	index.Concat(tmpIndexPath.Name(), indexFiles[i-1], indexFiles[i])
	//	t1 := time.Now()
	//	log.Printf("merged in %v\n", t1.Sub(t0))
	//}
	log.Printf("merged into shard %s\n", tmpIndexPath.Name())

	// If full.idx does not exist (i.e. on initial deployment), just move the
	// new index to full.idx, the dcs-index-backend will not be running anyway.
	fullIdxPath := filepath.Join(*unpackedPath, "full.idx")
	if _, err := os.Stat(fullIdxPath); os.IsNotExist(err) {
		if err := os.Rename(tmpIndexPath.Name(), fullIdxPath); err != nil {
			log.Fatal(err)
		}
		return
	}

	successfulMerges.Inc()

	// Replace the current index with the newly created index.
	resp, err := http.Get(fmt.Sprintf("http://localhost:28081/replace?shard=%s", filepath.Base(tmpIndexPath.Name())))
	if err != nil {
		log.Fatal(err)
	}

	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		body, _ := ioutil.ReadAll(resp.Body)
		log.Fatalf("dcs-index-backend /replace response: %+v (body: %s)\n", resp, body)
	}
}