Exemple #1
0
func read(list_path, keys_path string) {
	f, err := os.Open(keys_path)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(ErrorCodes["file-open"])
	}
	bf := bufio.NewReader(f)

	var keys []int64
	parse_key := func(s string) int64 {
		return int64(parse_int(s))
	}

	var line string
	for err != io.EOF {
		line, err = bf.ReadString('\n')

		if err != nil && err != io.EOF {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(ErrorCodes["file-read"])
		}

		line = strings.TrimSpace(line)
		if line != "" {
			keys = append(keys, parse_key(line))
		}
	}

	ibf := testfile(list_path)
	cf, err := file.OpenLRUCacheFile(ibf, 1024*1024*12)
	if err != nil {
		panic(err)
	}

	list := varchar.MakeVarcharList(cf)
	defer list.Close()

	err = pprof.StartCPUProfile(profile_writer)
	if err != nil {
		panic(err)
	}

	start := time.Now()
	for _, key := range keys {
		fmt.Print("key ", key)
		start := time.Now()
		list, err := list.GetList(key)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(ErrorCodes["getlist"])
		}
		end := time.Now()
		sum := 0
		for _, item := range list {
			sum += len(item)
		}
		fmt.Println(" duration", end.Sub(start).Seconds(), len(list), sum)
	}
	end := time.Now()
	fmt.Println("duration", end.Sub(start).Seconds())
	pprof.StopCPUProfile()
}
Exemple #2
0
func create(list_path, keys_path string) {
	ibf := testfile(list_path)
	// cf, err := file.OpenLRUCacheFile(ibf, 1024*1024*1024*2)
	// if err != nil { panic(err) }

	list := varchar.MakeVarcharList(ibf)
	defer list.Close()

	list_keys := make([]int64, list_count)
	list_size := make([]int, list_count)
	item_size := item_base_size + rand.Intn(item_variance)
	item := randslice(item_size)

	err := pprof.StartCPUProfile(profile_writer)
	if err != nil {
		panic(err)
	}

	start := time.Now()
	/*
	   for j, key := range list_keys {
	       for i := 0; i < list_size[j]; i++ {
	           item_size := item_base_size + rand.Intn(item_variance)
	           err := list.Push(key, randslice(item_size))
	           if err != nil { panic(err) }
	       }
	       fmt.Fprintln(os.Stderr, "j", j)
	   }
	*/
	max_extra_small := 100
	max_extra_big := 5
	i := 0
	for {
		any_left := false
		for j, key := range list_keys {
			if key == 0 {
				var err error
				list_keys[j], err = list.New()
				if err != nil {
					panic(err)
				}
				list_size[j] = list_base_size + rand.Intn(list_variance)
				if j == 7 {
					list_size[j] += rand.Intn(list_variance) * 2
				}
				key = list_keys[j]
			}
			if list_size[j] <= 0 {
				continue
			} else {
				list_size[j] -= 1
				any_left = true
			}
			err := list.Push(key, item)
			if err != nil {
				panic(err)
			}
			if (j*i+int(key))%17 == 0 && max_extra_small > 0 {
				var err error
				list_key, err := list.New()
				if err != nil {
					panic(err)
				}
				list_keys = append(list_keys, list_key)
				list_size = append(list_size, 1)
				max_extra_small += 1
			}
			if (j*i+int(key))%57 == 0 && max_extra_big > 0 {
				var err error
				list_key, err := list.New()
				if err != nil {
					panic(err)
				}
				list_keys = append(list_keys, list_key)
				list_size = append(list_size, list_base_size)
				max_extra_big -= 1
			}
		}
		if i%20 == 0 {
			fmt.Fprintln(os.Stderr, "i", i)
		}
		i += 1
		if !any_left {
			break
		}
	}
	end := time.Now()
	fmt.Println("duration", end.Sub(start).Seconds())

	pprof.StopCPUProfile()

	f, err := os.Create(keys_path)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(ErrorCodes["file-create"])
	}

	for i := range list_keys {
		j := rand.Intn(i + 1)
		list_keys[i], list_keys[j] = list_keys[j], list_keys[i]
	}

	for _, key := range list_keys {
		fmt.Fprintln(f, key)
	}

	// if err := cf.Persist(); err != nil {
	// panic(err)
	// }
}