func read(list_path, keys_path string) {
f, err := os.Open(keys_path)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(ErrorCodes["file-open"])
}
bf := bufio.NewReader(f)
var keys []int64
parse_key := func(s string) int64 {
return int64(parse_int(s))
}
var line string
for err != io.EOF {
line, err = bf.ReadString('\n')
if err != nil && err != io.EOF {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(ErrorCodes["file-read"])
}
line = strings.TrimSpace(line)
if line != "" {
keys = append(keys, parse_key(line))
}
}
ibf := testfile(list_path)
cf, err := file.OpenLRUCacheFile(ibf, 1024*1024*12)
if err != nil {
panic(err)
}
list := varchar.MakeVarcharList(cf)
defer list.Close()
err = pprof.StartCPUProfile(profile_writer)
if err != nil {
panic(err)
}
start := time.Now()
for _, key := range keys {
fmt.Print("key ", key)
start := time.Now()
list, err := list.GetList(key)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(ErrorCodes["getlist"])
}
end := time.Now()
sum := 0
for _, item := range list {
sum += len(item)
}
fmt.Println(" duration", end.Sub(start).Seconds(), len(list), sum)
}
end := time.Now()
fmt.Println("duration", end.Sub(start).Seconds())
pprof.StopCPUProfile()
}
func create(list_path, keys_path string) {
ibf := testfile(list_path)
// cf, err := file.OpenLRUCacheFile(ibf, 1024*1024*1024*2)
// if err != nil { panic(err) }
list := varchar.MakeVarcharList(ibf)
defer list.Close()
list_keys := make([]int64, list_count)
list_size := make([]int, list_count)
item_size := item_base_size + rand.Intn(item_variance)
item := randslice(item_size)
err := pprof.StartCPUProfile(profile_writer)
if err != nil {
panic(err)
}
start := time.Now()
/*
for j, key := range list_keys {
for i := 0; i < list_size[j]; i++ {
item_size := item_base_size + rand.Intn(item_variance)
err := list.Push(key, randslice(item_size))
if err != nil { panic(err) }
}
fmt.Fprintln(os.Stderr, "j", j)
}
*/
max_extra_small := 100
max_extra_big := 5
i := 0
for {
any_left := false
for j, key := range list_keys {
if key == 0 {
var err error
list_keys[j], err = list.New()
if err != nil {
panic(err)
}
list_size[j] = list_base_size + rand.Intn(list_variance)
if j == 7 {
list_size[j] += rand.Intn(list_variance) * 2
}
key = list_keys[j]
}
if list_size[j] <= 0 {
continue
} else {
list_size[j] -= 1
any_left = true
}
err := list.Push(key, item)
if err != nil {
panic(err)
}
if (j*i+int(key))%17 == 0 && max_extra_small > 0 {
var err error
list_key, err := list.New()
if err != nil {
panic(err)
}
list_keys = append(list_keys, list_key)
list_size = append(list_size, 1)
max_extra_small += 1
}
if (j*i+int(key))%57 == 0 && max_extra_big > 0 {
var err error
list_key, err := list.New()
if err != nil {
panic(err)
}
list_keys = append(list_keys, list_key)
list_size = append(list_size, list_base_size)
max_extra_big -= 1
}
}
if i%20 == 0 {
fmt.Fprintln(os.Stderr, "i", i)
}
i += 1
if !any_left {
break
}
}
end := time.Now()
fmt.Println("duration", end.Sub(start).Seconds())
pprof.StopCPUProfile()
f, err := os.Create(keys_path)
if err != nil {
fmt.Fprintln(os.Stderr, err.Error())
os.Exit(ErrorCodes["file-create"])
}
for i := range list_keys {
j := rand.Intn(i + 1)
list_keys[i], list_keys[j] = list_keys[j], list_keys[i]
}
for _, key := range list_keys {
fmt.Fprintln(f, key)
}
// if err := cf.Persist(); err != nil {
// panic(err)
// }
}