func main() {
log.Println("-----")
log.Println(0x01)
log.Printf("%0b \n", 0x01<<32)
log.Println("-----")
bmap := bitmap.NewBitmapSize(7)
origal := [5]uint64{4, 6, 3, 1, 7}
expected := [5]uint64{1, 3, 4, 6, 7}
actual := [5]uint64{}
for _, offset := range origal {
bmap.SetBit(offset, 1)
}
var i uint64
var offset, maxpos uint64 = 0, bmap.Maxpos() + 1
for ; offset < maxpos; offset++ {
if bmap.GetBit(offset) == 1 {
actual[i] = offset
i++
}
}
if expected != actual {
log.Println("error")
}
log.Println(bmap)
}
func main() {
var id int
var file *os.File
result := make(chan record)
todo := make(chan int, maxTodo)
idDb = make(map[int]string)
sem := make(chan int, maxProcessor)
//args := os.Args[1:]
//if args != nil {
// val, err := strconv.Atoi(args[0])
// if err == nil && val > 0 {
// useRand = false
// maxValidId = val
// tmpId = maxValidId
// }
//}
bitMap = bitmap.NewBitmapSize(50000000)
file, err := os.OpenFile(dbFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
fmt.Printf("[%s]: %s\n", dbFileName, err.Error())
return
}
defer file.Close()
totalLoaded, _ := loadRecord(file)
fmt.Printf("Load %d record(s) from %s\n", totalLoaded, dbFileName)
start := genStartId()
fmt.Println("Start id:", start)
todo <- start
go recorder(file, result)
timeout := make(chan bool)
go func() {
time.Sleep(10 * time.Second)
timeout <- true
}()
for {
lenTodo = len(todo)
select {
case id = <-todo:
case <-timeout:
if lenTodo < 2000 {
fmt.Println("genRandId triggerd")
go genRandId(todo)
}
go func() {
time.Sleep(10 * time.Second)
timeout <- true
}()
continue
}
//fmt.Println("get", id)
idDbLock.RLock()
_, ok := idDb[id]
idDbLock.RUnlock()
if ok {
continue
}
sem <- 1
go func() {
nid := id
url := fmt.Sprintf("http://www.zhihu.com/question/%d", nid)
processUrl(url, result, todo)
<-sem
}()
}
}
