Ejemplo n.º 1
0
func main() {
	log.Println("-----")
	log.Println(0x01)
	log.Printf("%0b \n", 0x01<<32)
	log.Println("-----")

	bmap := bitmap.NewBitmapSize(7)

	origal := [5]uint64{4, 6, 3, 1, 7}
	expected := [5]uint64{1, 3, 4, 6, 7}
	actual := [5]uint64{}

	for _, offset := range origal {
		bmap.SetBit(offset, 1)
	}

	var i uint64
	var offset, maxpos uint64 = 0, bmap.Maxpos() + 1
	for ; offset < maxpos; offset++ {
		if bmap.GetBit(offset) == 1 {
			actual[i] = offset
			i++
		}
	}

	if expected != actual {
		log.Println("error")
	}
	log.Println(bmap)
}
Ejemplo n.º 2
0
func main() {
	var id int
	var file *os.File
	result := make(chan record)
	todo := make(chan int, maxTodo)
	idDb = make(map[int]string)
	sem := make(chan int, maxProcessor)

	//args := os.Args[1:]
	//if args != nil {
	//	val, err := strconv.Atoi(args[0])
	//	if err == nil && val > 0 {
	//		useRand = false
	//		maxValidId = val
	//		tmpId = maxValidId
	//	}
	//}

	bitMap = bitmap.NewBitmapSize(50000000)

	file, err := os.OpenFile(dbFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Printf("[%s]: %s\n", dbFileName, err.Error())
		return
	}
	defer file.Close()

	totalLoaded, _ := loadRecord(file)
	fmt.Printf("Load %d record(s) from %s\n", totalLoaded, dbFileName)

	start := genStartId()
	fmt.Println("Start id:", start)

	todo <- start

	go recorder(file, result)

	timeout := make(chan bool)
	go func() {
		time.Sleep(10 * time.Second)
		timeout <- true
	}()

	for {
		lenTodo = len(todo)

		select {
		case id = <-todo:
		case <-timeout:
			if lenTodo < 2000 {
				fmt.Println("genRandId triggerd")
				go genRandId(todo)
			}
			go func() {
				time.Sleep(10 * time.Second)
				timeout <- true
			}()
			continue
		}

		//fmt.Println("get", id)
		idDbLock.RLock()
		_, ok := idDb[id]
		idDbLock.RUnlock()
		if ok {
			continue
		}
		sem <- 1
		go func() {
			nid := id
			url := fmt.Sprintf("http://www.zhihu.com/question/%d", nid)
			processUrl(url, result, todo)
			<-sem
		}()
	}
}