Beispiel #1
0
func (g *Game) getInitTweets(aliveNum int) {
	var progress *pb.ProgressBar
	if !g.debug {
		progress = pb.StartNew(aliveNum)
	}
	for y := 0; y < g.field.SizeY; y++ {
		for x := 0; x < g.field.SizeX; x++ {
			if g.field.Points[y][x].IsAlive {
				tweet := <-g.tweetCh
				if g.debug {
					emoji.Printf(":bird:Points[%v][%v]: %v\n", x, y, tweet)
				}
				g.field.Points[y][x].Str = tweet
				if !g.debug {
					progress.Increment()
				}
			}
		}
	}
	if g.debug {
		emoji.Println(":smile::smile::smile:Collected initial tweets:smile::smile::smile:")
	} else {
		e := emoji.Sprint(":smile::smile::smile:")
		progress.FinishPrint(e + "Collected initial tweets" + e)
	}
}
Beispiel #2
0
func indexAndSaveHits(ts *index.TokenSetSearcher, hits []HitInfo, idxs []int, saveFullHit func(*HitInfo) error) error {
	rank := 0
	var bar *pb.ProgressBar
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		bar = pb.New(len(idxs))
		bar.Start()
	}
	for i := range idxs {
		hit := &hits[idxs[i]]
		if i > 0 && hit.StaticScore < hits[idxs[i-1]].StaticScore {
			rank = i
		}
		hit.StaticRank = rank

		if err := saveFullHit(hit); err != nil {
			return err
		}

		var desc, readme string
		desc, hit.Description = hit.Description, ""
		readme, hit.ReadmeData = hit.ReadmeData, ""
		hit.Imported = nil
		hit.TestImported = nil

		var nameTokens stringsp.Set
		nameTokens = AppendTokens(nameTokens, []byte(hit.Name))

		var tokens stringsp.Set
		tokens.Add(nameTokens.Elements()...)
		tokens = AppendTokens(tokens, []byte(hit.Package))
		tokens = AppendTokens(tokens, []byte(desc))
		tokens = AppendTokens(tokens, []byte(readme))
		tokens = AppendTokens(tokens, []byte(hit.Author))
		for _, word := range hit.Exported {
			AppendTokens(tokens, []byte(word))
		}
		ts.AddDoc(map[string]stringsp.Set{
			IndexTextField: tokens,
			IndexNameField: nameTokens,
			IndexPkgField:  stringsp.NewSet(hit.Package),
		}, *hit)
		if bar != nil {
			bar.Increment()
		}
	}
	if bar != nil {
		bar.FinishPrint("Indexing finished!")
	}
	DumpMemStats()
	return nil
}
Beispiel #3
0
func main() {

	var progress *pb.ProgressBar

	page := 0
	category := "bugs"

	for {

		printf("Get page: %d", page)
		tickets, _, err := sfClient.Tracker.Info(category)

		if err != nil {
			log.Fatal(err)
		}

		if ghMilestones == nil {
			createMilestones(tickets)
			getMilestones()
		}

		if progress == nil {
			log.Println("Creating tickets")
			progress = pb.StartNew(tickets.Count)
		}

		if len(tickets.Tickets) == 0 {
			break
		}

		for _, ticket := range tickets.Tickets {
			ticket, _, err := sfClient.Tracker.Get(category, ticket.TicketNum)

			if err != nil {
				log.Fatal(err)
			}

			sfTicketToGhIssue(ticket, category)

			progress.Increment()
			time.Sleep(time.Millisecond * sleepTime)
		}

		page += 1
	}

	progress.FinishPrint("All tickets imported")
}
Beispiel #4
0
func scanWords(wordpath string, tupleLength int, makeLookup bool) (words map[string]int, tuples map[string]string, wordsLookup map[int]string, tuplesLookup map[string][]int) {
	totalLines := lineCount(wordpath)

	inFile, _ := os.Open(wordpath)
	defer inFile.Close()
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

	// initialize
	words = make(map[string]int)
	tuples = make(map[string]string)
	wordsLookup = make(map[int]string)
	tuplesLookup = make(map[string][]int)

	numTuples := 0
	numWords := 0
	lineNum := 0
	var bar *pb.ProgressBar
	if VERBOSE {
		fmt.Println("Parsing subsets...")
		bar = pb.StartNew(totalLines)
	}
	for scanner.Scan() {
		if VERBOSE {
			bar.Increment()
		}
		lineNum++
		s := strings.TrimSpace(scanner.Text())

		_, ok := words[s]
		if ok == false {
			if makeLookup {
				wordsLookup[numWords] = s
			} else {
				words[s] = numWords
			}

			partials := getPartials(s, tupleLength)
			for i := 0; i < len(partials); i++ {
				_, ok := tuples[partials[i]]
				if makeLookup {
					_, ok = tuplesLookup[partials[i]]
				}
				if ok == false {
					if makeLookup {
						tuplesLookup[partials[i]] = append([]int{}, numWords)
					} else {
						tuples[partials[i]] = strconv.Itoa(numWords)
					}
					numTuples++
				} else {
					if makeLookup {
						tuplesLookup[partials[i]] = append(tuplesLookup[partials[i]], numWords)
					} else {
						tuples[partials[i]] += " " + strconv.Itoa(numWords)
					}
				}
			}

			numWords++
		}

	}
	if VERBOSE {
		bar.FinishPrint("Finished parsing subsets")
	}
	return
}
Beispiel #5
0
func dumpToBoltDB(path string, words map[string]int, tuples map[string]string, tupleLength int) {
	var bar *pb.ProgressBar
	var start time.Time
	wordBuckets := int(len(words) / 600)
	if wordBuckets < 10 {
		wordBuckets = 10
	}
	if VERBOSE {
		fmt.Printf("Creating %v word buckets\n", wordBuckets)
	}

	if _, err := os.Stat(path); err == nil {
		os.Remove(path)
		if VERBOSE {
			fmt.Println("Removed old " + path)
		}
	}

	// Open a new bolt database
	db, err := bolt.Open(path, 0600, &bolt.Options{NoGrowSync: false})
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	if VERBOSE {
		fmt.Println("Creating subset buckets...")
		bar = pb.StartNew(len(tuples))
		start = time.Now()
	}
	err = db.Batch(func(tx *bolt.Tx) error {
		for k := range tuples {
			if VERBOSE {
				bar.Increment()
			}
			firstLetter := string(k[0])
			secondLetter := string(k[1])
			if strings.Contains(alphabet, firstLetter) && strings.Contains(alphabet, secondLetter) {
				_, err := tx.CreateBucketIfNotExists([]byte("tuples-" + firstLetter + secondLetter))
				if err != nil {
					return fmt.Errorf("create bucket: %s", err)
				}
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	if VERBOSE {
		elapsed := time.Since(start)
		bar.FinishPrint("Creating subset buckets took " + elapsed.String())
	}

	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucket([]byte("tuples"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return nil
	})

	if VERBOSE {
		fmt.Println("Creating words buckets...")
	}
	db.Batch(func(tx *bolt.Tx) error {
		for i := 0; i < wordBuckets; i++ {
			_, err := tx.CreateBucket([]byte("words-" + strconv.Itoa(i)))
			if err != nil {
				return fmt.Errorf("create bucket: %s", err)
			}
		}
		return nil
	})

	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucket([]byte("vars"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		return nil
	})

	// fmt.Printf("INSERT INTO words (id,word) values (%v,'%v');\n", v, k)
	if VERBOSE {
		fmt.Println("Loading words into db...")
		start = time.Now()
		bar = pb.StartNew(len(words))
	}
	err = db.Batch(func(tx *bolt.Tx) error {
		for k, v := range words {
			if VERBOSE {
				bar.Increment()
			}
			if len(k) > 0 {
				b := tx.Bucket([]byte("words-" + strconv.Itoa(int(math.Mod(float64(v), float64(wordBuckets))))))
				b.Put([]byte(strconv.Itoa(v)), []byte(k))
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	if VERBOSE {
		elapsed := time.Since(start)
		bar.FinishPrint("Words took " + elapsed.String())
	}

	if VERBOSE {
		fmt.Println("Loading subsets into db...")
		start = time.Now()
		bar = pb.StartNew(len(tuples))
	}
	err = db.Update(func(tx *bolt.Tx) error {
		for k, v := range tuples {
			if VERBOSE {
				bar.Increment()
			}
			firstLetter := string(k[0])
			secondLetter := string(k[1])
			if strings.Contains(alphabet, firstLetter) && strings.Contains(alphabet, secondLetter) {
				b := tx.Bucket([]byte("tuples-" + firstLetter + secondLetter))
				b.Put([]byte(k), []byte(v))
			} else {
				b := tx.Bucket([]byte("tuples"))
				b.Put([]byte(k), []byte(v))
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err) // BUG(schollz): Windows file resize error: https://github.com/schollz/goagrep/issues/6
	}
	if VERBOSE {
		elapsed := time.Since(start)
		bar.FinishPrint("Subsets took " + elapsed.String())
	}

	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("vars"))
		err := b.Put([]byte("tupleLength"), []byte(strconv.Itoa(tupleLength)))
		return err
	})

	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("vars"))
		err := b.Put([]byte("wordBuckets"), []byte(strconv.Itoa(wordBuckets)))
		return err
	})
}