func (g *Game) getInitTweets(aliveNum int) {
var progress *pb.ProgressBar
if !g.debug {
progress = pb.StartNew(aliveNum)
}
for y := 0; y < g.field.SizeY; y++ {
for x := 0; x < g.field.SizeX; x++ {
if g.field.Points[y][x].IsAlive {
tweet := <-g.tweetCh
if g.debug {
emoji.Printf(":bird:Points[%v][%v]: %v\n", x, y, tweet)
}
g.field.Points[y][x].Str = tweet
if !g.debug {
progress.Increment()
}
}
}
}
if g.debug {
emoji.Println(":smile::smile::smile:Collected initial tweets:smile::smile::smile:")
} else {
e := emoji.Sprint(":smile::smile::smile:")
progress.FinishPrint(e + "Collected initial tweets" + e)
}
}
func indexAndSaveHits(ts *index.TokenSetSearcher, hits []HitInfo, idxs []int, saveFullHit func(*HitInfo) error) error {
rank := 0
var bar *pb.ProgressBar
if terminal.IsTerminal(int(os.Stdout.Fd())) {
bar = pb.New(len(idxs))
bar.Start()
}
for i := range idxs {
hit := &hits[idxs[i]]
if i > 0 && hit.StaticScore < hits[idxs[i-1]].StaticScore {
rank = i
}
hit.StaticRank = rank
if err := saveFullHit(hit); err != nil {
return err
}
var desc, readme string
desc, hit.Description = hit.Description, ""
readme, hit.ReadmeData = hit.ReadmeData, ""
hit.Imported = nil
hit.TestImported = nil
var nameTokens stringsp.Set
nameTokens = AppendTokens(nameTokens, []byte(hit.Name))
var tokens stringsp.Set
tokens.Add(nameTokens.Elements()...)
tokens = AppendTokens(tokens, []byte(hit.Package))
tokens = AppendTokens(tokens, []byte(desc))
tokens = AppendTokens(tokens, []byte(readme))
tokens = AppendTokens(tokens, []byte(hit.Author))
for _, word := range hit.Exported {
AppendTokens(tokens, []byte(word))
}
ts.AddDoc(map[string]stringsp.Set{
IndexTextField: tokens,
IndexNameField: nameTokens,
IndexPkgField: stringsp.NewSet(hit.Package),
}, *hit)
if bar != nil {
bar.Increment()
}
}
if bar != nil {
bar.FinishPrint("Indexing finished!")
}
DumpMemStats()
return nil
}
func main() {
var progress *pb.ProgressBar
page := 0
category := "bugs"
for {
printf("Get page: %d", page)
tickets, _, err := sfClient.Tracker.Info(category)
if err != nil {
log.Fatal(err)
}
if ghMilestones == nil {
createMilestones(tickets)
getMilestones()
}
if progress == nil {
log.Println("Creating tickets")
progress = pb.StartNew(tickets.Count)
}
if len(tickets.Tickets) == 0 {
break
}
for _, ticket := range tickets.Tickets {
ticket, _, err := sfClient.Tracker.Get(category, ticket.TicketNum)
if err != nil {
log.Fatal(err)
}
sfTicketToGhIssue(ticket, category)
progress.Increment()
time.Sleep(time.Millisecond * sleepTime)
}
page += 1
}
progress.FinishPrint("All tickets imported")
}
func scanWords(wordpath string, tupleLength int, makeLookup bool) (words map[string]int, tuples map[string]string, wordsLookup map[int]string, tuplesLookup map[string][]int) {
totalLines := lineCount(wordpath)
inFile, _ := os.Open(wordpath)
defer inFile.Close()
scanner := bufio.NewScanner(inFile)
scanner.Split(bufio.ScanLines)
// initialize
words = make(map[string]int)
tuples = make(map[string]string)
wordsLookup = make(map[int]string)
tuplesLookup = make(map[string][]int)
numTuples := 0
numWords := 0
lineNum := 0
var bar *pb.ProgressBar
if VERBOSE {
fmt.Println("Parsing subsets...")
bar = pb.StartNew(totalLines)
}
for scanner.Scan() {
if VERBOSE {
bar.Increment()
}
lineNum++
s := strings.TrimSpace(scanner.Text())
_, ok := words[s]
if ok == false {
if makeLookup {
wordsLookup[numWords] = s
} else {
words[s] = numWords
}
partials := getPartials(s, tupleLength)
for i := 0; i < len(partials); i++ {
_, ok := tuples[partials[i]]
if makeLookup {
_, ok = tuplesLookup[partials[i]]
}
if ok == false {
if makeLookup {
tuplesLookup[partials[i]] = append([]int{}, numWords)
} else {
tuples[partials[i]] = strconv.Itoa(numWords)
}
numTuples++
} else {
if makeLookup {
tuplesLookup[partials[i]] = append(tuplesLookup[partials[i]], numWords)
} else {
tuples[partials[i]] += " " + strconv.Itoa(numWords)
}
}
}
numWords++
}
}
if VERBOSE {
bar.FinishPrint("Finished parsing subsets")
}
return
}
func dumpToBoltDB(path string, words map[string]int, tuples map[string]string, tupleLength int) {
var bar *pb.ProgressBar
var start time.Time
wordBuckets := int(len(words) / 600)
if wordBuckets < 10 {
wordBuckets = 10
}
if VERBOSE {
fmt.Printf("Creating %v word buckets\n", wordBuckets)
}
if _, err := os.Stat(path); err == nil {
os.Remove(path)
if VERBOSE {
fmt.Println("Removed old " + path)
}
}
// Open a new bolt database
db, err := bolt.Open(path, 0600, &bolt.Options{NoGrowSync: false})
if err != nil {
log.Fatal(err)
}
defer db.Close()
if VERBOSE {
fmt.Println("Creating subset buckets...")
bar = pb.StartNew(len(tuples))
start = time.Now()
}
err = db.Batch(func(tx *bolt.Tx) error {
for k := range tuples {
if VERBOSE {
bar.Increment()
}
firstLetter := string(k[0])
secondLetter := string(k[1])
if strings.Contains(alphabet, firstLetter) && strings.Contains(alphabet, secondLetter) {
_, err := tx.CreateBucketIfNotExists([]byte("tuples-" + firstLetter + secondLetter))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
}
}
return nil
})
if err != nil {
log.Fatal(err)
}
if VERBOSE {
elapsed := time.Since(start)
bar.FinishPrint("Creating subset buckets took " + elapsed.String())
}
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("tuples"))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
return nil
})
if VERBOSE {
fmt.Println("Creating words buckets...")
}
db.Batch(func(tx *bolt.Tx) error {
for i := 0; i < wordBuckets; i++ {
_, err := tx.CreateBucket([]byte("words-" + strconv.Itoa(i)))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
}
return nil
})
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("vars"))
if err != nil {
return fmt.Errorf("create bucket: %s", err)
}
return nil
})
// fmt.Printf("INSERT INTO words (id,word) values (%v,'%v');\n", v, k)
if VERBOSE {
fmt.Println("Loading words into db...")
start = time.Now()
bar = pb.StartNew(len(words))
}
err = db.Batch(func(tx *bolt.Tx) error {
for k, v := range words {
if VERBOSE {
bar.Increment()
}
if len(k) > 0 {
b := tx.Bucket([]byte("words-" + strconv.Itoa(int(math.Mod(float64(v), float64(wordBuckets))))))
b.Put([]byte(strconv.Itoa(v)), []byte(k))
}
}
return nil
})
if err != nil {
log.Fatal(err)
}
if VERBOSE {
elapsed := time.Since(start)
bar.FinishPrint("Words took " + elapsed.String())
}
if VERBOSE {
fmt.Println("Loading subsets into db...")
start = time.Now()
bar = pb.StartNew(len(tuples))
}
err = db.Update(func(tx *bolt.Tx) error {
for k, v := range tuples {
if VERBOSE {
bar.Increment()
}
firstLetter := string(k[0])
secondLetter := string(k[1])
if strings.Contains(alphabet, firstLetter) && strings.Contains(alphabet, secondLetter) {
b := tx.Bucket([]byte("tuples-" + firstLetter + secondLetter))
b.Put([]byte(k), []byte(v))
} else {
b := tx.Bucket([]byte("tuples"))
b.Put([]byte(k), []byte(v))
}
}
return nil
})
if err != nil {
log.Fatal(err) // BUG(schollz): Windows file resize error: https://github.com/schollz/goagrep/issues/6
}
if VERBOSE {
elapsed := time.Since(start)
bar.FinishPrint("Subsets took " + elapsed.String())
}
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("vars"))
err := b.Put([]byte("tupleLength"), []byte(strconv.Itoa(tupleLength)))
return err
})
db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("vars"))
err := b.Put([]byte("wordBuckets"), []byte(strconv.Itoa(wordBuckets)))
return err
})
}
