示例#1
0
func streamCollection(s []interface{}) chan ptrval {
	ch := make(chan ptrval)
	go func() {
		defer close(ch)
		t := time.Unix(1347255646, 418514126).UTC()
		for _, r := range s {
			t = t.Add(time.Second)
			ts := t.Format(time.RFC3339Nano)
			ch <- ptrval{gouchstore.NewDocumentInfo(ts), r, true}
		}
		t = t.Add(time.Second)
		ts := t.Format(time.RFC3339Nano)
		ch <- ptrval{gouchstore.NewDocumentInfo(ts), nextValue, false}
	}()
	return ch
}
示例#2
0
func deleteBulk(args []string, w http.ResponseWriter, req *http.Request) {
	// Parse the params

	req.ParseForm()

	compactAfter := strings.ToLower(req.FormValue("compact"))

	from, err := cleanupRangeParam(req.FormValue("from"), "")
	if err != nil {
		emitError(400, w, "Bad from value", err.Error())
		return
	}
	to, err := cleanupRangeParam(req.FormValue("to"), "")
	if err != nil {
		emitError(400, w, "Bad to value", err.Error())
		return
	}

	db, err := dbopen(args[0])
	if err != nil {
		emitError(400, w, "Unable to open Database", err.Error())
		return
	}

	bulk := db.Bulk()
	deleteCount := 0
	commitThreshold := 10000

	err = dbwalkKeys(args[0], from, to, func(k string) error {

		bulk.Delete(gouchstore.NewDocumentInfo(k))
		deleteCount++
		if deleteCount >= commitThreshold {
			bulk.Commit()
			deleteCount = 0
		}
		return err
	})

	if deleteCount > 0 {
		bulk.Commit()
	}

	bulk.Close()

	if compactAfter == "true" {
		err = dbcompact(args[0])
	}

	w.WriteHeader(201)

}
示例#3
0
func TestPairRateConversion(t *testing.T) {
	ch := make(chan ptrval, 2)
	rch := convertTofloat64Rate(ch)

	tm := time.Now().UTC()
	val1 := "20"
	ch <- ptrval{gouchstore.NewDocumentInfo(tm.Format(time.RFC3339Nano)),
		val1, true}

	tm = tm.Add(5 * time.Second)
	val2 := "25"
	ch <- ptrval{gouchstore.NewDocumentInfo(tm.Format(time.RFC3339Nano)),
		val2, false}

	close(ch)
	exp := 1.0
	val, got := <-rch
	if !got {
		t.Fatalf("Expected value, got empty channel")
	}
	if val != exp {
		t.Fatalf("Expected %v, got %v", exp, val)
	}
}
示例#4
0
func TestPointers(t *testing.T) {
	docID := "2013-02-22T16:29:19.750264Z"
	di := gouchstore.NewDocumentInfo(docID)
	tests := []struct {
		pointer string
		exp     interface{}
	}{
		{"/kind", "Listing"},
		{"_id", docID},
	}

	for _, test := range tests {
		chans := make([]chan ptrval, 0, 1)
		chans = append(chans, make(chan ptrval))
		go processDoc(di, chans, bigInput, []string{test.pointer}, []string{}, []string{}, true)
		got := <-chans[0]
		if test.exp != got.val {
			t.Errorf("Expected %v for %v, got %v",
				test.exp, test.pointer, got.val)
		}
	}

}
示例#5
0
func dbWriteLoop(dq *dbWriter) {
	defer dbWg.Done()

	queued := 0
	bulk := dq.db.Bulk()

	t := time.NewTimer(*flushTime)
	defer t.Stop()
	liveTracker := time.NewTicker(*liveTime)
	defer liveTracker.Stop()
	liveOps := 0

	dbst := dbStats.getOrCreate(dq.dbname)
	defer atomic.StoreUint32(&dbst.qlen, 0)
	defer atomic.AddUint32(&dbst.closes, 1)

	for {
		atomic.StoreUint32(&dbst.qlen, uint32(queued))

		select {
		case <-dq.quit:
			sdt := time.Now()
			bulk.Commit()
			bulk.Close()
			closeDBConn(dq.db)
			dbRemoveConn(dq.dbname)
			log.Printf("Closed %v with %v items in %v",
				dq.dbname, queued, time.Since(sdt))
			return
		case <-liveTracker.C:
			if queued == 0 && liveOps == 0 {
				log.Printf("Closing idle DB: %v", dq.dbname)
				close(dq.quit)
			}
			liveOps = 0
		case qi := <-dq.ch:
			liveOps++
			switch qi.op {
			case opStoreItem:
				bulk.Set(gouchstore.NewDocumentInfo(qi.k),
					gouchstore.NewDocument(qi.k, qi.data))
				queued++
			case opDeleteItem:
				queued++
				bulk.Delete(gouchstore.NewDocumentInfo(qi.k))
			case opCompact:
				var err error
				bulk, err = dbCompact(dq, bulk, queued, qi)
				qi.cherr <- err
				atomic.AddUint64(&dbst.written, uint64(queued))
				queued = 0
			default:
				log.Panicf("Unhandled case: %v", qi.op)
			}
			if queued >= *maxOpQueue {
				start := time.Now()
				bulk.Commit()
				if *verbose {
					log.Printf("Flush of %d items took %v",
						queued, time.Since(start))
				}
				atomic.AddUint64(&dbst.written, uint64(queued))
				queued = 0
				t.Reset(*flushTime)
			}
		case <-t.C:
			if queued > 0 {
				start := time.Now()
				bulk.Commit()
				if *verbose {
					log.Printf("Flush of %d items from timer took %v",
						queued, time.Since(start))
				}
				atomic.AddUint64(&dbst.written, uint64(queued))
				queued = 0
			}
			t.Reset(*flushTime)
		}
	}
}
示例#6
0
文件: main.go 项目: ngaut/gouchstore
func main() {

	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("Must specify path to a use")
		return
	}
	db, err := gouchstore.Open(flag.Args()[0], gouchstore.OPEN_CREATE)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	var docCount uint64 = 0
	var deletedCount uint64 = 0
	nextDocId := 0
	docs := make(map[string]*gouchstore.Document)
	deletedDocs := make(map[string]bool)
	docInfos := make(map[string]*gouchstore.DocumentInfo)
	docKeys := make([]string, 0)
	docsBySeq := make(map[uint64]*gouchstore.Document)
	deletedDocsBySeq := make(map[uint64]bool)
	// iterate through this many operations
	for i := 0; i < *numOps; i++ {
		operation := rand.Intn(100)
		if operation < 70 {
			// add doc
			docId := "doc-" + strconv.Itoa(nextDocId)
			doc := &gouchstore.Document{
				ID:   docId,
				Body: []byte(`{"content":123,"filler":"hf7BB83mKJ5APQ0OBqrj1wnYw2mjArEFmkuPaiNIYbBa4cMkOdI/JuTsGZTddqW5cxLYT1md1kWPxwuNUYliPAcrY/eoibeOMJ/3HaBUu88S/unTrR/P8iVMR1RcV0jyVzw475E3ckff8AqomvglH3ij7gsJyrAMmF2TjLlJU2EURrp+Sq5sM3bboeHf7iyuG+LnUL1hKbYju+yxC7AaoWDoiUKB2KlrZrcKwolFQnTKkZcSMEuI9sXsUvdaoq+1MNMsFKnnlQsuricwTTfcFIFQFHlyGrV+ghrljAl7RY6kxNpPtFft8s0x8xYR8Po8SxTLgA/YNEa2ERbHPUCdheAZZ3MDMSeBlv9Sy7UOCgKVH1tW8S9kidJQLwnYoDcDlz1HCxrvvakdyrYPoeohZ5Ub44xo1/2hhO0Pypk11dWbTqO8MV/3Z+BIQWdVWbvx7zrLSG+kzxh1tMBnrluuZWXjT5w/+ZUSBnLU3Ru17TpIGu5I8pp4Xh2vjWgbRsdKHGga7li5tqtO5fb59rhA+8q+lLlBTcL1AR1oZiPL/O7O+2jqHE10gE7CO4GFqfX/iM4LhrxzcnPd3taeiVGp3/zTx1JJKiYn9H8yn7e4/KkpNObag7k+bqrpE64GPWOHvlYXwm7L8kVIesaBfZEwenlOWJgK73IleBzhoYVt6iKgS+MPZqNvQBN/ndXXffoyUoM9dbiV/dYEt7pmP14N3UVqJh3N/7Kz3bXrrQH0msmso1FnfkoZTUnPEK3toXgnub1dEPkbmJnwbiXBaNhHc7KPBnMYH7HlVWN4h7R0tH9u0zgAzemdBkARViJidkYhieig5K/31OfVSVFjbqbdWNtanAU1kSzcqX+O9+3H3kFQxB8Ez7Lo08Zbx2IojmjmVnJUH+oNs0qhYwlrmhH1lK2/TOIdaEASnYsANTxnlJvYWpSbNfpk2jXcyBQCvhaIOO2E8Hq+eRhdMtdqLUwoOtI8+Q0lJVaCxuM4RhVEtvBm7frqC3fm5fCJjYmjNOAF6QGWhRWnkeaEQgA0j40DbvRQypSPGr2y6mLOpsCeELtxXKpm929CngxLzWRsGZWvmCOZNtpH6zqHGmn5PTcm2KyTC1jhoD3maicGvGF/S7MJI5Pftuzp8GVx+EieqONwPZXln6g0L66Md2V8igH+Z134RXMN4mk2j6MGhZiZ4oUokbBZ9WR/QvGUGdxzoyfFnGt1Bh+9uUaydq1l6Lk1OHH3XrRdkexMPT2CFEpX+0JonOtNwaXTmdbclFjJIA9ObfNww0iBfxQ7TZYZSv4O0mHIBXcCkhM2/kqlFbomL3/eJXhJhDaGsc+KoVBQCU4x9wydSiiXGFmskX5gWV37bA=="}`),
			}
			nextDocId++
			docInfo := gouchstore.NewDocumentInfo(docId)
			docInfo.Rev = 1
			err = db.SaveDocument(doc, docInfo)
			if err != nil {
				fmt.Printf("err: %v\n", err)
				return
			}
			docs[docId] = doc
			docsBySeq[docInfo.Seq] = doc
			docInfos[docId] = docInfo
			docKeys = append(docKeys, docId)
			docCount++
		} else if operation < 90 {
			// update doc
			if len(docKeys) > 0 {
				docIdIndexToUpdate := rand.Intn(len(docKeys))
				docIdToUpdate := docKeys[docIdIndexToUpdate]

				// get the old seq
				oldSeq := docInfos[docIdToUpdate].Seq

				// bump the rev of this doc
				docInfos[docIdToUpdate].Rev++
				// now udpate it
				err = db.SaveDocument(docs[docIdToUpdate], docInfos[docIdToUpdate])
				if err != nil {
					fmt.Printf("err: %v\n", err)
					return
				}
				delete(docsBySeq, oldSeq)
				docsBySeq[docInfos[docIdToUpdate].Seq] = docs[docIdToUpdate]
			} else {
				continue
			}
		} else {
			// delete doc
			if len(docKeys) > 0 {
				docIdIndexToDelete := rand.Intn(len(docKeys))
				docIdToDelete := docKeys[docIdIndexToDelete]
				// get the old seq
				oldSeq := docInfos[docIdToDelete].Seq
				err = db.SaveDocument(nil, docInfos[docIdToDelete])
				if err != nil {
					fmt.Printf("err: %v\n", err)
					return
				}
				oldInfo := docInfos[docIdToDelete]
				docKeys = append(docKeys[:docIdIndexToDelete], docKeys[docIdIndexToDelete+1:]...)
				delete(docs, docIdToDelete)
				deletedDocs[docIdToDelete] = true
				delete(docInfos, docIdToDelete)
				docCount--
				deletedCount++
				delete(docsBySeq, oldSeq)
				deletedDocsBySeq[oldInfo.Seq] = true
			} else {
				continue
			}
		}

		if i%*commitEvery == 0 {
			err = db.Commit()
			if err != nil {
				fmt.Printf("error committing %d: %v\n", i, err)
				return
			}
		}

		// final commit
		if *commitAtEnd {
			err = db.Commit()
			if err != nil {
				fmt.Printf("error committing end: %v\n", err)
				return
			}
		}

	}
}