func Count(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "must-revalidate")
w.Header().Set("Content-Type", "text/plain")
var col, q string
if !Require(w, r, "col", &col) {
return
}
if !Require(w, r, "q", &q) {
return
}
var qJson interface{}
if err := json.Unmarshal([]byte(q), &qJson); err != nil {
http.Error(w, fmt.Sprintf("'%v' is not valid JSON.", q), 400)
return
}
V3Sync.RLock()
defer V3Sync.RUnlock()
dbcol := V3DB.Use(col)
if dbcol == nil {
http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
return
}
queryResult := make(map[uint64]struct{})
if err := db.EvalQueryV2(qJson, dbcol, &queryResult); err != nil {
http.Error(w, fmt.Sprint(err), 400)
return
}
w.Write([]byte(strconv.Itoa(len(queryResult))))
}
func Query(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "must-revalidate")
w.Header().Set("Content-Type", "text/plain")
var col, q string
if !Require(w, r, "col", &col) {
return
}
if !Require(w, r, "q", &q) {
return
}
var qJson interface{}
if err := json.Unmarshal([]byte(q), &qJson); err != nil {
http.Error(w, fmt.Sprintf("'%v' is not valid JSON.", q), 400)
return
}
V3Sync.RLock()
defer V3Sync.RUnlock()
dbcol := V3DB.Use(col)
if dbcol == nil {
http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
return
}
// evaluate the query
queryResult := make(map[uint64]struct{})
if err := db.EvalQueryV2(qJson, dbcol, &queryResult); err != nil {
http.Error(w, fmt.Sprint(err), 400)
return
}
// write each document on a new line
for k := range queryResult {
var doc interface{}
dbcol.Read(k, &doc)
if doc == nil {
continue
}
resp, err := json.Marshal(doc)
if err != nil {
tdlog.Printf("Query returned invalid JSON '%v'", doc)
continue
}
w.Write([]byte(string(resp) + "\r\n"))
}
}
func embeddedExample() {
// ****************** Collection Management ******************
// Create and open database
dir := "/tmp/MyDatabase"
os.RemoveAll(dir)
defer os.RemoveAll(dir)
myDB, err := db.OpenDB(dir)
if err != nil {
panic(err)
}
// Create two collections Feeds and Votes
if err := myDB.Create("Feeds"); err != nil {
panic(err)
}
if err := myDB.Create("Votes"); err != nil {
panic(err)
}
// What collections do I now have?
for name := range myDB.StrCol {
fmt.Printf("I have a collection called %s\n", name)
}
// Rename collection "Votes" to "Points"
if err := myDB.Rename("Votes", "Points"); err != nil {
panic(err)
}
// Drop (delete) collection "Points"
if err := myDB.Drop("Points"); err != nil {
panic(err)
}
// Scrub (repair and compact) "Feeds"
if err := myDB.Scrub("Feeds"); err != nil {
panic(err)
}
// ****************** Document Management ******************
// Start using a collection
feeds := myDB.Use("Feeds")
// Insert document (document must be map[string]interface{})
docID, err := feeds.Insert(map[string]interface{}{
"name": "Go 1.2 is released",
"url": "golang.org"})
if err != nil {
panic(err)
}
// Read document
var readBack interface{}
feeds.Read(docID, &readBack) // pass in document's physical ID
fmt.Println(readBack)
// Update document (document must be map[string]interface{})
newID, err := feeds.Update(docID, map[string]interface{}{
"name": "Go is very popular",
"url": "google.com"})
if err != nil {
panic(err)
}
// Delete document
feeds.Delete(newID)
// Delete document
feeds.Delete(123) // An ID which does not exist does no harm
// ****************** Index Management ******************
// Create index (path leads to document JSON attribute)
if err := feeds.Index([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// What indexes do I have on collection A?
for path := range feeds.StrHT {
fmt.Printf("I have an index on path %s\n", path)
}
// Remove index
if err := feeds.Unindex([]string{"author", "name", "first_name"}); err != nil {
panic(err)
}
// ****************** Queries ******************
// Let's prepare a number of docments for a start
feeds.Insert(map[string]interface{}{"Title": "New Go release", "Source": "golang.org", "Age": 3})
feeds.Insert(map[string]interface{}{"Title": "Kitkat is here", "Source": "google.com", "Age": 2})
feeds.Insert(map[string]interface{}{"Title": "Good Slackware", "Source": "slackware.com", "Age": 1})
// There are two ways to construct query - by deserializing tiedot query string
// Or construct a Golang map[string]interface{} (essentially, deserialized JSON)
queryStr := `[{"eq": "New Go release", "in": ["Title"]}, {"eq": "slackware.com", "in": ["Source"]}]`
var query interface{}
json.Unmarshal([]byte(queryStr), &query)
queryResult := make(map[uint64]struct{}) // query result goes into map keys
if err := db.EvalQueryV2(query, feeds, &queryResult); err != nil {
panic(err)
}
// Query results are physical document IDs
for id := range queryResult {
fmt.Printf("Query returned document ID %d\n", id)
}
// To use the document itself, simply read it back
for id := range queryResult {
feeds.Read(id, &readBack)
fmt.Printf("Query returned document %v\n", readBack)
}
// Gracefully close database
myDB.Close()
}
// Benchmark document insert, read, query, update and delete.
func benchmark(benchSize int) {
docs := make([]interface{}, benchSize)
ids := make([]uint64, 0)
// Prepare serialized documents to be inserted
for i := range docs {
if err := json.Unmarshal([]byte(
`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(benchSize))+`}},`+
`"c": {"d": `+strconv.Itoa(rand.Intn(benchSize))+`},`+
`"more": "abcdefghijklmnopqrstuvwxyz"}`), &docs[i]); err != nil {
panic("json error")
}
}
// Prepare a collection with two indexes
tmp := "/tmp/tiedot_bench"
os.RemoveAll(tmp)
defer os.RemoveAll(tmp)
col, err := db.OpenCol(tmp)
if err != nil {
panic(err)
}
col.Index([]string{"a", "b", "c"})
col.Index([]string{"c", "d"})
// Benchmark document insert
average("insert", benchSize, func() {}, func() {
if _, err := col.Insert(docs[rand.Intn(benchSize)]); err != nil {
panic("insert error")
}
})
// Collect all document IDs and benchmark document read
average("read", benchSize, func() {
col.ForAll(func(id uint64, doc interface{}) bool {
ids = append(ids, id)
return true
})
}, func() {
var doc interface{}
col.Read(ids[rand.Intn(benchSize)], &doc)
if doc == nil {
panic("read error")
}
})
// Benchmark lookup query (two attributes)
average("lookup", benchSize, func() {}, func() {
var query interface{}
if err := json.Unmarshal([]byte(`{"c": [{"eq": `+strconv.Itoa(rand.Intn(benchSize))+`, "in": ["a", "b", "c"], "limit": 1}, `+
`{"eq": `+strconv.Itoa(rand.Intn(benchSize))+`, "in": ["c", "d"], "limit": 1}]}`), &query); err != nil {
panic("json error")
}
result := make(map[uint64]struct{})
if err := db.EvalQueryV2(query, col, &result); err != nil {
panic("query error")
}
})
// Benchmark document update
average("update", benchSize, func() {}, func() {
if _, err := col.Update(ids[rand.Intn(benchSize)], docs[rand.Intn(benchSize)]); err != nil {
panic("update error")
}
})
// Benchmark document delete
average("delete", benchSize, func() {}, func() {
col.Delete(ids[rand.Intn(benchSize)])
})
col.Close()
}
// Run document opearations (insert, read, query, update and delete) all at once.
func benchmark2(benchSize int) {
docs := make([]uint64, 0, benchSize*2+1000)
wp := new(sync.WaitGroup)
numThreads := runtime.GOMAXPROCS(-1)
wp.Add(5 * numThreads) // There are 5 goroutines: insert, read, query, update and delete
// Prepare a collection with two indexes
tmp := "/tmp/tiedot_bench"
os.RemoveAll(tmp)
col, err := db.OpenCol(tmp)
if err != nil {
panic(err)
}
col.Index([]string{"a", "b", "c"})
col.Index([]string{"c", "d"})
// Insert 1000 documents to make a start
var docToInsert interface{}
for j := 0; j < 1000; j++ {
if err = json.Unmarshal([]byte(
`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(benchSize))+`}},`+
`"c": {"d": `+strconv.Itoa(rand.Intn(benchSize))+`},`+
`"more": "abcdefghijklmnopqrstuvwxyz"}`), &docToInsert); err != nil {
panic(err)
}
if newID, err := col.Insert(docToInsert); err == nil {
docs = append(docs, newID)
} else {
panic(err)
}
}
start := float64(time.Now().UTC().UnixNano())
// Insert benchSize * 2 documents
for i := 0; i < numThreads; i++ {
go func(i int) {
fmt.Printf("Insert thread %d starting\n", i)
defer wp.Done()
var docToInsert interface{}
var err error
for j := 0; j < benchSize/numThreads*2; j++ {
if err = json.Unmarshal([]byte(
`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(benchSize))+`}},`+
`"c": {"d": `+strconv.Itoa(rand.Intn(benchSize))+`},`+
`"more": "abcdefghijklmnopqrstuvwxyz"}`), &docToInsert); err != nil {
panic(err)
}
if newID, err := col.Insert(docToInsert); err == nil {
docs = append(docs, newID)
} else {
panic(err)
}
}
fmt.Printf("Insert thread %d completed\n", i)
}(i)
}
// Read benchSize * 2 documents
for i := 0; i < numThreads; i++ {
go func(i int) {
fmt.Printf("Read thread %d starting\n", i)
defer wp.Done()
var doc interface{}
for j := 0; j < benchSize/numThreads*2; j++ {
col.Read(docs[uint64(rand.Intn(len(docs)))], &doc)
}
fmt.Printf("Read thread %d completed\n", i)
}(i)
}
// Query benchSize times (lookup on two attributes)
for i := 0; i < numThreads; i++ {
go func(i int) {
fmt.Printf("Query thread %d starting\n", i)
defer wp.Done()
var query interface{}
var err error
for j := 0; j < benchSize/numThreads; j++ {
if err = json.Unmarshal([]byte(`{"c": [{"eq": `+strconv.Itoa(rand.Intn(benchSize))+`, "in": ["a", "b", "c"], "limit": 1}, `+
`{"eq": `+strconv.Itoa(rand.Intn(benchSize))+`, "in": ["c", "d"], "limit": 1}]}`), &query); err != nil {
panic("json error")
}
result := make(map[uint64]struct{})
if err = db.EvalQueryV2(query, col, &result); err != nil {
panic("query error")
}
}
fmt.Printf("Query thread %d completed\n", i)
}(i)
}
// Update benchSize documents
for i := 0; i < numThreads; i++ {
go func(i int) {
fmt.Printf("Update thread %d starting\n", i)
defer wp.Done()
var updated interface{}
var err error
for j := 0; j < benchSize/numThreads; j++ {
if err = json.Unmarshal([]byte(
`{"a": {"b": {"c": `+strconv.Itoa(rand.Intn(benchSize))+`}},`+
`"c": {"d": `+strconv.Itoa(rand.Intn(benchSize))+`},`+
`"more": "abcdefghijklmnopqrstuvwxyz"}`), &updated); err != nil {
panic(err)
}
if _, err = col.Update(docs[uint64(rand.Intn(len(docs)))], updated); err != nil {
// "does not exist" indicates that a deleted document is being updated, it is safe to ignore
if !strings.Contains(fmt.Sprint(err), "does not exist") {
fmt.Println(err)
}
}
}
fmt.Printf("Update thread %d completed\n", i)
}(i)
}
// Delete benchSize documents
for i := 0; i < numThreads; i++ {
go func(i int) {
fmt.Printf("Delete thread %d starting\n", i)
defer wp.Done()
for j := 0; j < benchSize/numThreads; j++ {
col.Delete(docs[uint64(rand.Intn(len(docs)))])
}
fmt.Printf("Delete thread %d completed\n", i)
}(i)
}
// Wait for all goroutines to finish, then print summary
wp.Wait()
end := float64(time.Now().UTC().UnixNano())
fmt.Printf("Total operations %d: %d ns/iter, %d iter/sec\n", benchSize*7, int((end-start)/float64(benchSize)/7), int(1000000000/((end-start)/float64(benchSize)/7)))
}
