// Create creates a new file with the provided name in the GridFS. If the file
// name already exists, a new version will be inserted with an up-to-date
// uploadDate that will cause it to be atomically visible to the Open and
// OpenId methods. If the file name is not important, an empty name may be
// provided and the file Id used instead.
//
// It's important to Close files whether they are being written to
// or read from, and to check the err result to ensure the operation
// completed successfully.
//
// A simple example inserting a new file:
//
// func check(err os.Error) {
// if err != nil {
// panic(err.String())
// }
// }
// file, err := db.GridFS("fs").Create("myfile.txt")
// check(err)
// n, err := file.Write([]byte("Hello world!")
// check(err)
// err = file.Close()
// check(err)
// fmt.Printf("%d bytes written\n", n)
//
// The io.Writer interface is implemented by *GridFile and may be used to
// help on the file creation. For example:
//
// file, err := db.GridFS("fs").Create("myfile.txt")
// check(err)
// messages, err := os.Open("/var/log/messages")
// check(err)
// defer messages.Close()
// err = io.Copy(file, messages)
// check(err)
// err = file.Close()
// check(err)
//
func (gfs *GridFS) Create(name string) (file *GridFile, err error) {
file = gfs.newFile()
file.mode = gfsWriting
file.wsum = md5.New()
file.doc = gfsFile{Id: bson.NewObjectId(), ChunkSize: 256 * 1024, Filename: name}
return
}
// UnmarshalJSON turns *dal.ObjectId into a json.Unmarshaller.
func (id *ObjectID) UnmarshalJSON(data []byte) (err error) {
a := bson.NewObjectId()
err = a.UnmarshalJSON(data)
if err != nil {
log.Printf("ERROR: %s\n", err)
return
}
*id = ObjectID(a)
return
}
func (s *S) TestInfo(c *C) {
ops := []txn.Op{{
C: "accounts",
Id: 0,
Assert: txn.DocMissing,
}}
id := bson.NewObjectId()
err := s.runner.Run(ops, id, M{"n": 42})
c.Assert(err, IsNil)
var t struct{ I struct{ N int } }
err = s.tc.FindId(id).One(&t)
c.Assert(err, IsNil)
c.Assert(t.I.N, Equals, 42)
}
func (s *S) TestDirect(c *C) {
session, err := mgo.Dial("localhost:40012?connect=direct")
c.Assert(err, IsNil)
defer session.Close()
// We know that server is a slave.
session.SetMode(mgo.Monotonic, true)
result := &struct{ Host string }{}
err = session.Run("serverStatus", result)
c.Assert(err, IsNil)
c.Assert(strings.HasSuffix(result.Host, ":40012"), Equals, true)
stats := mgo.GetStats()
c.Assert(stats.SocketsAlive, Equals, 1)
c.Assert(stats.SocketsInUse, Equals, 1)
c.Assert(stats.SocketRefs, Equals, 1)
// We've got no master, so it'll timeout.
session.SetSyncTimeout(5e8 * time.Nanosecond)
coll := session.DB("mydb").C("mycoll")
err = coll.Insert(M{"test": 1})
c.Assert(err, ErrorMatches, "no reachable servers")
// Writing to the local database is okay.
coll = session.DB("local").C("mycoll")
defer coll.RemoveAll(nil)
id := bson.NewObjectId()
err = coll.Insert(M{"_id": id})
c.Assert(err, IsNil)
// Data was stored in the right server.
n, err := coll.Find(M{"_id": id}).Count()
c.Assert(err, IsNil)
c.Assert(n, Equals, 1)
// Server hasn't changed.
result.Host = ""
err = session.Run("serverStatus", result)
c.Assert(err, IsNil)
c.Assert(strings.HasSuffix(result.Host, ":40012"), Equals, true)
}
func (s *S) TestQueueStashing(c *C) {
txn.SetChaos(txn.Chaos{
KillChance: 1,
Breakpoint: "set-applying",
})
opses := [][]txn.Op{{{
C: "accounts",
Id: 0,
Insert: M{"balance": 100},
}}, {{
C: "accounts",
Id: 0,
Remove: true,
}}, {{
C: "accounts",
Id: 0,
Insert: M{"balance": 200},
}}, {{
C: "accounts",
Id: 0,
Update: M{"$inc": M{"balance": 100}},
}}}
var last bson.ObjectId
for _, ops := range opses {
last = bson.NewObjectId()
err := s.runner.Run(ops, last, nil)
c.Assert(err, Equals, txn.ErrChaos)
}
txn.SetChaos(txn.Chaos{})
err := s.runner.Resume(last)
c.Assert(err, IsNil)
var account Account
err = s.accounts.FindId(0).One(&account)
c.Assert(err, IsNil)
c.Assert(account.Balance, Equals, 300)
}
func (file *GridFile) insertChunk(data []byte) {
n := file.chunk
file.chunk++
debugf("GridFile %p: adding to checksum: %q", file, string(data))
file.wsum.Write(data)
for file.doc.ChunkSize*file.wpending >= 1024*1024 {
// Hold on.. we got a MB pending.
file.c.Wait()
if file.err != nil {
return
}
}
file.wpending++
debugf("GridFile %p: inserting chunk %d with %d bytes", file, n, len(data))
// We may not own the memory of data, so rather than
// simply copying it, we'll marshal the document ahead of time.
data, err := bson.Marshal(gfsChunk{bson.NewObjectId(), file.doc.Id, n, data})
if err != nil {
file.err = err
return
}
go func() {
err := file.gfs.Chunks.Insert(bson.Raw{Data: data})
file.m.Lock()
file.wpending--
if err != nil && file.err == nil {
file.err = err
}
file.c.Broadcast()
file.m.Unlock()
}()
}
func simulate(c *C, params params) {
seed := *seed
if seed == 0 {
seed = time.Now().UnixNano()
}
rand.Seed(seed)
c.Logf("Seed: %v", seed)
txn.SetChaos(txn.Chaos{
KillChance: params.killChance,
SlowdownChance: params.slowdownChance,
Slowdown: params.slowdown,
})
defer txn.SetChaos(txn.Chaos{})
session, err := mgo.Dial(mgoaddr)
c.Assert(err, IsNil)
defer session.Close()
db := session.DB("test")
tc := db.C("tc")
runner := txn.NewRunner(tc)
tclog := db.C("tc.log")
if params.changelog {
info := mgo.CollectionInfo{
Capped: true,
MaxBytes: 1000000,
}
err := tclog.Create(&info)
c.Assert(err, IsNil)
runner.ChangeLog(tclog)
}
accounts := db.C("accounts")
for i := 0; i < params.accounts; i++ {
err := accounts.Insert(M{"_id": i, "balance": 300})
c.Assert(err, IsNil)
}
var stop time.Time
if params.changes <= 0 {
stop = time.Now().Add(*duration)
}
max := params.accounts
if params.reinsertCopy || params.reinsertZeroed {
max = int(float64(params.accounts) * 1.5)
}
changes := make(chan balanceChange, 1024)
//session.SetMode(mgo.Eventual, true)
for i := 0; i < params.workers; i++ {
go func() {
n := 0
for {
if n > 0 && n == params.changes {
break
}
if !stop.IsZero() && time.Now().After(stop) {
break
}
change := balanceChange{
id: bson.NewObjectId(),
origin: rand.Intn(max),
target: rand.Intn(max),
amount: 100,
}
var old Account
var oldExists bool
if params.reinsertCopy || params.reinsertZeroed {
if err := accounts.FindId(change.origin).One(&old); err != mgo.ErrNotFound {
c.Check(err, IsNil)
change.amount = old.Balance
oldExists = true
}
}
var ops []txn.Op
switch {
case params.reinsertCopy && oldExists:
ops = []txn.Op{{
C: "accounts",
Id: change.origin,
Assert: M{"balance": change.amount},
Remove: true,
}, {
C: "accounts",
Id: change.target,
Assert: txn.DocMissing,
Insert: M{"balance": change.amount},
}}
case params.reinsertZeroed && oldExists:
ops = []txn.Op{{
C: "accounts",
Id: change.target,
Assert: txn.DocMissing,
Insert: M{"balance": 0},
}, {
C: "accounts",
Id: change.origin,
Assert: M{"balance": change.amount},
Remove: true,
}, {
C: "accounts",
Id: change.target,
Assert: txn.DocExists,
Update: M{"$inc": M{"balance": change.amount}},
}}
case params.changeHalf:
ops = []txn.Op{{
C: "accounts",
Id: change.origin,
Assert: M{"balance": M{"$gte": change.amount}},
Update: M{"$inc": M{"balance": -change.amount / 2}},
}, {
C: "accounts",
Id: change.target,
Assert: txn.DocExists,
Update: M{"$inc": M{"balance": change.amount / 2}},
}, {
C: "accounts",
Id: change.origin,
Update: M{"$inc": M{"balance": -change.amount / 2}},
}, {
C: "accounts",
Id: change.target,
Update: M{"$inc": M{"balance": change.amount / 2}},
}}
default:
ops = []txn.Op{{
C: "accounts",
Id: change.origin,
Assert: M{"balance": M{"$gte": change.amount}},
Update: M{"$inc": M{"balance": -change.amount}},
}, {
C: "accounts",
Id: change.target,
Assert: txn.DocExists,
Update: M{"$inc": M{"balance": change.amount}},
}}
}
err = runner.Run(ops, change.id, nil)
if err != nil && err != txn.ErrAborted && err != txn.ErrChaos {
c.Check(err, IsNil)
}
n++
changes <- change
}
changes <- balanceChange{}
}()
}
alive := params.workers
changeLog := make([]balanceChange, 0, 1024)
for alive > 0 {
change := <-changes
if change.id == "" {
alive--
} else {
changeLog = append(changeLog, change)
}
}
c.Check(len(changeLog), Not(Equals), 0, Commentf("No operations were even attempted."))
txn.SetChaos(txn.Chaos{})
err = runner.ResumeAll()
c.Assert(err, IsNil)
n, err := accounts.Count()
c.Check(err, IsNil)
c.Check(n, Equals, params.accounts, Commentf("Number of accounts has changed."))
n, err = accounts.Find(M{"balance": M{"$lt": 0}}).Count()
c.Check(err, IsNil)
c.Check(n, Equals, 0, Commentf("There are %d accounts with negative balance.", n))
globalBalance := 0
iter := accounts.Find(nil).Iter()
account := Account{}
for iter.Next(&account) {
globalBalance += account.Balance
}
c.Check(iter.Close(), IsNil)
c.Check(globalBalance, Equals, params.accounts*300, Commentf("Total amount of money should be constant."))
// Compute and verify the exact final state of all accounts.
balance := make(map[int]int)
for i := 0; i < params.accounts; i++ {
balance[i] += 300
}
var applied, aborted int
for _, change := range changeLog {
err := runner.Resume(change.id)
if err == txn.ErrAborted {
aborted++
continue
} else if err != nil {
c.Fatalf("resuming %s failed: %v", change.id, err)
}
balance[change.origin] -= change.amount
balance[change.target] += change.amount
applied++
}
iter = accounts.Find(nil).Iter()
for iter.Next(&account) {
c.Assert(account.Balance, Equals, balance[account.Id])
}
c.Check(iter.Close(), IsNil)
c.Logf("Total transactions: %d (%d applied, %d aborted)", len(changeLog), applied, aborted)
if params.changelog {
n, err := tclog.Count()
c.Assert(err, IsNil)
// Check if the capped collection is full.
dummy := make([]byte, 1024)
tclog.Insert(M{"_id": bson.NewObjectId(), "dummy": dummy})
m, err := tclog.Count()
c.Assert(err, IsNil)
if m == n+1 {
// Wasn't full, so it must have seen it all.
c.Assert(err, IsNil)
c.Assert(n, Equals, applied)
}
}
}
func (s *S) TestPurgeMissing(c *C) {
txn.SetChaos(txn.Chaos{
KillChance: 1,
Breakpoint: "set-applying",
})
err := s.accounts.Insert(M{"_id": 0, "balance": 100})
c.Assert(err, IsNil)
err = s.accounts.Insert(M{"_id": 1, "balance": 100})
c.Assert(err, IsNil)
ops1 := []txn.Op{{
C: "accounts",
Id: 3,
Insert: M{"balance": 100},
}}
ops2 := []txn.Op{{
C: "accounts",
Id: 0,
Remove: true,
}, {
C: "accounts",
Id: 1,
Update: M{"$inc": M{"balance": 100}},
}, {
C: "accounts",
Id: 2,
Insert: M{"balance": 100},
}}
err = s.runner.Run(ops1, "", nil)
c.Assert(err, Equals, txn.ErrChaos)
last := bson.NewObjectId()
err = s.runner.Run(ops2, last, nil)
c.Assert(err, Equals, txn.ErrChaos)
err = s.tc.RemoveId(last)
c.Assert(err, IsNil)
txn.SetChaos(txn.Chaos{})
err = s.runner.ResumeAll()
c.Assert(err, IsNil)
err = s.runner.Run(ops2, "", nil)
c.Assert(err, ErrorMatches, "cannot find transaction .*")
err = s.runner.PurgeMissing("accounts")
c.Assert(err, IsNil)
err = s.runner.Run(ops2, "", nil)
c.Assert(err, IsNil)
expect := []struct{ Id, Balance int }{
{0, -1},
{1, 200},
{2, 100},
{3, 100},
}
var got Account
for _, want := range expect {
err = s.accounts.FindId(want.Id).One(&got)
if want.Balance == -1 {
if err != mgo.ErrNotFound {
c.Errorf("Account %d should not exist, find got err=%#v", err)
}
} else if err != nil {
c.Errorf("Account %d should have balance of %d, but wasn't found", want.Id, want.Balance)
} else if got.Balance != want.Balance {
c.Errorf("Account %d should have balance of %d, got %d", want.Id, want.Balance, got.Balance)
}
}
}
func (s *S) TestChangeLog(c *C) {
chglog := s.db.C("chglog")
s.runner.ChangeLog(chglog)
ops := []txn.Op{{
C: "debts",
Id: 0,
Assert: txn.DocMissing,
}, {
C: "accounts",
Id: 0,
Insert: M{"balance": 300},
}, {
C: "accounts",
Id: 1,
Insert: M{"balance": 300},
}, {
C: "people",
Id: "joe",
Insert: M{"accounts": []int64{0, 1}},
}}
id := bson.NewObjectId()
err := s.runner.Run(ops, id, nil)
c.Assert(err, IsNil)
type IdList []interface{}
type Log struct {
Docs IdList "d"
Revnos []int64 "r"
}
var m map[string]*Log
err = chglog.FindId(id).One(&m)
c.Assert(err, IsNil)
c.Assert(m["accounts"], DeepEquals, &Log{IdList{0, 1}, []int64{2, 2}})
c.Assert(m["people"], DeepEquals, &Log{IdList{"joe"}, []int64{2}})
c.Assert(m["debts"], IsNil)
ops = []txn.Op{{
C: "accounts",
Id: 0,
Update: M{"$inc": M{"balance": 100}},
}, {
C: "accounts",
Id: 1,
Update: M{"$inc": M{"balance": 100}},
}}
id = bson.NewObjectId()
err = s.runner.Run(ops, id, nil)
c.Assert(err, IsNil)
m = nil
err = chglog.FindId(id).One(&m)
c.Assert(err, IsNil)
c.Assert(m["accounts"], DeepEquals, &Log{IdList{0, 1}, []int64{3, 3}})
c.Assert(m["people"], IsNil)
ops = []txn.Op{{
C: "accounts",
Id: 0,
Remove: true,
}, {
C: "people",
Id: "joe",
Remove: true,
}}
id = bson.NewObjectId()
err = s.runner.Run(ops, id, nil)
c.Assert(err, IsNil)
m = nil
err = chglog.FindId(id).One(&m)
c.Assert(err, IsNil)
c.Assert(m["accounts"], DeepEquals, &Log{IdList{0}, []int64{-4}})
c.Assert(m["people"], DeepEquals, &Log{IdList{"joe"}, []int64{-3}})
}
func NewObjectID() ObjectID {
return ObjectID(bson.NewObjectId())
}