func (db *DB) applyTransactionInternal(tx DBTransaction, writeTxLogFlag bool) (TxID, error) { logger.Debugf(mylog, "applyTransactionInternal(%+v, writeTxLog: %t)", tx, writeTxLogFlag) if tx.TxID == AnyVersion { tx.TxID = db.state.version + 1 } else if tx.TxID != db.state.version+1 { return 0, fmt.Errorf("Attempted to apply tx %d to dbver %d. Next accepted tx is %d", tx.TxID, db.state.version, db.state.version+1) } for _, op := range tx.Ops { if err := op.Apply(db.state); err != nil { if rerr := db.RestoreVersion(db.state.version); rerr != nil { logger.Panicf(mylog, "Following Error: %v. DB rollback failed!!!: %v", err, rerr) } return 0, err } } if writeTxLogFlag == writeTxLog { if err := db.txLogIO.AppendTransaction(tx); err != nil { if rerr := db.RestoreVersion(db.state.version); rerr != nil { logger.Panicf(mylog, "Failed to write txlog: %v. DB rollback failed!!!: %v", err, rerr) } return 0, fmt.Errorf("Failed to write txlog: %v", err) } } db.state.version = tx.TxID db.stats.LastTx = time.Now() return tx.TxID, nil }
func GenHostName() string { hostname, err := os.Hostname() if err != nil { logger.Panicf(mylog, "Failed to query local hostname: %v", err) } pid := os.Getpid() return fmt.Sprintf("%s-%d", hostname, pid) }
func (h FileBlobHandle) Size() int64 { fi, err := h.Fp.Stat() if err != nil { logger.Panicf(mylog, "Stat failed: %v", err) } return fi.Size() }
func (h ChunkHeader) WriteTo(w io.Writer, c btncrypt.Cipher) error { h.FrameEncapsulation = CurrentFrameEncapsulation if h.PayloadLen > MaxChunkPayloadLen { return fmt.Errorf("payload length too big: %d", h.PayloadLen) } if len(h.OrigFilename) > MaxOrigFilenameLen { h.OrigFilename = filepath.Base(h.OrigFilename) if len(h.OrigFilename) > MaxOrigFilenameLen { h.OrigFilename = "<filename_too_long>" } } if _, err := w.Write([]byte{ChunkSignatureMagic1, ChunkSignatureMagic2}); err != nil { return fmt.Errorf("Failed to write signature magic: %v", err) } if _, err := w.Write([]byte{CurrentFormat}); err != nil { return fmt.Errorf("Failed to write format byte: %v", err) } var b bytes.Buffer enc := gob.NewEncoder(&b) if err := enc.Encode(h); err != nil { return err } framelen := ChunkHeaderLength - c.FrameOverhead() - SignatureLength - 1 paddinglen := framelen - b.Len() if paddinglen < 0 { logger.Panicf(mylog, "SHOULD NOT BE REACHED: Marshaled ChunkHeader size too large") } bew, err := btncrypt.NewWriteCloser(w, c, framelen) if _, err := b.WriteTo(bew); err != nil { return fmt.Errorf("Failed to initialize frame encryptor: %v", err) } if err != nil { return fmt.Errorf("Header frame gob payload write failed: %v", err) } // zero padding if _, err := bew.Write(make([]byte, paddinglen)); err != nil { return fmt.Errorf("Header frame zero padding write failed: %v", err) } if err := bew.Close(); err != nil { return fmt.Errorf("Header frame close failed: %v", err) } return nil }
func (be *CachedBlobEntry) markDirtyWithLock() { now := time.Now() be.lastUsed = now be.lastWrite = now if be.state == cacheEntryDirty { return } if be.state != cacheEntryClean { logger.Panicf(mylog, "markDirty called from unexpected state: %+v", be.infoWithLock()) } be.state = cacheEntryDirty if be.lastSync.IsZero() { be.lastSync = time.Now() } }
func (f *frameEncryptor) Sync() ([]byte, error) { if f.Written() > BtnFrameMaxPayload { return nil, fmt.Errorf("frame payload size exceeding max len: %d > %d", f.Written(), BtnFrameMaxPayload) } nonce := util.RandomBytes(f.c.gcm.NonceSize()) f.encrypted = f.encrypted[:len(nonce)] copy(f.encrypted, nonce) f.encrypted = f.c.gcm.Seal(f.encrypted, nonce, f.b.Bytes(), nil) if len(f.encrypted) != f.c.EncryptedFrameSize(f.Written()) { logger.Panicf(mylog, "EncryptedFrameSize mismatch. expected: %d, actual: %v", f.c.EncryptedFrameSize(f.Written()), len(f.encrypted)) } f.b.Reset() return f.encrypted, nil }
func NewOtaru(cfg *Config, oneshotcfg *OneshotConfig) (*Otaru, error) { o := &Otaru{} var err error key := btncrypt.KeyFromPassword(cfg.Password) o.C, err = btncrypt.NewCipher(key) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init Cipher: %v", err) } o.S = scheduler.NewScheduler() if !cfg.LocalDebug { o.Tsrc, err = auth.GetGCloudTokenSource(context.TODO(), cfg.CredentialsFilePath, cfg.TokenCacheFilePath, false) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init GCloudClientSource: %v", err) } o.DSCfg = datastore.NewConfig(cfg.ProjectName, cfg.BucketName, o.C, o.Tsrc) o.GL = datastore.NewGlobalLocker(o.DSCfg, GenHostName(), "FIXME: fill info") if err := o.GL.Lock(); err != nil { return nil, err } } o.CacheTgtBS, err = blobstore.NewFileBlobStore(cfg.CacheDir, oflags.O_RDWRCREATE) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init FileBlobStore: %v", err) } if !cfg.LocalDebug { o.DefaultBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, cfg.BucketName, o.Tsrc, oflags.O_RDWRCREATE) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init GCSBlobStore: %v", err) } if !cfg.UseSeparateBucketForMetadata { o.BackendBS = o.DefaultBS } else { metabucketname := fmt.Sprintf("%s-meta", cfg.BucketName) o.MetadataBS, err = gcs.NewGCSBlobStore(cfg.ProjectName, metabucketname, o.Tsrc, oflags.O_RDWRCREATE) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init GCSBlobStore (metadata): %v", err) } o.BackendBS = blobstore.Mux{ blobstore.MuxEntry{metadata.IsMetadataBlobpath, o.MetadataBS}, blobstore.MuxEntry{nil, o.DefaultBS}, } } } else { o.BackendBS, err = blobstore.NewFileBlobStore(path.Join(os.Getenv("HOME"), ".otaru", "bbs"), oflags.O_RDWRCREATE) } queryFn := chunkstore.NewQueryChunkVersion(o.C) o.CBS, err = cachedblobstore.New(o.BackendBS, o.CacheTgtBS, o.S, oflags.O_RDWRCREATE /* FIXME */, queryFn) if err != nil { o.Close() return nil, fmt.Errorf("Failed to init CachedBlobStore: %v", err) } if err := o.CBS.RestoreState(o.C); err != nil { logger.Warningf(mylog, "Attempted to restore cachedblobstore state but failed: %v", err) } o.CSS = cachedblobstore.NewCacheSyncScheduler(o.CBS) if !cfg.LocalDebug { o.SSLoc = datastore.NewINodeDBSSLocator(o.DSCfg) } else { logger.Panicf(mylog, "Implement mock sslocator that doesn't depend on gcloud/datastore") } o.SIO = blobstoredbstatesnapshotio.New(o.CBS, o.C, o.SSLoc) if !cfg.LocalDebug { txio := datastore.NewDBTransactionLogIO(o.DSCfg) o.TxIO = txio o.TxIOSS = util.NewSyncScheduler(txio, 300*time.Millisecond) } else { o.TxIO = inodedb.NewSimpleDBTransactionLogIO() } o.CTxIO = inodedb.NewCachedDBTransactionLogIO(o.TxIO) if oneshotcfg.Mkfs { o.IDBBE, err = inodedb.NewEmptyDB(o.SIO, o.CTxIO) if err != nil { o.Close() return nil, fmt.Errorf("NewEmptyDB failed: %v", err) } } else { o.IDBBE, err = inodedb.NewDB(o.SIO, o.CTxIO) if err != nil { o.Close() return nil, fmt.Errorf("NewDB failed: %v", err) } } o.IDBS = inodedb.NewDBService(o.IDBBE) o.IDBSS = util.NewSyncScheduler(o.IDBS, 30*time.Second) o.FS = otaru.NewFileSystem(o.IDBS, o.CBS, o.C) o.MGMT = mgmt.NewServer() if err := o.runMgmtServer(); err != nil { o.Close() return nil, fmt.Errorf("Mgmt server run failed: %v", err) } return o, nil }