func Install(srv *mgmt.Server) {
rtr := srv.APIRouter().PathPrefix("/logger").Subrouter()
rtr.HandleFunc("/categories", mgmt.JSONHandler(func(req *http.Request) interface{} {
return logger.Registry().Categories()
}))
rtr.HandleFunc("/category/{cat:\\w+}", mgmt.JSONHandler(func(req *http.Request) interface{} {
vars := mux.Vars(req)
cat := vars["cat"]
cl := logger.Registry().Category(cat)
if cl == nil {
return fmt.Errorf("Cateogory not found")
}
if req.Method == "GET" {
return cl.View()
} else if req.Method == "POST" {
levelp := req.URL.Query().Get("level")
nlevel, err := strconv.ParseUint(levelp, 10, 32)
if err != nil {
return fmt.Errorf("Failed to parse level")
}
cl.Level = logger.Level(nlevel)
return cl.View()
} else {
return fmt.Errorf("Unknown method!")
}
}))
}
func SetupFluentLogger(cfg *Config) error {
if cfg.Fluent.FluentHost == "" {
logger.Infof(mylog, "The fluentd host is not specified. Skipping fluent logger instantiation.")
return nil
}
logger.Infof(mylog, "Initializing fluent logger based on config: %+v", cfg.Fluent)
fcli, err := gfluent.New(cfg.Fluent)
if err != nil {
return err
}
logger.Registry().AddOutput(fluent.FluentLogger{fcli})
return nil
}
func main() {
log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
logger.Registry().AddOutput(logger.WriterLogger{os.Stderr})
flag.Usage = Usage
flag.Parse()
cfg, err := facade.NewConfig(*flagConfigDir)
if err != nil {
logger.Criticalf(mylog, "%v", err)
Usage()
os.Exit(2)
}
if flag.NArg() != 1 {
Usage()
os.Exit(2)
}
mountpoint := flag.Arg(0)
if err := facade.SetupFluentLogger(cfg); err != nil {
logger.Criticalf(mylog, "Failed to setup fluentd logger: %v", err)
os.Exit(1)
}
o, err := facade.NewOtaru(cfg, &facade.OneshotConfig{Mkfs: *flagMkfs})
if err != nil {
logger.Criticalf(mylog, "NewOtaru failed: %v", err)
os.Exit(1)
}
var muClose sync.Mutex
closeOtaruAndExit := func(exitCode int) {
muClose.Lock()
defer muClose.Unlock()
if err := bfuse.Unmount(mountpoint); err != nil {
logger.Warningf(mylog, "umount err: %v", err)
}
if o != nil {
if err := o.Close(); err != nil {
logger.Warningf(mylog, "Otaru.Close() returned errs: %v", err)
}
o = nil
}
os.Exit(exitCode)
}
defer closeOtaruAndExit(0)
sigC := make(chan os.Signal, 1)
signal.Notify(sigC, os.Interrupt)
signal.Notify(sigC, syscall.SIGTERM)
go func() {
for s := range sigC {
logger.Warningf(mylog, "Received signal: %v", s)
closeOtaruAndExit(1)
}
}()
logger.Registry().AddOutput(logger.HandleCritical(func() {
logger.Warningf(mylog, "Starting shutdown due to critical event.")
closeOtaruAndExit(1)
}))
bfuseLogger := logger.Registry().Category("bfuse")
bfuse.Debug = func(msg interface{}) { logger.Debugf(bfuseLogger, "%v", msg) }
if err := fuse.ServeFUSE(cfg.BucketName, mountpoint, o.FS, nil); err != nil {
logger.Warningf(mylog, "ServeFUSE failed: %v", err)
closeOtaruAndExit(1)
}
logger.Infof(mylog, "ServeFUSE end!")
}
"flag"
"fmt"
"log"
"os"
"os/signal"
"sync"
"syscall"
bfuse "bazil.org/fuse"
"github.com/nyaxt/otaru/facade"
"github.com/nyaxt/otaru/fuse"
"github.com/nyaxt/otaru/logger"
)
var mylog = logger.Registry().Category("cli")
var Usage = func() {
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s MOUNTPOINT\n", os.Args[0])
flag.PrintDefaults()
}
var (
flagMkfs = flag.Bool("mkfs", false, "Reset metadata if no existing metadata exists")
flagConfigDir = flag.String("configDir", facade.DefaultConfigDir(), "Config dirpath")
)
func main() {
log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
logger.Registry().AddOutput(logger.WriterLogger{os.Stderr})
import (
"fmt"
"io"
"os"
"path"
"syscall"
"time"
"github.com/dustin/go-humanize"
fl "github.com/nyaxt/otaru/flags"
"github.com/nyaxt/otaru/logger"
)
var mylog = logger.Registry().Category("filebs")
const (
ENOENT = syscall.Errno(syscall.ENOENT)
EPERM = syscall.Errno(syscall.EPERM)
)
type FileBlobHandle struct {
Fp *os.File
}
func (h FileBlobHandle) PRead(p []byte, offset int64) error {
if _, err := h.Fp.Seek(offset, os.SEEK_SET); err != nil {
return err
}
if _, err := io.ReadFull(h.Fp, p); err != nil {
package datastore
import (
"errors"
"time"
"golang.org/x/net/context"
"google.golang.org/cloud/datastore"
gcutil "github.com/nyaxt/otaru/gcloud/util"
"github.com/nyaxt/otaru/logger"
)
var sslog = logger.Registry().Category("inodedbsslocator")
const kindINodeDBSS = "OtaruINodeDBSS"
var EEMPTY = errors.New("Failed to find any snapshot location entry.")
type INodeDBSSLocator struct {
cfg *Config
rootKey *datastore.Key
}
func NewINodeDBSSLocator(cfg *Config) *INodeDBSSLocator {
return &INodeDBSSLocator{
cfg: cfg,
rootKey: datastore.NewKey(ctxNoNamespace, kindINodeDBSS, cfg.rootKeyStr, 0, nil),
}
}
package chunkstore
import (
"fmt"
"io"
"math"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/blobstore/version"
"github.com/nyaxt/otaru/btncrypt"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("chunkstore")
const (
ContentFramePayloadLength = btncrypt.BtnFrameMaxPayload
)
var (
ZeroContent = make([]byte, ContentFramePayloadLength)
)
func NewQueryChunkVersion(c btncrypt.Cipher) version.QueryFunc {
return func(r io.Reader) (version.Version, error) {
var h ChunkHeader
if err := h.ReadFrom(r, c); err != nil {
if err == io.EOF {
return 0, nil
}
package otaru
import (
"math"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/intn"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/util"
)
var wclog = logger.Registry().Category("filewritecache")
type FileWriteCache struct {
ps intn.Patches
}
func NewFileWriteCache() *FileWriteCache {
return &FileWriteCache{ps: intn.NewPatches()}
}
func (wc *FileWriteCache) PWrite(p []byte, offset int64) error {
pcopy := make([]byte, len(p))
copy(pcopy, p)
newp := intn.Patch{Offset: offset, P: pcopy}
logger.Debugf(wclog, "PWrite: %v", newp)
// logger.Debugf(wclog, "PWrite: p=%v", pcopy)
wc.ps = wc.ps.Merge(newp)
return nil
"syscall"
"time"
"github.com/dustin/go-humanize"
"golang.org/x/net/context"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/blobstore/version"
"github.com/nyaxt/otaru/btncrypt"
fl "github.com/nyaxt/otaru/flags"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/scheduler"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("cachedbs")
const (
EPERM = syscall.Errno(syscall.EPERM)
ENFILE = syscall.Errno(syscall.ENFILE)
)
type CachedBlobStore struct {
backendbs blobstore.BlobStore
cachebs blobstore.RandomAccessBlobStore
s *scheduler.Scheduler
flags int
queryVersion version.QueryFunc
bever *CachedBackendVersion
package util
import (
"time"
"github.com/nyaxt/otaru/logger"
)
var synclog = logger.Registry().Category("syncsched")
func NewSyncScheduler(s Syncer, wait time.Duration) *PeriodicRunner {
return NewPeriodicRunner(func() {
err := s.Sync()
if err != nil {
logger.Warningf(synclog, "Sync err: %v", err)
}
}, wait)
}
"github.com/nyaxt/otaru/btncrypt"
"github.com/nyaxt/otaru/chunkstore"
oflags "github.com/nyaxt/otaru/flags"
"github.com/nyaxt/otaru/gcloud/auth"
"github.com/nyaxt/otaru/gcloud/datastore"
"github.com/nyaxt/otaru/gcloud/gcs"
"github.com/nyaxt/otaru/inodedb"
"github.com/nyaxt/otaru/inodedb/blobstoredbstatesnapshotio"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/metadata"
"github.com/nyaxt/otaru/mgmt"
"github.com/nyaxt/otaru/scheduler"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("facade")
type Otaru struct {
C btncrypt.Cipher
S *scheduler.Scheduler
Tsrc oauth2.TokenSource
DSCfg *datastore.Config
GL *datastore.GlobalLocker
MetadataBS blobstore.BlobStore
DefaultBS blobstore.BlobStore
BackendBS blobstore.BlobStore
import (
"bytes"
"compress/zlib"
"encoding/gob"
"fmt"
"io"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/btncrypt"
"github.com/nyaxt/otaru/chunkstore"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("statess")
type EncodeCallback func(enc *gob.Encoder) error
func Save(blobpath string, c btncrypt.Cipher, bs blobstore.BlobStore, cb EncodeCallback) error {
var buf bytes.Buffer
zw := zlib.NewWriter(&buf)
enc := gob.NewEncoder(zw)
es := []error{}
if err := cb(enc); err != nil {
es = append(es, fmt.Errorf("Failed to encode state: %v", err))
}
if err := zw.Close(); err != nil {
es = append(es, fmt.Errorf("Failed to close zlib Writer: %v", err))
}
package inodedb
import (
"errors"
"fmt"
"math"
"strconv"
"syscall"
"time"
bfuse "bazil.org/fuse"
"github.com/nyaxt/otaru/logger"
)
var mylog = logger.Registry().Category("inodedb")
type Errno syscall.Errno
func (e Errno) Errno() bfuse.Errno {
return bfuse.Errno(e)
}
func (e Errno) Error() string {
return syscall.Errno(e).Error()
}
var (
EEXIST = Errno(syscall.EEXIST)
ENOENT = Errno(syscall.ENOENT)
EISDIR = Errno(syscall.EISDIR)
// better than nothing cryptography.
// This code has not gone through any security audit, so don't trust this code / otaru encryption.
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"fmt"
"io"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("btncrypt")
const (
BtnFrameMaxPayload = 256 * 1024
)
func gcmFromKey(key []byte) (cipher.AEAD, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, fmt.Errorf("Failed to initialize AES: %v", err)
}
gcm, err := cipher.NewGCM(block)
if err != nil {
panic(err)
}
"bytes"
"fmt"
"sync"
"syscall"
"time"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/btncrypt"
"github.com/nyaxt/otaru/chunkstore"
fl "github.com/nyaxt/otaru/flags"
"github.com/nyaxt/otaru/inodedb"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/util"
)
var fslog = logger.Registry().Category("filesystem")
const (
EACCES = syscall.Errno(syscall.EACCES)
EBADF = syscall.Errno(syscall.EBADF)
EEXIST = syscall.Errno(syscall.EEXIST)
EISDIR = syscall.Errno(syscall.EISDIR)
ENOENT = syscall.Errno(syscall.ENOENT)
ENOTDIR = syscall.Errno(syscall.ENOTDIR)
ENOTEMPTY = syscall.Errno(syscall.ENOTEMPTY)
EPERM = syscall.Errno(syscall.EPERM)
)
const (
FileWriteCacheMaxPatches = 32
FileWriteCacheMaxPatchContentLen = 256 * 1024
package gc
import (
"fmt"
"time"
"golang.org/x/net/context"
"github.com/nyaxt/otaru/blobstore"
"github.com/nyaxt/otaru/inodedb"
"github.com/nyaxt/otaru/logger"
"github.com/nyaxt/otaru/metadata"
"github.com/nyaxt/otaru/util"
)
var mylog = logger.Registry().Category("gc")
type GCableBlobStore interface {
blobstore.BlobLister
blobstore.BlobRemover
}
func GC(ctx context.Context, bs GCableBlobStore, idb inodedb.DBFscker, dryrun bool) error {
start := time.Now()
logger.Infof(mylog, "GC start. Dryrun: %t. Listing blobs.", dryrun)
allbs, err := bs.ListBlobs()
if err != nil {
return fmt.Errorf("ListBlobs failed: %v", err)
}
logger.Infof(mylog, "List blobs done. %d blobs found.", len(allbs))
package fuse
import (
"fmt"
"github.com/nyaxt/otaru"
"github.com/nyaxt/otaru/inodedb"
"github.com/nyaxt/otaru/logger"
bfuse "bazil.org/fuse"
bfs "bazil.org/fuse/fs"
)
var mylog = logger.Registry().Category("fuse")
type FileSystem struct {
ofs *otaru.FileSystem
}
func (fs FileSystem) Root() (bfs.Node, error) {
return DirNode{fs: fs.ofs, id: inodedb.RootDirID}, nil
}
func ServeFUSE(bucketName string, mountpoint string, ofs *otaru.FileSystem, ready chan<- bool) error {
fsName := fmt.Sprintf("otaru+gs://%s", bucketName)
volName := fmt.Sprintf("Otaru %s", bucketName)
c, err := bfuse.Mount(
mountpoint,
bfuse.FSName(fsName),
bfuse.Subtype("otarufs"),
package inodedb
import (
"github.com/nyaxt/otaru/logger"
)
var clog = logger.Registry().Category("cacheddbtxlogio")
type CachedDBTransactionLogIO struct {
be DBTransactionLogIO
ringbuf []DBTransaction
next int
oldestTxID TxID
}
var _ = DBTransactionLogIO(&CachedDBTransactionLogIO{})
const ringbufLen = 256 // FIXME: this should be >2k
func NewCachedDBTransactionLogIO(be DBTransactionLogIO) *CachedDBTransactionLogIO {
txio := &CachedDBTransactionLogIO{
be: be,
ringbuf: make([]DBTransaction, ringbufLen),
next: 0,
oldestTxID: LatestVersion,
}
for i, _ := range txio.ringbuf {
txio.ringbuf[i].TxID = 0
}
import (
"fmt"
"sort"
"sync"
"time"
"golang.org/x/net/context"
"google.golang.org/cloud/datastore"
"github.com/nyaxt/otaru/btncrypt"
gcutil "github.com/nyaxt/otaru/gcloud/util"
"github.com/nyaxt/otaru/inodedb"
"github.com/nyaxt/otaru/logger"
)
var txlog = logger.Registry().Category("dbtxlogio")
type DBTransactionLogIO struct {
cfg *Config
rootKey *datastore.Key
mu sync.Mutex
nextbatch []inodedb.DBTransaction
muSync sync.Mutex
committing []inodedb.DBTransaction
}
const kindTransaction = "OtaruINodeDBTx"
var _ = inodedb.DBTransactionLogIO(&DBTransactionLogIO{})