Example #1
0
func Install(srv *mgmt.Server) {
	rtr := srv.APIRouter().PathPrefix("/logger").Subrouter()

	rtr.HandleFunc("/categories", mgmt.JSONHandler(func(req *http.Request) interface{} {
		return logger.Registry().Categories()
	}))
	rtr.HandleFunc("/category/{cat:\\w+}", mgmt.JSONHandler(func(req *http.Request) interface{} {
		vars := mux.Vars(req)
		cat := vars["cat"]
		cl := logger.Registry().Category(cat)
		if cl == nil {
			return fmt.Errorf("Cateogory not found")
		}
		if req.Method == "GET" {
			return cl.View()
		} else if req.Method == "POST" {
			levelp := req.URL.Query().Get("level")
			nlevel, err := strconv.ParseUint(levelp, 10, 32)
			if err != nil {
				return fmt.Errorf("Failed to parse level")
			}
			cl.Level = logger.Level(nlevel)

			return cl.View()
		} else {
			return fmt.Errorf("Unknown method!")
		}
	}))
}
Example #2
0
func SetupFluentLogger(cfg *Config) error {
	if cfg.Fluent.FluentHost == "" {
		logger.Infof(mylog, "The fluentd host is not specified. Skipping fluent logger instantiation.")
		return nil
	}
	logger.Infof(mylog, "Initializing fluent logger based on config: %+v", cfg.Fluent)

	fcli, err := gfluent.New(cfg.Fluent)
	if err != nil {
		return err
	}

	logger.Registry().AddOutput(fluent.FluentLogger{fcli})
	return nil
}
Example #3
0
func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	logger.Registry().AddOutput(logger.WriterLogger{os.Stderr})
	flag.Usage = Usage
	flag.Parse()

	cfg, err := facade.NewConfig(*flagConfigDir)
	if err != nil {
		logger.Criticalf(mylog, "%v", err)
		Usage()
		os.Exit(2)
	}
	if flag.NArg() != 1 {
		Usage()
		os.Exit(2)
	}
	mountpoint := flag.Arg(0)

	if err := facade.SetupFluentLogger(cfg); err != nil {
		logger.Criticalf(mylog, "Failed to setup fluentd logger: %v", err)
		os.Exit(1)
	}

	o, err := facade.NewOtaru(cfg, &facade.OneshotConfig{Mkfs: *flagMkfs})
	if err != nil {
		logger.Criticalf(mylog, "NewOtaru failed: %v", err)
		os.Exit(1)
	}
	var muClose sync.Mutex
	closeOtaruAndExit := func(exitCode int) {
		muClose.Lock()
		defer muClose.Unlock()

		if err := bfuse.Unmount(mountpoint); err != nil {
			logger.Warningf(mylog, "umount err: %v", err)
		}
		if o != nil {
			if err := o.Close(); err != nil {
				logger.Warningf(mylog, "Otaru.Close() returned errs: %v", err)
			}
			o = nil
		}
		os.Exit(exitCode)
	}
	defer closeOtaruAndExit(0)

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, os.Interrupt)
	signal.Notify(sigC, syscall.SIGTERM)
	go func() {
		for s := range sigC {
			logger.Warningf(mylog, "Received signal: %v", s)
			closeOtaruAndExit(1)
		}
	}()
	logger.Registry().AddOutput(logger.HandleCritical(func() {
		logger.Warningf(mylog, "Starting shutdown due to critical event.")
		closeOtaruAndExit(1)
	}))

	bfuseLogger := logger.Registry().Category("bfuse")
	bfuse.Debug = func(msg interface{}) { logger.Debugf(bfuseLogger, "%v", msg) }
	if err := fuse.ServeFUSE(cfg.BucketName, mountpoint, o.FS, nil); err != nil {
		logger.Warningf(mylog, "ServeFUSE failed: %v", err)
		closeOtaruAndExit(1)
	}
	logger.Infof(mylog, "ServeFUSE end!")
}
Example #4
0
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	bfuse "bazil.org/fuse"

	"github.com/nyaxt/otaru/facade"
	"github.com/nyaxt/otaru/fuse"
	"github.com/nyaxt/otaru/logger"
)

var mylog = logger.Registry().Category("cli")

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s MOUNTPOINT\n", os.Args[0])
	flag.PrintDefaults()
}

var (
	flagMkfs      = flag.Bool("mkfs", false, "Reset metadata if no existing metadata exists")
	flagConfigDir = flag.String("configDir", facade.DefaultConfigDir(), "Config dirpath")
)

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	logger.Registry().AddOutput(logger.WriterLogger{os.Stderr})
Example #5
0
import (
	"fmt"
	"io"
	"os"
	"path"
	"syscall"
	"time"

	"github.com/dustin/go-humanize"

	fl "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/logger"
)

var mylog = logger.Registry().Category("filebs")

const (
	ENOENT = syscall.Errno(syscall.ENOENT)
	EPERM  = syscall.Errno(syscall.EPERM)
)

type FileBlobHandle struct {
	Fp *os.File
}

func (h FileBlobHandle) PRead(p []byte, offset int64) error {
	if _, err := h.Fp.Seek(offset, os.SEEK_SET); err != nil {
		return err
	}
	if _, err := io.ReadFull(h.Fp, p); err != nil {
Example #6
0
package datastore

import (
	"errors"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/cloud/datastore"

	gcutil "github.com/nyaxt/otaru/gcloud/util"
	"github.com/nyaxt/otaru/logger"
)

var sslog = logger.Registry().Category("inodedbsslocator")

const kindINodeDBSS = "OtaruINodeDBSS"

var EEMPTY = errors.New("Failed to find any snapshot location entry.")

type INodeDBSSLocator struct {
	cfg     *Config
	rootKey *datastore.Key
}

func NewINodeDBSSLocator(cfg *Config) *INodeDBSSLocator {
	return &INodeDBSSLocator{
		cfg:     cfg,
		rootKey: datastore.NewKey(ctxNoNamespace, kindINodeDBSS, cfg.rootKeyStr, 0, nil),
	}
}
Example #7
0
package chunkstore

import (
	"fmt"
	"io"
	"math"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/blobstore/version"
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("chunkstore")

const (
	ContentFramePayloadLength = btncrypt.BtnFrameMaxPayload
)

var (
	ZeroContent = make([]byte, ContentFramePayloadLength)
)

func NewQueryChunkVersion(c btncrypt.Cipher) version.QueryFunc {
	return func(r io.Reader) (version.Version, error) {
		var h ChunkHeader
		if err := h.ReadFrom(r, c); err != nil {
			if err == io.EOF {
				return 0, nil
			}
Example #8
0
package otaru

import (
	"math"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/intn"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/util"
)

var wclog = logger.Registry().Category("filewritecache")

type FileWriteCache struct {
	ps intn.Patches
}

func NewFileWriteCache() *FileWriteCache {
	return &FileWriteCache{ps: intn.NewPatches()}
}

func (wc *FileWriteCache) PWrite(p []byte, offset int64) error {
	pcopy := make([]byte, len(p))
	copy(pcopy, p)

	newp := intn.Patch{Offset: offset, P: pcopy}
	logger.Debugf(wclog, "PWrite: %v", newp)
	// logger.Debugf(wclog, "PWrite: p=%v", pcopy)

	wc.ps = wc.ps.Merge(newp)
	return nil
Example #9
0
	"syscall"
	"time"

	"github.com/dustin/go-humanize"
	"golang.org/x/net/context"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/blobstore/version"
	"github.com/nyaxt/otaru/btncrypt"
	fl "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/scheduler"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("cachedbs")

const (
	EPERM  = syscall.Errno(syscall.EPERM)
	ENFILE = syscall.Errno(syscall.ENFILE)
)

type CachedBlobStore struct {
	backendbs blobstore.BlobStore
	cachebs   blobstore.RandomAccessBlobStore
	s         *scheduler.Scheduler

	flags int

	queryVersion version.QueryFunc
	bever        *CachedBackendVersion
Example #10
0
package util

import (
	"time"

	"github.com/nyaxt/otaru/logger"
)

var synclog = logger.Registry().Category("syncsched")

func NewSyncScheduler(s Syncer, wait time.Duration) *PeriodicRunner {
	return NewPeriodicRunner(func() {
		err := s.Sync()
		if err != nil {
			logger.Warningf(synclog, "Sync err: %v", err)
		}
	}, wait)
}
Example #11
0
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/chunkstore"
	oflags "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/gcloud/auth"
	"github.com/nyaxt/otaru/gcloud/datastore"
	"github.com/nyaxt/otaru/gcloud/gcs"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/inodedb/blobstoredbstatesnapshotio"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/metadata"
	"github.com/nyaxt/otaru/mgmt"
	"github.com/nyaxt/otaru/scheduler"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("facade")

type Otaru struct {
	C btncrypt.Cipher

	S *scheduler.Scheduler

	Tsrc  oauth2.TokenSource
	DSCfg *datastore.Config
	GL    *datastore.GlobalLocker

	MetadataBS blobstore.BlobStore
	DefaultBS  blobstore.BlobStore

	BackendBS blobstore.BlobStore
Example #12
0
import (
	"bytes"
	"compress/zlib"
	"encoding/gob"
	"fmt"
	"io"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/chunkstore"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("statess")

type EncodeCallback func(enc *gob.Encoder) error

func Save(blobpath string, c btncrypt.Cipher, bs blobstore.BlobStore, cb EncodeCallback) error {
	var buf bytes.Buffer
	zw := zlib.NewWriter(&buf)
	enc := gob.NewEncoder(zw)

	es := []error{}
	if err := cb(enc); err != nil {
		es = append(es, fmt.Errorf("Failed to encode state: %v", err))
	}
	if err := zw.Close(); err != nil {
		es = append(es, fmt.Errorf("Failed to close zlib Writer: %v", err))
	}
Example #13
0
package inodedb

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"syscall"
	"time"

	bfuse "bazil.org/fuse"

	"github.com/nyaxt/otaru/logger"
)

var mylog = logger.Registry().Category("inodedb")

type Errno syscall.Errno

func (e Errno) Errno() bfuse.Errno {
	return bfuse.Errno(e)
}

func (e Errno) Error() string {
	return syscall.Errno(e).Error()
}

var (
	EEXIST         = Errno(syscall.EEXIST)
	ENOENT         = Errno(syscall.ENOENT)
	EISDIR         = Errno(syscall.EISDIR)
Example #14
0
// better than nothing cryptography.
// This code has not gone through any security audit, so don't trust this code / otaru encryption.

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"fmt"
	"io"

	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("btncrypt")

const (
	BtnFrameMaxPayload = 256 * 1024
)

func gcmFromKey(key []byte) (cipher.AEAD, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, fmt.Errorf("Failed to initialize AES: %v", err)
	}

	gcm, err := cipher.NewGCM(block)
	if err != nil {
		panic(err)
	}
Example #15
0
	"bytes"
	"fmt"
	"sync"
	"syscall"
	"time"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/btncrypt"
	"github.com/nyaxt/otaru/chunkstore"
	fl "github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/util"
)

var fslog = logger.Registry().Category("filesystem")

const (
	EACCES    = syscall.Errno(syscall.EACCES)
	EBADF     = syscall.Errno(syscall.EBADF)
	EEXIST    = syscall.Errno(syscall.EEXIST)
	EISDIR    = syscall.Errno(syscall.EISDIR)
	ENOENT    = syscall.Errno(syscall.ENOENT)
	ENOTDIR   = syscall.Errno(syscall.ENOTDIR)
	ENOTEMPTY = syscall.Errno(syscall.ENOTEMPTY)
	EPERM     = syscall.Errno(syscall.EPERM)
)

const (
	FileWriteCacheMaxPatches         = 32
	FileWriteCacheMaxPatchContentLen = 256 * 1024
Example #16
0
File: gc.go Project: postfix/otaru
package gc

import (
	"fmt"
	"time"

	"golang.org/x/net/context"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/logger"
	"github.com/nyaxt/otaru/metadata"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("gc")

type GCableBlobStore interface {
	blobstore.BlobLister
	blobstore.BlobRemover
}

func GC(ctx context.Context, bs GCableBlobStore, idb inodedb.DBFscker, dryrun bool) error {
	start := time.Now()

	logger.Infof(mylog, "GC start. Dryrun: %t. Listing blobs.", dryrun)
	allbs, err := bs.ListBlobs()
	if err != nil {
		return fmt.Errorf("ListBlobs failed: %v", err)
	}
	logger.Infof(mylog, "List blobs done. %d blobs found.", len(allbs))
Example #17
0
package fuse

import (
	"fmt"

	"github.com/nyaxt/otaru"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/logger"

	bfuse "bazil.org/fuse"
	bfs "bazil.org/fuse/fs"
)

var mylog = logger.Registry().Category("fuse")

type FileSystem struct {
	ofs *otaru.FileSystem
}

func (fs FileSystem) Root() (bfs.Node, error) {
	return DirNode{fs: fs.ofs, id: inodedb.RootDirID}, nil
}

func ServeFUSE(bucketName string, mountpoint string, ofs *otaru.FileSystem, ready chan<- bool) error {
	fsName := fmt.Sprintf("otaru+gs://%s", bucketName)
	volName := fmt.Sprintf("Otaru %s", bucketName)

	c, err := bfuse.Mount(
		mountpoint,
		bfuse.FSName(fsName),
		bfuse.Subtype("otarufs"),
Example #18
0
package inodedb

import (
	"github.com/nyaxt/otaru/logger"
)

var clog = logger.Registry().Category("cacheddbtxlogio")

type CachedDBTransactionLogIO struct {
	be DBTransactionLogIO

	ringbuf    []DBTransaction
	next       int
	oldestTxID TxID
}

var _ = DBTransactionLogIO(&CachedDBTransactionLogIO{})

const ringbufLen = 256 // FIXME: this should be >2k

func NewCachedDBTransactionLogIO(be DBTransactionLogIO) *CachedDBTransactionLogIO {
	txio := &CachedDBTransactionLogIO{
		be:         be,
		ringbuf:    make([]DBTransaction, ringbufLen),
		next:       0,
		oldestTxID: LatestVersion,
	}

	for i, _ := range txio.ringbuf {
		txio.ringbuf[i].TxID = 0
	}
Example #19
0
import (
	"fmt"
	"sort"
	"sync"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/cloud/datastore"

	"github.com/nyaxt/otaru/btncrypt"
	gcutil "github.com/nyaxt/otaru/gcloud/util"
	"github.com/nyaxt/otaru/inodedb"
	"github.com/nyaxt/otaru/logger"
)

var txlog = logger.Registry().Category("dbtxlogio")

type DBTransactionLogIO struct {
	cfg     *Config
	rootKey *datastore.Key

	mu        sync.Mutex
	nextbatch []inodedb.DBTransaction

	muSync     sync.Mutex
	committing []inodedb.DBTransaction
}

const kindTransaction = "OtaruINodeDBTx"

var _ = inodedb.DBTransactionLogIO(&DBTransactionLogIO{})