func (s *server) Read(ctx context.Context, req *pb.ReadRequest) (*pb.ReadReply, error) {
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: read error (%v)", err)
return &pb.ReadReply{}, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: read error (cannot find disk %s)", dn)
return &pb.ReadReply{}, nil
}
stats.Counter("cfs_read_ops_total").Disk(dn).Add()
// TODO: reuse buffer
data := make([]byte, req.Length)
n, err := d.ReadAt(fn, data, req.Offset)
// TODO: add error
if err != nil {
log.Infof("server: read error (%v)", err)
return &pb.ReadReply{}, nil
}
reply := &pb.ReadReply{BytesRead: int64(n), Data: data}
return reply, nil
}
func (s *server) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (*pb.ReadDirReply, error) {
reply := &pb.ReadDirReply{}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: readDir error (%v)", err)
return reply, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: readDir error (cannot find disk %s)", dn)
return reply, nil
}
stats.Counter("cfs_readdir_ops_total").Disk(dn).Add()
stats, err := d.ReadDir(fn)
if err != nil {
log.Infof("server: readDir error (%v)", err)
return reply, nil
}
reply.FileInfos = make([]*pb.FileInfo, len(stats))
for i, stat := range stats {
reply.FileInfos[i] = &pb.FileInfo{
Name: stat.Name(),
// TODO: Add size
TotalSize: stat.Size(),
IsDir: stat.IsDir(),
}
}
return reply, nil
}
func (s *server) Read(ctx context.Context, req *pb.ReadRequest) (*pb.ReadReply, error) {
if !enforce.HasQuota(req.Header.ClientID) {
log.Infof("server: out of quota for client %d", req.Header.ClientID)
return &pb.ReadReply{}, nil
}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: read error (%v)", err)
return &pb.ReadReply{}, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: read error (cannot find disk %s)", dn)
return &pb.ReadReply{}, nil
}
stats.Counter(dn, "read").Client(req.Header.ClientID).Add()
// TODO: reuse buffer
data := make([]byte, req.Length)
n, err := d.ReadAt(fn, data, req.Offset)
// TODO: add error
if err == io.EOF {
log.Infof("server: read %d bytes until EOF", n)
return &pb.ReadReply{BytesRead: int64(n), Data: data[:n]}, nil
}
if err != nil {
log.Infof("server: read error (%v)", err)
return &pb.ReadReply{}, nil
}
reply := &pb.ReadReply{BytesRead: int64(n), Data: data}
return reply, nil
}
func (s *server) Write(ctx context.Context, req *pb.WriteRequest) (*pb.WriteReply, error) {
if !enforce.HasQuota(req.Header.ClientID) {
log.Infof("server: out of quota for client %d", req.Header.ClientID)
return &pb.WriteReply{}, nil
}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: write error (%v)", err)
return &pb.WriteReply{}, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: write error (cannot find disk %s)", dn)
return &pb.WriteReply{}, nil
}
stats.Counter(dn, "write").Client(req.Header.ClientID).Add()
n, err := d.WriteAt(fn, req.Data, req.Offset)
// TODO: add error
if err != nil {
log.Infof("server: write error (%v)", err)
return &pb.WriteReply{}, nil
}
reply := &pb.WriteReply{BytesWritten: int64(n)}
return reply, nil
}
func (s *server) Mkdir(ctx context.Context, req *pb.MkdirRequest) (*pb.MkdirReply, error) {
reply := &pb.MkdirReply{}
if !enforce.HasQuota(req.Header.ClientID) {
log.Infof("server: out of quota for client %d", req.Header.ClientID)
return reply, nil
}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: mkdir error (%v)", err)
return reply, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: mkdir error (cannot find disk %s)", dn)
return reply, nil
}
stats.Counter(dn, "mkdir").Client(req.Header.ClientID).Add()
err = d.Mkdir(fn, req.All)
if err != nil {
log.Infof("server: mkdir error (%v)", err)
return reply, nil
}
return reply, nil
}
func main() {
configfn := "nameserver.conf"
data, err := ioutil.ReadFile(configfn)
if err != nil {
log.Fatalf("server: cannot load configuration file[%s] (%v)", configfn, err)
}
var conf config.Server
if _, err := toml.Decode(string(data), &conf); err != nil {
log.Fatalf("server: configuration file[%s] is not valid (%v)", configfn, err)
}
server := NewServer()
for i, v := range conf.Disks {
log.Infof("Adding %v to disks", v)
server.registeredDisks = append(server.registeredDisks, &conf.Disks[i])
}
log.Infof("server: starting server...")
lis, err := net.Listen("tcp", net.JoinHostPort(conf.Bind, conf.Port))
if err != nil {
log.Fatalf("server: failed to listen: %v", err)
}
log.Infof("server: listening on %s", net.JoinHostPort(conf.Bind, conf.Port))
s := grpc.NewServer()
pb.RegisterNameServer(s, server)
log.Infof("server: ready to serve clients")
s.Serve(lis)
}
func (s *server) Rename(ctx context.Context, req *pb.RenameRequest) (*pb.RenameReply, error) {
dn0, ofn, err := splitDiskAndFile(req.Oldname)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
dn1, nfn, err := splitDiskAndFile(req.Newname)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
if dn0 != dn1 {
log.Infof("server: rename error (%v)", "not same disk")
return &pb.RenameReply{}, nil
}
d := s.Disk(dn0)
if d == nil {
log.Infof("server: read error (cannot find disk %s)", dn0)
return &pb.RenameReply{}, nil
}
stats.Counter("cfs_rename_ops_total").Disk(dn0).Add()
err = d.Rename(ofn, nfn)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
reply := &pb.RenameReply{}
return reply, nil
}
func main() {
configfn := flag.String("config", "default.conf", "location of configuration file")
flag.Parse()
data, err := ioutil.ReadFile(*configfn)
if err != nil {
log.Fatalf("server: cannot load configuration file[%s] (%v)", *configfn, err)
}
var conf config.Server
if _, err := toml.Decode(string(data), &conf); err != nil {
log.Fatalf("server: configuration file[%s] is not valid (%v)", *configfn, err)
}
// default is that cfs is bootstrapped using docker
cname, err := detectDockerContainer()
if err != nil {
log.Printf("server: failed to detect docker container (%v)", err)
} else {
stats.SetContainerName(cname)
log.Printf("server: detect docker container %q", cname)
}
log.Infof("server: starting server...")
lis, err := net.Listen("tcp", net.JoinHostPort(conf.Bind, conf.Port))
if err != nil {
log.Fatalf("server: failed to listen: %v", err)
}
log.Infof("server: listening on %s", net.JoinHostPort(conf.Bind, conf.Port))
cfs := NewServer()
for _, d := range conf.Disks {
err = cfs.AddDisk(d.Name, d.Root)
if err != nil {
log.Fatalf("server: failed to add disk (%v)", err)
}
}
// 0x1234 is the client ID for cfsctl, and its quota is 10 req/sec.
enforce.SetQuota(0x1234, 10)
// TODO report with influxSinker
stats.Report(nil, 3*time.Second)
s := grpc.NewServer()
pb.RegisterCfsServer(s, cfs)
pb.RegisterStatsServer(s, stats.Server())
log.Infof("server: ready to serve clients")
s.Serve(lis)
}
// Start sync dir, it will loop until the Stop is called
func (s *Syncer) Start() {
log.Infof("Start sync %v/%s", s.dir, s.matchPattern)
for {
if err := s.sync(); err != nil && !os.IsNotExist(err) {
log.Errorf("Sync %s failed - %v", s.dir, err)
}
select {
case <-s.endch:
log.Infof("Stop sync %v/%s", s.dir, s.matchPattern)
return
default:
}
time.Sleep(time.Second * time.Duration(s.IntervalSecond))
}
}
func initContainerManager() {
sysFs, err := sysfs.NewRealSysFs()
if err != nil {
log.Infof("stats: failed to create a system interface (%v)", err)
return
}
// TODO: support influxdb or other backend storage
cmgr, err = manager.New(memory.New(storageDuration, nil), sysFs)
if err != nil {
log.Infof("stats: failed to create a container Manager (%v)", err)
return
}
if err := cmgr.Start(); err != nil {
log.Infof("stats: failed to start container manager (%v)", err)
return
}
}
func handleRename(ctx context.Context, c *client.Client) error {
err := c.Rename(ctx, renameOld, renameNew)
if err != nil {
log.Fatalf("Rename err (%v)", err)
}
log.Infof("rename %s into %s", renameOld, renameNew)
return nil
}
func handleWrite(ctx context.Context, c *client.Client) error {
n, err := c.Write(ctx, writeName, writeOffset, []byte(writeData), writeAppend)
if err != nil {
log.Fatalf("Write err (%v)", err)
}
log.Infof("%d bytes written to %s at offset %d", n, writeName, writeOffset)
return nil
}
// FIXME(ssx): maybe need to return error
func (p *Process) buildCommand() *kexec.KCommand {
cmd := kexec.CommandString(p.Command)
// cmd := kexec.Command(p.Command[0], p.Command[1:]...)
logDir := filepath.Join(defaultConfigDir, "log", sanitize.Name(p.Name))
if !IsDir(logDir) {
os.MkdirAll(logDir, 0755)
}
var fout io.Writer
var err error
p.OutputFile, err = os.OpenFile(filepath.Join(logDir, "output.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.Warn("create stdout log failed:", err)
fout = ioutil.Discard
} else {
fout = p.OutputFile
}
cmd.Stdout = io.MultiWriter(p.Stdout, p.Output, fout)
cmd.Stderr = io.MultiWriter(p.Stderr, p.Output, fout)
// config environ
cmd.Env = os.Environ() // inherit current vars
environ := map[string]string{}
if p.User != "" {
err := cmd.SetUser(p.User)
if err != nil {
log.Warnf("[%s] chusr to %s failed", p.Name, p.User)
} else {
var homeDir string
switch runtime.GOOS {
case "linux":
homeDir = "/home/" + p.User // FIXME(ssx): maybe there is a better way
case "darwin":
homeDir = "/Users/" + p.User
}
cmd.Env = append(cmd.Env, "HOME="+homeDir, "USER="******"HOME"] = homeDir
environ["USER"] = p.User
}
}
cmd.Env = append(cmd.Env, p.Environ...)
mapping := func(key string) string {
val := os.Getenv(key)
if val != "" {
return val
}
return environ[key]
}
cmd.Dir = os.Expand(p.Dir, mapping)
if strings.HasPrefix(cmd.Dir, "~") {
cmd.Dir = mapping("HOME") + cmd.Dir[1:]
}
log.Infof("[%s] use dir: %s\n", p.Name, cmd.Dir)
return cmd
}
func (s *server) Rename(ctx context.Context, req *pb.RenameRequest) (*pb.RenameReply, error) {
if !enforce.HasQuota(req.Header.ClientID) {
log.Infof("server: out of quota for client %d", req.Header.ClientID)
return &pb.RenameReply{}, nil
}
dn0, ofn, err := splitDiskAndFile(req.Oldname)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
dn1, nfn, err := splitDiskAndFile(req.Newname)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
if dn0 != dn1 {
log.Infof("server: rename error (%v)", "not same disk")
return &pb.RenameReply{}, nil
}
d := s.Disk(dn0)
if d == nil {
log.Infof("server: read error (cannot find disk %s)", dn0)
return &pb.RenameReply{}, nil
}
stats.Counter(dn0, "rename").Client(req.Header.ClientID).Add()
err = d.Rename(ofn, nfn)
if err != nil {
log.Infof("server: rename error (%v)", err)
return &pb.RenameReply{}, nil
}
reply := &pb.RenameReply{}
return reply, nil
}
func (s *server) Mkdir(ctx context.Context, req *pb.MkdirRequest) (*pb.MkdirReply, error) {
reply := &pb.MkdirReply{}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: mkdir error (%v)", err)
return reply, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: mkdir error (cannot find disk %s)", dn)
return reply, nil
}
stats.Counter("cfs_mkdir_ops_total").Disk(dn).Add()
err = d.Mkdir(fn, req.All)
if err != nil {
log.Infof("server: mkdir error (%v)", err)
return reply, nil
}
return reply, nil
}
func (s *server) Remove(ctx context.Context, req *pb.RemoveRequest) (*pb.RemoveReply, error) {
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: remove error (%v)", err)
return &pb.RemoveReply{}, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: remove error (cannot find disk %s)", dn)
return &pb.RemoveReply{}, nil
}
stats.Counter("cfs_remove_ops_total").Disk(dn).Add()
err = d.Remove(fn, req.All)
if err != nil {
log.Infof("server: read error (%v)", err)
return &pb.RemoveReply{}, nil
}
reply := &pb.RemoveReply{}
return reply, nil
}
func (s *server) Write(ctx context.Context, req *pb.WriteRequest) (*pb.WriteReply, error) {
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: write error (%v)", err)
return &pb.WriteReply{}, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: write error (cannot find disk %s)", dn)
return &pb.WriteReply{}, nil
}
stats.Counter("cfs_write_ops_total").Disk(dn).Add()
n, err := d.WriteAt(fn, req.Data, req.Offset)
// TODO: add error
if err != nil {
log.Infof("server: write error (%v)", err)
return &pb.WriteReply{}, nil
}
reply := &pb.WriteReply{BytesWritten: int64(n)}
return reply, nil
}
func (s *server) AddDisk(name, root string) error {
s.disks[name] = &disk.Disk{Name: name, Root: root}
err := os.MkdirAll(root, 0700)
if err != nil {
return err
}
pwd, err := os.Getwd()
if err != nil {
log.Panicf("server: cannot get current working directory (%v)", err)
}
log.Infof("server: created disk[%s] at root path[%s]", name, path.Join(pwd, root))
return nil
}
func (s *server) ReadDir(ctx context.Context, req *pb.ReadDirRequest) (*pb.ReadDirReply, error) {
reply := &pb.ReadDirReply{}
if !enforce.HasQuota(req.Header.ClientID) {
log.Infof("server: out of quota for client %d", req.Header.ClientID)
return reply, nil
}
dn, fn, err := splitDiskAndFile(req.Name)
if err != nil {
log.Infof("server: readDir error (%v)", err)
return reply, nil
}
d := s.Disk(dn)
if d == nil {
log.Infof("server: readDir error (cannot find disk %s)", dn)
return reply, nil
}
stats.Counter(dn, "readdir").Client(req.Header.ClientID).Add()
stats, err := d.ReadDir(fn)
if err != nil {
log.Infof("server: readDir error (%v)", err)
return reply, nil
}
reply.FileInfos = make([]*pb.FileInfo, len(stats))
for i, stat := range stats {
reply.FileInfos[i] = &pb.FileInfo{
Name: stat.Name(),
// TODO: Add size
TotalSize: stat.Size(),
IsDir: stat.IsDir(),
}
}
return reply, nil
}
func proxyHandler(w http.ResponseWriter, r *http.Request) {
var proxyFor = r.Header.Get("X-Proxy-For")
log.Infof("[remote %s] X-Proxy-For [%s]", r.RemoteAddr, proxyFor)
connC, ok := namedConnection[proxyFor]
if !ok {
http.Error(w, "inside error: proxy not ready to receive conn", http.StatusInternalServerError)
return
}
conn, err := hijackHTTPRequest(w)
if err != nil {
log.Warnf("hijeck failed, %v", err)
connC <- nil
return
}
connC <- conn
}
// msg comes from px server by websocket
// 1: connect to px server, use msg.Name to identify self.
// 2: change conn to reverse conn
func handleWsMsg(msg message, sURL *url.URL, rnl *reverseNetListener) {
u := sURL
switch msg.Type {
case TYPE_NEWCONN:
log.Debug("dial remote:", u.Host)
sconn, err := net.Dial("tcp", u.Host)
if err != nil {
log.Println(err)
break
}
log.Infof("proxy for: %s", msg.Name)
_, err = sconn.Write([]byte(fmt.Sprintf(
"GET /proxyhijack HTTP/1.1\r\nHost: proxylocal\r\nX-Proxy-For: %s \r\n\r\n", msg.Name)))
if err != nil {
log.Println(err)
break
}
rnl.connCh <- sconn
case TYPE_MESSAGE:
fmt.Printf("Recv Message: %v\n", msg.Body)
default:
log.Warnf("Type: %v not support", msg.Type)
}
}
func (ps *ProxyServer) newControlHandler() func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
// read listen port from request
protocol, subdomain, port := parseConnectRequest(r)
log.Debugf("proxy listen proto: %v, subdomain: %v port: %v", protocol, subdomain, port)
// create websocket connection
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
http.Error(w, err.Error(), 502)
return
}
defer conn.Close()
log.Debug("remote client addr:", conn.RemoteAddr())
tunnel := &Tunnel{
wsconn: conn,
}
// TCP: create new port to listen
log.Infof("New %s proxy for %v", protocol, conn.RemoteAddr())
switch protocol {
case "tcp":
// proxyAddr := fmt.Sprintf("0.0.0.0:%d", port)
listener, err := NewTcpProxyListener(tunnel, port)
if err != nil {
log.Warnf("new tcp proxy err: %v", err)
http.Error(w, err.Error(), 501)
return
}
defer listener.Close()
_, port, _ := net.SplitHostPort(listener.Addr().String())
wsSendMessage(conn, fmt.Sprintf(
"Local tcp conn is now publicly available via:\n%v:%v\n", ps.domain, port))
case "http", "https":
tr := &http.Transport{
Dial: tunnel.generateTransportDial(),
}
revProxy := &httputil.ReverseProxy{
Director: func(req *http.Request) {
log.Println("director:", req.RequestURI)
},
Transport: tr,
}
// should hook here
// hook(HOOK_CREATE_HTTP_SUBDOMAIN, subdomain)
// generate a uniq domain
if subdomain == "" {
subdomain = uniqName(5) + ".t"
}
pxDomain := subdomain + "." + ps.domain
log.Println("http px use domain:", pxDomain)
if _, exists := ps.revProxies[pxDomain]; exists {
wsSendMessage(conn, fmt.Sprintf("subdomain [%s] has already been taken", pxDomain))
return
}
ps.Lock()
ps.revProxies[pxDomain] = revProxy
ps.Unlock()
wsSendMessage(conn, fmt.Sprintf(
"Local server is now publicly available via:\nhttp://%s\n", pxDomain))
defer func() {
ps.Lock()
delete(ps.revProxies, pxDomain)
ps.Unlock()
}()
default:
log.Warn("unknown protocol:", protocol)
return
}
// HTTP: use httputil.ReverseProxy
for {
var msg Msg
if err := conn.ReadJSON(&msg); err != nil {
log.Warn(err)
break
}
log.Info("recv json:", msg)
}
}
}