func (self *Runner) handlerOneMail(m *mailproto.Message) {
uid, err := self.db.GetUserByEmail(m.From)
if err != nil {
log.Warn("GetUserByEmail failed:", m.From, err)
return
}
appName, err := mailproto.EmailAddress2App(m.To[0])
if err != nil {
log.Warn("EmailAddress2App failed:", m.To, err)
return
}
err = self.db.CheckUserApp(uid, appName)
if err != nil {
log.Warn("CheckUserApp failed:", err)
return
}
addr, err := self.db.GetUsersMail([]string{uid})
if err != nil {
log.Warn("GetUsersMail failed:", err)
return
}
m.From = ""
m.To = addr
_, err = self.mail.Send(m)
if err != nil {
log.Warn("Send failed", err)
return
}
}
func QiniuUpload(cmd string, params ...string) {
if len(params) == 1 || len(params) == 2 {
var uploadConfigFile string
var threadCount int64
var err error
if len(params) == 2 {
threadCount, err = strconv.ParseInt(params[0], 10, 64)
if err != nil {
log.Error("Invalid <ThreadCount> value,", params[0])
return
}
uploadConfigFile = params[1]
} else {
uploadConfigFile = params[0]
}
if threadCount < qshell.MIN_UPLOAD_THREAD_COUNT ||
threadCount > qshell.MAX_UPLOAD_THREAD_COUNT {
log.Warn("<ThreadCount> can only between 1 and 100")
threadCount = qshell.MIN_UPLOAD_THREAD_COUNT
}
qshell.QiniuUpload(int(threadCount), uploadConfigFile)
} else {
CmdHelp(cmd)
}
}
func QiniuDownload(cmd string, params ...string) {
if len(params) == 1 || len(params) == 2 {
var threadCount int64 = 5
var downConfig string
var err error
if len(params) == 1 {
downConfig = params[0]
} else {
threadCount, err = strconv.ParseInt(params[0], 10, 64)
if err != nil {
log.Error("Invalid value for <ThreadCount>", params[0])
return
}
downConfig = params[1]
}
if threadCount < qshell.MIN_DOWNLOAD_THREAD_COUNT ||
threadCount > qshell.MAX_DOWNLOAD_THREAD_COUNT {
log.Warn("<ThreadCount> can only between 1 and 100")
threadCount = qshell.MIN_DOWNLOAD_THREAD_COUNT
}
qshell.QiniuDownload(int(threadCount), downConfig)
} else {
CmdHelp(cmd)
}
}
// Listen and forward connections
func NewTcpProxyListener(tunnel *Tunnel, port int) (listener *net.TCPListener, err error) {
var laddr *net.TCPAddr
if port != 0 {
laddr, _ = net.ResolveTCPAddr("tcp", ":"+strconv.Itoa(port))
listener, err = net.ListenTCP("tcp", laddr)
} else {
laddr, listener, err = freeport.ListenTCP()
}
if err != nil {
return nil, err
}
port = laddr.Port
//port, listener, err = listenTcpInRangePort(port, TCP_MIN_PORT, TCP_MAX_PORT)
//if err != nil {
//return nil, err
//}
// hook here
err = hook(HOOK_TCP_POST_CONNECT, []string{
"PORT=" + strconv.Itoa(port),
"REMOTE_ADDR=" + tunnel.wsconn.RemoteAddr().String(),
"CLIENT_ADDRESS=" + tunnel.wsconn.RemoteAddr().String(),
"REMOTE_DATA=" + tunnel.data,
})
if err != nil {
listener.Close()
return
}
go func() {
for {
rconn, err := listener.AcceptTCP()
if err != nil {
log.Warn(err)
break
}
// find proxy to where
log.Debug("Receive new connections from", rconn.RemoteAddr())
lconn, err := tunnel.RequestNewConn(rconn.RemoteAddr().String())
if err != nil {
log.Debug("request new conn err:", err)
rconn.Close()
continue
}
log.Debug("request new conn:", lconn, err)
pc := &ProxyConn{
lconn: lconn,
rconn: rconn,
stats: proxyStats,
}
go pc.start()
}
}()
return listener, nil
}
// FIXME(ssx): maybe need to return error
func (p *Process) buildCommand() *kexec.KCommand {
cmd := kexec.CommandString(p.Command)
// cmd := kexec.Command(p.Command[0], p.Command[1:]...)
logDir := filepath.Join(defaultConfigDir, "log", sanitize.Name(p.Name))
if !IsDir(logDir) {
os.MkdirAll(logDir, 0755)
}
var fout io.Writer
var err error
p.OutputFile, err = os.OpenFile(filepath.Join(logDir, "output.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.Warn("create stdout log failed:", err)
fout = ioutil.Discard
} else {
fout = p.OutputFile
}
cmd.Stdout = io.MultiWriter(p.Stdout, p.Output, fout)
cmd.Stderr = io.MultiWriter(p.Stderr, p.Output, fout)
// config environ
cmd.Env = os.Environ() // inherit current vars
environ := map[string]string{}
if p.User != "" {
err := cmd.SetUser(p.User)
if err != nil {
log.Warnf("[%s] chusr to %s failed", p.Name, p.User)
} else {
var homeDir string
switch runtime.GOOS {
case "linux":
homeDir = "/home/" + p.User // FIXME(ssx): maybe there is a better way
case "darwin":
homeDir = "/Users/" + p.User
}
cmd.Env = append(cmd.Env, "HOME="+homeDir, "USER="******"HOME"] = homeDir
environ["USER"] = p.User
}
}
cmd.Env = append(cmd.Env, p.Environ...)
mapping := func(key string) string {
val := os.Getenv(key)
if val != "" {
return val
}
return environ[key]
}
cmd.Dir = os.Expand(p.Dir, mapping)
if strings.HasPrefix(cmd.Dir, "~") {
cmd.Dir = mapping("HOME") + cmd.Dir[1:]
}
log.Infof("[%s] use dir: %s\n", p.Name, cmd.Dir)
return cmd
}
func (t *Task) Synced() (cl changelog.Logger, urls []string, err error) {
dir, err := cc.GetConfigDir("fetch/")
if err != nil {
log.Warn(err, "GetConfigDir failed")
return
}
pos := strings.LastIndex(t.SyncSrc, "/")
if pos == -1 {
pos = 0
}
eigen := t.SyncSrc[pos:] + t.Bucket + t.Style
cl, urls, err = OpenChangeLog(dir, eigen)
return
}
// download src
func (b *Builder) get() (err error) {
log.Debug("start get src to:", b.srcDir)
exists := beeutils.FileExists(b.srcDir)
b.sh.Command("go", "version").Run()
if !exists {
err = b.sh.Command("go", "get", "-v", "-d", b.project).Run()
if err != nil {
return
}
}
b.sh.SetDir(b.srcDir)
if b.ref == "-" {
b.ref = "master"
}
// get code from remote
if err = b.sh.Command("git", "fetch", "origin").Run(); err != nil {
return
}
// change branch
if err = b.sh.Command("git", "checkout", "-q", b.ref).Run(); err != nil {
return
}
// update code
if err = b.sh.Command("git", "merge", "origin/"+b.ref).Run(); err != nil {
log.Warn("git merge error:", err)
//return
}
// get sha
out, err := sh.Command("git", "rev-parse", "HEAD", sh.Dir(b.srcDir)).Output()
if err != nil {
return
}
b.sha = strings.TrimSpace(string(out))
// parse .gobuild
b.rc = new(Assembly)
rcfile := "public/gobuildrc"
if b.sh.Test("f", ".gobuild") {
rcfile = filepath.Join(b.srcDir, ".gobuild")
}
data, err := ioutil.ReadFile(rcfile)
if err != nil {
return
}
err = goyaml.Unmarshal(data, b.rc)
return
}
func main() {
flag.Parse()
if flag.NArg() < 1 {
help()
return
}
var task Task
var taskFile = flag.Arg(0)
err := config.LoadEx(&task, taskFile)
if err != nil || !task.Isvalid() {
log.Warn(err, "config.Load failed:", taskFile)
return
}
task.Run()
}
func NewScriptHandler(script string, contentType string) func(*macaron.Context) {
ext := filepath.Ext(script)
return func(ctx *macaron.Context) {
var err error
var output []byte
errquit := func(err error) {
ctx.Error(500, err.Error()+"\n"+string(output))
return
}
switch ext {
case ".sh":
output, err = runShellScript(ctx, script)
render(ctx, output, err, contentType)
case ".html":
output, err = ioutil.ReadFile(script)
render(ctx, output, err, contentType)
case ".md":
output, err = ioutil.ReadFile(script)
if err != nil {
errquit(err)
return
}
unsafe := blackfriday.MarkdownCommon(output)
html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
render(ctx, html, nil, "html")
case ".jade":
tmpl, err := jade.CompileFile(script, jade.Options{})
if err != nil {
errquit(err)
return
}
buf := bytes.NewBuffer(nil)
err = tmpl.Execute(buf, nil)
render(ctx, buf.Bytes(), err, "html")
default:
log.Warn("Unknown script ext", script, ext)
output, err = ioutil.ReadFile(script)
render(ctx, output, err, detectCType(script))
}
}
}
func serveRevConn(proto ProxyProtocol, pAddr string, lis net.Listener) error {
switch proto {
case TCP:
for {
rconn, err := lis.Accept()
if err != nil {
log.Errorf("accept error: %v", err)
return err
}
log.Info("local dial tcp", pAddr)
lconn, err := net.Dial("tcp", pAddr)
if err != nil {
log.Warn(err)
rconn.Close()
return err
}
// start forward local proxy
pc := &proxyConn{
lconn: lconn,
rconn: rconn,
stats: proxyStats,
}
go pc.start()
}
case HTTP:
rp := &httputil.ReverseProxy{
Director: func(req *http.Request) {
req.Host = pAddr
req.URL.Scheme = "http"
req.URL.Host = pAddr
},
}
return http.Serve(lis, rp)
default:
log.Println("Unknown protocol:", proto)
return ErrUnknownProtocol
}
}
func (p *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
h := p.route[req.URL.Host]
if h == nil {
log.Warn("Server not found:", req.Host)
return nil, ErrServerNotFound
}
cp := *req
cp.Body = &mockServerRequestBody{req.Body, false}
req = &cp
req.RemoteAddr = "127.0.0.1:8000"
rw := httptest.NewRecorder()
h.ServeHTTP(rw, req)
req.Body.Close()
ctlen := int64(-1)
if v := rw.HeaderMap.Get("Content-Length"); v != "" {
ctlen, _ = strconv.ParseInt(v, 10, 64)
}
return &http.Response{
Status: "",
StatusCode: rw.Code,
Header: rw.HeaderMap,
Body: ioutil.NopCloser(rw.Body),
ContentLength: ctlen,
TransferEncoding: nil,
Close: false,
Trailer: nil,
Request: req,
}, nil
}
func put(
l rpc.Logger, ret interface{}, uptoken string, key string, hasKey bool, f io.ReaderAt, fsize int64, extra *PutExtra) error {
once.Do(initWorkers)
blockCnt := BlockCount(fsize)
if extra == nil {
extra = new(PutExtra)
}
if extra.Progresses == nil {
extra.Progresses = make([]BlkputRet, blockCnt)
} else if len(extra.Progresses) != blockCnt {
return ErrInvalidPutProgress
}
if extra.ChunkSize == 0 {
extra.ChunkSize = settings.ChunkSize
}
if extra.TryTimes == 0 {
extra.TryTimes = settings.TryTimes
}
if extra.Notify == nil {
extra.Notify = notifyNil
}
if extra.NotifyErr == nil {
extra.NotifyErr = notifyErrNil
}
var wg sync.WaitGroup
wg.Add(blockCnt)
last := blockCnt - 1
blkSize := 1 << blockBits
nfails := 0
c := NewClient(uptoken, nil)
for i := 0; i < blockCnt; i++ {
blkIdx := i
blkSize1 := blkSize
if i == last {
offbase := int64(blkIdx) << blockBits
blkSize1 = int(fsize - offbase)
}
task := func() {
defer wg.Done()
tryTimes := extra.TryTimes
lzRetry:
err := ResumableBlockput(c, l, &extra.Progresses[blkIdx], f, blkIdx, blkSize1, extra)
if err != nil {
if tryTimes > 1 {
tryTimes--
log.Info("resumable.Put retrying ...")
goto lzRetry
}
log.Warn("resumable.Put", blkIdx, "failed:", err)
extra.NotifyErr(blkIdx, blkSize1, err)
nfails++
}
}
tasks <- task
}
wg.Wait()
if nfails != 0 {
return ErrPutFailed
}
return Mkfile(c, l, ret, key, hasKey, fsize, extra)
}
// init + build + publish + clean
func (j *Builder) Auto() (addr string, err error) {
lock := utils.NewNameLock(j.project)
lock.Lock()
defer func() {
lock.Unlock()
if j.wbc != nil {
j.wbc.CloseWriters()
}
}()
if err = j.init(); err != nil {
return
}
// defer clean should start when GOPATH success created
defer func() {
er := j.clean()
if er != nil {
log.Warn(er)
}
if err != nil && j.wbc != nil {
io.WriteString(j.wbc, err.Error())
}
// FIXME: delete WriteBroadcaster after finish
// if build error, output will not saved, it is not a good idea
// better to change database func to -..
// SearchProject(project) AddProject(project)
// SearchFile(pid, ref, os, arch)
// AddFile(pid, ref, os, arch, sha)
}()
// download src (in order to get sha)
err = j.get()
if err != nil {
log.Error("download src error:", err)
return
}
// search db for history project record
log.Info("request project:", j.project)
log.Info("current sha:", j.sha)
p, err := database.SearchProject(j.project, j.sha)
if err != nil {
log.Info("exists in db", j.project, j.ref, j.sha)
pid, er := database.AddProject(j.project, j.ref, j.sha)
if er != nil {
err = er
return
}
j.pid = pid // project id
} else {
j.pid = p.Id
}
// generate tag for builded-file search
j.tag = fmt.Sprintf("%d-%s-%s", j.pid, j.os, j.arch)
//log.Info("tag:", j.tag)
// search memory history
hisAddr, ok := history[j.tag]
if ok {
return hisAddr, nil
}
// search database history
f, err := database.SearchFile(j.pid, j.tag)
log.Debugf("search db: %v", f)
if err == nil {
addr = f.Addr
return
}
// build xc
// file maybe empty
file, err := j.build(j.os, j.arch)
if err != nil {
log.Error("build error:", err)
return
}
// package build file(include upload)
addr, err = j.publish(file)
if err != nil {
return
}
return
}
func (ps *ProxyServer) newControlHandler() func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
// read listen port from request
protocol, subdomain, port := parseConnectRequest(r)
log.Debugf("proxy listen proto: %v, subdomain: %v port: %v", protocol, subdomain, port)
// create websocket connection
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
http.Error(w, err.Error(), 502)
return
}
defer conn.Close()
log.Debug("remote client addr:", conn.RemoteAddr())
tunnel := &Tunnel{
wsconn: conn,
}
// TCP: create new port to listen
log.Infof("New %s proxy for %v", protocol, conn.RemoteAddr())
switch protocol {
case "tcp":
// proxyAddr := fmt.Sprintf("0.0.0.0:%d", port)
listener, err := NewTcpProxyListener(tunnel, port)
if err != nil {
log.Warnf("new tcp proxy err: %v", err)
http.Error(w, err.Error(), 501)
return
}
defer listener.Close()
_, port, _ := net.SplitHostPort(listener.Addr().String())
wsSendMessage(conn, fmt.Sprintf(
"Local tcp conn is now publicly available via:\n%v:%v\n", ps.domain, port))
case "http", "https":
tr := &http.Transport{
Dial: tunnel.generateTransportDial(),
}
revProxy := &httputil.ReverseProxy{
Director: func(req *http.Request) {
log.Println("director:", req.RequestURI)
},
Transport: tr,
}
// should hook here
// hook(HOOK_CREATE_HTTP_SUBDOMAIN, subdomain)
// generate a uniq domain
if subdomain == "" {
subdomain = uniqName(5) + ".t"
}
pxDomain := subdomain + "." + ps.domain
log.Println("http px use domain:", pxDomain)
if _, exists := ps.revProxies[pxDomain]; exists {
wsSendMessage(conn, fmt.Sprintf("subdomain [%s] has already been taken", pxDomain))
return
}
ps.Lock()
ps.revProxies[pxDomain] = revProxy
ps.Unlock()
wsSendMessage(conn, fmt.Sprintf(
"Local server is now publicly available via:\nhttp://%s\n", pxDomain))
defer func() {
ps.Lock()
delete(ps.revProxies, pxDomain)
ps.Unlock()
}()
default:
log.Warn("unknown protocol:", protocol)
return
}
// HTTP: use httputil.ReverseProxy
for {
var msg Msg
if err := conn.ReadJSON(&msg); err != nil {
log.Warn(err)
break
}
log.Info("recv json:", msg)
}
}
}
func ResumableBlockput(
c rpc.Client, l rpc.Logger, ret *BlkputRet, f io.ReaderAt, blkIdx, blkSize int, extra *PutExtra) (err error) {
h := crc32.NewIEEE()
offbase := int64(blkIdx) << blockBits
chunkSize := extra.ChunkSize
var bodyLength int
if ret.Ctx == "" {
if chunkSize < blkSize {
bodyLength = chunkSize
} else {
bodyLength = blkSize
}
body1 := io.NewSectionReader(f, offbase, int64(bodyLength))
body := io.TeeReader(body1, h)
err = Mkblock(c, l, ret, blkSize, body, bodyLength)
if err != nil {
return
}
if ret.Crc32 != h.Sum32() || int(ret.Offset) != bodyLength {
err = ErrUnmatchedChecksum
return
}
extra.Notify(blkIdx, blkSize, ret)
}
for int(ret.Offset) < blkSize {
if chunkSize < blkSize-int(ret.Offset) {
bodyLength = chunkSize
} else {
bodyLength = blkSize - int(ret.Offset)
}
tryTimes := extra.TryTimes
lzRetry:
h.Reset()
body1 := io.NewSectionReader(f, offbase+int64(ret.Offset), int64(bodyLength))
body := io.TeeReader(body1, h)
err = Blockput(c, l, ret, body, bodyLength)
if err == nil {
if ret.Crc32 == h.Sum32() {
extra.Notify(blkIdx, blkSize, ret)
continue
}
log.Warn("ResumableBlockput: invalid checksum, retry")
err = ErrUnmatchedChecksum
} else {
if ei, ok := err.(*rpc.ErrorInfo); ok && ei.Code == InvalidCtx {
ret.Ctx = "" // reset
log.Warn("ResumableBlockput: invalid ctx, please retry")
return
}
log.Warn("ResumableBlockput: bput failed -", err)
}
if tryTimes > 1 {
tryTimes--
log.Info("ResumableBlockput retrying ...")
goto lzRetry
}
break
}
return
}