func (self *Runner) handlerOneMail(m *mailproto.Message) { uid, err := self.db.GetUserByEmail(m.From) if err != nil { log.Warn("GetUserByEmail failed:", m.From, err) return } appName, err := mailproto.EmailAddress2App(m.To[0]) if err != nil { log.Warn("EmailAddress2App failed:", m.To, err) return } err = self.db.CheckUserApp(uid, appName) if err != nil { log.Warn("CheckUserApp failed:", err) return } addr, err := self.db.GetUsersMail([]string{uid}) if err != nil { log.Warn("GetUsersMail failed:", err) return } m.From = "" m.To = addr _, err = self.mail.Send(m) if err != nil { log.Warn("Send failed", err) return } }
func QiniuUpload(cmd string, params ...string) { if len(params) == 1 || len(params) == 2 { var uploadConfigFile string var threadCount int64 var err error if len(params) == 2 { threadCount, err = strconv.ParseInt(params[0], 10, 64) if err != nil { log.Error("Invalid <ThreadCount> value,", params[0]) return } uploadConfigFile = params[1] } else { uploadConfigFile = params[0] } if threadCount < qshell.MIN_UPLOAD_THREAD_COUNT || threadCount > qshell.MAX_UPLOAD_THREAD_COUNT { log.Warn("<ThreadCount> can only between 1 and 100") threadCount = qshell.MIN_UPLOAD_THREAD_COUNT } qshell.QiniuUpload(int(threadCount), uploadConfigFile) } else { CmdHelp(cmd) } }
func QiniuDownload(cmd string, params ...string) { if len(params) == 1 || len(params) == 2 { var threadCount int64 = 5 var downConfig string var err error if len(params) == 1 { downConfig = params[0] } else { threadCount, err = strconv.ParseInt(params[0], 10, 64) if err != nil { log.Error("Invalid value for <ThreadCount>", params[0]) return } downConfig = params[1] } if threadCount < qshell.MIN_DOWNLOAD_THREAD_COUNT || threadCount > qshell.MAX_DOWNLOAD_THREAD_COUNT { log.Warn("<ThreadCount> can only between 1 and 100") threadCount = qshell.MIN_DOWNLOAD_THREAD_COUNT } qshell.QiniuDownload(int(threadCount), downConfig) } else { CmdHelp(cmd) } }
// Listen and forward connections func NewTcpProxyListener(tunnel *Tunnel, port int) (listener *net.TCPListener, err error) { var laddr *net.TCPAddr if port != 0 { laddr, _ = net.ResolveTCPAddr("tcp", ":"+strconv.Itoa(port)) listener, err = net.ListenTCP("tcp", laddr) } else { laddr, listener, err = freeport.ListenTCP() } if err != nil { return nil, err } port = laddr.Port //port, listener, err = listenTcpInRangePort(port, TCP_MIN_PORT, TCP_MAX_PORT) //if err != nil { //return nil, err //} // hook here err = hook(HOOK_TCP_POST_CONNECT, []string{ "PORT=" + strconv.Itoa(port), "REMOTE_ADDR=" + tunnel.wsconn.RemoteAddr().String(), "CLIENT_ADDRESS=" + tunnel.wsconn.RemoteAddr().String(), "REMOTE_DATA=" + tunnel.data, }) if err != nil { listener.Close() return } go func() { for { rconn, err := listener.AcceptTCP() if err != nil { log.Warn(err) break } // find proxy to where log.Debug("Receive new connections from", rconn.RemoteAddr()) lconn, err := tunnel.RequestNewConn(rconn.RemoteAddr().String()) if err != nil { log.Debug("request new conn err:", err) rconn.Close() continue } log.Debug("request new conn:", lconn, err) pc := &ProxyConn{ lconn: lconn, rconn: rconn, stats: proxyStats, } go pc.start() } }() return listener, nil }
// FIXME(ssx): maybe need to return error func (p *Process) buildCommand() *kexec.KCommand { cmd := kexec.CommandString(p.Command) // cmd := kexec.Command(p.Command[0], p.Command[1:]...) logDir := filepath.Join(defaultConfigDir, "log", sanitize.Name(p.Name)) if !IsDir(logDir) { os.MkdirAll(logDir, 0755) } var fout io.Writer var err error p.OutputFile, err = os.OpenFile(filepath.Join(logDir, "output.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { log.Warn("create stdout log failed:", err) fout = ioutil.Discard } else { fout = p.OutputFile } cmd.Stdout = io.MultiWriter(p.Stdout, p.Output, fout) cmd.Stderr = io.MultiWriter(p.Stderr, p.Output, fout) // config environ cmd.Env = os.Environ() // inherit current vars environ := map[string]string{} if p.User != "" { err := cmd.SetUser(p.User) if err != nil { log.Warnf("[%s] chusr to %s failed", p.Name, p.User) } else { var homeDir string switch runtime.GOOS { case "linux": homeDir = "/home/" + p.User // FIXME(ssx): maybe there is a better way case "darwin": homeDir = "/Users/" + p.User } cmd.Env = append(cmd.Env, "HOME="+homeDir, "USER="******"HOME"] = homeDir environ["USER"] = p.User } } cmd.Env = append(cmd.Env, p.Environ...) mapping := func(key string) string { val := os.Getenv(key) if val != "" { return val } return environ[key] } cmd.Dir = os.Expand(p.Dir, mapping) if strings.HasPrefix(cmd.Dir, "~") { cmd.Dir = mapping("HOME") + cmd.Dir[1:] } log.Infof("[%s] use dir: %s\n", p.Name, cmd.Dir) return cmd }
func (t *Task) Synced() (cl changelog.Logger, urls []string, err error) { dir, err := cc.GetConfigDir("fetch/") if err != nil { log.Warn(err, "GetConfigDir failed") return } pos := strings.LastIndex(t.SyncSrc, "/") if pos == -1 { pos = 0 } eigen := t.SyncSrc[pos:] + t.Bucket + t.Style cl, urls, err = OpenChangeLog(dir, eigen) return }
// download src func (b *Builder) get() (err error) { log.Debug("start get src to:", b.srcDir) exists := beeutils.FileExists(b.srcDir) b.sh.Command("go", "version").Run() if !exists { err = b.sh.Command("go", "get", "-v", "-d", b.project).Run() if err != nil { return } } b.sh.SetDir(b.srcDir) if b.ref == "-" { b.ref = "master" } // get code from remote if err = b.sh.Command("git", "fetch", "origin").Run(); err != nil { return } // change branch if err = b.sh.Command("git", "checkout", "-q", b.ref).Run(); err != nil { return } // update code if err = b.sh.Command("git", "merge", "origin/"+b.ref).Run(); err != nil { log.Warn("git merge error:", err) //return } // get sha out, err := sh.Command("git", "rev-parse", "HEAD", sh.Dir(b.srcDir)).Output() if err != nil { return } b.sha = strings.TrimSpace(string(out)) // parse .gobuild b.rc = new(Assembly) rcfile := "public/gobuildrc" if b.sh.Test("f", ".gobuild") { rcfile = filepath.Join(b.srcDir, ".gobuild") } data, err := ioutil.ReadFile(rcfile) if err != nil { return } err = goyaml.Unmarshal(data, b.rc) return }
func main() { flag.Parse() if flag.NArg() < 1 { help() return } var task Task var taskFile = flag.Arg(0) err := config.LoadEx(&task, taskFile) if err != nil || !task.Isvalid() { log.Warn(err, "config.Load failed:", taskFile) return } task.Run() }
func NewScriptHandler(script string, contentType string) func(*macaron.Context) { ext := filepath.Ext(script) return func(ctx *macaron.Context) { var err error var output []byte errquit := func(err error) { ctx.Error(500, err.Error()+"\n"+string(output)) return } switch ext { case ".sh": output, err = runShellScript(ctx, script) render(ctx, output, err, contentType) case ".html": output, err = ioutil.ReadFile(script) render(ctx, output, err, contentType) case ".md": output, err = ioutil.ReadFile(script) if err != nil { errquit(err) return } unsafe := blackfriday.MarkdownCommon(output) html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) render(ctx, html, nil, "html") case ".jade": tmpl, err := jade.CompileFile(script, jade.Options{}) if err != nil { errquit(err) return } buf := bytes.NewBuffer(nil) err = tmpl.Execute(buf, nil) render(ctx, buf.Bytes(), err, "html") default: log.Warn("Unknown script ext", script, ext) output, err = ioutil.ReadFile(script) render(ctx, output, err, detectCType(script)) } } }
func serveRevConn(proto ProxyProtocol, pAddr string, lis net.Listener) error { switch proto { case TCP: for { rconn, err := lis.Accept() if err != nil { log.Errorf("accept error: %v", err) return err } log.Info("local dial tcp", pAddr) lconn, err := net.Dial("tcp", pAddr) if err != nil { log.Warn(err) rconn.Close() return err } // start forward local proxy pc := &proxyConn{ lconn: lconn, rconn: rconn, stats: proxyStats, } go pc.start() } case HTTP: rp := &httputil.ReverseProxy{ Director: func(req *http.Request) { req.Host = pAddr req.URL.Scheme = "http" req.URL.Host = pAddr }, } return http.Serve(lis, rp) default: log.Println("Unknown protocol:", proto) return ErrUnknownProtocol } }
func (p *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { h := p.route[req.URL.Host] if h == nil { log.Warn("Server not found:", req.Host) return nil, ErrServerNotFound } cp := *req cp.Body = &mockServerRequestBody{req.Body, false} req = &cp req.RemoteAddr = "127.0.0.1:8000" rw := httptest.NewRecorder() h.ServeHTTP(rw, req) req.Body.Close() ctlen := int64(-1) if v := rw.HeaderMap.Get("Content-Length"); v != "" { ctlen, _ = strconv.ParseInt(v, 10, 64) } return &http.Response{ Status: "", StatusCode: rw.Code, Header: rw.HeaderMap, Body: ioutil.NopCloser(rw.Body), ContentLength: ctlen, TransferEncoding: nil, Close: false, Trailer: nil, Request: req, }, nil }
func put( l rpc.Logger, ret interface{}, uptoken string, key string, hasKey bool, f io.ReaderAt, fsize int64, extra *PutExtra) error { once.Do(initWorkers) blockCnt := BlockCount(fsize) if extra == nil { extra = new(PutExtra) } if extra.Progresses == nil { extra.Progresses = make([]BlkputRet, blockCnt) } else if len(extra.Progresses) != blockCnt { return ErrInvalidPutProgress } if extra.ChunkSize == 0 { extra.ChunkSize = settings.ChunkSize } if extra.TryTimes == 0 { extra.TryTimes = settings.TryTimes } if extra.Notify == nil { extra.Notify = notifyNil } if extra.NotifyErr == nil { extra.NotifyErr = notifyErrNil } var wg sync.WaitGroup wg.Add(blockCnt) last := blockCnt - 1 blkSize := 1 << blockBits nfails := 0 c := NewClient(uptoken, nil) for i := 0; i < blockCnt; i++ { blkIdx := i blkSize1 := blkSize if i == last { offbase := int64(blkIdx) << blockBits blkSize1 = int(fsize - offbase) } task := func() { defer wg.Done() tryTimes := extra.TryTimes lzRetry: err := ResumableBlockput(c, l, &extra.Progresses[blkIdx], f, blkIdx, blkSize1, extra) if err != nil { if tryTimes > 1 { tryTimes-- log.Info("resumable.Put retrying ...") goto lzRetry } log.Warn("resumable.Put", blkIdx, "failed:", err) extra.NotifyErr(blkIdx, blkSize1, err) nfails++ } } tasks <- task } wg.Wait() if nfails != 0 { return ErrPutFailed } return Mkfile(c, l, ret, key, hasKey, fsize, extra) }
// init + build + publish + clean func (j *Builder) Auto() (addr string, err error) { lock := utils.NewNameLock(j.project) lock.Lock() defer func() { lock.Unlock() if j.wbc != nil { j.wbc.CloseWriters() } }() if err = j.init(); err != nil { return } // defer clean should start when GOPATH success created defer func() { er := j.clean() if er != nil { log.Warn(er) } if err != nil && j.wbc != nil { io.WriteString(j.wbc, err.Error()) } // FIXME: delete WriteBroadcaster after finish // if build error, output will not saved, it is not a good idea // better to change database func to -.. // SearchProject(project) AddProject(project) // SearchFile(pid, ref, os, arch) // AddFile(pid, ref, os, arch, sha) }() // download src (in order to get sha) err = j.get() if err != nil { log.Error("download src error:", err) return } // search db for history project record log.Info("request project:", j.project) log.Info("current sha:", j.sha) p, err := database.SearchProject(j.project, j.sha) if err != nil { log.Info("exists in db", j.project, j.ref, j.sha) pid, er := database.AddProject(j.project, j.ref, j.sha) if er != nil { err = er return } j.pid = pid // project id } else { j.pid = p.Id } // generate tag for builded-file search j.tag = fmt.Sprintf("%d-%s-%s", j.pid, j.os, j.arch) //log.Info("tag:", j.tag) // search memory history hisAddr, ok := history[j.tag] if ok { return hisAddr, nil } // search database history f, err := database.SearchFile(j.pid, j.tag) log.Debugf("search db: %v", f) if err == nil { addr = f.Addr return } // build xc // file maybe empty file, err := j.build(j.os, j.arch) if err != nil { log.Error("build error:", err) return } // package build file(include upload) addr, err = j.publish(file) if err != nil { return } return }
func (ps *ProxyServer) newControlHandler() func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { // read listen port from request protocol, subdomain, port := parseConnectRequest(r) log.Debugf("proxy listen proto: %v, subdomain: %v port: %v", protocol, subdomain, port) // create websocket connection conn, err := upgrader.Upgrade(w, r, nil) if err != nil { http.Error(w, err.Error(), 502) return } defer conn.Close() log.Debug("remote client addr:", conn.RemoteAddr()) tunnel := &Tunnel{ wsconn: conn, } // TCP: create new port to listen log.Infof("New %s proxy for %v", protocol, conn.RemoteAddr()) switch protocol { case "tcp": // proxyAddr := fmt.Sprintf("0.0.0.0:%d", port) listener, err := NewTcpProxyListener(tunnel, port) if err != nil { log.Warnf("new tcp proxy err: %v", err) http.Error(w, err.Error(), 501) return } defer listener.Close() _, port, _ := net.SplitHostPort(listener.Addr().String()) wsSendMessage(conn, fmt.Sprintf( "Local tcp conn is now publicly available via:\n%v:%v\n", ps.domain, port)) case "http", "https": tr := &http.Transport{ Dial: tunnel.generateTransportDial(), } revProxy := &httputil.ReverseProxy{ Director: func(req *http.Request) { log.Println("director:", req.RequestURI) }, Transport: tr, } // should hook here // hook(HOOK_CREATE_HTTP_SUBDOMAIN, subdomain) // generate a uniq domain if subdomain == "" { subdomain = uniqName(5) + ".t" } pxDomain := subdomain + "." + ps.domain log.Println("http px use domain:", pxDomain) if _, exists := ps.revProxies[pxDomain]; exists { wsSendMessage(conn, fmt.Sprintf("subdomain [%s] has already been taken", pxDomain)) return } ps.Lock() ps.revProxies[pxDomain] = revProxy ps.Unlock() wsSendMessage(conn, fmt.Sprintf( "Local server is now publicly available via:\nhttp://%s\n", pxDomain)) defer func() { ps.Lock() delete(ps.revProxies, pxDomain) ps.Unlock() }() default: log.Warn("unknown protocol:", protocol) return } // HTTP: use httputil.ReverseProxy for { var msg Msg if err := conn.ReadJSON(&msg); err != nil { log.Warn(err) break } log.Info("recv json:", msg) } } }
func ResumableBlockput( c rpc.Client, l rpc.Logger, ret *BlkputRet, f io.ReaderAt, blkIdx, blkSize int, extra *PutExtra) (err error) { h := crc32.NewIEEE() offbase := int64(blkIdx) << blockBits chunkSize := extra.ChunkSize var bodyLength int if ret.Ctx == "" { if chunkSize < blkSize { bodyLength = chunkSize } else { bodyLength = blkSize } body1 := io.NewSectionReader(f, offbase, int64(bodyLength)) body := io.TeeReader(body1, h) err = Mkblock(c, l, ret, blkSize, body, bodyLength) if err != nil { return } if ret.Crc32 != h.Sum32() || int(ret.Offset) != bodyLength { err = ErrUnmatchedChecksum return } extra.Notify(blkIdx, blkSize, ret) } for int(ret.Offset) < blkSize { if chunkSize < blkSize-int(ret.Offset) { bodyLength = chunkSize } else { bodyLength = blkSize - int(ret.Offset) } tryTimes := extra.TryTimes lzRetry: h.Reset() body1 := io.NewSectionReader(f, offbase+int64(ret.Offset), int64(bodyLength)) body := io.TeeReader(body1, h) err = Blockput(c, l, ret, body, bodyLength) if err == nil { if ret.Crc32 == h.Sum32() { extra.Notify(blkIdx, blkSize, ret) continue } log.Warn("ResumableBlockput: invalid checksum, retry") err = ErrUnmatchedChecksum } else { if ei, ok := err.(*rpc.ErrorInfo); ok && ei.Code == InvalidCtx { ret.Ctx = "" // reset log.Warn("ResumableBlockput: invalid ctx, please retry") return } log.Warn("ResumableBlockput: bput failed -", err) } if tryTimes > 1 { tryTimes-- log.Info("ResumableBlockput retrying ...") goto lzRetry } break } return }