Beispiel #1
0
func (self *Runner) handlerOneMail(m *mailproto.Message) {

	uid, err := self.db.GetUserByEmail(m.From)
	if err != nil {
		log.Warn("GetUserByEmail failed:", m.From, err)
		return
	}
	appName, err := mailproto.EmailAddress2App(m.To[0])
	if err != nil {
		log.Warn("EmailAddress2App failed:", m.To, err)
		return
	}
	err = self.db.CheckUserApp(uid, appName)
	if err != nil {
		log.Warn("CheckUserApp failed:", err)
		return
	}
	addr, err := self.db.GetUsersMail([]string{uid})
	if err != nil {
		log.Warn("GetUsersMail failed:", err)
		return
	}

	m.From = ""
	m.To = addr
	_, err = self.mail.Send(m)
	if err != nil {
		log.Warn("Send failed", err)
		return
	}
}
Beispiel #2
0
func QiniuUpload(cmd string, params ...string) {
	if len(params) == 1 || len(params) == 2 {
		var uploadConfigFile string
		var threadCount int64
		var err error
		if len(params) == 2 {
			threadCount, err = strconv.ParseInt(params[0], 10, 64)
			if err != nil {
				log.Error("Invalid <ThreadCount> value,", params[0])
				return
			}
			uploadConfigFile = params[1]
		} else {
			uploadConfigFile = params[0]
		}
		if threadCount < qshell.MIN_UPLOAD_THREAD_COUNT ||
			threadCount > qshell.MAX_UPLOAD_THREAD_COUNT {
			log.Warn("<ThreadCount> can only between 1 and 100")
			threadCount = qshell.MIN_UPLOAD_THREAD_COUNT
		}
		qshell.QiniuUpload(int(threadCount), uploadConfigFile)
	} else {
		CmdHelp(cmd)
	}
}
Beispiel #3
0
func QiniuDownload(cmd string, params ...string) {
	if len(params) == 1 || len(params) == 2 {
		var threadCount int64 = 5
		var downConfig string
		var err error
		if len(params) == 1 {
			downConfig = params[0]
		} else {
			threadCount, err = strconv.ParseInt(params[0], 10, 64)
			if err != nil {
				log.Error("Invalid value for <ThreadCount>", params[0])
				return
			}
			downConfig = params[1]
		}
		if threadCount < qshell.MIN_DOWNLOAD_THREAD_COUNT ||
			threadCount > qshell.MAX_DOWNLOAD_THREAD_COUNT {
			log.Warn("<ThreadCount> can only between 1 and 100")
			threadCount = qshell.MIN_DOWNLOAD_THREAD_COUNT
		}
		qshell.QiniuDownload(int(threadCount), downConfig)
	} else {
		CmdHelp(cmd)
	}
}
Beispiel #4
0
// Listen and forward connections
func NewTcpProxyListener(tunnel *Tunnel, port int) (listener *net.TCPListener, err error) {
	var laddr *net.TCPAddr
	if port != 0 {
		laddr, _ = net.ResolveTCPAddr("tcp", ":"+strconv.Itoa(port))
		listener, err = net.ListenTCP("tcp", laddr)
	} else {
		laddr, listener, err = freeport.ListenTCP()
	}
	if err != nil {
		return nil, err
	}
	port = laddr.Port
	//port, listener, err = listenTcpInRangePort(port, TCP_MIN_PORT, TCP_MAX_PORT)
	//if err != nil {
	//return nil, err
	//}
	// hook here
	err = hook(HOOK_TCP_POST_CONNECT, []string{
		"PORT=" + strconv.Itoa(port),
		"REMOTE_ADDR=" + tunnel.wsconn.RemoteAddr().String(),
		"CLIENT_ADDRESS=" + tunnel.wsconn.RemoteAddr().String(),
		"REMOTE_DATA=" + tunnel.data,
	})
	if err != nil {
		listener.Close()
		return
	}

	go func() {
		for {
			rconn, err := listener.AcceptTCP()
			if err != nil {
				log.Warn(err)
				break
			}
			// find proxy to where
			log.Debug("Receive new connections from", rconn.RemoteAddr())
			lconn, err := tunnel.RequestNewConn(rconn.RemoteAddr().String())
			if err != nil {
				log.Debug("request new conn err:", err)
				rconn.Close()
				continue
			}

			log.Debug("request new conn:", lconn, err)
			pc := &ProxyConn{
				lconn: lconn,
				rconn: rconn,
				stats: proxyStats,
			}
			go pc.start()
		}
	}()
	return listener, nil
}
Beispiel #5
0
// FIXME(ssx): maybe need to return error
func (p *Process) buildCommand() *kexec.KCommand {
	cmd := kexec.CommandString(p.Command)
	// cmd := kexec.Command(p.Command[0], p.Command[1:]...)
	logDir := filepath.Join(defaultConfigDir, "log", sanitize.Name(p.Name))
	if !IsDir(logDir) {
		os.MkdirAll(logDir, 0755)
	}
	var fout io.Writer
	var err error
	p.OutputFile, err = os.OpenFile(filepath.Join(logDir, "output.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		log.Warn("create stdout log failed:", err)
		fout = ioutil.Discard
	} else {
		fout = p.OutputFile
	}
	cmd.Stdout = io.MultiWriter(p.Stdout, p.Output, fout)
	cmd.Stderr = io.MultiWriter(p.Stderr, p.Output, fout)
	// config environ
	cmd.Env = os.Environ() // inherit current vars
	environ := map[string]string{}
	if p.User != "" {
		err := cmd.SetUser(p.User)
		if err != nil {
			log.Warnf("[%s] chusr to %s failed", p.Name, p.User)
		} else {
			var homeDir string
			switch runtime.GOOS {
			case "linux":
				homeDir = "/home/" + p.User // FIXME(ssx): maybe there is a better way
			case "darwin":
				homeDir = "/Users/" + p.User
			}
			cmd.Env = append(cmd.Env, "HOME="+homeDir, "USER="******"HOME"] = homeDir
			environ["USER"] = p.User
		}
	}
	cmd.Env = append(cmd.Env, p.Environ...)
	mapping := func(key string) string {
		val := os.Getenv(key)
		if val != "" {
			return val
		}
		return environ[key]
	}
	cmd.Dir = os.Expand(p.Dir, mapping)
	if strings.HasPrefix(cmd.Dir, "~") {
		cmd.Dir = mapping("HOME") + cmd.Dir[1:]
	}
	log.Infof("[%s] use dir: %s\n", p.Name, cmd.Dir)
	return cmd
}
Beispiel #6
0
func (t *Task) Synced() (cl changelog.Logger, urls []string, err error) {
	dir, err := cc.GetConfigDir("fetch/")
	if err != nil {
		log.Warn(err, "GetConfigDir failed")
		return
	}

	pos := strings.LastIndex(t.SyncSrc, "/")
	if pos == -1 {
		pos = 0
	}
	eigen := t.SyncSrc[pos:] + t.Bucket + t.Style
	cl, urls, err = OpenChangeLog(dir, eigen)
	return
}
Beispiel #7
0
// download src
func (b *Builder) get() (err error) {
	log.Debug("start get src to:", b.srcDir)
	exists := beeutils.FileExists(b.srcDir)
	b.sh.Command("go", "version").Run()
	if !exists {
		err = b.sh.Command("go", "get", "-v", "-d", b.project).Run()
		if err != nil {
			return
		}
	}
	b.sh.SetDir(b.srcDir)
	if b.ref == "-" {
		b.ref = "master"
	}
	// get code from remote
	if err = b.sh.Command("git", "fetch", "origin").Run(); err != nil {
		return
	}
	// change branch
	if err = b.sh.Command("git", "checkout", "-q", b.ref).Run(); err != nil {
		return
	}
	// update code
	if err = b.sh.Command("git", "merge", "origin/"+b.ref).Run(); err != nil {
		log.Warn("git merge error:", err)
		//return
	}
	// get sha
	out, err := sh.Command("git", "rev-parse", "HEAD", sh.Dir(b.srcDir)).Output()
	if err != nil {
		return
	}
	b.sha = strings.TrimSpace(string(out))

	// parse .gobuild
	b.rc = new(Assembly)
	rcfile := "public/gobuildrc"
	if b.sh.Test("f", ".gobuild") {
		rcfile = filepath.Join(b.srcDir, ".gobuild")
	}
	data, err := ioutil.ReadFile(rcfile)
	if err != nil {
		return
	}
	err = goyaml.Unmarshal(data, b.rc)
	return
}
Beispiel #8
0
func main() {

	flag.Parse()

	if flag.NArg() < 1 {
		help()
		return
	}

	var task Task
	var taskFile = flag.Arg(0)
	err := config.LoadEx(&task, taskFile)
	if err != nil || !task.Isvalid() {
		log.Warn(err, "config.Load failed:", taskFile)
		return
	}

	task.Run()
}
Beispiel #9
0
func NewScriptHandler(script string, contentType string) func(*macaron.Context) {
	ext := filepath.Ext(script)
	return func(ctx *macaron.Context) {
		var err error
		var output []byte
		errquit := func(err error) {
			ctx.Error(500, err.Error()+"\n"+string(output))
			return
		}
		switch ext {
		case ".sh":
			output, err = runShellScript(ctx, script)
			render(ctx, output, err, contentType)
		case ".html":
			output, err = ioutil.ReadFile(script)
			render(ctx, output, err, contentType)
		case ".md":
			output, err = ioutil.ReadFile(script)
			if err != nil {
				errquit(err)
				return
			}
			unsafe := blackfriday.MarkdownCommon(output)
			html := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
			render(ctx, html, nil, "html")
		case ".jade":
			tmpl, err := jade.CompileFile(script, jade.Options{})
			if err != nil {
				errquit(err)
				return
			}
			buf := bytes.NewBuffer(nil)
			err = tmpl.Execute(buf, nil)
			render(ctx, buf.Bytes(), err, "html")
		default:
			log.Warn("Unknown script ext", script, ext)
			output, err = ioutil.ReadFile(script)
			render(ctx, output, err, detectCType(script))
		}
	}
}
Beispiel #10
0
func serveRevConn(proto ProxyProtocol, pAddr string, lis net.Listener) error {
	switch proto {
	case TCP:
		for {
			rconn, err := lis.Accept()
			if err != nil {
				log.Errorf("accept error: %v", err)
				return err
			}
			log.Info("local dial tcp", pAddr)
			lconn, err := net.Dial("tcp", pAddr)
			if err != nil {
				log.Warn(err)
				rconn.Close()
				return err
			}
			// start forward local proxy
			pc := &proxyConn{
				lconn: lconn,
				rconn: rconn,
				stats: proxyStats,
			}
			go pc.start()
		}
	case HTTP:
		rp := &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.Host = pAddr
				req.URL.Scheme = "http"
				req.URL.Host = pAddr
			},
		}
		return http.Serve(lis, rp)
	default:
		log.Println("Unknown protocol:", proto)
		return ErrUnknownProtocol
	}
}
func (p *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {

	h := p.route[req.URL.Host]
	if h == nil {
		log.Warn("Server not found:", req.Host)
		return nil, ErrServerNotFound
	}

	cp := *req
	cp.Body = &mockServerRequestBody{req.Body, false}
	req = &cp

	req.RemoteAddr = "127.0.0.1:8000"

	rw := httptest.NewRecorder()
	h.ServeHTTP(rw, req)

	req.Body.Close()

	ctlen := int64(-1)
	if v := rw.HeaderMap.Get("Content-Length"); v != "" {
		ctlen, _ = strconv.ParseInt(v, 10, 64)
	}

	return &http.Response{
		Status:           "",
		StatusCode:       rw.Code,
		Header:           rw.HeaderMap,
		Body:             ioutil.NopCloser(rw.Body),
		ContentLength:    ctlen,
		TransferEncoding: nil,
		Close:            false,
		Trailer:          nil,
		Request:          req,
	}, nil
}
Beispiel #12
0
func put(
	l rpc.Logger, ret interface{}, uptoken string, key string, hasKey bool, f io.ReaderAt, fsize int64, extra *PutExtra) error {

	once.Do(initWorkers)

	blockCnt := BlockCount(fsize)

	if extra == nil {
		extra = new(PutExtra)
	}
	if extra.Progresses == nil {
		extra.Progresses = make([]BlkputRet, blockCnt)
	} else if len(extra.Progresses) != blockCnt {
		return ErrInvalidPutProgress
	}

	if extra.ChunkSize == 0 {
		extra.ChunkSize = settings.ChunkSize
	}
	if extra.TryTimes == 0 {
		extra.TryTimes = settings.TryTimes
	}
	if extra.Notify == nil {
		extra.Notify = notifyNil
	}
	if extra.NotifyErr == nil {
		extra.NotifyErr = notifyErrNil
	}

	var wg sync.WaitGroup
	wg.Add(blockCnt)

	last := blockCnt - 1
	blkSize := 1 << blockBits
	nfails := 0
	c := NewClient(uptoken, nil)

	for i := 0; i < blockCnt; i++ {
		blkIdx := i
		blkSize1 := blkSize
		if i == last {
			offbase := int64(blkIdx) << blockBits
			blkSize1 = int(fsize - offbase)
		}
		task := func() {
			defer wg.Done()
			tryTimes := extra.TryTimes
		lzRetry:
			err := ResumableBlockput(c, l, &extra.Progresses[blkIdx], f, blkIdx, blkSize1, extra)
			if err != nil {
				if tryTimes > 1 {
					tryTimes--
					log.Info("resumable.Put retrying ...")
					goto lzRetry
				}
				log.Warn("resumable.Put", blkIdx, "failed:", err)
				extra.NotifyErr(blkIdx, blkSize1, err)
				nfails++
			}
		}
		tasks <- task
	}

	wg.Wait()
	if nfails != 0 {
		return ErrPutFailed
	}

	return Mkfile(c, l, ret, key, hasKey, fsize, extra)
}
Beispiel #13
0
// init + build + publish + clean
func (j *Builder) Auto() (addr string, err error) {
	lock := utils.NewNameLock(j.project)
	lock.Lock()
	defer func() {
		lock.Unlock()
		if j.wbc != nil {
			j.wbc.CloseWriters()
		}
	}()
	if err = j.init(); err != nil {
		return
	}
	// defer clean should start when GOPATH success created
	defer func() {
		er := j.clean()
		if er != nil {
			log.Warn(er)
		}
		if err != nil && j.wbc != nil {
			io.WriteString(j.wbc, err.Error())
		}
		// FIXME: delete WriteBroadcaster after finish
		// if build error, output will not saved, it is not a good idea
		// better to change database func to -..
		//		SearchProject(project) AddProject(project)
		//		SearchFile(pid, ref, os, arch)
		//		AddFile(pid, ref, os, arch, sha)
	}()
	// download src (in order to get sha)
	err = j.get()
	if err != nil {
		log.Error("download src error:", err)
		return
	}
	// search db for history project record
	log.Info("request project:", j.project)
	log.Info("current sha:", j.sha)
	p, err := database.SearchProject(j.project, j.sha)
	if err != nil {
		log.Info("exists in db", j.project, j.ref, j.sha)
		pid, er := database.AddProject(j.project, j.ref, j.sha)
		if er != nil {
			err = er
			return
		}
		j.pid = pid // project id
	} else {
		j.pid = p.Id
	}
	// generate tag for builded-file search
	j.tag = fmt.Sprintf("%d-%s-%s", j.pid, j.os, j.arch)
	//log.Info("tag:", j.tag)

	// search memory history
	hisAddr, ok := history[j.tag]
	if ok {
		return hisAddr, nil
	}
	// search database history
	f, err := database.SearchFile(j.pid, j.tag)
	log.Debugf("search db: %v", f)
	if err == nil {
		addr = f.Addr
		return
	}

	// build xc
	// file maybe empty
	file, err := j.build(j.os, j.arch)
	if err != nil {
		log.Error("build error:", err)
		return
	}
	// package build file(include upload)
	addr, err = j.publish(file)
	if err != nil {
		return
	}
	return
}
Beispiel #14
0
func (ps *ProxyServer) newControlHandler() func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		// read listen port from request
		protocol, subdomain, port := parseConnectRequest(r)
		log.Debugf("proxy listen proto: %v, subdomain: %v port: %v", protocol, subdomain, port)

		// create websocket connection
		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			http.Error(w, err.Error(), 502)
			return
		}
		defer conn.Close()
		log.Debug("remote client addr:", conn.RemoteAddr())

		tunnel := &Tunnel{
			wsconn: conn,
		}
		// TCP: create new port to listen
		log.Infof("New %s proxy for %v", protocol, conn.RemoteAddr())
		switch protocol {
		case "tcp":
			// proxyAddr := fmt.Sprintf("0.0.0.0:%d", port)
			listener, err := NewTcpProxyListener(tunnel, port)
			if err != nil {
				log.Warnf("new tcp proxy err: %v", err)
				http.Error(w, err.Error(), 501)
				return
			}
			defer listener.Close()
			_, port, _ := net.SplitHostPort(listener.Addr().String())
			wsSendMessage(conn, fmt.Sprintf(
				"Local tcp conn is now publicly available via:\n%v:%v\n", ps.domain, port))
		case "http", "https":
			tr := &http.Transport{
				Dial: tunnel.generateTransportDial(),
			}
			revProxy := &httputil.ReverseProxy{
				Director: func(req *http.Request) {
					log.Println("director:", req.RequestURI)
				},
				Transport: tr,
			}
			// should hook here
			// hook(HOOK_CREATE_HTTP_SUBDOMAIN, subdomain)
			// generate a uniq domain
			if subdomain == "" {
				subdomain = uniqName(5) + ".t"
			}
			pxDomain := subdomain + "." + ps.domain
			log.Println("http px use domain:", pxDomain)
			if _, exists := ps.revProxies[pxDomain]; exists {
				wsSendMessage(conn, fmt.Sprintf("subdomain [%s] has already been taken", pxDomain))
				return
			}
			ps.Lock()
			ps.revProxies[pxDomain] = revProxy
			ps.Unlock()
			wsSendMessage(conn, fmt.Sprintf(
				"Local server is now publicly available via:\nhttp://%s\n", pxDomain))

			defer func() {
				ps.Lock()
				delete(ps.revProxies, pxDomain)
				ps.Unlock()
			}()
		default:
			log.Warn("unknown protocol:", protocol)
			return
		}
		// HTTP: use httputil.ReverseProxy
		for {
			var msg Msg
			if err := conn.ReadJSON(&msg); err != nil {
				log.Warn(err)
				break
			}
			log.Info("recv json:", msg)
		}
	}
}
Beispiel #15
0
func ResumableBlockput(
	c rpc.Client, l rpc.Logger, ret *BlkputRet, f io.ReaderAt, blkIdx, blkSize int, extra *PutExtra) (err error) {

	h := crc32.NewIEEE()
	offbase := int64(blkIdx) << blockBits
	chunkSize := extra.ChunkSize

	var bodyLength int

	if ret.Ctx == "" {

		if chunkSize < blkSize {
			bodyLength = chunkSize
		} else {
			bodyLength = blkSize
		}

		body1 := io.NewSectionReader(f, offbase, int64(bodyLength))
		body := io.TeeReader(body1, h)

		err = Mkblock(c, l, ret, blkSize, body, bodyLength)
		if err != nil {
			return
		}
		if ret.Crc32 != h.Sum32() || int(ret.Offset) != bodyLength {
			err = ErrUnmatchedChecksum
			return
		}
		extra.Notify(blkIdx, blkSize, ret)
	}

	for int(ret.Offset) < blkSize {

		if chunkSize < blkSize-int(ret.Offset) {
			bodyLength = chunkSize
		} else {
			bodyLength = blkSize - int(ret.Offset)
		}

		tryTimes := extra.TryTimes

	lzRetry:
		h.Reset()
		body1 := io.NewSectionReader(f, offbase+int64(ret.Offset), int64(bodyLength))
		body := io.TeeReader(body1, h)

		err = Blockput(c, l, ret, body, bodyLength)
		if err == nil {
			if ret.Crc32 == h.Sum32() {
				extra.Notify(blkIdx, blkSize, ret)
				continue
			}
			log.Warn("ResumableBlockput: invalid checksum, retry")
			err = ErrUnmatchedChecksum
		} else {
			if ei, ok := err.(*rpc.ErrorInfo); ok && ei.Code == InvalidCtx {
				ret.Ctx = "" // reset
				log.Warn("ResumableBlockput: invalid ctx, please retry")
				return
			}
			log.Warn("ResumableBlockput: bput failed -", err)
		}
		if tryTimes > 1 {
			tryTimes--
			log.Info("ResumableBlockput retrying ...")
			goto lzRetry
		}
		break
	}
	return
}