Пример #1
0
func GetSampleData(itempath string) (sample string) {
	var filename string
	for _, v := range SampleFiles {
		filename = itempath + "/" + v
		if isFileExists(filename) == true {
			if bytes, err := ioutil.ReadFile(filename); err == nil {
				sample = string(bytes)
				return sample
			} else {
				l := log.Error(err)
				logq.LogPutqueue(l)
			}
		}
	}
	d, err := os.Open(itempath) //ppen dir
	if err != nil {
		log.Println(err)
		return ""
	}
	defer d.Close()
	ff, _ := d.Readdir(10) //  return []fileinfo
	for i, fi := range ff {
		log.Printf("sample filename %d: %+v\n", i, fi.Name())
		filename = strings.ToLower(fi.Name())
		if filename != "sample.md" && filename != "meta.md" && filename != PriceFile {
			f, err := os.Open(itempath + "/" + fi.Name())
			log.Println("filename:", itempath+"/"+fi.Name())
			if err != nil {
				continue
			}
			defer f.Close()
			scanner := bufio.NewScanner(f)
			scanner.Split(bufio.ScanLines)
			var i = 0
			for scanner.Scan() {
				if i > 9 {
					break
				}
				i++
				sample += scanner.Text() + "  \n" //md "  \n" is a new line
				//log.Println(scanner.Text())
			}
			break
		}
	}
	log.Debug("sample data:", sample)
	//need lenth check
	return sample
}
Пример #2
0
func dl(uri, ip string, p ds.DsPull, w http.ResponseWriter, c chan int) error {

	if len(ip) == 0 {
		ip = "http://localhost:65535"
		//TODO return
	}

	target := ip + uri
	log.Println(target)
	n, err := download(target, p, w, c)
	if err != nil {
		log.Printf("[%d bytes returned.]\n", n)
		log.Println(err)
	}
	return err
}
Пример #3
0
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */
func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto)
	logq.LogPutqueue(l)

	r.ParseForm()
	sRepoName := ps.ByName("repo")
	sDataItem := ps.ByName("dataitem")
	sTag := ps.ByName("tag")

	log.Info(sRepoName, sDataItem, sTag)
	jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag)
	var irpdmid, idpid int
	var stagdetail, itemdesc, dpconn, dpname, dptype string
	msg := &ds.MsgResp{}
	msg.Msg = "OK."

	irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem)
	if len(itemdesc) == 0 {
		itemdesc = sRepoName + "_" + sDataItem
	}
	log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc)

	stagdetail = GetTagDetail(irpdmid, sTag)
	log.Debug("tagdetail", stagdetail)
	if len(stagdetail) == 0 {
		l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag)
		logq.LogPutqueue(l)
		http.Error(rw, sTag+" not found", http.StatusNotFound)
		return
	}

	dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid)
	log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype)

	datapool, err := dpdriver.New(dptype)
	if err != nil {
		WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
		return
	}

	filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail)
	//filepathname := dpconn + "/" + itemdesc + "/" + stagdetail
	log.Println("filename:", filepathname)
	if exists := isFileExists(filepathname); !exists {
		l := log.Error(filepathname, "not found")
		logq.LogPutqueue(l)
		putToJobQueue(jobtag, filepathname, "N/A", -1)
		msg.Msg = fmt.Sprintf("Tag:%s not found", sTag)
		resp, _ := json.Marshal(msg)
		respStr := string(resp)
		rw.WriteHeader(http.StatusNotFound)
		fmt.Fprintln(rw, respStr)
		return
	}

	tokenValid := false
	retmsg := ""
	token := r.Form.Get("token")
	username := r.Form.Get("username")

	log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username"))
	if len(token) > 0 && len(username) > 0 {
		log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag +
			"?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err)
		logq.LogPutqueue(l)
	}
	log.Printf("Tag file full path name :%v, size:%v", filepathname, size)
	//rw.Header().Set("Source-FileName", stagdetail)
	bmd5, err := ComputeMd5(filepathname)
	strmd5 := fmt.Sprintf("%x", bmd5)
	if err != nil {
		log.Error(filepathname, err, bmd5, strmd5)
	} else {
		rw.Header().Set("X-Source-MD5", strmd5)
	}
	rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10))

	l = log.Info("transfering", filepathname, bmd5, strmd5)
	logq.LogPutqueue(l)

	jobid := putToJobQueue(jobtag, filepathname, "transfering", size)
	http.ServeFile(rw, r, filepathname)
	updateJobQueue(jobid, "transfered", 0)

	return
}
Пример #4
0
func RunDaemon() {
	//fmt.Println("Run daemon..")
	// Daemonizing echo server application.

	switch isDaemon, err := daemonigo.Daemonize(); {
	case !isDaemon:
		return
	case err != nil:
		log.Fatal("main(): could not start daemon, reason -> %s", err.Error())
	}
	//fmt.Printf("server := http.Server{}\n")

	if false == isDirExists(g_strDpPath) {
		err := os.MkdirAll(g_strDpPath, 0755)
		if err != nil {
			log.Printf("mkdir %s error! %v ", g_strDpPath, err)
		}

	}

	DaemonAuthrization = utils.Getguid()
	log.Println("DaemonAuthrization", DaemonAuthrization)

	dbinit()

	if len(DaemonID) == 40 {
		log.Println("daemonid", DaemonID)
		saveDaemonID(DaemonID)
	} else {
		log.Println("get daemonid from db")
		DaemonID = getDaemonid()
	}

	LoadJobFromDB()

	os.Chdir(g_strDpPath)
	originalListener, err := net.Listen("tcp", DaemonCliServer)
	if err != nil {
		log.Fatal(err)
	} //else {
	// 	if err = os.Chmod(cmd.UnixSock, os.ModePerm); err != nil {
	// 		l := log.Error(err)
	// 		logq.LogPutqueue(l)
	// 	}
	// }

	sl, err = tcpNew(originalListener)
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	router.GET("/", serverFileHandler)
	router.POST("/api/datapools", dpPostOneHandler)
	router.GET("/api/datapools", dpGetAllHandler)
	router.GET("/api/datapools/:dpname", dpGetOneHandler)
	router.DELETE("/api/datapools/:dpname", dpDeleteOneHandler)

	router.GET("/api/ep", epGetHandler)
	router.POST("/api/ep", epPostHandler)
	router.DELETE("/api/ep", epDeleteHandler)

	router.GET("/api/repositories/:repo/:item/:tag", repoTagHandler)
	router.GET("/api/repositories/:repo/:item", repoItemHandler)
	router.GET("/api/repositories/:repo", repoRepoNameHandler)
	router.GET("/api/repositories", repoHandler)
	router.GET("/api/repositories/:repo/:item/:tag/judge", judgeTagExistHandler)
	router.DELETE("/api/repositories/:repo/:item", repoDelOneItemHandler)
	router.DELETE("/api/repositories/:repo/:item/:tag", repoDelTagHandler)

	router.GET("/api/subscriptions/dataitems", subsHandler)
	router.GET("/api/subscriptions/pull/:repo/:item", subsHandler)

	router.POST("/api/repositories/:repo/:item", pubItemHandler)
	router.POST("/api/repositories/:repo/:item/:tag", newPubTagHandler)

	router.POST("/api/subscriptions/:repo/:item/pull", pullHandler)

	router.GET("/api/job", jobHandler)
	router.GET("/api/job/:id", jobDetailHandler)
	router.DELETE("/api/job/:id", jobRmHandler)
	router.DELETE("/api/job", jobRmAllHandler)

	router.GET("/api/daemon/:repo/:item/:tag", tagStatusHandler)
	router.GET("/api/daemon/:repo/:item", tagOfItemStatusHandler)

	router.GET("/api/heartbeat/status/:user", userStatusHandler)

	http.Handle("/", router)
	http.HandleFunc("/api/stop", stopHttp)
	http.HandleFunc("/api/users/auth", loginHandler)
	http.HandleFunc("/api/users/logout", logoutHandler)

	router.GET("/api/users/whoami", whoamiHandler)
	router.GET("/api/pulled/:repo/:item", itemPulledHandler)

	router.GET("/api/datapool/published/:dpname", publishedOfDatapoolHandler)
	router.GET("/api/datapool/pulled/:dpname", pulledOfDatapoolHandler)
	router.GET("/api/datapool/published/:dpname/:repo", publishedOfRepoHandler)
	router.GET("/api/datapool/pulled/:dpname/:repo", pulledOfRepoHandler)
	router.POST("/api/datapool/check", checkDpConnectHandler)
	router.GET("/api/datapool/other/:dpname", dpGetOtherDataHandler)
	router.GET("/api/datapool/pulled/:dpname/:repo/:item", pulledTagOfItemHandler)
	router.GET("/api/datapool/published/:dpname/:repo/:item", publishedTagOfItemHandler)
	//router.POST("/api/datapool/newpublishtag", newPublishTagHandler)

	router.NotFound = &mux{}

	server := http.Server{}

	go func() {

		stop := make(chan os.Signal)
		signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)

		select {
		case signal := <-stop:
			log.Printf("Got signal:%v", signal)
		}

		sl.Stop()
		if len(DaemonID) > 0 {
			p2psl.Stop()
		}

	}()

	if len(DaemonID) > 0 {
		go startP2PServer()
		go HeartBeat()
		go CheckHealthClock()
		//go datapoolMonitor()  //Temporarily not use
		go GetMessages()
		go PullTagAutomatic()
	} else {
		l := log.Error("no daemonid specificed.")
		logq.LogPutqueue(l)
		fmt.Println("You don't have a daemonid specificed.")
	}

	/*
		wg.Add(1)
		defer wg.Done()
	*/
	log.Info("starting daemon listener...")
	server.Serve(sl)
	log.Info("Stopping daemon listener...")

	if len(DaemonID) > 0 {
		wg.Wait()
	}

	daemonigo.UnlockPidFile()
	g_ds.Db.Close()

	log.Info("daemon exit....")
	log.CloseLogFile()

}
Пример #5
0
// This function wraps application with daemonization.
// Returns isDaemon value to distinguish parent and daemonized processes.
func Daemonize() (isDaemon bool, err error) {
	const errLoc = "daemonigo.Daemonize()"
	isDaemon = os.Getenv(EnvVarName) == EnvVarValue
	//log.Println("IsDaemon: ", isDaemon)
	if WorkDir != "" {
		if err = os.Chdir(WorkDir); err != nil {
			err = fmt.Errorf(
				"%s: changing working directory failed, reason -> %s",
				errLoc, err.Error(),
			)
			return
		}
	}
	if isDaemon {
		screenlog := os.Getenv(OutPutLogScreen)
		if screenlog == sCREENLOG {

		} else {
			log.SetLogFile(logfile)
		}

		oldmask := syscall.Umask(int(Umask))
		defer syscall.Umask(oldmask)
		if _, err = syscall.Setsid(); err != nil {
			err = fmt.Errorf(
				"%s: setsid failed, reason -> %s", errLoc, err.Error(),
			)
			return
		}
		if pidFile, err = lockPidFile(); err != nil {
			err = fmt.Errorf(
				"%s: locking PID file failed, reason -> %s",
				errLoc, err.Error(),
			)
		}
	} else {
		flag.Usage = func() {
			arr := make([]string, 0, len(actions))
			for k, _ := range actions {
				arr = append(arr, k)
			}
			fmt.Fprintf(os.Stderr, "Usage: %s {%s}\n",
				os.Args[0], strings.Join(arr, "|"),
			)
			flag.PrintDefaults()
		}

		if !flag.Parsed() {
			log.Printf(" flag.Parse\n")
			flag.Parse()
		}
		//fmt.Printf("after flag.Parse\n")
		//fmt.Printf("flag.Arg(0):%s\n", flag.Arg(0))

		//action, exist := actions[flag.Arg(0)]
		action, exist := actions["daemon"]
		//fmt.Println("exist:", exist)
		if exist {
			action()
		} else {
			flag.Usage()
		}

	}
	return
}
Пример #6
0
/*download routine, supports resuming broken downloads.*/
func download(uri string, p ds.DsPull, w http.ResponseWriter, c chan int) (int64, error) {
	log.Printf("we are going to download %s, save to dp=%s,name=%s\n", uri, p.Datapool, p.DestName)

	var out *os.File
	var err error
	var destfilename, tmpdestfilename, tmpdir, dpconn, dptype string

	dpconn, dptype = GetDataPoolDpconnAndDptype(p.Datapool)
	if len(dpconn) == 0 {
		err = fmt.Errorf("dpconn is null! datapool:%s ", p.Datapool)
		return ErrLogAndResp(c, w, http.StatusBadRequest, cmd.ErrorNoRecord, err)
	}

	//New a datapool object
	datapool, err := dpdriver.New(dptype)
	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err)
	}
	destfilename, tmpdir, tmpdestfilename = datapool.GetDestFileName(dpconn, p.ItemDesc, p.DestName)

	os.MkdirAll(tmpdir, 0777)

	log.Info("open tmp destfile name:", tmpdestfilename)
	out, err = os.OpenFile(tmpdestfilename, os.O_RDWR|os.O_CREATE, 0644)

	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorOpenFile, err)
	}

	stat, err := out.Stat()
	if err != nil {
		return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorStatFile, err)
	}
	out.Seek(stat.Size(), 0)

	req, err := http.NewRequest("GET", uri, nil)
	req.Header.Set("User-Agent", "go-downloader")
	/* Set download starting position with 'Range' in HTTP header*/
	req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-")
	log.Printf("%v bytes had already been downloaded.\n", stat.Size())

	log.Debug(EnvDebug("http_proxy", false))

	resp, err := http.DefaultClient.Do(req)

	/*Save response body to file only when HTTP 2xx received. TODO*/
	if err != nil || (resp != nil && resp.StatusCode/100 != 2) {
		log.Error("http error", err)
		if resp != nil {
			body, _ := ioutil.ReadAll(resp.Body)
			l := log.Error("http status code:", resp.StatusCode, "response Body:", string(body), err)
			logq.LogPutqueue(l)
			struMsg := &ds.MsgResp{}
			json.Unmarshal(body, struMsg)
			msg := struMsg.Msg
			if resp.StatusCode == 416 {
				msg = tmpdestfilename + " has already been downloaded."
			}
			r, _ := buildResp(resp.StatusCode, msg, nil)

			w.WriteHeader(resp.StatusCode)
			w.Write(r)
		} else {
			HttpNoData(w, http.StatusInternalServerError, cmd.ErrorOtherError, err.Error())
		}
		filesize := stat.Size()
		out.Close()
		if filesize == 0 {
			os.Remove(tmpdestfilename)
		}
		c <- -1
		return 0, err
	}
	defer resp.Body.Close()

	HttpNoData(w, http.StatusOK, cmd.ResultOK, strret)

	//write channel
	c <- 1

	jobtag := p.Repository + "/" + p.Dataitem + ":" + p.Tag

	srcsize, err := strconv.ParseInt(resp.Header.Get("X-Source-FileSize"), DECIMAL_BASE, INT_SIZE_64)
	md5str := resp.Header.Get("X-Source-MD5")
	status := "downloading"
	log.Info("pull tag:", jobtag, tmpdestfilename, status, srcsize)
	jobid := putToJobQueue(jobtag, tmpdestfilename, status, srcsize)

	n, err := io.Copy(out, resp.Body)
	if err != nil {
		out.Close()
		bl := log.Error(err)
		logq.LogPutqueue(bl)
		dlsize, e := GetFileSize(tmpdestfilename)
		if e != nil {
			l := log.Error(e)
			logq.LogPutqueue(l)
		}
		status = "failed"
		updateJobQueue(jobid, status, dlsize)
		return 0, err
	}
	out.Close()

	status = "downloaded"

	if len(md5str) > 0 {
		bmd5, err := ComputeMd5(tmpdestfilename)
		bmd5str := fmt.Sprintf("%x", bmd5)
		log.Debug("md5", md5str, tmpdestfilename, bmd5str)
		if err != nil {
			log.Error(tmpdestfilename, err, bmd5)
		} else if md5str != bmd5str {
			l := log.Errorf("check md5 code error! src md5:%v,  local md5:%v", md5str, bmd5str)
			logq.LogPutqueue(l)
			status = "md5 error"
			updateJobQueue(jobid, status, 0)
			return n, nil
		}
	}
	log.Printf("%d bytes downloaded.", n)

	if err := MoveFromTmp(tmpdestfilename, destfilename); err != nil {
		status = "MoveFromTmp error"
	}

	dlsize, e := GetFileSize(destfilename)
	if e != nil {
		l := log.Error(e)
		logq.LogPutqueue(l)
	}

	status = datapool.StoreFile(status, destfilename, dpconn, p.Datapool, p.ItemDesc, p.DestName)
	updateJobQueue(jobid, status, dlsize)

	tagComment := GetTagComment(p.Repository, p.Dataitem, p.Tag)

	InsertTagToDb(true, p, tagComment)
	return n, nil
}