func GetSampleData(itempath string) (sample string) { var filename string for _, v := range SampleFiles { filename = itempath + "/" + v if isFileExists(filename) == true { if bytes, err := ioutil.ReadFile(filename); err == nil { sample = string(bytes) return sample } else { l := log.Error(err) logq.LogPutqueue(l) } } } d, err := os.Open(itempath) //ppen dir if err != nil { log.Println(err) return "" } defer d.Close() ff, _ := d.Readdir(10) // return []fileinfo for i, fi := range ff { log.Printf("sample filename %d: %+v\n", i, fi.Name()) filename = strings.ToLower(fi.Name()) if filename != "sample.md" && filename != "meta.md" && filename != PriceFile { f, err := os.Open(itempath + "/" + fi.Name()) log.Println("filename:", itempath+"/"+fi.Name()) if err != nil { continue } defer f.Close() scanner := bufio.NewScanner(f) scanner.Split(bufio.ScanLines) var i = 0 for scanner.Scan() { if i > 9 { break } i++ sample += scanner.Text() + " \n" //md " \n" is a new line //log.Println(scanner.Text()) } break } } log.Debug("sample data:", sample) //need lenth check return sample }
func dl(uri, ip string, p ds.DsPull, w http.ResponseWriter, c chan int) error { if len(ip) == 0 { ip = "http://localhost:65535" //TODO return } target := ip + uri log.Println(target) n, err := download(target, p, w, c) if err != nil { log.Printf("[%d bytes returned.]\n", n) log.Println(err) } return err }
/*pull parses filename and target IP from HTTP GET method, and start downloading routine. */ func p2p_pull(rw http.ResponseWriter, r *http.Request, ps httprouter.Params) { l := log.Info("P2P PULL FROM", r.RemoteAddr, r.Method, r.URL.RequestURI(), r.Proto) logq.LogPutqueue(l) r.ParseForm() sRepoName := ps.ByName("repo") sDataItem := ps.ByName("dataitem") sTag := ps.ByName("tag") log.Info(sRepoName, sDataItem, sTag) jobtag := fmt.Sprintf("%s/%s:%s", sRepoName, sDataItem, sTag) var irpdmid, idpid int var stagdetail, itemdesc, dpconn, dpname, dptype string msg := &ds.MsgResp{} msg.Msg = "OK." irpdmid, idpid, itemdesc = GetRpdmidDpidItemdesc(sRepoName, sDataItem) if len(itemdesc) == 0 { itemdesc = sRepoName + "_" + sDataItem } log.Debug("dpid:", idpid, "rpdmid:", irpdmid, "itemdesc:", itemdesc) stagdetail = GetTagDetail(irpdmid, sTag) log.Debug("tagdetail", stagdetail) if len(stagdetail) == 0 { l := log.Warnf("%s(tag:%s) not found", stagdetail, sTag) logq.LogPutqueue(l) http.Error(rw, sTag+" not found", http.StatusNotFound) return } dpconn, dpname, dptype = GetDpconnDpnameDptypeByDpid(idpid) log.Debug("dpconn:", dpconn, "dpname:", dpname, "dptype:", dptype) datapool, err := dpdriver.New(dptype) if err != nil { WriteErrLogAndResp(rw, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err) return } filepathname := datapool.GetFileTobeSend(dpconn, dpname, itemdesc, stagdetail) //filepathname := dpconn + "/" + itemdesc + "/" + stagdetail log.Println("filename:", filepathname) if exists := isFileExists(filepathname); !exists { l := log.Error(filepathname, "not found") logq.LogPutqueue(l) putToJobQueue(jobtag, filepathname, "N/A", -1) msg.Msg = fmt.Sprintf("Tag:%s not found", sTag) resp, _ := json.Marshal(msg) respStr := string(resp) rw.WriteHeader(http.StatusNotFound) fmt.Fprintln(rw, respStr) return } tokenValid := false retmsg := "" token := r.Form.Get("token") username := r.Form.Get("username") log.Debug(r.URL, "----", r.FormValue("username"), "----", r.Form.Get("username")) if len(token) > 0 && len(username) > 0 { log.Println(r.URL.Path, "token:", token, "username:"******"/transaction/" + sRepoName + "/" + sDataItem + "/" + sTag + "?cypt_accesstoken=" + token + "&username="******"Get %s size error, %v", filepathname, err) logq.LogPutqueue(l) } log.Printf("Tag file full path name :%v, size:%v", filepathname, size) //rw.Header().Set("Source-FileName", stagdetail) bmd5, err := ComputeMd5(filepathname) strmd5 := fmt.Sprintf("%x", bmd5) if err != nil { log.Error(filepathname, err, bmd5, strmd5) } else { rw.Header().Set("X-Source-MD5", strmd5) } rw.Header().Set("X-Source-FileSize", strconv.FormatInt(size, 10)) l = log.Info("transfering", filepathname, bmd5, strmd5) logq.LogPutqueue(l) jobid := putToJobQueue(jobtag, filepathname, "transfering", size) http.ServeFile(rw, r, filepathname) updateJobQueue(jobid, "transfered", 0) return }
func RunDaemon() { //fmt.Println("Run daemon..") // Daemonizing echo server application. switch isDaemon, err := daemonigo.Daemonize(); { case !isDaemon: return case err != nil: log.Fatal("main(): could not start daemon, reason -> %s", err.Error()) } //fmt.Printf("server := http.Server{}\n") if false == isDirExists(g_strDpPath) { err := os.MkdirAll(g_strDpPath, 0755) if err != nil { log.Printf("mkdir %s error! %v ", g_strDpPath, err) } } DaemonAuthrization = utils.Getguid() log.Println("DaemonAuthrization", DaemonAuthrization) dbinit() if len(DaemonID) == 40 { log.Println("daemonid", DaemonID) saveDaemonID(DaemonID) } else { log.Println("get daemonid from db") DaemonID = getDaemonid() } LoadJobFromDB() os.Chdir(g_strDpPath) originalListener, err := net.Listen("tcp", DaemonCliServer) if err != nil { log.Fatal(err) } //else { // if err = os.Chmod(cmd.UnixSock, os.ModePerm); err != nil { // l := log.Error(err) // logq.LogPutqueue(l) // } // } sl, err = tcpNew(originalListener) if err != nil { panic(err) } router := httprouter.New() router.GET("/", serverFileHandler) router.POST("/api/datapools", dpPostOneHandler) router.GET("/api/datapools", dpGetAllHandler) router.GET("/api/datapools/:dpname", dpGetOneHandler) router.DELETE("/api/datapools/:dpname", dpDeleteOneHandler) router.GET("/api/ep", epGetHandler) router.POST("/api/ep", epPostHandler) router.DELETE("/api/ep", epDeleteHandler) router.GET("/api/repositories/:repo/:item/:tag", repoTagHandler) router.GET("/api/repositories/:repo/:item", repoItemHandler) router.GET("/api/repositories/:repo", repoRepoNameHandler) router.GET("/api/repositories", repoHandler) router.GET("/api/repositories/:repo/:item/:tag/judge", judgeTagExistHandler) router.DELETE("/api/repositories/:repo/:item", repoDelOneItemHandler) router.DELETE("/api/repositories/:repo/:item/:tag", repoDelTagHandler) router.GET("/api/subscriptions/dataitems", subsHandler) router.GET("/api/subscriptions/pull/:repo/:item", subsHandler) router.POST("/api/repositories/:repo/:item", pubItemHandler) router.POST("/api/repositories/:repo/:item/:tag", newPubTagHandler) router.POST("/api/subscriptions/:repo/:item/pull", pullHandler) router.GET("/api/job", jobHandler) router.GET("/api/job/:id", jobDetailHandler) router.DELETE("/api/job/:id", jobRmHandler) router.DELETE("/api/job", jobRmAllHandler) router.GET("/api/daemon/:repo/:item/:tag", tagStatusHandler) router.GET("/api/daemon/:repo/:item", tagOfItemStatusHandler) router.GET("/api/heartbeat/status/:user", userStatusHandler) http.Handle("/", router) http.HandleFunc("/api/stop", stopHttp) http.HandleFunc("/api/users/auth", loginHandler) http.HandleFunc("/api/users/logout", logoutHandler) router.GET("/api/users/whoami", whoamiHandler) router.GET("/api/pulled/:repo/:item", itemPulledHandler) router.GET("/api/datapool/published/:dpname", publishedOfDatapoolHandler) router.GET("/api/datapool/pulled/:dpname", pulledOfDatapoolHandler) router.GET("/api/datapool/published/:dpname/:repo", publishedOfRepoHandler) router.GET("/api/datapool/pulled/:dpname/:repo", pulledOfRepoHandler) router.POST("/api/datapool/check", checkDpConnectHandler) router.GET("/api/datapool/other/:dpname", dpGetOtherDataHandler) router.GET("/api/datapool/pulled/:dpname/:repo/:item", pulledTagOfItemHandler) router.GET("/api/datapool/published/:dpname/:repo/:item", publishedTagOfItemHandler) //router.POST("/api/datapool/newpublishtag", newPublishTagHandler) router.NotFound = &mux{} server := http.Server{} go func() { stop := make(chan os.Signal) signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { case signal := <-stop: log.Printf("Got signal:%v", signal) } sl.Stop() if len(DaemonID) > 0 { p2psl.Stop() } }() if len(DaemonID) > 0 { go startP2PServer() go HeartBeat() go CheckHealthClock() //go datapoolMonitor() //Temporarily not use go GetMessages() go PullTagAutomatic() } else { l := log.Error("no daemonid specificed.") logq.LogPutqueue(l) fmt.Println("You don't have a daemonid specificed.") } /* wg.Add(1) defer wg.Done() */ log.Info("starting daemon listener...") server.Serve(sl) log.Info("Stopping daemon listener...") if len(DaemonID) > 0 { wg.Wait() } daemonigo.UnlockPidFile() g_ds.Db.Close() log.Info("daemon exit....") log.CloseLogFile() }
// This function wraps application with daemonization. // Returns isDaemon value to distinguish parent and daemonized processes. func Daemonize() (isDaemon bool, err error) { const errLoc = "daemonigo.Daemonize()" isDaemon = os.Getenv(EnvVarName) == EnvVarValue //log.Println("IsDaemon: ", isDaemon) if WorkDir != "" { if err = os.Chdir(WorkDir); err != nil { err = fmt.Errorf( "%s: changing working directory failed, reason -> %s", errLoc, err.Error(), ) return } } if isDaemon { screenlog := os.Getenv(OutPutLogScreen) if screenlog == sCREENLOG { } else { log.SetLogFile(logfile) } oldmask := syscall.Umask(int(Umask)) defer syscall.Umask(oldmask) if _, err = syscall.Setsid(); err != nil { err = fmt.Errorf( "%s: setsid failed, reason -> %s", errLoc, err.Error(), ) return } if pidFile, err = lockPidFile(); err != nil { err = fmt.Errorf( "%s: locking PID file failed, reason -> %s", errLoc, err.Error(), ) } } else { flag.Usage = func() { arr := make([]string, 0, len(actions)) for k, _ := range actions { arr = append(arr, k) } fmt.Fprintf(os.Stderr, "Usage: %s {%s}\n", os.Args[0], strings.Join(arr, "|"), ) flag.PrintDefaults() } if !flag.Parsed() { log.Printf(" flag.Parse\n") flag.Parse() } //fmt.Printf("after flag.Parse\n") //fmt.Printf("flag.Arg(0):%s\n", flag.Arg(0)) //action, exist := actions[flag.Arg(0)] action, exist := actions["daemon"] //fmt.Println("exist:", exist) if exist { action() } else { flag.Usage() } } return }
/*download routine, supports resuming broken downloads.*/ func download(uri string, p ds.DsPull, w http.ResponseWriter, c chan int) (int64, error) { log.Printf("we are going to download %s, save to dp=%s,name=%s\n", uri, p.Datapool, p.DestName) var out *os.File var err error var destfilename, tmpdestfilename, tmpdir, dpconn, dptype string dpconn, dptype = GetDataPoolDpconnAndDptype(p.Datapool) if len(dpconn) == 0 { err = fmt.Errorf("dpconn is null! datapool:%s ", p.Datapool) return ErrLogAndResp(c, w, http.StatusBadRequest, cmd.ErrorNoRecord, err) } //New a datapool object datapool, err := dpdriver.New(dptype) if err != nil { return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorNoDatapoolDriver, err) } destfilename, tmpdir, tmpdestfilename = datapool.GetDestFileName(dpconn, p.ItemDesc, p.DestName) os.MkdirAll(tmpdir, 0777) log.Info("open tmp destfile name:", tmpdestfilename) out, err = os.OpenFile(tmpdestfilename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorOpenFile, err) } stat, err := out.Stat() if err != nil { return ErrLogAndResp(c, w, http.StatusInternalServerError, cmd.ErrorStatFile, err) } out.Seek(stat.Size(), 0) req, err := http.NewRequest("GET", uri, nil) req.Header.Set("User-Agent", "go-downloader") /* Set download starting position with 'Range' in HTTP header*/ req.Header.Set("Range", "bytes="+strconv.FormatInt(stat.Size(), 10)+"-") log.Printf("%v bytes had already been downloaded.\n", stat.Size()) log.Debug(EnvDebug("http_proxy", false)) resp, err := http.DefaultClient.Do(req) /*Save response body to file only when HTTP 2xx received. TODO*/ if err != nil || (resp != nil && resp.StatusCode/100 != 2) { log.Error("http error", err) if resp != nil { body, _ := ioutil.ReadAll(resp.Body) l := log.Error("http status code:", resp.StatusCode, "response Body:", string(body), err) logq.LogPutqueue(l) struMsg := &ds.MsgResp{} json.Unmarshal(body, struMsg) msg := struMsg.Msg if resp.StatusCode == 416 { msg = tmpdestfilename + " has already been downloaded." } r, _ := buildResp(resp.StatusCode, msg, nil) w.WriteHeader(resp.StatusCode) w.Write(r) } else { HttpNoData(w, http.StatusInternalServerError, cmd.ErrorOtherError, err.Error()) } filesize := stat.Size() out.Close() if filesize == 0 { os.Remove(tmpdestfilename) } c <- -1 return 0, err } defer resp.Body.Close() HttpNoData(w, http.StatusOK, cmd.ResultOK, strret) //write channel c <- 1 jobtag := p.Repository + "/" + p.Dataitem + ":" + p.Tag srcsize, err := strconv.ParseInt(resp.Header.Get("X-Source-FileSize"), DECIMAL_BASE, INT_SIZE_64) md5str := resp.Header.Get("X-Source-MD5") status := "downloading" log.Info("pull tag:", jobtag, tmpdestfilename, status, srcsize) jobid := putToJobQueue(jobtag, tmpdestfilename, status, srcsize) n, err := io.Copy(out, resp.Body) if err != nil { out.Close() bl := log.Error(err) logq.LogPutqueue(bl) dlsize, e := GetFileSize(tmpdestfilename) if e != nil { l := log.Error(e) logq.LogPutqueue(l) } status = "failed" updateJobQueue(jobid, status, dlsize) return 0, err } out.Close() status = "downloaded" if len(md5str) > 0 { bmd5, err := ComputeMd5(tmpdestfilename) bmd5str := fmt.Sprintf("%x", bmd5) log.Debug("md5", md5str, tmpdestfilename, bmd5str) if err != nil { log.Error(tmpdestfilename, err, bmd5) } else if md5str != bmd5str { l := log.Errorf("check md5 code error! src md5:%v, local md5:%v", md5str, bmd5str) logq.LogPutqueue(l) status = "md5 error" updateJobQueue(jobid, status, 0) return n, nil } } log.Printf("%d bytes downloaded.", n) if err := MoveFromTmp(tmpdestfilename, destfilename); err != nil { status = "MoveFromTmp error" } dlsize, e := GetFileSize(destfilename) if e != nil { l := log.Error(e) logq.LogPutqueue(l) } status = datapool.StoreFile(status, destfilename, dpconn, p.Datapool, p.ItemDesc, p.DestName) updateJobQueue(jobid, status, dlsize) tagComment := GetTagComment(p.Repository, p.Dataitem, p.Tag) InsertTagToDb(true, p, tagComment) return n, nil }