Beispiel #1
0
func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	fileInfo, err := os.Stat(*volumeFolder)
	if err != nil {
		log.Fatalf("No Existing Folder:%s", *volumeFolder)
	}
	if !fileInfo.IsDir() {
		log.Fatalf("Volume Folder should not be a file:%s", *volumeFolder)
	}
	perm := fileInfo.Mode().Perm()
	log.Println("Volume Folder permission:", perm)

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}

	store = storage.NewStore(*vport, *ip, *publicUrl, *volumeFolder, *maxVolumeCount)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/admin/assign_volume", assignVolumeHandler)
	http.HandleFunc("/admin/vacuum_volume_check", vacuumVolumeCheckHandler)
	http.HandleFunc("/admin/vacuum_volume_compact", vacuumVolumeCompactHandler)
	http.HandleFunc("/admin/vacuum_volume_commit", vacuumVolumeCommitHandler)

	go func() {
		connected := true
		store.SetMaster(*masterNode)
		for {
			err := store.Join()
			if err == nil {
				if !connected {
					connected = true
					log.Println("Reconnected with master")
				}
			} else {
				if connected {
					connected = false
				}
			}
			time.Sleep(time.Duration(float32(*vpulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *masterNode)

	log.Println("Start Weed volume server", VERSION, "at http://"+*ip+":"+strconv.Itoa(*vport))
	srv := &http.Server{
		Addr:        ":" + strconv.Itoa(*vport),
		Handler:     http.DefaultServeMux,
		ReadTimeout: (time.Duration(*vReadTimeout) * time.Second),
	}
	e := srv.ListenAndServe()
	if e != nil {
		log.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}
Beispiel #2
0
func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, folders []string, maxCounts []int,
	masterNode string, pulseSeconds int,
	dataCenter string, rack string,
	whiteList []string, fixJpgOrientation bool) *VolumeServer {
	publicUrl := publicIp + ":" + strconv.Itoa(port)
	vs := &VolumeServer{
		masterNode:        masterNode,
		pulseSeconds:      pulseSeconds,
		dataCenter:        dataCenter,
		rack:              rack,
		whiteList:         whiteList,
		FixJpgOrientation: fixJpgOrientation,
	}
	vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts)

	r.HandleFunc("/submit", secure(vs.whiteList, vs.submitFromVolumeServerHandler))
	r.HandleFunc("/status", secure(vs.whiteList, vs.statusHandler))
	r.HandleFunc("/admin/assign_volume", secure(vs.whiteList, vs.assignVolumeHandler))
	r.HandleFunc("/admin/vacuum_volume_check", secure(vs.whiteList, vs.vacuumVolumeCheckHandler))
	r.HandleFunc("/admin/vacuum_volume_compact", secure(vs.whiteList, vs.vacuumVolumeCompactHandler))
	r.HandleFunc("/admin/vacuum_volume_commit", secure(vs.whiteList, vs.vacuumVolumeCommitHandler))
	r.HandleFunc("/admin/freeze_volume", secure(vs.whiteList, vs.freezeVolumeHandler))
	r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler))
	r.HandleFunc("/stats/counter", secure(vs.whiteList, statsCounterHandler))
	r.HandleFunc("/stats/memory", secure(vs.whiteList, statsMemoryHandler))
	r.HandleFunc("/stats/disk", secure(vs.whiteList, vs.statsDiskHandler))
	r.HandleFunc("/delete", secure(vs.whiteList, vs.batchDeleteHandler))
	r.HandleFunc("/", vs.storeHandler)

	go func() {
		connected := true
		vs.store.SetBootstrapMaster(vs.masterNode)
		vs.store.SetDataCenter(vs.dataCenter)
		vs.store.SetRack(vs.rack)
		for {
			master, err := vs.store.Join()
			if err == nil {
				if !connected {
					connected = true
					glog.V(0).Infoln("Volume Server Connected with master at", master)
				}
			} else {
				glog.V(4).Infoln("Volume Server Failed to talk with master:", err.Error())
				if connected {
					connected = false
				}
			}
			if connected {
				time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*(1+rand.Float32())) * time.Millisecond)
			} else {
				time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*0.25) * time.Millisecond)
			}
		}
	}()

	return vs
}
Beispiel #3
0
func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	folders := strings.Split(*volumeFolders, ",")
	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			log.Fatalf("The max specified in -max not a valid number %s", max)
		}
	}
	if len(folders) != len(maxCounts) {
		log.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		fileInfo, err := os.Stat(folder)
		if err != nil {
			log.Fatalf("No Existing Folder:%s", folder)
		}
		if !fileInfo.IsDir() {
			log.Fatalf("Volume Folder should not be a file:%s", folder)
		}
		perm := fileInfo.Mode().Perm()
		log.Println("Volume Folder", folder)
		log.Println("Permission:", perm)
	}

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}

	store = storage.NewStore(*vport, *ip, *publicUrl, folders, maxCounts)
	defer store.Close()
	http.HandleFunc("/", storeHandler)
	http.HandleFunc("/status", statusHandler)
	http.HandleFunc("/admin/assign_volume", assignVolumeHandler)
	http.HandleFunc("/admin/vacuum_volume_check", vacuumVolumeCheckHandler)
	http.HandleFunc("/admin/vacuum_volume_compact", vacuumVolumeCompactHandler)
	http.HandleFunc("/admin/vacuum_volume_commit", vacuumVolumeCommitHandler)
	http.HandleFunc("/admin/freeze_volume", freezeVolumeHandler)

	go func() {
		connected := true
		store.SetMaster(*masterNode)
		store.SetDataCenter(*dataCenter)
		store.SetRack(*rack)
		for {
			err := store.Join()
			if err == nil {
				if !connected {
					connected = true
					log.Println("Reconnected with master")
				}
			} else {
				if connected {
					connected = false
				}
			}
			time.Sleep(time.Duration(float32(*vpulse*1e3)*(1+rand.Float32())) * time.Millisecond)
		}
	}()
	log.Println("store joined at", *masterNode)

	log.Println("Start Weed volume server", VERSION, "at http://"+*ip+":"+strconv.Itoa(*vport))
	srv := &http.Server{
		Addr:        ":" + strconv.Itoa(*vport),
		Handler:     http.DefaultServeMux,
		ReadTimeout: (time.Duration(*vReadTimeout) * time.Second),
	}
	e := srv.ListenAndServe()
	if e != nil {
		log.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}