func runVolume(cmd *Command, args []string) bool {
if *vMaxCpu < 1 {
*vMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*vMaxCpu)
folders := strings.Split(*volumeFolders, ",")
maxCountStrings := strings.Split(*maxVolumeCounts, ",")
maxCounts := make([]int, 0)
for _, maxString := range maxCountStrings {
if max, e := strconv.Atoi(maxString); e == nil {
maxCounts = append(maxCounts, max)
} else {
glog.Fatalf("The max specified in -max not a valid number %s", maxString)
}
}
if len(folders) != len(maxCounts) {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
}
for _, folder := range folders {
if err := util.TestFolderWritable(folder); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
}
if *publicIp == "" {
if *ip == "" {
*publicIp = "localhost"
} else {
*publicIp = *ip
}
}
if *volumeWhiteListOption != "" {
volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
}
r := http.NewServeMux()
volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts,
*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,
*fixJpgOrientation,
)
listeningAddress := *bindIp + ":" + strconv.Itoa(*vport)
glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second)
if e != nil {
glog.Fatalf(e.Error())
}
OnInterrupt(func() {
volumeServer.Shutdown()
})
if e := http.Serve(listener, r); e != nil {
glog.Fatalf("Fail to serve:%s", e.Error())
}
return true
}
func (m *SequencerImpl) saveSequence() {
glog.V(0).Infoln("Saving file id sequence", m.FileIdSequence, "to", path.Join(m.dir, m.fileName+".seq"))
seqFile, e := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_CREATE|os.O_WRONLY, 0644)
if e != nil {
glog.Fatalf("Sequence File Save [ERROR] %s", e)
}
defer seqFile.Close()
encoder := gob.NewEncoder(seqFile)
if e = encoder.Encode(m.FileIdSequence); e != nil {
glog.Fatalf("Sequence File Save [ERROR] %s", e)
}
}
func runMaster(cmd *Command, args []string) bool {
if *mMaxCpu < 1 {
*mMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*mMaxCpu)
if err := util.TestFolderWritable(*metaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
}
if *masterWhiteListOption != "" {
masterWhiteList = strings.Split(*masterWhiteListOption, ",")
}
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList,
)
listeningAddress := *masterIp + ":" + strconv.Itoa(*mport)
glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
if e != nil {
glog.Fatalf(e.Error())
}
go func() {
time.Sleep(100 * time.Millisecond)
if *mPublicIp == "" {
if *masterIp == "" {
*mPublicIp = "localhost"
} else {
*mPublicIp = *masterIp
}
}
myPublicMasterAddress := *mPublicIp + ":" + strconv.Itoa(*mport)
var peers []string
if *masterPeers != "" {
peers = strings.Split(*masterPeers, ",")
}
raftServer := weed_server.NewRaftServer(r, peers, myPublicMasterAddress, *metaFolder, ms.Topo, *mpulse)
ms.SetRaftServer(raftServer)
}()
if e := http.Serve(listener, r); e != nil {
glog.Fatalf("Fail to serve:%s", e.Error())
}
return true
}
func readFileIds(fileName string, fileIdLineChan chan string) {
file, err := os.Open(fileName) // For read access.
if err != nil {
glog.Fatalf("File to read file %s: %s\n", fileName, err)
}
defer file.Close()
r := bufio.NewReader(file)
if *b.sequentialRead {
for {
if line, err := Readln(r); err == nil {
fileIdLineChan <- string(line)
} else {
break
}
}
} else {
lines := make([]string, 0, readStats.total)
for {
if line, err := Readln(r); err == nil {
lines = append(lines, string(line))
} else {
break
}
}
if len(lines) > 0 {
for i := 0; i < readStats.total; i++ {
fileIdLineChan <- lines[rand.Intn(len(lines))]
}
}
}
close(fileIdLineChan)
}
func runCompact(cmd *Command, args []string) bool {
if *compactVolumeId == -1 {
return false
}
vid := storage.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, nil, nil)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
if err = v.Compact(); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
return true
}
// Loads config information from a JSON string
func LoadConfigString(s string) *Config {
result := newConfig()
err := json.Unmarshal([]byte(s), &result.data)
if err != nil {
glog.Fatalf("error parsing config string %s: %s", s, err)
}
return result
}
// Loads config information from a JSON file
func LoadConfig(filename string) *Config {
result := newConfig()
result.filename = filename
err := result.parse()
if err != nil {
glog.Fatalf("error loading config file %s: %s", filename, err)
}
return result
}
func runMaster(cmd *Command, args []string) bool {
if *mMaxCpu < 1 {
*mMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*mMaxCpu)
if *masterWhiteListOption != "" {
masterWhiteList = strings.Split(*masterWhiteListOption, ",")
}
var e error
if topo, e = topology.NewTopology("topo", *confFile, *metaFolder, "weed",
uint64(*volumeSizeLimitMB)*1024*1024, *mpulse); e != nil {
glog.Fatalf("cannot create topology:%s", e)
}
vg = replication.NewDefaultVolumeGrowth()
glog.V(0).Infoln("Volume Size Limit is", *volumeSizeLimitMB, "MB")
http.HandleFunc("/dir/assign", secure(masterWhiteList, dirAssignHandler))
http.HandleFunc("/dir/lookup", secure(masterWhiteList, dirLookupHandler))
http.HandleFunc("/dir/join", secure(masterWhiteList, dirJoinHandler))
http.HandleFunc("/dir/status", secure(masterWhiteList, dirStatusHandler))
http.HandleFunc("/vol/grow", secure(masterWhiteList, volumeGrowHandler))
http.HandleFunc("/vol/status", secure(masterWhiteList, volumeStatusHandler))
http.HandleFunc("/vol/vacuum", secure(masterWhiteList, volumeVacuumHandler))
http.HandleFunc("/submit", secure(masterWhiteList, submitFromMasterServerHandler))
http.HandleFunc("/", redirectHandler)
topo.StartRefreshWritableVolumes(*garbageThreshold)
glog.V(0).Infoln("Start Weed Master", VERSION, "at port", strconv.Itoa(*mport))
srv := &http.Server{
Addr: ":" + strconv.Itoa(*mport),
Handler: http.DefaultServeMux,
ReadTimeout: time.Duration(*mReadTimeout) * time.Second,
}
e = srv.ListenAndServe()
if e != nil {
glog.Fatalf("Fail to start:%s", e)
}
return true
}
func runFix(cmd *Command, args []string) bool {
if *fixVolumeId == -1 {
return false
}
fileName := strconv.Itoa(*fixVolumeId)
if *fixVolumeCollection != "" {
fileName = *fixVolumeCollection + "_" + fileName
}
indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
}
defer indexFile.Close()
nm := storage.NewNeedleMap(indexFile)
defer nm.Close()
vid := storage.VolumeId(*fixVolumeId)
err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, func(superBlock storage.SuperBlock) error {
return nil
}, false, func(n *storage.Needle, offset int64) error {
debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
if n.Size > 0 {
count, pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size)
debug("saved", count, "with error", pe)
} else {
debug("skipping deleted file ...")
return nm.Delete(n.Id)
}
return nil
})
if err != nil {
glog.Fatalf("Export Volume File [ERROR] %s\n", err)
}
return true
}
func runFiler(cmd *Command, args []string) bool {
if err := util.TestFolderWritable(*f.dir); err != nil {
glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
}
r := http.NewServeMux()
_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection)
if nfs_err != nil {
glog.Fatalf(nfs_err.Error())
}
glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
filerListener, e := util.NewListener(
":"+strconv.Itoa(*f.port),
time.Duration(10)*time.Second,
)
if e != nil {
glog.Fatalf(e.Error())
}
if e := http.Serve(filerListener, r); e != nil {
glog.Fatalf("Filer Fail to serve:%s", e.Error())
}
return true
}
func NewMasterServer(r *mux.Router, port int, metaFolder string,
volumeSizeLimitMB uint,
pulseSeconds int,
confFile string,
defaultReplicaPlacement string,
garbageThreshold string,
whiteList []string,
) *MasterServer {
ms := &MasterServer{
port: port,
volumeSizeLimitMB: volumeSizeLimitMB,
pulseSeconds: pulseSeconds,
defaultReplicaPlacement: defaultReplicaPlacement,
garbageThreshold: garbageThreshold,
whiteList: whiteList,
}
ms.bounedLeaderChan = make(chan int, 16)
seq := sequence.NewMemorySequencer()
var e error
if ms.Topo, e = topology.NewTopology("topo", confFile, seq,
uint64(volumeSizeLimitMB)*1024*1024, pulseSeconds); e != nil {
glog.Fatalf("cannot create topology:%s", e)
}
ms.vg = topology.NewDefaultVolumeGrowth()
glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB")
r.HandleFunc("/dir/assign", ms.proxyToLeader(secure(ms.whiteList, ms.dirAssignHandler)))
r.HandleFunc("/dir/lookup", ms.proxyToLeader(secure(ms.whiteList, ms.dirLookupHandler)))
r.HandleFunc("/dir/join", ms.proxyToLeader(secure(ms.whiteList, ms.dirJoinHandler)))
r.HandleFunc("/dir/status", ms.proxyToLeader(secure(ms.whiteList, ms.dirStatusHandler)))
r.HandleFunc("/col/delete", ms.proxyToLeader(secure(ms.whiteList, ms.collectionDeleteHandler)))
r.HandleFunc("/vol/lookup", ms.proxyToLeader(secure(ms.whiteList, ms.volumeLookupHandler)))
r.HandleFunc("/vol/grow", ms.proxyToLeader(secure(ms.whiteList, ms.volumeGrowHandler)))
r.HandleFunc("/vol/status", ms.proxyToLeader(secure(ms.whiteList, ms.volumeStatusHandler)))
r.HandleFunc("/vol/vacuum", ms.proxyToLeader(secure(ms.whiteList, ms.volumeVacuumHandler)))
r.HandleFunc("/submit", secure(ms.whiteList, ms.submitFromMasterServerHandler))
r.HandleFunc("/delete", secure(ms.whiteList, ms.deleteFromMasterServerHandler))
r.HandleFunc("/{fileId}", ms.redirectHandler)
r.HandleFunc("/stats/counter", secure(ms.whiteList, statsCounterHandler))
r.HandleFunc("/stats/memory", secure(ms.whiteList, statsMemoryHandler))
ms.Topo.StartRefreshWritableVolumes(garbageThreshold)
return ms
}
func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
glog.Fatalf("File to create file %s: %s\n", fileName, err)
}
defer file.Close()
for {
select {
case <-finishChan:
wait.Done()
return
case line := <-fileIdLineChan:
file.Write([]byte(line))
file.Write([]byte("\n"))
}
}
}
func runServer(cmd *Command, args []string) bool {
if *serverOptions.cpuprofile != "" {
f, err := os.Create(*serverOptions.cpuprofile)
if err != nil {
glog.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
if *serverPublicIp == "" {
if *serverIp == "" {
*serverPublicIp = "localhost"
} else {
*serverPublicIp = *serverIp
}
}
*filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort)
if *filerOptions.defaultReplicaPlacement == "" {
*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
}
if *serverMaxCpu < 1 {
*serverMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*serverMaxCpu)
folders := strings.Split(*volumeDataFolders, ",")
maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
maxCounts := make([]int, 0)
for _, maxString := range maxCountStrings {
if max, e := strconv.Atoi(maxString); e == nil {
maxCounts = append(maxCounts, max)
} else {
glog.Fatalf("The max specified in -max not a valid number %s", maxString)
}
}
if len(folders) != len(maxCounts) {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
}
for _, folder := range folders {
if err := util.TestFolderWritable(folder); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
}
if *masterMetaFolder == "" {
*masterMetaFolder = folders[0]
}
if *filerOptions.dir == "" {
*filerOptions.dir = *masterMetaFolder + "/filer"
os.MkdirAll(*filerOptions.dir, 0700)
}
if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
}
if err := util.TestFolderWritable(*filerOptions.dir); err != nil {
glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err)
}
if *serverWhiteListOption != "" {
serverWhiteList = strings.Split(*serverWhiteListOption, ",")
}
if *isStartingFiler {
go func() {
r := http.NewServeMux()
_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection)
if nfs_err != nil {
glog.Fatalf(nfs_err.Error())
}
glog.V(0).Infoln("Start Weed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
filerListener, e := util.NewListener(
":"+strconv.Itoa(*filerOptions.port),
time.Duration(10)*time.Second,
)
if e != nil {
glog.Fatalf(e.Error())
}
if e := http.Serve(filerListener, r); e != nil {
glog.Fatalf("Filer Fail to serve:%s", e.Error())
}
}()
}
var raftWaitForMaster sync.WaitGroup
var volumeWait sync.WaitGroup
raftWaitForMaster.Add(1)
volumeWait.Add(1)
go func() {
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *garbageThreshold, serverWhiteList,
)
glog.V(0).Infoln("Start Weed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
masterListener, e := util.NewListener(*serverIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
if e != nil {
glog.Fatalf(e.Error())
}
go func() {
raftWaitForMaster.Wait()
time.Sleep(100 * time.Millisecond)
myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort)
var peers []string
if *serverPeers != "" {
peers = strings.Split(*serverPeers, ",")
}
raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
ms.SetRaftServer(raftServer)
volumeWait.Done()
}()
raftWaitForMaster.Done()
if e := http.Serve(masterListener, r); e != nil {
glog.Fatalf("Master Fail to serve:%s", e.Error())
}
}()
volumeWait.Wait()
time.Sleep(100 * time.Millisecond)
r := http.NewServeMux()
volumeServer := weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
)
glog.V(0).Infoln("Start Weed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
volumeListener, e := util.NewListener(
*serverIp+":"+strconv.Itoa(*volumePort),
time.Duration(*serverTimeout)*time.Second,
)
if e != nil {
glog.Fatalf(e.Error())
}
OnInterrupt(func() {
volumeServer.Shutdown()
pprof.StopCPUProfile()
})
if e := http.Serve(volumeListener, r); e != nil {
glog.Fatalf("Fail to serve:%s", e.Error())
}
return true
}
func runExport(cmd *Command, args []string) bool {
if *exportVolumeId == -1 {
return false
}
var err error
if *dest != "" {
if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
fmt.Println("the output file", *dest, "should be '-' or end with .tar")
return false
}
if fnTmpl, err = template.New("name").Parse(*format); err != nil {
fmt.Println("cannot parse format " + *format + ": " + err.Error())
return false
}
var fh *os.File
if *dest == "-" {
fh = os.Stdout
} else {
if fh, err = os.Create(*dest); err != nil {
glog.Fatalf("cannot open output tar %s: %s", *dest, err)
}
}
defer fh.Close()
tarFh = tar.NewWriter(fh)
defer tarFh.Close()
t := time.Now()
tarHeader = tar.Header{Mode: 0644,
ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
Typeflag: tar.TypeReg,
AccessTime: t, ChangeTime: t}
}
fileName := strconv.Itoa(*exportVolumeId)
if *exportCollection != "" {
fileName = *exportCollection + "_" + fileName
}
vid := storage.VolumeId(*exportVolumeId)
indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
if err != nil {
glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
}
defer indexFile.Close()
nm, err := storage.LoadNeedleMap(indexFile)
if err != nil {
glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
}
var version storage.Version
err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error {
version = superBlock.Version()
return nil
}, true, func(n *storage.Needle, offset int64) error {
nv, ok := nm.Get(n.Id)
glog.V(3).Infoln("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped(), "ok", ok, "nv", nv)
if ok && nv.Size > 0 {
return walker(vid, n, version)
} else {
if !ok {
debug("This seems deleted", n.Id, "size", n.Size)
} else {
debug("Id", n.Id, "size", n.Size)
}
}
return nil
})
if err != nil {
glog.Fatalf("Export Volume File [ERROR] %s\n", err)
}
return true
}
func runVolume(cmd *Command, args []string) bool {
if *vMaxCpu < 1 {
*vMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*vMaxCpu)
folders := strings.Split(*volumeFolders, ",")
maxCountStrings := strings.Split(*maxVolumeCounts, ",")
maxCounts := make([]int, 0)
for _, maxString := range maxCountStrings {
if max, e := strconv.Atoi(maxString); e == nil {
maxCounts = append(maxCounts, max)
} else {
glog.Fatalf("The max specified in -max not a valid number %s", max)
}
}
if len(folders) != len(maxCounts) {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
}
for _, folder := range folders {
fileInfo, err := os.Stat(folder)
if err != nil {
glog.Fatalf("No Existing Folder:%s", folder)
}
if !fileInfo.IsDir() {
glog.Fatalf("Volume Folder should not be a file:%s", folder)
}
perm := fileInfo.Mode().Perm()
glog.V(0).Infoln("Volume Folder", folder)
glog.V(0).Infoln("Permission:", perm)
}
if *publicUrl == "" {
*publicUrl = *ip + ":" + strconv.Itoa(*vport)
}
if *volumeWhiteListOption != "" {
volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
}
store = storage.NewStore(*vport, *ip, *publicUrl, folders, maxCounts)
defer store.Close()
http.HandleFunc("/", storeHandler)
http.HandleFunc("/submit", secure(volumeWhiteList, submitFromVolumeServerHandler))
http.HandleFunc("/status", secure(volumeWhiteList, statusHandler))
http.HandleFunc("/admin/assign_volume", secure(volumeWhiteList, assignVolumeHandler))
http.HandleFunc("/admin/vacuum_volume_check", secure(volumeWhiteList, vacuumVolumeCheckHandler))
http.HandleFunc("/admin/vacuum_volume_compact", secure(volumeWhiteList, vacuumVolumeCompactHandler))
http.HandleFunc("/admin/vacuum_volume_commit", secure(volumeWhiteList, vacuumVolumeCommitHandler))
http.HandleFunc("/admin/freeze_volume", secure(volumeWhiteList, freezeVolumeHandler))
go func() {
connected := true
store.SetMaster(*masterNode)
store.SetDataCenter(*dataCenter)
store.SetRack(*rack)
for {
err := store.Join()
if err == nil {
if !connected {
connected = true
glog.V(0).Infoln("Reconnected with master")
}
} else {
if connected {
connected = false
}
}
time.Sleep(time.Duration(float32(*vpulse*1e3)*(1+rand.Float32())) * time.Millisecond)
}
}()
glog.V(0).Infoln("store joined at", *masterNode)
glog.V(0).Infoln("Start Weed volume server", VERSION, "at http://"+*ip+":"+strconv.Itoa(*vport))
srv := &http.Server{
Addr: ":" + strconv.Itoa(*vport),
Handler: http.DefaultServeMux,
ReadTimeout: (time.Duration(*vReadTimeout) * time.Second),
}
e := srv.ListenAndServe()
if e != nil {
glog.Fatalf("Fail to start:%s", e.Error())
}
return true
}
