// ==========================API DOCS=======================================
// API Name: Create & Initiate Container by Force
// Action: Create a new container and format it for pseudo-fs by force
// API URL: /cn/{Container-Name}
// REQUEST: PUT
// Parameters:
// - Container-Name(in URL): the container name to create
// Returns:
// - HTTP 201: No problem and the container has been created.
// - HTTP 405: Parameters not specifed. Info will be provided in body.
// - HTTP 500: Error. The body is supposed to return error info.
// ==========================API DOCS END===================================
func createContainerHandlerByForce(req Request, res Response) {
var containerName = req.Path()
if containerName == "" || containerName[0] != '/' {
res.Status("Path /container/{Container-Name} is required.", 405)
return
}
containerName = containerName[1:]
var ioAPI = outapi.NewSwiftio(outapi.DefaultConnector, containerName)
var _, err = ioAPI.EnsureSpace()
if err != nil {
logger.Secretary.Error("inapi.container.create", err)
res.Status("Internal Error: "+err.Error(), 500)
return
}
// Format it.
var theFS = filesystem.GetFs(ioAPI)
if theFS == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer theFS.Release()
if err = theFS.FormatFS(); err != nil {
logger.Secretary.Error("inapi.container.create", err)
res.Status("Internal Error: "+err.Error(), 500)
return
}
res.SendCode(201)
}
func downloader(req Request, res Response) {
var pathDetail, _ = req.F()["HandledRR"].([]string)
if pathDetail == nil {
pathDetail = req.F()["RR"].([]string)
pathDetail = append(pathDetail, filesystem.ROOT_INODE_NAME)
}
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, pathDetail[1]))
if fs == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer fs.Release()
var hasSent bool = false
if base, filename := pathman.SplitPath(pathDetail[2]); filename == "" {
var err = fs.Get("", pathDetail[3], func(fileInode string, oriName string, oriHeader map[string]string) io.Writer {
for k, v := range oriHeader {
res.Set(ORIGINAL_HEADER+k, v)
}
res.Set(FILE_NODE, fileInode)
res.Set(HEADER_CONTENT_DISPOSE, "inline; filename=\""+oriName+"\"")
res.SendCode(200)
hasSent = true
return res.R()
})
if err != nil && !hasSent {
if err == exception.EX_FILE_NOT_EXIST {
res.Status("File Not Found.", 404)
} else {
res.Status("Internal Error: "+err.Error(), 500)
}
}
} else {
var nodeName, err = fs.Locate(base, pathDetail[3])
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
err = fs.Get(filename, nodeName, func(fileInode string, oriName string, oriHeader map[string]string) io.Writer {
for k, v := range oriHeader {
res.Set(ORIGINAL_HEADER+k, v)
}
res.Set(PARENT_NODE, nodeName)
res.Set(HEADER_CONTENT_DISPOSE, "inline; filename=\""+oriName+"\"")
res.Set(FILE_NODE, fileInode)
res.SendCode(200)
hasSent = true
return res.R()
})
if err != nil && !hasSent {
if err == exception.EX_FILE_NOT_EXIST {
res.Status("File Not Found.", 404)
} else {
res.Status("Internal Error: "+err.Error(), 500)
}
}
}
}
func main() {
flag.Parse()
var args = flag.Args()
if len(args) < 1 {
fmt.Fprintln(os.Stderr, "The input file should be specified.")
os.Exit(1)
}
var path = args[0]
if strings.HasPrefix(path, SWIFT_LOCALE) {
var whole = strings.SplitN(path[len(SWIFT_LOCALE):], "/", 2)
if len(whole) < 2 {
fmt.Fprintln(os.Stderr, "The container in Swift should be specified.")
os.Exit(1)
}
var io = outapi.NewSwiftio(outapi.DefaultConnector, whole[0])
var meta, file, err = io.Get(whole[1])
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if meta == nil {
fmt.Fprintln(os.Stderr, "File non exists.")
os.Exit(1)
}
fmt.Fprintln(os.Stdout, "==FILE META==")
for k, v := range meta {
fmt.Fprintln(os.Stdout, k, ":\t\t", v)
}
switch file := file.(type) {
case *filetype.Kvmap:
fmt.Fprintln(os.Stdout, "==KVMAP FILE==")
file.CheckOut()
for k, v := range file.Kvm {
fmt.Fprintln(os.Stdout, k, ":\t\t", *v)
}
case *filetype.Nnode:
fmt.Fprintln(os.Stdout, "==NNODE FILE==")
fmt.Fprintln(os.Stdout, "Pointed:\t\t", file.DesName)
default:
fmt.Fprintln(os.Stdout, "==TYPE NOT FOUND==")
}
os.Exit(0)
} else {
var res, err = ioutil.ReadFile(path)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Fprintln(os.Stdout, string(res))
os.Exit(0)
}
}
// ==========================API DOCS=======================================
// API Name: List all the object in the directory
// Action: Return all the file in the format of JSON
// API URL: /fs/{contianer}/{followingpath}
// REQUEST: GET
// Parameters:
// - contianer(in URL): the container name
// - followingpath(in URL): the path to be listed
// - Show-All(in Header): TRUE to show all in the kvfile
// Returns:
// - HTTP 200: No error and the result will be returned in JSON in the body.
// When success, 'Parent-Node' will indicate the listed directory.
// - HTTP 404: Either the container or the filepath does not exist.
// - HTTP 500: Error. The body is supposed to return error info.
// ==========================API DOCS END===================================
func lsDirectory(req Request, res Response) {
//===============measurement========================
var startTime int64 = 0
if req.Get("Enable-Measure") == "True" {
startTime = time.Now().UnixNano()
}
//==================================================
var pathDetail, _ = req.F()["HandledRR"].([]string)
if pathDetail == nil {
pathDetail = req.F()["RR"].([]string)
pathDetail = append(pathDetail, filesystem.ROOT_INODE_NAME)
}
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, pathDetail[1]))
if fs == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer fs.Release()
var nodeName, err = fs.Locate(pathDetail[2], pathDetail[3])
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
var resultList []*filetype.KvmapEntry
if req.Get("Show-All") == "TRUE" {
resultList, err = fs.ListXPP(nodeName)
} else {
resultList, err = fs.ListX(nodeName)
}
if err != nil {
res.Status("Reading error: "+err.Error(), 404)
return
}
res.Set(LAST_PARENT_NODE, nodeName)
//=============================================
if startTime > 0 {
res.Set("Time-Consumed", strconv.FormatInt(time.Now().UnixNano()-startTime, 10))
}
//=============================================
res.JSON(resultList)
}
func batchPutHandler(req Request, res Response) {
var container = req.Get("P-Container")
var frominode = req.Get("P-From-Inode")
var fromn = req.Get("P-From")
var ton = req.Get("P-To")
var prefix = req.Get("P-Prefix")
var content = "The quick brown fox jumps over the lazy dog"
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, container))
i, _ := strconv.Atoi(fromn)
j, _ := strconv.Atoi(ton)
if err := fs.BatchPutDir(prefix, frominode, i, j, content); err != nil {
res.Send(err.Error())
} else {
res.Send("OK")
}
}
// ==========================API DOCS=======================================
// API Name: Stream data to specified file
// Action: Write the data from http to the file streamingly
// API URL: /io/{contianer}/{followingpath}
// REQUEST: PUT
// Parameters:
// - contianer(in URL): the container name
// Returns:
// - HTTP 200: No error, the file is written by force
// When success, the returned header Parent-Node(if accessed) will
// contain its parent inode and File-Node will indicate the file itself.
// - HTTP 404: Either the container or the filepath does not exist.
// - HTTP 500: Error. The body is supposed to return error info.
// ==========================API DOCS END===================================
func uploader(req Request, res Response) {
var pathDetail, _ = req.F()["HandledRR"].([]string)
if pathDetail == nil {
pathDetail = req.F()["RR"].([]string)
pathDetail = append(pathDetail, filesystem.ROOT_INODE_NAME)
}
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, pathDetail[1]))
if fs == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer fs.Release()
var putErr error
if base, filename := pathman.SplitPath(pathDetail[2]); filename == "" {
// TODO: glean user meta
putErr, _ = fs.Put("", pathDetail[3], nil, req.R().Body)
res.Set(FILE_NODE, pathDetail[3])
} else {
var nodeName, err = fs.Locate(base, pathDetail[3])
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
// TODO: glean user meta
var targetNode string
putErr, targetNode = fs.Put(filename, nodeName, nil, req.R().Body)
if targetNode != "" {
res.Set(FILE_NODE, targetNode)
}
res.Set(PARENT_NODE, nodeName)
}
if !egg.Nil(putErr) {
if egg.In(putErr, exception.EX_FILE_NOT_EXIST) {
res.Status("Nonexist container or path. Or you cannot refer to a non-existing inode in ovveride mode.", 404)
return
}
res.Status("Internal Error: "+putErr.Error(), 500)
return
}
res.SendCode(200)
}
func rawGetHandler(req Request, res Response) {
var rr = req.F()["RR"].([]string)
var io = outapi.NewSwiftio(outapi.DefaultConnector, rr[1])
var _, rc, err = io.GetStreamX(rr[2])
if err != nil {
res.SendCode(500)
} else if rc == nil {
res.SendCode(404)
} else {
var w = res.R()
if _, copyErr := sysio.Copy(w, rc); copyErr != nil {
rc.Close()
return
}
if err2 := rc.Close(); err2 != nil {
return
}
}
}
func mvDirectory(req Request, res Response) {
//===============measurement========================
var startTime int64 = 0
if req.Get("Enable-Measure") == "True" {
startTime = time.Now().UnixNano()
}
//==================================================
var pathDetail, _ = req.F()["HandledRR"].([]string)
if pathDetail == nil {
pathDetail = req.F()["RR"].([]string)
pathDetail = append(pathDetail, filesystem.ROOT_INODE_NAME)
}
var base, filename = pathman.SplitPath(pathDetail[2])
if filename == "" {
res.Status("The directory/file to move should be specified.", 404)
return
}
var destinationALL = req.Get(HEADER_DESTINATION)
if destinationALL == "" {
res.Status("Destination path should be specified in the Header "+HEADER_DESTINATION, 403)
return
}
var destinationSC, destinationPath = pathman.ShortcutResolver(destinationALL)
if destinationSC == "" {
destinationSC = filesystem.ROOT_INODE_NAME
} else {
if len(destinationPath) > 0 {
destinationPath = destinationPath[1:]
}
}
var desBase, desFilename = pathman.SplitPath(destinationPath)
if desFilename == "" {
res.Status("The destination directory/file should be specified.", 404)
return
}
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, pathDetail[1]))
if fs == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer fs.Release()
var srcNodeNames, desNodeNames string
var err error
if req.Get(HEADER_DISTABLE_PARALLEL) == "TRUE" {
srcNodeNames, err = fs.Locate(base, pathDetail[3])
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
desNodeNames, err = fs.Locate(desBase, destinationSC)
} else {
var wg sync.WaitGroup
var lock sync.Mutex
wg.Add(2)
go (func() {
defer wg.Done()
var err2 error
srcNodeNames, err2 = fs.Locate(base, pathDetail[3])
if err2 != nil {
lock.Lock()
if err == nil {
err = err2
}
lock.Unlock()
}
})()
go (func() {
defer wg.Done()
var err2 error
desNodeNames, err = fs.Locate(desBase, destinationSC)
if err2 != nil {
lock.Lock()
if err == nil {
err = err2
}
lock.Unlock()
}
})()
wg.Wait()
}
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
var byForce = req.Get(HEADER_MOVE_BY_FORCE) == "TRUE"
if req.Get(HEADER_DISTABLE_PARALLEL) == "TRUE" {
err = fs.MvX(filename, srcNodeNames, desFilename, desNodeNames, byForce)
} else {
err = fs.MvXParalleled(filename, srcNodeNames, desFilename, desNodeNames, byForce)
}
if !egg.Nil(err) {
if egg.In(err, exception.EX_FILE_NOT_EXIST) {
res.Status("Not Found", 404)
return
}
if egg.In(err, exception.EX_FOLDER_ALREADY_EXIST) {
res.Status("The destination has already existed.", 202)
return
}
res.Status("Internal error: "+err.Error(), 500)
return
}
//=============================================
if startTime > 0 {
res.Set("Time-Consumed", strconv.FormatInt(time.Now().UnixNano()-startTime, 10))
}
//=============================================
res.SendCode(201)
}
// ==========================API DOCS=======================================
// API Name: Remove one directory
// Action: remove the directory only if it exists and its parent path exists
// API URL: /fs/{contianer}/{followingpath}
// REQUEST: DELETE
// Parameters:
// - contianer(in URL): the container name
// - followingpath(in URL): the path to be removed. Please guarantee its parent node exists.
// - Disable-Parallel(in Header): if set to TRUE, a non-parallelized mkdir will be
// operated. Default to FALSE, it is only for test and not recommend to set.
// Returns:
// - HTTP 204: The deletion succeeds but it is only a patch. to ensure created, another list
// operation should be carried.
// When success, 'Parent-Node' will indicate the parent of removed directory.
// - HTTP 404: Either the container or the parent filepath or the file itself does not exist.
// - HTTP 500: Error. The body is supposed to return error info.
// ==========================API DOCS END===================================
func rmDirectory(req Request, res Response) {
//===============measurement========================
var startTime int64 = 0
if req.Get("Enable-Measure") == "True" {
startTime = time.Now().UnixNano()
}
//==================================================
var pathDetail, _ = req.F()["HandledRR"].([]string)
if pathDetail == nil {
pathDetail = req.F()["RR"].([]string)
pathDetail = append(pathDetail, filesystem.ROOT_INODE_NAME)
}
var trimer = pathDetail[2]
var i int
for i = len(trimer) - 1; i >= 0; i-- {
if trimer[i] != '/' {
break
}
}
if i < 0 {
res.Status("The directory to remove should be specified.", 404)
return
}
trimer = trimer[:i+1]
var j int
for j = i; j >= 0; j-- {
if trimer[j] == '/' {
break
}
}
var base = trimer[:j+1]
trimer = trimer[j+1:]
// now trimer holds the last foldername
// base holds the parent folder path
var fs = filesystem.GetFs(outapi.NewSwiftio(outapi.DefaultConnector, pathDetail[1]))
if fs == nil {
res.Status("Internal Error: the FS pool is full.", 500)
}
defer fs.Release()
var nodeName, err = fs.Locate(base, pathDetail[3])
if err != nil {
res.Status("Nonexist container or path. "+err.Error(), 404)
return
}
res.Set(LAST_PARENT_NODE, nodeName)
// TODO: what if the src file does not exist?
if req.Get(HEADER_DISTABLE_PARALLEL) == "TRUE" {
err = fs.Rm(trimer, nodeName)
} else {
err = fs.RmParalleled(trimer, nodeName)
}
if !egg.Nil(err) {
if egg.In(err, exception.EX_INODE_NONEXIST) {
res.Status("Nonexist container or path.", 404)
return
}
if egg.In(err, exception.EX_FILE_NOT_EXIST) {
res.Status("Nonexist container or path.", 404)
return
}
res.Status("Internal Error: "+err.Error(), 500)
return
}
//=============================================
if startTime > 0 {
res.Set("Time-Consumed", strconv.FormatInt(time.Now().UnixNano()-startTime, 10))
}
//=============================================
res.SendCode(204)
}
func main() {
var pContainer = flag.String("container", "", "The container to manipulate.")
var pFromPath = flag.String("from", "", "The from path.")
var pToPath = flag.String("to", "", "The to path")
var pThread = flag.Int("thread", 1, "The thread to issue concurrently")
var pDelete = flag.Bool("delete", true, "Perform delete")
var pCopy = flag.Bool("copy", true, "Perform copy")
flag.Parse()
if *pContainer == "" {
fmt.Println("Container must be specified.")
os.Exit(1)
}
if *pFromPath == *pToPath {
fmt.Println("FromPath==ToPath, abort.")
return
}
var nowTime = time.Now().UnixNano()
var objList, err = c.ObjectsAll(*pContainer, &swift.ObjectsOpts{
Prefix: *pFromPath,
})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
var io = outapi.NewSwiftio(outapi.DefaultConnector, *pContainer)
var wg sync.WaitGroup
var rollingList = func(begg int, endd int) {
if endd < 0 || endd > len(objList) {
endd = len(objList)
}
for i := begg; i < endd; i++ {
var e = objList[i]
var fromName = e.Name
var toName = *pToPath + e.Name[len(*pFromPath):]
fmt.Println("Moving", fromName, "->", toName)
if *pCopy {
if err := io.Copy(fromName, toName, nil); err != nil {
fmt.Println("Error:", err, "when trying to copy", fromName)
os.Exit(1)
}
}
if *pDelete {
if err := io.Delete(fromName); err != nil {
fmt.Println("Error:", err, "when trying to delete", fromName)
os.Exit(1)
}
}
}
wg.Done()
}
var slice = len(objList) / (*pThread)
if slice < 1 {
slice = 1
}
var now = 0
for now < len(objList) {
go rollingList(now, now+slice)
now = now + slice
wg.Add(1)
}
wg.Wait()
fmt.Println("Time consumed:", time.Now().UnixNano()-nowTime, "ns")
return
}