Ejemplo n.º 1
0
Archivo: main.go Proyecto: ndlib/bendo
func doGet(item string, files []string) int {
	var json *jason.Object
	var jsonFetchErr error
	filesToGet := make(chan string)
	var getFileDone sync.WaitGroup

	// if file or dir exists in target path named after the item, give error mesg and exit
	pathPrefix := path.Join(*fileroot, item)

	// set up communication to the bendo server, and init local and remote filelists

	thisItem := bclientapi.New(*server, item, *fileroot, *chunksize, *wait, *token)
	fileLists := fileutil.NewLists(*fileroot)

	// Fetch Item Info from bclientapi
	json, jsonFetchErr = thisItem.GetItemInfo()

	// If not found or error, we're done

	switch {
	case jsonFetchErr == bclientapi.ErrNotFound:
		fmt.Printf("\n Item %s was not found on server %s\n", item, *server)
		return 1
	case jsonFetchErr != nil:
		fmt.Println(jsonFetchErr)
		return 1
	}

	// if item only, get all of the files; otherwise, only those asked for

	if len(files) == 0 {
		fileLists.BuildLocalList(json)
	} else {
		fileLists.BuildRemoteList(json)
		fileLists.BuildLocalFromFiles(files)
	}

	// At this point, the local list contains files, verified to exist on server

	// set up our barrier, that will wait for all the file chunks to be uploaded
	getFileDone.Add(*numuploaders)

	//Spin off desire number of upload workers
	for cnt := int(0); cnt < *numuploaders; cnt++ {
		go func() {
			err := thisItem.GetFiles(filesToGet, pathPrefix)
			if err != nil {
				goReturnMutex.Lock()
				goReturn = 1
				goReturnMutex.Unlock()
			}
			getFileDone.Done()
		}()
	}

	fileLists.QueueFiles(filesToGet)

	getFileDone.Wait()

	// If a file upload failed, return an error to main
	if goReturn == 1 {
		return 1
	}

	return 0
}
Ejemplo n.º 2
0
Archivo: main.go Proyecto: ndlib/bendo
func doUpload(item string, files string) int {

	filesToSend := make(chan string)
	var upLoadDone sync.WaitGroup
	var sendFileDone sync.WaitGroup
	var json *jason.Object
	var jsonFetchErr error

	thisItem := bclientapi.New(*server, item, *fileroot, *chunksize, *wait, *token)
	fileLists := fileutil.NewLists(*fileroot)

	// Set up Barrier for 3 goroutines below
	upLoadDone.Add(3)

	// Fire 1!
	go func() {
		fileLists.CreateUploadList(files)
		upLoadDone.Done()
	}()

	// Fire 2!
	go func() {
		fileLists.ComputeLocalChecksums()
		upLoadDone.Done()
	}()

	// Fire 3!
	go func() {
		json, jsonFetchErr = thisItem.GetItemInfo()
		upLoadDone.Done()
	}()

	// Wait for everyone to finish
	upLoadDone.Wait()

	if *verbose {
		fmt.Printf("\nLocal Files:\n")
		fileLists.PrintLocalList()
	}

	// If GetItemInfo returns ErrNotFound it's a new item- upload whole local list
	// If GetItemInfo returns other error, bendo unvavailable for upload- abort!
	// default: build remote filelist of returned json, diff against local list, upload remainder

	switch {
	case jsonFetchErr == bclientapi.ErrNotFound:
		break
	case jsonFetchErr != nil:
		fmt.Println(jsonFetchErr)
		return 1
	default:
		fileLists.BuildRemoteList(json)

		if *verbose {
			fmt.Printf("\nRemote Files:\n")
			fileLists.PrintRemoteList()
			fmt.Printf("\nBlobs:\n")
			fileLists.PrintBlobList()
			fmt.Printf("\n")
		}

		// This compares the local list with the remote list (if the item already exists)
		// and eliminates any unnneeded duplicates
		fileLists.CullLocalList()
		break
	}

	// Culled list is empty, nothing to upload

	if fileLists.IsLocalListEmpty() {
		fmt.Printf("Nothing to do:\nThe vesions of All Files given for upload in item %s\nare already present on the server\n", item)
		return 0
	}

	if *verbose {
		fmt.Printf("\nFiles to Upload:\n")
		fileLists.PrintLocalList()
	}

	// set up our barrier, that will wait for all the file chunks to be uploaded
	sendFileDone.Add(*numuploaders)

	//Spin off desire number of upload workers
	for cnt := int(0); cnt < *numuploaders; cnt++ {
		go func() {
			err := thisItem.SendFiles(filesToSend, fileLists)
			if err != nil {
				goReturnMutex.Lock()
				goReturn = 1
				goReturnMutex.Unlock()
			}
			sendFileDone.Done()
		}()
	}

	fileLists.QueueFiles(filesToSend)

	// wait for all file chunks to be uploaded
	sendFileDone.Wait()

	// If a file upload failed, return an error to main
	if goReturn == 1 {
		return 1
	}

	// chunks uploaded- submit trnsaction to add FileIDs to item
	transaction, transErr := thisItem.SendNewTransactionRequest()

	if transErr != nil {
		fmt.Println(transErr)
		return 1
	}

	if *verbose {
		fmt.Printf("\n Transaction id is %s\n", transaction)
	}

	if *wait {
		thisItem.WaitForCommitFinish(transaction)
	}

	return 0
}