Exemple #1
0
func upload_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {

	file, _ := os.Open(filePath)

	finfo := agro_pb.FileInfo{
		Name: proto.String(path.Base(filePath)),
		Id:   proto.String(fileID),
	}
	dbi.CreateFile(context.Background(), &finfo)
	buffer := make([]byte, BLOCK_SIZE)
	bytes_written := int64(0)
	for {
		n, _ := file.Read(buffer)
		if n == 0 {
			break
		}
		packet := agro_pb.DataBlock{
			Id:    proto.String(fileID),
			Start: proto.Int64(bytes_written),
			Len:   proto.Int64(int64(n)),
			Data:  buffer[:n],
		}
		dbi.WriteFile(context.Background(), &packet)
		bytes_written += int64(n)
	}
	file.Close()
	dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	return nil
}
Exemple #2
0
func TestFileTest(t *testing.T) {

	dbi, err := NewMongo("localhost")
	if err != nil {
		panic(err)
	}
	fmt.Printf("Testing")

	u, _ := uuid.NewV4()
	file_id := u.String()

	finfo := agro_pb.FileInfo{
		Name: proto.String("README"),
		Id:   proto.String(file_id),
	}
	w, _ := os.Getwd()
	fmt.Printf("Dis: %s\n", w)
	data, _ := ioutil.ReadFile("../../../README")
	dbi.CreateFile(context.Background(), &finfo)

	fmt.Printf("Sending: %s\n", data)

	packet := agro_pb.DataBlock{
		Id:    proto.String(file_id),
		Start: proto.Int64(0),
		Len:   proto.Int64(int64(len(data))),
		Data:  data,
	}
	dbi.WriteFile(context.Background(), &packet)
	dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})

	info, _ := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
	fmt.Printf("FileSize: %d\n", *info.Size)

	block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
		Id:    proto.String(file_id),
		Start: proto.Int64(0),
		Size:  info.Size,
	})

	fmt.Printf("ReadOut:%d '%s'\n", *block.Len, string(block.Data))
}
Exemple #3
0
func (self *MongoInterface) ReadFile(ctx context.Context, req *agro_pb.ReadRequest) (*agro_pb.DataBlock, error) {
	f, _ := self.db.GridFS("fs").OpenId(*req.Id)
	f.Seek(*req.Start, 0)
	data := make([]byte, *req.Size)
	l, _ := f.Read(data)
	log.Printf("GridRead: %d (%d)", l, *req.Size)
	o := agro_pb.DataBlock{
		Id:    req.Id,
		Start: req.Start,
		Len:   proto.Int64(int64(l)),
		Data:  data[:l],
	}
	f.Close()
	return &o, nil
}
Exemple #4
0
func (self *Webserver) GetFile(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	fileID := params["id"]

	info, _ := self.dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
		block, _ := self.dbi.ReadFile(context.Background(),
			&agro_pb.ReadRequest{
				Id:    info.Id,
				Start: &i,
				Size:  proto.Int64(BLOCK_SIZE),
			})
		w.Write(block.Data)
	}
}
Exemple #5
0
func (self *MongoInterface) GetFileInfo(ctx context.Context, i *agro_pb.FileID) (*agro_pb.FileInfo, error) {
	f, err := self.db.GridFS("fs").OpenId(*i.Id)
	if err != nil {
		return &agro_pb.FileInfo{
			Name:  proto.String(fmt.Sprintf("%s error: %s", *i.Id, err)),
			Id:    i.Id,
			State: agro_pb.State_ERROR.Enum(),
		}, nil
	}
	o := agro_pb.FileInfo{
		Name:  proto.String(f.Name()),
		Id:    proto.String(f.Id().(string)),
		Size:  proto.Int64(f.Size()),
		State: agro_pb.State_OK.Enum(),
	}
	f.Close()
	return &o, nil
}
Exemple #6
0
func download_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
	f, err := os.Create(filePath)
	if err != nil {
		log.Printf("Unable to create workfile")
		return err
	}
	info, err := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
		block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
			Id:    info.Id,
			Start: &i,
			Size:  proto.Int64(BLOCK_SIZE),
		})
		f.Write(block.Data)
	}
	f.Close()
	return nil
}
Exemple #7
0
func (self *ForkManager) watcher(sched agro_pb.SchedulerClient, filestore agro_pb.FileStoreClient) {
	self.sched = sched
	self.files = filestore
	hostname, _ := os.Hostname()
	jobchan := make(chan agro_pb.Job, 10)
	for i := 0; i < self.procCount; i++ {
		go self.worker(jobchan)
	}
	var sleep_size int64 = 1
	for self.running {
		if self.check_func != nil {
			self.check_func(self.status)
		}
		job_stream, _ := self.sched.GetJobToRun(self.ctx,
			&agro_pb.JobRequest{
				Max: proto.Int32(1),
				Worker: &agro_pb.WorkerInfo{
					Id:       proto.String(self.workerId),
					Hostname: proto.String(hostname),
					LastPing: proto.Int64(time.Now().Unix()),
				},
			})
		job, _ := job_stream.Recv()
		if job != nil {
			sleep_size = 1
			log.Printf("Found job: %s", job)
			jobchan <- *job
		} else {
			log.Printf("No jobs found")
			if sleep_size < 20 {
				//  sleep_size += 1
			}
			time.Sleep(time.Second * time.Duration(sleep_size))
		}
	}
	close(jobchan)
}