func upload_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
file, _ := os.Open(filePath)
finfo := agro_pb.FileInfo{
Name: proto.String(path.Base(filePath)),
Id: proto.String(fileID),
}
dbi.CreateFile(context.Background(), &finfo)
buffer := make([]byte, BLOCK_SIZE)
bytes_written := int64(0)
for {
n, _ := file.Read(buffer)
if n == 0 {
break
}
packet := agro_pb.DataBlock{
Id: proto.String(fileID),
Start: proto.Int64(bytes_written),
Len: proto.Int64(int64(n)),
Data: buffer[:n],
}
dbi.WriteFile(context.Background(), &packet)
bytes_written += int64(n)
}
file.Close()
dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
return nil
}
func TestFileTest(t *testing.T) {
dbi, err := NewMongo("localhost")
if err != nil {
panic(err)
}
fmt.Printf("Testing")
u, _ := uuid.NewV4()
file_id := u.String()
finfo := agro_pb.FileInfo{
Name: proto.String("README"),
Id: proto.String(file_id),
}
w, _ := os.Getwd()
fmt.Printf("Dis: %s\n", w)
data, _ := ioutil.ReadFile("../../../README")
dbi.CreateFile(context.Background(), &finfo)
fmt.Printf("Sending: %s\n", data)
packet := agro_pb.DataBlock{
Id: proto.String(file_id),
Start: proto.Int64(0),
Len: proto.Int64(int64(len(data))),
Data: data,
}
dbi.WriteFile(context.Background(), &packet)
dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
info, _ := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
fmt.Printf("FileSize: %d\n", *info.Size)
block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
Id: proto.String(file_id),
Start: proto.Int64(0),
Size: info.Size,
})
fmt.Printf("ReadOut:%d '%s'\n", *block.Len, string(block.Data))
}
func (self *MongoInterface) ReadFile(ctx context.Context, req *agro_pb.ReadRequest) (*agro_pb.DataBlock, error) {
f, _ := self.db.GridFS("fs").OpenId(*req.Id)
f.Seek(*req.Start, 0)
data := make([]byte, *req.Size)
l, _ := f.Read(data)
log.Printf("GridRead: %d (%d)", l, *req.Size)
o := agro_pb.DataBlock{
Id: req.Id,
Start: req.Start,
Len: proto.Int64(int64(l)),
Data: data[:l],
}
f.Close()
return &o, nil
}
func (self *Webserver) GetFile(w http.ResponseWriter, r *http.Request) {
params := mux.Vars(r)
fileID := params["id"]
info, _ := self.dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
block, _ := self.dbi.ReadFile(context.Background(),
&agro_pb.ReadRequest{
Id: info.Id,
Start: &i,
Size: proto.Int64(BLOCK_SIZE),
})
w.Write(block.Data)
}
}
func (self *MongoInterface) GetFileInfo(ctx context.Context, i *agro_pb.FileID) (*agro_pb.FileInfo, error) {
f, err := self.db.GridFS("fs").OpenId(*i.Id)
if err != nil {
return &agro_pb.FileInfo{
Name: proto.String(fmt.Sprintf("%s error: %s", *i.Id, err)),
Id: i.Id,
State: agro_pb.State_ERROR.Enum(),
}, nil
}
o := agro_pb.FileInfo{
Name: proto.String(f.Name()),
Id: proto.String(f.Id().(string)),
Size: proto.Int64(f.Size()),
State: agro_pb.State_OK.Enum(),
}
f.Close()
return &o, nil
}
func download_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
f, err := os.Create(filePath)
if err != nil {
log.Printf("Unable to create workfile")
return err
}
info, err := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
Id: info.Id,
Start: &i,
Size: proto.Int64(BLOCK_SIZE),
})
f.Write(block.Data)
}
f.Close()
return nil
}
func (self *ForkManager) watcher(sched agro_pb.SchedulerClient, filestore agro_pb.FileStoreClient) {
self.sched = sched
self.files = filestore
hostname, _ := os.Hostname()
jobchan := make(chan agro_pb.Job, 10)
for i := 0; i < self.procCount; i++ {
go self.worker(jobchan)
}
var sleep_size int64 = 1
for self.running {
if self.check_func != nil {
self.check_func(self.status)
}
job_stream, _ := self.sched.GetJobToRun(self.ctx,
&agro_pb.JobRequest{
Max: proto.Int32(1),
Worker: &agro_pb.WorkerInfo{
Id: proto.String(self.workerId),
Hostname: proto.String(hostname),
LastPing: proto.Int64(time.Now().Unix()),
},
})
job, _ := job_stream.Recv()
if job != nil {
sleep_size = 1
log.Printf("Found job: %s", job)
jobchan <- *job
} else {
log.Printf("No jobs found")
if sleep_size < 20 {
// sleep_size += 1
}
time.Sleep(time.Second * time.Duration(sleep_size))
}
}
close(jobchan)
}