func upload_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
file, _ := os.Open(filePath)
finfo := agro_pb.FileInfo{
Name: proto.String(path.Base(filePath)),
Id: proto.String(fileID),
}
dbi.CreateFile(context.Background(), &finfo)
buffer := make([]byte, BLOCK_SIZE)
bytes_written := int64(0)
for {
n, _ := file.Read(buffer)
if n == 0 {
break
}
packet := agro_pb.DataBlock{
Id: proto.String(fileID),
Start: proto.Int64(bytes_written),
Len: proto.Int64(int64(n)),
Data: buffer[:n],
}
dbi.WriteFile(context.Background(), &packet)
bytes_written += int64(n)
}
file.Close()
dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
return nil
}
func TestProtoConvert_2(t *testing.T) {
a := &agro_pb.Task{
Id: proto.String("test"),
Command: proto.String("/bin/echo Testing"),
Container: proto.String("ubuntu"),
}
doProtoConvert_task(a)
}
func (self *MesosManager) BuildExecutorInfo() *mesos.ExecutorInfo {
executorCommand := fmt.Sprintf("/Users/ellrott/workspaces/agro/agro_exec")
return &mesos.ExecutorInfo{
ExecutorId: &mesos.ExecutorID{
Value: proto.String("AgroExec"),
},
Name: proto.String("AgroExecutor"),
Source: proto.String("go_test"),
Command: &mesos.CommandInfo{
Value: proto.String(executorCommand),
Uris: []*mesos.CommandInfo_URI{},
},
}
}
func (self *MongoInterface) GetTaskStatus(ids *agro_pb.IDQuery, stream agro_pb.Scheduler_GetTaskStatusServer) error {
i_task := self.db.C("task").Find(bson.M{"_id": bson.M{"$in": ids.Ids}}).Iter()
result := make(map[string]interface{})
for i_task.Next(&result) {
i_job := self.db.C("job").Find(bson.M{"task_id": result["_id"]}).Iter()
runs := make([]string, 0)
var completed *string = nil
job_result := make(map[string]interface{})
for i_job.Next(&job_result) {
runs = append(runs, string(job_result["_id"].(string)))
if job_result["state"] == "OK" {
s := string(job_result["_id"].(string))
completed = &s
}
}
s := agro_pb.State(agro_pb.State_value[result["state"].(string)])
a := &agro_pb.TaskStatus{
Id: proto.String(string(result["_id"].(string))),
State: &s,
Runs: runs,
CompletedJob: completed,
}
stream.Send(a)
}
return nil
}
func (self *MongoInterface) GetFileInfo(ctx context.Context, i *agro_pb.FileID) (*agro_pb.FileInfo, error) {
f, err := self.db.GridFS("fs").OpenId(*i.Id)
if err != nil {
return &agro_pb.FileInfo{
Name: proto.String(fmt.Sprintf("%s error: %s", *i.Id, err)),
Id: i.Id,
State: agro_pb.State_ERROR.Enum(),
}, nil
}
o := agro_pb.FileInfo{
Name: proto.String(f.Name()),
Id: proto.String(f.Id().(string)),
Size: proto.Int64(f.Size()),
State: agro_pb.State_OK.Enum(),
}
f.Close()
return &o, nil
}
func (self *MongoInterface) GetJobStatus(ids *agro_pb.IDQuery, stream agro_pb.Scheduler_GetJobStatusServer) error {
i := self.db.C("job").Find(bson.M{"_id": bson.M{"$in": ids.Ids}}).Iter()
result := make(map[string]interface{})
for i.Next(&result) {
s := agro_pb.State(agro_pb.State_value[result["state"].(string)])
a := &agro_pb.JobStatus{
Id: proto.String(string(result["_id"].(string))),
State: &s,
}
stream.Send(a)
}
return nil
}
func TestDocument(t *testing.T) {
doc := agro_pb.Document{
Id: proto.String("test"),
Fields: []*agro_pb.Field{
&agro_pb.Field{Path: []string{"hello"}, Value: &agro_pb.Field_FloatValue{FloatValue: 35.0}},
},
}
dbi, err := NewMongo("localhost")
if err != nil {
t.Error("DB Failure")
panic(err)
}
t.Log("Testing")
fmt.Print("Testing2")
dbi.CreateDoc(context.Background(), &doc)
out, _ := dbi.GetDoc(context.Background(), &agro_pb.FileID{Id: proto.String("test")})
log.Print("output: ", out)
}
func (self *MesosManager) Start(engine *agro_engine.WorkEngine) {
self.engine = engine
self.executor = self.BuildExecutorInfo()
//frameworkIdStr := FRAMEWORK_ID
failoverTimeout := 0.0
//frameworkId := &mesos.FrameworkID{Value: &frameworkIdStr}
config := sched.DriverConfig{
Master: self.master,
Framework: &mesos.FrameworkInfo{
Name: proto.String("AgroFramework"),
User: proto.String(""),
FailoverTimeout: &failoverTimeout,
//Id: frameworkId,
},
Scheduler: self,
}
driver, err := sched.NewMesosSchedulerDriver(config)
if err != nil {
log.Printf("Driver Error: %s", err)
panic(err)
}
//driver.Init()
//defer driver.Destroy()
//go self.EventLoop()
status, err := driver.Start()
log.Printf("Driver Status:%s", status)
if err != nil {
log.Printf("Mesos Start Error: %s", err)
panic(err)
}
//<-self.exit
//log.Printf("Mesos Exit")
//driver.Stop(false)
}
func (self *ForkManager) watcher(sched agro_pb.SchedulerClient, filestore agro_pb.FileStoreClient) {
self.sched = sched
self.files = filestore
hostname, _ := os.Hostname()
jobchan := make(chan agro_pb.Job, 10)
for i := 0; i < self.procCount; i++ {
go self.worker(jobchan)
}
var sleep_size int64 = 1
for self.running {
if self.check_func != nil {
self.check_func(self.status)
}
job_stream, _ := self.sched.GetJobToRun(self.ctx,
&agro_pb.JobRequest{
Max: proto.Int32(1),
Worker: &agro_pb.WorkerInfo{
Id: proto.String(self.workerId),
Hostname: proto.String(hostname),
LastPing: proto.Int64(time.Now().Unix()),
},
})
job, _ := job_stream.Recv()
if job != nil {
sleep_size = 1
log.Printf("Found job: %s", job)
jobchan <- *job
} else {
log.Printf("No jobs found")
if sleep_size < 20 {
// sleep_size += 1
}
time.Sleep(time.Second * time.Duration(sleep_size))
}
}
close(jobchan)
}
func (self *MongoInterface) CreateTaskJob(task *agro_pb.Task) {
u, _ := uuid.NewV4()
job := &agro_pb.Job{
Id: proto.String(u.String()),
TaskId: task.Id,
State: agro_pb.State_WAITING.Enum(),
Command: task.Command,
Container: task.Container,
Requirements: task.Requirements,
Args: task.Args,
}
log.Printf("Creating job %s", job)
self.AddJob(job)
self.jobCache[*task.Id] = job
}
func (self *Webserver) GetFile(w http.ResponseWriter, r *http.Request) {
params := mux.Vars(r)
fileID := params["id"]
info, _ := self.dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
block, _ := self.dbi.ReadFile(context.Background(),
&agro_pb.ReadRequest{
Id: info.Id,
Start: &i,
Size: proto.Int64(BLOCK_SIZE),
})
w.Write(block.Data)
}
}
func PackDoc(id string, doc map[string]interface{}) *agro_pb.Document {
fields := make([]*agro_pb.Field, 0)
for f := range StreamFields(doc) {
a := f
if len(a.Path) != 1 || a.Path[0] != "_id" {
fields = append(fields, &a)
}
}
//for _, f := range fields {
// log.Printf("Got: %#v %s", f.Path, f.Value)
//}
//log.Printf("OutFields: %d", len(fields))
return &agro_pb.Document{Id: proto.String(id), Fields: fields}
}
func download_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
f, err := os.Create(filePath)
if err != nil {
log.Printf("Unable to create workfile")
return err
}
info, err := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
Id: info.Id,
Start: &i,
Size: proto.Int64(BLOCK_SIZE),
})
f.Write(block.Data)
}
f.Close()
return nil
}
func TestFileTest(t *testing.T) {
dbi, err := NewMongo("localhost")
if err != nil {
panic(err)
}
fmt.Printf("Testing")
u, _ := uuid.NewV4()
file_id := u.String()
finfo := agro_pb.FileInfo{
Name: proto.String("README"),
Id: proto.String(file_id),
}
w, _ := os.Getwd()
fmt.Printf("Dis: %s\n", w)
data, _ := ioutil.ReadFile("../../../README")
dbi.CreateFile(context.Background(), &finfo)
fmt.Printf("Sending: %s\n", data)
packet := agro_pb.DataBlock{
Id: proto.String(file_id),
Start: proto.Int64(0),
Len: proto.Int64(int64(len(data))),
Data: data,
}
dbi.WriteFile(context.Background(), &packet)
dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
info, _ := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
fmt.Printf("FileSize: %d\n", *info.Size)
block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
Id: proto.String(file_id),
Start: proto.Int64(0),
Size: info.Size,
})
fmt.Printf("ReadOut:%d '%s'\n", *block.Len, string(block.Data))
}