Exemple #1
0
func upload_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {

	file, _ := os.Open(filePath)

	finfo := agro_pb.FileInfo{
		Name: proto.String(path.Base(filePath)),
		Id:   proto.String(fileID),
	}
	dbi.CreateFile(context.Background(), &finfo)
	buffer := make([]byte, BLOCK_SIZE)
	bytes_written := int64(0)
	for {
		n, _ := file.Read(buffer)
		if n == 0 {
			break
		}
		packet := agro_pb.DataBlock{
			Id:    proto.String(fileID),
			Start: proto.Int64(bytes_written),
			Len:   proto.Int64(int64(n)),
			Data:  buffer[:n],
		}
		dbi.WriteFile(context.Background(), &packet)
		bytes_written += int64(n)
	}
	file.Close()
	dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	return nil
}
Exemple #2
0
func TestProtoConvert_2(t *testing.T) {
	a := &agro_pb.Task{
		Id:        proto.String("test"),
		Command:   proto.String("/bin/echo Testing"),
		Container: proto.String("ubuntu"),
	}
	doProtoConvert_task(a)
}
Exemple #3
0
func (self *MesosManager) BuildExecutorInfo() *mesos.ExecutorInfo {

	executorCommand := fmt.Sprintf("/Users/ellrott/workspaces/agro/agro_exec")

	return &mesos.ExecutorInfo{
		ExecutorId: &mesos.ExecutorID{
			Value: proto.String("AgroExec"),
		},
		Name:   proto.String("AgroExecutor"),
		Source: proto.String("go_test"),
		Command: &mesos.CommandInfo{
			Value: proto.String(executorCommand),
			Uris:  []*mesos.CommandInfo_URI{},
		},
	}
}
Exemple #4
0
func (self *MongoInterface) GetTaskStatus(ids *agro_pb.IDQuery, stream agro_pb.Scheduler_GetTaskStatusServer) error {
	i_task := self.db.C("task").Find(bson.M{"_id": bson.M{"$in": ids.Ids}}).Iter()
	result := make(map[string]interface{})
	for i_task.Next(&result) {

		i_job := self.db.C("job").Find(bson.M{"task_id": result["_id"]}).Iter()
		runs := make([]string, 0)
		var completed *string = nil

		job_result := make(map[string]interface{})
		for i_job.Next(&job_result) {
			runs = append(runs, string(job_result["_id"].(string)))
			if job_result["state"] == "OK" {
				s := string(job_result["_id"].(string))
				completed = &s
			}
		}
		s := agro_pb.State(agro_pb.State_value[result["state"].(string)])
		a := &agro_pb.TaskStatus{
			Id:           proto.String(string(result["_id"].(string))),
			State:        &s,
			Runs:         runs,
			CompletedJob: completed,
		}
		stream.Send(a)
	}
	return nil
}
Exemple #5
0
func (self *MongoInterface) GetFileInfo(ctx context.Context, i *agro_pb.FileID) (*agro_pb.FileInfo, error) {
	f, err := self.db.GridFS("fs").OpenId(*i.Id)
	if err != nil {
		return &agro_pb.FileInfo{
			Name:  proto.String(fmt.Sprintf("%s error: %s", *i.Id, err)),
			Id:    i.Id,
			State: agro_pb.State_ERROR.Enum(),
		}, nil
	}
	o := agro_pb.FileInfo{
		Name:  proto.String(f.Name()),
		Id:    proto.String(f.Id().(string)),
		Size:  proto.Int64(f.Size()),
		State: agro_pb.State_OK.Enum(),
	}
	f.Close()
	return &o, nil
}
Exemple #6
0
func (self *MongoInterface) GetJobStatus(ids *agro_pb.IDQuery, stream agro_pb.Scheduler_GetJobStatusServer) error {
	i := self.db.C("job").Find(bson.M{"_id": bson.M{"$in": ids.Ids}}).Iter()
	result := make(map[string]interface{})
	for i.Next(&result) {
		s := agro_pb.State(agro_pb.State_value[result["state"].(string)])
		a := &agro_pb.JobStatus{
			Id:    proto.String(string(result["_id"].(string))),
			State: &s,
		}
		stream.Send(a)
	}
	return nil
}
Exemple #7
0
func TestDocument(t *testing.T) {
	doc := agro_pb.Document{
		Id: proto.String("test"),
		Fields: []*agro_pb.Field{
			&agro_pb.Field{Path: []string{"hello"}, Value: &agro_pb.Field_FloatValue{FloatValue: 35.0}},
		},
	}

	dbi, err := NewMongo("localhost")
	if err != nil {
		t.Error("DB Failure")
		panic(err)
	}
	t.Log("Testing")
	fmt.Print("Testing2")
	dbi.CreateDoc(context.Background(), &doc)

	out, _ := dbi.GetDoc(context.Background(), &agro_pb.FileID{Id: proto.String("test")})

	log.Print("output: ", out)

}
Exemple #8
0
func (self *MesosManager) Start(engine *agro_engine.WorkEngine) {
	self.engine = engine
	self.executor = self.BuildExecutorInfo()

	//frameworkIdStr := FRAMEWORK_ID
	failoverTimeout := 0.0
	//frameworkId := &mesos.FrameworkID{Value: &frameworkIdStr}
	config := sched.DriverConfig{
		Master: self.master,
		Framework: &mesos.FrameworkInfo{
			Name:            proto.String("AgroFramework"),
			User:            proto.String(""),
			FailoverTimeout: &failoverTimeout,
			//Id:              frameworkId,
		},
		Scheduler: self,
	}

	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		log.Printf("Driver Error: %s", err)
		panic(err)
	}
	//driver.Init()
	//defer driver.Destroy()
	//go self.EventLoop()

	status, err := driver.Start()
	log.Printf("Driver Status:%s", status)
	if err != nil {
		log.Printf("Mesos Start Error: %s", err)
		panic(err)
	}
	//<-self.exit
	//log.Printf("Mesos Exit")
	//driver.Stop(false)
}
Exemple #9
0
func (self *ForkManager) watcher(sched agro_pb.SchedulerClient, filestore agro_pb.FileStoreClient) {
	self.sched = sched
	self.files = filestore
	hostname, _ := os.Hostname()
	jobchan := make(chan agro_pb.Job, 10)
	for i := 0; i < self.procCount; i++ {
		go self.worker(jobchan)
	}
	var sleep_size int64 = 1
	for self.running {
		if self.check_func != nil {
			self.check_func(self.status)
		}
		job_stream, _ := self.sched.GetJobToRun(self.ctx,
			&agro_pb.JobRequest{
				Max: proto.Int32(1),
				Worker: &agro_pb.WorkerInfo{
					Id:       proto.String(self.workerId),
					Hostname: proto.String(hostname),
					LastPing: proto.Int64(time.Now().Unix()),
				},
			})
		job, _ := job_stream.Recv()
		if job != nil {
			sleep_size = 1
			log.Printf("Found job: %s", job)
			jobchan <- *job
		} else {
			log.Printf("No jobs found")
			if sleep_size < 20 {
				//  sleep_size += 1
			}
			time.Sleep(time.Second * time.Duration(sleep_size))
		}
	}
	close(jobchan)
}
Exemple #10
0
func (self *MongoInterface) CreateTaskJob(task *agro_pb.Task) {
	u, _ := uuid.NewV4()
	job := &agro_pb.Job{
		Id:           proto.String(u.String()),
		TaskId:       task.Id,
		State:        agro_pb.State_WAITING.Enum(),
		Command:      task.Command,
		Container:    task.Container,
		Requirements: task.Requirements,
		Args:         task.Args,
	}
	log.Printf("Creating job %s", job)
	self.AddJob(job)
	self.jobCache[*task.Id] = job
}
Exemple #11
0
func (self *Webserver) GetFile(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	fileID := params["id"]

	info, _ := self.dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
		block, _ := self.dbi.ReadFile(context.Background(),
			&agro_pb.ReadRequest{
				Id:    info.Id,
				Start: &i,
				Size:  proto.Int64(BLOCK_SIZE),
			})
		w.Write(block.Data)
	}
}
Exemple #12
0
func PackDoc(id string, doc map[string]interface{}) *agro_pb.Document {
	fields := make([]*agro_pb.Field, 0)
	for f := range StreamFields(doc) {
		a := f
		if len(a.Path) != 1 || a.Path[0] != "_id" {
			fields = append(fields, &a)
		}
	}

	//for _, f := range fields {
	//  log.Printf("Got: %#v %s", f.Path, f.Value)
	//}

	//log.Printf("OutFields: %d", len(fields))
	return &agro_pb.Document{Id: proto.String(id), Fields: fields}
}
Exemple #13
0
func download_file(fileID string, filePath string, dbi agro_pb.FileStoreClient) error {
	f, err := os.Create(filePath)
	if err != nil {
		log.Printf("Unable to create workfile")
		return err
	}
	info, err := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(fileID)})
	for i := int64(0); i < *info.Size; i += BLOCK_SIZE {
		block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
			Id:    info.Id,
			Start: &i,
			Size:  proto.Int64(BLOCK_SIZE),
		})
		f.Write(block.Data)
	}
	f.Close()
	return nil
}
Exemple #14
0
func TestFileTest(t *testing.T) {

	dbi, err := NewMongo("localhost")
	if err != nil {
		panic(err)
	}
	fmt.Printf("Testing")

	u, _ := uuid.NewV4()
	file_id := u.String()

	finfo := agro_pb.FileInfo{
		Name: proto.String("README"),
		Id:   proto.String(file_id),
	}
	w, _ := os.Getwd()
	fmt.Printf("Dis: %s\n", w)
	data, _ := ioutil.ReadFile("../../../README")
	dbi.CreateFile(context.Background(), &finfo)

	fmt.Printf("Sending: %s\n", data)

	packet := agro_pb.DataBlock{
		Id:    proto.String(file_id),
		Start: proto.Int64(0),
		Len:   proto.Int64(int64(len(data))),
		Data:  data,
	}
	dbi.WriteFile(context.Background(), &packet)
	dbi.CommitFile(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})

	info, _ := dbi.GetFileInfo(context.Background(), &agro_pb.FileID{Id: proto.String(file_id)})
	fmt.Printf("FileSize: %d\n", *info.Size)

	block, _ := dbi.ReadFile(context.Background(), &agro_pb.ReadRequest{
		Id:    proto.String(file_id),
		Start: proto.Int64(0),
		Size:  info.Size,
	})

	fmt.Printf("ReadOut:%d '%s'\n", *block.Len, string(block.Data))
}