Пример #1
0
func main() {
	log.Infof("starting schema...")

	options := struct {
		Driver string `goptions:"-t,--type, obligatory, description='Type of database backend'"`
		DSN    string `goptions:"-d,--database, obligatory, description='DSN of the database backend'"`
	}{
	// No defaults
	}
	goptions.ParseAndFail(&options)

	database := &db.DB{
		Driver: options.Driver,
		DSN:    options.DSN,
	}

	log.Debugf("connecting to %s database at %s", database.Driver, database.DSN)
	if err := database.Connect(); err != nil {
		log.Errorf("failed to connect to %s database at %s: %s",
			database.Driver, database.DSN, err)
	}

	if err := database.Setup(); err != nil {
		log.Errorf("failed to set up schema in %s database at %s: %s",
			database.Driver, database.DSN, err)
		return
	}

	log.Infof("deployed schema version %d", db.CurrentSchema)
}
Пример #2
0
func (s *Supervisor) PurgeArchives() {
	log.Debugf("scanning for archives needing to be expired")

	// mark archives past their retention policy as expired
	toExpire, err := s.Database.GetExpiredArchives()
	if err != nil {
		log.Errorf("error retrieving archives needing to be expired: %s", err.Error())
	}
	for _, archive := range toExpire {
		log.Infof("marking archive %s has expiration date %s, marking as expired", archive.UUID, archive.ExpiresAt)
		err := s.Database.ExpireArchive(archive.UUID)
		if err != nil {
			log.Errorf("error marking archive %s as expired: %s", archive.UUID, err)
			continue
		}
	}

	// get archives that are not valid or purged
	toPurge, err := s.Database.GetArchivesNeedingPurge()
	if err != nil {
		log.Errorf("error retrieving archives to purge: %s", err.Error())
	}

	for _, archive := range toPurge {
		log.Infof("requesting purge of archive %s due to status '%s'", archive.UUID, archive.Status)
		task, err := s.Database.CreatePurgeTask("system", archive, s.PurgeAgent)
		if err != nil {
			log.Errorf("error scheduling purge of archive %s: %s", archive.UUID, err)
			continue
		}
		s.ScheduleTask(task)
	}
}
Пример #3
0
func (ba BasicAuthenticator) IsAuthenticated(r *http.Request) bool {
	authType, authToken := AuthHeader(r)
	log.Debugf("Checking `Authorization: %s %s` against our configuration", authType, authToken)

	if strings.ToLower(authType) == "basic" {
		decoded, err := base64.StdEncoding.DecodeString(authToken)
		if err != nil {
			log.Infof("Authorization header is corrupt: %s", err)
			return false
		}

		creds := strings.SplitN(string(decoded), ":", 2)
		if len(creds) != 2 {
			log.Infof("Authorization header is corrupt: '%s' does not contain a ':' delimiter",
				string(decoded))
			return false
		}

		log.Debugf("Received Authorization credentials for user '%s', password '%s'", creds[0], obfuscate(creds[1]))
		log.Debugf("checking against the configured credentials '%s', password '%s'", ba.Cfg.User, obfuscate(ba.Cfg.Password))
		return creds[0] == ba.Cfg.User && creds[1] == ba.Cfg.Password
	}

	log.Infof("Received an invalid Authorization header type '%s' (not 'Basic')", authType)
	return false
}
Пример #4
0
func main() {
	log.Infof("starting schema...")

	options := struct {
		Help    bool   `goptions:"-h, --help, description='Show the help screen'"`
		Driver  string `goptions:"-t, --type, description='Type of database backend (postgres or mysql)'"`
		DSN     string `goptions:"-d,--database, description='DSN of the database backend'"`
		Version bool   `goptions:"-v, --version, description='Display the SHIELD version'"`
	}{
	// No defaults
	}
	if err := goptions.Parse(&options); err != nil {
		fmt.Printf("%s\n", err)
		goptions.PrintHelp()
		return
	}
	if options.Help {
		goptions.PrintHelp()
		os.Exit(0)
	}
	if options.Version {
		if Version == "" {
			fmt.Printf("shield-schema (development)%s\n", Version)
		} else {
			fmt.Printf("shield-schema v%s\n", Version)
		}
		os.Exit(0)
	}
	if options.Driver == "" {
		fmt.Fprintf(os.Stderr, "You must indicate which type of database you wish to manage, via the `--type` option.\n")
		os.Exit(1)
	}
	if options.DSN == "" {
		fmt.Fprintf(os.Stderr, "You must specify the DSN of your database, via the `--database` option.\n")
		os.Exit(1)
	}

	database := &db.DB{
		Driver: options.Driver,
		DSN:    options.DSN,
	}

	log.Debugf("connecting to %s database at %s", database.Driver, database.DSN)
	if err := database.Connect(); err != nil {
		log.Errorf("failed to connect to %s database at %s: %s",
			database.Driver, database.DSN, err)
	}

	if err := database.Setup(); err != nil {
		log.Errorf("failed to set up schema in %s database at %s: %s",
			database.Driver, database.DSN, err)
		return
	}

	log.Infof("deployed schema version %d", db.CurrentSchema)
}
Пример #5
0
func main() {
	var opts ShieldAgentOpts
	opts.Log = "Info"
	if err := goptions.Parse(&opts); err != nil {
		fmt.Printf("%s\n", err)
		goptions.PrintHelp()
		return
	}
	if opts.Help {
		goptions.PrintHelp()
		os.Exit(0)
	}
	if opts.Version {
		if Version == "" {
			fmt.Printf("shield-agent (development)%s\n", Version)
		} else {
			fmt.Printf("shield-agent v%s\n", Version)
		}
		os.Exit(0)
	}
	if opts.ConfigFile == "" {
		fmt.Fprintf(os.Stderr, "You must specify a configuration file via `--config`\n")
		os.Exit(1)
	}

	log.SetupLogging(log.LogConfig{Type: "console", Level: opts.Log})
	log.Infof("starting agent")

	ag := agent.NewAgent()
	if err := ag.ReadConfig(opts.ConfigFile); err != nil {
		log.Errorf("configuration failed: %s", err)
		return
	}
	ag.Run()
}
Пример #6
0
func main() {
	supervisor.Version = Version
	var opts ShielddOpts
	opts.Log = "Info"
	if err := goptions.Parse(&opts); err != nil {
		fmt.Printf("%s\n", err)
		goptions.PrintHelp()
		return
	}

	if opts.Help {
		goptions.PrintHelp()
		os.Exit(0)
	}
	if opts.Version {
		if Version == "" {
			fmt.Printf("shieldd (development)\n")
		} else {
			fmt.Printf("shieldd v%s\n", Version)
		}
		os.Exit(0)
	}

	if opts.ConfigFile == "" {
		fmt.Fprintf(os.Stderr, "No config specified. Please try again using the -c/--config argument\n")
		os.Exit(1)
	}

	log.SetupLogging(log.LogConfig{Type: "console", Level: opts.Log})
	log.Infof("starting shield daemon")

	s := supervisor.NewSupervisor()
	if err := s.ReadConfig(opts.ConfigFile); err != nil {
		log.Errorf("Failed to load config: %s", err)
		return
	}

	s.SpawnAPI()
	s.SpawnWorkers()

	if err := s.Run(); err != nil {
		log.Errorf("shield daemon failed: %s", err)
	}
	log.Infof("stopping daemon")
}
Пример #7
0
func (s *Supervisor) CheckSchedule() {
	for _, job := range s.jobq {
		if !job.Runnable() {
			continue
		}

		log.Infof("scheduling execution of job %s [%s]", job.Name, job.UUID)
		task, err := s.Database.CreateBackupTask("system", job)
		if err != nil {
			log.Errorf("job -> task conversion / database update failed: %s", err)
			continue
		}
		s.ScheduleTask(task)

		err = job.Reschedule()
		if err != nil {
			log.Errorf("error encountered while determining next run of %s (%s): %s",
				job.UUID, job.Spec, err)
		} else {
			log.Infof("next run of %s [%s] which runs %s is at %s",
				job.Name, job.UUID, job.Spec, job.NextRun)
		}
	}
}
Пример #8
0
func main() {
	var opts ShieldAgentOpts
	opts.Log = "Info"
	if err := goptions.Parse(&opts); err != nil {
		fmt.Printf("%s\n", err)
		goptions.PrintHelp()
		return
	}

	log.SetupLogging(log.LogConfig{Type: "console", Level: opts.Log})
	log.Infof("starting agent")

	ag := agent.NewAgent()
	if err := ag.ReadConfig(opts.ConfigFile); err != nil {
		log.Errorf("configuration failed: %s", err)
		return
	}
	ag.Run()
}
Пример #9
0
func (s *Supervisor) Resync() error {
	jobq, err := s.Database.GetAllJobs(nil)
	if err != nil {
		return err
	}

	// calculate the initial run of each job
	for _, job := range jobq {
		err := job.Reschedule()
		if err != nil {
			log.Errorf("error encountered while determining next run of %s [%s] which runs %s: %s",
				job.Name, job.UUID, job.Spec, err)
		} else {
			log.Infof("initial run of %s [%s] which runs %s is at %s",
				job.Name, job.UUID, job.Spec, job.NextRun)
		}
	}

	s.jobq = jobq
	return nil
}
Пример #10
0
func (req *Request) Run(output chan string) error {
	cmd := exec.Command("shield-pipe")

	log.Infof("Executing %s request using target %s and store %s via shield-pipe", req.Operation, req.TargetPlugin, req.StorePlugin)
	log.Debugf("Target Endpoint config: %s", req.TargetEndpoint)
	log.Debugf("Store Endpoint config: %s", req.StoreEndpoint)

	cmd.Env = []string{
		fmt.Sprintf("HOME=%s", os.Getenv("HOME")),
		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
		fmt.Sprintf("USER=%s", os.Getenv("USER")),
		fmt.Sprintf("LANG=%s", os.Getenv("LANG")),

		fmt.Sprintf("SHIELD_OP=%s", req.Operation),
		fmt.Sprintf("SHIELD_STORE_PLUGIN=%s", req.StorePlugin),
		fmt.Sprintf("SHIELD_STORE_ENDPOINT=%s", req.StoreEndpoint),
		fmt.Sprintf("SHIELD_TARGET_PLUGIN=%s", req.TargetPlugin),
		fmt.Sprintf("SHIELD_TARGET_ENDPOINT=%s", req.TargetEndpoint),
		fmt.Sprintf("SHIELD_RESTORE_KEY=%s", req.RestoreKey),
	}

	if log.LogLevel() == syslog.LOG_DEBUG {
		cmd.Env = append(cmd.Env, "DEBUG=true")
	}

	log.Debugf("ENV: %s", strings.Join(cmd.Env, ","))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	drain := func(prefix string, out chan string, in io.Reader) {
		defer wg.Done()
		s := bufio.NewScanner(in)
		for s.Scan() {
			out <- fmt.Sprintf("%s:%s\n", prefix, s.Text())
		}
	}

	wg.Add(2)
	go drain("E", output, stderr)
	go drain("O", output, stdout)

	err = cmd.Start()
	if err != nil {
		close(output)
		return err
	}

	wg.Wait()
	close(output)

	err = cmd.Wait()
	if err != nil {
		return err
	}

	return nil
}
Пример #11
0
func (s *Supervisor) ScheduleTask(t *db.Task) {
	t.TimeoutAt = timestamp.Now().Add(s.Timeout)
	log.Infof("schedule task %s with deadline %v", t.UUID, t.TimeoutAt)
	s.schedq = append(s.schedq, t)
}
Пример #12
0
func (s *Supervisor) Run() error {
	if err := s.Database.Connect(); err != nil {
		return fmt.Errorf("failed to connect to %s database at %s: %s\n",
			s.Database.Driver, s.Database.DSN, err)
	}

	if err := s.Database.CheckCurrentSchema(); err != nil {
		return fmt.Errorf("database failed schema version check: %s\n", err)
	}

	if err := s.Resync(); err != nil {
		return err
	}
	if err := s.FailUnfinishedTasks(); err != nil {
		return err
	}
	if err := s.ReschedulePendingTasks(); err != nil {
		return err
	}

	for {
		select {
		case <-s.resync:
			if err := s.Resync(); err != nil {
				log.Errorf("resync error: %s", err)
			}

		case <-s.purge.C:
			s.PurgeArchives()

		case <-s.tick.C:
			s.CheckSchedule()

			// see if any tasks have been running past the timeout period
			if len(s.runq) > 0 {
				ok := true
				lst := make([]*db.Task, 0)
				now := timestamp.Now()

				for _, runtask := range s.runq {
					if now.After(runtask.TimeoutAt) {
						s.Database.CancelTask(runtask.UUID, now.Time())
						log.Errorf("shield timed out task '%s' after running for %v", runtask.UUID, s.Timeout)
						ok = false

					} else {
						lst = append(lst, runtask)
					}
				}

				if !ok {
					s.runq = lst
				}
			}

			// see if we have anything in the schedule queue
		SchedQueue:
			for len(s.schedq) > 0 {
				select {
				case s.workers <- s.schedq[0]:
					s.Database.StartTask(s.schedq[0].UUID, time.Now())
					s.schedq[0].Attempts++
					log.Infof("sent a task to a worker")
					s.runq = append(s.runq, s.schedq[0])
					log.Debugf("added task to the runq")
					s.schedq = s.schedq[1:]
				default:
					break SchedQueue
				}
			}

		case adhoc := <-s.adhoc:
			s.ScheduleAdhoc(adhoc)

		case u := <-s.updates:
			switch u.Op {
			case STOPPED:
				log.Infof("  %s: job stopped at %s", u.Task, u.StoppedAt)
				s.RemoveTaskFromRunq(u.Task)
				if err := s.Database.CompleteTask(u.Task, u.StoppedAt); err != nil {
					log.Errorf("  %s: !! failed to update database - %s", u.Task, err)
				}

			case FAILED:
				log.Warnf("  %s: task failed!", u.Task)
				s.RemoveTaskFromRunq(u.Task)
				if err := s.Database.FailTask(u.Task, u.StoppedAt); err != nil {
					log.Errorf("  %s: !! failed to update database - %s", u.Task, err)
				}

			case OUTPUT:
				log.Infof("  %s> %s", u.Task, strings.Trim(u.Output, "\n"))
				if err := s.Database.UpdateTaskLog(u.Task, u.Output); err != nil {
					log.Errorf("  %s: !! failed to update database - %s", u.Task, err)
				}

			case RESTORE_KEY:
				log.Infof("  %s: restore key is %s", u.Task, u.Output)
				if id, err := s.Database.CreateTaskArchive(u.Task, u.Output, time.Now()); err != nil {
					log.Errorf("  %s: !! failed to update database - %s", u.Task, err)
				} else {
					if !u.TaskSuccess {
						s.Database.InvalidateArchive(id)
					}
				}

			case PURGE_ARCHIVE:
				log.Infof("  %s: archive %s purged from storage", u.Task, u.Archive)
				if err := s.Database.PurgeArchive(u.Archive); err != nil {
					log.Errorf("  %s: !! failed to update database - %s", u.Task, err)
				}

			default:
				log.Errorf("  %s: !! unrecognized op type", u.Task)
			}
		}
	}
}
Пример #13
0
func (s *Supervisor) ScheduleAdhoc(a *db.Task) {
	log.Infof("schedule adhoc %s job", a.Op)

	switch a.Op {
	case db.BackupOperation:
		// expect a JobUUID to move to the schedq Immediately
		for _, job := range s.jobq {
			if !uuid.Equal(job.UUID, a.JobUUID) {
				continue
			}

			log.Infof("scheduling immediate (ad hoc) execution of job %s [%s]", job.Name, job.UUID)
			task, err := s.Database.CreateBackupTask(a.Owner, job)
			if err != nil {
				log.Errorf("job -> task conversion / database update failed: %s", err)
				if a.TaskUUIDChan != nil {
					a.TaskUUIDChan <- &db.TaskInfo{
						Err:  true,
						Info: err.Error(),
					}
				}
				continue
			}
			if a.TaskUUIDChan != nil {
				a.TaskUUIDChan <- &db.TaskInfo{
					Err:  false,
					Info: task.UUID.String(),
				}
			}

			s.ScheduleTask(task)
		}

	case db.RestoreOperation:
		archive, err := s.Database.GetArchive(a.ArchiveUUID)
		if err != nil {
			log.Errorf("unable to find archive %s for restore task: %s", a.ArchiveUUID, err)
			return
		}
		target, err := s.Database.GetTarget(a.TargetUUID)
		if err != nil {
			log.Errorf("unable to find target %s for restore task: %s", a.TargetUUID, err)
			return
		}
		task, err := s.Database.CreateRestoreTask(a.Owner, archive, target)
		if err != nil {
			log.Errorf("restore task database creation failed: %s", err)
			if a.TaskUUIDChan != nil {
				a.TaskUUIDChan <- &db.TaskInfo{
					Err:  true,
					Info: err.Error(),
				}
			}
			return
		}
		if a.TaskUUIDChan != nil {
			a.TaskUUIDChan <- &db.TaskInfo{
				Err:  false,
				Info: task.UUID.String(),
			}
		}

		s.ScheduleTask(task)
	}
}
Пример #14
0
func (agent *Agent) handleConn(conn *ssh.ServerConn, chans <-chan ssh.NewChannel, reqs <-chan *ssh.Request) {
	defer conn.Close()

	for newChannel := range chans {
		if newChannel.ChannelType() != "session" {
			log.Errorf("rejecting unknown channel type: %s\n", newChannel.ChannelType())
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}

		channel, requests, err := newChannel.Accept()
		if err != nil {
			log.Errorf("failed to accept channel: %s\n", err)
			return
		}

		defer channel.Close()

		for req := range requests {
			if req.Type != "exec" {
				log.Errorf("rejecting non-exec channel request (type=%s)\n", req.Type)
				req.Reply(false, nil)
				continue
			}

			request, err := ParseRequest(req)
			if err != nil {
				log.Errorf("%s\n", err)
				req.Reply(false, nil)
				continue
			}

			if err = request.ResolvePaths(agent); err != nil {
				log.Errorf("%s\n", err)
				req.Reply(false, nil)
				continue
			}

			//log.Errorf("got an agent-request [%s]\n", request.JSON)
			req.Reply(true, nil)

			// drain output to the SSH channel stream
			output := make(chan string)
			done := make(chan int)
			go func(out io.Writer, in chan string, done chan int) {
				for {
					s, ok := <-in
					if !ok {
						break
					}
					fmt.Fprintf(out, "%s", s)
					log.Debugf("%s", s)
				}
				close(done)
			}(channel, output, done)

			// run the agent request
			err = request.Run(output)
			<-done
			var rc int
			if exitErr, ok := err.(*exec.ExitError); ok {
				sys := exitErr.ProcessState.Sys()
				// os.ProcessState.Sys() may not return syscall.WaitStatus on non-UNIX machines,
				// so currently this feature only works on UNIX, but shouldn't crash on other OSes
				if ws, ok := sys.(syscall.WaitStatus); ok {
					if ws.Exited() {
						rc = ws.ExitStatus()
					} else {
						var signal syscall.Signal
						if ws.Signaled() {
							signal = ws.Signal()
						}
						if ws.Stopped() {
							signal = ws.StopSignal()
						}
						sigStr, ok := SIGSTRING[signal]
						if !ok {
							sigStr = "ABRT" // use ABRT as catch-all signal for any that don't translate
							log.Infof("Task execution terminted due to %s, translating as ABRT for ssh transport", signal)
						} else {
							log.Infof("Task execution terminated due to SIG%s", sigStr)
						}
						sigMsg := struct {
							Signal     string
							CoreDumped bool
							Error      string
							Lang       string
						}{
							Signal:     sigStr,
							CoreDumped: false,
							Error:      fmt.Sprintf("shield-pipe terminated due to SIG%s", sigStr),
							Lang:       "en-US",
						}
						channel.SendRequest("exit-signal", false, ssh.Marshal(&sigMsg))
						channel.Close()
						continue
					}
				}
			} else if err != nil {
				// we got some kind of error that isn't a command execution error,
				// from a UNIX system, use an magical error code to signal this to
				// the shield daemon - 16777216
				log.Infof("Task could not execute: %s", err)
				rc = 16777216
			}

			log.Infof("Task completed with rc=%d", rc)
			byteCode := make([]byte, 4)
			binary.BigEndian.PutUint32(byteCode, uint32(rc)) // SSH protocol is big-endian byte ordering
			channel.SendRequest("exit-status", false, byteCode)
			channel.Close()
		}
	}
}
Пример #15
0
func (oa OAuthenticator) OAuthCallback() http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Debugf("Incoming Auth request: %s", r)
		sess, err := gothic.Store.Get(r, gothic.SessionName)
		if err != nil {
			log.Errorf("Error retrieving session info: %s", err)
			w.WriteHeader(500)
			return
		}
		log.Debugf("Processing oauth callback for '%s'", sess.ID)
		if gothic.GetState(r) != sess.Values["state"] {
			w.WriteHeader(403)
			w.Write([]byte("Unauthorized"))
			return
		}

		if r.URL.Query().Get("code") == "" {
			log.Errorf("No code detected in oauth callback: %v", r)
			w.WriteHeader(403)
			w.Write([]byte("No oauth code issued from provider"))
			return
		}

		user, err := gothic.CompleteUserAuth(w, r)
		if err != nil {
			log.Errorf("Error verifying oauth success: %s. Request: %v", err, r)
			w.WriteHeader(403)
			w.Write([]byte("UnOAuthorized"))
			return
		}

		log.Debugf("Authenticated user %#v", user)

		ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: user.AccessToken})
		ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, oa.Cfg.Client)
		tc := oauth2.NewClient(ctx, ts)

		log.Debugf("Checking authorization...")
		membership, err := OAuthVerifier.Membership(user, tc)
		if err != nil {
			log.Errorf("Error retreiving user membership: %s", err)
			w.WriteHeader(403)
			w.Write([]byte("Unable to verify your membership"))
			return
		}

		if !OAuthVerifier.Verify(user.NickName, membership) {
			log.Debugf("Authorization denied")
			w.WriteHeader(403)
			w.Write([]byte("You are not authorized to view this content"))
			return
		}

		log.Infof("Successful login for %s", user.NickName)

		redirect := "/"
		if flashes := sess.Flashes(); len(flashes) > 0 {
			if flash, ok := flashes[0].(string); ok {
				// don't redirect back to api calls, to prevent auth redirection loops
				if !apiCall.MatchString(flash) || cliAuthCall.MatchString(flash) {
					redirect = flash
				}
			}
		}

		sess.Values["User"] = user.NickName
		sess.Values["Membership"] = membership
		err = sess.Save(r, w)
		if err != nil {
			log.Errorf("Error saving session: %s", err)
			w.WriteHeader(500)
			w.Write([]byte("Unable to save authentication data. Check the SHIELD logs for more info."))
			return
		}

		http.Redirect(w, r, redirect, 302) // checks auth
	})
}