Esempio n. 1
0
// Start statrs a sub app in its own goroutine
func (a *App) subAppStart(app subapp.App, log *logrus.Entry) {
	log.Debugf("starting sub app %q", app.Name())
	a.wg.Add(1)
	go func() {
		defer a.wg.Done()

		if err := app.Run(log); err != nil {
			// Check the error type, if it comes from a panic recovery
			// reload the app
			switch e := err.(type) {
			case *errors.Error:
				errors.LogErrors(log.WithField("app", app.Name()), e)

				// Notify the safeguard of the error
				a.safeguard.Event()

				// Write to the reload channel in a goroutine to prevent deadlocks
				go func() {
					a.reload <- app
				}()
			// Only log the error
			default:
				log.Error(err)
				go a.Stop(log)
			}
		}
	}()
	log.Debugf("sub app %q started", app.Name())
}
Esempio n. 2
0
// Stop stops the app
func (b *Base) Stop(log *logrus.Entry) {
	log.WithField("app", b.AppName).Debug("stopping app asynchronously")
	if b.AppStatus == Started {
		close(b.Done)
		b.AppStatus = Stopped
	}
}
Esempio n. 3
0
func (c *Cleaner) cleanDoneVideos(log *logrus.Entry) {
	list, err := c.config.Downloader.Client.List()
	if err != nil {
		log.Errorf("error while getting torrent list: %q", err)
		return
	}

	for _, t := range list {
		torrentInfos := t.Infos()

		log = log.WithField("torrent_name", torrentInfos.Name)

		// Check if the file is ready to be cleaned
		isReady := c.isReadyToBeCleaned(t, log)
		if !isReady {
			log.Debug("torrent is not ready to be cleaned")
			continue
		}

		// We remove the torrent
		log.Debugf("removing torrent")
		err := c.config.Downloader.Client.Remove(t)
		if err != nil {
			log.Errorf("got error when removing torrent : %q", err)
			continue
		}

		log.Debug("removing files")
		if err = c.clean(t, log); err != nil {
			log.Errorf("failed to clean torrent files: %q", err)
			continue
		}
	}
}
Esempio n. 4
0
// GetDetails helps getting infos for a movie
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetDetails(log *logrus.Entry) error {
	c := errors.NewCollector()

	if len(m.Detailers) == 0 {
		c.Push(errors.Wrap("No detailer available").Fatal())
		return c
	}

	var done bool
	for _, d := range m.Detailers {
		detailerLog := log.WithField("detailer", d.Name())
		err := d.GetDetails(m, detailerLog)
		if err == nil {
			done = true
			break
		}
		c.Push(errors.Wrap(err).Ctx("Detailer", d.Name()))
	}
	if !done {
		c.Push(errors.Wrap("All detailers failed").Fatal())
	}

	if c.HasErrors() {
		return c
	}

	return nil
}
Esempio n. 5
0
// GetSubtitle implements the subtitle interface
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetSubtitle(log *logrus.Entry) error {
	c := errors.NewCollector()

	var subtitle Subtitle
	for _, subtitler := range m.Subtitlers {
		var err error
		subtitlerLog := log.WithField("subtitler", subtitler.Name())
		subtitle, err = subtitler.GetMovieSubtitle(m, subtitlerLog)
		if err == nil {
			break
		}

		c.Push(errors.Wrap(err).Ctx("Subtitler", subtitler.Name()))
	}

	if subtitle != nil {
		file, err := os.Create(m.File.SubtitlePath())
		if err != nil {
			c.Push(errors.Wrap(err).Fatal())
			return c
		}
		defer file.Close()
		defer subtitle.Close()

		if _, err := io.Copy(file, subtitle); err != nil {
			c.Push(errors.Wrap(err).Fatal())
			return c
		}
	}

	if c.HasErrors() {
		return c
	}
	return nil
}
Esempio n. 6
0
// Run runs the safeguard
func (s *Safeguard) Run(log *logrus.Entry) error {
	s.wg.Add(1)
	defer s.wg.Done()

	log = log.WithField("module", "safeguard")
	log.Debug("safeguard started")

	for {
		select {
		case <-s.done:
			log.Debug("safeguard stopped")
			return nil
		case <-s.event:
			// Increase the event count
			s.count++

			if s.count >= MaxEventCount {
				ctx := errors.Context{
					"current_count": s.count,
					"max_count":     MaxEventCount,
				}

				return errors.New("got %d safeguard events in less than %s",
					s.count, MaxEventDelay).Fatal().AddContext(ctx)
			}
		case <-time.After(MaxEventDelay):
			// Reset the panic count is there was not panic during the
			// MaxPanicDelay
			s.count = 0
		}
	}
}
Esempio n. 7
0
// Watch implements the modules fsNotifier interface
func (fs *FsNotify) Watch(watchPath string, ctx polochon.FsNotifierCtx, log *logrus.Entry) error {
	// Create a new watcher
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	fs.watcher = watcher

	// Ensure that the watch path exists
	if _, err := os.Stat(watchPath); os.IsNotExist(err) {
		return err
	}

	log = log.WithField("module", moduleName)

	// Run the event handler
	go fs.eventHandler(ctx, log)

	// Watch the path
	if err := fs.watcher.Add(watchPath); err != nil {
		return err
	}

	return nil
}
func reportCompleted(client client.Queue, task *TaskRun, log *logrus.Entry) *updateError {
	_, err := client.ReportCompleted(task.TaskID, strconv.Itoa(task.RunID))
	if err != nil {
		log.WithField("error", err).Warn("Not able to report successful completion for task.")
		return &updateError{err: err.Error()}
	}
	return nil
}
Esempio n. 9
0
// Notify sends video to the notifiers
func (o *Organizer) Notify(v polochon.Video, log *logrus.Entry) {
	log = log.WithField("function", "notify")
	for _, n := range o.config.Notifiers {
		if err := n.Notify(v, log); err != nil {
			log.Warnf("failed to send a notification from notifier: %q: %q", n.Name(), err)
		}
	}
}
Esempio n. 10
0
// DecorateRuntimeContext appends line, file and function context to the logger
func DecorateRuntimeContext(logger *logrus.Entry) *logrus.Entry {
	if pc, file, line, ok := runtime.Caller(1); ok {
		fName := runtime.FuncForPC(pc).Name()
		return logger.WithField("file", file).WithField("line", line).WithField("func", fName)
	} else {
		return logger
	}
}
Esempio n. 11
0
func (k *Kickass) listMoviesByUser(movies map[string]*polochon.Movie, user string, log *logrus.Entry) error {
	query := &kickass.Query{
		User:     user,
		OrderBy:  "seeders",
		Order:    "desc",
		Category: string(MoviesCategory),
	}
	log = log.WithField("explore_user", user)

	torrents, err := k.client.ListByUser(query)
	if err != nil {
		return err
	}

	for _, t := range torrents {
		torrentStr := torrentGuessitStr(t)
		guess, err := guessit.Guess(torrentStr)
		if err != nil {
			continue
		}

		// Get the torrent quality
		torrentQuality := polochon.Quality(guess.Quality)
		if !torrentQuality.IsAllowed() {
			log.Infof("kickass: unhandled quality: %q", torrentQuality)
			continue
		}

		// Get the movie if its already in the map
		m, ok := movies[guess.Title]
		if !ok {
			// Create a new movie
			m = polochon.NewMovie(polochon.MovieConfig{})
			m.Title = guess.Title
			if guess.Year != 0 {
				m.Year = guess.Year
			}
		}

		log.WithFields(logrus.Fields{
			"torrent_quality": guess.Quality,
			"movie_title":     guess.Title,
		}).Debug("Adding torrent to the list")

		m.Torrents = append(m.Torrents, polochon.Torrent{
			Quality:    torrentQuality,
			URL:        t.MagnetURL,
			Seeders:    t.Seed,
			Leechers:   t.Leech,
			Source:     moduleName,
			UploadUser: user,
		})

		movies[m.Title] = m
	}

	return nil
}
Esempio n. 12
0
// Run starts the server
func (s *Server) Run(log *logrus.Entry) error {
	s.log = log.WithField("app", AppName)

	// Init the app
	s.InitStart(log)

	s.gracefulServer = manners.NewWithServer(s.httpServer(s.log))
	return s.gracefulServer.ListenAndServe()
}
func reportException(client client.Queue, task *TaskRun, reason runtime.ExceptionReason, log *logrus.Entry) *updateError {
	payload := queue.TaskExceptionRequest{Reason: string(reason)}
	_, err := client.ReportException(task.TaskID, strconv.Itoa(task.RunID), &payload)
	if err != nil {
		log.WithField("error", err).Warn("Not able to report exception for task")
		return &updateError{err: err.Error()}
	}
	return nil
}
Esempio n. 14
0
func handleConnection(conn net.Conn, jobd *jobserver.JobServer, cbor *codec.CborHandle, reqLogger *logrus.Logger) {
	defer conn.Close()

	var reqLog, errLog *logrus.Entry
	fields := logrus.Fields{
		"remote": conn.RemoteAddr(),
	}
	errLog = logrus.WithFields(fields)
	if reqLogger != nil {
		reqLog = reqLogger.WithFields(fields)
	}

	jobdv := reflect.ValueOf(jobd)

	reader := bufio.NewReader(conn)
	decoder := codec.NewDecoder(reader, cbor)
	writer := bufio.NewWriter(conn)
	encoder := codec.NewEncoder(writer, cbor)

	for {
		var request cborrpc.Request
		err := decoder.Decode(&request)
		if err == io.EOF {
			if reqLog != nil {
				reqLog.Debug("Connection closed")
			}
			return
		} else if err != nil {
			errLog.WithError(err).Error("Error reading message")
			return
		}
		if reqLog != nil {
			reqLog.WithFields(logrus.Fields{
				"id":     request.ID,
				"method": request.Method,
			}).Debug("Request")
		}
		response := doRequest(jobdv, request)
		if reqLog != nil {
			entry := reqLog.WithField("id", response.ID)
			if response.Error != "" {
				entry = entry.WithField("error", response.Error)
			}
			entry.Debug("Response")
		}
		err = encoder.Encode(response)
		if err != nil {
			errLog.WithError(err).Error("Error encoding response")
			return
		}
		err = writer.Flush()
		if err != nil {
			errLog.WithError(err).Error("Error writing response")
			return
		}
	}
}
Esempio n. 15
0
func (k *Kickass) searchUser(s Searcher, log *logrus.Entry, user string) ([]polochon.Torrent, error) {
	query := &kickass.Query{
		User:     user,
		OrderBy:  "seeders",
		Order:    "desc",
		Category: string(s.category()),
		Search:   s.searchStr(),
	}
	log = log.WithField("search_user", user)

	torrents, err := k.client.Search(query)
	if err != nil {
		return nil, err
	}

	result := []polochon.Torrent{}
	for _, t := range torrents {
		torrentStr := torrentGuessitStr(t)
		guess, err := guessit.Guess(torrentStr)
		if err != nil {
			continue
		}

		if !s.isValidGuess(guess, log) {
			continue
		}

		// Default quality
		if s.category() == ShowsCategory && guess.Quality == "" {
			guess.Quality = string(polochon.Quality480p)
		}

		// Get the torrent quality
		torrentQuality := polochon.Quality(guess.Quality)
		if !torrentQuality.IsAllowed() {
			log.Infof("kickass: unhandled quality: %q", torrentQuality)
			continue
		}

		log.WithFields(logrus.Fields{
			"torrent_quality": guess.Quality,
			"torrent_name":    torrentStr,
		}).Debug("Adding torrent to the list")

		// Add the torrent
		result = append(result, polochon.Torrent{
			Quality:    torrentQuality,
			URL:        t.MagnetURL,
			Seeders:    t.Seed,
			Leechers:   t.Leech,
			Source:     moduleName,
			UploadUser: user,
		})
	}

	return result, nil
}
Esempio n. 16
0
// BlockingStop stops the app a nd waits for it to be done
func (b *Base) BlockingStop(log *logrus.Entry) {
	log.WithField("app", b.AppName).Debug("stopping app synchronously")

	// Send a signal to stop the app
	b.Stop(log)

	// Wait for all the goroutines to be done
	b.Wg.Wait()

	log.WithField("app_name", b.AppName).Info("app stopped")
}
Esempio n. 17
0
// validateSignature validates the request payload with the user provided key using the
// HMAC algo
func validateSignature(requestLog *logrus.Entry, r *http.Request, key string, payload []byte) bool {
	// if we don't have a secret to validate then just return true
	// because the user does not care about security
	if key == "" {
		return true
	}
	actual := r.Header.Get("X-Hub-Signature")
	expected, err := getExpectedSignature([]byte(key), payload)
	if err != nil {
		requestLog.WithField("gh_signature", actual).WithField("error", err).Error("parse expected signature")
		return false
	}
	return hmac.Equal([]byte(expected), []byte(actual))
}
Esempio n. 18
0
// startFsNotifier starts the FsNotifier
func (o *Organizer) startFsNotifier(log *logrus.Entry) error {
	ctx := polochon.FsNotifierCtx{
		Event: o.event,
		Done:  o.Done,
		Wg:    &o.Wg,
	}

	// Send a notification to organize the whole folder on app start
	watcherPath := o.config.Watcher.Dir
	ctx.Event <- watcherPath

	// Launch the FsNotifier
	if err := o.config.Watcher.FsNotifier.Watch(watcherPath, ctx, log); err != nil {
		return err
	}

	var err error
	o.Wg.Add(1)
	go func() {
		defer func() {
			o.Wg.Done()
			if r := recover(); r != nil {
				err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
					"sub_app": AppName,
				})
				o.Stop(log)
			}
		}()

		for {
			select {
			case file := <-ctx.Event:
				log.WithField("event", file).Debugf("got an event")
				if err := o.organize(file, log); err != nil {
					log.Errorf("failed to organize file: %q", err)
				}
			case <-o.Done:
				log.Debug("organizer done handling events")
				return
			}
		}
	}()

	o.Wg.Wait()

	return err
}
Esempio n. 19
0
// Run starts the downloader
func (o *Organizer) Run(log *logrus.Entry) error {
	// Create the channels
	o.event = make(chan string, 1)
	// Init the app
	o.InitStart(log)

	log = log.WithField("app", AppName)

	defer log.Debug("organizer stopped")

	// Start the file system notifier
	if err := o.startFsNotifier(log); err != nil {
		return err
	}

	return nil
}
Esempio n. 20
0
// GetTorrents helps getting the torrent files for a movie
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetTorrents(log *logrus.Entry) error {
	c := errors.NewCollector()

	for _, t := range m.Torrenters {
		torrenterLog := log.WithField("torrenter", t.Name())
		err := t.GetTorrents(m, torrenterLog)
		if err == nil {
			break
		}
		c.Push(errors.Wrap(err).Ctx("Torrenter", t.Name()))
	}

	if c.HasErrors() {
		return c
	}
	return nil
}
Esempio n. 21
0
/*
Load problem from directory
*/
func Load(path string, logger *logrus.Entry) (Problem, error) {
	logger = logger.WithField("state", "load_problem")
	path = filepath.Join(path, "problem.toml")
	var problem Problem
	_, err := toml.DecodeFile(path, &problem)
	if err != nil {
		logger.WithFields(logrus.Fields{
			"type":  "error",
			"error": IE.String(),
			"info":  err.Error()}).Error("Loading problem: Failed")
		return problem, err
	}
	logger.WithFields(logrus.Fields{
		"type": "info",
		"info": problem}).Infof("Loaded Problem: %s", problem.Name)
	return problem, nil
}
Esempio n. 22
0
func new(host string, log *logrus.Entry) *guestTools {
	got := got.New()
	got.Client = &http.Client{Timeout: 5 * time.Second}
	got.MaxSize = 10 * 1024 * 1024
	got.Log = log.WithField("guest-tools", "http-got")
	got.Retries = 15
	got.BackOff = backOff

	ctx, cancel := context.WithCancel(context.Background())
	return &guestTools{
		baseURL:       "http://" + host + "/",
		got:           got,
		log:           log,
		pollingCtx:    ctx,
		cancelPolling: cancel,
	}
}
Esempio n. 23
0
func WithValue(parent Context, key string, value interface{}) Context {
	c := context.WithValue(parent, key, value)

	var entry *logrus.Entry
	if ctx, ok := parent.(*ctx); ok {
		entry = ctx.Entry
	} else {
		entry = std
	}

	ctx := &ctx{
		Context: c,
		Entry:   entry.WithField(key, value),
	}

	return ctx
}
// Create a new instance of the task manager that will be responsible for claiming,
// executing, and resolving units of work (tasks).
func newTaskManager(
	config *configType, engine engines.Engine, pluginManager plugins.Plugin,
	environment *runtime.Environment, log *logrus.Entry, gc *gc.GarbageCollector,
) (*Manager, error) {
	queue := tcqueue.New(
		&tcclient.Credentials{
			ClientID:    config.Credentials.ClientID,
			AccessToken: config.Credentials.AccessToken,
			Certificate: config.Credentials.Certificate,
		},
	)

	if config.QueueBaseURL != "" {
		queue.BaseURL = config.QueueBaseURL
	}

	service := &queueService{
		capacity:         config.Capacity,
		interval:         config.PollingInterval,
		client:           queue,
		provisionerID:    config.ProvisionerID,
		workerGroup:      config.WorkerGroup,
		workerID:         config.WorkerID,
		workerType:       config.WorkerType,
		log:              log.WithField("component", "Queue Service"),
		expirationOffset: config.ReclaimOffset,
	}

	m := &Manager{
		tasks:         make(map[string]*TaskRun),
		done:          make(chan struct{}),
		config:        config,
		engine:        engine,
		environment:   environment,
		log:           log,
		queue:         service,
		provisionerID: config.ProvisionerID,
		workerGroup:   config.WorkerGroup,
		workerID:      config.WorkerID,
		gc:            gc,
	}

	m.pluginManager = pluginManager
	return m, nil
}
Esempio n. 25
0
func (c *Cleaner) isReadyToBeCleaned(d polochon.Downloadable, log *logrus.Entry) bool {
	torrent := d.Infos()
	log = log.WithField("torrent_name", torrent.Name)

	// First check that the torrent download is finished
	if !torrent.IsFinished {
		log.Debugf("torrent is not yet finished")
		return false
	}

	// Check that the ratio is reached
	if torrent.Ratio < c.config.Downloader.Cleaner.Ratio {
		log.Debugf("ratio is not reached (%.02f / %.02f)", torrent.Ratio, c.config.Downloader.Cleaner.Ratio)
		return false
	}

	return true
}
Esempio n. 26
0
/*
Save problem to path
*/
func Save(path string, problem Problem, logger *logrus.Entry) error {
	logger = logger.WithField("state", "save_problem")
	out := CreateFile(filepath.Join(path, "problem.toml"), logger)
	defer out.Close()
	encoder := toml.NewEncoder(out)
	err := encoder.Encode(problem)
	if err != nil {
		logger.WithFields(logrus.Fields{
			"type":  "error",
			"error": IE,
			"info":  err.Error()}).Error("Encode problem.toml: Failed")
		return err
	}
	logger.WithFields(logrus.Fields{
		"type": "info",
		"info": problem}).Infof("Saved problem: %s", problem.Name)
	return nil
}
Esempio n. 27
0
// GetMovieList implements the explorer interface
func (y *Yts) GetMovieList(option polochon.ExplorerOption, log *logrus.Entry) ([]*polochon.Movie, error) {
	log = log.WithField("explore_category", "movies")

	opt, err := translateMovieOptions(option)
	if err != nil {
		return nil, err
	}

	movieList, err := yts.GetList(1, 6, opt, yts.OrderDesc)
	if err != nil {
		return nil, err
	}

	result := []*polochon.Movie{}
	for _, movie := range movieList {
		// Create the movie
		m := polochon.NewMovie(polochon.MovieConfig{})
		m.Title = movie.Title
		m.Year = movie.Year
		m.ImdbID = movie.ImdbID

		// Add the torrents
		for _, t := range movie.Torrents {
			// Get the torrent quality
			torrentQuality := polochon.Quality(t.Quality)
			if !torrentQuality.IsAllowed() {
				log.Debugf("yts: unhandled quality: %q", torrentQuality)
				continue
			}
			m.Torrents = append(m.Torrents, polochon.Torrent{
				Quality:  torrentQuality,
				URL:      t.URL,
				Seeders:  t.Seeds,
				Leechers: t.Peers,
				Source:   moduleName,
			})
		}

		// Append the movie
		result = append(result, m)
	}

	return result, nil
}
Esempio n. 28
0
// Run starts the downloader
func (d *Downloader) Run(log *logrus.Entry) error {
	log = log.WithField("app", AppName)

	// Init the app
	d.InitStart(log)

	log.Debug("downloader started")

	// Lauch the downloader at startup
	log.Debug("initial downloader launch")
	d.event = make(chan struct{}, 1)
	d.event <- struct{}{}

	// Start the ticker
	d.Wg.Add(1)
	go func() {
		defer d.Wg.Done()
		d.ticker(log)
	}()

	// Start the downloader
	var err error
	d.Wg.Add(1)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
					"sub_app": AppName,
				})
				d.Stop(log)
			}

			d.Wg.Done()
		}()
		d.downloader(log)
	}()

	defer log.Debug("downloader stopped")

	d.Wg.Wait()

	return err
}
Esempio n. 29
0
// Run starts the cleaner
func (c *Cleaner) Run(log *logrus.Entry) error {
	log = log.WithField("app", AppName)

	// Init the app
	c.InitStart(log)

	c.event = make(chan struct{}, 1)

	log.Debug("cleaner started")

	log.Debug("initial cleaner launch")
	c.event <- struct{}{}

	// Start the ticker
	c.Wg.Add(1)
	go func() {
		defer c.Wg.Done()
		c.ticker(log)
	}()

	// Start the cleaner
	var err error
	c.Wg.Add(1)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
					"sub_app": AppName,
				})
				c.Stop(log)
			}

			c.Wg.Done()
		}()
		c.cleaner(log)
	}()

	defer log.Debug("cleaner stopped")

	c.Wg.Wait()

	return err
}
Esempio n. 30
0
/*
Gen creates input files.
*/
func Gen(dir string, generator []Generator, isConcurrent bool, logger *logrus.Entry) {
	logger = logger.WithField("state", "input_gen")
	var ret []string

	for k, s := range generator {
		id := k + 1
		size := s.Count
		name := filepath.Base(s.Name)
		outdir := filepath.Join(dir, "temp", fmt.Sprintf("gen_%d", id))
		CreateDir(outdir, logger)
		info := ProgInfo{Src: s.Name, Out: filepath.Join(outdir, fmt.Sprintf("generator%d.out", id))}
		if gen(dir, name, id, size, info, isConcurrent, logger) {
			ret = append(ret, s.Name)
		}
	}

	logger.WithFields(logrus.Fields{
		"type":  "result",
		"value": ret}).Infof("Input Generation finished")
}