// Start statrs a sub app in its own goroutine
func (a *App) subAppStart(app subapp.App, log *logrus.Entry) {
log.Debugf("starting sub app %q", app.Name())
a.wg.Add(1)
go func() {
defer a.wg.Done()
if err := app.Run(log); err != nil {
// Check the error type, if it comes from a panic recovery
// reload the app
switch e := err.(type) {
case *errors.Error:
errors.LogErrors(log.WithField("app", app.Name()), e)
// Notify the safeguard of the error
a.safeguard.Event()
// Write to the reload channel in a goroutine to prevent deadlocks
go func() {
a.reload <- app
}()
// Only log the error
default:
log.Error(err)
go a.Stop(log)
}
}
}()
log.Debugf("sub app %q started", app.Name())
}
// Stop stops the app
func (b *Base) Stop(log *logrus.Entry) {
log.WithField("app", b.AppName).Debug("stopping app asynchronously")
if b.AppStatus == Started {
close(b.Done)
b.AppStatus = Stopped
}
}
func (c *Cleaner) cleanDoneVideos(log *logrus.Entry) {
list, err := c.config.Downloader.Client.List()
if err != nil {
log.Errorf("error while getting torrent list: %q", err)
return
}
for _, t := range list {
torrentInfos := t.Infos()
log = log.WithField("torrent_name", torrentInfos.Name)
// Check if the file is ready to be cleaned
isReady := c.isReadyToBeCleaned(t, log)
if !isReady {
log.Debug("torrent is not ready to be cleaned")
continue
}
// We remove the torrent
log.Debugf("removing torrent")
err := c.config.Downloader.Client.Remove(t)
if err != nil {
log.Errorf("got error when removing torrent : %q", err)
continue
}
log.Debug("removing files")
if err = c.clean(t, log); err != nil {
log.Errorf("failed to clean torrent files: %q", err)
continue
}
}
}
// GetDetails helps getting infos for a movie
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetDetails(log *logrus.Entry) error {
c := errors.NewCollector()
if len(m.Detailers) == 0 {
c.Push(errors.Wrap("No detailer available").Fatal())
return c
}
var done bool
for _, d := range m.Detailers {
detailerLog := log.WithField("detailer", d.Name())
err := d.GetDetails(m, detailerLog)
if err == nil {
done = true
break
}
c.Push(errors.Wrap(err).Ctx("Detailer", d.Name()))
}
if !done {
c.Push(errors.Wrap("All detailers failed").Fatal())
}
if c.HasErrors() {
return c
}
return nil
}
// GetSubtitle implements the subtitle interface
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetSubtitle(log *logrus.Entry) error {
c := errors.NewCollector()
var subtitle Subtitle
for _, subtitler := range m.Subtitlers {
var err error
subtitlerLog := log.WithField("subtitler", subtitler.Name())
subtitle, err = subtitler.GetMovieSubtitle(m, subtitlerLog)
if err == nil {
break
}
c.Push(errors.Wrap(err).Ctx("Subtitler", subtitler.Name()))
}
if subtitle != nil {
file, err := os.Create(m.File.SubtitlePath())
if err != nil {
c.Push(errors.Wrap(err).Fatal())
return c
}
defer file.Close()
defer subtitle.Close()
if _, err := io.Copy(file, subtitle); err != nil {
c.Push(errors.Wrap(err).Fatal())
return c
}
}
if c.HasErrors() {
return c
}
return nil
}
// Run runs the safeguard
func (s *Safeguard) Run(log *logrus.Entry) error {
s.wg.Add(1)
defer s.wg.Done()
log = log.WithField("module", "safeguard")
log.Debug("safeguard started")
for {
select {
case <-s.done:
log.Debug("safeguard stopped")
return nil
case <-s.event:
// Increase the event count
s.count++
if s.count >= MaxEventCount {
ctx := errors.Context{
"current_count": s.count,
"max_count": MaxEventCount,
}
return errors.New("got %d safeguard events in less than %s",
s.count, MaxEventDelay).Fatal().AddContext(ctx)
}
case <-time.After(MaxEventDelay):
// Reset the panic count is there was not panic during the
// MaxPanicDelay
s.count = 0
}
}
}
// Watch implements the modules fsNotifier interface
func (fs *FsNotify) Watch(watchPath string, ctx polochon.FsNotifierCtx, log *logrus.Entry) error {
// Create a new watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
fs.watcher = watcher
// Ensure that the watch path exists
if _, err := os.Stat(watchPath); os.IsNotExist(err) {
return err
}
log = log.WithField("module", moduleName)
// Run the event handler
go fs.eventHandler(ctx, log)
// Watch the path
if err := fs.watcher.Add(watchPath); err != nil {
return err
}
return nil
}
func reportCompleted(client client.Queue, task *TaskRun, log *logrus.Entry) *updateError {
_, err := client.ReportCompleted(task.TaskID, strconv.Itoa(task.RunID))
if err != nil {
log.WithField("error", err).Warn("Not able to report successful completion for task.")
return &updateError{err: err.Error()}
}
return nil
}
// Notify sends video to the notifiers
func (o *Organizer) Notify(v polochon.Video, log *logrus.Entry) {
log = log.WithField("function", "notify")
for _, n := range o.config.Notifiers {
if err := n.Notify(v, log); err != nil {
log.Warnf("failed to send a notification from notifier: %q: %q", n.Name(), err)
}
}
}
// DecorateRuntimeContext appends line, file and function context to the logger
func DecorateRuntimeContext(logger *logrus.Entry) *logrus.Entry {
if pc, file, line, ok := runtime.Caller(1); ok {
fName := runtime.FuncForPC(pc).Name()
return logger.WithField("file", file).WithField("line", line).WithField("func", fName)
} else {
return logger
}
}
func (k *Kickass) listMoviesByUser(movies map[string]*polochon.Movie, user string, log *logrus.Entry) error {
query := &kickass.Query{
User: user,
OrderBy: "seeders",
Order: "desc",
Category: string(MoviesCategory),
}
log = log.WithField("explore_user", user)
torrents, err := k.client.ListByUser(query)
if err != nil {
return err
}
for _, t := range torrents {
torrentStr := torrentGuessitStr(t)
guess, err := guessit.Guess(torrentStr)
if err != nil {
continue
}
// Get the torrent quality
torrentQuality := polochon.Quality(guess.Quality)
if !torrentQuality.IsAllowed() {
log.Infof("kickass: unhandled quality: %q", torrentQuality)
continue
}
// Get the movie if its already in the map
m, ok := movies[guess.Title]
if !ok {
// Create a new movie
m = polochon.NewMovie(polochon.MovieConfig{})
m.Title = guess.Title
if guess.Year != 0 {
m.Year = guess.Year
}
}
log.WithFields(logrus.Fields{
"torrent_quality": guess.Quality,
"movie_title": guess.Title,
}).Debug("Adding torrent to the list")
m.Torrents = append(m.Torrents, polochon.Torrent{
Quality: torrentQuality,
URL: t.MagnetURL,
Seeders: t.Seed,
Leechers: t.Leech,
Source: moduleName,
UploadUser: user,
})
movies[m.Title] = m
}
return nil
}
// Run starts the server
func (s *Server) Run(log *logrus.Entry) error {
s.log = log.WithField("app", AppName)
// Init the app
s.InitStart(log)
s.gracefulServer = manners.NewWithServer(s.httpServer(s.log))
return s.gracefulServer.ListenAndServe()
}
func reportException(client client.Queue, task *TaskRun, reason runtime.ExceptionReason, log *logrus.Entry) *updateError {
payload := queue.TaskExceptionRequest{Reason: string(reason)}
_, err := client.ReportException(task.TaskID, strconv.Itoa(task.RunID), &payload)
if err != nil {
log.WithField("error", err).Warn("Not able to report exception for task")
return &updateError{err: err.Error()}
}
return nil
}
func handleConnection(conn net.Conn, jobd *jobserver.JobServer, cbor *codec.CborHandle, reqLogger *logrus.Logger) {
defer conn.Close()
var reqLog, errLog *logrus.Entry
fields := logrus.Fields{
"remote": conn.RemoteAddr(),
}
errLog = logrus.WithFields(fields)
if reqLogger != nil {
reqLog = reqLogger.WithFields(fields)
}
jobdv := reflect.ValueOf(jobd)
reader := bufio.NewReader(conn)
decoder := codec.NewDecoder(reader, cbor)
writer := bufio.NewWriter(conn)
encoder := codec.NewEncoder(writer, cbor)
for {
var request cborrpc.Request
err := decoder.Decode(&request)
if err == io.EOF {
if reqLog != nil {
reqLog.Debug("Connection closed")
}
return
} else if err != nil {
errLog.WithError(err).Error("Error reading message")
return
}
if reqLog != nil {
reqLog.WithFields(logrus.Fields{
"id": request.ID,
"method": request.Method,
}).Debug("Request")
}
response := doRequest(jobdv, request)
if reqLog != nil {
entry := reqLog.WithField("id", response.ID)
if response.Error != "" {
entry = entry.WithField("error", response.Error)
}
entry.Debug("Response")
}
err = encoder.Encode(response)
if err != nil {
errLog.WithError(err).Error("Error encoding response")
return
}
err = writer.Flush()
if err != nil {
errLog.WithError(err).Error("Error writing response")
return
}
}
}
func (k *Kickass) searchUser(s Searcher, log *logrus.Entry, user string) ([]polochon.Torrent, error) {
query := &kickass.Query{
User: user,
OrderBy: "seeders",
Order: "desc",
Category: string(s.category()),
Search: s.searchStr(),
}
log = log.WithField("search_user", user)
torrents, err := k.client.Search(query)
if err != nil {
return nil, err
}
result := []polochon.Torrent{}
for _, t := range torrents {
torrentStr := torrentGuessitStr(t)
guess, err := guessit.Guess(torrentStr)
if err != nil {
continue
}
if !s.isValidGuess(guess, log) {
continue
}
// Default quality
if s.category() == ShowsCategory && guess.Quality == "" {
guess.Quality = string(polochon.Quality480p)
}
// Get the torrent quality
torrentQuality := polochon.Quality(guess.Quality)
if !torrentQuality.IsAllowed() {
log.Infof("kickass: unhandled quality: %q", torrentQuality)
continue
}
log.WithFields(logrus.Fields{
"torrent_quality": guess.Quality,
"torrent_name": torrentStr,
}).Debug("Adding torrent to the list")
// Add the torrent
result = append(result, polochon.Torrent{
Quality: torrentQuality,
URL: t.MagnetURL,
Seeders: t.Seed,
Leechers: t.Leech,
Source: moduleName,
UploadUser: user,
})
}
return result, nil
}
// BlockingStop stops the app a nd waits for it to be done
func (b *Base) BlockingStop(log *logrus.Entry) {
log.WithField("app", b.AppName).Debug("stopping app synchronously")
// Send a signal to stop the app
b.Stop(log)
// Wait for all the goroutines to be done
b.Wg.Wait()
log.WithField("app_name", b.AppName).Info("app stopped")
}
// validateSignature validates the request payload with the user provided key using the
// HMAC algo
func validateSignature(requestLog *logrus.Entry, r *http.Request, key string, payload []byte) bool {
// if we don't have a secret to validate then just return true
// because the user does not care about security
if key == "" {
return true
}
actual := r.Header.Get("X-Hub-Signature")
expected, err := getExpectedSignature([]byte(key), payload)
if err != nil {
requestLog.WithField("gh_signature", actual).WithField("error", err).Error("parse expected signature")
return false
}
return hmac.Equal([]byte(expected), []byte(actual))
}
// startFsNotifier starts the FsNotifier
func (o *Organizer) startFsNotifier(log *logrus.Entry) error {
ctx := polochon.FsNotifierCtx{
Event: o.event,
Done: o.Done,
Wg: &o.Wg,
}
// Send a notification to organize the whole folder on app start
watcherPath := o.config.Watcher.Dir
ctx.Event <- watcherPath
// Launch the FsNotifier
if err := o.config.Watcher.FsNotifier.Watch(watcherPath, ctx, log); err != nil {
return err
}
var err error
o.Wg.Add(1)
go func() {
defer func() {
o.Wg.Done()
if r := recover(); r != nil {
err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
"sub_app": AppName,
})
o.Stop(log)
}
}()
for {
select {
case file := <-ctx.Event:
log.WithField("event", file).Debugf("got an event")
if err := o.organize(file, log); err != nil {
log.Errorf("failed to organize file: %q", err)
}
case <-o.Done:
log.Debug("organizer done handling events")
return
}
}
}()
o.Wg.Wait()
return err
}
// Run starts the downloader
func (o *Organizer) Run(log *logrus.Entry) error {
// Create the channels
o.event = make(chan string, 1)
// Init the app
o.InitStart(log)
log = log.WithField("app", AppName)
defer log.Debug("organizer stopped")
// Start the file system notifier
if err := o.startFsNotifier(log); err != nil {
return err
}
return nil
}
// GetTorrents helps getting the torrent files for a movie
// If there is an error, it will be of type *errors.Collector
func (m *Movie) GetTorrents(log *logrus.Entry) error {
c := errors.NewCollector()
for _, t := range m.Torrenters {
torrenterLog := log.WithField("torrenter", t.Name())
err := t.GetTorrents(m, torrenterLog)
if err == nil {
break
}
c.Push(errors.Wrap(err).Ctx("Torrenter", t.Name()))
}
if c.HasErrors() {
return c
}
return nil
}
/*
Load problem from directory
*/
func Load(path string, logger *logrus.Entry) (Problem, error) {
logger = logger.WithField("state", "load_problem")
path = filepath.Join(path, "problem.toml")
var problem Problem
_, err := toml.DecodeFile(path, &problem)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": IE.String(),
"info": err.Error()}).Error("Loading problem: Failed")
return problem, err
}
logger.WithFields(logrus.Fields{
"type": "info",
"info": problem}).Infof("Loaded Problem: %s", problem.Name)
return problem, nil
}
func new(host string, log *logrus.Entry) *guestTools {
got := got.New()
got.Client = &http.Client{Timeout: 5 * time.Second}
got.MaxSize = 10 * 1024 * 1024
got.Log = log.WithField("guest-tools", "http-got")
got.Retries = 15
got.BackOff = backOff
ctx, cancel := context.WithCancel(context.Background())
return &guestTools{
baseURL: "http://" + host + "/",
got: got,
log: log,
pollingCtx: ctx,
cancelPolling: cancel,
}
}
func WithValue(parent Context, key string, value interface{}) Context {
c := context.WithValue(parent, key, value)
var entry *logrus.Entry
if ctx, ok := parent.(*ctx); ok {
entry = ctx.Entry
} else {
entry = std
}
ctx := &ctx{
Context: c,
Entry: entry.WithField(key, value),
}
return ctx
}
// Create a new instance of the task manager that will be responsible for claiming,
// executing, and resolving units of work (tasks).
func newTaskManager(
config *configType, engine engines.Engine, pluginManager plugins.Plugin,
environment *runtime.Environment, log *logrus.Entry, gc *gc.GarbageCollector,
) (*Manager, error) {
queue := tcqueue.New(
&tcclient.Credentials{
ClientID: config.Credentials.ClientID,
AccessToken: config.Credentials.AccessToken,
Certificate: config.Credentials.Certificate,
},
)
if config.QueueBaseURL != "" {
queue.BaseURL = config.QueueBaseURL
}
service := &queueService{
capacity: config.Capacity,
interval: config.PollingInterval,
client: queue,
provisionerID: config.ProvisionerID,
workerGroup: config.WorkerGroup,
workerID: config.WorkerID,
workerType: config.WorkerType,
log: log.WithField("component", "Queue Service"),
expirationOffset: config.ReclaimOffset,
}
m := &Manager{
tasks: make(map[string]*TaskRun),
done: make(chan struct{}),
config: config,
engine: engine,
environment: environment,
log: log,
queue: service,
provisionerID: config.ProvisionerID,
workerGroup: config.WorkerGroup,
workerID: config.WorkerID,
gc: gc,
}
m.pluginManager = pluginManager
return m, nil
}
func (c *Cleaner) isReadyToBeCleaned(d polochon.Downloadable, log *logrus.Entry) bool {
torrent := d.Infos()
log = log.WithField("torrent_name", torrent.Name)
// First check that the torrent download is finished
if !torrent.IsFinished {
log.Debugf("torrent is not yet finished")
return false
}
// Check that the ratio is reached
if torrent.Ratio < c.config.Downloader.Cleaner.Ratio {
log.Debugf("ratio is not reached (%.02f / %.02f)", torrent.Ratio, c.config.Downloader.Cleaner.Ratio)
return false
}
return true
}
/*
Save problem to path
*/
func Save(path string, problem Problem, logger *logrus.Entry) error {
logger = logger.WithField("state", "save_problem")
out := CreateFile(filepath.Join(path, "problem.toml"), logger)
defer out.Close()
encoder := toml.NewEncoder(out)
err := encoder.Encode(problem)
if err != nil {
logger.WithFields(logrus.Fields{
"type": "error",
"error": IE,
"info": err.Error()}).Error("Encode problem.toml: Failed")
return err
}
logger.WithFields(logrus.Fields{
"type": "info",
"info": problem}).Infof("Saved problem: %s", problem.Name)
return nil
}
// GetMovieList implements the explorer interface
func (y *Yts) GetMovieList(option polochon.ExplorerOption, log *logrus.Entry) ([]*polochon.Movie, error) {
log = log.WithField("explore_category", "movies")
opt, err := translateMovieOptions(option)
if err != nil {
return nil, err
}
movieList, err := yts.GetList(1, 6, opt, yts.OrderDesc)
if err != nil {
return nil, err
}
result := []*polochon.Movie{}
for _, movie := range movieList {
// Create the movie
m := polochon.NewMovie(polochon.MovieConfig{})
m.Title = movie.Title
m.Year = movie.Year
m.ImdbID = movie.ImdbID
// Add the torrents
for _, t := range movie.Torrents {
// Get the torrent quality
torrentQuality := polochon.Quality(t.Quality)
if !torrentQuality.IsAllowed() {
log.Debugf("yts: unhandled quality: %q", torrentQuality)
continue
}
m.Torrents = append(m.Torrents, polochon.Torrent{
Quality: torrentQuality,
URL: t.URL,
Seeders: t.Seeds,
Leechers: t.Peers,
Source: moduleName,
})
}
// Append the movie
result = append(result, m)
}
return result, nil
}
// Run starts the downloader
func (d *Downloader) Run(log *logrus.Entry) error {
log = log.WithField("app", AppName)
// Init the app
d.InitStart(log)
log.Debug("downloader started")
// Lauch the downloader at startup
log.Debug("initial downloader launch")
d.event = make(chan struct{}, 1)
d.event <- struct{}{}
// Start the ticker
d.Wg.Add(1)
go func() {
defer d.Wg.Done()
d.ticker(log)
}()
// Start the downloader
var err error
d.Wg.Add(1)
go func() {
defer func() {
if r := recover(); r != nil {
err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
"sub_app": AppName,
})
d.Stop(log)
}
d.Wg.Done()
}()
d.downloader(log)
}()
defer log.Debug("downloader stopped")
d.Wg.Wait()
return err
}
// Run starts the cleaner
func (c *Cleaner) Run(log *logrus.Entry) error {
log = log.WithField("app", AppName)
// Init the app
c.InitStart(log)
c.event = make(chan struct{}, 1)
log.Debug("cleaner started")
log.Debug("initial cleaner launch")
c.event <- struct{}{}
// Start the ticker
c.Wg.Add(1)
go func() {
defer c.Wg.Done()
c.ticker(log)
}()
// Start the cleaner
var err error
c.Wg.Add(1)
go func() {
defer func() {
if r := recover(); r != nil {
err = errors.New("panic recovered").Fatal().AddContext(errors.Context{
"sub_app": AppName,
})
c.Stop(log)
}
c.Wg.Done()
}()
c.cleaner(log)
}()
defer log.Debug("cleaner stopped")
c.Wg.Wait()
return err
}
/*
Gen creates input files.
*/
func Gen(dir string, generator []Generator, isConcurrent bool, logger *logrus.Entry) {
logger = logger.WithField("state", "input_gen")
var ret []string
for k, s := range generator {
id := k + 1
size := s.Count
name := filepath.Base(s.Name)
outdir := filepath.Join(dir, "temp", fmt.Sprintf("gen_%d", id))
CreateDir(outdir, logger)
info := ProgInfo{Src: s.Name, Out: filepath.Join(outdir, fmt.Sprintf("generator%d.out", id))}
if gen(dir, name, id, size, info, isConcurrent, logger) {
ret = append(ret, s.Name)
}
}
logger.WithFields(logrus.Fields{
"type": "result",
"value": ret}).Infof("Input Generation finished")
}