func (fs *FsNotify) eventHandler(ctx polochon.FsNotifierCtx, log *logrus.Entry) { // Notify the waitgroup ctx.Wg.Add(1) defer ctx.Wg.Done() // Close the watcher when done defer fs.watcher.Close() for { select { case <-ctx.Done: log.Debug("fsnotify is done watching") return case ev := <-fs.watcher.Events: if ev.Op != fsnotify.Create && ev.Op != fsnotify.Chmod { continue } // Wait for the delay time before sending an event. // Transmission creates the folder and move the files afterwards. // We need to wait for the file to be moved in before sending the // event. Delay is the estimated time to wait. go func() { time.Sleep(DELAY) ctx.Event <- ev.Name }() case err := <-fs.watcher.Errors: log.Error(err) } } }
// Run runs the safeguard func (s *Safeguard) Run(log *logrus.Entry) error { s.wg.Add(1) defer s.wg.Done() log = log.WithField("module", "safeguard") log.Debug("safeguard started") for { select { case <-s.done: log.Debug("safeguard stopped") return nil case <-s.event: // Increase the event count s.count++ if s.count >= MaxEventCount { ctx := errors.Context{ "current_count": s.count, "max_count": MaxEventCount, } return errors.New("got %d safeguard events in less than %s", s.count, MaxEventDelay).Fatal().AddContext(ctx) } case <-time.After(MaxEventDelay): // Reset the panic count is there was not panic during the // MaxPanicDelay s.count = 0 } } }
func (c *Cleaner) cleanDoneVideos(log *logrus.Entry) { list, err := c.config.Downloader.Client.List() if err != nil { log.Errorf("error while getting torrent list: %q", err) return } for _, t := range list { torrentInfos := t.Infos() log = log.WithField("torrent_name", torrentInfos.Name) // Check if the file is ready to be cleaned isReady := c.isReadyToBeCleaned(t, log) if !isReady { log.Debug("torrent is not ready to be cleaned") continue } // We remove the torrent log.Debugf("removing torrent") err := c.config.Downloader.Client.Remove(t) if err != nil { log.Errorf("got error when removing torrent : %q", err) continue } log.Debug("removing files") if err = c.clean(t, log); err != nil { log.Errorf("failed to clean torrent files: %q", err) continue } } }
func (d *Downloader) downloadMissingMovies(wl *polochon.Wishlist, log *logrus.Entry) { log = log.WithField("function", "download_movies") for _, wantedMovie := range wl.Movies { ok, err := d.library.HasMovie(wantedMovie.ImdbID) if err != nil { log.Error(err) continue } if ok { log.Debugf("movie %q already in the video store", wantedMovie.ImdbID) continue } m := polochon.NewMovie(d.config.Movie) m.ImdbID = wantedMovie.ImdbID log = log.WithField("imdb_id", m.ImdbID) if err := m.GetDetails(log); err != nil { errors.LogErrors(log, err) if errors.IsFatal(err) { continue } } log = log.WithField("title", m.Title) if err := m.GetTorrents(log); err != nil && err != polochon.ErrMovieTorrentNotFound { errors.LogErrors(log, err) if errors.IsFatal(err) { continue } } // Keep the torrent URL var torrentURL string quality_loop: for _, q := range wantedMovie.Qualities { for _, t := range m.Torrents { if t.Quality == q { torrentURL = t.URL break quality_loop } } } if torrentURL == "" { log.Debug("no torrent found") continue } if err := d.config.Downloader.Client.Download(torrentURL, log); err != nil { log.Error(err) continue } } }
func handleConnection(conn net.Conn, jobd *jobserver.JobServer, cbor *codec.CborHandle, reqLogger *logrus.Logger) { defer conn.Close() var reqLog, errLog *logrus.Entry fields := logrus.Fields{ "remote": conn.RemoteAddr(), } errLog = logrus.WithFields(fields) if reqLogger != nil { reqLog = reqLogger.WithFields(fields) } jobdv := reflect.ValueOf(jobd) reader := bufio.NewReader(conn) decoder := codec.NewDecoder(reader, cbor) writer := bufio.NewWriter(conn) encoder := codec.NewEncoder(writer, cbor) for { var request cborrpc.Request err := decoder.Decode(&request) if err == io.EOF { if reqLog != nil { reqLog.Debug("Connection closed") } return } else if err != nil { errLog.WithError(err).Error("Error reading message") return } if reqLog != nil { reqLog.WithFields(logrus.Fields{ "id": request.ID, "method": request.Method, }).Debug("Request") } response := doRequest(jobdv, request) if reqLog != nil { entry := reqLog.WithField("id", response.ID) if response.Error != "" { entry = entry.WithField("error", response.Error) } entry.Debug("Response") } err = encoder.Encode(response) if err != nil { errLog.WithError(err).Error("Error encoding response") return } err = writer.Flush() if err != nil { errLog.WithError(err).Error("Error writing response") return } } }
func (a *Access) InfoForGood(entry *logrus.Entry, req *http.Request) { if host := RemoteHost(req); !a.Seen(host) { entry.Data["comment"] = fmt.Sprintf( ";last info-logged successful request from %s", host) entry.Info("") } else { entry.Debug("") } }
// stopApps stops all the sub apps func (a *App) stopApps(log *logrus.Entry) { log.Debug("stopping the sub apps") for _, subApp := range a.subApps { log.Debugf("stopping sub app %q", subApp.Name()) subApp.Stop(log) } a.wg.Wait() log.Debug("sub apps stopped gracefully") }
func NewResolver(conn *tls.Conn, endpoint string, logger *logrus.Entry) (*Resolver, error) { certs := conn.ConnectionState().PeerCertificates chain := []ResolverPair{} for _, cert := range certs { cn := cert.Subject.CommonName if len(cert.Subject.OrganizationalUnit) > 0 && cn != "" { ou := cert.Subject.OrganizationalUnit[0] chain = append(chain, ResolverPair{cn, ou}) } } if len(chain) == 0 { return nil, errors.New("Cannot found CommonName or OrganizationalUnit in peer certificate") } userId := chain[0].userId role := chain[0].role endpoint = strings.Replace(endpoint, ":userId", userId, 1) endpoint = strings.Replace(endpoint, ":role", role, 1) logger.Debug("Resolve using ", endpoint) response, err := http.Get(endpoint) if err != nil { return nil, err } defer response.Body.Close() content, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } res := &Resolver{} err = json.Unmarshal(content, res) if err != nil { return nil, err } addrAndPort := strings.Join([]string{res.RawAddr, "2376"}, ":") addr, err := net.ResolveTCPAddr("tcp", addrAndPort) if err != nil { return nil, err } res.Addr = addr return res, nil }
func (c *Cleaner) cleaner(log *logrus.Entry) { for { select { case <-c.event: log.Debug("cleaner event") c.cleanDoneVideos(log) case <-c.Done: log.Debug("cleaner done handling events") return } } }
func (d *Downloader) downloader(log *logrus.Entry) { for { select { case <-d.event: log.Debug("downloader event") d.downloadMissingVideos(log) case <-d.Done: log.Debug("downloader done handling events") return } } }
func (c *Cleaner) ticker(log *logrus.Entry) { tick := time.Tick(c.config.Downloader.Cleaner.Timer) for { select { case <-tick: log.Debug("cleaner timer triggered") c.event <- struct{}{} case <-c.Done: log.Debug("cleaner timer stopped") return } } }
func (d *Downloader) ticker(log *logrus.Entry) { tick := time.Tick(d.config.Downloader.Timer) for { select { case <-tick: log.Debug("downloader timer triggered") d.event <- struct{}{} case <-d.Done: log.Debug("downloader timer stopped") return } } }
// startFsNotifier starts the FsNotifier func (o *Organizer) startFsNotifier(log *logrus.Entry) error { ctx := polochon.FsNotifierCtx{ Event: o.event, Done: o.Done, Wg: &o.Wg, } // Send a notification to organize the whole folder on app start watcherPath := o.config.Watcher.Dir ctx.Event <- watcherPath // Launch the FsNotifier if err := o.config.Watcher.FsNotifier.Watch(watcherPath, ctx, log); err != nil { return err } var err error o.Wg.Add(1) go func() { defer func() { o.Wg.Done() if r := recover(); r != nil { err = errors.New("panic recovered").Fatal().AddContext(errors.Context{ "sub_app": AppName, }) o.Stop(log) } }() for { select { case file := <-ctx.Event: log.WithField("event", file).Debugf("got an event") if err := o.organize(file, log); err != nil { log.Errorf("failed to organize file: %q", err) } case <-o.Done: log.Debug("organizer done handling events") return } } }() o.Wg.Wait() return err }
// Run starts the downloader func (o *Organizer) Run(log *logrus.Entry) error { // Create the channels o.event = make(chan string, 1) // Init the app o.InitStart(log) log = log.WithField("app", AppName) defer log.Debug("organizer stopped") // Start the file system notifier if err := o.startFsNotifier(log); err != nil { return err } return nil }
func (c *Cleaner) cleanDirectory(torrent *polochon.DownloadableInfos, log *logrus.Entry) error { if len(torrent.FilePaths) == 0 { return fmt.Errorf("no torrent files to clean") } // Get the path of one of the file to guess the directory that needs to be // deleted torrentFilePath := torrent.FilePaths[0] // Get the full path of the file filePath := filepath.Join(c.config.Watcher.Dir, torrentFilePath) // Get the directory of the file directoryPath := filepath.Dir(filePath) // Ensure the path is clean directoryPath = filepath.Clean(directoryPath) // We don't want to clean the DownloadDir if directoryPath == c.config.Downloader.DownloadDir { log.Debug("in the watching folder, no need to clean") return nil } // Get relative path of the directory to clean relDir, err := filepath.Rel(c.config.Downloader.DownloadDir, directoryPath) if err != nil { return err } // Get the first part of the directory to clean for filepath.Dir(relDir) != "." { relDir = filepath.Dir(relDir) log.Debugf("going higher : %s", relDir) } // Get the full path directoryToClean := filepath.Join(c.config.Downloader.DownloadDir, relDir) log.Debug("try to clean and delete") ok, err := IsEmpty(directoryToClean) if err != nil { log.Warnf("got error checking if directory is empty : %q", err) return err } if !ok { log.Debug("directory is not empty") return nil } log.Debug("everything is ready to delete the dir") // Delete the directory if err = c.deleteDirectory(directoryToClean, log); err != nil { return err } return nil }
// RemoveContainer removes a container func RemoveContainer(logger *log.Entry, client client.DockerClient, containerID string, expectContainer bool) { logger.Debug("Removing container") err := client.RemoveContainer(docker.RemoveContainerOptions{ ID: containerID, RemoveVolumes: true, }) switch err.(type) { case *docker.NoSuchContainer: if !expectContainer { return } case nil: return } logger.WithFields(log.Fields{"container": containerID}).Warnf( "Failed to remove container: %s", err) }
func (r RouteTableManagerEC2) checkRemoteHealthCheck(contextLogger *log.Entry, route *ec2.Route, rs ManageRoutesSpec) bool { contextLogger = contextLogger.WithFields(log.Fields{ "remote_healthcheck": rs.RemoteHealthcheckName, "current_eni": *(route.NetworkInterfaceId), }) contextLogger.Info("Has remote healthcheck ") if ip, ok := eniToIP[*route.NetworkInterfaceId]; ok { contextLogger = contextLogger.WithFields(log.Fields{"current_ip": ip}) if hc, ok := rs.remotehealthchecks[ip]; ok { contextLogger = contextLogger.WithFields(log.Fields{ "healthcheck_healthy": hc.IsHealthy(), "healthcheck_ready": hc.CanPassYet(), }) contextLogger.Info("Has remote healthcheck instance") if hc.CanPassYet() { if hc.IsHealthy() { contextLogger.Debug("Not replacing route, as current route is healthy") return false } else { contextLogger.Debug("Replacing route as remote healthcheck is unhealthy") } } else { contextLogger.Debug("Not replacing route as remote healthcheck cannot pass yet") return false } } else { contextLogger.Error("Cannot find healthcheck") return false } } else { contextLogger.Error("Cannot find ip for ENI") return false } return true }
// parse an 'app' tag of request and generate a corresponding 'app' tag of response func handleApiApp(logContext *logrus.Entry, localUrl string, appRequest, appResponse *omaha.App) { logContext = logContext.WithFields(logrus.Fields{ "machineId": appRequest.MachineID, }) if appRequest.Id != coreOSAppID { appResponse.Status = "error-unknownApplication" } else { appResponse.Status = "ok" } // <UpdateCheck> tag if appRequest.UpdateCheck != nil { logContext.Debug("Handling UpdateCheck") ucResp := appResponse.AddUpdateCheck() appVersion, err := parseVersionString(appRequest.Version) if err != nil { logContext.Errorf("Could not parse client's version string: %v", err.Error()) ucResp.Status = "error-invalidVersionString" } else { handleApiUpdateCheck(logContext, localUrl, appVersion, appRequest.Track, appRequest.UpdateCheck, ucResp) } } // <ping> tag if appRequest.Ping != nil { // TODO register info from the ping // response is always "ok" according to the specs responsePing := appResponse.AddPing() responsePing.Status = "ok" } // <Event> tag handleApiEvents(logContext, appRequest.MachineID, appRequest.Events) }
// searchSubtitles will search via hash, then filename, then info and return the best subtitle func (osp *osProxy) searchSubtitles(v interface{}, filePath string, log *logrus.Entry) (*openSubtitle, error) { // Look for subtitles with the hash sub, err := osp.checkConnAndExec(osp.searchSubtitlesByHash, v, filePath, log) if err != nil { log.Warnf("Got error looking for subtitle by hash : %q", err) } if sub != nil { log.Debug("We got the subtitle by hash") return &openSubtitle{os: sub, client: osp.client}, nil } log.Debug("Nothing in the result, need to check again with filename") // Look for subtitles with the filename sub, err = osp.checkConnAndExec(osp.searchSubtitlesByFilename, v, filePath, log) if err != nil { log.Warnf("Got error looking for subtitle by filename : %q", err) } if sub != nil { log.Debug("We got the subtitle by filename") return &openSubtitle{os: sub, client: osp.client}, nil } log.Debug("Still no good, need to check again with imdbID") // Look for subtitles with the title and episode and season or by imdbID sub, err = osp.checkConnAndExec(osp.searchSubtitlesByInfos, v, filePath, log) if err != nil { log.Warnf("Got error looking for subtitle by infos : %q", err) } if sub != nil { return &openSubtitle{os: sub, client: osp.client}, nil } return nil, polochon.ErrNoSubtitleFound }
// Run starts the downloader func (d *Downloader) Run(log *logrus.Entry) error { log = log.WithField("app", AppName) // Init the app d.InitStart(log) log.Debug("downloader started") // Lauch the downloader at startup log.Debug("initial downloader launch") d.event = make(chan struct{}, 1) d.event <- struct{}{} // Start the ticker d.Wg.Add(1) go func() { defer d.Wg.Done() d.ticker(log) }() // Start the downloader var err error d.Wg.Add(1) go func() { defer func() { if r := recover(); r != nil { err = errors.New("panic recovered").Fatal().AddContext(errors.Context{ "sub_app": AppName, }) d.Stop(log) } d.Wg.Done() }() d.downloader(log) }() defer log.Debug("downloader stopped") d.Wg.Wait() return err }
// Run starts the cleaner func (c *Cleaner) Run(log *logrus.Entry) error { log = log.WithField("app", AppName) // Init the app c.InitStart(log) c.event = make(chan struct{}, 1) log.Debug("cleaner started") log.Debug("initial cleaner launch") c.event <- struct{}{} // Start the ticker c.Wg.Add(1) go func() { defer c.Wg.Done() c.ticker(log) }() // Start the cleaner var err error c.Wg.Add(1) go func() { defer func() { if r := recover(); r != nil { err = errors.New("panic recovered").Fatal().AddContext(errors.Context{ "sub_app": AppName, }) c.Stop(log) } c.Wg.Done() }() c.cleaner(log) }() defer log.Debug("cleaner stopped") c.Wg.Wait() return err }
// AddMovie adds a movie to the store func (l *Library) AddMovie(movie *polochon.Movie, log *logrus.Entry) error { if movie.Path == "" { return ErrMissingMovieFilePath } // Check if the movie is already in the library ok, err := l.HasMovie(movie.ImdbID) if err != nil { return err } if ok { // Get the old movie path from the index oldMovie, err := l.GetMovie(movie.ImdbID) if err != nil { return err } // Delete it if err := l.DeleteMovie(oldMovie, log); err != nil { return err } } storePath := l.getMovieDir(movie) // If the movie already in the right dir there is nothing to do if path.Dir(movie.Path) == storePath { log.Debug("Movie already in the destination folder") return nil } // Remove movie dir if it exisits if ok := exists(storePath); ok { log.Debug("Movie folder exists, remove it") if err := os.RemoveAll(storePath); err != nil { return err } } // Create the folder if err := os.Mkdir(storePath, os.ModePerm); err != nil { return err } // Move the movie into the folder newPath := filepath.Join(storePath, path.Base(movie.Path)) // Save the old path oldPath := movie.Path log.Debugf("Old path: %q, new path %q", movie.Path, newPath) if err := os.Rename(movie.Path, newPath); err != nil { return err } // Set the new movie path movie.Path = newPath // Create a symlink between the new and the old location if err := os.Symlink(movie.Path, oldPath); err != nil { log.Warnf("Error while making symlink between %s and %s : %+v", oldPath, movie.Path, err) } // Write NFO into the file if err := writeNFOFile(movie.NfoPath(), movie); err != nil { return err } // At this point the video is stored if err := l.movieIndex.Add(movie); err != nil { return err } if movie.Fanart == "" || movie.Thumb == "" { return ErrMissingMovieImageURL } // Download images for _, img := range []struct { url string savePath string }{ { url: movie.Fanart, savePath: movie.MovieFanartPath(), }, { url: movie.Thumb, savePath: movie.MovieThumbPath(), }, } { if err := download(img.url, img.savePath); err != nil { return err } } return nil }
func (d *Downloader) downloadMissingShows(wl *polochon.Wishlist, log *logrus.Entry) { log = log.WithField("function", "download_shows") for _, wishedShow := range wl.Shows { s := polochon.NewShow(d.config.Show) s.ImdbID = wishedShow.ImdbID log = log.WithField("imdb_id", s.ImdbID) if err := s.GetDetails(log); err != nil { errors.LogErrors(log, err) if errors.IsFatal(err) { continue } } calendar, err := s.GetCalendar(log) if err != nil { errors.LogErrors(log, err) if errors.IsFatal(err) { continue } } for _, calEpisode := range calendar.Episodes { // Check if the episode should be downloaded if calEpisode.IsOlder(wishedShow) { continue } // Check if the episode has already been downloaded ok, err := d.library.HasShowEpisode(wishedShow.ImdbID, calEpisode.Season, calEpisode.Episode) if err != nil { log.Error(err) continue } if ok { continue } // Setup the episode e := polochon.NewShowEpisode(d.config.Show) e.ShowImdbID = wishedShow.ImdbID e.ShowTitle = s.Title e.Season = calEpisode.Season e.Episode = calEpisode.Episode log = log.WithFields(logrus.Fields{ "show_imdb_id": e.ShowImdbID, "show_title": e.ShowTitle, "season": e.Season, "episode": e.Episode, }) if err := e.GetTorrents(log); err != nil && err != polochon.ErrShowEpisodeTorrentNotFound { errors.LogErrors(log, err) if errors.IsFatal(err) { continue } } // Keep the torrent URL var torrentURL string quality_loop: for _, q := range wishedShow.Qualities { for _, t := range e.Torrents { if t.Quality == q { torrentURL = t.URL break quality_loop } } } if torrentURL == "" { log.Debug("no torrent found") continue } if err := d.config.Downloader.Client.Download(torrentURL, log); err != nil { log.Error(err) continue } } } }
// OrganizeFile stores the videos in the video library func (o *Organizer) organizeFile(filePath string, log *logrus.Entry) error { log = log.WithField("file_path", filePath) log.Debug("organize file") // Create a file file := polochon.NewFileWithConfig(filePath, o.config.File) // Check if file really exists if !file.Exists() { log.Warning("the file has been removed") return nil } // Check if file is a video if !file.IsVideo() { log.Debug("the file is not a video") return nil } // Check if file is ignored if file.IsIgnored() { log.Debug("the file is ignored") return nil } // Check if file is symlink if file.IsSymlink() { log.Debug("the file is a symlink") return nil } // Check if file is ignored if file.IsExcluded() { log.Debug("the file is excluded") return file.Ignore() } // Guess the video inforamtion video, err := file.Guess(o.config.Movie, o.config.Show, log) if err != nil { errors.LogErrors(log, err) return file.Ignore() } // Get video details if err := video.GetDetails(log); err != nil { errors.LogErrors(log, err) if errors.IsFatal(err) { return file.Ignore() } } // Store the video if err := o.library.Add(video, log); err != nil { errors.LogErrors(log, err) return file.Ignore() } // Get subtitle if err := video.GetSubtitle(log); err != nil { errors.LogErrors(log, err) } // Notify o.Notify(video, log) return nil }
// AddShowEpisode adds an episode to the store func (l *Library) AddShowEpisode(ep *polochon.ShowEpisode, log *logrus.Entry) error { if ep.Path == "" { return ErrMissingShowEpisodeFilePath } ok, err := l.HasShowEpisode(ep.ShowImdbID, ep.Season, ep.Episode) if err != nil { return err } if ok { // Get the old episode from the index oldEpisode, err := l.GetEpisode(ep.ShowImdbID, ep.Season, ep.Episode) if err != nil { return err } if err := l.DeleteShowEpisode(oldEpisode, log); err != nil { return err } } // Add the show if err := l.addShow(ep, log); err != nil { return err } // Create show season dir if necessary seasonDir := l.getSeasonDir(ep) if !exists(seasonDir) { if err := os.Mkdir(seasonDir, os.ModePerm); err != nil { return err } } // Move the file // If the show episode already in the right dir there is nothing to do if path.Dir(ep.Path) == seasonDir { log.Debug("show episode already in the destination folder") return nil } // Save the old path oldPath := ep.Path // Move the episode into the folder newPath := filepath.Join(seasonDir, path.Base(ep.Path)) log.Debugf("Moving episode to folder Old path: %q, New path: %q", ep.Path, newPath) if err := os.Rename(ep.Path, newPath); err != nil { return err } // Set the new movie path ep.Path = newPath // Create a symlink between the new and the old location if err := os.Symlink(ep.Path, oldPath); err != nil { log.Warnf("Error while making symlink between %s and %s : %+v", oldPath, ep.Path, err) } // Create show NFO if necessary if err := writeNFOFile(ep.NfoPath(), ep); err != nil { return err } return l.showIndex.Add(ep) }
// startSubApps launches all the sub app func (a *App) startSubApps(log *logrus.Entry) { log.Debug("starting the sub apps") for _, subApp := range a.subApps { a.subAppStart(subApp, log) } }