// Start statrs a sub app in its own goroutine
func (a *App) subAppStart(app subapp.App, log *logrus.Entry) {
log.Debugf("starting sub app %q", app.Name())
a.wg.Add(1)
go func() {
defer a.wg.Done()
if err := app.Run(log); err != nil {
// Check the error type, if it comes from a panic recovery
// reload the app
switch e := err.(type) {
case *errors.Error:
errors.LogErrors(log.WithField("app", app.Name()), e)
// Notify the safeguard of the error
a.safeguard.Event()
// Write to the reload channel in a goroutine to prevent deadlocks
go func() {
a.reload <- app
}()
// Only log the error
default:
log.Error(err)
go a.Stop(log)
}
}
}()
log.Debugf("sub app %q started", app.Name())
}
func (c *Cleaner) cleanDoneVideos(log *logrus.Entry) {
list, err := c.config.Downloader.Client.List()
if err != nil {
log.Errorf("error while getting torrent list: %q", err)
return
}
for _, t := range list {
torrentInfos := t.Infos()
log = log.WithField("torrent_name", torrentInfos.Name)
// Check if the file is ready to be cleaned
isReady := c.isReadyToBeCleaned(t, log)
if !isReady {
log.Debug("torrent is not ready to be cleaned")
continue
}
// We remove the torrent
log.Debugf("removing torrent")
err := c.config.Downloader.Client.Remove(t)
if err != nil {
log.Errorf("got error when removing torrent : %q", err)
continue
}
log.Debug("removing files")
if err = c.clean(t, log); err != nil {
log.Errorf("failed to clean torrent files: %q", err)
continue
}
}
}
// searchByImdbID searches on tmdb based on the imdb id
func (t *TmDB) searchByImdbID(m *polochon.Movie, log *logrus.Entry) error {
// No imdb id, no search
if m.ImdbID == "" {
return ErrNoMovieImDBID
}
// ID already found
if m.TmdbID != 0 {
return nil
}
// Search on tmdb
results, err := tmdbSearchByImdbID(t.client, m.ImdbID, "imdb_id", map[string]string{})
if err != nil {
return err
}
// Check if there is any results
if len(results.MovieResults) == 0 {
log.Debugf("Failed to find movie from imdb ID %q", m.ImdbID)
return ErrNoMovieFound
}
m.TmdbID = results.MovieResults[0].ID
log.Debugf("Found movie from imdb ID %q", m.ImdbID)
return nil
}
func (d *Downloader) downloadMissingMovies(wl *polochon.Wishlist, log *logrus.Entry) {
log = log.WithField("function", "download_movies")
for _, wantedMovie := range wl.Movies {
ok, err := d.library.HasMovie(wantedMovie.ImdbID)
if err != nil {
log.Error(err)
continue
}
if ok {
log.Debugf("movie %q already in the video store", wantedMovie.ImdbID)
continue
}
m := polochon.NewMovie(d.config.Movie)
m.ImdbID = wantedMovie.ImdbID
log = log.WithField("imdb_id", m.ImdbID)
if err := m.GetDetails(log); err != nil {
errors.LogErrors(log, err)
if errors.IsFatal(err) {
continue
}
}
log = log.WithField("title", m.Title)
if err := m.GetTorrents(log); err != nil && err != polochon.ErrMovieTorrentNotFound {
errors.LogErrors(log, err)
if errors.IsFatal(err) {
continue
}
}
// Keep the torrent URL
var torrentURL string
quality_loop:
for _, q := range wantedMovie.Qualities {
for _, t := range m.Torrents {
if t.Quality == q {
torrentURL = t.URL
break quality_loop
}
}
}
if torrentURL == "" {
log.Debug("no torrent found")
continue
}
if err := d.config.Downloader.Client.Download(torrentURL, log); err != nil {
log.Error(err)
continue
}
}
}
func (c *Cleaner) cleanDirectory(torrent *polochon.DownloadableInfos, log *logrus.Entry) error {
if len(torrent.FilePaths) == 0 {
return fmt.Errorf("no torrent files to clean")
}
// Get the path of one of the file to guess the directory that needs to be
// deleted
torrentFilePath := torrent.FilePaths[0]
// Get the full path of the file
filePath := filepath.Join(c.config.Watcher.Dir, torrentFilePath)
// Get the directory of the file
directoryPath := filepath.Dir(filePath)
// Ensure the path is clean
directoryPath = filepath.Clean(directoryPath)
// We don't want to clean the DownloadDir
if directoryPath == c.config.Downloader.DownloadDir {
log.Debug("in the watching folder, no need to clean")
return nil
}
// Get relative path of the directory to clean
relDir, err := filepath.Rel(c.config.Downloader.DownloadDir, directoryPath)
if err != nil {
return err
}
// Get the first part of the directory to clean
for filepath.Dir(relDir) != "." {
relDir = filepath.Dir(relDir)
log.Debugf("going higher : %s", relDir)
}
// Get the full path
directoryToClean := filepath.Join(c.config.Downloader.DownloadDir, relDir)
log.Debug("try to clean and delete")
ok, err := IsEmpty(directoryToClean)
if err != nil {
log.Warnf("got error checking if directory is empty : %q", err)
return err
}
if !ok {
log.Debug("directory is not empty")
return nil
}
log.Debug("everything is ready to delete the dir")
// Delete the directory
if err = c.deleteDirectory(directoryToClean, log); err != nil {
return err
}
return nil
}
// stopApps stops all the sub apps
func (a *App) stopApps(log *logrus.Entry) {
log.Debug("stopping the sub apps")
for _, subApp := range a.subApps {
log.Debugf("stopping sub app %q", subApp.Name())
subApp.Stop(log)
}
a.wg.Wait()
log.Debug("sub apps stopped gracefully")
}
func newK8sClient(conf utils.NetConf, logger *log.Entry) (*kubernetes.Clientset, error) {
// Some config can be passed in a kubeconfig file
kubeconfig := conf.Kubernetes.Kubeconfig
// Config can be overridden by config passed in explicitly in the network config.
configOverrides := &clientcmd.ConfigOverrides{}
// If an API root is given, make sure we're using using the name / port rather than
// the full URL. Earlier versions of the config required the full `/api/v1/` extension,
// so split that off to ensure compatibility.
conf.Policy.K8sAPIRoot = strings.Split(conf.Policy.K8sAPIRoot, "/api/")[0]
var overridesMap = []struct {
variable *string
value string
}{
{&configOverrides.ClusterInfo.Server, conf.Policy.K8sAPIRoot},
{&configOverrides.AuthInfo.ClientCertificate, conf.Policy.K8sClientCertificate},
{&configOverrides.AuthInfo.ClientKey, conf.Policy.K8sClientKey},
{&configOverrides.ClusterInfo.CertificateAuthority, conf.Policy.K8sCertificateAuthority},
{&configOverrides.AuthInfo.Token, conf.Policy.K8sAuthToken},
}
// Using the override map above, populate any non-empty values.
for _, override := range overridesMap {
if override.value != "" {
*override.variable = override.value
}
}
// Also allow the K8sAPIRoot to appear under the "kubernetes" block in the network config.
if conf.Kubernetes.K8sAPIRoot != "" {
configOverrides.ClusterInfo.Server = conf.Kubernetes.K8sAPIRoot
}
// Use the kubernetes client code to load the kubeconfig file and combine it with the overrides.
config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig},
configOverrides).ClientConfig()
if err != nil {
return nil, err
}
logger.Debugf("Kubernetes config %v", config)
// Create the clientset
return kubernetes.NewForConfig(config)
}
// SearchByTitle searches a movie by its title. It adds the tmdb id into the
// movie struct so it can get details later
func (t *TmDB) searchByTitle(m *polochon.Movie, log *logrus.Entry) error {
// No title, no search
if m.Title == "" {
return ErrNoMovieTitle
}
// ID already found
if m.TmdbID != 0 {
return nil
}
// Add year option if given
options := map[string]string{}
if m.Year != 0 {
options["year"] = fmt.Sprintf("%d", m.Year)
}
// Search on tmdb
r, err := tmdbSearchMovie(t.client, m.Title, options)
if err != nil {
return err
}
// Check if there is any results
if len(r.Results) == 0 {
log.Debugf("Failed to find movie from imdb title %q", m.Title)
return ErrNoMovieFound
}
// Find the most accurate serie based on the levenshtein distance
var movieShort tmdb.MovieShort
minDistance := 100
for _, result := range r.Results {
d := levenshtein.Distance(m.Title, result.Title)
if d < minDistance {
minDistance = d
movieShort = result
}
}
m.TmdbID = movieShort.ID
log.Debugf("Found movie from title %q", m.Title)
return nil
}
// getGoodMovieSubtitles will retrieve only the movies with the same imdbId
func (osp *osProxy) getGoodMovieSubtitles(m polochon.Movie, subs osdb.Subtitles, log *logrus.Entry) []osdb.Subtitle {
var goodSubs []osdb.Subtitle
for _, sub := range subs {
// Need to check that it's the good subtitle
imdbID := fmt.Sprintf("tt%07s", sub.IDMovieImdb)
if imdbID == m.ImdbID {
goodSubs = append(goodSubs, sub)
} else {
continue
}
}
if len(goodSubs) > 0 {
log.Debugf("Got %d subtitles", len(goodSubs))
}
return goodSubs
}
func (c *Cleaner) isReadyToBeCleaned(d polochon.Downloadable, log *logrus.Entry) bool {
torrent := d.Infos()
log = log.WithField("torrent_name", torrent.Name)
// First check that the torrent download is finished
if !torrent.IsFinished {
log.Debugf("torrent is not yet finished")
return false
}
// Check that the ratio is reached
if torrent.Ratio < c.config.Downloader.Cleaner.Ratio {
log.Debugf("ratio is not reached (%.02f / %.02f)", torrent.Ratio, c.config.Downloader.Cleaner.Ratio)
return false
}
return true
}
// GetMovieList implements the explorer interface
func (y *Yts) GetMovieList(option polochon.ExplorerOption, log *logrus.Entry) ([]*polochon.Movie, error) {
log = log.WithField("explore_category", "movies")
opt, err := translateMovieOptions(option)
if err != nil {
return nil, err
}
movieList, err := yts.GetList(1, 6, opt, yts.OrderDesc)
if err != nil {
return nil, err
}
result := []*polochon.Movie{}
for _, movie := range movieList {
// Create the movie
m := polochon.NewMovie(polochon.MovieConfig{})
m.Title = movie.Title
m.Year = movie.Year
m.ImdbID = movie.ImdbID
// Add the torrents
for _, t := range movie.Torrents {
// Get the torrent quality
torrentQuality := polochon.Quality(t.Quality)
if !torrentQuality.IsAllowed() {
log.Debugf("yts: unhandled quality: %q", torrentQuality)
continue
}
m.Torrents = append(m.Torrents, polochon.Torrent{
Quality: torrentQuality,
URL: t.URL,
Seeders: t.Seeds,
Leechers: t.Peers,
Source: moduleName,
})
}
// Append the movie
result = append(result, m)
}
return result, nil
}
func (c *Cleaner) clean(d polochon.Downloadable, log *logrus.Entry) error {
torrent := d.Infos()
// Going over all the files and remove only the allowed ones
for _, tPath := range torrent.FilePaths {
filePath := filepath.Join(c.config.Watcher.Dir, tPath)
file := polochon.NewFile(filePath)
// Check extension
ext := path.Ext(filePath)
if !stringInSlice(ext, c.config.File.AllowedExtentionsToDelete) {
if !file.IsSymlink() {
// Not allowed to delete these types of files
log.WithFields(logrus.Fields{
"extension": ext,
"file_to_clean": filePath,
}).Debug("protected extention")
continue
} else {
log.Debugf("file %q is a sym link, delete it", filePath)
}
}
// If it's a symlink, delete the file as it has already been organized
err := c.deleteFile(file.Path, log)
if err != nil {
log.Warnf("got error while removing file %q", err)
continue
}
}
// Need to check if we can delete the directory of the torrent
err := c.cleanDirectory(torrent, log)
if err != nil {
log.Warnf("got error while deleting directory : %q", err)
return err
}
return nil
}
// getGoodShowEpisodeSubtitles will retrieve only the shoes with the same
// imdbId / season nb / episode nb
func (osp *osProxy) getGoodShowEpisodeSubtitles(s polochon.ShowEpisode, subs osdb.Subtitles, log *logrus.Entry) []osdb.Subtitle {
var goodSubs []osdb.Subtitle
for _, sub := range subs {
// Need to check that it's the good subtitle
imdbID := fmt.Sprintf("tt%07s", sub.SeriesIMDBParent)
if imdbID != s.ShowImdbID {
continue
}
if sub.SeriesEpisode != strconv.Itoa(s.Episode) {
continue
}
if sub.SeriesSeason != strconv.Itoa(s.Season) {
continue
}
goodSubs = append(goodSubs, sub)
}
if len(goodSubs) > 0 {
log.Debugf("Got %d subtitles", len(goodSubs))
}
return goodSubs
}
// DeleteShowEpisode will delete the showEpisode
func (l *Library) DeleteShowEpisode(se *polochon.ShowEpisode, log *logrus.Entry) error {
// Delete the episode
log.Infof("Removing ShowEpisode %q", se.Path)
// Remove the episode
if err := os.RemoveAll(se.Path); err != nil {
return err
}
pathWithoutExt := se.PathWithoutExt()
// Remove also the .nfo and .srt files
for _, ext := range []string{"nfo", "srt"} {
fileToDelete := fmt.Sprintf("%s.%s", pathWithoutExt, ext)
log.Debugf("Removing %q", fileToDelete)
// Remove file
if err := os.RemoveAll(fileToDelete); err != nil {
return err
}
}
// Remove the episode from the index
if err := l.showIndex.RemoveEpisode(se, log); err != nil {
return err
}
// Season is empty, delete the whole season
ok, err := l.showIndex.IsSeasonEmpty(se.ShowImdbID, se.Season)
if err != nil {
return err
}
if ok {
// Delete the whole season
seasonDir := l.getSeasonDir(se)
if err := os.RemoveAll(seasonDir); err != nil {
return err
}
// Remove the season from the index
show := &polochon.Show{ImdbID: se.ShowImdbID}
if err := l.showIndex.RemoveSeason(show, se.Season, log); err != nil {
return err
}
}
// Show is empty, delete the whole show from the index
ok, err = l.showIndex.IsShowEmpty(se.ShowImdbID)
if err != nil {
return err
}
if ok {
// Delete the whole Show
showDir := l.getShowDir(se)
if err := os.RemoveAll(showDir); err != nil {
return err
}
// Remove the show from the index
show := &polochon.Show{ImdbID: se.ShowImdbID}
if err := l.showIndex.RemoveShow(show, log); err != nil {
return err
}
}
return nil
}
// AddShowEpisode adds an episode to the store
func (l *Library) AddShowEpisode(ep *polochon.ShowEpisode, log *logrus.Entry) error {
if ep.Path == "" {
return ErrMissingShowEpisodeFilePath
}
ok, err := l.HasShowEpisode(ep.ShowImdbID, ep.Season, ep.Episode)
if err != nil {
return err
}
if ok {
// Get the old episode from the index
oldEpisode, err := l.GetEpisode(ep.ShowImdbID, ep.Season, ep.Episode)
if err != nil {
return err
}
if err := l.DeleteShowEpisode(oldEpisode, log); err != nil {
return err
}
}
// Add the show
if err := l.addShow(ep, log); err != nil {
return err
}
// Create show season dir if necessary
seasonDir := l.getSeasonDir(ep)
if !exists(seasonDir) {
if err := os.Mkdir(seasonDir, os.ModePerm); err != nil {
return err
}
}
// Move the file
// If the show episode already in the right dir there is nothing to do
if path.Dir(ep.Path) == seasonDir {
log.Debug("show episode already in the destination folder")
return nil
}
// Save the old path
oldPath := ep.Path
// Move the episode into the folder
newPath := filepath.Join(seasonDir, path.Base(ep.Path))
log.Debugf("Moving episode to folder Old path: %q, New path: %q", ep.Path, newPath)
if err := os.Rename(ep.Path, newPath); err != nil {
return err
}
// Set the new movie path
ep.Path = newPath
// Create a symlink between the new and the old location
if err := os.Symlink(ep.Path, oldPath); err != nil {
log.Warnf("Error while making symlink between %s and %s : %+v", oldPath, ep.Path, err)
}
// Create show NFO if necessary
if err := writeNFOFile(ep.NfoPath(), ep); err != nil {
return err
}
return l.showIndex.Add(ep)
}
// AddMovie adds a movie to the store
func (l *Library) AddMovie(movie *polochon.Movie, log *logrus.Entry) error {
if movie.Path == "" {
return ErrMissingMovieFilePath
}
// Check if the movie is already in the library
ok, err := l.HasMovie(movie.ImdbID)
if err != nil {
return err
}
if ok {
// Get the old movie path from the index
oldMovie, err := l.GetMovie(movie.ImdbID)
if err != nil {
return err
}
// Delete it
if err := l.DeleteMovie(oldMovie, log); err != nil {
return err
}
}
storePath := l.getMovieDir(movie)
// If the movie already in the right dir there is nothing to do
if path.Dir(movie.Path) == storePath {
log.Debug("Movie already in the destination folder")
return nil
}
// Remove movie dir if it exisits
if ok := exists(storePath); ok {
log.Debug("Movie folder exists, remove it")
if err := os.RemoveAll(storePath); err != nil {
return err
}
}
// Create the folder
if err := os.Mkdir(storePath, os.ModePerm); err != nil {
return err
}
// Move the movie into the folder
newPath := filepath.Join(storePath, path.Base(movie.Path))
// Save the old path
oldPath := movie.Path
log.Debugf("Old path: %q, new path %q", movie.Path, newPath)
if err := os.Rename(movie.Path, newPath); err != nil {
return err
}
// Set the new movie path
movie.Path = newPath
// Create a symlink between the new and the old location
if err := os.Symlink(movie.Path, oldPath); err != nil {
log.Warnf("Error while making symlink between %s and %s : %+v", oldPath, movie.Path, err)
}
// Write NFO into the file
if err := writeNFOFile(movie.NfoPath(), movie); err != nil {
return err
}
// At this point the video is stored
if err := l.movieIndex.Add(movie); err != nil {
return err
}
if movie.Fanart == "" || movie.Thumb == "" {
return ErrMissingMovieImageURL
}
// Download images
for _, img := range []struct {
url string
savePath string
}{
{
url: movie.Fanart,
savePath: movie.MovieFanartPath(),
},
{
url: movie.Thumb,
savePath: movie.MovieThumbPath(),
},
} {
if err := download(img.url, img.savePath); err != nil {
return err
}
}
return nil
}
// httpServer returns an http server
func (s *Server) httpServer(log *logrus.Entry) *http.Server {
addr := fmt.Sprintf("%s:%d", s.config.HTTPServer.Host, s.config.HTTPServer.Port)
log.Debugf("http server will listen on: %s", addr)
mux := mux.NewRouter()
for _, route := range []struct {
// name of the route
name string
// path of the route
path string
// allowed methods for this route
methods string
// handler is the http handler to run if the route matches
handler func(http.ResponseWriter, *http.Request)
// excluded tells if the route should be added to the router,
// it's in the negative form so that the default behaviour is to add
// the route to the router
excluded bool
}{
{
name: "GetMovies",
path: "/movies",
methods: "GET",
handler: s.movieIds,
},
{
name: "GetMovie",
path: "/movies/{id}",
methods: "GET",
handler: s.getMovieDetails,
},
{
name: "DeleteMovie",
path: "/movies/{id}",
methods: "DELETE",
handler: s.deleteMovie,
},
{
name: "DownloadMovie",
path: "/movies/{id}/download",
methods: "GET",
handler: s.serveMovie,
excluded: !s.config.HTTPServer.ServeFiles,
},
{
name: "GetShows",
path: "/shows",
methods: "GET",
handler: s.showIds,
},
{
name: "GetShow",
path: "/shows/{id}",
methods: "GET",
handler: s.getShowDetails,
},
{
name: "GetSeason",
path: "/shows/{id}/seasons/{season:[0-9]+}",
methods: "GET",
handler: s.getSeasonDetails,
},
{
name: "GetEpisode",
path: "/shows/{id}/seasons/{season:[0-9]+}/episodes/{episode:[0-9]+}",
methods: "GET",
handler: s.getShowEpisodeIDDetails,
},
{
name: "DeleteEpisode",
path: "/shows/{id}/seasons/{season:[0-9]+}/episodes/{episode:[0-9]+}",
methods: "DELETE",
handler: s.deleteEpisode,
},
{
name: "DownloadEpisode",
path: "/shows/{id}/seasons/{season:[0-9]+}/episodes/{episode:[0-9]+}/download",
methods: "GET",
handler: s.serveShow,
excluded: !s.config.HTTPServer.ServeFiles,
},
{
name: "Wishlist",
path: "/wishlist",
methods: "GET",
handler: s.wishlist,
},
{
name: "AddTorrent",
path: "/torrents",
methods: "POST",
handler: s.addTorrent,
},
} {
if route.excluded {
continue
}
// Register the route
mux.HandleFunc(route.path, route.handler).Name(route.name).Methods(route.methods)
}
n := negroni.New()
// Panic recovery
n.Use(negroni.NewRecovery())
// Use logrus as logger
n.Use(negronilogrus.NewMiddlewareFromLogger(s.log.Logger, "httpServer"))
// gzip compression
n.Use(gzip.Gzip(gzip.DefaultCompression))
// Add basic auth if configured
if s.config.HTTPServer.BasicAuth {
log.Info("server will require basic authentication")
n.Use(NewBasicAuthMiddleware(s.config.HTTPServer.BasicAuthUser, s.config.HTTPServer.BasicAuthPassword))
}
// Add token auth middleware if token configuration file specified
if s.tokenManager != nil {
n.Use(token.NewMiddleware(s.tokenManager, mux))
mux.HandleFunc("/tokens/allowed", s.tokenGetAllowed).Name("TokenGetAllowed")
}
// Wrap the router
n.UseHandler(mux)
return &http.Server{Addr: addr, Handler: n}
}
