// SelectMissingIds accepts a table name as an input and a list of ids as a source.
// It returns a new list of ids that does not exist in the destination table
//
// * tblName : a table name
// * srcIds : a source ids
// * db : a pointer to connected databased
// * Returns a new list of ids which are not from the specified table
//
// The format of sql statement is:
// WITH dna (id) AS (VALUES (5),(6),(7),(8),(9))
// SELECT id FROM dna WHERE NOT EXISTS
// (SELECT 1 from ziartists WHERE id=dna.id)
func SelectMissingIds(tblName dna.String, srcIds *dna.IntArray, db *sqlpg.DB) (*dna.IntArray, error) {
if srcIds.Length() > 0 {
val := dna.StringArray(srcIds.Map(func(val dna.Int, idx dna.Int) dna.String {
return "(" + val.ToString() + ")"
}).([]dna.String))
selectStmt := "with dna (id) as (values " + val.Join(",") + ") \n"
selectStmt += "SELECT id FROM dna WHERE NOT EXISTS\n (SELECT 1 from " + tblName + " WHERE id=dna.id)"
ids := &[]dna.Int{}
err := db.Select(ids, selectStmt)
switch {
case err != nil:
return nil, err
case err == nil && ids != nil:
slice := dna.IntArray(*ids)
return &slice, nil
case err == nil && ids == nil:
return &dna.IntArray{}, nil
default:
panic("Default case triggered. Case is not expected. Cannot select non existed ids")
}
} else {
return nil, errors.New("Empty input array")
}
}
// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// Step 1: Initalizing db connection, loading site config and state handler.
// Step 2: Finding all movies possibly having new episodes in DB
// Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// Step 4: Checking consitency of movies' new found episodes from the source.
// Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// Step 6: Find newest movies from the source.
// Step 7: Updating newest episodes from newest found movies in Step 6.
// Step 8: Recovering failed sql statements.
func UpdateHDViet() {
var mvTable = dna.String("hdvmovies")
db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
dna.PanicError(err)
siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
dna.PanicError(err)
TIMEOUT_SECS = 100
// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
// STEP 2: Finding all movies possibly having new episodes in DB.
movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
dna.PanicError(err)
newEpisodeKeys := dna.IntArray{}
for movieid, currentEps := range movieCurrentEps {
newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
}
newEpisodeKeys.Sort()
// STEP 3: Checking and getting new episodes if available from source website.
state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
Update(state)
// STEP 4:Checking consitency of new found episodes of movies from the source.
movieIdList := dna.IntArray{}
for _, epKey := range hdv.LastestEpisodeKeyList {
mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
movieIdList.Push(mvid)
}
movieIdList = movieIdList.Unique()
dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())
if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
}
// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
if hdv.LastestEpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
Update(state)
RecoverErrorQueries(SqlErrorLogPath, db)
// dna.Log(hdv.LastestEpisodeKeyList)
hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
RecoverErrorQueries(SqlErrorLogPath, db)
} else {
dna.Log("No new episodes found. Update operation has been aborted!")
}
// PART 2: UPDATING NEW MOVIES FROM hdv site.
// STEP 6: Find newest movies from sources.
db.Ping()
state = NewStateHandler(new(hdv.Movie), siteConf, db)
Update(state)
// STEP 7: Updating newest episodes from newest found movies in Step 6.
hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
if hdv.EpisodeKeyList.Length() > 0 {
state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
Update(state)
} else {
dna.Log("No new movies found")
}
// STEP 8: Recovering failed sql statements.
RecoverErrorQueries(SqlErrorLogPath, db)
CountDown(3*time.Second, QuittingMessage, EndingMessage)
db.Close()
}