Пример #1
0
func amRecovery() {
	// Recovering failed ambum ids
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("am", SiteConfigPath)
	siteConf.NConcurrent = 30
	dna.PanicError(err)

	// r := NewRange(20987, 30000)
	ids := amGetErrIds("./log/http_error.log", 1)
	tmp := ids.Unique()
	ids = &tmp
	if ids.Length() > 0 {
		state := NewStateHandlerWithExtSlice(new(am.APIAlbum), ids, siteConf, db)
		Update(state)
	} else {
		dna.Log("No need to recover file")
	}

	// Recover failed SQL statements
	RecoverErrorQueries(SqlErrorLogPath, db)

	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}
Пример #2
0
// UpdateNhacso gets lastest items from nhacso.com.
// The update process goes through 7 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Updating new songs.
// 	Step 3: Updating new albums.
// 	Step 4: Updating new videos.
// 	Step 5: Updating catagories of new songs.
// 	Step 6: Updating catagories of new albums.
// 	Step 7: Recovering failed sql statements.
func UpdateNhacso() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("ns", SiteConfigPath)
	dna.PanicError(err)
	// update song
	state := NewStateHandler(new(ns.Song), siteConf, db)
	Update(state)
	//  update album
	state = NewStateHandler(new(ns.Album), siteConf, db)
	Update(state)
	// update video
	state = NewStateHandler(new(ns.Video), siteConf, db)
	Update(state)

	r := NewRange(0, dna.Int(len(*ns.SongGenreList))*ns.LastNPages-1)
	siteConf.NConcurrent = 10
	state = NewStateHandlerWithRange(new(ns.SongCategory), r, siteConf, db)
	Update(state)

	state = NewStateHandlerWithRange(new(ns.AlbumCategory), r, siteConf, db)
	Update(state)

	RecoverErrorQueries(SqlErrorLogPath, db)
	time.Sleep(3 * time.Second)

	CountDown(3*time.Second, QuittingMessage, EndingMessage)

	db.Close()

}
Пример #3
0
// UpdateNhacvui gets lastest items from nhacvui.vn.
// The update process goes through 5 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Updating new songs.
// 	Step 3: Updating new albums.
// 	Step 4: Updating new videos from FoundVideos var.
// 	Step 5: Recovering failed sql statements.
func UpdateNhacvui() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("nv", SiteConfigPath)
	dna.PanicError(err)

	state := NewStateHandler(new(nv.Song), siteConf, db)
	Update(state)
	//  update album
	state = NewStateHandler(new(nv.Album), siteConf, db)
	Update(state)

	if nv.FoundVideos.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(nv.Video), nv.FoundVideos, siteConf, db)
		Update(state)
	} else {
		dna.Log("No videos found!")
	}

	RecoverErrorQueries(SqlErrorLogPath, db)

	time.Sleep(3 * time.Second)
	db.Close()

}
Пример #4
0
// UpdateSongFreaks gets lastest songs,albums,artists and videos from songfreaks.com
// The update process goes through 4 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Finding new songs, insert new albums,artists and videos if found.
// 	Step 3: Updating found new albums in Step 2.
// 	Step 4: Recovering failed sql statements in Step 2.
func UpdateSongFreaks() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("sf", SiteConfigPath)
	siteConf.NConcurrent = 20
	dna.PanicError(err)

	// Update new songs
	state := NewStateHandler(new(sf.APISongFreaksTrack), siteConf, db)
	state.TableName = "sfsongs"
	Update(state)

	// Update "ratings", "songids", "review_author", "review" of song
	ids := &[]dna.Int{}
	query := dna.Sprintf("SELECT id FROM sfalbums where checktime > '%v' AND array_length(songids, 1) is NULL", time.Now().Format("2006-01-02"))
	// dna.Log(query)
	err = db.Select(ids, query)
	if err != nil {
		dna.PanicError(err)
	}
	idsSlice := dna.IntArray(*ids)
	if idsSlice.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(sf.APISongFreaksAlbum), &idsSlice, siteConf, db)
		Update(state)

	} else {
		dna.Log("No new albums found")
	}

	// Recover failed sql statements
	RecoverErrorQueries(SqlErrorLogPath, db)

	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}
Пример #5
0
// UpdateNhaccuatui gets lastest items from nhaccuatui.com.
//
// The update process goes through 6 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Updating new songs.
// 	Step 3: Updating new albums.
// 	Step 4: Updating new videos.
// 	Step 5: Updating new artists.
// 	Step 6: Recovering failed sql statements.
func UpdateNhaccuatui() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("nct", SiteConfigPath)
	dna.PanicError(err)
	TIMEOUT_SECS = 60 // for safe
	// siteConf.NConcurrent = 15

	state := NewStateHandler(new(nct.Song), siteConf, db)
	Update(state)

	state = NewStateHandler(new(nct.Album), siteConf, db)
	Update(state)

	state = NewStateHandler(new(nct.Video), siteConf, db)
	Update(state)

	state = NewStateHandler(new(nct.Artist), siteConf, db)
	// Because there is no setting for total concurrent artists failed,
	// the default value NCSongFail will be returned.
	// Now, it is set to new value
	state.SiteConfig.NCSongFail = 1000 // Set concurrency for artists
	Update(state)
	RecoverErrorQueries(SqlErrorLogPath, db)
	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}
Пример #6
0
func ExampleSelectMissingIds() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig("./app.ini"))
	dna.PanicError(err)
	ids, err := SelectMissingIds("ziartists", &dna.IntArray{5, 6, 7, 8, 9}, db)
	dna.PanicError(err)
	dna.Log(ids)
	db.Close()
	// Output:
	// &dna.IntArray{8}
}
Пример #7
0
func amUpdateNewAlbums() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("am", SiteConfigPath)
	siteConf.NConcurrent = 10
	dna.PanicError(err)

	state := NewStateHandler(new(am.APIAlbum), siteConf, db)
	state.TableName = "amalbums"
	Update(state)
	CountDown(5*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}
Пример #8
0
// UpdateZing gets lastest items from mp3.zing.vn.
// The update process goes through 8 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler
// 	Step 1: Updating new songs
// 	Step 3: Updating new albums
// 	Step 4: Updating new videos
// 	Step 5: Updating new artists
// 	Step 6: Updating new songids found in new albums
// 	Step 7: Updating new tvs
// 	Step 8: Recovering failed sql statements
func UpdateZing() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("zi", SiteConfigPath)
	dna.PanicError(err)
	// update song
	state := NewStateHandler(new(zi.Song), siteConf, db)
	Update(state)
	// update album
	state = NewStateHandler(new(zi.Album), siteConf, db)
	Update(state)
	// update video
	state = NewStateHandler(new(zi.Video), siteConf, db)
	Update(state)
	// update artist
	state = NewStateHandler(new(zi.Artist), siteConf, db)
	Update(state)

	// update new songids found in albums
	dna.Log("Update new songs from albums")
	ids := utils.SelectNewSidsFromAlbums("zialbums", time.Now(), db)
	nids, err := utils.SelectMissingIds("zisongs", ids, db)
	if err != nil {
		dna.Log(err.Error())
	} else {
		if nids != nil && nids.Length() > 0 {
			state = NewStateHandlerWithExtSlice(new(zi.Song), nids, siteConf, db)
			Update(state)
		} else {
			dna.Log("No new songs found")
		}

	}

	state = NewStateHandler(new(zi.TV), siteConf, db)
	Update(state)

	RecoverErrorQueries(SqlErrorLogPath, db)

	time.Sleep(3 * time.Second)
	db.Close()
}
Пример #9
0
// UpdateChacha updates chacha.vn.
// The update process goes through 5 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 1: Updating new songs.
// 	Step 3: Updating new albums.
// 	Step 4: Updating new videos.
// 	Step 5: Recovering failed sql statements.
func UpdateChacha() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("cc", SiteConfigPath)
	dna.PanicError(err)

	state := NewStateHandler(new(cc.Song), siteConf, db)
	Update(state)
	//  update album
	state = NewStateHandler(new(cc.Album), siteConf, db)
	Update(state)

	state = NewStateHandler(new(cc.Video), siteConf, db)
	Update(state)

	RecoverErrorQueries(SqlErrorLogPath, db)

	time.Sleep(3 * time.Second)
	db.Close()

}
Пример #10
0
// UpdateKeeng gets lastest items from keeng.com.
// The update process goes through 6 steps:
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Updating new artists.
// 	Step 3: Updating new songs.
// 	Step 4: Updating new albums.
// 	Step 5: Updating new videos.
// 	Step 6: Recovering failed sql statements.
func UpdateKeeng() {
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("ke", SiteConfigPath)
	dna.PanicError(err)

	state := NewStateHandler(new(ke.Artist), siteConf, db)
	Update(state)

	state = NewStateHandler(new(ke.Song), siteConf, db)
	Update(state)

	state = NewStateHandler(new(ke.Album), siteConf, db)
	Update(state)

	state = NewStateHandler(new(ke.Video), siteConf, db)
	Update(state)

	RecoverErrorQueries(SqlErrorLogPath, db)
	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}
Пример #11
0
func UpdateLyricFind(vpnAppName, vpnAppUrl dna.String, estConDuration time.Duration) {
	cmd := <-startVPN(vpnAppName, vpnAppUrl)
	CountDown(estConDuration*time.Second, "Establishing VPN connection. Estimated time remaining:", "")

	err := testingSongs()
	if err != nil {
		dna.Log("Error occurs: ", err.Error())
		dna.Log("Operation aborted!")
	} else {
		db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
		dna.PanicError(err)
		siteConf, err := LoadSiteConfig("lf", SiteConfigPath)
		dna.PanicError(err)

		state := NewStateHandler(new(lf.Song), siteConf, db)
		Update(state)

		RecoverErrorQueries(SqlErrorLogPath, db)
		CountDown(3*time.Second, QuittingMessage, EndingMessage)
		db.Close()
	}
	stopVPN(vpnAppName, cmd)
}
Пример #12
0
// UpdateChiasenhac updates new songs, videos and albums.
// New albums are created from new songs fetched.
// After getting new songs or videos, it runs re-fetching
// procedure until there is no missing songs, videos or
// the procesure runs 10 times.
//
// 	fillMode : only update records whose titles are empty.
//
// The update process goes through 6 steps:
// 	Step 1: Initializing db connection, site config, state handler and finding max id
// 	Step 2: Getting new songs from max id found in the table. If videos are found, then it inserts the videos into a video table.
// 	Step 3: Getting missing ids from last 10000 songids.
// 	Step 4: Refetching errors of songs having empty titles.
// 	Step 5: Recovering fail sql statement.
// 	Step 6: Creating new albums from new songs found.
func UpdateChiasenhac(fillMode dna.Bool) {
	// note: songid 1172662 1172663 1172664 are not continuos
	var errCount = 0
	var missingCount = 0
	var lastTotalMissing dna.Int = 0
	var done = false

	TIMEOUT_SECS = 240
	dna.Log("TIMEOUT:", TIMEOUT_SECS)
	// Step 1
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("csn", SiteConfigPath)
	dna.PanicError(err)
	// Getting LastSongId for SaveNewAlbums func
	csn.LastSongId, err = utils.GetMaxId("csnsongs", db)
	dna.PanicError(err)
	dna.Log("Max ID:", csn.LastSongId)

	// Step 2: Getting both new songs and videos and
	// inserting into appropriate tables respectively.
	if fillMode == false {
		state := NewStateHandler(new(csn.Song), siteConf, db)
		Update(state)

		// Step 3: Fetching missing ids.
		// It stops when total loop is 3 or the last total ids equals new total ones
		dna.Log(dna.String("\nGetting missing ids from last 10000 songids").ToUpperCase())
		for missingCount < 3 && done == false {
			temp := updateMissingIds(db, siteConf, 10000)
			if temp == lastTotalMissing {
				done = true
			}
			lastTotalMissing = temp
			missingCount += 1
		}

	}

	// SET LAST SONGID
	// csn.LastSongId = 1203603

	// Step 4: Re-fetching err songs
	db.Ping()
	dna.Log(dna.String("\nRe-fetching err from last 10000 songs having EMPTY titles & ID > " + csn.LastSongId.ToString()).ToUpperCase())
	for false == updateEmptyTitles(db, siteConf, csn.LastSongId) && errCount < 10 {
		db.Ping()
		errCount += 1
		dna.Log("RE-FETCHING ROUND:", errCount)
	}
	dna.Log("Re-fetching error done!")

	// Step 5: Recovering failed sql statments
	RecoverErrorQueries(SqlErrorLogPath, db)

	if fillMode == false {
		// Step 6: Saving new abums
		dna.Log("Finding and saving new albums from last songid:", csn.LastSongId)
		nAlbums, err := csn.SaveNewAlbums(db)
		if err != nil {
			dna.Log(err.Error())
		} else {
			dna.Log("New albums inserted:", nAlbums)
		}
	}

	dna.Log("SET TIMEOUT TO DEFAULT: 8s")
	TIMEOUT_SECS = 8
	time.Sleep(2 * time.Second)
	db.Close()
}
Пример #13
0
// UpdateHDViet gets lastest items from hdviet.com.
//
// The update process goes through 8 steps and 2 parts:
//
// 	PART I: FINDING AND UPDATING NEW EPS OF MOVIES IN DB
// 	Step 1: Initalizing db connection, loading site config and state handler.
// 	Step 2: Finding all movies possibly having new episodes in DB
// 	Step 3: Finding all movies found in Step 2 actually having new episodes available from source website.
// 	Step 4: Checking consitency of movies' new found episodes from the source.
// 	Step 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
//
// 	PART II: GETTING NEW MOVIES FROM HDVIET SITE.
// 	Step 6: Find newest movies from the source.
// 	Step 7: Updating newest episodes from newest found movies in Step 6.
// 	Step 8: Recovering failed sql statements.
func UpdateHDViet() {
	var mvTable = dna.String("hdvmovies")
	db, err := sqlpg.Connect(sqlpg.NewSQLConfig(SqlConfigPath))
	dna.PanicError(err)
	siteConf, err := LoadSiteConfig("hdv", SiteConfigPath)
	dna.PanicError(err)
	TIMEOUT_SECS = 100

	// PART 1: UPDATING NEW EPISODES OF MOVIES IN DB
	// STEP 2: Finding all movies possibly having new episodes in DB.
	movieCurrentEps, err := hdv.GetMoviesCurrentEps(db, mvTable)
	dna.PanicError(err)
	newEpisodeKeys := dna.IntArray{}
	for movieid, currentEps := range movieCurrentEps {
		newEpisodeKeys.Push(hdv.ToEpisodeKey(movieid, currentEps))
	}
	newEpisodeKeys.Sort()

	// STEP 3: Checking and getting new episodes if available from source website.
	state := NewStateHandlerWithExtSlice(new(hdv.EpUpdater), &newEpisodeKeys, siteConf, db)
	Update(state)

	// STEP 4:Checking consitency of new found episodes of movies from the source.
	movieIdList := dna.IntArray{}
	for _, epKey := range hdv.LastestEpisodeKeyList {
		mvid, _ := hdv.ToMovieIdAndEpisodeId(epKey)
		movieIdList.Push(mvid)
	}
	movieIdList = movieIdList.Unique()
	dna.Log("\nNumber of movies: ", movieIdList.Length(), "having new episodes:", hdv.LastestEpisodeKeyList.Length())

	if movieIdList.Length() != dna.Int(len(hdv.LastestMovieCurrentEps)) {
		dna.Log("LastestEpisodeKeyList & LastestMovieCurrentEps do not match! GOT:", movieIdList.Length(), len(hdv.LastestMovieCurrentEps))
	}

	// STEP 5: Getting new episode data from the source, save them to DB and update current_eps field of movies having new eps.
	if hdv.LastestEpisodeKeyList.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.LastestEpisodeKeyList, siteConf, db)
		Update(state)
		RecoverErrorQueries(SqlErrorLogPath, db)
		// dna.Log(hdv.LastestEpisodeKeyList)

		hdv.SaveLastestMovieCurrentEps(db, mvTable, SQLERROR)
		RecoverErrorQueries(SqlErrorLogPath, db)
	} else {
		dna.Log("No new episodes found. Update operation has been aborted!")
	}

	// PART 2: UPDATING NEW MOVIES FROM hdv site.
	// STEP 6: Find newest movies from sources.
	db.Ping()
	state = NewStateHandler(new(hdv.Movie), siteConf, db)
	Update(state)

	// STEP 7: Updating newest episodes from newest found movies in Step 6.
	hdv.EpisodeKeyList = hdv.EpisodeKeyList.Unique()
	if hdv.EpisodeKeyList.Length() > 0 {
		state = NewStateHandlerWithExtSlice(new(hdv.Episode), &hdv.EpisodeKeyList, siteConf, db)
		Update(state)
	} else {
		dna.Log("No new movies found")
	}

	// STEP 8: Recovering failed sql statements.
	RecoverErrorQueries(SqlErrorLogPath, db)

	CountDown(3*time.Second, QuittingMessage, EndingMessage)
	db.Close()
}