// GetEpisole returns an episole URL of a series.
func (urlb *URLBuilder) GetEpisole(movieid, ep dna.Int) dna.String {
str := dna.Sprintf("movieid=%v&accesstokenkey=%v&ep=%v", movieid, ACCESS_TOKEN_KEY, ep)
data := []byte(str.String())
strBase64 := base64.StdEncoding.EncodeToString(data)
sign := getMD5(dna.String(strBase64) + SECRET_KEY)
return dna.Sprintf("%vmovieid=%v&accesstokenkey=%v&ep=%v&sign=%v", BASE_URL, movieid, ACCESS_TOKEN_KEY, ep, sign)
}
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
var errCount = 0
var errStrings = dna.StringArray{}
b, err := ioutil.ReadFile(sqlErrFilePath.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
// dna.Log("\n", data.Length())
sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
for _, val := range sqlArr {
sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
if len(sqlStmtArr) > 0 {
_, err := db.Exec(sqlStmtArr[0][1].String())
if err != nil {
if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
errCount += 1
errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
}
}
}
}
if errCount == 0 {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
} else {
err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
}
if err != nil {
panic(err)
}
return dna.Int(errCount)
}
func getSrtContent(episode *Episode, isEn dna.Bool) <-chan bool {
channel := make(chan bool, 1)
go func() {
var result *http.Result
var err error
if isEn == true {
result, err = http.Get(episode.SubtitleExt[1])
} else {
result, err = http.Get(episode.SubtitleExt[0])
}
if err == nil {
if isEn == true {
// It is hard to detect an encoding of a string.
// Therefore we convert them to BASE64
episode.EnSrt = dna.String(base64.StdEncoding.EncodeToString(result.Data.ToBytes()))
// episode.EnSrt = ISO8859_1ToUTF8String(result.Data.ToBytes())
// ioutil.WriteFile("./dump/test_en_srt.srt", result.Data.ToBytes(), 0644)
} else {
// Vietnamese Subtitle encoded in UTF-16 Little Ending
// It has to be converted to UTF-8
if result.Data.Match(`^[0-9a-fA-F]+$`) == false {
// episode.ViSrt = UTF16ToUTF8String(result.Data.ToBytes(), binary.LittleEndian)
episode.ViSrt = dna.String(base64.StdEncoding.EncodeToString(result.Data.ToBytes()))
}
// dna.Log(result.Data.Substring(0, 100))
}
}
channel <- true
}()
return channel
}
func (c Counter) String() string {
c.mu.RLock()
if c.Total > 0 {
format := dna.String("%v✘ | %v✔")
return string(dna.Sprintf(format, c.Fail, c.Pass))
} else {
format := dna.String("t:%v|n:%v|f:%v✘|p:%v✔|v:%v")
return string(dna.Sprintf(format, getTimeFmt(c.ElapsedTime), c.Count, c.Fail, c.Pass, c.Speed))
}
c.mu.RUnlock()
return ""
}
// getVideoPlays returns video plays
func getVideoPlays(video *Video, body dna.String) {
link := "http://www.nhaccuatui.com/interaction/api/hit-counter?jsoncallback=nct"
http.DefaulHeader.Set("Content-Type", "application/x-www-form-urlencoded ")
result, err := http.Post(dna.String(link), body)
// Log(link)
if err == nil {
data := &result.Data
tpl := dna.String(`{"counter":([0-9]+)}`)
playsArr := data.FindAllStringSubmatch(tpl, -1)
if len(playsArr) > 0 {
video.Plays = playsArr[0][1].ToInt()
}
}
}
// getAPIAlbumAwards fetches album's awards
// with the following url format:
// http://www.allmusic.com/album/google-bot-mw0002585207/similar/mobile
func getAPIAlbumAwards(album *APIAlbum) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://www.allmusic.com/album/google-bot-mw" + album.Id.ToFormattedString(10, true) + "/awards/mobile"
result, err := http.Get(link)
if err == nil {
data := &result.Data
var awardSections = []APIAwardSection{}
sectionsArr := data.FindAllString(`(?mis)<section class=.+?</section>`, -1)
sectionsArr.ForEach(func(section dna.String, idx dna.Int) {
awardSections = append(awardSections, getSection(section))
})
if len(awardSections) > 0 {
bAwards, derr := json.Marshal(awardSections)
if derr == nil {
album.Awards = dna.String(string(bAwards))
}
}
}
channel <- true
}()
return channel
}
// GetChannel returns TV channel URL.
func (urlb *URLBuilder) GetChannel(channelid dna.Int) dna.String {
str := dna.Sprintf("channelid=%v&accesstokenkey=%v", channelid, ACCESS_TOKEN_KEY)
data := []byte(str.String())
strBase64 := base64.StdEncoding.EncodeToString(data)
sign := getMD5(dna.String(strBase64) + SECRET_KEY)
return dna.Sprintf("%vchannelid=%v&accesstokenkey=%v&sign=%v", CHANNEL_BASE_URL, channelid, ACCESS_TOKEN_KEY, sign)
}
func amGetErrIds(inputFile dna.String, mode dna.Int) *dna.IntArray {
var ret = dna.IntArray{}
b, err := ioutil.ReadFile(inputFile.String())
if err != nil {
panic(err)
}
data := dna.String(string(b))
lines := data.Split("\n")
for _, line := range lines {
switch mode {
case 1:
idArr := line.FindAllStringSubmatch(`([0-9]+) Post.+no such host`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
idArr = line.FindAllStringSubmatch(`Timeout.+at id :([0-9]+)`, 1)
if len(idArr) > 0 {
ret.Push(idArr[0][1].ToInt())
}
case 2:
ret.Push(line.ToInt())
}
}
if mode == 1 {
err = ioutil.WriteFile(inputFile.String(), []byte{}, 0644)
if err != nil {
dna.Log("Cannot write to file1:", err.Error())
}
}
ret = ret.Unique()
return &ret
}
func ISO8859_1ToUTF8String(iso8859_1_buf []byte) dna.String {
buf := make([]rune, len(iso8859_1_buf))
for i, b := range iso8859_1_buf {
buf[i] = rune(b)
}
return dna.String(string(buf))
}
func SaveLastedChecktime(db *sqlpg.DB, filePath dna.String) error {
cf, err := cfg.LoadConfigFile(filePath)
if err != nil {
return err
}
siteCts, err := GetLastedChecktime(db)
if err != nil {
return err
}
for site, checktime := range siteCts {
var key dna.String
var section dna.String
switch {
case site.Match("songs") == true:
key = "songs"
section = site.Replace("songs", "")
case site.Match("albums") == true:
key = "albums"
section = site.Replace("albums", "")
case site.Match("videos") == true:
key = "videos"
section = site.Replace("videos", "")
default:
panic("site is not valid")
}
cf.SetValue(section, key, dna.String(checktime.Format(utils.DefaultTimeLayout)))
}
return cfg.SaveConfigFile(cf, filePath)
}
// Get impliments getting site with basic properties.
// Enable gzip, deflat by default to reduce network data, redirect to new location from response.
// It returns data (String type) and error
// if err is nil then data is "" (empty).
func Get(url dna.String) (*Result, error) {
req, err := http.NewRequest("GET", url.ToPrimitiveValue(), nil)
req.Header = DefaulHeader
// req.Header.Add("Accept-Encoding", "gzip,deflate")
// req.Header.Add("Accept-Language", "en-US,en")
// req.Header.Add("Cache-Control", "max-age=0")
// req.Header.Add("Connection", "keep-alive")
req.Header.Add("Host", url.ToPrimitiveValue())
// req.Header.Add("User-Agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
// req.Header.Add("Cookie", "")
// dna.Log(req.Header)
res, err := client.Do(req)
if err != nil {
return new(Result), err
}
var data []byte
var myErr error
switch res.Header.Get("Content-Encoding") {
case "gzip":
var reader io.ReadCloser
reader, err := gzip.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
case "deflate":
// Logv("sdsafsd")
reader, err := zlib.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
default:
data, myErr = ioutil.ReadAll(res.Body)
}
if myErr != nil {
return new(Result), myErr
}
res.Body.Close()
return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
func (sf *APISongFreaksTrack) ToSongFreak() (*SongFreak, error) {
if sferr := sf.HasError(); sferr != nil {
return nil, sferr
}
sofre := NewSongFreak()
sofre.Id = sf.Id
track, err := xml.MarshalIndent(sf.Track, "", "\t")
if err == nil {
sofre.Track = dna.String(string(track))
}
videos, err := xml.MarshalIndent(sf.Videos, "", "\t")
if err == nil {
sofre.Videos = dna.String(string(videos))
}
sofre.Checktime = time.Now()
return sofre, nil
}
func ExampleURLBuilder() {
logURL := func(descStr, url dna.String) {
dna.Log(descStr+dna.String(" ").Repeat(23-descStr.Length()), url)
}
urlb := NewURLBuilder()
urlb.SetTimestamp(time.Date(2014, time.February, 4, 12, 12, 12, 12, time.UTC))
logURL("ARTIST INFO:", urlb.GetArtistInfo(49674))
logURL("CHECKVERSION:", urlb.GetCheckVersion())
dna.Log("--------------------------------------------------------")
logURL("LIST OF GENRES:", urlb.GetListOfGenres("song"))
logURL("LIST OF TOPICS:", urlb.GetListOfTopics(1, 30))
dna.Log("--------------------------------------------------------")
logURL("PLAYLIST INFO:", urlb.GetPlaylistInfo(12255234))
logURL("PLAYLISTS BY ARTIST:", urlb.GetPlaylistsByArtist(49674, 1, 30))
logURL("PLAYLISTS BY GENRE:", urlb.GetPlaylistsByGenre(0, 1, 30))
logURL("PLAYLISTS BY SEARCHING:", urlb.GetPlaylistsBySearching("tell", 1, 30))
logURL("PLAYLISTS BY TOPIC:", urlb.GetPlaylistsByTopic(51))
logURL("PLAYLISTS RELATED:", urlb.GetPlaylistsRelated(12336694))
dna.Log("--------------------------------------------------------")
logURL("SONG INFO:", urlb.GetSongInfo(2854574))
logURL("SONG LYRIC:", urlb.GetSongLyric(2909727))
logURL("SONGS BY ARTIST:", urlb.GetSongsByArtist(49674, 50, 30))
logURL("SONGS BY GENRE:", urlb.GetSongsByGenre(5015, 34, 30))
logURL("SONGS BY SEARCHING:", urlb.GetSongsBySearching("man", 1, 30))
logURL("SONGS OF PLAYLIST", urlb.GetSongsOfPlaylist(12095591))
dna.Log("--------------------------------------------------------")
logURL("VIDEO INFO:", urlb.GetVideoInfo(2876055))
logURL("VIDEOS BY ARTIST:", urlb.GetVideosByArist(49674, 1, 30))
logURL("VIDEOS BY GENRE:", urlb.GetVideosByGenre(5142, 1, 30))
logURL("VIDEOS BY SEARCHING:", urlb.GetVideosBySearching("man", 1, 30))
logURL("VIDEOS RELATED:", urlb.GetVideosRelated(12336694))
// Output:
// ARTIST INFO: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-artist-detail&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&artistid=49674&time=1391515932000&token=a6ffbbb6da2b822f647829ff6215c210
// CHECKVERSION: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=check-version&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&time=1391515932000&token=31842a81519047f5dd477b54608eb2d7
// --------------------------------------------------------
// LIST OF GENRES: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-list-genre&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&type=song&time=1391515932000&token=c1b31f50c1ea19250898a1a2e14ca60b
// LIST OF TOPICS: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-list-topic&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&pageindex=1&pagesize=30&time=1391515932000&token=d7abea98a130db44bce09600f25cb316
// --------------------------------------------------------
// PLAYLIST INFO: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-playlist-info&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&playlistid=12255234&time=1391515932000&token=f1f0b8e4827086b37c3c3accee79017f
// PLAYLISTS BY ARTIST: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-playlist-by-artist&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&artistid=49674&pageindex=1&pagesize=30&time=1391515932000&token=2be9e5d0c3fd1325d2a7ee2426922d5b
// PLAYLISTS BY GENRE: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-playlist-by-genre&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&genreid=0&pageindex=1&pagesize=30&time=1391515932000&token=ec761c1865e50560e61c5036fca893ce
// PLAYLISTS BY SEARCHING: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=search-playlist&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&keyword=tell&pageindex=1&pagesize=30&time=1391515932000&token=5370cbfb8040183ad5570b999c813c83
// PLAYLISTS BY TOPIC: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-playlist-by-topic&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&topicid=51&time=1391515932000&token=ff8f012db0d705b3cdf2698ec16536a7
// PLAYLISTS RELATED: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-playlist-related&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&playlistid=12336694&time=1391515932000&token=5c8efd0df7e283a58c28b943b85e1262
// --------------------------------------------------------
// SONG INFO: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-song-info&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&songid=2854574&time=1391515932000&token=b1db67e99010e541ea910e997350a75d
// SONG LYRIC: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-lyric&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&songid=2909727&time=1391515932000&token=c752ab44e4f60177474b3ff554c380f3
// SONGS BY ARTIST: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-song-by-artist&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&artistid=49674&pageindex=50&pagesize=30&time=1391515932000&token=cb00833e14a158c8eae098be2834e166
// SONGS BY GENRE: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-song-by-genre&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&genreid=5015&pageindex=34&pagesize=30&time=1391515932000&token=21bd96e68c4179d0459f5692e0144ca0
// SONGS BY SEARCHING: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=search-song&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&keyword=man&pageindex=1&pagesize=30&time=1391515932000&token=3ba6c692cd09ef54ee196bc8e3798078
// SONGS OF PLAYLIST http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-song-by-playlistid&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&playlistid=12095591&time=1391515932000&token=9e461944fa434c9e197b5bae1ad9cc3f
// --------------------------------------------------------
// VIDEO INFO: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-video-detail&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&videoid=2876055&time=1391515932000&token=ad13b3cee157e1ab1dfa337ec47ab20e
// VIDEOS BY ARTIST: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-video-by-artist&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&artistid=49674&pageindex=1&pagesize=30&time=1391515932000&token=c1641290c36d3a36590df55cbad083dc
// VIDEOS BY GENRE: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-video-by-genre&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&genreid=5142&pageindex=1&pagesize=30&time=1391515932000&token=270c039a73d299c688d8b3b086ae1e7e
// VIDEOS BY SEARCHING: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=search-video&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&keyword=man&pageindex=1&pagesize=30&time=1391515932000&token=4bce7e9c8333ba31076c3cb1a2b85544
// VIDEOS RELATED: http://api.m.nhaccuatui.com/mobile/v5.0/api?secretkey=nct@mobile_service&action=get-video-related&deviceinfo={"DeviceID":"90c18c4cb3c37d442e8386631d46b46f","OsName":"ANDROID","OsVersion":"10","AppName":"NhacCuaTui","AppVersion":"5.0.1","UserInfo":"","LocationInfo":""}&videoid=12336694&time=1391515932000&token=57aa7164a27ac3dacb25526d7f4a3cbc
}
// GetTableName returns table name from a struct.
// Ex: An instance of ns.Song will return nssongs
// An instance of ns.Album will return nsalbums
func getTableName(structValue interface{}) dna.String {
val := reflect.TypeOf(structValue)
if val.Kind() != reflect.Ptr {
panic("StructValue has to be pointer")
if val.Elem().Kind() != reflect.Struct {
panic("StructValue has to be struct type")
}
}
return dna.String(val.Elem().String()).Replace(".", "").ToLowerCase() + "s"
}
func UTF16ToUTF8String(b []byte, o binary.ByteOrder) dna.String {
utf := make([]uint16, (len(b)+(2-1))/2)
for i := 0; i+(2-1) < len(b); i += 2 {
utf[i/2] = o.Uint16(b[i:])
}
if len(b)/2 < len(utf) {
utf[len(utf)-1] = utf8.RuneError
}
return dna.String(string(utf16.Decode(utf)))
}
func Post(url dna.String, bodyStr dna.String) (*Result, error) {
client := &http.Client{}
// dna.Log(bodyStr)
req, err := http.NewRequest("POST", url.String(), strings.NewReader(bodyStr.String()))
req.Header = DefaulHeader
req.Header.Add("Host", url.ToPrimitiveValue())
res, err := client.Do(req)
if err != nil {
return new(Result), err
}
var data []byte
var myErr error
switch res.Header.Get("Content-Encoding") {
case "gzip":
var reader io.ReadCloser
reader, err := gzip.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
case "deflate":
// Logv("sdsafsd")
reader, err := zlib.NewReader(res.Body)
if err != nil {
return new(Result), err
}
data, myErr = ioutil.ReadAll(reader)
reader.Close()
default:
data, myErr = ioutil.ReadAll(res.Body)
}
if myErr != nil {
return new(Result), myErr
}
res.Body.Close()
return NewResult(dna.Int(res.StatusCode), dna.String(res.Status), res.Header, dna.String(data)), nil
}
// InsertIgnore runs exactly the same as Insert.
// However if the insert value has already existed in a table,
// it does not return any error.
// It only returns an error if and only if other errors occur.
//
// The error format is the same as the one of sqlpg.DB.Insert()
func (db *DB) InsertIgnore(structValue interface{}) error {
err := db.Insert(structValue)
if err != nil {
if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) {
return nil
} else {
return err
}
} else {
return nil
}
}
// DecodePath decodes encoded string such as "MjAxMyUyRjExJTJGMDUlMkYwJTJGMiUyRjAyN2UzN2M4NDUwMWFlOTEwNGNkZjgyMDZjYWE4OTkzLm1wMyU3QzI="
// into its real path on server such as "/2013/11/05/0/2/027e37c84501ae9104cdf8206caa8993.mp3|2"
func DecodePath(encodedPath dna.String) dna.String {
ret, err := base64.StdEncoding.DecodeString(encodedPath.String())
if err == nil {
escape, err := url.QueryUnescape(string(ret))
if err == nil {
return dna.String(escape)
} else {
return ""
}
} else {
return ""
}
}
func atomicUpdate(errChannel chan bool, state *StateHandler) {
var err error
var c1 = make(chan error, 1)
it := state.GetItem().New()
state.IncreaseCid()
n := state.GetCid()
it.Init(n)
go func() {
c1 <- it.Fetch()
}()
select {
case err = <-c1:
// do nothing
case <-time.After(time.Second * time.Duration(TIMEOUT_SECS)):
strErr := dna.Sprintf("%v - Timeout after %vs at id :%v", state.GetTableName(), TIMEOUT_SECS, it.GetId())
err = errors.New(strErr.String())
}
state.AddNcCount(1)
// if it != nil {
// dna.LogStruct(it)
// }
if err != nil {
// dna.Log(err.Error())
HTTPERROR.Println(it.GetId(), err.Error())
errChannel <- true
} else {
// checking this code.Working only with 1st pattern
// The goroutine continues to run after DB closed so it will invoke an error
// state.InsertIgnore(it)
saveErr := it.Save(state.GetDb())
if saveErr != nil {
SQLERROR.Println(dna.String(saveErr.Error()))
}
errChannel <- false
}
state.AddNcCount(-1)
if state.IsComplete() == false {
switch state.GetPattern() {
case 2:
if state.GetCid() < state.GetRange().Last {
go atomicUpdate(errChannel, state)
}
default:
go atomicUpdate(errChannel, state)
}
}
}
func (sf *APISongFreaksTrack) ToSong() (*Song, error) {
if sferr := sf.HasError(); sferr != nil {
return nil, sferr
}
song := NewSong()
song.Id = sf.Track.Id
song.TrackGroupId = sf.Track.TrackGroupId
song.AMG = sf.Track.AMG
song.UrlSlug = sf.Track.UrlSlug
song.IsInstrumental = sf.Track.IsInstrumental
song.Viewable = sf.Track.Viewable
song.Duration = utils.ToSeconds(sf.Track.Duration)
song.Lyricid = sf.Track.LyricId
song.HasLrc = sf.Track.HasLrc
song.TrackNumber = sf.Track.TrackNumber
song.DiscNumber = sf.Track.DiscNumber
song.Title = sf.Track.Title
song.Rating = dna.IntArray{sf.Track.Rating.AverageRating.ToInt(), sf.Track.Rating.UserRating, sf.Track.Rating.TotalRatings}
song.Albumid = sf.Track.Album.Id
artistIds := dna.IntArray{}
artists := dna.StringArray{}
for _, artist := range sf.Track.Artists {
artistIds.Push(artist.Id)
artists.Push(artist.Name)
}
song.Artistids = artistIds
song.Artists = artists
if sf.Track.Lrc.Lines != nil && len(sf.Track.Lrc.Lines) > 0 {
lines, err := json.Marshal(sf.Track.Lrc.Lines)
if err == nil {
song.Lrc = dna.String(string(lines))
}
}
song.Link = sf.Track.Link
song.Lyric = sf.Track.Lyrics
if song.Lyric != "" {
song.HasLyric = true
}
song.Copyright = sf.Track.Copyright
song.Writer = sf.Track.Writer
song.SubmittedLyric = sf.Track.SubmittedLyric
song.Checktime = time.Now()
return song, nil
}
func getStringifiedSongUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
songUrls := []SongUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`128kbps`) == true:
songUrl := getSongUrl(val, "128kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`320kbps`) == true:
songUrl := getSongUrl(val, "320kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`32kbps`) == true:
songUrl := getSongUrl(val, "32kbps")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
songUrls = append(songUrls, *songUrl)
case val.Match(`500kbps`) == true:
songUrl := getSongUrl(val, "500kbps")
songUrl.Link = baseLink + "m4a/file-name.m4a"
songUrls = append(songUrls, *songUrl)
case val.Match(`Lossless`) == true:
songUrl := getSongUrl(val, "Lossless")
songUrl.Link = baseLink + "flac/file-name.flac"
songUrls = append(songUrls, *songUrl)
}
})
// http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac
// replace the link 500kps and lossless with available link, apply for registered user only
// and reduce the link length
var ret = dna.StringArray{}
for _, songUrl := range songUrls {
var br dna.String
if songUrl.Bitrate == "Lossless" {
br = "1411"
} else {
br = songUrl.Bitrate.Replace("kbps", "")
}
t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}
func getTSGM(data *dna.String, kind dna.String) dna.String {
var itemArr dna.StringArray
switch kind {
case "genres":
itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
case "styles":
itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
case "moods":
itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
case "themes":
itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
default:
panic("Wrong kind!!!")
}
if itemArr.Length() > 0 {
catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
var idArr []dna.StringArray
var id dna.Int = 0
name := val.RemoveHtmlTags("")
if kind == "moods" {
idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
} else {
idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
}
if len(idArr) > 0 {
id = idArr[0][1].ToInt()
}
return Category{id, name}
}).([]Category)
if len(categories) > 0 {
bCat, merr := json.Marshal(categories)
if merr != nil {
return "[]"
} else {
return dna.String(string(bCat))
}
} else {
return "[]"
}
} else {
return "[]"
}
}
func getAlbumDesc(album *Album) <-chan bool {
channel := make(chan bool, 1)
go func() {
link := "http://www.nhaccuatui.com/ajax/get-description?key=" + album.Key
result, err := http.Get(dna.String(link))
// Log(link)
if err == nil {
data := &result.Data
albumDesc := &AlbumDesc{}
errJson := json.Unmarshal([]byte(data.String()), albumDesc)
if errJson == nil {
album.Description = albumDesc.Data
}
}
channel <- true
}()
return channel
}
func GetAPISong(id dna.Int) (*APISong, error) {
var apisong = new(APISong)
link := "https://api.lyricfind.com/lyric.do?apikey=" + API_KEY + "&reqtype=default&trackid=amg:" + id.ToString().String() + "&output=json"
// result, err := GetProxy(dna.String(link), 0)
result, err := http.Get(dna.String(link))
if err == nil {
data := &result.Data
json.Unmarshal([]byte(*data), apisong)
switch apisong.Response.Code {
case 100, 101, 111, 102:
return apisong, nil
default:
return nil, errors.New(dna.Sprintf("LyricFind - Song ID: %v - %v %v", id, apisong.Response.Code, apisong.Response.Description).String())
}
return apisong, nil
} else {
return nil, err
}
}
// ForeachLine loops through every line a file.
// An anynomous input function has line, index as params
func ForeachLine(filePath dna.String, lineFunc func(dna.String, dna.Int)) {
var err error
var line []byte
f, err := os.Open(filePath.String())
if err != nil {
dna.Log("error opening file: %v\n", err)
os.Exit(1)
}
defer f.Close()
r := bufio.NewReaderSize(f, 4*1024)
i := 0
err = nil
for err == nil {
i += 1
line, _, err = r.ReadLine()
lineFunc(dna.String(string(line)), dna.Int(i))
}
}
func getAPIFullSongWithMetadata(id dna.Int) (*APIFullSong, error) {
var apisong = new(APIFullSong)
var link string
link = "http://api.lyricfind.com/metadata.do?apikey=" + METADATA_KEY + "&reqtype=metadata&trackid=amg:" + id.ToString().String() + "&displaykey=" + API_KEY + "&output=json"
result, err := http.Get(dna.String(link))
if err == nil {
data := &result.Data
json.Unmarshal([]byte(*data), apisong)
switch apisong.Response.Code {
case 100, 101, 111, 102:
return apisong, nil
default:
return nil, errors.New(dna.Sprintf("LyricFind - Song ID: %v - %v %v", id, apisong.Response.Code, apisong.Response.Description).String())
}
return apisong, nil
} else {
return nil, err
}
}
func (apiMovie *APIMovie) ToEpisode() *Episode {
episode := NewEpisode()
episode.MovieId = apiMovie.MovieId
episode.EpId = apiMovie.EpId
episode.Title = apiMovie.Title
episode.LinkPlayBackup = apiMovie.LinkPlayBackup
episode.Link = apiMovie.Link
episode.LinkPlayOther = apiMovie.LinkPlayOther
episode.SubtitleExt = dna.StringArray{apiMovie.SubtitleExt.Vietnamese.Source, apiMovie.SubtitleExt.English.Source}
episode.SubtitleExtSe = dna.StringArray{apiMovie.SubtitleExtSe.Vietnamese.Source, apiMovie.SubtitleExtSe.English.Source}
episode.Subtitle = dna.StringArray{apiMovie.Subtitle.Vietnamese.Source, apiMovie.Subtitle.English.Source}
episode.EpisodeId = apiMovie.Episode.ToInt()
episode.Audiodub = apiMovie.Audiodub.ToInt()
episode.Audio = apiMovie.Audio.ToInt()
season, err := json.Marshal(apiMovie.Season)
if err == nil {
episode.Season = dna.String(string(season))
}
return episode
}
func (qv *QueryValues) Encode() dna.String {
qv.mutex.Lock()
defer qv.mutex.Unlock()
if qv.LrcEnable == true {
qv.AddLrcKey()
} else {
qv.RemoveLrcKey()
}
if qv.VideosEnable == true {
qv.AddVideosKey()
} else {
qv.RemoveVideosKey()
}
if qv.CommentsEnable == true {
qv.AddCommentsKey()
} else {
qv.RemoveCommentsKey()
}
return dna.String(qv.Values.Encode())
}
//DecryptLRC returns LRC string from encrypted string.
//
//For example: Given a song id 2882720 with a key 9Fd4zVvPMIbf
//and an XML source file http://www.nhaccuatui.com/flash/xml?key1=8bfe9db992afaff5dc931dffca1b5c7b
//
//The encrypted lyric file url lies in <lyric> tags
// http://lrc.nct.nixcdn.com/2013/12/29/0/a/5/b/1388320374102.lrc
func DecryptLRC(data dna.String) (dna.String, error) {
// CODE_SPECIAL "M z s 2 d k v t u 5 o d u" separated by space
keyStr := irrcrpt(dna.Sprintf("%s", []byte{0x4d, 0x7a, 0x73, 0x32, 0x64, 0x6b, 0x76, 0x74, 0x75, 0x35, 0x6f, 0x64, 0x75}), 1)
keyStrInHex := hex.EncodeToString(keyStr.ToBytes())
keyStrInBytes, err := hex.DecodeString(keyStrInHex)
if err != nil {
return "", err
}
ret, err := hex.DecodeString(data.String())
if err != nil {
return "", err
}
cipher, err := rc4.NewCipher(keyStrInBytes)
if err != nil {
return "", err
} else {
cipher.XORKeyStream(ret, ret)
return dna.String(string(ret)), nil
}
}
func getStringifiedVideoUrls(urls dna.StringArray) dna.String {
var baseLink = dna.String("")
videoUrls := []VideoUrl{}
urls.ForEach(func(val dna.String, idx dna.Int) {
// dna.Log(val)
// Finding bitrate
switch {
case val.Match(`MV 360p`) == true:
songUrl := getVideoUrl(val, "360p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 480p`) == true:
songUrl := getVideoUrl(val, "480p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`MV 180p`) == true:
songUrl := getVideoUrl(val, "180p")
baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 720p`) == true:
songUrl := getVideoUrl(val, "720p")
songUrl.Link = baseLink + "m4a/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
case val.Match(`HD 1080p`) == true:
songUrl := getVideoUrl(val, "1080p")
songUrl.Link = baseLink + "flac/file-name.mp4"
videoUrls = append(videoUrls, *songUrl)
}
})
var ret = dna.StringArray{}
for _, videoUrl := range videoUrls {
t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"`
ret.Push(t)
}
// dna.Log(`{` + ret.Join(",") + `}`)
return `{` + ret.Join(",") + `}`
}