Beispiel #1
1
func getAnchorTagsData(data dna.String) dna.StringArray {
	return dna.StringArray(data.Split("|").Map(func(val dna.String, idx dna.Int) dna.String {
		return val.RemoveHtmlTags("").Trim()
	}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
		if val != "" {
			return true
		} else {
			return false
		}
	})
}
Beispiel #2
1
Datei: zi.go Projekt: olragon/dna
// Decrypt decodes a key into an integer ID
func Decrypt(key dna.String) dna.Int {
	return dna.ParseInt(dna.StringArray(key.Split("").Map(func(v dna.String, i dna.Int) dna.String {
		return b[a.IndexOf(v)]
	}).([]dna.String)).Join(""), 16)
}
Beispiel #3
0
// SelectMissingKeys accepts a table name as an input and a list of keys as a source.
// It returns a new list of keys that does not exist in the destination table
//
// 	* tblName : a table name
// 	* srcKeys : a source keys
// 	* db : a pointer to connected databased
// 	* Returns a new list of keys which are not from the specified table
//
// Notice: Only applied to a table having a column named "key".
// The column has to be indexed to ensure good performance
//
// The format of sql statement is:
//	with dna (key) as (values ('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('RvFDlckJB5QU'),('uIF7rwd5wo4p'),('Kveukbhre1ry'),('oJ1lzAlKwJX6'),('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('hfhtyMdywMau'),('PpZuccjYqy1b'))
//	select key from dna where key not in
//	(select key from nctalbums where key in ('43f3HhhU6DGV','uFfgQhKbwAfN','RvFDlckJB5QU','uIF7rwd5wo4p','Kveukbhre1ry','oJ1lzAlKwJX6','43f3HhhU6DGV','uFfgQhKbwAfN','hfhtyMdywMau','PpZuccjYqy1b'))
func SelectMissingKeys(tblName dna.String, srcKeys *dna.StringArray, db *sqlpg.DB) (*dna.StringArray, error) {
	if srcKeys.Length() > 0 {
		val := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
			return `('` + val + `')`
		}).([]dna.String))
		val1 := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
			return `'` + val + `'`
		}).([]dna.String))
		selectStmt := "with dna (key) as (values " + val.Join(",") + ") \n"
		selectStmt += "select key from dna where key not in \n(select key from " + tblName + " where key in (" + val1.Join(",") + "))"
		keys := &[]dna.String{}
		err := db.Select(keys, selectStmt)
		switch {
		case err != nil:
			return nil, err
		case err == nil && keys != nil:
			slice := dna.StringArray(*keys)
			return &slice, nil
		case err == nil && keys == nil:
			return &dna.StringArray{}, nil
		default:
			panic("Default case triggered. Case is not expected. Cannot select non existed keys")
		}
	} else {
		return nil, errors.New("Empty input array")
	}
}
Beispiel #4
0
//GetAlbumFromAPI gets a album from API. It does not get content from main site.
func GetAlbumFromAPI(id dna.Int) (*Album, error) {
	var album *Album = NewAlbum()
	album.Id = id
	apialbum, err := GetAPIAlbum(id)
	if err != nil {
		return nil, err
	} else {
		if apialbum.Response.MsgCode == 1 {
			if GetKey(apialbum.Id) != GetKey(album.Id) {
				errMes := dna.Sprintf("Resulted key and computed key are not match. %v =/= %v , id: %v =/= %v", GetKey(apialbum.Id), GetKey(album.Id), id, apialbum.Id)
				panic(errMes.String())
			}

			album.Title = apialbum.Title
			album.Artists = dna.StringArray(apialbum.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).SplitWithRegexp(",").Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})

			album.Topics = dna.StringArray(apialbum.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			album.Plays = apialbum.Plays
			// album.Songids
			// album.Nsongs
			// album.EncodedKey
			// album.Coverart
			// album.DateCreated
			album.YearReleased = apialbum.YearReleased
			album.Description = apialbum.Description.RemoveHtmlTags("")

			album.ArtistIds = apialbum.ArtistIds.Split(",").ToIntArray()
			album.IsAlbum = apialbum.IsAlbum
			album.IsHit = apialbum.IsHit
			album.IsOfficial = apialbum.IsOfficial
			album.Likes = apialbum.Likes
			album.StatusId = apialbum.StatusId
			album.Comments = apialbum.Comments
			album.Checktime = time.Now()
			return album, nil
		} else {
			return nil, errors.New("Message code invalid " + apialbum.Response.MsgCode.ToString().String())
		}
	}
}
Beispiel #5
0
// Fill writes all fields found APIUser instance to a User variable.
func (user *User) Fill(apiUser *APIUser) {
	user.Id = apiUser.Id
	user.Email = apiUser.Email
	user.Mobile = dna.StringArray(apiUser.Mobile)
	user.ProfilePoint = apiUser.ProfilePoint
	user.StatusWall = apiUser.StatusWall
	user.DisplayName = apiUser.DisplayName.Trim()
	user.Point = apiUser.Point
	user.Username = apiUser.Username
	user.Avatar = apiUser.Avatar
	user.BirthdayType = apiUser.BirthdayType
	user.VipTotal = apiUser.Vip.Total
	user.VipBlock = apiUser.Vip.Block
	user.VipAvatar = apiUser.Vip.AvatarVip
	user.IsNull = apiUser.Benull
	user.Gender = apiUser.Gender
	user.Status = apiUser.Status
	user.Birthday = apiUser.Birthday
	user.FeedWriteWallAll = apiUser.Feed.WriteWallAll
	user.FeedViewWallAll = apiUser.Feed.ViewWallAll
	user.CoverUrl = apiUser.CoverUrl
	user.GoogleId = apiUser.GoogleId
	user.YahooId = apiUser.YahooId
	user.FriendTotal = apiUser.Friend.Total
	user.FriendBlock = apiUser.Friend.Block
	user.FriendAvatar = apiUser.Avatar
}
Beispiel #6
0
func (apiSong *APIFullSong) Save(db *sqlpg.DB) error {

	var queries = dna.StringArray{}
	var err error

	// Getting artist queries
	artists := apiSong.ToArtists()
	for _, artist := range artists {
		queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(artist), artist, "id", artist.Id, false))
	}

	// Getting album query
	album := apiSong.ToAlbum()
	queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(album), album, "id", album.Id, false))

	// Getting song query
	song := apiSong.ToSong()
	queries.Push(sqlpg.GetInsertStatement(sqlpg.GetTableName(song), song, false))

	for _, query := range queries {
		_, err = db.Exec(query.String())
	}

	if err != nil {
		errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
			return "$$$error$$$" + val + "$$$error$$$"
		}).([]dna.String))
		return errors.New(err.Error() + errQueries.Join("\n").String())
	} else {
		return nil
	}

}
Beispiel #7
0
Datei: zi.go Projekt: olragon/dna
func getCipherText(id dna.Int, tailArray dna.IntArray) dna.String {
	rand.Seed(time.Now().UnixNano())
	return dna.StringArray(dna.IntArray{1, 0, 8, 0, 10}.Concat((id - 307843200).ToString().Split("").ToIntArray()).Concat(tailArray).Map(
		func(v dna.Int, i dna.Int) dna.String {
			return c[v].Split("")[rand.Intn(len(c[v]))]
		}).([]dna.String)).Join("")
}
Beispiel #8
0
// SelectMissingIds accepts a table name as an input and a list of ids as a source.
// It returns a new list of ids that does not exist in the destination table
//
// 	* tblName : a table name
// 	* srcIds : a source ids
// 	* db : a pointer to connected databased
// 	* Returns a new list of ids which are not from the specified table
//
// The format of sql statement is:
// 	WITH dna (id) AS (VALUES (5),(6),(7),(8),(9))
// 	SELECT id FROM dna WHERE NOT EXISTS
// 	(SELECT 1 from ziartists WHERE id=dna.id)
func SelectMissingIds(tblName dna.String, srcIds *dna.IntArray, db *sqlpg.DB) (*dna.IntArray, error) {

	if srcIds.Length() > 0 {

		val := dna.StringArray(srcIds.Map(func(val dna.Int, idx dna.Int) dna.String {
			return "(" + val.ToString() + ")"
		}).([]dna.String))
		selectStmt := "with dna (id) as (values " + val.Join(",") + ") \n"
		selectStmt += "SELECT id FROM dna WHERE NOT EXISTS\n (SELECT 1 from " + tblName + "  WHERE id=dna.id)"
		ids := &[]dna.Int{}
		err := db.Select(ids, selectStmt)
		switch {
		case err != nil:
			return nil, err
		case err == nil && ids != nil:
			slice := dna.IntArray(*ids)
			return &slice, nil
		case err == nil && ids == nil:
			return &dna.IntArray{}, nil
		default:
			panic("Default case triggered. Case is not expected. Cannot select non existed ids")
		}
	} else {
		return nil, errors.New("Empty input array")
	}

}
Beispiel #9
0
// splitAndTruncateArtists splits stringarray by the key "feat:"
// and filter only string elements not equal to empty string.
func splitAndTruncateArtists(artists dna.StringArray) dna.StringArray {
	return dna.StringArray(artists.SplitWithRegexp("feat:").Map(func(val dna.String, idx dna.Int) dna.String {
		return val.Trim()
	}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
		if val != "" {
			return true
		} else {
			return false
		}
	})
}
Beispiel #10
0
func getGenresFromDesc(desc dna.String) dna.StringArray {
	var ret dna.StringArray
	genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
	// "Released:" found in album id: 836258
	if genres.Length() > 0 {
		ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
			return val.ReplaceWithRegexp(":", "").Trim()
		}).([]dna.String))
		if ret.Length() == 1 {
			arr := dna.StringArray{}
			if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
				arr.Push("Korean Pop")
				arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
				ret = arr
			}
		}
	}
	return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
Beispiel #11
0
// ExecQueriesInTransaction executes queries in a transaction.
// If one statement fails, the whole queries cannot commit.
//
// The returned error is nil if there is no error.
// If an error occurs, each statement will be enclosed in
// format $$$error$$$.
// 	$$$error$$$ Your Custom Query $$$error$$$
//
// This function is seen in songfreaks and allmusic sites.
func ExecQueriesInTransaction(db *DB, queries *dna.StringArray) error {
	var err error
	globalSqlTransactoNo += 1
	// tx, err := db.Begin()
	// if err != nil {
	// 	dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not create transaction\n")
	// }

	for idx, query := range *queries {
		_, err = db.Exec(query.String())
		// _, err = tx.Exec(query.String())
		if err != nil {
			dna.Log(dna.Sprintf("DNAError: Query series No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the statement"))
		}
		// stmt, err := tx.Prepare(query.String())
		// if err != nil {
		// 	dna.Log(dna.Sprintf("DNAError Transaction No: %v - %v - %v - %v \n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not prepare"))
		// } else {
		// 	_, err = stmt.Exec()
		// 	if err != nil {
		// 		dna.Log(dna.Sprintf("DNAError: Transaction No: %v - %v - %v - %v\n", dna.Sprintf("%v", globalSqlTransactoNo), idx, err.Error(), "Could not execute the prepared statement"))
		// 	}

		// 	err = stmt.Close()
		// 	if err != nil {
		// 		dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not close\n")
		// 	}
		// }
	}
	// err = tx.Commit()
	// if err != nil {
	// 	dna.Log("Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo).String() + err.Error() + " Could not commit transaction\n")
	// }

	if err != nil {
		errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
			return "Transaction No:" + dna.Sprintf("%v", globalSqlTransactoNo) + " $$$error$$$" + val + "$$$error$$$"
		}).([]dna.String))
		return errors.New(err.Error() + errQueries.Join("\n").String())
	} else {
		return nil
	}
}
Beispiel #12
0
func refineAuthorsOrArtists(str dna.String) dna.StringArray {
	tmp := str.ToStringArray().SplitWithRegexp(` / `).SplitWithRegexp(` - `).SplitWithRegexp(` – `)
	tmp = tmp.SplitWithRegexp(`, `).SplitWithRegexp(` ft `).SplitWithRegexp(` feat `).SplitWithRegexp(` ft. `)
	tmp = tmp.SplitWithRegexp(` feat. `).SplitWithRegexp(` Feat. `).SplitWithRegexp(` Ft. `)
	tmp = tmp.SplitWithRegexp(` & `).SplitWithRegexp(` vs. `).SplitWithRegexp(`- `).SplitWithRegexp(` & `)
	tmp = dna.StringArray(tmp.Map(func(val dna.String, idx dna.Int) dna.String {
		rv := val.Replace(`Đang Cập Nhật...`, ``).Replace(`Đang Cập Nhật (QT)`, ``)
		rv = rv.Replace(`Đang Cập Nhật (VN)`, ``).Replace(`Nhạc Phim QT`, `Nhạc Phim Quốc Tế`)
		rv = rv.Replace(`Đang cập nhật`, ``).Replace(`Nhiều Ca Sỹ`, `Various Artists`)
		return rv
	}).([]dna.String)).Filter(func(val dna.String, idx dna.Int) dna.Bool {
		if val != "" {
			return true
		} else {
			return false
		}
	})
	return tmp
}
Beispiel #13
0
func getSongFormats(song *Song) <-chan bool {

	channel := make(chan bool, 1)
	go func() {
		link := "http://download.chiasenhac.com/google-bot~" + song.Id.ToString() + "_download.html"
		result, err := http.Get(link)
		// dna.Log(link)
		if err == nil {
			data := &result.Data
			urlsArr := data.FindAllString(`<a href.+(Link|Mobile) Download.+`, -1)
			if urlsArr.Length() > 0 {
				// dna.Log(urlsArr.Length())
				song.Formats, song.Type = GetFormats(urlsArr)
				// dna.Log(song.Formats)
			}

			topicsArr := data.FindAllStringSubmatch(`<div class="plt-text">(.+)`, 1)
			if len(topicsArr) > 0 {
				topics := topicsArr[0][1].RemoveHtmlTags("").Trim().Split(`-&gt;`)
				topics.Pop()
				song.Topics = dna.StringArray(topics.Map(func(val dna.String, idx dna.Int) dna.String {
					return val.Replace("...", "").Replace("Nước khác", "Nhạc Các Nước Khác").Replace("Âu, Mỹ", "Âu Mỹ").Trim().Title()
				}).([]dna.String)).SplitWithRegexp(`, `).Unique()
			}

			hrefArr := data.FindAllString(`Download: <a.+`, 1)
			if hrefArr.Length() > 0 {
				song.Href = "http://chiasenhac.com/" + hrefArr[0].GetTagAttributes("href").Trim()
				titleArtists := hrefArr[0].Replace("Download:", "").Trim().RemoveHtmlTags("").Split(" - ")
				song.Artists = refineAuthorsOrArtists(titleArtists[titleArtists.Length()-1])
				titleArtists.Pop()
				song.Title = titleArtists.Join(" - ")

				<-getSongFromMainPage(song)

			}

			dlArr := data.FindAllStringSubmatch(`([0-9]+) downloads`, 1)
			if len(dlArr) > 0 {
				song.Downloads = dlArr[0][1].Trim().Replace(".", "").ToInt()
			}

			dateCreatedArr := data.FindAllStringSubmatch(`<img src="images/tain5.gif" title="Upload at (.+?)" />`, 1)
			if len(dateCreatedArr) > 0 {
				tmp := dateCreatedArr[0][1].Trim()
				switch {
				case tmp.Match(`Hôm qua`) == true:
					song.DateCreated = time.Now().AddDate(0, 0, -1)
				case tmp.Match(`Hôm nay`) == true:
					song.DateCreated = time.Now()
				default:
					val := tmp.ReplaceWithRegexp(`(\d+)/(\d+)/(\d+) (\d+):(\d+)`, "${3}-${2}-${1}T${4}:${5}:00Z")
					parsedDate, err := time.Parse(time.RFC3339, val.String())
					if err == nil {
						song.DateCreated = parsedDate
					}
				}
			}

		}
		channel <- true

	}()
	return channel
}
Beispiel #14
0
func getVideoFromMainPage(video *Video) <-chan bool {
	channel := make(chan bool, 1)
	go func() {
		link := "http://nhacso.net/xem-video/google-bot." + GetKey(video.Id) + "=.html"
		// Log(link)
		result, err := http.Get(link)
		if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
			data := &result.Data
			temp := data.FindAllString(`(?mis)<p class="title_video.+Đăng bởi`, 1)
			if temp.Length() > 0 {
				title := temp[0].FindAllString(`<h1 class="title">.+</h1>`, 1)
				if title.Length() > 0 {
					video.Title = title[0].RemoveHtmlTags("").Trim()
				}

				if temp[0].Match(`official`) {
					video.Official = 1
				}

				artists := temp[0].FindAllString(`<h2>.+</h2>`, -1)
				if artists.Length() > 0 {
					video.Artists = dna.StringArray(artists.Map(func(val dna.String, idx dna.Int) dna.String {
						return val.RemoveHtmlTags("").Trim()
					}).([]dna.String)).SplitWithRegexp(` / `).Unique()
				}

			}
			topics := data.FindAllString(`<li><a href="http://nhacso.net/the-loai-video/.+</a></li>`, 1)
			if topics.Length() > 0 {
				video.Topics = topics[0].RemoveHtmlTags("").ToStringArray().SplitWithRegexp(` - `).SplitWithRegexp(`/`)
			}

			plays := data.FindAllString(`<span>.+</span><ins>&nbsp;lượt xem</ins>`, 1)
			if plays.Length() > 0 {
				video.Plays = plays[0].GetTags("span")[0].RemoveHtmlTags("").Trim().Replace(".", "").ToInt()
			}

			thumbLink := data.FindAllString(`poster="(.+)" src="(.+)" data`, 1)
			if thumbLink.Length() > 0 {
				video.Thumbnail = thumbLink[0].FindAllStringSubmatch(`poster="(.+?)" `, 1)[0][1]
				video.Link = thumbLink[0].FindAllStringSubmatch(`src="(.+?)" `, 1)[0][1]
				if video.Link != "" {
					ts := video.Link.FindAllStringSubmatch(`([0-9]+)_`, 1)
					if len(ts) > 0 {
						secs := float64(ts[0][1].ToInt()) * math.Pow10(13-len(ts[0][1]))
						// Log(secs)
						video.DateCreated = dna.Float(secs / 1000).ToInt().ToTime()
					}

				}
			}
			producerid := data.FindAllStringSubmatch(`getProducerByListIds\('(\d+)', '#producer_'\);`, 1)
			if len(producerid) > 0 {
				video.Producerid = producerid[0][1].ToInt()
			}
		}
		channel <- true
	}()
	return channel

}
Beispiel #15
0
// getVideoFromMainPage returns song from main page
func getVideoFromMainPage(video *Video) <-chan bool {

	channel := make(chan bool, 1)
	go func() {
		link := "http://mp3.zing.vn/video-clip/google-bot/" + GetKey(video.Id) + ".html"
		result, err := http.Get(link)
		// Log(link)
		// Log(result.Data)
		data := &result.Data
		// dna.Log(data.Match("<title>Thông báo</title>"))
		if err == nil && !data.Match("<title>Thông báo</title>") {

			topicsArr := data.FindAllStringSubmatch(`Thể loại:(.+)\|`, -1)
			if len(topicsArr) > 0 {
				video.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(` / `).Unique()
			}

			playsArr := data.FindAllStringSubmatch(`Lượt xem:(.+)</p>`, -1)
			if len(playsArr) > 0 {
				video.Plays = playsArr[0][1].Trim().Replace(".", "").ToInt()
			}

			titleArr := data.FindAllStringSubmatch(`<h1 class="detail-title">(.+?)</h1>`, -1)
			if len(titleArr) > 0 {
				video.Title = titleArr[0][1].RemoveHtmlTags("").Trim()
			}

			artistsArr := data.FindAllStringSubmatch(`<h1 class="detail-title">.+(<a.+)`, -1)
			if len(artistsArr) > 0 {
				video.Artists = dna.StringArray(artistsArr[0][1].RemoveHtmlTags("").Trim().Split(" ft. ").Unique().Map(func(val dna.String, idx dna.Int) dna.String {
					return val.Trim()
				}).([]dna.String))
			}

			thumbnailArr := data.FindAllString(`<meta property="og:image".+`, -1)
			if thumbnailArr.Length() > 0 {
				video.Thumbnail = thumbnailArr[0].GetTagAttributes("content")
				// datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
				// if len(datecreatedArr) > 0 {
				// 	// Log(int64(datecreatedArr[0][1].ToInt()))
				// 	video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
				// }
				datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
				if len(datecreatedArr) > 0 && datecreatedArr[0][1].Length() > 6 {
					video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
				} else {
					dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
					if len(dateCreatedArr) > 0 {
						year := dateCreatedArr[0][1].FindAllStringSubmatch(`^(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
						month := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
						day := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
						video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)

					}
				}
			}

			video.Link = video.GetDirectLink(Resolution360p)

			lyricArr := data.FindAllStringSubmatch(`(?mis)<p class="_lyricContent.+</span></span>(.+?)<p class="seo">`, -1)
			if len(lyricArr) > 0 {
				video.Lyric = lyricArr[0][1].ReplaceWithRegexp(`(?mis)<p class="seo">.+`, "").Trim().Replace("</p>\r\n\t</div>\r\n\t\t</div>", "").Trim()
			}

		}
		channel <- true

	}()
	return channel
}
Beispiel #16
0
//GetVideoFromAPI gets a video from API. It does not get content from main site.
func GetVideoFromAPI(id dna.Int) (*Video, error) {
	var video *Video = NewVideo()
	video.Id = id
	apivideo, err := GetAPIVideo(id)
	if err != nil {
		return nil, err
	} else {
		if apivideo.Response.MsgCode == 1 {
			video.Title = apivideo.Title
			video.Artists = dna.StringArray(apivideo.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			video.Topics = dna.StringArray(apivideo.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			video.Plays = apivideo.Plays
			video.Thumbnail = "http://image.mp3.zdn.vn/" + apivideo.Thumbnail
			// video.Lyric =

			datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`_([0-9]+)\..+$`, -1)
			if len(datecreatedArr) > 0 && datecreatedArr[0][1].Length() > 6 {
				video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()), 0)
			} else {
				dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
				if len(dateCreatedArr) > 0 {
					year := dateCreatedArr[0][1].FindAllStringSubmatch(`^(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
					month := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
					day := dateCreatedArr[0][1].FindAllStringSubmatch(`^\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
					video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)

				}
			}
			video.Plays = apivideo.Plays
			video.ArtistIds = apivideo.ArtistIds.Split(",").ToIntArray()
			flags := 0
			for key, val := range apivideo.Source {
				video.Link = val
				switch {
				case key == "240" && val != "":
					flags = flags | LRe240p
				case key == "360" && val != "":
					flags = flags | LRe360p
				case key == "480" && val != "":
					flags = flags | LRe480p
				case key == "720" && val != "":
					flags = flags | LRe720p
				case key == "1080" && val != "":
					flags = flags | LRe1080p
				}
			}
			video.ResolutionFlags = dna.Int(flags)
			video.Duration = apivideo.Duration
			video.Likes = apivideo.Likes
			video.StatusId = apivideo.StatusId
			video.Comments = apivideo.Comments
			video.Checktime = time.Now()
			return video, nil
		} else {
			return nil, errors.New("Message code invalid " + apivideo.Response.MsgCode.ToString().String())
		}
	}
}
Beispiel #17
0
Datei: zi.go Projekt: olragon/dna
// Encrypt encodes an integer ID into a key
func Encrypt(id dna.Int) dna.String {
	return dna.StringArray(id.ToHex().Split("").Map(
		func(v dna.String, i dna.Int) dna.String {
			return a[b.IndexOf(v)]
		}).([]dna.String)).Join("")
}
Beispiel #18
0
Datei: ns.go Projekt: olragon/dna
// This function will encode the Id of nhacso into cipher text.
func Encrypt(id dna.Int) dna.String {
	return dna.StringArray(id.ToString().Split("").Map(
		func(v dna.String, i dna.Int) dna.String {
			return ns[6-i][v.ToInt()]
		}).([]dna.String)).Join("")
}
Beispiel #19
0
//GetSongFromAPI gets a song from API. It does not get content from main site.
func GetSongFromAPI(id dna.Int) (*Song, error) {
	var song *Song = NewSong()
	song.Id = id

	asong, err := GetAPISong(id)
	if err != nil {
		return nil, err
	} else {
		if asong.Response.MsgCode == 1 {
			if asong.Key != GetKey(song.Id) {
				panic("Resulted key and computed key are not match.")
			}
			song.Key = asong.Key
			song.Title = asong.Title
			song.Artists = dna.StringArray(asong.Artists.Split(" , ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).SplitWithRegexp(",").Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			song.ArtistIds = asong.ArtistIds.Split(",").ToIntArray()
			song.Authors = asong.Authors.Split(", ").Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			song.Plays = asong.Plays
			song.Topics = dna.StringArray(asong.Topics.Split(", ").Map(func(val dna.String, idx dna.Int) dna.String {
				return val.Trim()
			}).([]dna.String)).SplitWithRegexp(" / ").Unique().Filter(func(v dna.String, i dna.Int) dna.Bool {
				if v != "" {
					return true
				} else {
					return false
				}
			})
			// song.Link
			// song.Path
			// song.Lyric
			// song.DateCreated
			// song.Checktime = time.Time{}
			if asong.Video.Id > 0 {
				song.VideoId = asong.Video.Id + 307843200
			}
			if asong.AlbumId > 0 {
				song.AlbumId = asong.AlbumId + 307843200
			}

			song.IsHit = asong.IsHit
			song.IsOfficial = asong.IsOfficial
			song.DownloadStatus = asong.DownloadStatus
			song.Copyright = asong.Copyright
			flags := 0
			for key, val := range asong.Source {
				switch {
				case key == "128" && val != "":
					flags = flags | LBr128
				case key == "320" && val != "":
					flags = flags | LBr320
				case key == "lossless" && val != "":
					flags = flags | LBrLossless
				}
			}
			song.BitrateFlags = dna.Int(flags)
			song.Likes = asong.Likes
			song.Comments = asong.Comments
			song.Thumbnail = asong.Thumbnail
			song.Checktime = time.Now()
			return song, nil
		} else {
			return nil, errors.New("Message code invalid " + asong.Response.MsgCode.ToString().String())
		}

	}

}
Beispiel #20
0
func getAlbumFromMainPage(album *Album) <-chan bool {
	channel := make(chan bool, 1)
	go func() {
		link := "http://nhacso.net/nghe-album/ab." + GetKey(album.Id) + ".html"
		// Log(link)
		result, err := http.Get(link)
		if err == nil && !result.Data.Match("Rất tiếc, chúng tôi không tìm thấy thông tin bạn yêu cầu!") {
			data := &result.Data
			temp := data.FindAllString(`(?mis)class="intro_album_detail.+id="divPlayer`, 1)[0]
			if !temp.IsBlank() {
				title := temp.GetTags("strong")[0]
				if !title.IsBlank() {
					album.Title = title.RemoveHtmlTags("")
				}
				artists := temp.FindAllString(`strong.+`, 1)[0]
				if !artists.IsBlank() {
					album.Artists = artists.ReplaceWithRegexp(`^.+>`, "").ToStringArray().SplitWithRegexp(`\|\|`).SplitWithRegexp(` / `).SplitWithRegexp(" - ")
					artistid := artists.FindAllString(`\d+\.html`, 1)
					if artistid.Length() > 0 {
						album.Artistid = artistid[0].ReplaceWithRegexp(`\.html`, "").ToInt()
					}
				}

				// get multiple artists, overwrite the artists var above
				newArs := temp.FindAllString(`<p><span>.+?</span></p>`, -1)
				if newArs.Length() > 0 {
					album.Artists = dna.StringArray(newArs.Map(func(val dna.String, idx dna.Int) dna.String {
						return val.RemoveHtmlTags("").Trim()
					}).([]dna.String)).SplitWithRegexp(`\|\|`).SplitWithRegexp(` / `).SplitWithRegexp(" - ")
				}

				coverart := temp.GetTags(`img`)[0]
				if !coverart.IsBlank() {
					album.Coverart = coverart.GetTagAttributes("src")
				}
			}

			description := data.FindAllString(`<p class="desc".+?</p>`, 1)
			if description.Length() > 0 {
				album.Description = description[0].Trim().Replace("<br>", "\n").RemoveHtmlTags("")
				if album.Description.Match(`thưởng thức nhạc chất lượng cao và chia sẻ cảm xúc với bạn bè tại Nhacso.net`) {
					album.Description = ""
				}
				album.Genres = getGenresFromDesc(album.Description)
				album.Label = getLabelFromDesc(album.Description)
			}

			topics := data.FindAllString(`<li class="bg">.+</li>`, 1)[0]
			if !topics.IsBlank() {
				album.Topics = topics.RemoveHtmlTags("").ToStringArray().SplitWithRegexp(" / ").SplitWithRegexp(" - ")
			}

			songids := data.FindAllString(`songid_\d+`, -1)
			if songids.Length() > 0 {
				songids.ForEach(func(value dna.String, index dna.Int) {
					album.Songids.Push(value.ReplaceWithRegexp(`songid_`, "").ToInt())
				})
			}
		}
		channel <- true
	}()
	return channel
}
Beispiel #21
0
// getSongFromMainPage returns song from main page
func getSongFromMainPage(song *Song) <-chan bool {

	channel := make(chan bool, 1)
	go func() {
		link := "http://mp3.zing.vn/bai-hat/google-bot/" + song.Key + ".html"
		result, err := http.Get(link)
		// Log(link)
		// Log(result.Data)
		if err == nil {
			data := &result.Data

			titleArr := data.FindAllStringSubmatch(`<h1 class="detail-title">(.+?)</h1>`, -1)
			if len(titleArr) > 0 {
				song.Title = titleArr[0][1].Trim()
			}

			artistsArr := data.FindAllStringSubmatch(`<h1 class="detail-title">.+</h1><span>-</span>(.+)`, -1)
			if len(artistsArr) > 0 {
				artists := artistsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp("ft. ")
				song.Artists = dna.StringArray(artists.Map(func(val dna.String, i dna.Int) dna.String {
					return val.Trim()
				}).([]dna.String))
			}

			playsArr := data.FindAllStringSubmatch(`Lượt nghe: (.+)</p>`, -1)
			if len(playsArr) > 0 {
				song.Plays = playsArr[0][1].Replace(".", "").ToInt()
			}

			authorsArr := data.FindAllStringSubmatch(`Sáng tác:(.+?)\|`, -1)
			if len(authorsArr) > 0 {
				authors := authorsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(" / ").SplitWithRegexp(" & ")
				song.Authors = dna.StringArray(authors.Map(func(val dna.String, idx dna.Int) dna.String {
					switch {
					case val == "Đang Cập Nhật":
						return ""
					case val == "Đang cập nhật":
						return ""
					default:
						return val
					}
				}).([]dna.String)).Filter(func(v dna.String, i dna.Int) dna.Bool {
					if v != "" {
						return true
					} else {
						return false
					}
				})
			}

			topicsArr := data.FindAllStringSubmatch(`Thể loại:(.+?)\|`, -1)
			if len(topicsArr) > 0 {
				song.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ").SplitWithRegexp(" / ").Unique()
			}

			lyricsArr := data.FindAllStringSubmatch(`(?mis)<p class="_lyricContent.+</span></span>(.+?)</p>`, -1)
			if len(lyricsArr) > 0 {
				song.Lyric = lyricsArr[0][1].Trim()
			}

		}
		channel <- true

	}()
	return channel
}
Beispiel #22
0
// getVideoFromMainPage returns video from main page
func getVideoFromMainPage(video *Video) <-chan bool {
	channel := make(chan bool, 1)
	go func() {
		link := "http://www.nhaccuatui.com/video/google-bot." + video.Key + ".html"
		result, err := http.Get(link)
		if err == nil {
			data := &result.Data

			topicsArr := data.FindAllStringSubmatch(`<strong>Thể loại</strong></p>[\n\t\r]+(.+)`, 1)
			if len(topicsArr) > 0 {
				video.Topics = topicsArr[0][1].RemoveHtmlTags("").Trim().Split(", ")
			}

			linkkeyArr := data.FindAllStringSubmatch(`"flashPlayer", ".+", "(.+?)"`, 1)
			if len(linkkeyArr) > 0 {
				video.LinkKey = linkkeyArr[0][1].Trim()
			}

			relativesArr := data.FindAllString("<a href=.+[\n\t\r]+.+video_relative", -1)
			video.Relatives = dna.StringArray(relativesArr.Map(func(val dna.String, idx dna.Int) dna.String {
				valArr := val.GetTagAttributes("href").Replace(".html", "").ReplaceWithRegexp(`^.+\.`, "")
				return valArr
			}).([]dna.String))

			// new version in June 2013 does not support lyric of videos

			// idArr := data.FindAllStringSubmatch(`value="(.+)" id="inpHiddenId"`, 1)
			// if len(idArr) > 0 {
			// 	video.Id = idArr[0][1].ToInt()
			// }

			// titleArr := data.FindAllStringSubmatch(`<h1 itemprop="name">(.+?)</h1>`, 1)
			// if len(titleArr) > 0 {
			// 	video.Title = titleArr[0][1].Trim().SplitWithRegexp(" - ", 2)[0].Trim()
			// }

			// artistsArr := data.FindAllStringSubmatch(`<h1 itemprop="name">(.+?)</h1>`, 1)
			// if len(artistsArr) > 0 {
			// 	artists := artistsArr[0][1].RemoveHtmlTags("").SplitWithRegexp(" - ", 2)
			// 	if artists.Length() == 2 {
			// 		video.Artists = artists[1].Split(", ").Filter(func(v dna.String, i dna.Int) dna.Bool {
			// 			if v != "" {
			// 				return true
			// 			} else {
			// 				return false
			// 			}
			// 		})
			// 	}
			// }

			// durationArr := data.FindAllStringSubmatch(`<meta itemprop="duration" content="(.+)" />`, 1)
			// if len(durationArr) > 0 {
			// 	durationF := durationArr[0][1].Replace("PT", "").Replace("H", "h").Replace("M", "m").Replace("S", "s")
			// 	duration, perr := time.ParseDuration(durationF.String())
			// 	if perr == nil {
			// 		video.Duration = dna.Float(duration.Seconds()).Round()
			// 	}
			// }

			// thumbArr := data.FindAllString(`<link rel="image_src".+`, 1)
			// if thumbArr.Length() > 0 {
			// 	video.Thumbnail = thumbArr[0].GetTagAttributes("href").Trim()
			// 	datecreatedArr := video.Thumbnail.FindAllStringSubmatch(`/([0-9]+)_[0-9]+\..+$`, -1)
			// 	if len(datecreatedArr) > 0 {
			// 		// Log(int64(datecreatedArr[0][1].ToInt()))
			// 		video.DateCreated = time.Unix(int64(datecreatedArr[0][1].ToInt()/1000), 0)
			// 	} else {
			// 		dateCreatedArr := video.Thumbnail.FindAllStringSubmatch(`/?(\d{4}/\d{2}/\d{2})`, -1)
			// 		if len(dateCreatedArr) > 0 {
			// 			year := dateCreatedArr[0][1].FindAllStringSubmatch(`(\d{4})/\d{2}/\d{2}`, -1)[0][1].ToInt()
			// 			month := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/(\d{2})/\d{2}`, -1)[0][1].ToInt()
			// 			day := dateCreatedArr[0][1].FindAllStringSubmatch(`\d{4}/\d{2}/(\d{2})`, -1)[0][1].ToInt()
			// 			video.DateCreated = time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.UTC)
			// 		}
			// 	}
			// }

			// Find params for the number of video plays
			// itemIdArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('(.+?)'.+`, 1)
			// timeArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '(.+?)'.+\);`, 1)
			// signArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '.+?', '(.+?)'.+;`, 1)
			// typeArr := data.FindAllStringSubmatch(`NCTWidget.hitCounter\('.+?', '.+?', '.+?', "(.+?)"\);`, 1)
			// if len(itemIdArr) > 0 && len(timeArr) > 0 && len(signArr) > 0 && len(typeArr) > 0 {
			// 	// boday has post form:
			// 	// item_id=2870710&time=1389009424631&sign=2499ab08f6662842a02b06aad603d8ab&type=video
			// 	body := dna.Sprintf(`item_id=%v&time=%v&sign=%v&type=%v`, itemIdArr[0][1], timeArr[0][1], signArr[0][1], typeArr[0][1])
			// 	getVideoPlays(video, body)
			// 	video.Type = typeArr[0][1].Trim()
			// 	if video.Type == "video" {
			// 		video.Type = "mv"
			// 	}
			// 	video.Id = itemIdArr[0][1].ToInt()
			// }

			// GetRelevantPortions(&result.Data)
		}
		channel <- true
	}()
	return channel
}