예제 #1
0
파일: sqlrecover.go 프로젝트: olragon/dna
// RecoverSQLLogError re-executes failed sql queries in sql error log file from specified path.
// It returns the number of failed -reexec queries, and new failed
// queries will be written to the file
//
// The format of error file is:
// 	Error description - $$$error$$$SQL_QUERY$$$error$$$
// Therefore only get statements enclosed by special `$$$error$$$`
func RecoverSQLLogError(sqlErrFilePath dna.String, db *sqlpg.DB) dna.Int {
	var errCount = 0
	var errStrings = dna.StringArray{}
	b, err := ioutil.ReadFile(sqlErrFilePath.String())
	if err != nil {
		panic(err)
	}
	data := dna.String(string(b))
	// dna.Log("\n", data.Length())
	sqlArr := data.FindAllString(`(?mis)\$\$\$error\$\$\$.+?\$\$\$error\$\$\$`, -1)
	// dna.Log("\nTOTAL SQL STATEMENTS FOUND:", sqlArr.Length())
	for _, val := range sqlArr {
		sqlStmtArr := val.FindAllStringSubmatch(`(?mis)\$\$\$error\$\$\$(.+?)\$\$\$error\$\$\$`, -1)
		if len(sqlStmtArr) > 0 {
			_, err := db.Exec(sqlStmtArr[0][1].String())
			if err != nil {
				if dna.String(err.Error()).Contains(`duplicate key value violates unique constraint`) == false {
					errCount += 1
					errStrings.Push("$$$error$$$" + sqlStmtArr[0][1] + "$$$error$$$")
				}
			}
		}
	}
	if errCount == 0 {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte{}, 0644)
	} else {
		err = ioutil.WriteFile(sqlErrFilePath.String(), []byte(errStrings.Join("\n").String()), 0644)
	}
	if err != nil {
		panic(err)
	}
	return dna.Int(errCount)
}
예제 #2
0
파일: song.go 프로젝트: olragon/dna
func (apisong *APISong) ToSong() *Song {
	song := NewSong()
	song.Id = apisong.Id
	song.Title = apisong.Title.DecodeHTML()
	song.Duration = apisong.Duration

	artistids := dna.IntArray{}
	artists := dna.StringArray{}
	for _, artist := range apisong.Artists {
		artistids.Push(artist.Id)
		artists.Push(artist.Name)
	}
	song.Artistids = artistids
	song.Artists = artists

	composerids := dna.IntArray{}
	composers := dna.StringArray{}
	for _, composer := range apisong.Composers {
		composerids.Push(composer.Id)
		composers.Push(composer.Name)
	}
	song.Composerids = composerids
	song.Composers = composers
	song.Checktime = time.Now()

	return song
}
예제 #3
0
파일: ref.go 프로젝트: olragon/dna
func DumpFiles() {
	var ret = dna.StringArray{}
	for _, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
		ret.Push(`"` + table + `"`)
	}
	dna.Log(ret.Join(","))
}
예제 #4
0
파일: ref.go 프로젝트: olragon/dna
func GetAlbumTables() dna.StringArray {
	var tblArr = dna.StringArray{}
	for key, _ := range RefTables {
		if RefTables.HasAlbum(key) {
			tblArr.Push(key + "albums")
		}
	}
	return tblArr
}
예제 #5
0
파일: ref.go 프로젝트: olragon/dna
func GetSongTables() dna.StringArray {
	var tblArr = dna.StringArray{}
	for key, _ := range RefTables {
		if RefTables.HasSong(key) {
			tblArr.Push(key + "songs")
		}
	}
	return tblArr
}
예제 #6
0
파일: ref.go 프로젝트: olragon/dna
func GetArtistTables() dna.StringArray {
	var tblArr = dna.StringArray{}
	for key, _ := range RefTables {
		if RefTables.HasArtist(key) {
			tblArr.Push(key + "artists")
		}
	}
	return tblArr
}
예제 #7
0
파일: ref.go 프로젝트: olragon/dna
func GetVideoTables() dna.StringArray {
	var tblArr = dna.StringArray{}
	for key, _ := range RefTables {
		if RefTables.HasVideo(key) {
			tblArr.Push(key + "videos")
		}
	}
	return tblArr
}
예제 #8
0
파일: question.go 프로젝트: olragon/dna
func (q Question) ToRecord() []string {

	if q.Cat.Subject == "non-verbal-reasoning/questions-and-answers" {
		imagesArr := q.QuestionContent.FindAllString(`<img.+?src=.+?>`, -1)
		images := dna.StringArray{}
		for _, image := range imagesArr {
			images.Push("http://indiabix.com" + image.GetTagAttributes("src"))
		}
		return []string{"Non Verbal Reasoning", q.Cat.Name.String(), q.Description.String(), images.Join(",").String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
	} else {
		return []string{q.Cat.Subject.String(), q.Cat.Name.String(), q.Description.String(), q.QuestionContent.String(), q.OptionA.String(), q.OptionB.String(), q.OptionC.String(), q.OptionD.String(), q.OptionE.String(), q.Answer.String(), q.Explaination.String(), q.ExerciseNo.ToString().String()}
	}
}
예제 #9
0
파일: album.go 프로젝트: olragon/dna
func convertCategoryToStringArray(catStr dna.String) dna.StringArray {
	var cats []Category
	var ret = dna.StringArray{}
	err := json.Unmarshal([]byte(string(catStr)), &cats)
	if err != nil {
		dna.Log(catStr)
		dna.Log(err.Error())
		panic("Invalid category string input")
	} else {
		for _, cat := range cats {
			ret.Push(cat.Name)
		}
	}
	return ret
}
예제 #10
0
파일: funcs.go 프로젝트: olragon/dna
// GetUpdateStatement returns an update statement from specified snake-case columns.
// If columns's names are not found, it will return an error.
// It updates some fields from a struct.
//
// 	* tbName : A name of update table.
// 	* structValue : A struct-typed value being scanned. Its fields have to be dna basic type or time.Time.
// 	* conditionColumn : A snake-case column name in the condition, usually it's an id
// 	* columns : A list of args of column names in the table being updated.
// 	* Returns an update statement.
func GetUpdateStatement(tbName dna.String, structValue interface{}, conditionColumn dna.String, columns ...dna.String) (dna.String, error) {
	if reflect.TypeOf(structValue).Kind() != reflect.Ptr {
		panic("StructValue has to be pointer")
		if reflect.TypeOf(structValue).Elem().Kind() != reflect.Struct {
			panic("StructValue has to be struct type")
		}
	}
	query := "UPDATE " + tbName + " SET\n"
	result := dna.StringArray{}
	for _, column := range columns {
		result.Push(getPairValue(structValue, column))
	}
	conditionRet := "\nWHERE " + getPairValue(structValue, conditionColumn) + ";"
	return query + result.Join(",\n") + conditionRet, nil
}
예제 #11
0
func (sf *APISongFreaksTrack) ToSong() (*Song, error) {
	if sferr := sf.HasError(); sferr != nil {
		return nil, sferr
	}
	song := NewSong()
	song.Id = sf.Track.Id
	song.TrackGroupId = sf.Track.TrackGroupId
	song.AMG = sf.Track.AMG
	song.UrlSlug = sf.Track.UrlSlug
	song.IsInstrumental = sf.Track.IsInstrumental
	song.Viewable = sf.Track.Viewable
	song.Duration = utils.ToSeconds(sf.Track.Duration)
	song.Lyricid = sf.Track.LyricId
	song.HasLrc = sf.Track.HasLrc
	song.TrackNumber = sf.Track.TrackNumber
	song.DiscNumber = sf.Track.DiscNumber
	song.Title = sf.Track.Title
	song.Rating = dna.IntArray{sf.Track.Rating.AverageRating.ToInt(), sf.Track.Rating.UserRating, sf.Track.Rating.TotalRatings}
	song.Albumid = sf.Track.Album.Id

	artistIds := dna.IntArray{}
	artists := dna.StringArray{}
	for _, artist := range sf.Track.Artists {
		artistIds.Push(artist.Id)
		artists.Push(artist.Name)
	}
	song.Artistids = artistIds
	song.Artists = artists

	if sf.Track.Lrc.Lines != nil && len(sf.Track.Lrc.Lines) > 0 {
		lines, err := json.Marshal(sf.Track.Lrc.Lines)
		if err == nil {
			song.Lrc = dna.String(string(lines))

		}
	}

	song.Link = sf.Track.Link
	song.Lyric = sf.Track.Lyrics
	if song.Lyric != "" {
		song.HasLyric = true
	}
	song.Copyright = sf.Track.Copyright
	song.Writer = sf.Track.Writer
	song.SubmittedLyric = sf.Track.SubmittedLyric
	song.Checktime = time.Now()
	return song, nil
}
예제 #12
0
파일: link_formats.go 프로젝트: olragon/dna
func getStringifiedSongUrls(urls dna.StringArray) dna.String {
	var baseLink = dna.String("")
	songUrls := []SongUrl{}
	urls.ForEach(func(val dna.String, idx dna.Int) {
		// dna.Log(val)
		// Finding bitrate
		switch {
		case val.Match(`128kbps`) == true:
			songUrl := getSongUrl(val, "128kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`320kbps`) == true:
			songUrl := getSongUrl(val, "320kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`32kbps`) == true:
			songUrl := getSongUrl(val, "32kbps")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			songUrls = append(songUrls, *songUrl)
		case val.Match(`500kbps`) == true:
			songUrl := getSongUrl(val, "500kbps")
			songUrl.Link = baseLink + "m4a/file-name.m4a"
			songUrls = append(songUrls, *songUrl)
		case val.Match(`Lossless`) == true:
			songUrl := getSongUrl(val, "Lossless")
			songUrl.Link = baseLink + "flac/file-name.flac"
			songUrls = append(songUrls, *songUrl)
		}
	})
	// http://data.chiasenhac.com/downloads/1184/2/1183017-cfc5f7df/flac/file-name.flac
	// replace the link 500kps and lossless with available link,  apply for registered user only
	// and reduce the link length
	var ret = dna.StringArray{}
	for _, songUrl := range songUrls {
		var br dna.String
		if songUrl.Bitrate == "Lossless" {
			br = "1411"
		} else {
			br = songUrl.Bitrate.Replace("kbps", "")
		}
		t := `"(` + songUrl.Link + "," + songUrl.Type + "," + songUrl.Size.ToString() + "," + br + `)"`
		ret.Push(t)
	}
	// dna.Log(`{` + ret.Join(",") + `}`)
	return `{` + ret.Join(",") + `}`

}
예제 #13
0
파일: album.go 프로젝트: olragon/dna
func getGenresFromDesc(desc dna.String) dna.StringArray {
	var ret dna.StringArray
	genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
	// "Released:" found in album id: 836258
	if genres.Length() > 0 {
		ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
			return val.ReplaceWithRegexp(":", "").Trim()
		}).([]dna.String))
		if ret.Length() == 1 {
			arr := dna.StringArray{}
			if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
				arr.Push("Korean Pop")
				arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
				ret = arr
			}
		}
	}
	return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
예제 #14
0
파일: award.go 프로젝트: olragon/dna
func (apiaward *APIAward) ToAward() *Award {
	award := NewAward()
	award.Id = apiaward.Id
	award.Title = apiaward.Title
	award.Section = ""
	award.Year = apiaward.Year
	award.Chart = apiaward.Chart
	award.Peak = apiaward.Peak
	award.Type = apiaward.Type
	award.Prize = apiaward.Award
	winnerids := dna.IntArray{}
	winners := dna.StringArray{}
	for _, winner := range apiaward.Winners {
		winnerids.Push(winner.Id)
		winners.Push(winner.Name)
	}
	award.Winnerids = winnerids
	award.Winners = winners
	return award
}
예제 #15
0
파일: apifullsong.go 프로젝트: olragon/dna
func (apiSong *APIFullSong) Save(db *sqlpg.DB) error {

	var queries = dna.StringArray{}
	var err error

	// Getting artist queries
	artists := apiSong.ToArtists()
	for _, artist := range artists {
		queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(artist), artist, "id", artist.Id, false))
	}

	// Getting album query
	album := apiSong.ToAlbum()
	queries.Push(sqlpg.GetInsertIgnoreStatement(sqlpg.GetTableName(album), album, "id", album.Id, false))

	// Getting song query
	song := apiSong.ToSong()
	queries.Push(sqlpg.GetInsertStatement(sqlpg.GetTableName(song), song, false))

	for _, query := range queries {
		_, err = db.Exec(query.String())
	}

	if err != nil {
		errQueries := dna.StringArray(queries.Map(func(val dna.String, idx dna.Int) dna.String {
			return "$$$error$$$" + val + "$$$error$$$"
		}).([]dna.String))
		return errors.New(err.Error() + errQueries.Join("\n").String())
	} else {
		return nil
	}

}
예제 #16
0
파일: ref.go 프로젝트: olragon/dna
// DumpHashTables uses psql command. Set its path to ENV
func DumpHashTables() {
	var commands = dna.StringArray{}
	for index, table := range GetAlbumTables().Concat(GetSongTables()).Concat(GetVideoTables()) {
		shortForm := table.Replace("songs", "").Replace("albums", "").Replace("videos", "")
		stmt := dna.Sprintf(`SELECT dna_hash(title,artists), ROW(%v,id) from %v`, ToSiteid(shortForm), table)
		command := dna.Sprintf(`psql -c 'COPY (%v) TO STDOUT'`, stmt)
		switch {
		case table.Match("albums") == true:
			command += " >> data/hash_albums.log"
		case table.Match("songs") == true:
			command += " >> data/hash_songs.log"
		case table.Match("videos") == true:
			command += " >> data/hash_videos.log"
		}
		commands.Push(`INTERNAL_TIME=$(date +%s)`)
		commands.Push(dna.Sprintf(`printf "%-3v:Extracting hashids from %-12v "`, index+1, table+"..."))
		commands.Push(command)
		commands.Push(`echo "Completed in $(($(date +%s) - $INTERNAL_TIME)) seconds!"`)
		commands.Push("#--------------------------")
	}
	dna.Log(commands.Join("\n"))
}
예제 #17
0
파일: link_formats.go 프로젝트: olragon/dna
func getStringifiedVideoUrls(urls dna.StringArray) dna.String {
	var baseLink = dna.String("")
	videoUrls := []VideoUrl{}
	urls.ForEach(func(val dna.String, idx dna.Int) {
		// dna.Log(val)
		// Finding bitrate
		switch {
		case val.Match(`MV 360p`) == true:
			songUrl := getVideoUrl(val, "360p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`MV 480p`) == true:
			songUrl := getVideoUrl(val, "480p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`MV 180p`) == true:
			songUrl := getVideoUrl(val, "180p")
			baseLink = songUrl.Link.ReplaceWithRegexp(`[0-9]+/file-name.+`, "")
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`HD 720p`) == true:
			songUrl := getVideoUrl(val, "720p")
			songUrl.Link = baseLink + "m4a/file-name.mp4"
			videoUrls = append(videoUrls, *songUrl)
		case val.Match(`HD 1080p`) == true:
			songUrl := getVideoUrl(val, "1080p")
			songUrl.Link = baseLink + "flac/file-name.mp4"
			videoUrls = append(videoUrls, *songUrl)
		}
	})

	var ret = dna.StringArray{}
	for _, videoUrl := range videoUrls {
		t := `"(` + videoUrl.Link + "," + videoUrl.Type + "," + videoUrl.Size.ToString() + "," + videoUrl.Resolution.Replace("p", "") + `)"`
		ret.Push(t)
	}
	// dna.Log(`{` + ret.Join(",") + `}`)
	return `{` + ret.Join(",") + `}`
}
예제 #18
0
파일: funcs.go 프로젝트: olragon/dna
// GetInsertIgnoreStatement returns insert ignore statement from a struct. If input value is not struct, it will panic.
//	* tbName : A name of table in database you want to insert
//	* structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
//	* primaryColumn : A name of primary column. If a row is duplicate, it will be discarded.
//	* primaryValue : A value of row needed to be compared.
//	* isPrintable: A param determines where to print the pretty result statement
//	* Return an insert statement
// Notice:  Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertIgnoreStatement(tbName dna.String, structValue interface{}, primaryColumn dna.String, primaryValue interface{}, isPrintable dna.Bool) dna.String {
	var realKind string
	var columnNames, columnValues dna.StringArray
	tempintslice := []int{0}
	var ielements int
	var kind string = reflect.TypeOf(structValue).Kind().String()
	if kind == "ptr" {
		realKind = reflect.TypeOf(structValue).Elem().Kind().String()

	} else {
		realKind = reflect.TypeOf(structValue).Kind().String()

	}

	if realKind != "struct" {
		panic("Param has to be struct")
	}

	if kind == "ptr" {
		ielements = reflect.TypeOf(structValue).Elem().NumField()
	} else {
		ielements = reflect.TypeOf(structValue).NumField()
	}

	for i := 0; i < ielements; i++ {
		tempintslice[0] = i
		if kind == "ptr" {
			f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		} else {
			f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		}

	}
	condStr := dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", tbName, primaryValue)
	if isPrintable == true {
		return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + " SELECT " + columnValues.Join(",\n") + " \n" + condStr + ";" + " \n"
	} else {
		return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " SELECT " + columnValues.Join(",") + " " + condStr + ";"
	}
}
예제 #19
0
파일: funcs.go 프로젝트: olragon/dna
// GetInsertStatement returns insert statement from a struct. If input value is not struct, it will panic.
//	* tbName : A name of table in database you want to insert
//	* structValue : A struct-typed value. The struct's fields has to be dna basic types (dna.Int, dna.String..) or time.Time
//	* isPrintable: A param determines where to print the pretty result statement
//	* Return an insert statement
// Notice:  Insert statement uses Dollar-quoted String Constants with special tag "binhdna".
// So string or array is contained between $binhdna$ symbols.
// Therefore no need to escape values.
func GetInsertStatement(tbName dna.String, structValue interface{}, isPrintable dna.Bool) dna.String {
	var realKind string
	var columnNames, columnValues dna.StringArray
	tempintslice := []int{0}
	var ielements int
	var kind string = reflect.TypeOf(structValue).Kind().String()
	if kind == "ptr" {
		realKind = reflect.TypeOf(structValue).Elem().Kind().String()

	} else {
		realKind = reflect.TypeOf(structValue).Kind().String()

	}

	if realKind != "struct" {
		panic("Param has to be struct")
	}

	if kind == "ptr" {
		ielements = reflect.TypeOf(structValue).Elem().NumField()
	} else {
		ielements = reflect.TypeOf(structValue).NumField()
	}

	for i := 0; i < ielements; i++ {
		tempintslice[0] = i
		if kind == "ptr" {
			f := reflect.TypeOf(structValue).Elem().FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).Elem().FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		} else {
			f := reflect.TypeOf(structValue).FieldByIndex(tempintslice)
			v := reflect.ValueOf(structValue).FieldByIndex(tempintslice)
			clName, clValue := getColumn(f, v.Interface())
			columnNames.Push(clName)
			columnValues.Push(clValue)
		}

	}
	if isPrintable == true {
		return "INSERT INTO " + tbName + "\n(" + columnNames.Join(",") + ")\n" + "VALUES (\n" + columnValues.Join(",\n") + "\n);"
	} else {
		return "INSERT INTO " + tbName + "(" + columnNames.Join(",") + ")" + " VALUES (" + columnValues.Join(",") + ");"
	}
}
예제 #20
0
파일: apifullsong.go 프로젝트: olragon/dna
func (apiSong *APIFullSong) FillSong(song *Song) {
	song.Id = apiSong.Track.Id
	song.Title = apiSong.Track.Title

	artists := dna.StringArray{}
	for _, artist := range apiSong.Track.Artists {
		artists.Push(artist.Name)
	}
	song.Artists = artists
	artistids := dna.IntArray{}
	for _, artist := range apiSong.Track.Artists {
		artistids.Push(artist.Id)
	}

	song.Artistids = artistids
	song.Albumid = apiSong.Track.Album.Id
	song.AlbumTitle = apiSong.Track.Album.Title
	song.Duration = utils.ToSeconds(apiSong.Track.Duration)
	song.Instrumental = apiSong.Track.Instrumental
	song.Viewable = apiSong.Track.Viewable
	song.HasLrc = apiSong.Track.HasLrc
	song.LrcVerified = apiSong.Track.LrcVerified
	song.Lyricid = apiSong.Track.Lyricid
	song.Lyric = apiSong.Track.Lyrics
	song.Copyright = apiSong.Track.Copyright
	song.Writers = apiSong.Track.Writer
	// Mon Jan 2 15:04:05 MST 2006
	if apiSong.Track.LastUpdate != "" {
		lastUpdate, err := time.Parse("2006-01-02 15:04:05", apiSong.Track.LastUpdate.String())
		if err == nil {
			song.DateUpdated = lastUpdate
		} else {
			dna.Log(err.Error(), " Song ID:", song.Id, " GOT:", apiSong.Track.LastUpdate, "\n\n")
		}
	}
}
예제 #21
0
func (sf *APISongFreaksTrack) Save(db *sqlpg.DB) error {
	var queries = dna.StringArray{}
	album, artist, song, videos := sf.Convert()

	if artist != nil {
		queries.Push(getInsertStmt(artist, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(artist), artist.Id)))
	}

	if album != nil {
		queries.Push(getInsertStmt(album, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(album), album.Id)))
	}

	if song != nil {
		queries.Push(getInsertStmt(song, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(song), song.Id)))
	}

	for _, video := range videos {
		queries.Push(getInsertStmt(video, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE songid=%v AND youtube_id='%v')", getTableName(video), video.Songid, video.YoutubeId)))
	}
	// dna.Log(queries.Join("\n"))
	// dna.Log(queries.Join("\n"))
	// return nil
	return sqlpg.ExecQueriesInTransaction(db, &queries)
}
예제 #22
0
파일: question.go 프로젝트: olragon/dna
func getQuestions(cat Category, isFirstPage dna.Bool, pageUrl dna.String, pageLinks *dna.StringArray) <-chan Questions {
	channel := make(chan Questions)
	questions := Questions{}
	go func() {
		var url dna.String
		if isFirstPage == true {
			url = cat.Url
		} else {
			url = pageUrl
		}
		link := "http://www.indiabix.com" + url
		// dna.Log(link)
		result, err := http.Get(link)
		if err == nil {
			data := &result.Data

			if isFirstPage == true {
				// getting page links if the first page found
				pageLinkArr := data.FindAllString(`<p class="ib-pager">.+</p>`, 1)
				if pageLinkArr.Length() > 0 {
					links := pageLinkArr[0].FindAllString(`<a href.+?>`, -1)
					for _, link := range links {
						pageLinks.Push(link.GetTagAttributes("href"))
					}
				}
			}

			// Getting direction for the questions
			descriptionArr := data.FindAllString(`(?mis)id="divDirectionText".+?<div class="bix-div-container">`, 1)
			description := dna.String("")
			if descriptionArr.Length() > 0 {
				description = descriptionArr[0].ReplaceWithRegexp(`<div class="bix-div-container">$`, "").Trim().ReplaceWithRegexp(`<td style="padding-left:5px" valign="top">$`, "").ReplaceWithRegexp(`id="divDirectionText">?`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</div></div></td></tr>$`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
			}

			questionTables := data.FindAllString(`(?mis)<table class="bix-tbl-container".+?<hr />`, -1)
			// if questionTables.Length() > 0 {
			// 	dna.Log("# of questions: ", questionTables.Length())
			// }
			for _, questionTable := range questionTables {
				question := NewQuestion()
				question.Cat = cat
				question.Description = description

				// Getting question content
				questionContentArr := questionTable.FindAllString(`(?mis)<td class="bix-td-qtxt".+?<td class="bix-td-miscell"`, 1)
				if questionContentArr.Length() > 0 {
					question.QuestionContent = questionContentArr[0].ReplaceWithRegexp(`^<td class="bix-td-qtxt.+?>`, "").ReplaceWithRegexp(`<td class="bix-td-miscell"`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</tr>$`, "").Trim().ReplaceWithRegexp(`</p></td>$`, "").Trim()
				}
				// Getting question answers
				// answersArr is table html contains a list of options for the question
				optionArr := questionTable.FindAllString(`(?mis)<td class="bix-td-option.+tdOptionDt.+?<div class="bix-div-answer"`, -1)
				getOptionsForQuestion(optionArr[0], question)

				// Getting explaination for the question
				explantionArr := questionTable.FindAllString(`(?mis)Explanation:</b></span></p>.+?<div class="bix-div-workspace"`, 1)
				if explantionArr.Length() > 0 {
					question.Explaination = explantionArr[0].ReplaceWithRegexp(`^Explanation:</b></span></p>`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<div class="bix-div-workspace"$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
				}

				questions = append(questions, *question)

			}

			// Getting answers for all question
			// If length of answers != length of questions, raise an error
			answersArr := data.FindAllString(`<input id="hdnAjaxImageCacheKey".+`, -1)
			var answers = dna.StringArray{}
			if answersArr.Length() > 0 {
				answer := answersArr[0].GetTagAttributes("value")
				// $('input' + '#' + 'hdn' + 'Ajax' + 'Image' + 'Cache' + 'Key').val().substr(18).split('').reverse().join('').substr(17).toUpperCase().split('');
				tmp := answer.Substring(18, answer.Length()).Split("").Reverse().Join("")
				answers = tmp.Substring(17, tmp.Length()).ToUpperCase().Split("")
			}

			if answers.Length().ToPrimitiveValue() != len(questions) {
				panic("Answer length differs from question length - " + cat.ToString())
			}
			for idx, ans := range answers {
				questions[idx].Answer = ans
			}

		}
		channel <- questions

	}()
	return channel
}
예제 #23
0
// transformCats turns genre name into multiple genres if applicable
func transformCats(cats dna.StringArray) dna.StringArray {
	ret := dna.StringArray{}
	vnSongs := dna.StringArray{"Nhạc Trẻ", "Nhạc Trữ Tình", "Nhạc Cách Mạng", "Nhạc Trịnh", "Nhạc Tiền Chiến", "Nhạc Dân Tộc", "Nhạc Thiếu Nhi", "Rock Việt", "Nhạc Hải Ngoại", "Nhạc Quê Hương", "Rap Việt - Hiphop"}
	for _, cat := range cats {
		if vnSongs.IndexOf(cat) > -1 {
			ret.Push("Nhạc Việt Nam")
		}
		switch cat {
		case "Pop/Ballad":
			ret.Push("Pop")
			ret.Push("Ballad")
		case "Dance/Electronic":
			ret.Push("Dance")
			ret.Push("Electronic")
		case "Nhạc Spa | Thư Giãn":
			ret.Push("Nhạc Spa")
			ret.Push("Thư Giãn")
		case "Hiphop/Rap":
			ret.Push("Hiphop")
			ret.Push("Rap")
		case "Nhạc Bà Bầu & Baby":
			ret.Push("Nhạc Bà Bầu")
			ret.Push("Nhạc Baby")
		case "Rap Việt - Hiphop":
			ret.Push("Rap Việt")
			ret.Push("Hiphop")
		case "Radio - Cảm Xúc":
			ret.Push("Radio")
			ret.Push("Cảm Xúc")
		default:
			ret.Push(cat)
		}
	}
	return ret
}
예제 #24
0
파일: apialbum.go 프로젝트: olragon/dna
func (album *APIAlbum) Save(db *sqlpg.DB) error {
	var queries = dna.StringArray{}

	awards, credits, discos, releases, songs, mainAlbum := album.Convert()

	queries.Push(getInsertStmt(mainAlbum, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v)", getTableName(mainAlbum), album.Id)))

	for _, award := range awards {
		// dna.Log(award)
		queries.Push(getInsertStmt(&award, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v AND albumid=%v AND chart=$binhdna$%v$binhdna$ AND prize=$binhdna$%v$binhdna$)", getTableName(&award), award.Id, award.Albumid, award.Chart, award.Prize)))
	}

	for _, credit := range credits {
		queries.Push(getInsertStmt(&credit, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&credit), credit.Id, credit.Albumid)))
	}

	for _, disco := range discos {
		queries.Push(getInsertStmt(&disco, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and artistid=%v)", getTableName(&disco), disco.Id, disco.Artistid)).Replace("amdiscographys", "amdiscographies"))
	}

	for _, release := range releases {
		queries.Push(getInsertStmt(&release, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&release), release.Id, release.Albumid)))
	}

	for _, song := range songs {
		queries.Push(getInsertStmt(&song, dna.Sprintf("WHERE NOT EXISTS (SELECT 1 FROM %v WHERE id=%v and albumid=%v)", getTableName(&song), song.Id, song.Albumid)))
	}

	// dna.Log(queries.Join("\n"))
	// return db.Update(mainAlbum, "id", "coverart")
	return sqlpg.ExecQueriesInTransaction(db, &queries)
	// return db.InsertIgnore(album)
}
예제 #25
0
func getCategory(songs *[]*Song, genre Genre, page dna.Int) <-chan bool {
	channel := make(chan bool, 1)
	go func() {
		link := "http://nhacso.net/bai-hat-theo-the-loai-" + genre.Id.ToString() + "/joke-link-2-" + page.ToString() + ".html"
		// dna.Log(link)
		result, err := http.Get(link)
		if err == nil {
			data := &result.Data

			// transform string {"2":[0,3,5,7,9,11,13,15,29],"10":[1,2,4,6,8,]}
			// to  map[dna.Int]dna.Int{20:2, 28:2, 4:10, 12:10} Ex: map[29] = 2
			temp := data.FindAllStringSubmatch(`getCategory.+'(\{.+\})'`, -1)
			mapping := map[dna.Int]dna.Int{}
			if len(temp) > 0 && temp[0].Length() > 0 {
				vals := temp[0][1].FindAllString(`"[0-9]+":\[[0-9,]+?\]`, -1)
				if vals.Length() > 0 {
					for _, val := range vals {
						target := val.FindAllStringSubmatch(`"(.+)"`, -1)[0][1].ToInt()
						arr := val.FindAllStringSubmatch(`\[(.+)\]`, -1)[0][1].Split(",").ToIntArray()
						for _, itm := range arr {
							mapping[itm] = target
						}
					}
				}
			}
			// Finding cat id for each song. cats var is 2-dimentional array.
			// Each index of it represents the correspondent song, its value is the categories the song belongs to
			catStrings := data.FindAllString(`Thể loại :.+`, -1)
			cats := []dna.IntArray{}
			for _, val := range catStrings {
				tagids := dna.IntArray{}
				tmp := val.FindAllStringSubmatch(`cate_tag_song_([0-9]+)`, -1)
				if len(tmp) > 0 {
					for _, el := range tmp {
						tagids.Push(el[1].ToInt())
					}
				}
				cats = append(cats, tagids)
			}
			// Log(cats)

			// get songids
			temps := data.FindAllStringSubmatch(`play" id="blocksongtag_([0-9]+)`, -1)
			songids := dna.IntArray{}
			if len(temps) > 0 {
				for _, val := range temps {
					songids.Push(val[1].ToInt())
				}
			}

			tmpsongs := &[]*Song{}
			for idx, songid := range songids {
				song := NewSong()
				song.Id = songid
				category := dna.StringArray{}
				for _, val := range cats[idx] {
					if mapping[val] > 0 && mapping[val] < CatTags.Length() {
						if CatTags[mapping[val]] != "" {
							category.Push(CatTags[mapping[val]])
						}
					} else {
						mess := dna.Sprintf("WRONG INDEX AT CATTAGS: %v %v %v - %v", mapping[val], genre, page, link)
						panic(mess.String())
					}

				}
				category.Push(genre.Name)
				song.Category = transformCats(category.Unique()).Unique()
				*tmpsongs = append(*tmpsongs, song)
			}
			*songs = *tmpsongs

		}
		channel <- true

	}()
	return channel
}