Ejemplo n.º 1
0
Archivo: func.go Proyecto: olragon/dna
// SelectMissingKeys accepts a table name as an input and a list of keys as a source.
// It returns a new list of keys that does not exist in the destination table
//
// 	* tblName : a table name
// 	* srcKeys : a source keys
// 	* db : a pointer to connected databased
// 	* Returns a new list of keys which are not from the specified table
//
// Notice: Only applied to a table having a column named "key".
// The column has to be indexed to ensure good performance
//
// The format of sql statement is:
//	with dna (key) as (values ('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('RvFDlckJB5QU'),('uIF7rwd5wo4p'),('Kveukbhre1ry'),('oJ1lzAlKwJX6'),('43f3HhhU6DGV'),('uFfgQhKbwAfN'),('hfhtyMdywMau'),('PpZuccjYqy1b'))
//	select key from dna where key not in
//	(select key from nctalbums where key in ('43f3HhhU6DGV','uFfgQhKbwAfN','RvFDlckJB5QU','uIF7rwd5wo4p','Kveukbhre1ry','oJ1lzAlKwJX6','43f3HhhU6DGV','uFfgQhKbwAfN','hfhtyMdywMau','PpZuccjYqy1b'))
func SelectMissingKeys(tblName dna.String, srcKeys *dna.StringArray, db *sqlpg.DB) (*dna.StringArray, error) {
	if srcKeys.Length() > 0 {
		val := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
			return `('` + val + `')`
		}).([]dna.String))
		val1 := dna.StringArray(srcKeys.Map(func(val dna.String, idx dna.Int) dna.String {
			return `'` + val + `'`
		}).([]dna.String))
		selectStmt := "with dna (key) as (values " + val.Join(",") + ") \n"
		selectStmt += "select key from dna where key not in \n(select key from " + tblName + " where key in (" + val1.Join(",") + "))"
		keys := &[]dna.String{}
		err := db.Select(keys, selectStmt)
		switch {
		case err != nil:
			return nil, err
		case err == nil && keys != nil:
			slice := dna.StringArray(*keys)
			return &slice, nil
		case err == nil && keys == nil:
			return &dna.StringArray{}, nil
		default:
			panic("Default case triggered. Case is not expected. Cannot select non existed keys")
		}
	} else {
		return nil, errors.New("Empty input array")
	}
}
Ejemplo n.º 2
0
func getTSGM(data *dna.String, kind dna.String) dna.String {
	var itemArr dna.StringArray
	switch kind {
	case "genres":
		itemArr = data.FindAllString(`(?mis)<h4>Genre</h4>(.+?)</div>`, 1)
	case "styles":
		itemArr = data.FindAllString(`(?mis)<h4>Styles</h4>(.+?)</div>`, 1)
	case "moods":
		itemArr = data.FindAllString(`(?mis)<h4>Album Moods</h4>(.+?)</div>`, 1)
	case "themes":
		itemArr = data.FindAllString(`(?mis)<h4>Themes</h4>(.+?)</div>`, 1)
	default:
		panic("Wrong kind!!!")
	}
	if itemArr.Length() > 0 {
		catArr := itemArr[0].FindAllString(`<a href=.+?</a>`, -1)
		categories := catArr.Map(func(val dna.String, idx dna.Int) Category {
			var idArr []dna.StringArray
			var id dna.Int = 0
			name := val.RemoveHtmlTags("")
			if kind == "moods" {
				idArr = val.FindAllStringSubmatch(`xa([0-9]+)`, 1)
			} else {
				idArr = val.FindAllStringSubmatch(`ma([0-9]+)`, 1)
			}
			if len(idArr) > 0 {
				id = idArr[0][1].ToInt()
			}
			return Category{id, name}
		}).([]Category)

		if len(categories) > 0 {
			bCat, merr := json.Marshal(categories)
			if merr != nil {
				return "[]"
			} else {
				return dna.String(string(bCat))
			}
		} else {
			return "[]"
		}
	} else {
		return "[]"
	}

}
Ejemplo n.º 3
0
Archivo: album.go Proyecto: olragon/dna
func getGenresFromDesc(desc dna.String) dna.StringArray {
	var ret dna.StringArray
	genres := desc.FindAllString(`(?i)genres?(\s+)?:?.+`, 1)
	// "Released:" found in album id: 836258
	if genres.Length() > 0 {
		ret = dna.StringArray(genres[0].ReplaceWithRegexp(`(?mis)genres?(\s+)?:?`, "").ReplaceWithRegexp(`\.?\s*Released:.+`, "").Trim().Split(",").Map(func(val dna.String, idx dna.Int) dna.String {
			return val.ReplaceWithRegexp(":", "").Trim()
		}).([]dna.String))
		if ret.Length() == 1 {
			arr := dna.StringArray{}
			if ret[0].FindAllString(`(?mis)K-Pop`, 1).Length() > 0 {
				arr.Push("Korean Pop")
				arr.Push(ret[0].ReplaceWithRegexp(`(?mis)\(?K-Pop\)?`, "").Trim())
				ret = arr
			}
		}
	}
	return ret.SplitWithRegexp(` > `).SplitWithRegexp(`/`)
}
Ejemplo n.º 4
0
func getQuestions(cat Category, isFirstPage dna.Bool, pageUrl dna.String, pageLinks *dna.StringArray) <-chan Questions {
	channel := make(chan Questions)
	questions := Questions{}
	go func() {
		var url dna.String
		if isFirstPage == true {
			url = cat.Url
		} else {
			url = pageUrl
		}
		link := "http://www.indiabix.com" + url
		// dna.Log(link)
		result, err := http.Get(link)
		if err == nil {
			data := &result.Data

			if isFirstPage == true {
				// getting page links if the first page found
				pageLinkArr := data.FindAllString(`<p class="ib-pager">.+</p>`, 1)
				if pageLinkArr.Length() > 0 {
					links := pageLinkArr[0].FindAllString(`<a href.+?>`, -1)
					for _, link := range links {
						pageLinks.Push(link.GetTagAttributes("href"))
					}
				}
			}

			// Getting direction for the questions
			descriptionArr := data.FindAllString(`(?mis)id="divDirectionText".+?<div class="bix-div-container">`, 1)
			description := dna.String("")
			if descriptionArr.Length() > 0 {
				description = descriptionArr[0].ReplaceWithRegexp(`<div class="bix-div-container">$`, "").Trim().ReplaceWithRegexp(`<td style="padding-left:5px" valign="top">$`, "").ReplaceWithRegexp(`id="divDirectionText">?`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</div></div></td></tr>$`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
			}

			questionTables := data.FindAllString(`(?mis)<table class="bix-tbl-container".+?<hr />`, -1)
			// if questionTables.Length() > 0 {
			// 	dna.Log("# of questions: ", questionTables.Length())
			// }
			for _, questionTable := range questionTables {
				question := NewQuestion()
				question.Cat = cat
				question.Description = description

				// Getting question content
				questionContentArr := questionTable.FindAllString(`(?mis)<td class="bix-td-qtxt".+?<td class="bix-td-miscell"`, 1)
				if questionContentArr.Length() > 0 {
					question.QuestionContent = questionContentArr[0].ReplaceWithRegexp(`^<td class="bix-td-qtxt.+?>`, "").ReplaceWithRegexp(`<td class="bix-td-miscell"`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<tr>$`, "").Trim().ReplaceWithRegexp(`</tr>$`, "").Trim().ReplaceWithRegexp(`</p></td>$`, "").Trim()
				}
				// Getting question answers
				// answersArr is table html contains a list of options for the question
				optionArr := questionTable.FindAllString(`(?mis)<td class="bix-td-option.+tdOptionDt.+?<div class="bix-div-answer"`, -1)
				getOptionsForQuestion(optionArr[0], question)

				// Getting explaination for the question
				explantionArr := questionTable.FindAllString(`(?mis)Explanation:</b></span></p>.+?<div class="bix-div-workspace"`, 1)
				if explantionArr.Length() > 0 {
					question.Explaination = explantionArr[0].ReplaceWithRegexp(`^Explanation:</b></span></p>`, "").Trim().ReplaceWithRegexp(`^<p>`, "").Trim().ReplaceWithRegexp(`<div class="bix-div-workspace"$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</div>$`, "").Trim().ReplaceWithRegexp(`</p>$`, "").Trim()
				}

				questions = append(questions, *question)

			}

			// Getting answers for all question
			// If length of answers != length of questions, raise an error
			answersArr := data.FindAllString(`<input id="hdnAjaxImageCacheKey".+`, -1)
			var answers = dna.StringArray{}
			if answersArr.Length() > 0 {
				answer := answersArr[0].GetTagAttributes("value")
				// $('input' + '#' + 'hdn' + 'Ajax' + 'Image' + 'Cache' + 'Key').val().substr(18).split('').reverse().join('').substr(17).toUpperCase().split('');
				tmp := answer.Substring(18, answer.Length()).Split("").Reverse().Join("")
				answers = tmp.Substring(17, tmp.Length()).ToUpperCase().Split("")
			}

			if answers.Length().ToPrimitiveValue() != len(questions) {
				panic("Answer length differs from question length - " + cat.ToString())
			}
			for idx, ans := range answers {
				questions[idx].Answer = ans
			}

		}
		channel <- questions

	}()
	return channel
}