func main() {
flag.Parse()
file := flag.Arg(0)
csv := csv.NewWriter(os.Stdout)
selector := "meta[name]" // All meta tags with name attribute
webdevdata.ProcessMatchingTags(file, selector, func(node *html.Node) {
name := webdevdata.GetAttr("name", node.Attr)
csv.Write([]string{file, name})
})
csv.Flush()
}
func main() {
flag.Parse()
file := flag.Arg(0)
csv := csv.NewWriter(os.Stdout)
selector := "html[manifest]" // all html tags with manifest attribute
webdevdata.ProcessMatchingTags(file, selector, func(node *html.Node) {
manifest := webdevdata.GetAttr("manifest", node.Attr)
if manifest != "" {
csv.Write([]string{file, manifest})
}
})
csv.Flush()
}
func main() {
flag.Parse()
filesChan := make(chan string)
go webdevdata.GetFiles(filesChan, 0)
csv := csv.NewWriter(os.Stdout)
for file := range filesChan {
tags := make(map[string]int)
webdevdata.ProcessTags(file, func(token html.Token) {
tags[token.Data]++
})
for tag, count := range tags {
csv.Write([]string{file, tag, strconv.Itoa(count)})
}
}
csv.Flush()
}
func (o *output) writeLines() {
csv := csv.NewWriter(o.stream)
commLoop:
for {
select {
case r := <-o.receive:
switch r.Type {
case plumbing.UpdateQuestion:
csv.Write([]string{
r.Identifier,
r.Label,
r.Value,
})
case plumbing.Flush:
csv.Flush()
break commLoop
}
}
}
}
func main() {
attrs := flag.String("attrs", "", "CSV list of attributes to print")
flag.Parse()
if flag.NArg() == 0 {
fmt.Fprintln(os.Stderr, "You need to provide a CSS selector and one file")
os.Exit(-1)
}
selector := flag.Arg(0)
filesChan := make(chan string)
go webdevdata.GetFiles(filesChan, 1)
attrList := regexp.MustCompile(",").Split(*attrs, -1)
csv := csv.NewWriter(os.Stdout)
for file := range filesChan {
process(file, selector, attrList, csv)
}
csv.Flush()
}
func IdentifySongs(songs []*Song, outFile string) {
log.Printf("Identifying songs...")
if len(songs) == 0 {
log.Printf("No songs to identify.")
return
}
// How many identification routines can run concurrently
const concurrency = 10
var sem = make(chan bool, concurrency)
var songsOutCh = make(chan *Song)
var songsErrCh = make(chan SongErr)
var wg sync.WaitGroup
if len(songs) != 0 {
// from Params
for _, song := range songs {
if song.Hash == "" {
wg.Add(1)
go func(song *Song) {
sem <- true
defer func() { <-sem }()
defer func() { wg.Done() }()
hash, err := audioFileChecksum(song.Path)
if err != nil {
var songErr SongErr
songErr.Song = song
songErr.Err = err
songsErrCh <- songErr
//log.Printf("ident: err: %v for %s", err, song.Path)
return
}
song.Hash = hash
songsOutCh <- song
}(song)
}
}
}
// Close the output channel when all the audioFileCheksums complete.
go func() {
wg.Wait()
close(songsOutCh)
close(songsErrCh)
}()
// Consumes from songsErrCh and writes song_errors.txt for unparsable files.
go func() {
songHashesPath := path.Join(config.DataPath, "song_errors.txt")
f, err := os.OpenFile(songHashesPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
if err != nil {
panic(fmt.Errorf("%q: %v", songHashesPath, err))
}
for songErr := range songsErrCh {
f.WriteString(fmt.Sprintf("%q\t%q\n",
songErr.Song.Path,
songErr.Err))
}
}()
// Consumes songs via SongsOutChan, collects hash and metadata into song_hashes.txt
go func() {
songHashesPath := path.Join(config.DataPath, "song_hashes.txt")
f, err := os.OpenFile(songHashesPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
if err != nil {
panic(fmt.Errorf("%q: %v", songHashesPath, err))
}
csv := csv.NewWriter(f)
i := 0
for song := range songsOutCh {
meta := song.Meta()
csv.Write([]string{
song.Hash,
song.Path,
meta.Artist,
meta.Album,
meta.Title})
i++
if i%100 == 0 {
log.Printf("Attempted identification of %d songs\n", i)
}
}
csv.Flush()
log.Printf("Identifying songs...done\n")
}()
}
func (w *csvFormatter) WriteResponse(resp Response) (n int, err error) {
csv := csv.NewWriter(w)
for _, result := range resp.Results {
if result.StatementID != w.statementID {
// If there are no series in the result, skip past this result.
if len(result.Series) == 0 {
continue
}
// Set the statement id and print out a newline if this is not the first statement.
if w.statementID >= 0 {
// Flush the csv writer and write a newline.
csv.Flush()
if err := csv.Error(); err != nil {
return n, err
}
if out, err := io.WriteString(w, "\n"); err != nil {
return n, err
} else {
n += out
}
}
w.statementID = result.StatementID
// Print out the column headers from the first series.
w.columns = make([]string, 2+len(result.Series[0].Columns))
w.columns[0] = "name"
w.columns[1] = "tags"
copy(w.columns[2:], result.Series[0].Columns)
if err := csv.Write(w.columns); err != nil {
return n, err
}
}
for _, row := range result.Series {
w.columns[0] = row.Name
if len(row.Tags) > 0 {
w.columns[1] = string(models.NewTags(row.Tags).HashKey()[1:])
} else {
w.columns[1] = ""
}
for _, values := range row.Values {
for i, value := range values {
switch v := value.(type) {
case float64:
w.columns[i+2] = strconv.FormatFloat(v, 'f', -1, 64)
case int64:
w.columns[i+2] = strconv.FormatInt(v, 10)
case string:
w.columns[i+2] = v
case bool:
if v {
w.columns[i+2] = "true"
} else {
w.columns[i+2] = "false"
}
case time.Time:
w.columns[i+2] = strconv.FormatInt(v.UnixNano(), 10)
}
}
csv.Write(w.columns)
}
}
}
csv.Flush()
if err := csv.Error(); err != nil {
return n, err
}
return n, nil
}