// This example uses a Decoder to decode a stream of distinct JSON values.
func ExampleDecoder() {
const jsonStream = `
{Name: "Ed" Text: "Knock knock."}
{Name: "Sam" Text: "Who's there?"}
{Name: "Ed" Text: "Go fmt."}
{Name: "Sam" Text: "Go fmt who?"}
{Name: "Ed" Text: "Go fmt yourself!"}
`
type Message struct {
Name, Text string
}
dec := ljson.NewDecoder(strings.NewReader(jsonStream))
for {
var m Message
if err := dec.Decode(&m); err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
fmt.Printf("%s: %s\n", m.Name, m.Text)
}
// Output:
// Ed: Knock knock.
// Sam: Who's there?
// Ed: Go fmt.
// Sam: Go fmt who?
// Ed: Go fmt yourself!
}
func loadPackages(fn villa.Path) ([]string, error) {
f, err := fn.Open()
if err != nil {
return nil, villa.NestErrorf(err, "open file %s", fn)
}
defer f.Close()
dec := ljson.NewDecoder(f)
var list []string
if err := dec.Decode(&list); err != nil {
log.Printf("Decode %s failed: %v", fn, err)
return nil, villa.NestErrorf(err, "decode list")
}
return list, nil
}
func jsonfmt() int {
if len(os.Args) != 2 {
usage()
return 1
}
path := os.Args[1]
file, err := os.Open(path)
if err != nil {
fmt.Printf("err: %s", err)
return 1
}
fi, err := file.Stat()
if err != nil {
fmt.Printf("err: %s", err)
return 1
}
j := ljson.NewDecoder(file)
data := map[string]interface{}{}
if err := j.Decode(&data); err != nil {
fmt.Printf("err: %s", err)
return 1
}
if err := file.Close(); err != nil {
fmt.Printf("err: %s", err)
return 1
}
jsonStr, err := json.MarshalIndent(data, "", " ")
if err != nil {
fmt.Printf("err: %s", err)
return 1
}
if err := ioutil.WriteFile(path, jsonStr, fi.Mode()); err != nil {
fmt.Printf("err: %s", err)
return 1
}
return 0
}
func main() {
log.Printf("Server: %s", serverAddr)
var blackPackages villa.StrSet
if doBlackPackages {
func() {
f, err := blackPkgFn.Open()
if err != nil {
log.Printf("Open file %s failed: %v", blackPkgFn, err)
return
}
defer f.Close()
dec := ljson.NewDecoder(f)
var list []string
if err := dec.Decode(&list); err != nil {
log.Printf("Decode %s failed: %v", blackPkgFn, err)
return
}
blackPackages.Put(list...)
log.Printf("%d black packages loaded!", len(blackPackages))
}()
}
httpClient := genHttpClient(proxyServer)
rpcClient := rpc.NewClient(httpClient, serverAddr)
client := gcc.NewServiceClient(rpcClient)
for maxLoops != 0 {
var wg sync.WaitGroup
blackPackagesCnt := len(blackPackages)
morePackages := false
pkgs := client.FetchPackageList(nil, entriesPerLoop)
err := client.LastError()
if err != nil {
log.Printf("FetchPackageList failed: %v", err)
} else {
morePackages = len(pkgs) >= entriesPerLoop
groups := gcc.GroupPackages(pkgs)
log.Printf("Packages: %v, %d packages, %d groups", groups,
len(pkgs), len(groups))
wg.Add(len(groups))
for _, pkgs := range groups {
go func(pkgs []string) {
for _, pkg := range pkgs {
p, err := gcc.CrawlPackage(httpClient, pkg)
if err != nil {
log.Printf("Crawling pkg %s failed: %v", pkg, err)
if gcc.IsBadPackage(err) {
// a wrong path
client.ReportBadPackage(nil, pkg)
log.Printf("Remove wrong package %s: %v", pkg, client.LastError())
if doBlackPackages {
blackPackages.Put(pkg)
}
}
continue
}
log.Printf("Crawled package %s success!", pkg)
client.PushPackage(nil, p)
err = client.LastError()
if err != nil {
log.Printf("Push package %s failed: %v", pkg, err)
continue
}
log.Printf("Push package %s success!", pkg)
}
wg.Done()
}(pkgs)
}
}
hasNewPackage := false
morePersons := false
persons := client.FetchPersonList(nil, entriesPerLoop)
err = client.LastError()
if err != nil {
log.Printf("FetchPersonList failed: %v", err)
} else {
morePersons = len(persons) >= entriesPerLoop
groups := gcc.GroupPersons(persons)
log.Printf("persons: %v, %d persons, %d groups", groups,
len(persons), len(groups))
wg.Add(len(groups))
for _, ids := range groups {
go func(ids []string) {
for _, id := range ids {
p, err := gcc.CrawlPerson(httpClient, id)
if err != nil {
log.Printf("Crawling person %s failed: %v", id, err)
continue
}
log.Printf("Crawled person %s success!", id)
newPackage := client.PushPerson(nil, p)
err = client.LastError()
if err != nil {
log.Printf("Push person %s failed: %v", id, err)
continue
}
log.Printf("Push person %s success: %v", id, newPackage)
if newPackage {
hasNewPackage = true
}
}
wg.Done()
}(ids)
}
}
wg.Wait()
if blackPackagesCnt != len(blackPackages) {
func() {
log.Printf("Saving black packages...")
f, err := blackPkgFn.Create()
if err != nil {
log.Printf("Open file %s failed: %v", blackPkgFn, err)
return
}
enc := json.NewEncoder(f)
err = enc.Encode(blackPackages.Elements())
if err != nil {
log.Printf("Encoding black packages failed: %v", err)
}
}()
}
if !morePackages && !morePersons && !hasNewPackage {
log.Printf("Nothing to do, have a rest...(%d)", maxLoops)
time.Sleep(time.Duration(restSeconds) * time.Second)
if maxLoops > 0 {
maxLoops--
}
}
}
}
