Exemple #1
0
// This example uses a Decoder to decode a stream of distinct JSON values.
func ExampleDecoder() {
	const jsonStream = `
		{Name: "Ed"  Text: "Knock knock."}
		{Name: "Sam" Text: "Who's there?"}
		{Name: "Ed"  Text: "Go fmt."}
		{Name: "Sam" Text: "Go fmt who?"}
		{Name: "Ed"  Text: "Go fmt yourself!"}
	`
	type Message struct {
		Name, Text string
	}
	dec := ljson.NewDecoder(strings.NewReader(jsonStream))
	for {
		var m Message
		if err := dec.Decode(&m); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("%s: %s\n", m.Name, m.Text)
	}
	// Output:
	// Ed: Knock knock.
	// Sam: Who's there?
	// Ed: Go fmt.
	// Sam: Go fmt who?
	// Ed: Go fmt yourself!
}
Exemple #2
0
func loadPackages(fn villa.Path) ([]string, error) {
	f, err := fn.Open()
	if err != nil {
		return nil, villa.NestErrorf(err, "open file %s", fn)
	}
	defer f.Close()

	dec := ljson.NewDecoder(f)
	var list []string
	if err := dec.Decode(&list); err != nil {
		log.Printf("Decode %s failed: %v", fn, err)
		return nil, villa.NestErrorf(err, "decode list")
	}

	return list, nil
}
Exemple #3
0
func jsonfmt() int {
	if len(os.Args) != 2 {
		usage()
		return 1
	}
	path := os.Args[1]

	file, err := os.Open(path)
	if err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	fi, err := file.Stat()
	if err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	j := ljson.NewDecoder(file)
	data := map[string]interface{}{}
	if err := j.Decode(&data); err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	if err := file.Close(); err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	jsonStr, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	if err := ioutil.WriteFile(path, jsonStr, fi.Mode()); err != nil {
		fmt.Printf("err: %s", err)
		return 1
	}

	return 0
}
Exemple #4
0
func main() {
	log.Printf("Server: %s", serverAddr)

	var blackPackages villa.StrSet
	if doBlackPackages {
		func() {
			f, err := blackPkgFn.Open()
			if err != nil {
				log.Printf("Open file %s failed: %v", blackPkgFn, err)
				return
			}
			defer f.Close()

			dec := ljson.NewDecoder(f)
			var list []string
			if err := dec.Decode(&list); err != nil {
				log.Printf("Decode %s failed: %v", blackPkgFn, err)
				return
			}
			blackPackages.Put(list...)
			log.Printf("%d black packages loaded!", len(blackPackages))
		}()
	}

	httpClient := genHttpClient(proxyServer)
	rpcClient := rpc.NewClient(httpClient, serverAddr)
	client := gcc.NewServiceClient(rpcClient)

	for maxLoops != 0 {
		var wg sync.WaitGroup

		blackPackagesCnt := len(blackPackages)

		morePackages := false
		pkgs := client.FetchPackageList(nil, entriesPerLoop)
		err := client.LastError()
		if err != nil {
			log.Printf("FetchPackageList failed: %v", err)
		} else {
			morePackages = len(pkgs) >= entriesPerLoop

			groups := gcc.GroupPackages(pkgs)
			log.Printf("Packages: %v, %d packages, %d groups", groups,
				len(pkgs), len(groups))

			wg.Add(len(groups))

			for _, pkgs := range groups {
				go func(pkgs []string) {
					for _, pkg := range pkgs {
						p, err := gcc.CrawlPackage(httpClient, pkg)
						if err != nil {
							log.Printf("Crawling pkg %s failed: %v", pkg, err)

							if gcc.IsBadPackage(err) {
								// a wrong path
								client.ReportBadPackage(nil, pkg)
								log.Printf("Remove wrong package %s: %v", pkg, client.LastError())

								if doBlackPackages {
									blackPackages.Put(pkg)
								}
							}
							continue
						}

						log.Printf("Crawled package %s success!", pkg)

						client.PushPackage(nil, p)
						err = client.LastError()
						if err != nil {
							log.Printf("Push package %s failed: %v", pkg, err)
							continue
						}
						log.Printf("Push package %s success!", pkg)
					}

					wg.Done()
				}(pkgs)
			}
		}

		hasNewPackage := false
		morePersons := false
		persons := client.FetchPersonList(nil, entriesPerLoop)
		err = client.LastError()
		if err != nil {
			log.Printf("FetchPersonList failed: %v", err)
		} else {
			morePersons = len(persons) >= entriesPerLoop

			groups := gcc.GroupPersons(persons)
			log.Printf("persons: %v, %d persons, %d groups", groups,
				len(persons), len(groups))

			wg.Add(len(groups))

			for _, ids := range groups {
				go func(ids []string) {
					for _, id := range ids {
						p, err := gcc.CrawlPerson(httpClient, id)
						if err != nil {
							log.Printf("Crawling person %s failed: %v", id, err)
							continue
						}

						log.Printf("Crawled person %s success!", id)
						newPackage := client.PushPerson(nil, p)
						err = client.LastError()
						if err != nil {
							log.Printf("Push person %s failed: %v", id, err)
							continue
						}

						log.Printf("Push person %s success: %v", id, newPackage)
						if newPackage {
							hasNewPackage = true
						}
					}

					wg.Done()
				}(ids)
			}
		}

		wg.Wait()

		if blackPackagesCnt != len(blackPackages) {
			func() {
				log.Printf("Saving black packages...")
				f, err := blackPkgFn.Create()
				if err != nil {
					log.Printf("Open file %s failed: %v", blackPkgFn, err)
					return
				}

				enc := json.NewEncoder(f)
				err = enc.Encode(blackPackages.Elements())
				if err != nil {
					log.Printf("Encoding black packages failed: %v", err)
				}
			}()
		}

		if !morePackages && !morePersons && !hasNewPackage {
			log.Printf("Nothing to do, have a rest...(%d)", maxLoops)
			time.Sleep(time.Duration(restSeconds) * time.Second)
			if maxLoops > 0 {
				maxLoops--
			}
		}
	}
}