func UpdateAllMachines(flags *UpdateFlags, log *logging.Logger) error {
// Get all members
members, err := flags.GetClusterMembers(log)
if err != nil {
return maskAny(err)
}
// Pull image on all machines
log.Infof("Pulling gluon image on %d machines", len(members))
var pullGroup errgroup.Group
for _, m := range members {
m := m
pullGroup.Go(func() error {
return maskAny(pullImage(m, *flags, log))
})
}
if err := pullGroup.Wait(); err != nil {
return maskAny(err)
}
// Update all machines, one at a time
for index, m := range members {
if index > 0 {
log.Infof("Waiting %s...", flags.MachineDelay)
time.Sleep(flags.MachineDelay)
}
if err := updateMachine(m, *flags, log); err != nil {
return maskAny(err)
}
}
return nil
}
func TestZeroGroup(t *testing.T) {
err1 := errors.New("errgroup_test: 1")
err2 := errors.New("errgroup_test: 2")
cases := []struct {
errs []error
}{
{errs: []error{}},
{errs: []error{nil}},
{errs: []error{err1}},
{errs: []error{err1, nil}},
{errs: []error{err1, nil, err2}},
}
for _, tc := range cases {
var g errgroup.Group
var firstErr error
for i, err := range tc.errs {
err := err
g.Go(func() error { return err })
if firstErr == nil && err != nil {
firstErr = err
}
if gErr := g.Wait(); gErr != firstErr {
t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
"g.Wait() = %v; want %v",
g, tc.errs[:i+1], err, firstErr)
}
}
}
}
func TestConfigProcessContent(t *testing.T) {
Convey("Testing ProcessContent()", t, func() {
newCfg := func() *Config {
return NewConfig(
API("/bin/cli-shell-api"),
Arch(runtime.GOARCH),
Bash("/bin/bash"),
Cores(runtime.NumCPU()),
Dir("/:~/"),
DNSsvc("service dnsmasq restart"),
Ext("blacklist.conf"),
FileNameFmt("%v/%v.%v.%v"),
InCLI("inSession"),
Level("service dns forwarding"),
Method("GET"),
Nodes([]string{domains, hosts}),
Prefix("address="),
LTypes([]string{files, PreDomns, PreHosts, urls}),
Timeout(30*time.Second),
WCard(Wildcard{Node: "*s", Name: "*"}),
)
}
tests := []struct {
c *Config
cfg string
ct IFace
err error
expErr bool
name string
}{
{
c: newCfg(),
cfg: testCfg,
ct: FileObj,
err: fmt.Errorf("open /:~/=../testdata/blist.hosts.src: no such file or directory\nopen /:~//hosts.tasty.blacklist.conf: no such file or directory"),
expErr: true,
name: "File",
},
}
for _, tt := range tests {
So(tt.c.ReadCfg(&CFGstatic{Cfg: tt.cfg}), ShouldBeNil)
var g errgroup.Group
obj, err := tt.c.NewContent(tt.ct)
So(err, ShouldBeNil)
g.Go(func() error { return tt.c.ProcessContent(obj) })
err = g.Wait()
if (err != nil) == tt.expErr {
So(err.Error(), ShouldEqual, tt.err.Error())
}
}
Convey("Testing ProcessContent() if no arguments ", func() {
var g errgroup.Group
g.Go(func() error { return newCfg().ProcessContent() })
err := g.Wait()
So(err, ShouldNotBeNil)
})
})
}
func TestMarshalRace(t *testing.T) {
// unregistered extension
desc := &proto.ExtensionDesc{
ExtendedType: (*pb.MyMessage)(nil),
ExtensionType: (*bool)(nil),
Field: 101010100,
Name: "emptyextension",
Tag: "varint,0,opt",
}
m := &pb.MyMessage{Count: proto.Int32(4)}
if err := proto.SetExtension(m, desc, proto.Bool(true)); err != nil {
t.Errorf("proto.SetExtension(m, desc, true): got error %q, want nil", err)
}
var g errgroup.Group
for n := 3; n > 0; n-- {
g.Go(func() error {
_, err := proto.Marshal(m)
return err
})
}
if err := g.Wait(); err != nil {
t.Fatal(err)
}
}
func TestClients(t *testing.T) {
var (
builder = testutil.NewBuilder(nil)
idA = machine.ID("servA")
idB = machine.ID("servB")
servA = &testutil.Server{}
servB = &testutil.Server{}
)
cs, err := clients.New(testOptions(builder))
if err != nil {
t.Fatalf("want err = nil; got %v", err)
}
defer cs.Close()
var g errgroup.Group
create := map[machine.ID]client.DynamicAddrFunc{
idA: servA.AddrFunc(),
idB: servB.AddrFunc(),
idA: servA.AddrFunc(), // duplicate.
idA: servA.AddrFunc(), // duplicate.
}
for id, dynAddr := range create {
id, dynAddr := id, dynAddr // Local copy for concurrency.
g.Go(func() error {
return cs.Create(id, dynAddr)
})
}
if err := g.Wait(); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
if regs := cs.Registered(); len(regs) != 2 {
t.Fatalf("want clients count = 2; got %d", len(regs))
}
if _, err := cs.Client(idA); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
if err := cs.Drop(idA); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
if _, err := cs.Client(idA); err != machine.ErrMachineNotFound {
t.Fatalf("want machine not found; got %v", err)
}
if regs := cs.Registered(); len(regs) != 1 {
t.Fatalf("want clients count = 1; got %d", len(regs))
}
}
func (c Client) Get(
location *os.File,
contentURL string,
progressWriter io.Writer,
) error {
req, err := http.NewRequest("HEAD", contentURL, nil)
if err != nil {
return fmt.Errorf("failed to construct HEAD request: %s", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to make HEAD request: %s", err)
}
contentURL = resp.Request.URL.String()
ranges, err := c.ranger.BuildRange(resp.ContentLength)
if err != nil {
return fmt.Errorf("failed to construct range: %s", err)
}
c.bar.SetOutput(progressWriter)
c.bar.SetTotal(resp.ContentLength)
c.bar.Kickoff()
defer c.bar.Finish()
var g errgroup.Group
for _, r := range ranges {
byteRange := r
g.Go(func() error {
respBytes, err := c.retryableRequest(contentURL, byteRange.HTTPHeader)
if err != nil {
return fmt.Errorf("failed during retryable request: %s", err)
}
bytesWritten, err := location.WriteAt(respBytes, byteRange.Lower)
if err != nil {
return fmt.Errorf("failed to write file: %s", err)
}
c.bar.Add(bytesWritten)
return nil
})
}
if err := g.Wait(); err != nil {
return err
}
return nil
}
func fetchParallelizableUsageItems(group *models.Group, username string) (*Usage, error) {
var g errgroup.Group
var cus *stripe.Customer
var subscription *stripe.Sub
g.Go(func() (err error) {
cus, err = EnsureCustomerForGroup(username, group.Slug, nil)
if err != nil {
return err
}
subscription, err = EnsureSubscriptionForGroup(group.Slug, nil)
return err
})
// Active users count.
var activeCount int
g.Go(func() (err error) {
activeCount, err = (&socialapimodels.PresenceDaily{}).CountDistinctByGroupName(group.Slug)
return err
})
// Trialing user count is set if only sub is trialing.
var trialCount int
g.Go(func() (err error) {
if group.Payment.Subscription.Status != "trialing" {
return nil
}
trialCount, err = (&socialapimodels.PresenceDaily{}).CountDistinctProcessedByGroupName(group.Slug)
return err
})
if err := g.Wait(); err != nil {
return nil, err
}
usage := &Usage{
User: &UserInfo{
Total: activeCount,
},
ExpectedPlan: nil,
Due: 0,
NextBillingDate: time.Unix(subscription.PeriodEnd, 0),
Subscription: subscription,
Customer: cus,
Trial: &TrialInfo{
User: &UserInfo{
Total: trialCount,
},
},
}
return usage, nil
}
func TestDynamicClientContext(t *testing.T) {
var (
serv = &testutil.Server{}
builder = testutil.NewBuilder(nil)
)
dc, err := client.NewDynamic(testutil.DynamicOpts(serv, builder))
if err != nil {
t.Fatalf("want err = nil; got %v", err)
}
defer dc.Close()
ctx := dc.Client().Context()
serv.TurnOn()
if err := testutil.WaitForContextClose(ctx, time.Second); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
const ContextWorkers = 10
var g errgroup.Group
for i := 0; i < ContextWorkers; i++ {
g.Go(func() error {
select {
case <-dc.Client().Context().Done():
return errors.New("context closed unexpectedly")
case <-time.After(50 * time.Millisecond):
return nil
}
})
}
// Machine is on so dynamic client should not close its context.
if err := g.Wait(); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
ctx = dc.Client().Context()
serv.TurnOff()
for i := 0; i < ContextWorkers; i++ {
g.Go(func() error {
select {
case <-ctx.Done():
return nil
case <-time.After(time.Second):
return errors.New("timed out")
}
})
}
// Machine is off so its context channel should be closed by dynamic client.
if err := g.Wait(); err != nil {
t.Fatalf("want err = nil; got %v", err)
}
}
// JustErrors illustrates the use of a Group in place of a sync.WaitGroup to
// simplify goroutine counting and error handling. This example is derived from
// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
func ExampleGroup_justErrors() {
var g errgroup.Group
var urls = []string{
"http://www.golang.org/",
"http://www.google.com/",
"http://www.somestupidname.com/",
}
for _, url := range urls {
// Launch a goroutine to fetch the URL.
url := url // https://golang.org/doc/faq#closures_and_goroutines
g.Go(func() error {
// Fetch the URL.
resp, err := http.Get(url)
if err == nil {
resp.Body.Close()
}
return err
})
}
// Wait for all HTTP fetches to complete.
if err := g.Wait(); err == nil {
fmt.Println("Successfully fetched all URLs.")
}
}
// EnsureInfoForGroup ensures data validity of the group, if customer does not
// exist, creates if, if sub does not exist, creates it
func EnsureInfoForGroup(group *models.Group, username string) (*Usage, error) {
usage, err := fetchParallelizableUsageItems(group, username)
if err != nil {
return nil, err
}
var g errgroup.Group
g.Go(func() error {
expectedPlan, err := getPlan(usage.Subscription, usage.User.Total)
if err != nil {
return err
}
usage.ExpectedPlan = expectedPlan
usage.Due = uint64(usage.User.Total) * expectedPlan.Amount
return nil
})
g.Go(func() error {
if usage.Trial.User.Total == 0 {
return nil
}
trialPlan, err := getPlan(usage.Subscription, usage.Trial.User.Total)
if err != nil {
return err
}
usage.Trial.ExpectedPlan = trialPlan
usage.Trial.Due = uint64(usage.Trial.User.Total) * trialPlan.Amount
return nil
})
if err := g.Wait(); err != nil {
return nil, err
}
return usage, nil
}
// Main takes a list of urls and request parameters, then fetches the URLs and
// outputs the headers to stdout
func Main(cookie, etag string, gzip, ignoreBody bool, urls ...string) error {
for nurl, url := range urls {
// Separate subsequent lookups with newline
if nurl > 0 {
fmt.Println()
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
if gzip {
req.Header.Add("Accept-Encoding", "gzip, deflate")
}
if etag != "" {
req.Header.Add("If-None-Match", etag)
}
if cookie != "" {
req.Header.Add("Cookie", cookie)
}
start := time.Now()
resp, err := client.Do(req)
duration := time.Since(start)
var (
n int64
eg errgroup.Group
)
// Ignore the error if it's just our errRedirect
switch urlErr, ok := err.(*netURL.Error); {
case err == nil:
if ignoreBody {
break
}
eg.Go(func() error {
// Copying to /dev/null just to make sure this is real
n, err = io.Copy(ioutil.Discard, resp.Body)
duration = time.Since(start)
if err != nil {
return err
}
return nil
})
case ok && urlErr.Err == errRedirect:
default:
return err
}
fmt.Println("GET", url)
fmt.Println(resp.Proto, resp.Status)
fmt.Println()
fmt.Println(prettyprint.ResponseHeader(resp.Header))
if err := eg.Wait(); err != nil {
return err
}
if err := resp.Body.Close(); err != nil {
return err
}
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
fmt.Fprintf(tw, "Time\t%s\n", prettyprint.Duration(duration))
if n != 0 {
fmt.Fprintf(tw, "Content length\t%s\n", prettyprint.Size(n))
bps := prettyprint.Size(float64(n) / duration.Seconds())
fmt.Fprintf(tw, "Speed\t%s/s\n", bps)
}
if err := tw.Flush(); err != nil {
return err
}
}
return nil
}
func TestProcessContent(t *testing.T) {
Convey("Testing ProcessContent(), setting up temporary directory in /tmp", t, func() {
dir, err := ioutil.TempDir("/tmp", "testBlacklist")
So(err, ShouldBeNil)
defer os.RemoveAll(dir)
Convey("Testing ProcessContent()", func() {
c := NewConfig(
Dir(dir),
Ext("blacklist.conf"),
FileNameFmt("%v/%v.%v.%v"),
Method("GET"),
Nodes([]string{domains, hosts}),
Prefix("address="),
LTypes([]string{PreDomns, PreHosts, files, urls}),
)
tests := []struct {
err error
exp string
expDexMap list
expExcMap list
f string
fdata string
name string
obj IFace
}{
{
name: "ExRtObj",
err: nil,
exp: "[\nDesc:\t \"root-excludes exclusions\"\nDisabled: false\nFile:\t \"\"\nIP:\t \"0.0.0.0\"\nLtype:\t \"root-excludes\"\nName:\t \"root-excludes\"\nnType:\t \"excRoot\"\nPrefix:\t \"\"\nType:\t \"root-excludes\"\nURL:\t \"\"\n]",
expDexMap: list{entry: entry{"ytimg.com": 0}},
expExcMap: list{entry: entry{"ytimg.com": 0}},
obj: ExRtObj,
},
{
name: "ExDmObj",
err: nil,
exp: "[\nDesc:\t \"domn-excludes exclusions\"\nDisabled: false\nFile:\t \"\"\nIP:\t \"0.0.0.0\"\nLtype:\t \"domn-excludes\"\nName:\t \"domn-excludes\"\nnType:\t \"excDomn\"\nPrefix:\t \"\"\nType:\t \"domn-excludes\"\nURL:\t \"\"\n]",
expDexMap: list{RWMutex: &sync.RWMutex{}, entry: make(entry)},
expExcMap: list{RWMutex: &sync.RWMutex{}, entry: make(entry)},
obj: ExDmObj,
},
{
name: "ExHtObj",
err: nil,
exp: "[\nDesc:\t \"host-excludes exclusions\"\nDisabled: false\nFile:\t \"\"\nIP:\t \"192.168.168.1\"\nLtype:\t \"host-excludes\"\nName:\t \"host-excludes\"\nnType:\t \"excHost\"\nPrefix:\t \"\"\nType:\t \"host-excludes\"\nURL:\t \"\"\n]",
expDexMap: list{RWMutex: &sync.RWMutex{}, entry: make(entry)},
expExcMap: list{RWMutex: &sync.RWMutex{}, entry: make(entry)},
obj: ExHtObj,
},
{
name: "PreDObj",
err: nil,
exp: "[\nDesc:\t \"pre-configured-domain blacklist content\"\nDisabled: false\nFile:\t \"\"\nIP:\t \"0.0.0.0\"\nLtype:\t \"pre-configured-domain\"\nName:\t \"includes.[8]\"\nnType:\t \"preDomn\"\nPrefix:\t \"\"\nType:\t \"pre-configured-domain\"\nURL:\t \"\"\n]",
expDexMap: list{
entry: entry{
"adsrvr.org": 0,
"adtechus.net": 0,
"advertising.com": 0,
"centade.com": 0,
"doubleclick.net": 0,
"free-counter.co.uk": 0,
"intellitxt.com": 0,
"kiosked.com": 0,
},
},
expExcMap: list{entry: entry{"ytimg.com": 0}},
f: dir + "/pre-configured-domain.includes.[8].blacklist.conf",
fdata: "address=/adsrvr.org/0.0.0.0\naddress=/adtechus.net/0.0.0.0\naddress=/advertising.com/0.0.0.0\naddress=/centade.com/0.0.0.0\naddress=/doubleclick.net/0.0.0.0\naddress=/free-counter.co.uk/0.0.0.0\naddress=/intellitxt.com/0.0.0.0\naddress=/kiosked.com/0.0.0.0\n",
obj: PreDObj,
},
{
name: "PreHObj",
err: nil,
exp: "[\nDesc:\t \"pre-configured-host blacklist content\"\nDisabled: false\nFile:\t \"\"\nIP:\t \"192.168.168.1\"\nLtype:\t \"pre-configured-host\"\nName:\t \"includes.[1]\"\nnType:\t \"preHost\"\nPrefix:\t \"\"\nType:\t \"pre-configured-host\"\nURL:\t \"\"\n]",
expDexMap: list{entry: entry{"ytimg.com": 0}},
expExcMap: list{entry: entry{"ytimg.com": 0}},
f: dir + "/pre-configured-host.includes.[1].blacklist.conf",
fdata: "address=/beap.gemini.yahoo.com/192.168.168.1\n",
obj: PreHObj,
},
{
name: "FileObj",
err: fmt.Errorf("open %v/hosts./tasty.blacklist.conf: no such file or directory", dir),
exp: filesMin,
expDexMap: list{
entry: entry{
"cw.bad.ultraadverts.site.eu": 1,
"really.bad.phishing.site.ru": 1,
},
},
expExcMap: list{entry: entry{"ytimg.com": 0}},
f: dir + "/hosts.tasty.blacklist.conf",
fdata: "address=/cw.bad.ultraadverts.site.eu/10.10.10.10\naddress=/really.bad.phishing.site.ru/10.10.10.10\n",
obj: FileObj,
},
}
So(c.ReadCfg(&CFGstatic{Cfg: CfgMimimal}), ShouldBeNil)
for _, tt := range tests {
Convey("Testing "+tt.name+" ProcessContent()", func() {
obj, err := c.NewContent(tt.obj)
So(err, ShouldBeNil)
if tt.f != "" {
So(fmt.Sprint(obj), ShouldEqual, tt.exp)
}
var g errgroup.Group
g.Go(func() error { return c.ProcessContent(obj) })
err = g.Wait()
if err != nil {
Convey("Testing "+tt.name+" ProcessContent().Error():", func() {
Convey("Error should match expected", func() {
So(err, ShouldResemble, tt.err)
})
})
}
switch tt.f {
default:
reader, err := getFile(tt.f)
So(err, ShouldBeNil)
act, err := ioutil.ReadAll(reader)
So(err, ShouldBeNil)
Convey("Testing "+tt.name+" ProcessContent(): file data should match expected", func() {
So(string(act), ShouldEqual, tt.fdata)
})
case "":
Convey("Testing "+tt.name+" ProcessContent(): Dex map should match expected", func() {
So(c.Dex.entry, ShouldResemble, tt.expDexMap.entry)
})
Convey("Testing "+tt.name+" ProcessContent(): Exc map should match expected", func() {
So(c.Exc.entry, ShouldResemble, tt.expExcMap.entry)
})
Convey("Testing "+tt.name+" ProcessContent(): Obj should match expected", func() {
So(obj.String(), ShouldEqual, tt.exp)
})
}
})
}
})
})
}