/
mirror.go
140 lines (110 loc) · 3.09 KB
/
mirror.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
package command
import (
"flag"
"fmt"
"strings"
"github.com/funkygao/gafka/cmd/gk/command/mirror"
"github.com/funkygao/gocli"
)
type Mirror struct {
Ui cli.Ui
Cmd string
zone1, zone2 string
cluster1, cluster2 string
excludes string
topics string
debug bool
compress string
autoCommit bool
logLevel string
bandwidthLimit int64
progressStep int64
showStatus bool
}
func (this *Mirror) Run(args []string) (exitCode int) {
cmdFlags := flag.NewFlagSet("mirror", flag.ContinueOnError)
cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
cmdFlags.StringVar(&this.zone1, "z1", "", "")
cmdFlags.StringVar(&this.zone2, "z2", "", "")
cmdFlags.StringVar(&this.cluster1, "c1", "", "")
cmdFlags.StringVar(&this.cluster2, "c2", "", "")
cmdFlags.StringVar(&this.excludes, "excluded", "", "")
cmdFlags.StringVar(&this.topics, "t", "", "")
cmdFlags.StringVar(&this.logLevel, "level", "trace", "")
cmdFlags.BoolVar(&this.debug, "debug", false, "")
cmdFlags.BoolVar(&this.showStatus, "stat", false, "")
cmdFlags.StringVar(&this.compress, "compress", "", "")
cmdFlags.Int64Var(&this.bandwidthLimit, "net", 100, "")
cmdFlags.BoolVar(&this.autoCommit, "commit", true, "")
cmdFlags.Int64Var(&this.progressStep, "step", 10000, "")
if err := cmdFlags.Parse(args); err != nil {
return 1
}
if validateArgs(this, this.Ui).
require("-z1", "-z2", "-c1", "-c2").
invalid(args) {
return 2
}
topicsExcluded := make(map[string]struct{})
for _, e := range strings.Split(this.excludes, ",") {
if e != "" {
topicsExcluded[e] = struct{}{}
}
}
topicsOnly := make(map[string]struct{})
for _, t := range strings.Split(this.topics, ",") {
if t != "" {
topicsOnly[t] = struct{}{}
}
}
cf := mirror.DefaultConfig()
cf.ExcludedTopics = topicsExcluded
cf.TopicsOnly = topicsOnly
cf.Z1 = this.zone1
cf.Z2 = this.zone2
cf.C1 = this.cluster1
cf.C2 = this.cluster2
cf.Debug = this.debug
cf.ShowStatus = this.showStatus
cf.Compress = this.compress
cf.BandwidthLimit = this.bandwidthLimit
cf.AutoCommit = this.autoCommit
cf.ProgressStep = this.progressStep
setupLogging("mirror.log", this.logLevel, "panic")
m := mirror.New(cf)
return m.Main()
}
func (*Mirror) Synopsis() string {
return "Continuously copy data between two remote Kafka clusters"
}
func (this *Mirror) Help() string {
help := fmt.Sprintf(`
Usage: %s mirror [options]
%s
e,g.
gk mirror -z1 prod -c1 logstash -z2 mirror -c2 aggregator -net 100 -step 2000
Options:
-z1 from zone
-z2 to zone
-c1 from cluster
-c2 to cluster
-t comma separated topic names
Only mirror for the specified topics.
-level [debug|trace|info]
-stat
Show status. TODO
-exclude comma separated topic names
-net bandwidth limit in Mbps
Defaults 100Mbps.
0 means unlimited.
-step n
Defaults 5000.
-debug
-compress <gzip|snappy>
Defaults none.
-commit
Auto commit the checkpoint offset.
Defaults true.
`, this.Cmd, this.Synopsis())
return strings.TrimSpace(help)
}