func TestMaxDepth(t *testing.T) {
var (
r run
s search
)
var expectedfiles = []string{
basedir + "/" + TESTDATA[0].name,
}
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
s.Names = append(s.Names, "^"+TESTDATA[0].name+"$")
s.Contents = append(s.Contents, TESTDATA[0].content)
s.Options.MatchAll = true
s.Options.MaxDepth = 5
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
if evalResults([]byte(out), expectedfiles) != nil {
t.Fatal(err)
}
}
func TestContentSearch(t *testing.T) {
for _, tp := range TESTDATA {
var (
r run
s search
)
var expectedfiles = []string{
basedir + "/" + tp.name,
basedir + subdirs + tp.name,
}
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
s.Contents = append(s.Contents, tp.content)
s.Contents = append(s.Contents, "!^FOOBAR$")
s.Options.MatchAll = true
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
func TestAllHashes(t *testing.T) {
for _, tp := range TESTDATA {
var (
r run
s search
)
var expectedfiles = []string{
basedir + "/" + tp.name,
basedir + subdirs + tp.name,
}
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
s.MD5 = append(s.MD5, tp.md5)
s.SHA1 = append(s.SHA1, tp.sha1)
s.SHA2 = append(s.SHA2, tp.sha2)
s.SHA3 = append(s.SHA3, tp.sha3)
s.Options.MatchAll = true
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters, false)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
func TestMacroal(t *testing.T) {
var MacroalTestCases = []macroaltest{
macroaltest{
desc: "want testfile0 with all lines matching '^(.+)?$', should find 2 files",
name: "^" + TESTDATA[0].name + "$",
content: "^(.+)?$",
expectedfiles: []string{
basedir + "/" + TESTDATA[0].name,
basedir + subdirs + TESTDATA[0].name},
},
macroaltest{
desc: "want testfile0 with no line matching '^(.+)?$', should find 0 file",
name: "^" + TESTDATA[0].name + "$",
content: "!^(.+)?$",
expectedfiles: []string{""},
},
macroaltest{
desc: "want testfile0 with no line matching '!FOOBAR', should find 2 files",
name: "^" + TESTDATA[0].name + "$",
content: "!FOOBAR",
expectedfiles: []string{
basedir + "/" + TESTDATA[0].name,
basedir + subdirs + TESTDATA[0].name},
},
macroaltest{
desc: "want testfile0 with all lines matching 'FOOBAR', should find 0 file",
name: "^" + TESTDATA[0].name + "$",
content: "FOOBAR",
expectedfiles: []string{""},
},
}
for _, mt := range MacroalTestCases {
t.Log(mt.desc)
var r run
var s search
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
s.Names = append(s.Names, mt.name)
s.Contents = append(s.Contents, mt.content)
s.Options.MatchAll = true
s.Options.Macroal = true
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), mt.expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
func TestHashes(t *testing.T) {
for _, hashtype := range []string{`md5`, `sha1`, `sha256`, `sha384`, `sha512`,
`sha3_224`, `sha3_256`, `sha3_384`, `sha3_512`} {
for _, tp := range TESTDATA {
var (
r run
s search
)
var expectedfiles = []string{
basedir + "/" + tp.name,
basedir + subdirs + tp.name,
}
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
switch hashtype {
case `md5`:
s.MD5 = append(s.MD5, tp.md5)
case `sha1`:
s.SHA1 = append(s.SHA1, tp.sha1)
case `sha256`:
s.SHA256 = append(s.SHA256, tp.sha256)
case `sha384`:
s.SHA384 = append(s.SHA384, tp.sha384)
case `sha512`:
s.SHA512 = append(s.SHA512, tp.sha512)
case `sha3_224`:
s.SHA3_224 = append(s.SHA3_224, tp.sha3_224)
case `sha3_256`:
s.SHA3_256 = append(s.SHA3_256, tp.sha3_256)
case `sha3_384`:
s.SHA3_384 = append(s.SHA3_384, tp.sha3_384)
case `sha3_512`:
s.SHA3_512 = append(s.SHA3_512, tp.sha3_512)
}
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
}
func TestFindGoTestProcess(t *testing.T) {
var (
r run
s search
)
r.Parameters = *newParameters()
s.Names = append(s.Names, "go-build")
s.Bytes = append(s.Bytes, "7465737420736561726368206c6f6f6b696e6720666f722073656c66")
s.Contents = append(s.Contents, "test search looking for self")
s.Options.MatchAll = true
s.Options.Offset = 0.0
s.Options.MaxLength = 10000000
s.Options.LogFailures = true
r.Parameters.Searches["testsearch"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = json.Unmarshal([]byte(out), &r.Results)
if err != nil {
t.Fatal(err)
}
if !r.Results.Success {
t.Fatal("failed to run memory search")
}
if !r.Results.FoundAnything {
t.Fatal("should have found own go test process but didn't")
}
prints, err := r.PrintResults(r.Results, false)
if err != nil {
t.Fatal(err)
}
if len(prints) < 2 {
t.Fatal("not enough results printed")
}
prints, err = r.PrintResults(r.Results, true)
if err != nil {
t.Fatal(err)
}
if len(prints) != 1 {
t.Fatal("wrong number of results, should be one")
}
}
func TestDecompressedContentSearch(t *testing.T) {
for _, tp := range TESTDATA {
var (
r run
s search
)
var expectedfiles = []string{
basedir + "/" + tp.name,
basedir + subdirs + tp.name,
}
// testfile8 is a corrupt gzip and should fail to decompress
if tp.name == "testfile8" {
expectedfiles = []string{""}
}
r.Parameters = *newParameters()
s.Paths = append(s.Paths, basedir)
if tp.decompressedcontent != "" {
s.Contents = append(s.Contents, tp.decompressedcontent)
} else {
s.Contents = append(s.Contents, tp.content)
}
s.Contents = append(s.Contents, "!^FOOBAR$")
s.Options.MatchAll = true
s.Options.Decompress = true
r.Parameters.Searches["s1"] = s
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
// This function is used to call the file module from this module. In order to
// avoid exporting types from the file module, we construct parameters for the
// file module using the parameter creation functions (passing command line
// arguments).
//
// We use the file modules file system location functions here to avoid
// duplicating functionality in this module.
func fileModuleLocator(pattern string, regex bool, root string, depth int) ([]string, error) {
ret := make([]string, 0)
// Build a pseudo-run struct to let us call the file module.
run := modules.Available["file"].NewRun()
args := make([]string, 0)
args = append(args, "-path", root)
args = append(args, "-name", pattern)
args = append(args, "-maxdepth", strconv.Itoa(depth))
param, err := run.(modules.HasParamsParser).ParamsParser(args)
buf, err := modules.MakeMessage(modules.MsgClassParameters, param, false)
if err != nil {
return ret, nil
}
rdr := bytes.NewReader(buf)
res := run.Run(rdr)
var modresult modules.Result
var sr file.SearchResults
err = json.Unmarshal([]byte(res), &modresult)
if err != nil {
return ret, err
}
err = modresult.GetElements(&sr)
if err != nil {
return ret, err
}
p0, ok := sr["s1"]
if !ok {
return ret, fmt.Errorf("result in file module call was missing")
}
for _, x := range p0 {
ret = append(ret, x.File)
}
return ret, nil
}
func TestSearches(t *testing.T) {
var parameters = []testParams{
{true, `{"searches":{"s1":{"names":["go"]}}}`},
{false, `{"searches":{"s1":{"libraries":["^caribou.so$"]}}}`},
{true, `{"searches":{"s1":{"contents":["memory_test"], "names": ["go"]}}}`},
{false, `{"searches":{"s1":{"names":["1983yrotewdshhhoiufhes7fd29"],"bytes":["ffffffffaaaabbbbcccceeee"],"options":{"matchall": true}}}}`},
}
for _, tp := range parameters {
var r run
r.Parameters = *newParameters()
err := json.Unmarshal([]byte(tp.params), &r.Parameters)
if err != nil {
t.Fatal(err)
}
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = json.Unmarshal([]byte(out), &r.Results)
if err != nil {
t.Fatal(err)
}
if !r.Results.Success {
t.Fatal("failed to run memory search")
}
if r.Results.FoundAnything && !tp.expect {
t.Fatalf("found something for search '%s' and shouldn't have", tp.params)
} else if !r.Results.FoundAnything && tp.expect {
t.Fatalf("found nothing for search '%s' and should have", tp.params)
}
}
}
func main() {
var (
conf client.Configuration
cli client.Client
err error
op mig.Operation
a mig.Action
migrc, show, render, target, expiration, afile string
verbose bool
modargs []string
run interface{}
)
defer func() {
if e := recover(); e != nil {
fmt.Fprintf(os.Stderr, "%v\n", e)
}
}()
homedir := client.FindHomedir()
fs := flag.NewFlagSet("mig flag", flag.ContinueOnError)
fs.Usage = continueOnFlagError
fs.StringVar(&migrc, "c", homedir+"/.migrc", "alternative configuration file")
fs.StringVar(&show, "show", "found", "type of results to show")
fs.StringVar(&render, "render", "text", "results rendering mode")
fs.StringVar(&target, "t", fmt.Sprintf("status='%s' AND mode='daemon'", mig.AgtStatusOnline), "action target")
fs.StringVar(&expiration, "e", "300s", "expiration")
fs.StringVar(&afile, "i", "/path/to/file", "Load action from file")
fs.BoolVar(&verbose, "v", false, "Enable verbose output")
// if first argument is missing, or is help, print help
// otherwise, pass the remainder of the arguments to the module for parsing
// this client is agnostic to module parameters
if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
usage()
}
if len(os.Args) < 2 || os.Args[1] == "-V" {
fmt.Println(version)
os.Exit(0)
}
// when reading the action from a file, go directly to launch
if os.Args[1] == "-i" {
err = fs.Parse(os.Args[1:])
if err != nil {
panic(err)
}
if afile == "/path/to/file" {
panic("-i flag must take an action file path as argument")
}
a, err = mig.ActionFromFile(afile)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "[info] launching action from file, all flags are ignored\n")
goto readytolaunch
}
// arguments parsing works as follow:
// * os.Args[1] must contain the name of the module to launch. we first verify
// that a module exist for this name and then continue parsing
// * os.Args[2:] contains both global options and module parameters. We parse the
// whole []string to extract global options, and module parameters will be left
// unparsed in fs.Args()
// * fs.Args() with the module parameters is passed as a string to the module parser
// which will return a module operation to store in the action
op.Module = os.Args[1]
if _, ok := modules.Available[op.Module]; !ok {
panic("Unknown module " + op.Module)
}
// -- Ugly hack Warning --
// Parse() will fail on the first flag that is not defined, but in our case module flags
// are defined in the module packages and not in this program. Therefore, the flag parse error
// is expected. Unfortunately, Parse() writes directly to stderr and displays the error to
// the user, which confuses them. The right fix would be to prevent Parse() from writing to
// stderr, since that's really the job of the calling program, but in the meantime we work around
// it by redirecting stderr to null before calling Parse(), and put it back to normal afterward.
// for ref, issue is at https://github.com/golang/go/blob/master/src/flag/flag.go#L793
fs.SetOutput(os.NewFile(uintptr(87592), os.DevNull))
err = fs.Parse(os.Args[2:])
fs.SetOutput(nil)
if err != nil {
// ignore the flag not defined error, which is expected because
// module parameters are defined in modules and not in main
if len(err.Error()) > 30 && err.Error()[0:29] == "flag provided but not defined" {
// requeue the parameter that failed
modargs = append(modargs, err.Error()[31:])
} else {
// if it's another error, panic
panic(err)
}
}
for _, arg := range fs.Args() {
modargs = append(modargs, arg)
}
run = modules.Available[op.Module].NewRun()
if _, ok := run.(modules.HasParamsParser); !ok {
fmt.Fprintf(os.Stderr, "[error] module '%s' does not support command line invocation\n", op.Module)
os.Exit(2)
}
op.Parameters, err = run.(modules.HasParamsParser).ParamsParser(modargs)
if err != nil || op.Parameters == nil {
panic(err)
}
// If running against the local target, don't post the action to the MIG API
// but run it locally instead.
if target == "local" {
msg, err := modules.MakeMessage(modules.MsgClassParameters, op.Parameters)
if err != nil {
panic(err)
}
out := run.(modules.Runner).Run(bytes.NewBuffer(msg))
if len(out) == 0 {
panic("got empty results, run failed")
}
if _, ok := run.(modules.HasResultsPrinter); ok {
var modres modules.Result
err := json.Unmarshal([]byte(out), &modres)
if err != nil {
panic(err)
}
outRes, err := run.(modules.HasResultsPrinter).PrintResults(modres, true)
if err != nil {
panic(err)
}
for _, resLine := range outRes {
fmt.Println(resLine)
}
} else {
out = fmt.Sprintf("%s\n", out)
}
os.Exit(0)
}
a.Operations = append(a.Operations, op)
for _, arg := range os.Args[1:] {
a.Name += arg + " "
}
a.Target = target
readytolaunch:
// instanciate an API client
conf, err = client.ReadConfiguration(migrc)
if err != nil {
panic(err)
}
cli, err = client.NewClient(conf, "cmd-"+version)
if err != nil {
panic(err)
}
if verbose {
cli.EnableDebug()
}
// set the validity 60 second in the past to deal with clock skew
a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
period, err := time.ParseDuration(expiration)
if err != nil {
panic(err)
}
a.ExpireAfter = a.ValidFrom.Add(period)
// add extra 60 seconds taken for clock skew
a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()
asig, err := cli.SignAction(a)
if err != nil {
panic(err)
}
a = asig
// evaluate target before launch, give a change to cancel before going out to agents
agents, err := cli.EvaluateAgentTarget(a.Target)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "\x1b[33m%d agents will be targeted. ctrl+c to cancel. launching in \x1b[0m", len(agents))
for i := 5; i > 0; i-- {
time.Sleep(1 * time.Second)
fmt.Fprintf(os.Stderr, "\x1b[33m%d\x1b[0m ", i)
}
fmt.Fprintf(os.Stderr, "\x1b[33mGO\n\x1b[0m")
// launch and follow
a, err = cli.PostAction(a)
if err != nil {
panic(err)
}
c := make(chan os.Signal, 1)
done := make(chan bool, 1)
signal.Notify(c, os.Interrupt)
go func() {
err = cli.FollowAction(a)
if err != nil {
panic(err)
}
done <- true
}()
select {
case <-c:
fmt.Fprintf(os.Stderr, "stop following action. agents may still be running. printing available results:\n")
goto printresults
case <-done:
goto printresults
}
printresults:
err = cli.PrintActionResults(a, show, render)
if err != nil {
panic(err)
}
}
func TestMismatch(t *testing.T) {
var MismatchTestCases = []mismatchtest{
mismatchtest{
desc: "want files that don't match name '^testfile0' with maxdept=1, should find testfile1 and testfile2",
search: search{
Paths: []string{basedir},
Names: []string{"^" + TESTDATA[0].name + "$"},
Options: options{
MaxDepth: 1,
Mismatch: []string{"name"},
},
},
expectedfiles: []string{
basedir + "/" + TESTDATA[1].name,
basedir + "/" + TESTDATA[2].name},
},
mismatchtest{
desc: "want files that don't have a size of 190 bytes or larger than 10{k,m,g,t} or smaller than 10 bytes, should find testfile1 and testfile2",
search: search{
Paths: []string{basedir},
Sizes: []string{"190", ">10k", ">10m", ">10g", ">10t", "<10"},
Options: options{
MaxDepth: 1,
MatchAll: true,
Mismatch: []string{"size"},
},
},
expectedfiles: []string{
basedir + "/" + TESTDATA[1].name,
basedir + "/" + TESTDATA[2].name},
},
mismatchtest{
desc: "want files that have not been modified in the last hour ago, should find nothing",
search: search{
Paths: []string{basedir + subdirs, basedir},
Mtimes: []string{"<1h"},
Options: options{
Mismatch: []string{"mtime"},
},
},
expectedfiles: []string{""},
},
mismatchtest{
desc: "want files that don't have 644 permissions, should find nothing",
search: search{
Paths: []string{basedir},
Modes: []string{"-rw-r--r--"},
Options: options{
Mismatch: []string{"mode"},
},
},
expectedfiles: []string{""},
},
mismatchtest{
desc: "want files that don't a name different than testfile0, should find testfile0",
search: search{
Paths: []string{basedir},
Names: []string{"!^testfile0$"},
Options: options{
Mismatch: []string{"name"},
},
},
expectedfiles: []string{
basedir + "/" + TESTDATA[0].name,
basedir + subdirs + TESTDATA[0].name},
},
mismatchtest{
desc: "test matchall+macroal+mismatch: want file where at least one line fails to match the regex on testfile0, should find testfile1 that has the extra line 'some other other text'",
search: search{
Paths: []string{basedir},
Names: []string{"^testfile(0|1)$"},
Contents: []string{`^((---.+)|(#.+)|(\s+)|(some (other )?text))?$`},
Options: options{
MatchAll: true,
Macroal: true,
Mismatch: []string{"content"},
},
},
expectedfiles: []string{
basedir + "/" + TESTDATA[1].name,
basedir + subdirs + TESTDATA[1].name},
},
mismatchtest{
desc: "want files that don't match the hashes of testfile2, should find testfile0 & 1",
search: search{
Paths: []string{basedir},
MD5: []string{TESTDATA[2].md5},
SHA1: []string{TESTDATA[2].sha1},
SHA256: []string{TESTDATA[2].sha256},
SHA384: []string{TESTDATA[2].sha384},
SHA512: []string{TESTDATA[2].sha512},
SHA3_224: []string{TESTDATA[2].sha3_224},
SHA3_256: []string{TESTDATA[2].sha3_256},
SHA3_384: []string{TESTDATA[2].sha3_384},
SHA3_512: []string{TESTDATA[2].sha3_512},
Options: options{
MaxDepth: 1,
MatchAll: true,
Mismatch: []string{`md5`, `sha1`, `sha256`, `sha384`, `sha512`,
`sha3_224`, `sha3_256`, `sha3_384`, `sha3_512`},
},
},
expectedfiles: []string{
basedir + "/" + TESTDATA[0].name,
basedir + "/" + TESTDATA[1].name},
},
}
for _, mt := range MismatchTestCases {
t.Log(mt.desc)
var r run
r.Parameters = *newParameters()
r.Parameters.Searches["s1"] = mt.search
msg, err := modules.MakeMessage(modules.MsgClassParameters, r.Parameters)
if err != nil {
t.Fatal(err)
}
t.Logf("%s\n", msg)
out := r.Run(bytes.NewBuffer(msg))
if len(out) == 0 {
t.Fatal("run failed")
}
t.Log(out)
err = evalResults([]byte(out), mt.expectedfiles)
if err != nil {
t.Fatal(err)
}
}
}
