func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
flag.Parse()
logger := log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile)
pcap, err := pcap.OpenOffline(*pcapFile)
if err != nil {
fmt.Fprintln(os.Stderr, "error opening pcap file:", err)
os.Exit(1)
}
h := mongocaputils.NewPacketHandler(pcap)
m := mongocaputils.NewMongoOpStream(*packetBufSize)
ch := make(chan struct{})
go func() {
defer close(ch)
for op := range m.Ops {
// TODO: add other op types
if opQuery, ok := op.Op.(*mongoproto.OpQuery); ok {
fbOp := map[string]interface{}{}
fbOp["ns"] = opQuery.FullCollectionName
fbOp["ntoskip"] = opQuery.NumberToSkip
fbOp["ntoreturn"] = opQuery.NumberToReturn
fbOp["ts"] = json.Date(op.Seen.Unix())
query, err := rawBSONToJSON(opQuery.Query)
if err != nil {
logger.Println(err)
if !*continueOnError {
os.Exit(1)
}
}
if strings.HasSuffix(opQuery.FullCollectionName, ".$cmd") {
fbOp["op"] = "command"
fbOp["command"] = query
} else {
fbOp["op"] = "query"
fbOp["query"] = query
}
fbOpStr, err := json.Marshal(fbOp)
if err != nil {
logger.Println(err)
if !*continueOnError {
os.Exit(1)
}
}
fmt.Println(string(fbOpStr))
}
}
}()
if err := h.Handle(m, -1); err != nil {
fmt.Fprintln(os.Stderr, "pcap_converter: error handling packet stream:", err)
}
<-ch
}
// GetBSONValueAsJSON is equivalent to ConvertBSONValueToJSON, but does not mutate its argument.
func GetBSONValueAsJSON(x interface{}) (interface{}, error) {
switch v := x.(type) {
case nil:
return nil, nil
case bool:
return v, nil
case *bson.M: // document
doc, err := getConvertedKeys(*v)
if err != nil {
return nil, err
}
return doc, err
case bson.M: // document
return getConvertedKeys(v)
case map[string]interface{}:
return getConvertedKeys(v)
case bson.D:
out := bson.D{}
for _, value := range v {
jsonValue, err := GetBSONValueAsJSON(value.Value)
if err != nil {
return nil, err
}
out = append(out, bson.DocElem{
Name: value.Name,
Value: jsonValue,
})
}
return MarshalD(out), nil
case MarshalD:
out, err := GetBSONValueAsJSON(bson.D(v))
if err != nil {
return nil, err
}
return MarshalD(out.(bson.D)), nil
case []interface{}: // array
out := []interface{}{}
for _, value := range v {
jsonValue, err := GetBSONValueAsJSON(value)
if err != nil {
return nil, err
}
out = append(out, jsonValue)
}
return out, nil
case string:
return v, nil // require no conversion
case int:
return json.NumberInt(v), nil
case bson.ObjectId: // ObjectId
return json.ObjectId(v.Hex()), nil
case bson.Decimal128:
return json.Decimal128{v}, nil
case time.Time: // Date
return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil
case int64: // NumberLong
return json.NumberLong(v), nil
case int32: // NumberInt
return json.NumberInt(v), nil
case float64:
return json.NumberFloat(v), nil
case float32:
return json.NumberFloat(float64(v)), nil
case []byte: // BinData (with generic type)
data := base64.StdEncoding.EncodeToString(v)
return json.BinData{0x00, data}, nil
case bson.Binary: // BinData
data := base64.StdEncoding.EncodeToString(v.Data)
return json.BinData{v.Kind, data}, nil
case mgo.DBRef: // DBRef
return json.DBRef{v.Collection, v.Id, v.Database}, nil
case bson.DBPointer: // DBPointer
return json.DBPointer{v.Namespace, v.Id}, nil
case bson.RegEx: // RegExp
return json.RegExp{v.Pattern, v.Options}, nil
case bson.MongoTimestamp: // Timestamp
timestamp := int64(v)
return json.Timestamp{
Seconds: uint32(timestamp >> 32),
Increment: uint32(timestamp),
}, nil
case bson.JavaScript: // JavaScript
var scope interface{}
var err error
if v.Scope != nil {
scope, err = GetBSONValueAsJSON(v.Scope)
if err != nil {
return nil, err
}
}
return json.JavaScript{v.Code, scope}, nil
default:
switch x {
case bson.MinKey: // MinKey
return json.MinKey{}, nil
case bson.MaxKey: // MaxKey
return json.MaxKey{}, nil
case bson.Undefined: // undefined
return json.Undefined{}, nil
}
}
return nil, fmt.Errorf("conversion of BSON value '%v' of type '%T' not supported", x, x)
}
func (d *LogDoc) Date(value string) mongo_json.Date {
n, _ := strconv.Atoi(value)
return mongo_json.Date(n)
}
func TestDateValue(t *testing.T) {
Convey("When converting JSON with Date values", t, func() {
Convey("works for Date object", func() {
key := "key"
jsonMap := map[string]interface{}{
key: json.Date(100),
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(time.Unix(0, int64(100*time.Millisecond))), ShouldBeTrue)
})
Convey("works for Date document", func() {
dates := []string{
"2006-01-02T15:04:05.000Z",
"2006-01-02T15:04:05.000-0700",
"2006-01-02T15:04:05Z",
"2006-01-02T15:04:05-0700",
"2006-01-02T15:04Z",
"2006-01-02T15:04-0700",
}
for _, dateString := range dates {
example := fmt.Sprintf(`{ "$date": "%v" }`, dateString)
Convey(fmt.Sprintf("of string ('%v')", example), func() {
key := "key"
jsonMap := map[string]interface{}{
key: map[string]interface{}{
"$date": dateString,
},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
// dateString is a valid time format string
date, err := time.Parse(dateString, dateString)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(date), ShouldBeTrue)
})
}
date := time.Unix(0, int64(time.Duration(1136214245000)*time.Millisecond))
Convey(`of $numberLong ('{ "$date": { "$numberLong": "1136214245000" } }')`, func() {
key := "key"
jsonMap := map[string]interface{}{
key: map[string]interface{}{
"$date": map[string]interface{}{
"$numberLong": "1136214245000",
},
},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(date), ShouldBeTrue)
})
Convey(`of json.Number ('{ "$date": 1136214245000 }')`, func() {
key := "key"
jsonMap := map[string]interface{}{
key: map[string]interface{}{
"$date": json.Number("1136214245000"),
},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(date), ShouldBeTrue)
})
Convey(`of numeric int64 ('{ "$date": 1136214245000 }')`, func() {
key := "key"
jsonMap := map[string]interface{}{
key: map[string]interface{}{
"$date": int64(1136214245000),
},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(date), ShouldBeTrue)
})
Convey(`of numeric float64 ('{ "$date": 1136214245000 }')`, func() {
key := "key"
jsonMap := map[string]interface{}{
key: map[string]interface{}{
"$date": float64(1136214245000),
},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
jsonValue, ok := jsonMap[key].(time.Time)
So(ok, ShouldBeTrue)
So(jsonValue.Equal(date), ShouldBeTrue)
})
})
})
}
// ConvertBSONValueToJSON walks through a document or an array and
// replaces any BSON value with its corresponding extended JSON type.
func ConvertBSONValueToJSON(x interface{}) (interface{}, error) {
switch v := x.(type) {
case nil:
return nil, nil
case bool:
return v, nil
case *bson.M: // document
doc, err := convertKeys(*v)
if err != nil {
return nil, err
}
return doc, err
case bson.M: // document
return convertKeys(v)
case []interface{}: // array
for i, value := range v {
jsonValue, err := ConvertBSONValueToJSON(value)
if err != nil {
return nil, err
}
v[i] = jsonValue
}
return v, nil
case string, float64:
return v, nil // require no conversion
case int:
return json.NumberInt(v), nil
case bson.ObjectId: // ObjectId
return json.ObjectId(v.Hex()), nil
case time.Time: // Date
return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil
case int64: // NumberLong
return json.NumberLong(v), nil
case int32: // NumberInt
return json.NumberInt(v), nil
case []byte: // BinData (with generic type)
data := base64.StdEncoding.EncodeToString(v)
return json.BinData{0x00, data}, nil
case bson.Binary: // BinData
data := base64.StdEncoding.EncodeToString(v.Data)
return json.BinData{v.Kind, data}, nil
case mgo.DBRef: // DBRef
return json.DBRef{v.Collection, v.Id, v.Database}, nil
case bson.RegEx: // RegExp
return json.RegExp{v.Pattern, v.Options}, nil
case bson.MongoTimestamp: // Timestamp
inc := uint32(int64(v) & 0xffff)
secs := uint32((int64(v) & (0xffff << 32)) >> 32)
return json.Timestamp{secs, inc}, nil
default:
switch x {
case bson.MinKey: // MinKey
return json.MinKey{}, nil
case bson.MaxKey: // MaxKey
return json.MaxKey{}, nil
case bson.Undefined: // undefined
return json.Undefined{}, nil
}
}
return nil, fmt.Errorf("Conversion of BSON type '%v' not supported %v", reflect.TypeOf(x), x)
}
// ConvertBSONValueToJSON walks through a document or an array and
// converts any BSON value to its corresponding extended JSON type.
// It returns the converted JSON document and any error encountered.
func ConvertBSONValueToJSON(x interface{}) (interface{}, error) {
switch v := x.(type) {
case nil:
return nil, nil
case bool:
return v, nil
case *bson.M: // document
doc, err := convertKeys(*v)
if err != nil {
return nil, err
}
return doc, err
case bson.M: // document
return convertKeys(v)
case map[string]interface{}:
return convertKeys(v)
case bson.D:
for i, value := range v {
jsonValue, err := ConvertBSONValueToJSON(value.Value)
if err != nil {
return nil, err
}
v[i].Value = jsonValue
}
return MarshalD(v), nil
case MarshalD:
return v, nil
case []interface{}: // array
for i, value := range v {
jsonValue, err := ConvertBSONValueToJSON(value)
if err != nil {
return nil, err
}
v[i] = jsonValue
}
return v, nil
case string:
return v, nil // require no conversion
case int:
return json.NumberInt(v), nil
case bson.ObjectId: // ObjectId
return json.ObjectId(v.Hex()), nil
case time.Time: // Date
return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil
case int64: // NumberLong
return json.NumberLong(v), nil
case int32: // NumberInt
return json.NumberInt(v), nil
case float64:
return json.NumberFloat(v), nil
case float32:
return json.NumberFloat(float64(v)), nil
case []byte: // BinData (with generic type)
data := base64.StdEncoding.EncodeToString(v)
return json.BinData{0x00, data}, nil
case bson.Binary: // BinData
data := base64.StdEncoding.EncodeToString(v.Data)
return json.BinData{v.Kind, data}, nil
case mgo.DBRef: // DBRef
return json.DBRef{v.Collection, v.Id, v.Database}, nil
case bson.DBPointer: // DBPointer
return json.DBPointer{v.Namespace, v.Id}, nil
case bson.RegEx: // RegExp
return json.RegExp{v.Pattern, v.Options}, nil
case bson.MongoTimestamp: // Timestamp
timestamp := int64(v)
return json.Timestamp{
Seconds: uint32(timestamp >> 32),
Increment: uint32(timestamp),
}, nil
case bson.JavaScript: // JavaScript
var scope interface{}
var err error
if v.Scope != nil {
scope, err = ConvertBSONValueToJSON(v.Scope)
if err != nil {
return nil, err
}
}
return json.JavaScript{v.Code, scope}, nil
default:
switch x {
case bson.MinKey: // MinKey
return json.MinKey{}, nil
case bson.MaxKey: // MaxKey
return json.MaxKey{}, nil
case bson.Undefined: // undefined
return json.Undefined{}, nil
}
}
return nil, fmt.Errorf("conversion of BSON type '%v' not supported %v", reflect.TypeOf(x), x)
}
// ConvertBSONValueToJSON walks through a document or an array and converts any
// BSON value to its corresponding extended JSON type. It returns the converted
// JSON document and any error encountered.
func ConvertBSONValueToJSON(x interface{}) (interface{}, error) {
switch v := x.(type) {
case nil:
return nil, nil
case bool:
return v, nil
case *bson.M: // document
doc, err := convertKeys(*v)
if err != nil {
return nil, err
}
return doc, err
case bson.M: // document
return convertKeys(v)
case map[string]interface{}:
return convertKeys(v)
case []bson.Raw:
out := make([]interface{}, len(v))
for i, value := range v {
out[i] = value
}
return ConvertBSONValueToJSON(out)
case bson.Raw:
// Unmarshal the raw into a bson.D, then process that.
convertedFromRaw := bson.D{}
err := v.Unmarshal(&convertedFromRaw)
if err != nil {
return nil, err
}
return ConvertBSONValueToJSON(convertedFromRaw)
case (*bson.Raw):
return ConvertBSONValueToJSON(*v)
case (*bson.D):
return ConvertBSONValueToJSON(*v)
case bson.D:
for i, value := range v {
jsonValue, err := ConvertBSONValueToJSON(value.Value)
if err != nil {
return nil, err
}
v[i].Value = jsonValue
}
return v.Map(), nil
case []bson.D:
out := make([]interface{}, len(v))
for i, value := range v {
out[i] = value
}
return ConvertBSONValueToJSON(out)
case []interface{}: // array
for i, value := range v {
jsonValue, err := ConvertBSONValueToJSON(value)
if err != nil {
return nil, err
}
v[i] = jsonValue
}
return v, nil
case string:
return v, nil // require no conversion
case int:
return json.NumberInt(v), nil
case bson.ObjectId: // ObjectId
return json.ObjectId(v.Hex()), nil
case time.Time: // Date
return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil
case int64: // NumberLong
return json.NumberLong(v), nil
case int32: // NumberInt
return json.NumberInt(v), nil
case float64:
return json.NumberFloat(v), nil
case float32:
return json.NumberFloat(float64(v)), nil
case []byte: // BinData (with generic type)
data := base64.StdEncoding.EncodeToString(v)
return json.BinData{0x00, data}, nil
case bson.Binary: // BinData
data := base64.StdEncoding.EncodeToString(v.Data)
return json.BinData{v.Kind, data}, nil
case mgo.DBRef: // DBRef
return map[string]interface{}{"$ref": v.Collection, "$id": v.Id}, nil
//case bson.DBPointer: // DBPointer
//return json.DBPointer{v.Namespace, v.Id}, nil
case bson.RegEx: // RegExp
return json.RegExp{v.Pattern, v.Options}, nil
case bson.MongoTimestamp: // Timestamp
timestamp := int64(v)
return json.Timestamp{
Seconds: uint32(timestamp >> 32),
Increment: uint32(timestamp),
}, nil
case bson.JavaScript: // JavaScript
var scope interface{}
var err error
if v.Scope != nil {
scope, err = ConvertBSONValueToJSON(v.Scope)
if err != nil {
return nil, err
}
}
return json.JavaScript{v.Code, scope}, nil
default:
switch x {
case bson.MinKey: // MinKey
return json.MinKey{}, nil
case bson.MaxKey: // MaxKey
return json.MaxKey{}, nil
case bson.Undefined: // undefined
return json.Undefined{}, nil
}
}
if valueOfX := reflect.ValueOf(x); valueOfX.Kind() == reflect.Slice || valueOfX.Kind() == reflect.Array {
result := make([]interface{}, 0, valueOfX.Len())
for i := 0; i < (valueOfX.Len()); i++ {
v := valueOfX.Index(i).Interface()
jsonResult, err := ConvertBSONValueToJSON(v)
if err != nil {
return nil, err
}
result = append(result, jsonResult)
}
return result, nil
}
return nil, fmt.Errorf("conversion of BSON type '%v' not supported %v", reflect.TypeOf(x), x)
}