func idempotentRecordChapter(bid interface{}, chapter Chapter, col *mgo.Collection) interface{} {
them := col.Find(bson.M{
"position": chapter.Position,
"book": bid,
}).Limit(1)
cpt, e := them.Count()
if e != nil {
panic(e)
}
if cpt > 0 {
ans := make(map[string]interface{})
e = them.One(ans)
if e != nil {
panic(e)
}
return ans["_id"]
}
e = col.Insert(bson.M{
"position": chapter.Position,
"book": bid,
})
if e != nil {
panic(e)
}
return idempotentRecordChapter(bid, chapter, col)
}
func AddUser(collection *mgo.Collection, name, username, password string) {
// Index
index := mgo.Index{
Key: []string{"username", "email"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err := collection.EnsureIndex(index)
if err != nil {
panic(err)
}
bcryptPassword, _ := bcrypt.GenerateFromPassword(
[]byte(password), bcrypt.DefaultCost)
// Insert Dataz
err = collection.Insert(&models.User{Name: name, Username: username, Password: bcryptPassword})
if err != nil {
panic(err)
}
}
func idempotentRecordBook(vid interface{}, book Book, col *mgo.Collection) interface{} {
them := col.Find(bson.M{
"name": book.Name,
"version": vid,
}).Limit(1)
cpt, e := them.Count()
if e != nil {
panic(e)
}
if cpt > 0 {
ans := make(map[string]interface{})
e = them.One(ans)
if e != nil {
panic(e)
}
return ans["_id"]
}
e = col.Insert(bson.M{
"name": book.Name,
"position": book.Position,
"version": vid,
})
if e != nil {
panic(e)
}
return idempotentRecordBook(vid, book, col)
}
func addPost(database *mgo.Database, collection *mgo.Collection, title, subtitle, slug, category, body, image string) (post models.Post) {
// Index
index := mgo.Index{
Key: []string{"shortid", "timestamp", "title", "tags"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
err := collection.EnsureIndex(index)
if err != nil {
panic(err)
}
// Insert Dataz
err = collection.Insert(&models.Post{
ShortID: models.GetNextSequence(database),
Title: title,
Category: category,
Slug: slug,
Subtitle: subtitle,
Body: body,
Timestamp: time.Now(),
Published: false})
if err != nil {
panic(err)
}
result := models.Post{}
collection.Find(bson.M{"title": title}).One(&result)
return result
}
func assertSequenceExists(collection *mgo.Collection) {
c, _ := collection.Find(nil).Count()
if c == 0 {
collection.Insert(&Counter{
Seq: 0})
}
}
func insertFakeAssets(c *mgo.Collection) {
assets := []Asset{
{
SlotID: 1,
AssetType: 2,
DivisionCode: "Truzzardi",
Environment: "preview",
},
{
SlotID: 1,
AssetType: 2,
DivisionCode: "Prada",
Environment: "preview",
},
{
SlotID: 1,
AssetType: 2,
DivisionCode: "Truzzardi",
Environment: "production",
},
{
SlotID: 1,
AssetType: 2,
DivisionCode: "VinDiesel",
Environment: "preview",
},
}
for _, asset := range assets {
err := c.Insert(asset)
if err != nil {
panic(err)
}
}
}
func (fs *BFS) copyDirectoryDocument(d *Directory, newprefix, oldprefix, newname string, c *mgo.Collection) error {
// update parent path prefix with new prefix
_parent_path := d.Header.Parent
_parent_path = strings.Replace(_parent_path, oldprefix, newprefix, 1)
// update header info
uuid, err := makeUUID()
if err != nil {
return err
}
nw := time.Now()
dt := formatDatetime(&nw)
id := fmt.Sprintf("%s:%s", uuid, dt)
d.Header.Parent = _parent_path
if newname != "" {
err = fs.validateDirName(newname)
if err != nil {
return err
}
d.Header.Name = newname
}
d.Header.Created = dt
d.Id = id
// save to mongodb
err = c.Insert(&d)
if err != nil {
return err
}
return nil
}
func AddAutoIncrementingField(c *mgo.Collection) {
i, e := c.Find(bson.M{"_id": "users"}).Count()
if e != nil {
panic(fmt.Sprintf("Could not Add Auto Incrementing Field! collection:%s err:%v\n", c.FullName, e))
}
if i > 0 {
return
}
c.Insert(bson.M{"_id": "users", "seq": uint32(0)})
}
func bulkInsert(col *mgo.Collection) {
msg := uuid.NewRandom().String()
_len := 100
bulk := make([]bson.M, _len)
for i := 0; i < _len; i++ {
bulk[i] = bson.M{"msg": msg, "counter": i}
}
err := col.Insert(bulk)
fmt.Print("Bulk Insert")
fmt.Print(err)
}
func CreateUser(w http.ResponseWriter, rew *http.Request) {
w.Header().Add("Content-Type", "text/html; charset=UTF-8")
if rew.Method == "POST" {
var Users *mgo.Collection = MongoDB.C("users")
md5 := crypto.MD5.New()
md5.Write([]byte(rew.FormValue("password")))
passwordhash := fmt.Sprintf("%x", md5.Sum(nil))
err := Users.Insert(bson.M{"username": rew.FormValue("username"), "password": passwordhash})
if err == nil {
io.WriteString(w, "Success.")
} else {
io.WriteString(w, err.Error())
}
} else if rew.Method == "GET" {
Template(TemplateCreateUser).Execute(w, nil)
}
}
func (fs *BFS) copyFileDocument(f *File, newprefix, oldprefix, newname string, c *mgo.Collection) error {
// update parent path prefix with new prefix
_parent_path := f.Header.Parent
_parent_path = strings.Replace(_parent_path, oldprefix, newprefix, 1)
// update header info
uuid, err := makeUUID()
if err != nil {
return err
}
nw := time.Now()
dt := formatDatetime(&nw)
id := fmt.Sprintf("%s:%s", uuid, dt)
f.Header.Parent = _parent_path
f.Header.Created = dt
if newname != "" {
err = fs.validateFileName(newname)
if err != nil {
return err
}
f.Header.Name = newname
}
f.Id = id
// check if file has an attachment and copy if true
_attch_path := f.AHeader.Filepointer
if _attch_path != "" {
_attch_dir := path.Dir(_attch_path)
_new_attch_path := path.Join(_attch_dir, id)
_, err = exec.Command("cp", _attch_path, _new_attch_path).Output()
if err != nil {
return err
}
// set new attachment filepointer
f.AHeader.Filepointer = _new_attch_path
}
// save to mongodb
err = c.Insert(&f)
if err != nil {
return err
}
return nil
}
// 开始导入文本小说内容
func TxtImport(story *mgo.Collection, book *mgo.Collection, sysFile *sysFile) {
f, _ := os.OpenFile(sysFile.fName, os.O_RDONLY, 0666)
defer f.Close()
m := bufio.NewReader(f)
// char := 0
words := 0
lines := 0
for {
s, ok := m.ReadString('\n')
words = len(strings.Fields(s))
if words > 0 {
story.Insert(&Story{sysFile.fShortName, lines, s, words})
}
lines++
if ok != nil {
break
}
}
book.Insert(&Book{f.Name(), sysFile.fSize, sysFile.fMtime, lines})
}
func SaveNewKotoba(user_id bson.ObjectId, word string, ll string, diff int, hatsuon string, h_ string,
imi string, i_ string, c *mgo.Collection) (*MongoKotoba, error) {
timenow := time.Now().Local()
hours, _ := time.ParseDuration(DEFAULT_REVIEW_TIM)
r := timenow.Add(hours)
imis := SplitImi(imi)
hatsuons := SplitHatsuon(hatsuon)
labels := SplitLabels(ll)
// add meanings
newK := MongoKotoba{Id: bson.NewObjectId(),
User_id: user_id,
Goi: word,
Hatsuons: *hatsuons,
Hatsuon_: h_,
Imis: *imis,
Imi_: i_,
Labels: *labels,
Level: 2,
Review: r,
Difficulty: diff,
Unlocked: time.Now().Local()}
return &newK, c.Insert(&newK)
}
func idempotentRecordVersion(bible *Bible, col *mgo.Collection) interface{} {
versions := col.Find(bson.M{
"name": bible.Version,
}).Limit(1)
cpt, e := versions.Count()
if e != nil {
panic(e)
}
if cpt > 0 {
ans := make(map[string]interface{})
e = versions.One(ans)
if e != nil {
panic(e)
}
return ans["_id"]
}
e = col.Insert(bson.M{
"name": bible.Version,
})
if e != nil {
panic(e)
}
return idempotentRecordVersion(bible, col)
}
func Insert(collection *mgo.Collection, i interface{}) bool {
err := collection.Insert(i)
return Err(err)
}
// insertonedoc inserts one doc
func insertonedoc(c *mgo.Collection) error {
//defer nmendtimer(starttimer())
err := c.Insert(&Mongodoc{123456789, "CalvinandHobbes", 123456789, 123456789})
return err
}
// insertdoc inserts docs into mongo, called from insertworker
func insertdoc(c *mgo.Collection, i int) error {
defer endtimer(starttimer(), i)
err := c.Insert(&Mongodoc{i, "Facebook", random(), random()}, &Mongodoc{i, "Twitter", random(), random()}, &Mongodoc{i, "Instagram", random(), random()})
return err
}
func Create(col *mgo.Collection, s Sample) error {
return col.Insert(s)
}
// Insert Account into Mongo
func insert(u Account, s *mgo.Collection) {
err := s.Insert(&u)
if err != nil {
panic(err)
}
}
/* UDP Handlers */
func saveMessage(dbc *mgo.Collection, msg string) {
err := dbc.Insert(prepareMessage(msg))
if err != nil {
log.Panic(err)
}
}
// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *ToMongoDB) Run() {
var collectionname string
var dbname string
var collection *mgo.Collection
var session *mgo.Session
var host = ""
var err error
var batch = 0
var count = 0
var maxindex = 0
var list []interface{}
for {
select {
case msgI := <-b.inrule:
// set host string for MongoDB server
host, err = util.ParseRequiredString(msgI, "Host")
if err != nil {
b.Error(err.Error())
continue
}
// set database name
dbname, err = util.ParseRequiredString(msgI, "Database")
if err != nil {
b.Error(err.Error())
continue
}
// set collection name
collectionname, err = util.ParseRequiredString(msgI, "Collection")
if err != nil {
b.Error(err.Error())
continue
}
// set number of records to insert at a time
batch, err = util.ParseInt(msgI, "BatchSize")
if err != nil || batch < 0 {
b.Error(errors.New("Error parsing batch size....setting to 0"))
batch = 0
} else {
if batch > 1 {
list = make([]interface{}, batch, batch)
// set maxindex to 1 minus batch size
// use maxindex for looping everywhere for consistency
maxindex = batch - 1
}
}
// create MongoDB connection
session, err = mgo.Dial(host)
if err != nil {
// swallowing a panic from mgo here - streamtools must not die
b.Error(errors.New("Could not initiate connection with MongoDB service"))
continue
}
// use the specified DB and collection
collection = session.DB(dbname).C(collectionname)
case <-b.quit:
// close connection to MongoDB and quit
if session != nil {
session.Close()
}
return
case msg := <-b.in:
// deal with inbound data
if session != nil {
// mgo is so cool - it will check if the message can be serialized to valid bson.
// So, no need to do a json.Marshal on the inbound. just append to the list and
// batch insert
if maxindex >= 1 {
if count <= maxindex {
list[count] = msg
count = count + 1
}
if count == maxindex {
// insert batch if count reaches batch size
err = collection.Insert(list...)
if err != nil {
b.Error(err.Error())
}
// reset list and count
list = make([]interface{}, batch, batch)
count = 0
}
} else {
// mgo coolness again. No need to do a json.Marshal on the inbound.
err = collection.Insert(msg)
if err != nil {
b.Error(err.Error())
}
}
} else {
b.Error(errors.New("MongoDB connection not initated or lost. Please check your MongoDB server or block settings."))
}
case respChan := <-b.queryrule:
// deal with a query request
respChan <- map[string]interface{}{
"Collection": collectionname,
"Database": dbname,
"Host": host,
"BatchSize": batch,
}
}
}
}
func (executor *OpsExecutor) ExecInsert(
content Document, coll *mgo.Collection) error {
return coll.Insert(content["o"])
}
func main() {
verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
mongourl := flag.String("mongourl", "", "record request/response in mongodb")
mock := flag.Bool("m", false, "send fake responses")
addr := flag.String("l", ":8080", "on which address should the proxy listen")
flag.Parse()
tmpdir := filepath.Join(os.TempDir(), "proxy-service")
if _, err := os.Stat(tmpdir); err != nil {
if os.IsNotExist(err) {
// Create temp directory to store body response
err = os.MkdirAll(tmpdir, 0777)
}
// err should be nil if we just created the directory
if err != nil {
panic(err)
}
}
db := new(mgo.Database)
c := new(mgo.Collection)
h := new(mgo.Collection)
rules := new(mgo.Collection)
if len(*mongourl) != 0 {
// Mongo DB connection
session, err := mgo.Dial(*mongourl)
if err != nil {
panic(err)
}
defer session.Close()
// Optional. Switch the session to a monotonic behavior.
session.SetMode(mgo.Monotonic, true)
db = session.DB("proxyservice")
c = db.C("log_logentry")
h = db.C("log_hostrewrite")
rules = db.C("log_rules")
} else {
db = nil
c = nil
h = nil
rules = nil
}
uuid.SwitchFormat(uuid.CleanHyphen, false)
proxy := goproxy.NewProxyHttpServer()
proxy.Verbose = *verbose
proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm)
proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
origin := ipAddrFromRemoteAddr(ctx.Req.RemoteAddr)
ctx.Logf("Origin: %s", origin)
/*ctx.RoundTripper = goproxy.RoundTripperFunc(func (req *http.Request, ctx *goproxy.ProxyCtx) (resp *http.Response, err error) {
//data := transport.RoundTripDetails{}
data, resp, err := tr.DetailedRoundTrip(req)
//log.Printf("%+v", data)
return
})*/
rewrite := Rewrite{}
if h != nil && h.Database != nil {
err := h.Find(bson.M{"host": req.Host, "active": true}).One(&rewrite)
if err == nil {
req.URL.Scheme = rewrite.DProtocol
req.URL.Host = rewrite.DHost
req.Host = rewrite.DHost
ctx.Logf("Rewrite: %+v, URL: %+v", rewrite, req.URL)
}
}
//log.Printf("%+v", getHost(req.RemoteAddr))
/*if ctx.UserData != nil {
from = ctx.UserData.(*transport.RoundTripDetails).TCPAddr.String()
}*/
//log.Printf("Request: %s %s %s", req.Method, req.Host, req.RequestURI)
//host := req.Host
// Request to domain--name-co-uk.mocky.dev
// will be forwarded to domain-name.co.uk
/*if strings.Contains(host, ".mocky.dev") {
host = strings.Replace(host, ".mocky.dev", "", 1)
host = strings.Replace(host, "-", ".", -1)
host = strings.Replace(host, "..", "-", -1)
} else if strings.Contains(host, ".proxy.dev") {
host = strings.Replace(host, ".proxy.dev", "", 1)
host = strings.Replace(host, "-", ".", -1)
host = strings.Replace(host, "..", "-", -1)
}*/
/*r, _ := regexp.Compile(".([0-9]+)$")
// Check if host is hostname.80 (host with port number)
res := r.FindStringSubmatch(host)
if res != nil && len(res[1]) != 0 {
host = strings.Replace(host, strings.Join([]string{".", res[1]}, ""), "", 1)
host = strings.Join([]string{host, res[1]}, ":")
log.Printf("Changing host to %v", host);
}*/
//log.Printf("Target Host: %s - Headers: %+v", host, req.Header)
//req.Host = host
//log.Printf("%+v", req)
var reqbody []byte
var bodyreader io.Reader
if rules != nil && rules.Database != nil && *mock && req.Method != "CONNECT" {
//reqbody := string(body[:])
//log.Printf("request body: %s", reqbody)
rule := Rule{}
//ctx.Logf("Looking for existing request")
/*fmt.Println("RequestURI:", req.RequestURI)
fmt.Println("Path:", req.URL.Path)
fmt.Println("Host:", req.Host)
fmt.Println("Method:", req.Method)*/
b := bson.M{"$and": []bson.M{
bson.M{"active": true},
//bson.M{"dynamic": false},
bson.M{"origin": bson.M{"$in": []interface{}{origin, false}}},
bson.M{"host": bson.M{"$in": []interface{}{req.Host, false}}},
bson.M{"method": bson.M{"$in": []interface{}{req.Method, false}}},
bson.M{"path": bson.M{"$in": []interface{}{req.URL.Path, false}}},
bson.M{"query": bson.M{"$in": []interface{}{req.URL.Query().Encode(), false}}},
}}
//b := bson.M{"active": true, "dynamic": false, "host": req.Host, "method": req.Method, "path": req.URL.Path, "query": req.URL.Query().Encode()}
ctx.Logf("Looking for a rule: %+v", b)
err := rules.Find(b).Sort("dynamic").One(&rule)
//log.Printf("Query: %+v, Res: %+v", b, rule)
if err == nil {
ctx.Logf("Found rule: %+v", rule)
status, err := strconv.Atoi(rule.Status)
if rule.Dynamic && c != nil && c.Database != nil {
result := Content{}
reqQuery := bson.M{"$and": []bson.M{
/*bson.M{"origin": bson.M{"$in": []interface{}{origin, false}},
},*/
bson.M{"request.host": bson.M{"$in": []interface{}{req.Host}}},
bson.M{"request.method": bson.M{"$in": []interface{}{req.Method}}},
bson.M{"request.path": bson.M{"$in": []interface{}{req.URL.Path}}},
bson.M{"response.status": bson.M{"$in": []interface{}{status}}},
/*bson.M{"query": bson.M{"$in": []interface{}{req.URL.Query().Encode()}},
},*/
}}
ctx.Logf("Query %+v", reqQuery)
//reqQuery := bson.M{"request.host": rule.Host, "request.method": rule.Method, "response.status": status, "request.path": rule.Path}
err = c.Find(reqQuery).Sort("-date").One(&result)
if err == nil && db != nil {
ctx.Logf("Found a dynamic rule matching, returning it: %+v", result)
respId := result.Response.FileId
//reqfile, _ := getMongoFileContent(ctx, *db, result.Request.FileId)
respfile, err := getMongoFileContent(ctx, *db, respId)
if respfile != nil && err == nil {
//reqbody := ioutil.NopCloser(bytes.NewBufferString(rule.ReqBody))
//respbody := ioutil.NopCloser(bytes.NewBufferString(rule.Body))
ctx.Logf("Header: %+v", result.Response.Headers)
resp := NewResponse(req, result.Response.Headers, status, respfile)
ctx.UserData = ContextUserData{Store: true, Time: 0, Body: req.Body, Header: req.Header, Origin: origin}
return req, resp
} else {
ctx.Logf("Couldn't retrieve the response body: %+v", err)
}
} else {
ctx.Logf("Couldn't find a dynamic response matching: %+v", err)
}
} else {
reqbody := ioutil.NopCloser(bytes.NewBufferString(rule.ReqBody))
respbody := ioutil.NopCloser(bytes.NewBufferString(rule.Body))
ctx.Logf("Found a static rule matching, returning it: %+v", rule)
resp := NewResponse(req, rule.RespHeader, status, respbody)
ctx.Delay = rule.Delay
ctx.UserData = ContextUserData{Store: true, Time: 0, Body: reqbody, Header: req.Header, Origin: origin}
return req, resp
}
/*result := Content{}
err = c.Find(bson.M{"_id": bson.ObjectIdHex(rule.Response)}).One(&result)*/
//err := c.Find(bson.M{"request.host": req.Host, "request.method": req.Method, "response.status": 200, "request.path": req.URL.Path}).Sort("-date").One(&result)
if err == nil {
//log.Printf("Found %+v", result)
//respbody := result.Response.Body
/*file, err := getMongoFileContent(*db, result.Response.FileId)
if err == nil {
resp := NewResponse(req, result.Response.Headers.Get("Content-Type"), result.Response.Status, file)
ctx.UserData = ContextUserData{Store: false, Time: 0, Header: result.Request.Headers, Origin: origin}
return req, resp
}*/
//ctx.Logf("Found one")
/*fmt.Println("Path:", result.Request.Path)
//fmt.Println("Body:", result.Request.Body)
fmt.Println("Method:", result.Request.Method)
fmt.Println("Host:", result.Request.Host)
fmt.Println("Time:", result.Request.Time)
fmt.Println("Date:", result.Request.Date)
fmt.Println("Headers:", result.Request.Headers)
//fmt.Println("Body:", result.Response.Body)
fmt.Println("Status:", result.Response.Status)
fmt.Println("Headers:", result.Response.Headers)*/
//resp := goproxy.NewResponse(req, result.Response.Headers.Get("Content-Type"), result.Response.Status, result.Response.Body)
//ctx.UserData = ContextUserData{Store: false, Time: 0}
//return req, resp
}
}
// read the whole body
reqbody, err = ioutil.ReadAll(req.Body)
if err != nil {
ctx.Warnf("Cannot read request body %s", err)
}
defer req.Body.Close()
req.Body = ioutil.NopCloser(bytes.NewBuffer(reqbody))
bodyreader = bytes.NewReader(reqbody)
} else {
bodyreader = req.Body
}
ctx.UserData = ContextUserData{Store: true, Time: time.Now().UnixNano(), Body: bodyreader, Header: req.Header, Origin: origin}
return req, nil
})
proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
//ctx.Logf("Method: %s - host: %s", ctx.Resp.Request.Method, ctx.Resp.Request.Host)
if c != nil && c.Database != nil && ctx.UserData != nil && ctx.UserData.(ContextUserData).Store && ctx.Resp.Request.Method != "CONNECT" && db != nil {
// get response content type
respctype := getContentType(ctx.Resp.Header.Get("Content-Type"))
//log.Printf("Resp Contenttype %s", respctype)
respid := bson.NewObjectId()
//log.Printf("Resp id: %s, host: %s", respid.Hex(), ctx.Resp.Request.Host)
filename := filepath.Join(tmpdir, respid.Hex())
//log.Printf("Duplicating Body file id: %s", respid.String())
fs := NewFileStream(filename, *db, respctype, respid, ctx)
reqctype := getContentType(ctx.Resp.Request.Header.Get("Content-Type"))
//log.Printf("Req Contenttype %s", reqctype)
if reqctype == "application/x-www-form-urlencoded" {
//log.Printf("setting req content type to text/plain for saving to mongo")
reqctype = "text/plain"
}
reqid := bson.NewObjectId()
//log.Printf("Req id: %s, host: %s", reqid.Hex(), ctx.Resp.Request.Host)
saveFileToMongo(*db, reqid, reqctype, ctx.UserData.(ContextUserData).Body, reqid.Hex(), ctx)
// prepare document
content := Content{
//Id: docid,
Request: Request{
Origin: ctx.UserData.(ContextUserData).Origin,
Path: ctx.Resp.Request.URL.Path,
Query: ctx.Resp.Request.URL.Query().Encode(),
FileId: reqid,
Url: ctx.Resp.Request.URL.String(),
Scheme: ctx.Resp.Request.URL.Scheme,
Host: ctx.Resp.Request.Host,
Method: ctx.Resp.Request.Method,
Time: float32(time.Now().UnixNano()-ctx.UserData.(ContextUserData).Time) / 1.0e9,
Headers: ctx.UserData.(ContextUserData).Header},
Response: Response{
Status: ctx.Resp.StatusCode,
Headers: ctx.Resp.Header,
FileId: respid},
SocketUUID: ctx.Uuid.Bytes(),
Date: time.Now(),
}
err := c.Insert(content)
if err != nil {
ctx.Logf("Can't insert document: %v", err)
} else {
ctx.Logf("MongoDB document saved: %+v", content)
}
resp.Body = NewTeeReadCloser(resp.Body, fs)
}
return resp
})
log.Println("Starting Proxy")
log.Fatalln(http.ListenAndServe(*addr, proxy))
}