func NewAPI(baseURL string, username string, password string) *API {
auth := &gopencils.BasicAuth{username, password}
return &API{
rest: gopencils.Api(baseURL+"/rest/api", auth),
json: gopencils.Api(
baseURL+"/rpc/json-rpc/confluenceservice-v2",
auth,
),
}
}
// GetProbes returns data for a collection of probes
func GetProbes(opts map[string]string) (p []Probe, err error) {
auth := WantAuth()
api := gopencils.Api(apiEndpoint, auth)
rawlist, err := fetchOneProbePage(api, opts)
// Empty answer
if rawlist.Count == 0 {
return nil, fmt.Errorf("empty probe list")
}
var res []Probe
res = append(res, rawlist.Results...)
if rawlist.Next != "" {
// We have pagination
for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
opts["page"] = pn
rawlist, err = fetchOneProbePage(api, opts)
if err != nil {
return
}
res = append(res, rawlist.Results...)
}
}
p = res
return
}
// The user will be redirected back to this handler, that takes the
// "code" query parameter and Exchanges it for an access token.
func handler(w http.ResponseWriter, r *http.Request) {
t := &oauth.Transport{Config: config}
t.Exchange(r.FormValue("code"))
// The Transport now has a valid Token. Create an *http.Client
// with which we can make authenticated API requests.
c := t.Client()
// Now you can pass the authenticated Client to gopencils, and
// it will be used to make all the requests
api := gopencils.Api("http://your-api-url.com/api/", c)
// Create a pointer to our response struct, which will hold the response
re := &respStruct{}
// Maybe some payload to send along with the request?
payload := map[string]interface{}{"Key1": "Value1"}
// Perform a GET request
// URL Requested: http://your-api-url.com/api/users
api.Res("users", re).Get()
// Perform a GET request with Querystring
querystring := map[string]string{"page": "100"}
// URL Requested: http://your-api-url.com/api/users/123/items?page=100
api.Res("users").Id(123).Res("items", re).Get(querystring)
// Or perform a POST Request
// URL Requested: http://your-api-url.com/api/items/123 with payload as json Data
api.Res("items", re).Id(123).Post(payload)
}
func main() {
// Load defaultd if any
config, err := LoadConfig(TAG)
if err != nil {
log.Printf("Warning: missing or unreadable config.toml as %s\n", TAG)
}
flag.Parse()
// Check if we have anything on the command-line
if fId == "" {
if err != nil {
log.Fatalf("Error: no default Id and nothing on command-line")
}
fId = config.Id
}
api := gopencils.Api(API)
query := map[string]string{"id": fId}
answer := new(RestAnswer)
api.Res("probe/", answer).Get(query)
fmt.Printf("Result:\n %v", answer)
}
func getStashAPI(host, user, pass string) (*StashAPI, error) {
return &StashAPI{
gopencils.Api(
"http://"+host+"/rest/api/1.0",
&gopencils.BasicAuth{user, pass},
),
}, nil
}
func Start(url string) (*CveClient, error) {
if url != "" {
api := gopencils.Api(url)
cve_client := CveClient{api: api}
return &cve_client, nil
} else {
return nil, errors.New("You must specify an url")
}
}
// GetMeasurement gets info for a single one
func GetMeasurement(id int) (m *Measurement, err error) {
auth := WantAuth()
api := gopencils.Api(apiEndpoint, auth)
r, err := api.Res("measurements").Id(id, &m).Get()
if err != nil {
err = fmt.Errorf("%v - r:%#v\n", err, r.Raw)
return
}
return
}
// GetProbe returns data for a single probe
func GetProbe(id int) (p *Probe, err error) {
auth := WantAuth()
api := gopencils.Api(apiEndpoint, auth)
r, err := api.Res("probes").Id(id, &p).Get()
if err != nil {
err = fmt.Errorf("err: %v - r:%v\n", err, r)
return
}
return
}
func getHistory(auth *gopencils.BasicAuth, profid string) ([]*model.Change, error) {
api := gopencils.Api(apiurl, auth)
changes := []*model.Change{}
res, err := api.Res("profile").Id(profid).Res("history", &changes).Get()
if err = checkErr(res, err); err != nil {
return nil, err
}
return changes, nil
}
func updateProfile(auth *gopencils.BasicAuth, profile *model.Profile) (*model.Profile, error) {
api := gopencils.Api(apiurl, auth)
updatedProfile := &model.Profile{}
res, err := api.Res("profile", updatedProfile).Put(profile)
if err = checkErr(res, err); err != nil {
return nil, err
}
return profile, nil
}
func getProfile(profid string) (*model.Profile, error) {
api := gopencils.Api(apiurl)
profile := &model.Profile{}
res, err := api.Res("profile", profile).Id(profid).Get()
if err = checkErr(res, err); err != nil {
return nil, err
}
return profile, nil
}
func allProfiles() ([]*model.Profile, error) {
api := gopencils.Api(apiurl)
profiles := []*model.Profile{}
res, err := api.Res("profile").Res("all", &profiles).Get()
if err = checkErr(res, err); err != nil {
return nil, err
}
return profiles, nil
}
func main() {
api := gopencils.Api("http://localhost:5080/api")
resp := new(interface{})
// Perform a GET request
// URL Requested: /api/users
//Authorization: Bearer W8Ijq0B78JMzYIF26XdakL7od9a4OR
res := api.Res("todos", resp)
res.SetHeader("Authorization", "Bearer O6fm69RSKPICKQ2xcNXM7wTp5pMsHl")
res, _ = res.Get()
fmt.Println(*resp)
}
func (k *KibanaApi) Delete(item KibanaItem) (map[string]interface{}, error) {
api := gopencils.Api(k.Uri)
var response interface{}
var err error
if api, err = api.Res(item.Type, &response).Id(item.Id).Delete(); err != nil {
return nil, err
}
if response == nil {
return nil, nil
}
return response.(map[string]interface{}), nil
}
func (k *KibanaApi) Get(items []KibanaItem) (map[string]interface{}, error) {
api := gopencils.Api(k.Uri)
var response interface{}
postData := make(map[string]interface{})
postData["docs"] = items
//postJson, _ := json.Marshal(postData);
_, err := api.Res("/_mget", &response).Post(postData)
if err != nil {
return nil, err
}
return response.(map[string]interface{}), nil
}
func (k *KibanaApi) Create(item map[string]interface{}) (map[string]interface{}, error) {
querystring := map[string]string{"op_type": "create"}
api := gopencils.Api(k.Uri)
var response interface{}
var err error
if api, err = api.Res(item["_type"], &response).Id(item["_id"]).SetQuery(querystring).Post(item["_source"]); err != nil {
return nil, err
}
if response == nil {
return nil, nil
}
return response.(map[string]interface{}), nil
}
// returns profile id
func confirm(userid string) (string, error) {
api := gopencils.Api(apiurl)
outmap := map[string]interface{}{}
res, err := api.Res("user").Id(userid).Res("confirm", &outmap).Get()
if err = checkErr(res, err); err != nil {
return "", err
}
id, ok := outmap["id"].(string)
if !ok {
return "", fmt.Errorf("can't get user id, resp: %#v", outmap)
}
return id, nil
}
func (k *KibanaApi) Search(itemType string) ([]KibanaItem, error) {
var response interface{}
api := gopencils.Api(k.Uri)
querystring := map[string]string{"size": "100"}
if _, err := api.Res(itemType+"/_search", &response).SetQuery(querystring).Get(); err != nil {
return nil, err
}
hits := response.(map[string]interface{})["hits"].(map[string]interface{})["hits"]
items := hits.([]interface{})
result := make([]KibanaItem, 0)
for _, x := range items {
m := x.(map[string]interface{})
result = append(result, KibanaItem{m["_id"].(string), m["_type"].(string)})
}
return result, nil
}
// returns user id
func createUser(email, pass string) (string, error) {
api := gopencils.Api(apiurl)
user := &model.User{
Email: email,
Password: pass,
}
outmap := map[string]interface{}{}
res, err := api.Res("user", &outmap).Post(user)
if err = checkErr(res, err); err != nil {
return "", err
}
id, ok := outmap["id"].(string)
if !ok {
return "", fmt.Errorf("can't get user id, resp: %#v", outmap)
}
return id, nil
}
func main() {
// Create Basic Auth
auth := gopencils.BasicAuth{"username", "password"}
// Create New Api with our auth
api := gopencils.Api("http://your-api-url.com/api/", &auth)
// Maybe some payload to send along with the request?
payload := map[string]interface{}{"Key": "Value1"}
resp := new(respStruct)
// Perform a GET request
// URL Requested: http://your-api-url.com/api/users
api.Res("users", &respStruct{}).Get()
// Get Single Item
api.Res("users", &respStruct{}).Id(1).Get()
// Perform a GET request with Querystring
querystring := map[string]string{"page": "100", "per_page": "1000"}
// URL Requested: http://your-api-url.com/api/users/123/items?page=100&per_page=1000
api.Res("users").Id(123).Res("items", resp).Get(querystring)
// Or perform a POST Request
// URL Requested: http://your-api-url.com/api/items/123 with payload as json Data
api.Res("items", resp).Id(123).Post(payload)
// Users endpoint
users := api.Res("users")
for i := 0; i < 10; i++ {
// Create a new pointer to response Struct
user := new(respStruct)
// Get user with id i into the newly created response struct
users.Id(i, user).Get()
}
}
func fetchNewData(configuration Configuration) {
db, err := sql.Open("mysql", configuration.Datasource)
if err != nil {
panic(err.Error())
}
defer db.Close()
err = db.Ping()
if err != nil {
panic(err.Error())
}
logsOut, err := db.Prepare("SELECT `time`, `index` FROM `logs` ORDER BY `time` DESC, `index` DESC LIMIT 1")
if err != nil {
log.Println(err.Error())
}
defer logsOut.Close()
logsIns, err := db.Prepare("INSERT INTO logs VALUES( ?, ?, ?, ?, ?, ? )")
if err != nil {
log.Println(err.Error())
}
defer logsIns.Close()
logAttributesIns, err := db.Prepare("INSERT INTO log_attributes VALUES( ?, ?, ?, ? )")
if err != nil {
log.Println(err.Error())
}
defer logAttributesIns.Close()
var (
start time.Time
elapsed time.Duration
logEntryTime time.Time
index, preCount int64
extraLogs int64 = 10
serverLogLimit int64 = 10000
api = gopencils.Api(configuration.BaseUrl)
resp = new(RespStruct)
firstLog, logCount, refid, participantid float64
finalCount string
querystring map[string]string
entries int
)
for {
start = time.Now()
err = logsOut.QueryRow().Scan(&logEntryTime, &index)
if err != nil {
log.Println(err.Error())
}
log.Printf("Last log entry was from: %v", logEntryTime)
log.Printf("Last log entry index was: %d", index)
_, err = api.Res("log/overview", resp).Get()
firstLog = resp.Response["first"].(float64)
logCount = resp.Response["count"].(float64) + firstLog
if logCount == 0 {
log.Println("Logcount was 0")
// Fresh restarted server, not much to do here
elapsed := time.Since(start)
log.Println("Server just restarted, nothing to fetch")
log.Printf("Fetching data took %s", elapsed)
log.Println("Fetcher idling for 15 minutes")
time.Sleep(15 * time.Minute)
continue
}
preCount = (int64(logCount) - index) + extraLogs
if preCount < 0 {
log.Println("Precount was less than 0!")
preCount = 1
}
if preCount > serverLogLimit+extraLogs {
log.Println("precount was larger than server log limit + extra logs")
preCount = serverLogLimit + extraLogs
}
finalCount = strconv.Itoa(int(preCount))
log.Printf("Fetching %s log entries from server", finalCount)
querystring = map[string]string{"offset": "-" + finalCount, "count": finalCount}
_, err = api.Res("log/range", resp).Get(querystring)
entries = 0
if err != nil {
log.Println(err)
} else {
for _, event := range resp.Response["events"].([]interface{}) {
event, _ := event.(map[string]interface{})
if int64(event["index"].(float64)) <= index && preCount != 1 {
continue
}
entries++
switch event["refid"].(type) {
case float64:
refid = event["refid"].(float64)
default:
refid = 0
}
switch event["participantid"].(type) {
case float64:
participantid = event["participantid"].(float64)
default:
participantid = 0
}
result, err := logsIns.Exec(
nil,
event["index"].(float64),
time.Unix(int64(event["time"].(float64)), 0),
event["name"].(string),
refid,
participantid,
)
if err != nil {
log.Println(err.Error())
}
lastInsertedId, err := result.LastInsertId()
if err != nil {
log.Println(err.Error())
}
for key, value := range event["attributes"].(map[string]interface{}) {
_, err = logAttributesIns.Exec(nil, lastInsertedId, key, value)
if err != nil {
log.Println(err.Error())
}
}
}
}
log.Printf("Added %d new log entries out of %s fetched ones", entries, finalCount)
elapsed = time.Since(start)
log.Printf("Fetching data took %s", elapsed)
log.Println("Fetcher idling for 15 minutes")
time.Sleep(15 * time.Minute)
}
}
func (self *Momonga) SendPublishMessage(msg *codec.PublishMessage, client_id string, is_bridged bool) {
// Don't pass wrong message here. user should validate the message be ore using this API.
if len(msg.TopicName) < 1 {
return
}
rpc, jsonObj := stringutils.IsJsonRpc(string(msg.Payload))
if rpc {
fmt.Println("******************** message *****************", jsonObj)
obj := jsonObj.(map[string]interface{})
method := obj["method"]
paramsObj := obj["params"].(map[string]interface{})
// lets pick params and create a query string
params := url.Values{}
for k, v := range paramsObj {
if str, ok := v.(string); ok {
params.Add(k, str)
}
}
api := gopencils.Api("http://localhost:5080/api")
resp := new(interface{})
res := api.Res(strings.Replace(msg.TopicName, "/data/", "", 1), resp)
res.SetHeader("Authorization", "Bearer 2eEkMA6x7NmNQwzpE3qLvgvQgdILER")
switch method {
case "get":
res.Get()
fmt.Println(client_id, "GET ", msg.TopicName, params.Encode())
case "create":
res.Post(paramsObj)
fmt.Println(client_id, "POST ", msg.TopicName, params.Encode())
case "update":
fmt.Println(client_id, "PUT ", msg.TopicName, params.Encode())
case "delete":
fmt.Println(client_id, "DELETE ", msg.TopicName, params.Encode())
}
//fmt.Println(*resp)
result, _ := json.Marshal(*resp)
fmt.Println(string(result))
fmt.Println(msg)
msg.TopicName = "/client/" + client_id
msg.Payload = result
fmt.Println(msg)
}
// TODO: Have to persist retain message.
if msg.Retain > 0 {
if len(msg.Payload) == 0 {
log.Debug("[DELETE RETAIN: %s]\n%s", msg.TopicName, hex.Dump([]byte(msg.TopicName)))
err := self.DataStore.Del([]byte(msg.TopicName), []byte(msg.TopicName))
if err != nil {
log.Error("Error: %s\n", err)
}
// これ配送したらおかしいべ
log.Debug("Deleted retain: %s", msg.TopicName)
// あれ、ackとかかえすんだっけ？
Metrics.System.Broker.Messages.RetainedCount.Add(-1)
return
} else {
buffer := bytes.NewBuffer(nil)
codec.WriteMessageTo(msg, buffer)
self.DataStore.Put([]byte(msg.TopicName), buffer.Bytes())
Metrics.System.Broker.Messages.RetainedCount.Add(1)
}
}
if Mflags["experimental.qos1"] {
if msg.QosLevel == 1 {
targets := self.TopicMatcher.Match(msg.TopicName)
go func(msg *codec.PublishMessage, set []interface{}) {
p := make(chan string, 1000)
wg := sync.WaitGroup{}
wg.Add(3) // bulk sernder, retry kun, receive kun
mchan := make(chan string, 256)
term := make(chan bool, 1)
cnt := len(set)
mng := make(map[string]*codec.PublishMessage)
// retry kun。こういう実装だととても楽
go func(term chan bool, mchan chan string, mng map[string]*codec.PublishMessage) {
for {
select {
case m := <-mchan:
if msg, ok := mng[m]; ok {
conn, err := self.GetConnectionByClientId(m)
if err != nil {
fmt.Printf("something wrong: %s %s", m, err)
continue
}
if err == nil {
log.Debug("sending a retry msg: %s", msg)
conn.WriteMessageQueue(msg)
} else {
log.Debug("connection not exist. next retry")
}
}
case <-term:
log.Debug(" retry finished.")
wg.Done()
return
}
}
}(term, mchan, mng)
// reader
go func(p chan string, term chan bool, cnt int, mng map[string]*codec.PublishMessage, mchan chan string) {
limit := time.After(time.Second * 60)
for {
select {
case id := <-p:
cnt--
delete(mng, id)
// これはcallbackでやってもいいようなきもするけど、wait groupとかもろもろ渡すの面倒くさい
if cnt < 1 {
log.Debug(" all delivery finished.")
term <- true
wg.Done()
return
}
case <-time.After(time.Second * 20):
// 終わってないやつをなめてリトライさせる
for cid, m := range mng {
m.Dupe = true
mchan <- cid
}
case <-limit:
log.Debug(" gave up retry.")
term <- true
wg.Done()
return
}
}
}(p, term, cnt, mng, mchan)
// sender. これは勝手に終わる
go func(msg *codec.PublishMessage, set []interface{}, p chan string, mng map[string]*codec.PublishMessage) {
dp := make(map[string]bool)
for i := range targets {
var tbl *util.MessageTable
var ok bool
myset := targets[i].(*SubscribeSet)
fmt.Printf("myset: %s", myset)
// NOTE (from interoperability/client_test.py):
//
// overlapping subscriptions. When there is more than one matching subscription for the same client for a topic,
// the server may send back one message with the highest QoS of any matching subscription, or one message for
// each subscription with a matching QoS.
//
// Currently, We choose one message for each subscription with a matching QoS.
//
if _, ok := dp[myset.ClientId]; ok {
continue
}
dp[myset.ClientId] = true
x, _ := codec.CopyPublishMessage(msg)
x.QosLevel = myset.QoS
conn, err := self.GetConnectionByClientId(myset.ClientId)
// これは面倒臭い。clean sessionがtrueで再接続した時はもはや別人として扱わなければならない
if x.QosLevel == 0 {
// QoS 0にダウングレードしたらそのまま終わらせる
conn.WriteMessageQueue(x)
p <- myset.ClientId
continue
}
if tbl, ok = self.InflightTable[myset.ClientId]; !ok {
self.InflightTable[myset.ClientId] = util.NewMessageTable()
// callback仕込めるんだよなー。QoS1なら使わなくてもいいかなー。とかおもったり
tbl = self.InflightTable[myset.ClientId]
}
id := tbl.NewId()
x.PacketIdentifier = id
x.Opaque = p
tbl.Register2(id, x, 1, x)
if err != nil {
continue
}
mng[myset.ClientId] = x
conn.WriteMessageQueue(x)
}
log.Debug(" all fisrt delivery finished.")
wg.Done()
}(msg, targets, p, mng)
wg.Wait()
close(p)
close(mchan)
close(term)
mng = nil
log.Debug(" okay, cleanup qos1 sending thread.")
}(msg, targets)
return
}
}
// Publishで受け取ったMessageIdのやつのCountをとっておく
// で、Pubackが帰ってきたらrefcountを下げて0になったらMessageを消す
//log.Debug("TopicName: %s %s", m.TopicName, m.Payload)
targets := self.TopicMatcher.Match(msg.TopicName)
if msg.TopicName[0:1] == "#" {
// TODO: [MQTT-4.7.2-1] The Server MUST NOT match Topic Filters starting with a wildcard character
// (# or +) with Topic Names beginning with a $ character
}
// list つくってからとって、だとタイミング的に居ない奴も出てくるんだよな。マジカオス
// ここで必要なのは, connection(clientId), subscribed qosがわかればあとは投げるだけ
// なんで、Qlobberがかえすのはであるべきなんだけど。これすっげー消しづらいのよね・・・
// {
// Connection: Connection or client id
// QoS:
// }
// いやまぁエラーハンドリングちゃんとやってれば問題ない。
// client idのほうがベターだな。Connectionを無駄に参照つけると後が辛い
dp := make(map[string]bool)
count := 0
for i := range targets {
var cn Connection
var ok error
myset := targets[i].(*SubscribeSet)
clientId := myset.ClientId
//clientId := targets[i].(string)
// NOTE (from interoperability/client_test.py):
//
// overlapping subscriptions. When there is more than one matching subscription for the same client for a topic,
// the server may send back one message with the highest QoS of any matching subscription, or one message for
// each subscription with a matching QoS.
//
// Currently, We choose one message for each subscription with a matching QoS.
//
if _, ok := dp[clientId]; ok {
continue
}
dp[clientId] = true
cn, ok = self.GetConnectionByClientId(clientId)
if ok != nil {
// どちらかというとClientが悪いと思うよ！
// リスト拾った瞬間にはいたけど、その後いなくなったんだから配送リストからこれらは無視すべき
log.Info("(can't fetch %s. already disconnected, or unsubscribed?)", clientId)
continue
}
if cn.IsBridge() && clientId == client_id {
// Don't send message to same bridge
continue
}
var x *codec.PublishMessage
if msg.QosLevel == 0 {
// we don't need to copy message on QoS 0
x = msg
} else {
x = codec.MustCopyPublishMessage(msg)
}
subscriberQos := myset.QoS
// Downgrade QoS
if subscriberQos < x.QosLevel {
x.QosLevel = subscriberQos
}
if x.QosLevel > 0 {
// TODO: ClientごとにInflightTableを持つ
// engineのOutGoingTableなのはとりあえず、ということ
id := self.OutGoingTable.NewId()
x.PacketIdentifier = id
if sender, ok := x.Opaque.(Connection); ok {
// TODO: ここ(でなにをするつもりだったのか・・ｗ)
self.OutGoingTable.Register2(x.PacketIdentifier, x, len(targets), sender)
}
}
cn.WriteMessageQueue(x)
count++
}
Metrics.System.Broker.Messages.Sent.Add(int64(count))
}
