// HTTP Dynamic Streaming prober.
// Parse and probe F4M playlists and report time statistics and errors.
func SanjoseProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) {
for {
task := <-tasks
result := ExecHTTP(task)
task.ReplyTo <- result
debugvars.Add("hds-tasks-done", 1)
}
}
func writeHistogramForDuration(expvarMap *expvar.Map, duration time.Duration, prefix string) {
if base.LogEnabled("PerfStats") {
var durationMs int
if duration < 1*time.Second {
durationMs = int(duration/(100*time.Millisecond)) * 100
} else {
durationMs = int(duration/(1000*time.Millisecond)) * 1000
}
expvarMap.Add(fmt.Sprintf("%s-%06dms", prefix, durationMs), 1)
}
}
func TestSnapshotExpvarsMap(t *testing.T) {
test := expvar.NewMap("testMap")
test.Add("hello", 42)
map2 := new(expvar.Map).Init()
map2.Add("test", 5)
test.Set("map2", map2)
vals := map[string]int64{}
snapshotExpvars(vals)
assert.Equal(t, vals["testMap.hello"], int64(42))
assert.Equal(t, vals["testMap.map2.test"], int64(5))
}
// TODO к реализации
// Probe HTTP with additional checks for Widevine.
// Really now only http-range check supported.
func WidevineProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) {
var result *Result
defer func() {
if r := recover(); r != nil {
fmt.Println("trace dumped in Widevine prober:", r)
}
}()
for {
queueCount := debugvars.Get("wv-tasks-queue")
queueCount.(*expvar.Int).Set(int64(len(tasks)))
task := <-tasks
if time.Now().Before(task.TTL) {
result = ExecHTTP(task)
debugvars.Add("wv-tasks-done", 1)
} else {
result = TaskExpired(task)
debugvars.Add("wv-tasks-expired", 1)
}
task.ReplyTo <- result
}
}
// incrementMetric increments a value in the specified expvar.Map. The key
// should be a windows syscall.Errno or a string. Any other types will be
// reported under the "other" key.
func incrementMetric(v *expvar.Map, key interface{}) {
switch t := key.(type) {
default:
v.Add("other", 1)
case string:
v.Add(t, 1)
case syscall.Errno:
v.Add(strconv.Itoa(int(t)), 1)
}
}
func getMetricSetExpvarMap(module, name string) (*expvar.Map, error) {
key := fmt.Sprintf("%s-%s", module, name)
fetchesLock.Lock()
defer fetchesLock.Unlock()
expVar := fetches.Get(key)
switch m := expVar.(type) {
case nil:
expMap := new(expvar.Map).Init()
fetches.Set(key, expMap)
expMap.Add(successesKey, 0)
expMap.Add(failuresKey, 0)
expMap.Add(eventsKey, 0)
return expMap, nil
case *expvar.Map:
return m, nil
default:
return nil, fmt.Errorf("unexpected expvar.Var type (%T) found for key '%s'", m, key)
}
}
// HTTP Live Streaming support.
// Parse and probe M3U8 playlists (multi- and single bitrate)
// and report time statistics and errors
func CupertinoProber(ctl *bcast.Group, tasks chan *Task, debugvars *expvar.Map) {
var result *Result
defer func() {
if r := recover(); r != nil {
fmt.Println("trace dumped in HLS prober:", r)
}
}()
queueCount := debugvars.Get("hls-tasks-queue")
queueCount.(*expvar.Int).Set(int64(len(tasks)))
for {
task := <-tasks
if time.Now().Before(task.TTL) {
result = ExecHTTP(task)
if result.ErrType < ERROR_LEVEL && result.HTTPCode < 400 && result.ContentLength > 0 {
playlist, listType, err := m3u8.Decode(result.Body, true)
if err != nil {
result.ErrType = BADFORMAT
} else {
switch listType {
case m3u8.MASTER:
m := playlist.(*m3u8.MasterPlaylist)
subresult := make(chan *Result, 24)
mainuri, err := url.Parse(task.URI)
if err != nil {
result.ErrType = UNKERR
goto End
}
for _, variant := range m.Variants {
uri, err := url.Parse(variant.URI)
if err != nil {
subresult <- &Result{Task: &Task{Tid: task.Tid, Stream: Stream{variant.URI, HLS, task.Name, task.Title, task.Group}}, ErrType: BADURI, Started: time.Now()}
continue
}
var suburi string
if uri.IsAbs() { // absolute URI
suburi = variant.URI
} else { // relative URI
if variant.URI[0] == '/' { // from the root
suburi = fmt.Sprintf("%s://%s%s", mainuri.Scheme, mainuri.Host, variant.URI)
} else { // last element
splitted := strings.Split(task.URI, "/")
splitted[len(splitted)-1] = variant.URI
suburi = strings.Join(splitted, "/")
}
}
subtask := &Task{Tid: task.Tid, Stream: Stream{suburi, HLS, task.Name, task.Title, task.Group}, ReadBody: task.ReadBody, TTL: task.TTL}
go func(subtask *Task) {
subresult <- ExecHTTP(subtask)
}(subtask)
}
taskCount := len(m.Variants)
for taskCount > 0 {
select {
case data := <-subresult:
result.SubResults = append(result.SubResults, data)
case <-time.After(60 * time.Second):
}
taskCount--
}
case m3u8.MEDIA:
p := playlist.(*m3u8.MediaPlaylist)
p.Encode().String()
default:
result.ErrType = BADFORMAT
}
}
}
debugvars.Add("hls-tasks-done", 1)
} else {
result = TaskExpired(task)
debugvars.Add("hls-tasks-expired", 1)
}
End:
task.ReplyTo <- result
debugvars.Add("hls-tasks-done", 1)
}
}
func Listen(host string, port int, counts *expvar.Map, MessageRead func([]byte)) {
conn, err := amqp.Dial(fmt.Sprintf("amqp://*****:*****@%s:%v/", host, port))
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
err = ch.ExchangeDeclare(
"logs", // name
"fanout", // type
true, // durable
false, // auto-deleted
false, // internal
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare an exchange")
q, err := ch.QueueDeclare(
"", // name
false, // durable
false, // delete when usused
true, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.QueueBind(
q.Name, // queue name
"", // routing key
"logs", // exchange
false,
nil)
failOnError(err, "Failed to bind a queue")
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
forever := make(chan bool)
go func() {
for d := range msgs {
counts.Add("Rabbitmq get", 1)
MessageRead(d.Body)
}
}()
log.Printf(" [*] Waiting for logs. To exit press CTRL+C")
<-forever
}
// Container keep single stream properties and regulary make tasks for appropriate probers.
func StreamBox(ctl *bcast.Group, stream Stream, streamType StreamType, taskq chan *Task, debugvars *expvar.Map) {
var checkCount uint64 // число прошедших проверок
var addSleepToBrokenStream time.Duration
var tid int64 = time.Now().Unix() // got increasing offset on each program start
var min, max int
var command Command
var online bool = false
var stats Stats
defer func() {
if r := recover(); r != nil {
fmt.Printf("Stream %s trace: %s\n", stream.Name, r)
}
}()
task := &Task{Stream: stream, ReplyTo: make(chan *Result)}
switch streamType {
case HTTP:
task.ReadBody = false
case HLS:
task.ReadBody = true
case HDS:
task.ReadBody = true
case WV:
task.ReadBody = false
default:
task.ReadBody = false
}
ctlrcv := ctl.Join() // управление мониторингом
timer := time.Tick(3 * time.Second)
for {
select {
case recv := <-ctlrcv.In:
command = recv.(Command)
switch command {
case START_MON:
online = true
case STOP_MON:
online = false
}
case <-timer:
SaveStats(stream, stats)
default:
if !online {
time.Sleep(1 * time.Second)
continue
}
max = int(cfg.Params(stream.Group).TimeBetweenTasks)
min = int(cfg.Params(stream.Group).TimeBetweenTasks / 4. * 3.)
time.Sleep(time.Duration(rand.Intn(max-min)+min)*time.Second + addSleepToBrokenStream) // randomize streams order
tid++
task.Tid = tid
task.TTL = time.Now().Add(time.Duration(cfg.Params(stream.Group).TaskTTL * time.Second))
stats.Checks++ // TODO potentially overflow
taskq <- task
debugvars.Add("requested-tasks", 1)
result := <-task.ReplyTo
if result.ErrType == TTLEXPIRED {
continue
} else {
checkCount++
if checkCount > 144 {
fmt.Printf("Repeated %d times %s\n", checkCount, task.Name)
}
}
for _, subres := range result.SubResults {
subres.Pid = result
go SaveResult(stream, *subres)
}
go SaveResult(stream, *result)
max = int(cfg.Params(stream.Group).CheckBrokenTime)
min = int(cfg.Params(stream.Group).CheckBrokenTime / 4. * 3.)
switch {
// permanent error, not a timeout:
case result.ErrType > CRITICAL_LEVEL, result.ErrType == TTLEXPIRED:
addSleepToBrokenStream = time.Duration(rand.Intn(max-min)+min) * time.Second
// works ok:
case result.ErrType == SUCCESS:
addSleepToBrokenStream = 0
default:
addSleepToBrokenStream = 0
}
if result.ErrType != TTLEXPIRED {
if result.ErrType >= WARNING_LEVEL {
go Log(ERROR, stream, *result)
} else {
if result.Elapsed >= cfg.Params(stream.Group).VerySlowWarningTimeout*time.Second {
result.ErrType = VERYSLOW
go Log(WARNING, stream, *result)
} else if result.Elapsed >= cfg.Params(stream.Group).SlowWarningTimeout*time.Second {
result.ErrType = SLOW
go Log(WARNING, stream, *result)
}
}
}
}
}
}