Log & resolve incidents, add readme

This commit is contained in:
Matej Kramny
2015-03-16 21:02:29 +01:00
parent f478ad9895
commit 92d8791a4f
4 changed files with 133 additions and 14 deletions

View File

@@ -9,6 +9,7 @@ import (
const timeout = time.Duration(time.Second)
type Monitor struct {
Name string `json:"name"`
Url string `json:"url"`
MetricId int `json:"metric_id"`
Threshold float32 `json:"threshold"`
@@ -30,9 +31,8 @@ func (monitor *Monitor) Run() {
monitor.History = append(monitor.History, isUp)
monitor.AnalyseData()
if isUp == true {
if isUp == true && monitor.MetricId > 0 {
SendMetric(monitor.MetricId, lag)
return
}
}
@@ -52,11 +52,6 @@ func (monitor *Monitor) doRequest() bool {
func (monitor *Monitor) AnalyseData() {
// look at the past few incidents
if len(monitor.History) != 10 {
// not enough data
return
}
numDown := 0
for _, wasUp := range monitor.History {
if wasUp == false {
@@ -65,14 +60,31 @@ func (monitor *Monitor) AnalyseData() {
}
t := (float32(numDown) / float32(len(monitor.History))) * 100
fmt.Printf("%s %.2f%% Down. Threshold: %.2f%%\n", monitor.Url, t, monitor.Threshold)
fmt.Printf("%s %.2f%% Down at %v. Threshold: %.2f%%\n", monitor.Url, t, time.Now().UnixNano() / int64(time.Second), monitor.Threshold)
if len(monitor.History) != 10 {
// not enough data
return
}
if t > monitor.Threshold && monitor.Incident == nil {
// is down, create an incident
fmt.Println("Creating incident...")
monitor.Incident = &Incident{}
monitor.Incident = &Incident{
Name: monitor.Name,
Message: monitor.Name + " is unreachable.",
}
monitor.Incident.SetInvestigating()
monitor.Incident.Send()
} else if t < monitor.Threshold && monitor.Incident != nil {
// was down, created an incident, its now ok, make it resolved.
fmt.Println("Updating incident to resolved...")
monitor.Incident.SetFixed()
monitor.Incident.Send()
monitor.Incident = nil
}
}