- compile message templates

- send metrics to cachet
- fix http default configuration
This commit is contained in:
Matej Kramny
2017-02-05 19:27:01 -08:00
parent b4fa33b8ad
commit b3bc1d4405
9 changed files with 135 additions and 87 deletions

49
api.go
View File

@@ -3,9 +3,13 @@ package cachet
import ( import (
"bytes" "bytes"
"crypto/tls" "crypto/tls"
"encoding/json"
"errors" "errors"
"io/ioutil"
"net/http" "net/http"
"strconv"
"time"
"github.com/Sirupsen/logrus"
) )
type CachetAPI struct { type CachetAPI struct {
@@ -14,6 +18,10 @@ type CachetAPI struct {
Insecure bool `json:"insecure"` Insecure bool `json:"insecure"`
} }
type CachetResponse struct {
Data json.RawMessage `json:"data"`
}
func (api CachetAPI) Ping() error { func (api CachetAPI) Ping() error {
resp, _, err := api.NewRequest("GET", "/ping", nil) resp, _, err := api.NewRequest("GET", "/ping", nil)
if err != nil { if err != nil {
@@ -27,30 +35,43 @@ func (api CachetAPI) Ping() error {
return nil return nil
} }
func (api CachetAPI) NewRequest(requestType, url string, reqBody []byte) (*http.Response, []byte, error) { // SendMetric adds a data point to a cachet monitor
func (api CachetAPI) SendMetric(id int, lag int64) {
logrus.Debugf("Sending lag metric ID:%d %vms", id, lag)
jsonBytes, _ := json.Marshal(map[string]interface{}{
"value": lag,
"timestamp": time.Now().Unix(),
})
resp, _, err := api.NewRequest("POST", "/metrics/"+strconv.Itoa(id)+"/points", jsonBytes)
if err != nil || resp.StatusCode != 200 {
logrus.Warnf("Could not log metric! ID: %d, err: %v", id, err)
}
}
// NewRequest wraps http.NewRequest
func (api CachetAPI) NewRequest(requestType, url string, reqBody []byte) (*http.Response, CachetResponse, error) {
req, err := http.NewRequest(requestType, api.URL+url, bytes.NewBuffer(reqBody)) req, err := http.NewRequest(requestType, api.URL+url, bytes.NewBuffer(reqBody))
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Cachet-Token", api.Token) req.Header.Set("X-Cachet-Token", api.Token)
transport := &http.Transport{ transport := http.DefaultTransport.(*http.Transport)
Proxy: http.ProxyFromEnvironment, transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: api.Insecure}
}
if api.Insecure {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
client := &http.Client{ client := &http.Client{
Transport: transport, Transport: transport,
} }
res, err := client.Do(req) res, err := client.Do(req)
if err != nil { if err != nil {
return nil, []byte{}, err return nil, CachetResponse{}, err
} }
defer res.Body.Close() var body struct {
body, _ := ioutil.ReadAll(res.Body) Data json.RawMessage `json:"data"`
}
return res, body, nil err = json.NewDecoder(res.Body).Decode(&body)
return res, body, err
} }

View File

@@ -95,7 +95,7 @@ func main() {
logrus.Infof("Starting Monitor #%d: ", index) logrus.Infof("Starting Monitor #%d: ", index)
logrus.Infof("Features: \n - %v", strings.Join(monitor.Describe(), "\n - ")) logrus.Infof("Features: \n - %v", strings.Join(monitor.Describe(), "\n - "))
go monitor.ClockStart(cfg, wg) go monitor.ClockStart(cfg, monitor, wg)
} }
signals := make(chan os.Signal, 1) signals := make(chan os.Signal, 1)
@@ -164,6 +164,7 @@ func getConfiguration(path string) (*cachet.CachetMonitor, error) {
var t cachet.MonitorInterface var t cachet.MonitorInterface
var err error var err error
// get default type
monType := cachet.GetMonitorType("") monType := cachet.GetMonitorType("")
if t, ok := rawMonitor["type"].(string); ok { if t, ok := rawMonitor["type"].(string); ok {
monType = cachet.GetMonitorType(t) monType = cachet.GetMonitorType(t)
@@ -175,11 +176,17 @@ func getConfiguration(path string) (*cachet.CachetMonitor, error) {
err = mapstructure.Decode(rawMonitor, &s) err = mapstructure.Decode(rawMonitor, &s)
t = &s t = &s
case "dns": case "dns":
// t = cachet.DNSMonitor var s cachet.DNSMonitor
err = mapstructure.Decode(rawMonitor, &s)
t = &s
case "icmp": case "icmp":
// t = cachet.ICMPMonitor var s cachet.ICMPMonitor
err = mapstructure.Decode(rawMonitor, &s)
t = &s
case "tcp": case "tcp":
// t = cachet.TCPMonitor var s cachet.TCPMonitor
err = mapstructure.Decode(rawMonitor, &s)
t = &s
default: default:
logrus.Errorf("Invalid monitor type (index: %d) %v", index, monType) logrus.Errorf("Invalid monitor type (index: %d) %v", index, monType)
continue continue

4
dns.go
View File

@@ -1,3 +1,5 @@
package cachet package cachet
type DNSMonitor struct{} type DNSMonitor struct {
AbstractMonitor `mapstructure:",squash"`
}

67
http.go
View File

@@ -10,23 +10,23 @@ import (
"time" "time"
) )
// // Investigating template // Investigating template
// var HTTPTemplate = MessageTemplate{ var defaultHTTPInvestigatingTpl = MessageTemplate{
// Subject: `{{ .Name }} - {{ .config.SystemName }}`, Subject: `{{ .Name }} - {{ .config.SystemName }}`,
// Message: `{{ .Name }} check **failed** - {{ .now }} Message: `{{ .Name }} check **failed** - {{ .now }}
// {{ .lastFailReason }}`, {{ .lastFailReason }}`,
// } }
// // Fixed template // Fixed template
// var HTTPTemplate = MessageTemplate{ var defaultHTTPFixedTpl = MessageTemplate{
// Subject: `{{ .Name }} - {{ .config.SystemName }}`, Subject: `{{ .Name }} - {{ .config.SystemName }}`,
// Message: `**Resolved** - {{ .now }} Message: `**Resolved** - {{ .now }}
// - - - - - -
// {{ .incident.Message }}`, {{ .incident.Message }}`,
// } }
type HTTPMonitor struct { type HTTPMonitor struct {
AbstractMonitor `mapstructure:",squash"` AbstractMonitor `mapstructure:",squash"`
@@ -41,24 +41,21 @@ type HTTPMonitor struct {
} }
func (monitor *HTTPMonitor) test() bool { func (monitor *HTTPMonitor) test() bool {
client := &http.Client{
Timeout: time.Duration(monitor.Timeout * time.Second),
}
if monitor.Strict == false {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
req, err := http.NewRequest(monitor.Method, monitor.Target, nil) req, err := http.NewRequest(monitor.Method, monitor.Target, nil)
for k, v := range monitor.Headers { for k, v := range monitor.Headers {
req.Header.Add(k, v) req.Header.Add(k, v)
} }
transport := http.DefaultTransport.(*http.Transport)
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: monitor.Strict == false}
client := &http.Client{
Timeout: time.Duration(monitor.Timeout * time.Second),
Transport: transport,
}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
monitor.lastFailReason = err.Error() monitor.lastFailReason = err.Error()
return false return false
} }
@@ -66,7 +63,6 @@ func (monitor *HTTPMonitor) test() bool {
if monitor.ExpectedStatusCode > 0 && resp.StatusCode != monitor.ExpectedStatusCode { if monitor.ExpectedStatusCode > 0 && resp.StatusCode != monitor.ExpectedStatusCode {
monitor.lastFailReason = "Unexpected response code: " + strconv.Itoa(resp.StatusCode) + ". Expected " + strconv.Itoa(monitor.ExpectedStatusCode) monitor.lastFailReason = "Unexpected response code: " + strconv.Itoa(resp.StatusCode) + ". Expected " + strconv.Itoa(monitor.ExpectedStatusCode)
return false return false
} }
@@ -75,7 +71,6 @@ func (monitor *HTTPMonitor) test() bool {
responseBody, err := ioutil.ReadAll(resp.Body) responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
monitor.lastFailReason = err.Error() monitor.lastFailReason = err.Error()
return false return false
} }
@@ -91,6 +86,9 @@ func (monitor *HTTPMonitor) test() bool {
} }
func (mon *HTTPMonitor) Validate() []string { func (mon *HTTPMonitor) Validate() []string {
mon.Template.Investigating.SetDefault(defaultHTTPInvestigatingTpl)
mon.Template.Fixed.SetDefault(defaultHTTPFixedTpl)
errs := mon.AbstractMonitor.Validate() errs := mon.AbstractMonitor.Validate()
if len(mon.ExpectedBody) > 0 { if len(mon.ExpectedBody) > 0 {
@@ -125,22 +123,3 @@ func (mon *HTTPMonitor) Describe() []string {
return features return features
} }
// SendMetric sends lag metric point
/*func (monitor *Monitor) SendMetric(delay int64) error {
if monitor.MetricID == 0 {
return nil
}
jsonBytes, _ := json.Marshal(&map[string]interface{}{
"value": delay,
})
resp, _, err := monitor.config.makeRequest("POST", "/metrics/"+strconv.Itoa(monitor.MetricID)+"/points", jsonBytes)
if err != nil || resp.StatusCode != 200 {
return fmt.Errorf("Could not log data point!\n%v\n", err)
}
return nil
}
*/

View File

@@ -1,3 +1,5 @@
package cachet package cachet
type ICMPMonitor struct{} type ICMPMonitor struct {
AbstractMonitor `mapstructure:",squash"`
}

View File

@@ -57,15 +57,13 @@ func (incident *Incident) Send(cfg *CachetMonitor) error {
} }
var data struct { var data struct {
Incident struct {
ID int `json:"id"` ID int `json:"id"`
} `json:"data"`
} }
if err := json.Unmarshal(body, &data); err != nil { if err := json.Unmarshal(body.Data, &data); err != nil {
return fmt.Errorf("Cannot parse incident body: %v, %v", err, string(body)) return fmt.Errorf("Cannot parse incident body: %v, %v", err, string(body.Data))
} }
incident.ID = data.Incident.ID incident.ID = data.ID
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return fmt.Errorf("Could not create/update incident!") return fmt.Errorf("Could not create/update incident!")
} }
@@ -84,15 +82,13 @@ func (incident *Incident) GetComponentStatus(cfg *CachetMonitor) (int, error) {
} }
var data struct { var data struct {
Component struct {
Status int `json:"status"` Status int `json:"status"`
} `json:"data"`
} }
if err := json.Unmarshal(body, &data); err != nil { if err := json.Unmarshal(body.Data, &data); err != nil {
return 0, fmt.Errorf("Cannot parse component body: %v. Err = %v", string(body), err) return 0, fmt.Errorf("Cannot parse component body: %v. Err = %v", string(body.Data), err)
} }
return data.Component.Status, nil return data.Status, nil
} }
// SetInvestigating sets status to Investigating // SetInvestigating sets status to Investigating

View File

@@ -13,9 +13,9 @@ const DefaultTimeFormat = "15:04:05 Jan 2 MST"
const HistorySize = 10 const HistorySize = 10
type MonitorInterface interface { type MonitorInterface interface {
ClockStart(*CachetMonitor, *sync.WaitGroup) ClockStart(*CachetMonitor, MonitorInterface, *sync.WaitGroup)
ClockStop() ClockStop()
tick() tick(MonitorInterface)
test() bool test() bool
Validate() []string Validate() []string
@@ -70,6 +70,10 @@ func (mon *AbstractMonitor) Validate() []string {
mon.Timeout = DefaultTimeout mon.Timeout = DefaultTimeout
} }
if mon.Timeout > mon.Interval {
errs = append(errs, "Timeout greater than interval")
}
if mon.ComponentID == 0 && mon.MetricID == 0 { if mon.ComponentID == 0 && mon.MetricID == 0 {
errs = append(errs, "component_id & metric_id are unset") errs = append(errs, "component_id & metric_id are unset")
} }
@@ -78,6 +82,10 @@ func (mon *AbstractMonitor) Validate() []string {
mon.Threshold = 100 mon.Threshold = 100
} }
if err := mon.Template.Fixed.Compile(); err != nil {
errs = append(errs, "Could not compile template: "+err.Error())
}
return errs return errs
} }
func (mon *AbstractMonitor) GetMonitor() *AbstractMonitor { func (mon *AbstractMonitor) GetMonitor() *AbstractMonitor {
@@ -93,19 +101,19 @@ func (mon *AbstractMonitor) Describe() []string {
return features return features
} }
func (mon *AbstractMonitor) ClockStart(cfg *CachetMonitor, wg *sync.WaitGroup) { func (mon *AbstractMonitor) ClockStart(cfg *CachetMonitor, iface MonitorInterface, wg *sync.WaitGroup) {
wg.Add(1) wg.Add(1)
mon.config = cfg mon.config = cfg
mon.stopC = make(chan bool) mon.stopC = make(chan bool)
if cfg.Immediate { if cfg.Immediate {
mon.tick() mon.tick(iface)
} }
ticker := time.NewTicker(mon.Interval * time.Second) ticker := time.NewTicker(mon.Interval * time.Second)
for { for {
select { select {
case <-ticker.C: case <-ticker.C:
mon.tick() mon.tick(iface)
case <-mon.stopC: case <-mon.stopC:
wg.Done() wg.Done()
return return
@@ -124,9 +132,9 @@ func (mon *AbstractMonitor) ClockStop() {
func (mon *AbstractMonitor) test() bool { return false } func (mon *AbstractMonitor) test() bool { return false }
func (mon *AbstractMonitor) tick() { func (mon *AbstractMonitor) tick(iface MonitorInterface) {
reqStart := getMs() reqStart := getMs()
up := mon.test() up := iface.test()
lag := getMs() - reqStart lag := getMs() - reqStart
if len(mon.history) == HistorySize-1 { if len(mon.history) == HistorySize-1 {
@@ -139,9 +147,8 @@ func (mon *AbstractMonitor) tick() {
mon.AnalyseData() mon.AnalyseData()
// report lag // report lag
if up && mon.MetricID > 0 { if mon.MetricID > 0 {
logrus.Infof("%v", lag) go mon.config.API.SendMetric(mon.MetricID, lag)
// mon.SendMetric(lag)
} }
} }
@@ -158,7 +165,7 @@ func (monitor *AbstractMonitor) AnalyseData() {
t := (float32(numDown) / float32(len(monitor.history))) * 100 t := (float32(numDown) / float32(len(monitor.history))) * 100
logrus.Printf("%s %.2f%%/%.2f%% down at %v\n", monitor.Name, t, monitor.Threshold, time.Now().UnixNano()/int64(time.Second)) logrus.Printf("%s %.2f%%/%.2f%% down at %v\n", monitor.Name, t, monitor.Threshold, time.Now().UnixNano()/int64(time.Second))
if len(monitor.history) != 10 { if len(monitor.history) != HistorySize {
// not saturated // not saturated
return return
} }

2
tcp.go
View File

@@ -1,6 +1,8 @@
package cachet package cachet
type TCPMonitor struct { type TCPMonitor struct {
AbstractMonitor `mapstructure:",squash"`
// same as output from net.JoinHostPort // same as output from net.JoinHostPort
// defaults to parsed config from /etc/resolv.conf when empty // defaults to parsed config from /etc/resolv.conf when empty
DNSServer string DNSServer string

View File

@@ -1,6 +1,38 @@
package cachet package cachet
import "text/template"
type MessageTemplate struct { type MessageTemplate struct {
Subject string `json:"subject"` Subject string `json:"subject"`
Message string `json:"message"` Message string `json:"message"`
subjectTpl *template.Template
messageTpl *template.Template
}
func (t *MessageTemplate) SetDefault(d MessageTemplate) {
if len(t.Subject) == 0 {
t.Subject = d.Subject
}
if len(t.Message) == 0 {
t.Message = d.Message
}
}
func (t *MessageTemplate) Compile() error {
var err error
if len(t.Subject) > 0 {
t.subjectTpl, err = compileTemplate(t.Subject)
}
if err != nil && len(t.Message) > 0 {
t.messageTpl, err = compileTemplate(t.Message)
}
return err
}
func compileTemplate(text string) (*template.Template, error) {
return template.New("").Parse(text)
} }