a compiling proof of concept
- abstract type - http, tcp, icmp & dns monitor types - unmarshal from json into any monitor type
This commit is contained in:
53
api.go
53
api.go
@@ -1,7 +1,56 @@
|
|||||||
package cachet
|
package cachet
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
type CachetAPI struct {
|
type CachetAPI struct {
|
||||||
Url string `json:"api_url"`
|
URL string `json:"url"`
|
||||||
Token string `json:"api_token"`
|
Token string `json:"token"`
|
||||||
Insecure bool `json:"insecure"`
|
Insecure bool `json:"insecure"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (api CachetAPI) Ping() error {
|
||||||
|
resp, _, err := api.NewRequest("GET", "/ping", nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
return errors.New("API Responded with non-200 status code")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (api CachetAPI) NewRequest(requestType, url string, reqBody []byte) (*http.Response, []byte, error) {
|
||||||
|
req, err := http.NewRequest(requestType, api.URL+url, bytes.NewBuffer(reqBody))
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("X-Cachet-Token", api.Token)
|
||||||
|
|
||||||
|
transport := &http.Transport{
|
||||||
|
Proxy: http.ProxyFromEnvironment,
|
||||||
|
}
|
||||||
|
if api.Insecure {
|
||||||
|
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, []byte{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer res.Body.Close()
|
||||||
|
body, _ := ioutil.ReadAll(res.Body)
|
||||||
|
|
||||||
|
return res, body, nil
|
||||||
|
}
|
||||||
|
|||||||
105
cli/main.go
105
cli/main.go
@@ -10,12 +10,9 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/Sirupsen/logrus"
|
"github.com/Sirupsen/logrus"
|
||||||
cachet "github.com/castawaylabs/cachet-monitor"
|
cachet "github.com/castawaylabs/cachet-monitor"
|
||||||
docopt "github.com/docopt/docopt-go"
|
docopt "github.com/docopt/docopt-go"
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const usage = `cachet-monitor
|
const usage = `cachet-monitor
|
||||||
@@ -26,16 +23,16 @@ Usage:
|
|||||||
cachet-monitor print-config
|
cachet-monitor print-config
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
PATH path to config.yml
|
PATH path to config.json
|
||||||
LOGPATH path to log output (defaults to STDOUT)
|
LOGPATH path to log output (defaults to STDOUT)
|
||||||
NAME name of this logger
|
NAME name of this logger
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
cachet-monitor -c /root/cachet-monitor.yml
|
cachet-monitor -c /root/cachet-monitor.json
|
||||||
cachet-monitor -c /root/cachet-monitor.yml --log=/var/log/cachet-monitor.log --name="development machine"
|
cachet-monitor -c /root/cachet-monitor.json --log=/var/log/cachet-monitor.log --name="development machine"
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-c PATH.yml --config PATH Path to configuration file
|
-c PATH.json --config PATH Path to configuration file
|
||||||
-h --help Show this screen.
|
-h --help Show this screen.
|
||||||
--version Show version
|
--version Show version
|
||||||
print-config Print example configuration
|
print-config Print example configuration
|
||||||
@@ -59,43 +56,50 @@ func main() {
|
|||||||
logrus.SetOutput(getLogger(arguments["--log"]))
|
logrus.SetOutput(getLogger(arguments["--log"]))
|
||||||
|
|
||||||
if len(os.Getenv("CACHET_API")) > 0 {
|
if len(os.Getenv("CACHET_API")) > 0 {
|
||||||
cfg.APIUrl = os.Getenv("CACHET_API")
|
cfg.API.URL = os.Getenv("CACHET_API")
|
||||||
}
|
}
|
||||||
if len(os.Getenv("CACHET_TOKEN")) > 0 {
|
if len(os.Getenv("CACHET_TOKEN")) > 0 {
|
||||||
cfg.APIToken = os.Getenv("CACHET_TOKEN")
|
cfg.API.Token = os.Getenv("CACHET_TOKEN")
|
||||||
}
|
}
|
||||||
if len(os.Getenv("CACHET_DEV")) > 0 {
|
if len(os.Getenv("CACHET_DEV")) > 0 {
|
||||||
logrus.SetLevel(logrus.DebugLevel)
|
logrus.SetLevel(logrus.DebugLevel)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cfg.ValidateConfiguration(); err != nil {
|
if valid := cfg.Validate(); !valid {
|
||||||
panic(err)
|
logrus.Errorf("Invalid configuration")
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
logrus.Infof("System: %s\nAPI: %s\nMonitors: %d\n\n", cfg.SystemName, cfg.APIUrl, len(cfg.Monitors))
|
logrus.Infof("System: %s\nAPI: %s\nMonitors: %d\n\n", cfg.SystemName, cfg.API.URL, len(cfg.Monitors))
|
||||||
|
logrus.Infof("Pinging cachet")
|
||||||
|
if err := cfg.API.Ping(); err != nil {
|
||||||
|
logrus.Warnf("Cannot ping cachet!\n%v", err)
|
||||||
|
}
|
||||||
|
|
||||||
wg := &sync.WaitGroup{}
|
wg := &sync.WaitGroup{}
|
||||||
for _, mon := range cfg.Monitors {
|
for _, monitorInterface := range cfg.Monitors {
|
||||||
|
mon := monitorInterface.GetMonitor()
|
||||||
l := logrus.WithFields(logrus.Fields{
|
l := logrus.WithFields(logrus.Fields{
|
||||||
"name": mon.Name,
|
"name": mon.Name,
|
||||||
"interval": mon.CheckInterval,
|
"interval": mon.Interval,
|
||||||
"method": mon.Method,
|
"target": mon.Target,
|
||||||
"url": mon.URL,
|
"timeout": mon.Timeout,
|
||||||
"timeout": mon.HttpTimeout,
|
|
||||||
})
|
})
|
||||||
l.Info(" Starting monitor")
|
l.Info(" Starting monitor")
|
||||||
|
|
||||||
// print features
|
// print features
|
||||||
if len(mon.HttpHeaders) > 0 {
|
if mon.Type == "http" {
|
||||||
for _, h := range mon.HttpHeaders {
|
httpMonitor := monitorInterface.(*cachet.HTTPMonitor)
|
||||||
logrus.Infof(" - HTTP-Header '%s' '%s'", h.Name, h.Value)
|
|
||||||
|
for k, v := range httpMonitor.Headers {
|
||||||
|
logrus.Infof(" - HTTP-Header '%s' '%s'", k, v)
|
||||||
}
|
}
|
||||||
|
if httpMonitor.ExpectedStatusCode > 0 {
|
||||||
|
l.Infof(" - Expect HTTP %d", httpMonitor.ExpectedStatusCode)
|
||||||
}
|
}
|
||||||
if mon.ExpectedStatusCode > 0 {
|
if len(httpMonitor.ExpectedBody) > 0 {
|
||||||
l.Infof(" - Expect HTTP %d", mon.ExpectedStatusCode)
|
l.Infof(" - Expect Body to match \"%v\"", httpMonitor.ExpectedBody)
|
||||||
}
|
}
|
||||||
if len(mon.ExpectedBody) > 0 {
|
|
||||||
l.Infof(" - Expect Body to match \"%v\"", mon.ExpectedBody)
|
|
||||||
}
|
}
|
||||||
if mon.MetricID > 0 {
|
if mon.MetricID > 0 {
|
||||||
l.Infof(" - Log lag to metric id %d\n", mon.MetricID)
|
l.Infof(" - Log lag to metric id %d\n", mon.MetricID)
|
||||||
@@ -113,23 +117,24 @@ func main() {
|
|||||||
|
|
||||||
logrus.Warnf("Abort: Waiting monitors to finish")
|
logrus.Warnf("Abort: Waiting monitors to finish")
|
||||||
for _, mon := range cfg.Monitors {
|
for _, mon := range cfg.Monitors {
|
||||||
mon.Stop()
|
mon.(*cachet.AbstractMonitor).Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func getLogger(logPath *string) *os.File {
|
func getLogger(logPath interface{}) *os.File {
|
||||||
if logPath == nil || len(*logPath) == 0 {
|
if logPath == nil || len(logPath.(string)) == 0 {
|
||||||
return os.Stdout
|
return os.Stdout
|
||||||
}
|
}
|
||||||
|
|
||||||
if file, err := os.Create(logPath); err != nil {
|
file, err := os.Create(logPath.(string))
|
||||||
|
if err != nil {
|
||||||
logrus.Errorf("Unable to open file '%v' for logging: \n%v", logPath, err)
|
logrus.Errorf("Unable to open file '%v' for logging: \n%v", logPath, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
} else {
|
|
||||||
return file
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return file
|
||||||
}
|
}
|
||||||
|
|
||||||
func getConfiguration(path string) (*cachet.CachetMonitor, error) {
|
func getConfiguration(path string) (*cachet.CachetMonitor, error) {
|
||||||
@@ -157,15 +162,43 @@ func getConfiguration(path string) (*cachet.CachetMonitor, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// test file path for yml
|
if err = json.Unmarshal(data, &cfg); err != nil {
|
||||||
if strings.HasSuffix(path, ".yml") || strings.HasSuffix(path, ".yaml") {
|
logrus.Warnf("Unable to parse configuration file")
|
||||||
err = yaml.Unmarshal(data, &cfg)
|
}
|
||||||
} else {
|
|
||||||
err = json.Unmarshal(data, &cfg)
|
cfg.Monitors = make([]cachet.MonitorInterface, len(cfg.RawMonitors))
|
||||||
|
for index, rawMonitor := range cfg.RawMonitors {
|
||||||
|
var abstract cachet.AbstractMonitor
|
||||||
|
if err := json.Unmarshal(rawMonitor, &abstract); err != nil {
|
||||||
|
logrus.Errorf("Unable to unmarshal monitor (index: %d): %v", index, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var t cachet.MonitorInterface
|
||||||
|
var err error
|
||||||
|
|
||||||
|
switch abstract.Type {
|
||||||
|
case "http", "":
|
||||||
|
var s cachet.HTTPMonitor
|
||||||
|
err = json.Unmarshal(rawMonitor, &s)
|
||||||
|
t = &s
|
||||||
|
case "dns":
|
||||||
|
// t = cachet.DNSMonitor
|
||||||
|
case "icmp":
|
||||||
|
// t = cachet.ICMPMonitor
|
||||||
|
case "tcp":
|
||||||
|
// t = cachet.TCPMonitor
|
||||||
|
default:
|
||||||
|
logrus.Errorf("Invalid monitor type (index: %d) %v", index, abstract.Type)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Warnf("Unable to parse configuration file")
|
logrus.Errorf("Unable to unmarshal monitor to type (index: %d): %v", index, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Monitors[index] = t
|
||||||
}
|
}
|
||||||
|
|
||||||
return &cfg, err
|
return &cfg, err
|
||||||
|
|||||||
28
config.go
28
config.go
@@ -1,27 +1,35 @@
|
|||||||
package cachet
|
package cachet
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
"github.com/Sirupsen/logrus"
|
"github.com/Sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CachetMonitor struct {
|
type CachetMonitor struct {
|
||||||
Name string `json:"system_name"`
|
SystemName string `json:"system_name"`
|
||||||
API CachetAPI `json:"api"`
|
API CachetAPI `json:"api"`
|
||||||
Monitors []*Monitor `json:"monitors"`
|
RawMonitors []json.RawMessage `json:"monitors"`
|
||||||
|
|
||||||
|
Monitors []MonitorInterface `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate configuration
|
||||||
func (cfg *CachetMonitor) Validate() bool {
|
func (cfg *CachetMonitor) Validate() bool {
|
||||||
valid := true
|
valid := true
|
||||||
|
|
||||||
if len(cfg.Name) == 0 {
|
if len(cfg.SystemName) == 0 {
|
||||||
// get hostname
|
// get hostname
|
||||||
cfg.Name = getHostname()
|
cfg.SystemName = getHostname()
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(cfg.API.Token) == 0 || len(cfg.API.Url) == 0 {
|
fmt.Println(cfg.API)
|
||||||
|
if len(cfg.API.Token) == 0 || len(cfg.API.URL) == 0 {
|
||||||
logrus.Warnf("API URL or API Token missing.\nGet help at https://github.com/castawaylabs/cachet-monitor")
|
logrus.Warnf("API URL or API Token missing.\nGet help at https://github.com/castawaylabs/cachet-monitor")
|
||||||
valid = false
|
valid = false
|
||||||
}
|
}
|
||||||
@@ -32,7 +40,7 @@ func (cfg *CachetMonitor) Validate() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, monitor := range cfg.Monitors {
|
for _, monitor := range cfg.Monitors {
|
||||||
if err := monitor.Validate(); !valid {
|
if errs := monitor.Validate(); len(errs) > 0 {
|
||||||
valid = false
|
valid = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -48,11 +56,13 @@ func getHostname() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
addrs, err := net.InterfaceAddrs()
|
addrs, err := net.InterfaceAddrs()
|
||||||
if err != nil {
|
if err != nil || len(addrs) == 0 {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, addr := range addrs {
|
return addrs[0].String()
|
||||||
return addr.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getMs() int64 {
|
||||||
|
return time.Now().UnixNano() / int64(time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
{
|
{
|
||||||
"api_url": "https://demo.cachethq.io/api/v1",
|
"api": {
|
||||||
"api_token": "9yMHsdioQosnyVK4iCVR",
|
"url": "https://demo.cachethq.io/api/v1",
|
||||||
"insecure_api": true,
|
"token": "9yMHsdioQosnyVK4iCVR",
|
||||||
|
"insecure": true
|
||||||
|
},
|
||||||
"monitors": [
|
"monitors": [
|
||||||
{
|
{
|
||||||
"name": "google",
|
"name": "google",
|
||||||
@@ -10,12 +12,9 @@
|
|||||||
"component_id": 1,
|
"component_id": 1,
|
||||||
"interval": 10,
|
"interval": 10,
|
||||||
"timeout": 5,
|
"timeout": 5,
|
||||||
"headers": [
|
"headers": {
|
||||||
{
|
"Authorization": "Basic <hash>"
|
||||||
"header": "Authorization",
|
},
|
||||||
"value": "Basic <hash>"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"expected_status_code": 200,
|
"expected_status_code": 200,
|
||||||
"strict_tls": true
|
"strict_tls": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +0,0 @@
|
|||||||
apiurl: https://demo.cachethq.io/api/v1
|
|
||||||
apitoken: 9yMHsdioQosnyVK4iCVR
|
|
||||||
insecureapi: true
|
|
||||||
monitors:
|
|
||||||
- name: google
|
|
||||||
type: http
|
|
||||||
url: https://google.com
|
|
||||||
stricttls: true
|
|
||||||
threshold: 80
|
|
||||||
componentid: 1
|
|
||||||
interval: 10
|
|
||||||
timeout: 5
|
|
||||||
expectedstatuscode: 200
|
|
||||||
headers:
|
|
||||||
- name: Authorization
|
|
||||||
value: Basic <xyz>
|
|
||||||
129
http.go
129
http.go
@@ -1,63 +1,127 @@
|
|||||||
package cachet
|
package cachet
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HttpMonitor struct {
|
type HTTPMonitor struct {
|
||||||
URL string `json:"url"`
|
*AbstractMonitor
|
||||||
Method string `json:"method"`
|
|
||||||
StrictTLS bool `json:"strict_tls"`
|
|
||||||
CheckInterval time.Duration `json:"interval"`
|
|
||||||
HttpTimeout time.Duration `json:"timeout"`
|
|
||||||
|
|
||||||
// Threshold = percentage
|
Method string `json:"method"`
|
||||||
Threshold float32 `json:"threshold"`
|
|
||||||
ExpectedStatusCode int `json:"expected_status_code"`
|
ExpectedStatusCode int `json:"expected_status_code"`
|
||||||
|
Headers map[string]string `json:"headers"`
|
||||||
|
|
||||||
// compiled to Regexp
|
// compiled to Regexp
|
||||||
ExpectedBody string `json:"expected_body"`
|
ExpectedBody string `json:"expected_body"`
|
||||||
bodyRegexp *regexp.Regexp
|
bodyRegexp *regexp.Regexp
|
||||||
}
|
}
|
||||||
|
|
||||||
type TCPMonitor struct{}
|
func (monitor *HTTPMonitor) do() bool {
|
||||||
type ICMPMonitor struct{}
|
client := &http.Client{
|
||||||
type DNSMonitor struct{}
|
Timeout: time.Duration(monitor.Timeout * time.Second),
|
||||||
|
}
|
||||||
func (monitor *CachetMonitor) makeRequest(requestType string, url string, reqBody []byte) (*http.Response, []byte, error) {
|
if monitor.Strict == false {
|
||||||
req, err := http.NewRequest(requestType, monitor.APIUrl+url, bytes.NewBuffer(reqBody))
|
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
req.Header.Set("X-Cachet-Token", monitor.APIToken)
|
|
||||||
|
|
||||||
client := &http.Client{}
|
|
||||||
if monitor.InsecureAPI == true {
|
|
||||||
client.Transport = &http.Transport{
|
client.Transport = &http.Transport{
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||||
Proxy: http.ProxyFromEnvironment,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := client.Do(req)
|
req, err := http.NewRequest(monitor.Method, monitor.Target, nil)
|
||||||
|
for k, v := range monitor.Headers {
|
||||||
|
req.Header.Add(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, []byte{}, err
|
monitor.lastFailReason = err.Error()
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
defer res.Body.Close()
|
defer resp.Body.Close()
|
||||||
body, _ := ioutil.ReadAll(res.Body)
|
|
||||||
|
|
||||||
return res, body, nil
|
if monitor.ExpectedStatusCode > 0 && resp.StatusCode != monitor.ExpectedStatusCode {
|
||||||
|
monitor.lastFailReason = "Unexpected response code: " + strconv.Itoa(resp.StatusCode) + ". Expected " + strconv.Itoa(monitor.ExpectedStatusCode)
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if monitor.bodyRegexp != nil {
|
||||||
|
// check body
|
||||||
|
responseBody, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
monitor.lastFailReason = err.Error()
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
match := monitor.bodyRegexp.Match(responseBody)
|
||||||
|
if !match {
|
||||||
|
monitor.lastFailReason = "Unexpected body: " + string(responseBody) + ". Expected to match " + monitor.ExpectedBody
|
||||||
|
}
|
||||||
|
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (monitor *HTTPMonitor) Validate() []string {
|
||||||
|
errs := []string{}
|
||||||
|
if len(monitor.ExpectedBody) > 0 {
|
||||||
|
exp, err := regexp.Compile(monitor.ExpectedBody)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, "Regexp compilation failure: "+err.Error())
|
||||||
|
} else {
|
||||||
|
monitor.bodyRegexp = exp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(monitor.ExpectedBody) == 0 && monitor.ExpectedStatusCode == 0 {
|
||||||
|
errs = append(errs, "Both 'expected_body' and 'expected_status_code' fields empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
if monitor.Interval < 1 {
|
||||||
|
monitor.Interval = DefaultInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
if monitor.Timeout < 1 {
|
||||||
|
monitor.Timeout = DefaultTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
monitor.Method = strings.ToUpper(monitor.Method)
|
||||||
|
switch monitor.Method {
|
||||||
|
case "GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD":
|
||||||
|
break
|
||||||
|
case "":
|
||||||
|
monitor.Method = "GET"
|
||||||
|
default:
|
||||||
|
errs = append(errs, "Unsupported check method: "+monitor.Method)
|
||||||
|
}
|
||||||
|
|
||||||
|
if monitor.ComponentID == 0 && monitor.MetricID == 0 {
|
||||||
|
errs = append(errs, "component_id & metric_id are unset")
|
||||||
|
}
|
||||||
|
|
||||||
|
if monitor.Threshold <= 0 {
|
||||||
|
monitor.Threshold = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mon *HTTPMonitor) GetMonitor() *AbstractMonitor {
|
||||||
|
return mon.AbstractMonitor
|
||||||
}
|
}
|
||||||
|
|
||||||
// SendMetric sends lag metric point
|
// SendMetric sends lag metric point
|
||||||
func (monitor *Monitor) SendMetric(delay int64) error {
|
/*func (monitor *Monitor) SendMetric(delay int64) error {
|
||||||
if monitor.MetricID == 0 {
|
if monitor.MetricID == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -73,7 +137,4 @@ func (monitor *Monitor) SendMetric(delay int64) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
func getMs() int64 {
|
|
||||||
return time.Now().UnixNano() / int64(time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/Sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Incident Cachet data model
|
// Incident Cachet data model
|
||||||
@@ -33,7 +35,7 @@ func (incident *Incident) Send(cfg *CachetMonitor) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cfg.Logger.Printf("cannot fetch component: %v", err)
|
logrus.Warnf("cannot fetch component: %v", err)
|
||||||
}
|
}
|
||||||
case 4:
|
case 4:
|
||||||
// fixed
|
// fixed
|
||||||
@@ -49,7 +51,7 @@ func (incident *Incident) Send(cfg *CachetMonitor) error {
|
|||||||
|
|
||||||
jsonBytes, _ := json.Marshal(incident)
|
jsonBytes, _ := json.Marshal(incident)
|
||||||
|
|
||||||
resp, body, err := cfg.makeRequest(requestType, requestURL, jsonBytes)
|
resp, body, err := cfg.API.NewRequest(requestType, requestURL, jsonBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -72,7 +74,7 @@ func (incident *Incident) Send(cfg *CachetMonitor) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (incident *Incident) GetComponentStatus(cfg *CachetMonitor) (int, error) {
|
func (incident *Incident) GetComponentStatus(cfg *CachetMonitor) (int, error) {
|
||||||
resp, body, err := cfg.makeRequest("GET", "/components/"+strconv.Itoa(incident.ComponentID), nil)
|
resp, body, err := cfg.API.NewRequest("GET", "/components/"+strconv.Itoa(incident.ComponentID), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|||||||
193
monitor.go
193
monitor.go
@@ -1,46 +1,48 @@
|
|||||||
package cachet
|
package cachet
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/tls"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/Sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
const DefaultInterval = 60
|
const DefaultInterval = time.Second * 60
|
||||||
const DefaultTimeout = 1
|
const DefaultTimeout = time.Second
|
||||||
const DefaultTimeFormat = "15:04:05 Jan 2 MST"
|
const DefaultTimeFormat = "15:04:05 Jan 2 MST"
|
||||||
|
const HistorySize = 10
|
||||||
|
|
||||||
type HttpHeader struct {
|
type MonitorInterface interface {
|
||||||
Name string `json:"header"`
|
do() bool
|
||||||
Value string `json:"value"`
|
Validate() []string
|
||||||
|
GetMonitor() *AbstractMonitor
|
||||||
}
|
}
|
||||||
|
|
||||||
// Monitor data model
|
// AbstractMonitor data model
|
||||||
type Monitor struct {
|
type AbstractMonitor struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
URL string `json:"url"`
|
Target string `json:"target"`
|
||||||
Method string `json:"method"`
|
|
||||||
StrictTLS bool `json:"strict_tls"`
|
// (default)http, tcp, dns, icmp
|
||||||
CheckInterval time.Duration `json:"interval"`
|
Type string `json:"type"`
|
||||||
HttpTimeout time.Duration `json:"timeout"`
|
|
||||||
HttpHeaders []*HttpHeader `json:"headers"`
|
// defaults true
|
||||||
|
Strict bool `json:"strict"`
|
||||||
|
|
||||||
|
Interval time.Duration `json:"interval"`
|
||||||
|
Timeout time.Duration `json:"timeout"`
|
||||||
|
|
||||||
MetricID int `json:"metric_id"`
|
MetricID int `json:"metric_id"`
|
||||||
ComponentID int `json:"component_id"`
|
ComponentID int `json:"component_id"`
|
||||||
|
|
||||||
|
// Templating stuff
|
||||||
|
Template struct {
|
||||||
|
Investigating MessageTemplate `json:"investigating"`
|
||||||
|
Fixed MessageTemplate `json:"fixed"`
|
||||||
|
} `json:"template"`
|
||||||
|
|
||||||
// Threshold = percentage
|
// Threshold = percentage
|
||||||
Threshold float32 `json:"threshold"`
|
Threshold float32 `json:"threshold"`
|
||||||
ExpectedStatusCode int `json:"expected_status_code"`
|
|
||||||
// compiled to Regexp
|
|
||||||
ExpectedBody string `json:"expected_body"`
|
|
||||||
bodyRegexp *regexp.Regexp
|
|
||||||
|
|
||||||
history []bool
|
history []bool
|
||||||
lastFailReason string
|
lastFailReason string
|
||||||
@@ -51,13 +53,23 @@ type Monitor struct {
|
|||||||
stopC chan bool
|
stopC chan bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mon *Monitor) Start(cfg *CachetMonitor, wg *sync.WaitGroup) {
|
func (mon *AbstractMonitor) do() bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
func (mon *AbstractMonitor) Validate() []string {
|
||||||
|
return []string{}
|
||||||
|
}
|
||||||
|
func (mon *AbstractMonitor) GetMonitor() *AbstractMonitor {
|
||||||
|
return mon
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mon *AbstractMonitor) Start(cfg *CachetMonitor, wg *sync.WaitGroup) {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
mon.config = cfg
|
mon.config = cfg
|
||||||
mon.stopC = make(chan bool)
|
mon.stopC = make(chan bool)
|
||||||
mon.Tick()
|
mon.Tick()
|
||||||
|
|
||||||
ticker := time.NewTicker(mon.CheckInterval * time.Second)
|
ticker := time.NewTicker(mon.Interval * time.Second)
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
@@ -69,7 +81,7 @@ func (mon *Monitor) Start(cfg *CachetMonitor, wg *sync.WaitGroup) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) Stop() {
|
func (monitor *AbstractMonitor) Stop() {
|
||||||
if monitor.Stopped() {
|
if monitor.Stopped() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -77,7 +89,7 @@ func (monitor *Monitor) Stop() {
|
|||||||
close(monitor.stopC)
|
close(monitor.stopC)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) Stopped() bool {
|
func (monitor *AbstractMonitor) Stopped() bool {
|
||||||
select {
|
select {
|
||||||
case <-monitor.stopC:
|
case <-monitor.stopC:
|
||||||
return true
|
return true
|
||||||
@@ -86,77 +98,29 @@ func (monitor *Monitor) Stopped() bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) Tick() {
|
func (monitor *AbstractMonitor) Tick() {
|
||||||
reqStart := getMs()
|
reqStart := getMs()
|
||||||
isUp := monitor.doRequest()
|
up := monitor.do()
|
||||||
lag := getMs() - reqStart
|
lag := getMs() - reqStart
|
||||||
|
|
||||||
if len(monitor.history) == 9 {
|
if len(monitor.history) == HistorySize-1 {
|
||||||
monitor.config.Logger.Printf("%v is now saturated\n", monitor.Name)
|
logrus.Warnf("%v is now saturated\n", monitor.Name)
|
||||||
}
|
}
|
||||||
if len(monitor.history) >= 10 {
|
if len(monitor.history) >= HistorySize {
|
||||||
monitor.history = monitor.history[len(monitor.history)-9:]
|
monitor.history = monitor.history[len(monitor.history)-(HistorySize-1):]
|
||||||
}
|
}
|
||||||
monitor.history = append(monitor.history, isUp)
|
monitor.history = append(monitor.history, up)
|
||||||
monitor.AnalyseData()
|
monitor.AnalyseData()
|
||||||
|
|
||||||
if isUp == true && monitor.MetricID > 0 {
|
// report lag
|
||||||
monitor.SendMetric(lag)
|
if up && monitor.MetricID > 0 {
|
||||||
|
logrus.Infof("%v", lag)
|
||||||
|
// monitor.SendMetric(lag)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) doRequest() bool {
|
|
||||||
client := &http.Client{
|
|
||||||
Timeout: time.Duration(monitor.HttpTimeout * time.Second),
|
|
||||||
}
|
|
||||||
if monitor.StrictTLS == false {
|
|
||||||
client.Transport = &http.Transport{
|
|
||||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(monitor.Method, monitor.URL, nil)
|
|
||||||
for _, h := range monitor.HttpHeaders {
|
|
||||||
req.Header.Add(h.Name, h.Value)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
monitor.lastFailReason = err.Error()
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if monitor.ExpectedStatusCode > 0 && resp.StatusCode != monitor.ExpectedStatusCode {
|
|
||||||
monitor.lastFailReason = "Unexpected response code: " + strconv.Itoa(resp.StatusCode) + ". Expected " + strconv.Itoa(monitor.ExpectedStatusCode)
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.bodyRegexp != nil {
|
|
||||||
// check body
|
|
||||||
responseBody, err := ioutil.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
monitor.lastFailReason = err.Error()
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
match := monitor.bodyRegexp.Match(responseBody)
|
|
||||||
if !match {
|
|
||||||
monitor.lastFailReason = "Unexpected body: " + string(responseBody) + ". Expected to match " + monitor.ExpectedBody
|
|
||||||
}
|
|
||||||
|
|
||||||
return match
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// AnalyseData decides if the monitor is statistically up or down and creates / resolves an incident
|
// AnalyseData decides if the monitor is statistically up or down and creates / resolves an incident
|
||||||
func (monitor *Monitor) AnalyseData() {
|
func (monitor *AbstractMonitor) AnalyseData() {
|
||||||
// look at the past few incidents
|
// look at the past few incidents
|
||||||
numDown := 0
|
numDown := 0
|
||||||
for _, wasUp := range monitor.history {
|
for _, wasUp := range monitor.history {
|
||||||
@@ -166,7 +130,7 @@ func (monitor *Monitor) AnalyseData() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
t := (float32(numDown) / float32(len(monitor.history))) * 100
|
t := (float32(numDown) / float32(len(monitor.history))) * 100
|
||||||
monitor.config.Logger.Printf("%s %.2f%%/%.2f%% down at %v\n", monitor.Name, t, monitor.Threshold, time.Now().UnixNano()/int64(time.Second))
|
logrus.Printf("%s %.2f%%/%.2f%% down at %v\n", monitor.Name, t, monitor.Threshold, time.Now().UnixNano()/int64(time.Second))
|
||||||
|
|
||||||
if len(monitor.history) != 10 {
|
if len(monitor.history) != 10 {
|
||||||
// not saturated
|
// not saturated
|
||||||
@@ -186,16 +150,16 @@ func (monitor *Monitor) AnalyseData() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// is down, create an incident
|
// is down, create an incident
|
||||||
monitor.config.Logger.Printf("%v creating incident. Monitor is down: %v", monitor.Name, monitor.lastFailReason)
|
logrus.Printf("%v creating incident. Monitor is down: %v", monitor.Name, monitor.lastFailReason)
|
||||||
// set investigating status
|
// set investigating status
|
||||||
monitor.incident.SetInvestigating()
|
monitor.incident.SetInvestigating()
|
||||||
// create/update incident
|
// create/update incident
|
||||||
if err := monitor.incident.Send(monitor.config); err != nil {
|
if err := monitor.incident.Send(monitor.config); err != nil {
|
||||||
monitor.config.Logger.Printf("Error sending incident: %v\n", err)
|
logrus.Printf("Error sending incident: %v\n", err)
|
||||||
}
|
}
|
||||||
} else if t < monitor.Threshold && monitor.incident != nil {
|
} else if t < monitor.Threshold && monitor.incident != nil {
|
||||||
// was down, created an incident, its now ok, make it resolved.
|
// was down, created an incident, its now ok, make it resolved.
|
||||||
monitor.config.Logger.Printf("%v resolved downtime incident", monitor.Name)
|
logrus.Printf("%v resolved downtime incident", monitor.Name)
|
||||||
|
|
||||||
// resolve incident
|
// resolve incident
|
||||||
monitor.incident.Message = "\n**Resolved** - " + time.Now().Format(DefaultTimeFormat) + "\n\n - - - \n\n" + monitor.incident.Message
|
monitor.incident.Message = "\n**Resolved** - " + time.Now().Format(DefaultTimeFormat) + "\n\n - - - \n\n" + monitor.incident.Message
|
||||||
@@ -206,46 +170,3 @@ func (monitor *Monitor) AnalyseData() {
|
|||||||
monitor.incident = nil
|
monitor.incident = nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (monitor *Monitor) Validate() error {
|
|
||||||
if len(monitor.ExpectedBody) > 0 {
|
|
||||||
exp, err := regexp.Compile(monitor.ExpectedBody)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor.bodyRegexp = exp
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(monitor.ExpectedBody) == 0 && monitor.ExpectedStatusCode == 0 {
|
|
||||||
return errors.New("Nothing to check, both 'expected_body' and 'expected_status_code' fields empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.CheckInterval < 1 {
|
|
||||||
monitor.CheckInterval = DefaultInterval
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.HttpTimeout < 1 {
|
|
||||||
monitor.HttpTimeout = DefaultTimeout
|
|
||||||
}
|
|
||||||
|
|
||||||
monitor.Method = strings.ToUpper(monitor.Method)
|
|
||||||
switch monitor.Method {
|
|
||||||
case "GET", "POST", "DELETE", "OPTIONS", "HEAD":
|
|
||||||
break
|
|
||||||
case "":
|
|
||||||
monitor.Method = "GET"
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("Unsupported check method: %v", monitor.Method)
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.ComponentID == 0 && monitor.MetricID == 0 {
|
|
||||||
return errors.New("component_id & metric_id are unset")
|
|
||||||
}
|
|
||||||
|
|
||||||
if monitor.Threshold <= 0 {
|
|
||||||
monitor.Threshold = 100
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
13
tcp.go
Normal file
13
tcp.go
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
package cachet
|
||||||
|
|
||||||
|
type TCPMonitor struct {
|
||||||
|
// same as output from net.JoinHostPort
|
||||||
|
// defaults to parsed config from /etc/resolv.conf when empty
|
||||||
|
DNSServer string
|
||||||
|
|
||||||
|
// Will be converted to FQDN
|
||||||
|
Domain string
|
||||||
|
Type string
|
||||||
|
// expected answers (regex)
|
||||||
|
Expect []string
|
||||||
|
}
|
||||||
6
template.go
Normal file
6
template.go
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
package cachet
|
||||||
|
|
||||||
|
type MessageTemplate struct {
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user