-
Notifications
You must be signed in to change notification settings - Fork 3
/
alert.go
192 lines (159 loc) · 4.76 KB
/
alert.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
package main
import (
"bytes"
"errors"
"fmt"
"os/exec"
"strings"
"text/template"
"time"
"git.iamthefij.com/iamthefij/slog"
)
var (
errNoTemplate = errors.New("no template")
// ErrAlertFailed indicates that an alert failed to send
ErrAlertFailed = errors.New("alert failed")
)
// Alert is a config driven mechanism for sending a notice
type Alert struct {
Name string
Command CommandOrShell
commandTemplate []*template.Template
commandShellTemplate *template.Template
}
// AlertNotice captures the context for an alert to be sent
type AlertNotice struct {
AlertCount int16
FailureCount int16
IsUp bool
LastSuccess time.Time
MonitorName string
LastCheckOutput string
}
// IsValid returns a boolean indicating if the Alert has been correctly
// configured
func (alert Alert) IsValid() bool {
return !alert.Command.Empty()
}
// BuildTemplates compiles command templates for the Alert
func (alert *Alert) BuildTemplates() error {
// TODO: Remove legacy template support later after 1.0
legacy := strings.NewReplacer(
"{alert_count}", "{{.AlertCount}}",
"{alert_message}", "{{.MonitorName}} check has failed {{.FailureCount}} times",
"{failure_count}", "{{.FailureCount}}",
"{last_output}", "{{.LastCheckOutput}}",
"{last_success}", "{{.LastSuccess}}",
"{monitor_name}", "{{.MonitorName}}",
)
slog.Debugf("Building template for alert %s", alert.Name)
// Time format func factory
tff := func(formatString string) func(time.Time) string {
return func(t time.Time) string {
return t.Format(formatString)
}
}
// Create some functions for formatting datetimes in popular formats
timeFormatFuncs := template.FuncMap{
"ANSIC": tff(time.ANSIC),
"UnixDate": tff(time.UnixDate),
"RubyDate": tff(time.RubyDate),
"RFC822Z": tff(time.RFC822Z),
"RFC850": tff(time.RFC850),
"RFC1123": tff(time.RFC1123),
"RFC1123Z": tff(time.RFC1123Z),
"RFC3339": tff(time.RFC3339),
"RFC3339Nano": tff(time.RFC3339Nano),
"FormatTime": func(t time.Time, timeFormat string) string {
return t.Format(timeFormat)
},
"InTZ": func(t time.Time, tzName string) (time.Time, error) {
tz, err := time.LoadLocation(tzName)
if err != nil {
return t, fmt.Errorf("failed to convert time to specified tz: %w", err)
}
return t.In(tz), nil
},
}
switch {
case alert.commandTemplate == nil && alert.Command.Command != nil:
alert.commandTemplate = []*template.Template{}
for i, cmdPart := range alert.Command.Command {
if PyCompat {
cmdPart = legacy.Replace(cmdPart)
}
alert.commandTemplate = append(alert.commandTemplate, template.Must(
template.New(alert.Name+fmt.Sprint(i)).Funcs(timeFormatFuncs).Parse(cmdPart),
))
}
case alert.commandShellTemplate == nil && alert.Command.ShellCommand != "":
shellCmd := alert.Command.ShellCommand
if PyCompat {
shellCmd = legacy.Replace(shellCmd)
}
alert.commandShellTemplate = template.Must(
template.New(alert.Name).Funcs(timeFormatFuncs).Parse(shellCmd),
)
default:
return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
}
return nil
}
// Send will send an alert notice by executing the command template
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
var cmd *exec.Cmd
switch {
case alert.commandTemplate != nil:
command := []string{}
for _, cmdTmp := range alert.commandTemplate {
var commandBuffer bytes.Buffer
err = cmdTmp.Execute(&commandBuffer, notice)
if err != nil {
return
}
command = append(command, commandBuffer.String())
}
cmd = exec.Command(command[0], command[1:]...)
case alert.commandShellTemplate != nil:
var commandBuffer bytes.Buffer
err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
if err != nil {
return
}
shellCommand := commandBuffer.String()
cmd = ShellCommand(shellCommand)
default:
err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate)
return
}
// Exit if we're not ready to run the command
if cmd == nil || err != nil {
return
}
var output []byte
output, err = cmd.CombinedOutput()
outputStr = string(output)
slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
if err != nil {
err = fmt.Errorf(
"Alert %s failed to send. Returned %w: %w",
alert.Name,
err,
ErrAlertFailed,
)
}
return outputStr, err
}
// NewLogAlert creates an alert that does basic logging using echo
func NewLogAlert() *Alert {
return &Alert{
Name: "log",
Command: CommandOrShell{
Command: []string{
"echo",
"{{.MonitorName}} {{if .IsUp}}has recovered{{else}}check has failed {{.FailureCount}} times{{end}}",
},
},
}
}