notify when we don't receive a thing in the last 24 hours
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Elizabeth Hunt 2024-04-21 21:14:58 -07:00
parent bef72712c2
commit 9d96d1a942
Signed by: simponic
GPG Key ID: 2909B9A7FF6213EE
7 changed files with 130 additions and 21 deletions

View File

@ -11,4 +11,4 @@ RUN go build -o /app/backupnotify
EXPOSE 8080
CMD ["/app/nackupnotify", "--server", "--migrate", "--port", "8080", "--template-path", "/app/templates", "--database-path", "/app/db/nackupnotify.db", "--static-path", "/app/static", "--scheduler", "--ntfy-endpoint", "https://ntfy.internal.simponic.xyz", "--ntfy-topics", "server-backups"]
CMD ["/app/nackupnotify", "--server", "--migrate", "--port", "8080", "--template-path", "/app/templates", "--database-path", "/app/db/nackupnotify.db", "--static-path", "/app/static", "--scheduler", "--ntfy-endpoint", "https://ntfy.internal.simponic.xyz", "--ntfy-topics", "server-backups", "--ntfy-alert-topics", "alerts"]

View File

@ -13,8 +13,9 @@ type Arguments struct {
Migrate bool
Scheduler bool
NtfyEndpoint string
NtfyTopics []string
NtfyEndpoint string
NtfyBackupTopics []string
NtfyAlertTopics []string
Port int
Server bool
@ -29,7 +30,8 @@ func GetArgs() (*Arguments, error) {
scheduler := flag.Bool("scheduler", false, "Run scheduled jobs via cron")
migrate := flag.Bool("migrate", false, "Run the migrations")
ntfyEndpoint := flag.String("ntfy-endpoint", "https://ntfy.sh", "")
ntfyTopics := flag.String("ntfy-topics", "server-backup", "")
ntfyBackupTopics := flag.String("ntfy-topics", "server-backup", "")
ntfyAlertTopics := flag.String("ntfy-alert-topics", "server-backup", "")
port := flag.Int("port", 8080, "Port to listen on")
server := flag.Bool("server", false, "Run the server")
@ -37,15 +39,16 @@ func GetArgs() (*Arguments, error) {
flag.Parse()
arguments := &Arguments{
DatabasePath: *databasePath,
TemplatePath: *templatePath,
StaticPath: *staticPath,
Port: *port,
Server: *server,
Migrate: *migrate,
Scheduler: *scheduler,
NtfyEndpoint: *ntfyEndpoint,
NtfyTopics: strings.Split(*ntfyTopics, ","),
DatabasePath: *databasePath,
TemplatePath: *templatePath,
StaticPath: *staticPath,
Port: *port,
Server: *server,
Migrate: *migrate,
Scheduler: *scheduler,
NtfyEndpoint: *ntfyEndpoint,
NtfyBackupTopics: strings.Split(*ntfyBackupTopics, ","),
NtfyAlertTopics: strings.Split(*ntfyAlertTopics, ","),
}
return arguments, nil

View File

@ -49,3 +49,47 @@ func ReceivedBackup(dbConn *sql.DB, hostname string) error {
return err
}
func BackupedHostnamesInTheLastDay(dbConn *sql.DB) ([]string, error) {
log.Println("listing hostnames that have been backed up in the last day")
rows, err := dbConn.Query(`SELECT DISTINCT hostname FROM backups WHERE received_on > ?;`, time.Now().Add(-24*time.Hour))
if err != nil {
return nil, err
}
defer rows.Close()
hostnames := []string{}
for rows.Next() {
var hostname string
err := rows.Scan(&hostname)
if err != nil {
return nil, err
}
hostnames = append(hostnames, hostname)
}
return hostnames, nil
}
func ReceivedHostnames(dbConn *sql.DB) ([]string, error) {
log.Println("listing received hostnames")
rows, err := dbConn.Query(`SELECT DISTINCT hostname FROM backups;`)
if err != nil {
return nil, err
}
defer rows.Close()
hostnames := []string{}
for rows.Next() {
var hostname string
err := rows.Scan(&hostname)
if err != nil {
return nil, err
}
hostnames = append(hostnames, hostname)
}
return hostnames, nil
}

View File

@ -37,16 +37,15 @@ func main() {
}
if argv.Scheduler {
scheduler.StartScheduler(dbConn)
scheduler.StartScheduler(dbConn, argv)
}
if argv.NtfyEndpoint != "" {
ntfy := ntfy.MakeNtfyWatcher(argv.NtfyEndpoint, argv.NtfyTopics)
ntfy := ntfy.MakeNtfyWatcher(argv.NtfyEndpoint, argv.NtfyBackupTopics)
notifications := ntfy.Watch()
go func() {
for notification := range notifications {
// message type is a struct, so we can marshal it to JSON
message := notification.Text
messageStruct := struct {
Id string `json:"id"`

16
ntfy/publisher.go Normal file
View File

@ -0,0 +1,16 @@
package ntfy
import (
"net/http"
"strings"
)
func SendMessage(message string, endpoint string, topics []string) error {
for _, topic := range topics {
_, err := http.Post(endpoint+"/"+topic, "text/plain", strings.NewReader(message))
if err != nil {
return err
}
}
return nil
}

View File

@ -30,7 +30,7 @@ func (w *NtfyWatcher) Watch() chan Message {
retryTime := 5 * time.Second
retries := retryCount
retry := func() {
sleepAndDecrementRetry := func() {
log.Println("waiting 5 seconds before reconnecting. retries left:", retries, "topic:", topic, "endpoint:", w.Endpoint)
time.Sleep(retryTime)
retries--
@ -45,7 +45,7 @@ func (w *NtfyWatcher) Watch() chan Message {
resp, err := http.Get(endpoint)
if err != nil {
log.Println("error connecting to endpoint:", err)
retry()
sleepAndDecrementRetry()
continue
}
@ -60,7 +60,7 @@ func (w *NtfyWatcher) Watch() chan Message {
if err := scanner.Err(); err != nil {
log.Println("error reading response body:", err)
retry()
sleepAndDecrementRetry()
}
}
}()

View File

@ -2,16 +2,63 @@ package scheduler
import (
"database/sql"
"log"
"strings"
"time"
"git.simponic.xyz/simponic/backup-notify/args"
"git.simponic.xyz/simponic/backup-notify/database"
"git.simponic.xyz/simponic/backup-notify/ntfy"
"github.com/go-co-op/gocron"
)
func StartScheduler(dbConn *sql.DB) {
func contains(hostnames []string, hostname string) bool {
for _, h := range hostnames {
if h == hostname {
return true
}
}
return false
}
func StartScheduler(dbConn *sql.DB, argv *args.Arguments) {
scheduler := gocron.NewScheduler(time.Local)
scheduler.Every(1).Hour().Do(func() {
database.DeleteOldBackups(dbConn, 31)
err := database.DeleteOldBackups(dbConn, 31)
if err != nil {
log.Println("could not delete old backups:", err)
}
})
scheduler.Every(1).Day().Do(func() {
hostnames, err := database.BackupedHostnamesInTheLastDay(dbConn)
if err != nil {
log.Println("could not list hostnames that have been backed up in the last day:", err)
}
allHostnames, err := database.ReceivedHostnames(dbConn)
if err != nil {
log.Println("could not list received hostnames:", err)
}
failedHostnames := []string{}
for _, hostname := range allHostnames {
if !contains(hostnames, hostname) {
failedHostnames = append(failedHostnames, hostname)
}
}
if len(failedHostnames) > 0 {
msg := "the following hostnames have not been backed up in the last day:\n" + strings.Join(failedHostnames, ", ")
log.Println(msg)
err := ntfy.SendMessage(msg, argv.NtfyEndpoint, argv.NtfyAlertTopics)
if err != nil {
log.Println("could not send alert:", err)
}
}
})
scheduler.StartAsync()
}