notify when we don't receive a thing in the last 24 hours

Dockerfile

@@ -11,4 +11,4 @@ RUN go build -o /app/backupnotify
 
 EXPOSE 8080
 
-CMD ["/app/nackupnotify", "--server", "--migrate", "--port", "8080", "--template-path", "/app/templates", "--database-path", "/app/db/nackupnotify.db", "--static-path", "/app/static", "--scheduler", "--ntfy-endpoint", "https://ntfy.internal.simponic.xyz", "--ntfy-topics", "server-backups"]
+CMD ["/app/nackupnotify", "--server", "--migrate", "--port", "8080", "--template-path", "/app/templates", "--database-path", "/app/db/nackupnotify.db", "--static-path", "/app/static", "--scheduler", "--ntfy-endpoint", "https://ntfy.internal.simponic.xyz", "--ntfy-topics", "server-backups", "--ntfy-alert-topics", "alerts"]

args/args.go

@@ -13,8 +13,9 @@ type Arguments struct {
 	Migrate   bool
 	Scheduler bool
 
-	NtfyEndpoint string
+	NtfyEndpoint     string
-	NtfyTopics   []string
+	NtfyBackupTopics []string
+	NtfyAlertTopics  []string
 
 	Port   int
 	Server bool
@@ -29,7 +30,8 @@ func GetArgs() (*Arguments, error) {
 	scheduler := flag.Bool("scheduler", false, "Run scheduled jobs via cron")
 	migrate := flag.Bool("migrate", false, "Run the migrations")
 	ntfyEndpoint := flag.String("ntfy-endpoint", "https://ntfy.sh", "")
-	ntfyTopics := flag.String("ntfy-topics", "server-backup", "")
+	ntfyBackupTopics := flag.String("ntfy-topics", "server-backup", "")
+	ntfyAlertTopics := flag.String("ntfy-alert-topics", "server-backup", "")
 
 	port := flag.Int("port", 8080, "Port to listen on")
 	server := flag.Bool("server", false, "Run the server")
@@ -37,15 +39,16 @@ func GetArgs() (*Arguments, error) {
 	flag.Parse()
 
 	arguments := &Arguments{
-		DatabasePath: *databasePath,
+		DatabasePath:     *databasePath,
-		TemplatePath: *templatePath,
+		TemplatePath:     *templatePath,
-		StaticPath:   *staticPath,
+		StaticPath:       *staticPath,
-		Port:         *port,
+		Port:             *port,
-		Server:       *server,
+		Server:           *server,
-		Migrate:      *migrate,
+		Migrate:          *migrate,
-		Scheduler:    *scheduler,
+		Scheduler:        *scheduler,
-		NtfyEndpoint: *ntfyEndpoint,
+		NtfyEndpoint:     *ntfyEndpoint,
-		NtfyTopics:   strings.Split(*ntfyTopics, ","),
+		NtfyBackupTopics: strings.Split(*ntfyBackupTopics, ","),
+		NtfyAlertTopics:  strings.Split(*ntfyAlertTopics, ","),
 	}
 
 	return arguments, nil

database/backups.go

@@ -49,3 +49,47 @@ func ReceivedBackup(dbConn *sql.DB, hostname string) error {
 
 	return err
 }
+
+func BackupedHostnamesInTheLastDay(dbConn *sql.DB) ([]string, error) {
+	log.Println("listing hostnames that have been backed up in the last day")
+
+	rows, err := dbConn.Query(`SELECT DISTINCT hostname FROM backups WHERE received_on > ?;`, time.Now().Add(-24*time.Hour))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	hostnames := []string{}
+	for rows.Next() {
+		var hostname string
+		err := rows.Scan(&hostname)
+		if err != nil {
+			return nil, err
+		}
+		hostnames = append(hostnames, hostname)
+	}
+
+	return hostnames, nil
+}
+
+func ReceivedHostnames(dbConn *sql.DB) ([]string, error) {
+	log.Println("listing received hostnames")
+
+	rows, err := dbConn.Query(`SELECT DISTINCT hostname FROM backups;`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	hostnames := []string{}
+	for rows.Next() {
+		var hostname string
+		err := rows.Scan(&hostname)
+		if err != nil {
+			return nil, err
+		}
+		hostnames = append(hostnames, hostname)
+	}
+
+	return hostnames, nil
+}

main.go

@@ -37,16 +37,15 @@ func main() {
 	}
 
 	if argv.Scheduler {
-		scheduler.StartScheduler(dbConn)
+		scheduler.StartScheduler(dbConn, argv)
 	}
 
 	if argv.NtfyEndpoint != "" {
-		ntfy := ntfy.MakeNtfyWatcher(argv.NtfyEndpoint, argv.NtfyTopics)
+		ntfy := ntfy.MakeNtfyWatcher(argv.NtfyEndpoint, argv.NtfyBackupTopics)
 		notifications := ntfy.Watch()
 
 		go func() {
 			for notification := range notifications {
-				// message type is a struct, so we can marshal it to JSON
 				message := notification.Text
 				messageStruct := struct {
 					Id      string `json:"id"`

ntfy/publisher.go

@@ -0,0 +1,16 @@
+package ntfy
+
+import (
+	"net/http"
+	"strings"
+)
+
+func SendMessage(message string, endpoint string, topics []string) error {
+	for _, topic := range topics {
+		_, err := http.Post(endpoint+"/"+topic, "text/plain", strings.NewReader(message))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

ntfy/watcher.go

@@ -30,7 +30,7 @@ func (w *NtfyWatcher) Watch() chan Message {
 			retryTime := 5 * time.Second
 			retries := retryCount
 
-			retry := func() {
+			sleepAndDecrementRetry := func() {
 				log.Println("waiting 5 seconds before reconnecting. retries left:", retries, "topic:", topic, "endpoint:", w.Endpoint)
 				time.Sleep(retryTime)
 				retries--
@@ -45,7 +45,7 @@ func (w *NtfyWatcher) Watch() chan Message {
 				resp, err := http.Get(endpoint)
 				if err != nil {
 					log.Println("error connecting to endpoint:", err)
-					retry()
+					sleepAndDecrementRetry()
 					continue
 				}
 
@@ -60,7 +60,7 @@ func (w *NtfyWatcher) Watch() chan Message {
 
 				if err := scanner.Err(); err != nil {
 					log.Println("error reading response body:", err)
-					retry()
+					sleepAndDecrementRetry()
 				}
 			}
 		}()

scheduler/scheduler.go

@@ -2,16 +2,63 @@ package scheduler
 
 import (
 	"database/sql"
+	"log"
+	"strings"
 	"time"
 
+	"git.simponic.xyz/simponic/backup-notify/args"
 	"git.simponic.xyz/simponic/backup-notify/database"
+	"git.simponic.xyz/simponic/backup-notify/ntfy"
 	"github.com/go-co-op/gocron"
 )
 
-func StartScheduler(dbConn *sql.DB) {
+func contains(hostnames []string, hostname string) bool {
+	for _, h := range hostnames {
+		if h == hostname {
+			return true
+		}
+	}
+	return false
+}
+
+func StartScheduler(dbConn *sql.DB, argv *args.Arguments) {
 	scheduler := gocron.NewScheduler(time.Local)
+
 	scheduler.Every(1).Hour().Do(func() {
-		database.DeleteOldBackups(dbConn, 31)
+		err := database.DeleteOldBackups(dbConn, 31)
+		if err != nil {
+			log.Println("could not delete old backups:", err)
+		}
 	})
+
+	scheduler.Every(1).Day().Do(func() {
+		hostnames, err := database.BackupedHostnamesInTheLastDay(dbConn)
+		if err != nil {
+			log.Println("could not list hostnames that have been backed up in the last day:", err)
+		}
+
+		allHostnames, err := database.ReceivedHostnames(dbConn)
+		if err != nil {
+			log.Println("could not list received hostnames:", err)
+		}
+
+		failedHostnames := []string{}
+		for _, hostname := range allHostnames {
+			if !contains(hostnames, hostname) {
+				failedHostnames = append(failedHostnames, hostname)
+			}
+		}
+
+		if len(failedHostnames) > 0 {
+			msg := "the following hostnames have not been backed up in the last day:\n" + strings.Join(failedHostnames, ", ")
+			log.Println(msg)
+
+			err := ntfy.SendMessage(msg, argv.NtfyEndpoint, argv.NtfyAlertTopics)
+			if err != nil {
+				log.Println("could not send alert:", err)
+			}
+		}
+	})
+
 	scheduler.StartAsync()
 }