gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!!!!WIP!!!!

Browse Source 
adding InfluxDB

- influxdb added to dockerfile
- influxdb s6 service
- influxdb config
- adding defaults to config
- creating a DeviceRepo interface (multiple db backends)
- implemented DeviceRepo interface as ScruitnyRepository
This commit is contained in:
Jason Kulatunga
2021-06-27 10:55:18 -07:00
parent fd4f0429e4
commit 8a46931399
53 changed files with 4343 additions and 10759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/web/handler/get_device_details.go
+6 -25
View File
@@ -1,44 +1,25 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func GetDeviceDetails(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
device := dbModels.Device{}
if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC").Limit(40)
}).
Preload("SmartResults.AtaAttributes").
Preload("SmartResults.NvmeAttributes").
Preload("SmartResults.ScsiAttributes").
Where("wwn = ?", c.Param("wwn")).
First(&device).Error; err != nil {
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn"))
if err != nil {
logger.Errorln("An error occurred while retrieving device details", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := device.SquashHistory(); err != nil {
logger.Errorln("An error occurred while squashing device history", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := device.ApplyMetadataRules(); err != nil {
logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), "", nil)
var deviceMetadata interface{}
if device.IsAta() {
@@ -49,5 +30,5 @@ func GetDeviceDetails(c *gin.Context) {
deviceMetadata = metadata.ScsiMetadata
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "metadata": deviceMetadata})
c.JSON(http.StatusOK, gin.H{"success": true, "data": map[string]interface{}{"device": device, "smart_results": smartResults}, "metadata": deviceMetadata})
}
webapp/backend/pkg/web/handler/get_devices_summary.go
+9 -12
View File
@@ -1,31 +1,28 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func GetDevicesSummary(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
devices := []dbModels.Device{}
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
//We also need the last
if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
}).
Find(&devices).Error; err != nil {
logger.Errorln("Could not get device summary from DB", err)
summary, err := deviceRepo.GetSummary(c)
if err != nil {
logger.Errorln("An error occurred while retrieving device summary", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": devices,
"data": map[string]interface{}{
"summary": summary,
//"temperature": tem
},
})
}
webapp/backend/pkg/web/handler/register_devices.go
+7 -13
View File
@@ -1,22 +1,20 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/sirupsen/logrus"
"net/http"
)
// register devices that are detected by various collectors.
// This function is run everytime a collector is about to start a run. It can be used to update device data.
// This function is run everytime a collector is about to start a run. It can be used to update device metadata.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
var collectorDeviceWrapper dbModels.DeviceWrapper
var collectorDeviceWrapper models.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
logger.Errorln("Cannot parse detected devices", err)
@@ -28,11 +26,7 @@ func RegisterDevices(c *gin.Context) {
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB (and update specified columns if device is already registered)
// update device fields that may change: (DeviceType, HostID)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "wwn"}},
DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
}).Create(&dev).Error; err != nil {
if err := deviceRepo.RegisterDevice(c, dev); err != nil {
errs = append(errs, err)
}
}
@@ -44,7 +38,7 @@ func RegisterDevices(c *gin.Context) {
})
return
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
c.JSON(http.StatusOK, models.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
webapp/backend/pkg/web/handler/send_test_notification.go
+4 -3
View File
@@ -1,8 +1,9 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
@@ -20,7 +21,7 @@ func SendTestNotification(c *gin.Context) {
Payload: notify.Payload{
FailureType: "EmailTest",
DeviceSerial: "FAKEWDDJ324KSO",
DeviceType: dbModels.DeviceProtocolAta,
DeviceType: pkg.DeviceProtocolAta,
DeviceName: "/dev/sda",
Test: true,
},
@@ -33,7 +34,7 @@ func SendTestNotification(c *gin.Context) {
"errors": []string{err.Error()},
})
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
c.JSON(http.StatusOK, models.DeviceWrapper{
Success: true,
})
}
webapp/backend/pkg/web/handler/upload_device_metrics.go
+22 -18
View File
@@ -1,20 +1,24 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func UploadDeviceMetrics(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
//db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
appConfig := c.MustGet("CONFIG").(config.Interface)
//influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
//appConfig := c.MustGet("CONFIG").(config.Interface)
var collectorSmartData collector.SmartInfo
err := c.BindJSON(&collectorSmartData)
@@ -25,39 +29,39 @@ func UploadDeviceMetrics(c *gin.Context) {
}
//update the device information if necessary
var device dbModels.Device
db.Where("wwn = ?", c.Param("wwn")).First(&device)
device.UpdateFromCollectorSmartInfo(collectorSmartData)
if err := db.Model(&device).Updates(device).Error; err != nil {
updatedDevice, err := deviceRepo.UpdateDevice(c, c.Param("wwn"), collectorSmartData)
if err != nil {
logger.Errorln("An error occurred while updating device data from smartctl metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
// insert smart info
deviceSmartData := dbModels.Smart{}
err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
_, err = deviceRepo.SaveSmartAttributes(c, c.Param("wwn"), collectorSmartData)
if err != nil {
logger.Errorln("Could not process SMART metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := db.Create(&deviceSmartData).Error; err != nil {
logger.Errorln("An error occurred while saving smartctl metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
// save smart temperature data (ignore failures)
err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData)
if err != nil {
logger.Errorln("An error occurred while saving smartctl temp data", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
//check for error
if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed {
//send notifications
testNotify := notify.Notify{
Config: appConfig,
Payload: notify.Payload{
FailureType: notify.NotifyFailureTypeSmartFailure,
DeviceName: device.DeviceName,
DeviceType: device.DeviceProtocol,
DeviceSerial: device.SerialNumber,
DeviceName: updatedDevice.DeviceName,
DeviceType: updatedDevice.DeviceProtocol,
DeviceSerial: updatedDevice.SerialNumber,
Test: false,
},
Logger: logger,
webapp/backend/pkg/web/middleware/repository.go
+22
View File
@@ -0,0 +1,22 @@
package middleware
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
)
func RepositoryMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
deviceRepo, err := database.NewScrutinyRepository(appConfig, globalLogger)
if err != nil {
panic(err)
}
//TODO: determine where we can call defer deviceRepo.Close()
return func(c *gin.Context) {
c.Set("DEVICE_REPOSITORY", deviceRepo)
c.Next()
}
}
webapp/backend/pkg/web/middleware/sqlite3.go
-59
View File
@@ -1,59 +0,0 @@
package middleware
import (
"fmt"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func DatabaseMiddleware(appConfig config.Interface, globalLogger logrus.FieldLogger) gin.HandlerFunc {
//var database *gorm.DB
fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
//TODO: figure out how to log database queries again.
//Logger: logger
})
if err != nil {
panic("Failed to connect to database!")
}
//database.SetLogger()
database.AutoMigrate(&db.Device{})
database.AutoMigrate(&db.SelfTest{})
database.AutoMigrate(&db.Smart{})
database.AutoMigrate(&db.SmartAtaAttribute{})
database.AutoMigrate(&db.SmartNvmeAttribute{})
database.AutoMigrate(&db.SmartScsiAttribute{})
//TODO: detrmine where we can call defer database.Close()
return func(c *gin.Context) {
c.Set("DB", database)
c.Next()
}
}
// GormLogger is a custom logger for Gorm, making it use logrus.
type GormLogger struct{ Logger logrus.FieldLogger }
// Print handles log events from Gorm for the custom logger.
func (gl *GormLogger) Print(v ...interface{}) {
switch v[0] {
case "sql":
gl.Logger.WithFields(
logrus.Fields{
"module": "gorm",
"type": "sql",
"rows": v[5],
"src_ref": v[1],
"values": v[4],
},
).Debug(v[3])
case "log":
gl.Logger.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2])
}
}
webapp/backend/pkg/web/server.go
+1 -1
View File
@@ -23,7 +23,7 @@ func (ae *AppEngine) Setup(logger logrus.FieldLogger) *gin.Engine {
r := gin.New()
r.Use(middleware.LoggerMiddleware(logger))
r.Use(middleware.DatabaseMiddleware(ae.Config, logger))
r.Use(middleware.RepositoryMiddleware(ae.Config, logger))
r.Use(middleware.ConfigMiddleware(ae.Config))
r.Use(gin.Recovery())
webapp/backend/pkg/web/server_test.go
+2 -2
View File
@@ -3,7 +3,7 @@ package web_test
import (
"encoding/json"
mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/web"
"github.com/golang/mock/gomock"
"github.com/sirupsen/logrus"
@@ -319,7 +319,7 @@ func TestGetDevicesSummaryRoute_Nvme(t *testing.T) {
req, _ = http.NewRequest("GET", "/api/summary", nil)
router.ServeHTTP(sr, req)
require.Equal(t, 200, sr.Code)
var device dbModels.DeviceWrapper
var device models.DeviceWrapper
json.Unmarshal(sr.Body.Bytes(), &device)
//assert