gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!!!!WIP!!!!

Browse Source 
adding InfluxDB

- influxdb added to dockerfile
- influxdb s6 service
- influxdb config
- adding defaults to config
- creating a DeviceRepo interface (multiple db backends)
- implemented DeviceRepo interface as ScruitnyRepository
This commit is contained in:
Jason Kulatunga
2021-06-27 10:55:18 -07:00
parent fd4f0429e4
commit 8a46931399
53 changed files with 4343 additions and 10759 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/web/handler/get_device_details.go
+6 -25
View File
@@ -1,44 +1,25 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/metadata"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func GetDeviceDetails(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
device := dbModels.Device{}
if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC").Limit(40)
}).
Preload("SmartResults.AtaAttributes").
Preload("SmartResults.NvmeAttributes").
Preload("SmartResults.ScsiAttributes").
Where("wwn = ?", c.Param("wwn")).
First(&device).Error; err != nil {
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
device, err := deviceRepo.GetDeviceDetails(c, c.Param("wwn"))
if err != nil {
logger.Errorln("An error occurred while retrieving device details", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := device.SquashHistory(); err != nil {
logger.Errorln("An error occurred while squashing device history", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := device.ApplyMetadataRules(); err != nil {
logger.Errorln("An error occurred while applying scrutiny thresholds & rules", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
smartResults, err := deviceRepo.GetSmartAttributeHistory(c, c.Param("wwn"), "", nil)
var deviceMetadata interface{}
if device.IsAta() {
@@ -49,5 +30,5 @@ func GetDeviceDetails(c *gin.Context) {
deviceMetadata = metadata.ScsiMetadata
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": device, "metadata": deviceMetadata})
c.JSON(http.StatusOK, gin.H{"success": true, "data": map[string]interface{}{"device": device, "smart_results": smartResults}, "metadata": deviceMetadata})
}
webapp/backend/pkg/web/handler/get_devices_summary.go
+9 -12
View File
@@ -1,31 +1,28 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func GetDevicesSummary(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
devices := []dbModels.Device{}
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
//We need the last x (for now all) Smart objects for each Device, so that we can graph Temperature
//We also need the last
if err := db.Preload("SmartResults", func(db *gorm.DB) *gorm.DB {
return db.Order("smarts.created_at DESC") //OLD: .Limit(devicesCount)
}).
Find(&devices).Error; err != nil {
logger.Errorln("Could not get device summary from DB", err)
summary, err := deviceRepo.GetSummary(c)
if err != nil {
logger.Errorln("An error occurred while retrieving device summary", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": devices,
"data": map[string]interface{}{
"summary": summary,
//"temperature": tem
},
})
}
webapp/backend/pkg/web/handler/register_devices.go
+7 -13
View File
@@ -1,22 +1,20 @@
package handler
import (
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/sirupsen/logrus"
"net/http"
)
// register devices that are detected by various collectors.
// This function is run everytime a collector is about to start a run. It can be used to update device data.
// This function is run everytime a collector is about to start a run. It can be used to update device metadata.
func RegisterDevices(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
var collectorDeviceWrapper dbModels.DeviceWrapper
var collectorDeviceWrapper models.DeviceWrapper
err := c.BindJSON(&collectorDeviceWrapper)
if err != nil {
logger.Errorln("Cannot parse detected devices", err)
@@ -28,11 +26,7 @@ func RegisterDevices(c *gin.Context) {
for _, dev := range collectorDeviceWrapper.Data {
//insert devices into DB (and update specified columns if device is already registered)
// update device fields that may change: (DeviceType, HostID)
if err := db.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "wwn"}},
DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type"}),
}).Create(&dev).Error; err != nil {
if err := deviceRepo.RegisterDevice(c, dev); err != nil {
errs = append(errs, err)
}
}
@@ -44,7 +38,7 @@ func RegisterDevices(c *gin.Context) {
})
return
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
c.JSON(http.StatusOK, models.DeviceWrapper{
Success: true,
Data: collectorDeviceWrapper.Data,
})
webapp/backend/pkg/web/handler/send_test_notification.go
+4 -3
View File
@@ -1,8 +1,9 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
@@ -20,7 +21,7 @@ func SendTestNotification(c *gin.Context) {
Payload: notify.Payload{
FailureType: "EmailTest",
DeviceSerial: "FAKEWDDJ324KSO",
DeviceType: dbModels.DeviceProtocolAta,
DeviceType: pkg.DeviceProtocolAta,
DeviceName: "/dev/sda",
Test: true,
},
@@ -33,7 +34,7 @@ func SendTestNotification(c *gin.Context) {
"errors": []string{err.Error()},
})
} else {
c.JSON(http.StatusOK, dbModels.DeviceWrapper{
c.JSON(http.StatusOK, models.DeviceWrapper{
Success: true,
})
}
webapp/backend/pkg/web/handler/upload_device_metrics.go
+22 -18
View File
@@ -1,20 +1,24 @@
package handler
import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/database"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
dbModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/db"
"github.com/analogj/scrutiny/webapp/backend/pkg/notify"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
"net/http"
)
func UploadDeviceMetrics(c *gin.Context) {
db := c.MustGet("DB").(*gorm.DB)
//db := c.MustGet("DB").(*gorm.DB)
logger := c.MustGet("LOGGER").(logrus.FieldLogger)
appConfig := c.MustGet("CONFIG").(config.Interface)
//influxWriteDb := c.MustGet("INFLUXDB_WRITE").(*api.WriteAPIBlocking)
deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo)
//appConfig := c.MustGet("CONFIG").(config.Interface)
var collectorSmartData collector.SmartInfo
err := c.BindJSON(&collectorSmartData)
@@ -25,39 +29,39 @@ func UploadDeviceMetrics(c *gin.Context) {
}
//update the device information if necessary
var device dbModels.Device
db.Where("wwn = ?", c.Param("wwn")).First(&device)
device.UpdateFromCollectorSmartInfo(collectorSmartData)
if err := db.Model(&device).Updates(device).Error; err != nil {
updatedDevice, err := deviceRepo.UpdateDevice(c, c.Param("wwn"), collectorSmartData)
if err != nil {
logger.Errorln("An error occurred while updating device data from smartctl metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
// insert smart info
deviceSmartData := dbModels.Smart{}
err = deviceSmartData.FromCollectorSmartInfo(c.Param("wwn"), collectorSmartData)
_, err = deviceRepo.SaveSmartAttributes(c, c.Param("wwn"), collectorSmartData)
if err != nil {
logger.Errorln("Could not process SMART metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
if err := db.Create(&deviceSmartData).Error; err != nil {
logger.Errorln("An error occurred while saving smartctl metrics", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
// save smart temperature data (ignore failures)
err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData)
if err != nil {
logger.Errorln("An error occurred while saving smartctl temp data", err)
c.JSON(http.StatusInternalServerError, gin.H{"success": false})
return
}
//check for error
if deviceSmartData.SmartStatus == dbModels.SmartStatusFailed {
if updatedDevice.DeviceStatus != pkg.DeviceStatusPassed {
//send notifications
testNotify := notify.Notify{
Config: appConfig,
Payload: notify.Payload{
FailureType: notify.NotifyFailureTypeSmartFailure,
DeviceName: device.DeviceName,
DeviceType: device.DeviceProtocol,
DeviceSerial: device.SerialNumber,
DeviceName: updatedDevice.DeviceName,
DeviceType: updatedDevice.DeviceProtocol,
DeviceSerial: updatedDevice.SerialNumber,
Test: false,
},
Logger: logger,