gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Identify drives by a Scrutiny UUID instead of wwn (#960)

Browse Source 
* Generate a UUIDv5 from a random namespace  based on WWN, model name, and serial number
* Migrate sqlite and influxdb data accordingly
* Update frontend API routes and components
* Fixes #923
This commit is contained in:
Aram Akhavan
2026-03-25 20:16:17 -07:00
committed by GitHub
parent e4c40f7e80
commit c3b2eb2b4f
69 changed files with 815 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/backend/pkg/database/interface.go
+11 -10
View File
@@ -7,6 +7,7 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
"github.com/gofrs/uuid/v5"
)
// Create mock using:
@@ -17,19 +18,19 @@ type DeviceRepo interface {
RegisterDevice(ctx context.Context, dev models.Device) error
GetDevices(ctx context.Context) ([]models.Device, error)
UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error)
UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error)
GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error)
UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error
DeleteDevice(ctx context.Context, wwn string) error
UpdateDevice(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (models.Device, error)
UpdateDeviceStatus(ctx context.Context, scrutiny_uuid uuid.UUID, status pkg.DeviceStatus) (models.Device, error)
GetDeviceDetails(ctx context.Context, scrutiny_uuid uuid.UUID) (models.Device, error)
UpdateDeviceArchived(ctx context.Context, scrutiny_uuid uuid.UUID, archived bool) error
DeleteDevice(ctx context.Context, scrutiny_uuid uuid.UUID) error
SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error)
SaveSmartAttributes(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (measurements.Smart, error)
GetSmartAttributeHistory(ctx context.Context, scrutiny_uuid uuid.UUID, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error)
SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error
SaveSmartTemperature(ctx context.Context, scrutiny_uuid uuid.UUID, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error
GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error)
GetSummary(ctx context.Context) (map[uuid.UUID]*models.DeviceSummary, error)
GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[uuid.UUID][]measurements.SmartTemperature, error)
LoadSettings(ctx context.Context) (*models.Settings, error)
SaveSettings(ctx context.Context, settings models.Settings) error
webapp/backend/pkg/database/migrations/m20250221084400/device.go
+3 -1
View File
@@ -1,10 +1,12 @@
package m20250221084400
import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"time"
"github.com/analogj/scrutiny/webapp/backend/pkg"
)
// Deprecated: m20250221084400.Device is deprecated, only used by db migrations
type Device struct {
Archived bool `json:"archived"`
//GORM attributes, see: http://gorm.io/docs/conventions.html
webapp/backend/pkg/database/migrations/m20260216155600/device.go
+44
View File
@@ -0,0 +1,44 @@
package m20260216155600
import (
"time"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/gofrs/uuid/v5"
)
type Device struct {
//GORM attributes, see: http://gorm.io/docs/conventions.html
Archived bool `json:"archived"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt *time.Time
WWN string `json:"wwn"`
DeviceName string `json:"device_name"`
DeviceUUID string `json:"device_uuid"`
DeviceSerialID string `json:"device_serial_id"`
DeviceLabel string `json:"device_label"`
Manufacturer string `json:"manufacturer"`
ModelName string `json:"model_name"`
InterfaceType string `json:"interface_type"`
InterfaceSpeed string `json:"interface_speed"`
SerialNumber string `json:"serial_number"`
Firmware string `json:"firmware"`
RotationSpeed int `json:"rotational_speed"`
Capacity int64 `json:"capacity"`
FormFactor string `json:"form_factor"`
SmartSupport bool `json:"smart_support"`
DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector.
// User provided metadata
Label string `json:"label"`
HostId string `json:"host_id"`
// Data set by Scrutiny
DeviceStatus pkg.DeviceStatus `json:"device_status"`
ScrutinyUUID uuid.UUID `json:"scrutiny_uuid" gorm:"primaryKey;uniqueIndex"`
}
webapp/backend/pkg/database/mock/mock_database.go
+60 -53
View File
@@ -1,5 +1,10 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: webapp/backend/pkg/database/interface.go
//
// Generated by this command:
//
// mockgen -source=webapp/backend/pkg/database/interface.go -destination=webapp/backend/pkg/database/mock/mock_database.go
//
// Package mock_database is a generated GoMock package.
package mock_database
@@ -12,6 +17,7 @@ import (
models "github.com/analogj/scrutiny/webapp/backend/pkg/models"
collector "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
measurements "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
uuid "github.com/gofrs/uuid/v5"
gomock "go.uber.org/mock/gomock"
)
@@ -19,6 +25,7 @@ import (
type MockDeviceRepo struct {
ctrl *gomock.Controller
recorder *MockDeviceRepoMockRecorder
isgomock struct{}
}
// MockDeviceRepoMockRecorder is the mock recorder for MockDeviceRepo.
@@ -52,47 +59,33 @@ func (mr *MockDeviceRepoMockRecorder) Close() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDeviceRepo)(nil).Close))
}
// UpdateDeviceArchived mocks base method.
func (m *MockDeviceRepo) UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateDeviceArchived", ctx, wwn)
ret0, _ := ret[0].(error)
return ret0
}
// UpdateDeviceArchived indicates an expected call of UpdateDeviceArchived.
func (mr *MockDeviceRepoMockRecorder) UpdateDeviceArchived(ctx, wwn, archived interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceArchived", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceArchived), ctx, wwn, archived)
}
// DeleteDevice mocks base method.
func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, wwn string) error {
func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, scrutiny_uuid uuid.UUID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "DeleteDevice", ctx, wwn)
ret := m.ctrl.Call(m, "DeleteDevice", ctx, scrutiny_uuid)
ret0, _ := ret[0].(error)
return ret0
}
// DeleteDevice indicates an expected call of DeleteDevice.
func (mr *MockDeviceRepoMockRecorder) DeleteDevice(ctx, wwn interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) DeleteDevice(ctx, scrutiny_uuid any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDevice", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteDevice), ctx, wwn)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDevice", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteDevice), ctx, scrutiny_uuid)
}
// GetDeviceDetails mocks base method.
func (m *MockDeviceRepo) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
func (m *MockDeviceRepo) GetDeviceDetails(ctx context.Context, scrutiny_uuid uuid.UUID) (models.Device, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetDeviceDetails", ctx, wwn)
ret := m.ctrl.Call(m, "GetDeviceDetails", ctx, scrutiny_uuid)
ret0, _ := ret[0].(models.Device)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetDeviceDetails indicates an expected call of GetDeviceDetails.
func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, wwn interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, scrutiny_uuid any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, wwn)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, scrutiny_uuid)
}
// GetDevices mocks base method.
@@ -105,52 +98,52 @@ func (m *MockDeviceRepo) GetDevices(ctx context.Context) ([]models.Device, error
}
// GetDevices indicates an expected call of GetDevices.
func (mr *MockDeviceRepoMockRecorder) GetDevices(ctx interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) GetDevices(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevices", reflect.TypeOf((*MockDeviceRepo)(nil).GetDevices), ctx)
}
// GetSmartAttributeHistory mocks base method.
func (m *MockDeviceRepo) GetSmartAttributeHistory(ctx context.Context, wwn, durationKey string, selectEntries, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
func (m *MockDeviceRepo) GetSmartAttributeHistory(ctx context.Context, scrutiny_uuid uuid.UUID, durationKey string, selectEntries, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetSmartAttributeHistory", ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
ret := m.ctrl.Call(m, "GetSmartAttributeHistory", ctx, scrutiny_uuid, durationKey, selectEntries, selectEntriesOffset, attributes)
ret0, _ := ret[0].([]measurements.Smart)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetSmartAttributeHistory indicates an expected call of GetSmartAttributeHistory.
func (mr *MockDeviceRepoMockRecorder) GetSmartAttributeHistory(ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) GetSmartAttributeHistory(ctx, scrutiny_uuid, durationKey, selectEntries, selectEntriesOffset, attributes any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartAttributeHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartAttributeHistory), ctx, wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartAttributeHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartAttributeHistory), ctx, scrutiny_uuid, durationKey, selectEntries, selectEntriesOffset, attributes)
}
// GetSmartTemperatureHistory mocks base method.
func (m *MockDeviceRepo) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
func (m *MockDeviceRepo) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[uuid.UUID][]measurements.SmartTemperature, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetSmartTemperatureHistory", ctx, durationKey)
ret0, _ := ret[0].(map[string][]measurements.SmartTemperature)
ret0, _ := ret[0].(map[uuid.UUID][]measurements.SmartTemperature)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetSmartTemperatureHistory indicates an expected call of GetSmartTemperatureHistory.
func (mr *MockDeviceRepoMockRecorder) GetSmartTemperatureHistory(ctx, durationKey interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) GetSmartTemperatureHistory(ctx, durationKey any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSmartTemperatureHistory", reflect.TypeOf((*MockDeviceRepo)(nil).GetSmartTemperatureHistory), ctx, durationKey)
}
// GetSummary mocks base method.
func (m *MockDeviceRepo) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
func (m *MockDeviceRepo) GetSummary(ctx context.Context) (map[uuid.UUID]*models.DeviceSummary, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetSummary", ctx)
ret0, _ := ret[0].(map[string]*models.DeviceSummary)
ret0, _ := ret[0].(map[uuid.UUID]*models.DeviceSummary)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetSummary indicates an expected call of GetSummary.
func (mr *MockDeviceRepoMockRecorder) GetSummary(ctx interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) GetSummary(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSummary", reflect.TypeOf((*MockDeviceRepo)(nil).GetSummary), ctx)
}
@@ -164,7 +157,7 @@ func (m *MockDeviceRepo) HealthCheck(ctx context.Context) error {
}
// HealthCheck indicates an expected call of HealthCheck.
func (mr *MockDeviceRepoMockRecorder) HealthCheck(ctx interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) HealthCheck(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HealthCheck", reflect.TypeOf((*MockDeviceRepo)(nil).HealthCheck), ctx)
}
@@ -179,7 +172,7 @@ func (m *MockDeviceRepo) LoadSettings(ctx context.Context) (*models.Settings, er
}
// LoadSettings indicates an expected call of LoadSettings.
func (mr *MockDeviceRepoMockRecorder) LoadSettings(ctx interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) LoadSettings(ctx any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadSettings", reflect.TypeOf((*MockDeviceRepo)(nil).LoadSettings), ctx)
}
@@ -193,7 +186,7 @@ func (m *MockDeviceRepo) RegisterDevice(ctx context.Context, dev models.Device)
}
// RegisterDevice indicates an expected call of RegisterDevice.
func (mr *MockDeviceRepoMockRecorder) RegisterDevice(ctx, dev interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) RegisterDevice(ctx, dev any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterDevice", reflect.TypeOf((*MockDeviceRepo)(nil).RegisterDevice), ctx, dev)
}
@@ -207,66 +200,80 @@ func (m *MockDeviceRepo) SaveSettings(ctx context.Context, settings models.Setti
}
// SaveSettings indicates an expected call of SaveSettings.
func (mr *MockDeviceRepoMockRecorder) SaveSettings(ctx, settings interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) SaveSettings(ctx, settings any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSettings", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSettings), ctx, settings)
}
// SaveSmartAttributes mocks base method.
func (m *MockDeviceRepo) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
func (m *MockDeviceRepo) SaveSmartAttributes(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveSmartAttributes", ctx, wwn, collectorSmartData)
ret := m.ctrl.Call(m, "SaveSmartAttributes", ctx, scrutiny_uuid, collectorSmartData)
ret0, _ := ret[0].(measurements.Smart)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// SaveSmartAttributes indicates an expected call of SaveSmartAttributes.
func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, scrutiny_uuid, collectorSmartData any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartAttributes", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartAttributes), ctx, wwn, collectorSmartData)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartAttributes", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartAttributes), ctx, scrutiny_uuid, collectorSmartData)
}
// SaveSmartTemperature mocks base method.
func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, scrutiny_uuid uuid.UUID, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory)
ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, scrutiny_uuid, deviceProtocol, collectorSmartData, discardSCTTempHistory)
ret0, _ := ret[0].(error)
return ret0
}
// SaveSmartTemperature indicates an expected call of SaveSmartTemperature.
func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, scrutiny_uuid, deviceProtocol, collectorSmartData, discardSCTTempHistory any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData, discardSCTTempHistory)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, scrutiny_uuid, deviceProtocol, collectorSmartData, discardSCTTempHistory)
}
// UpdateDevice mocks base method.
func (m *MockDeviceRepo) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
func (m *MockDeviceRepo) UpdateDevice(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (models.Device, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateDevice", ctx, wwn, collectorSmartData)
ret := m.ctrl.Call(m, "UpdateDevice", ctx, scrutiny_uuid, collectorSmartData)
ret0, _ := ret[0].(models.Device)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// UpdateDevice indicates an expected call of UpdateDevice.
func (mr *MockDeviceRepoMockRecorder) UpdateDevice(ctx, wwn, collectorSmartData interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) UpdateDevice(ctx, scrutiny_uuid, collectorSmartData any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDevice", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDevice), ctx, wwn, collectorSmartData)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDevice", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDevice), ctx, scrutiny_uuid, collectorSmartData)
}
// UpdateDeviceArchived mocks base method.
func (m *MockDeviceRepo) UpdateDeviceArchived(ctx context.Context, scrutiny_uuid uuid.UUID, archived bool) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateDeviceArchived", ctx, scrutiny_uuid, archived)
ret0, _ := ret[0].(error)
return ret0
}
// UpdateDeviceArchived indicates an expected call of UpdateDeviceArchived.
func (mr *MockDeviceRepoMockRecorder) UpdateDeviceArchived(ctx, scrutiny_uuid, archived any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceArchived", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceArchived), ctx, scrutiny_uuid, archived)
}
// UpdateDeviceStatus mocks base method.
func (m *MockDeviceRepo) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
func (m *MockDeviceRepo) UpdateDeviceStatus(ctx context.Context, scrutiny_uuid uuid.UUID, status pkg.DeviceStatus) (models.Device, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "UpdateDeviceStatus", ctx, wwn, status)
ret := m.ctrl.Call(m, "UpdateDeviceStatus", ctx, scrutiny_uuid, status)
ret0, _ := ret[0].(models.Device)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// UpdateDeviceStatus indicates an expected call of UpdateDeviceStatus.
func (mr *MockDeviceRepoMockRecorder) UpdateDeviceStatus(ctx, wwn, status interface{}) *gomock.Call {
func (mr *MockDeviceRepoMockRecorder) UpdateDeviceStatus(ctx, scrutiny_uuid, status any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceStatus", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceStatus), ctx, wwn, status)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceStatus", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceStatus), ctx, scrutiny_uuid, status)
}
webapp/backend/pkg/database/scrutiny_repository.go
+19 -16
View File
@@ -13,10 +13,12 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/config"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/glebarez/sqlite"
"github.com/gofrs/uuid/v5"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
"github.com/influxdata/influxdb-client-go/v2/api"
"github.com/influxdata/influxdb-client-go/v2/domain"
"github.com/sirupsen/logrus"
"gorm.io/gorm"
)
@@ -333,16 +335,16 @@ func (sr *scrutinyRepository) EnsureBuckets(ctx context.Context, org *domain.Org
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// get a map of all devices and associated SMART data
func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) {
func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[uuid.UUID]*models.DeviceSummary, error) {
devices, err := sr.GetDevices(ctx)
if err != nil {
return nil, err
}
summaries := map[string]*models.DeviceSummary{}
summaries := map[uuid.UUID]*models.DeviceSummary{}
for _, device := range devices {
summaries[device.WWN] = &models.DeviceSummary{Device: device}
summaries[device.ScrutinyUUID] = &models.DeviceSummary{Device: device}
}
// Get parser flux query result
@@ -357,7 +359,7 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
|> last()
|> schema.fieldsAsCols()
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
weeklyData = from(bucket: bucketBaseName + "_weekly")
|> range(start: -10y, stop: now())
@@ -365,7 +367,7 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
|> last()
|> schema.fieldsAsCols()
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
monthlyData = from(bucket: bucketBaseName + "_monthly")
|> range(start: -10y, stop: now())
@@ -373,7 +375,7 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
|> last()
|> schema.fieldsAsCols()
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
yearlyData = from(bucket: bucketBaseName + "_yearly")
|> range(start: -10y, stop: now())
@@ -381,12 +383,12 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
|> last()
|> schema.fieldsAsCols()
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
union(tables: [dailyData, weeklyData, monthlyData, yearlyData])
|> sort(columns: ["_time"], desc: false)
|> group(columns: ["device_wwn"])
|> last(column: "device_wwn")
|> group(columns: ["scrutiny_uuid"])
|> last(column: "scrutiny_uuid")
|> yield(name: "last")
`,
sr.appConfig.GetString("web.influxdb.bucket"),
@@ -404,14 +406,15 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
//get summary data from Influxdb.
//result.Record().Values()
if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
if scrutinyUUIDString, ok := result.Record().Values()["scrutiny_uuid"]; ok {
scrutinyUUID := uuid.Must(uuid.FromString(scrutinyUUIDString.(string)))
//ensure summaries is intialized for this wwn
if _, exists := summaries[deviceWWN.(string)]; !exists {
summaries[deviceWWN.(string)] = &models.DeviceSummary{}
//ensure summaries is intialized for this scrutiny_uuid
if _, exists := summaries[scrutinyUUID]; !exists {
summaries[scrutinyUUID] = &models.DeviceSummary{}
}
summaries[deviceWWN.(string)].SmartResults = &models.SmartSummary{
summaries[scrutinyUUID].SmartResults = &models.SmartSummary{
Temp: result.Record().Values()["temp"].(int64),
PowerOnHours: result.Record().Values()["power_on_hours"].(int64),
CollectorDate: result.Record().Values()["_time"].(time.Time),
@@ -434,8 +437,8 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
sr.logger.Printf("========================>>>>>>>>======================")
sr.logger.Printf("Error: %v", err)
}
for wwn, tempHistory := range deviceTempHistory {
summaries[wwn].TempHistory = tempHistory
for scutiny_uuid, tempHistory := range deviceTempHistory {
summaries[scutiny_uuid].TempHistory = tempHistory
}
return summaries, nil
webapp/backend/pkg/database/scrutiny_repository_device.go
+15 -14
View File
@@ -8,6 +8,7 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/gofrs/uuid/v5"
"gorm.io/gorm/clause"
)
@@ -19,7 +20,7 @@ import (
// update device fields that may change: (DeviceType, HostID)
func (sr *scrutinyRepository) RegisterDevice(ctx context.Context, dev models.Device) error {
if err := sr.gormClient.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "wwn"}},
Columns: []clause.Column{{Name: "scrutiny_uuid"}},
DoUpdates: clause.AssignmentColumns([]string{"host_id", "device_name", "device_type", "device_uuid", "device_serial_id", "device_label"}),
}).Create(&dev).Error; err != nil {
return err
@@ -38,9 +39,9 @@ func (sr *scrutinyRepository) GetDevices(ctx context.Context) ([]models.Device,
}
// update device (only metadata) from collector
func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (models.Device, error) {
func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (models.Device, error) {
var device models.Device
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
if err := sr.gormClient.WithContext(ctx).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).First(&device).Error; err != nil {
return device, fmt.Errorf("could not get device from DB: %v", err)
}
@@ -53,9 +54,9 @@ func (sr *scrutinyRepository) UpdateDevice(ctx context.Context, wwn string, coll
}
// Update Device Status
func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) {
func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, scrutiny_uuid uuid.UUID, status pkg.DeviceStatus) (models.Device, error) {
var device models.Device
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
if err := sr.gormClient.WithContext(ctx).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).First(&device).Error; err != nil {
return device, fmt.Errorf("could not get device from DB: %v", err)
}
@@ -63,12 +64,12 @@ func (sr *scrutinyRepository) UpdateDeviceStatus(ctx context.Context, wwn string
return device, sr.gormClient.Model(&device).Updates(device).Error
}
func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) {
func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, scrutiny_uuid uuid.UUID) (models.Device, error) {
var device models.Device
fmt.Println("GetDeviceDetails from GORM")
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
if err := sr.gormClient.WithContext(ctx).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).First(&device).Error; err != nil {
return models.Device{}, err
}
@@ -76,17 +77,17 @@ func (sr *scrutinyRepository) GetDeviceDetails(ctx context.Context, wwn string)
}
// Update Device Archived State
func (sr *scrutinyRepository) UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error {
func (sr *scrutinyRepository) UpdateDeviceArchived(ctx context.Context, scrutiny_uuid uuid.UUID, archived bool) error {
var device models.Device
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil {
if err := sr.gormClient.WithContext(ctx).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).First(&device).Error; err != nil {
return fmt.Errorf("could not get device from DB: %v", err)
}
return sr.gormClient.Model(&device).Where("wwn = ?", wwn).Update("archived", archived).Error
return sr.gormClient.Model(&device).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).Update("archived", archived).Error
}
func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) error {
if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).Delete(&models.Device{}).Error; err != nil {
func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, scrutiny_uuid uuid.UUID) error {
if err := sr.gormClient.WithContext(ctx).Where("scrutiny_uuid = ?", scrutiny_uuid.String()).Delete(&models.Device{}).Error; err != nil {
return err
}
@@ -99,14 +100,14 @@ func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) erro
}
for _, bucket := range buckets {
sr.logger.Infof("Deleting data for %s in bucket: %s", wwn, bucket)
sr.logger.Infof("Deleting data for %s in bucket: %s", scrutiny_uuid.String(), bucket)
if err := sr.influxClient.DeleteAPI().DeleteWithName(
ctx,
sr.appConfig.GetString("web.influxdb.org"),
bucket,
time.Now().AddDate(-10, 0, 0),
time.Now(),
fmt.Sprintf(`device_wwn="%s"`, wwn),
fmt.Sprintf(`scrutiny_uuid="%s"`, scrutiny_uuid.String()),
); err != nil {
return err
}
webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go
+17 -16
View File
@@ -8,17 +8,18 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
"github.com/gofrs/uuid/v5"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
"github.com/influxdata/influxdb-client-go/v2/api"
log "github.com/sirupsen/logrus"
)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// SMART
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, scrutiny_uuid uuid.UUID, collectorSmartData collector.SmartInfo) (measurements.Smart, error) {
deviceSmartData := measurements.Smart{}
err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData)
err := deviceSmartData.FromCollectorSmartInfo(scrutiny_uuid, collectorSmartData)
if err != nil {
sr.logger.Errorln("Could not process SMART metrics", err)
return measurements.Smart{}, err
@@ -34,14 +35,14 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
// When selectEntries is > 0, only the most recent selectEntries database entries are returned, starting from the selectEntriesOffset entry.
// For example, with selectEntries = 5, selectEntries = 0, the most recent 5 are returned. With selectEntries = 3, selectEntries = 2, entries
// 2 to 4 are returned (2 being the third newest, since it is zero-indexed)
func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, scrutiny_uuid uuid.UUID, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) {
// Get SMartResults from InfluxDB
//TODO: change the filter startrange to a real number.
// Get parser flux query result
//appConfig.GetString("web.influxdb.bucket")
queryStr := sr.aggregateSmartAttributesQuery(wwn, durationKey, selectEntries, selectEntriesOffset, attributes)
queryStr := sr.aggregateSmartAttributesQuery(scrutiny_uuid, durationKey, selectEntries, selectEntriesOffset, attributes)
log.Infoln(queryStr)
smartResults := []measurements.Smart{}
@@ -100,7 +101,7 @@ func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking,
return influxWriteApi.WritePoint(ctx, p)
}
func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
func (sr *scrutinyRepository) aggregateSmartAttributesQuery(scrutiny_uuid uuid.UUID, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
/*
@@ -108,28 +109,28 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
weekData = from(bucket: "metrics")
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
|> filter(fn: (r) => r["scrutiny_uuid"] == "32bda933-15be-56a3-902f-9f3674b03d59" )
|> tail(n: 10, offset: 0)
|> schema.fieldsAsCols()
monthData = from(bucket: "metrics_weekly")
|> range(start: -1mo, stop: -1w)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
|> filter(fn: (r) => r["scrutiny_uuid"] == "32bda933-15be-56a3-902f-9f3674b03d59" )
|> tail(n: 10, offset: 0)
|> schema.fieldsAsCols()
yearData = from(bucket: "metrics_monthly")
|> range(start: -1y, stop: -1mo)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
|> filter(fn: (r) => r["scrutiny_uuid"] == "32bda933-15be-56a3-902f-9f3674b03d59" )
|> tail(n: 10, offset: 0)
|> schema.fieldsAsCols()
foreverData = from(bucket: "metrics_yearly")
|> range(start: -10y, stop: -1y)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
|> filter(fn: (r) => r["scrutiny_uuid"] == "32bda933-15be-56a3-902f-9f3674b03d59" )
|> tail(n: 10, offset: 0)
|> schema.fieldsAsCols()
@@ -150,7 +151,7 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
if len(nestedDurationKeys) == 1 {
//there's only one bucket being queried, no need to union, just aggregate the dataset and return
partialQueryStr = append(partialQueryStr, []string{
sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes),
sr.generateSmartAttributesSubquery(scrutiny_uuid, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes),
fmt.Sprintf(`%sData`, nestedDurationKeys[0]),
`|> sort(columns: ["_time"], desc: true)`,
`|> yield()`,
@@ -165,9 +166,9 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
if selectEntries > 0 {
// We only need the last `n + offset` # of entries from each table to guarantee we can
// get the last `n` # of entries starting from `offset` of the union
subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, selectEntries+selectEntriesOffset, 0, attributes))
subQueries = append(subQueries, sr.generateSmartAttributesSubquery(scrutiny_uuid, nestedDurationKey, selectEntries+selectEntriesOffset, 0, attributes))
} else {
subQueries = append(subQueries, sr.generateSmartAttributesSubquery(wwn, nestedDurationKey, 0, 0, attributes))
subQueries = append(subQueries, sr.generateSmartAttributesSubquery(scrutiny_uuid, nestedDurationKey, 0, 0, attributes))
}
}
partialQueryStr = append(partialQueryStr, subQueries...)
@@ -184,7 +185,7 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration
return strings.Join(partialQueryStr, "\n")
}
func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
func (sr *scrutinyRepository) generateSmartAttributesSubquery(scrutiny_uuid uuid.UUID, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) string {
bucketName := sr.lookupBucketName(durationKey)
durationRange := sr.lookupDuration(durationKey)
@@ -192,7 +193,7 @@ func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durati
fmt.Sprintf(`%sData = from(bucket: "%s")`, durationKey, bucketName),
fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
`|> filter(fn: (r) => r["_measurement"] == "smart" )`,
fmt.Sprintf(`|> filter(fn: (r) => r["device_wwn"] == "%s" )`, wwn),
fmt.Sprintf(`|> filter(fn: (r) => r["scrutiny_uuid"] == "%s" )`, scrutiny_uuid.String()),
}
partialQueryStr = append(partialQueryStr, `|> aggregateWindow(every: 1d, fn: last, createEmpty: false)`)
webapp/backend/pkg/database/scrutiny_repository_migrations.go
+134
View File
@@ -7,12 +7,14 @@ import (
"strconv"
"time"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/analogj/scrutiny/webapp/backend/pkg"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220716214900"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20250221084400"
"github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20260216155600"
"github.com/analogj/scrutiny/webapp/backend/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
@@ -424,6 +426,53 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error {
return tx.Create(&defaultSettings).Error
},
},
{
ID: "m20260216155600", // add ScrutinyUUID as primary key
Migrate: func(tx *gorm.DB) error {
devices := []m20260216155600.Device{}
if err := tx.Find(&devices).Error; err != nil {
return err
}
sr.logger.Debug("Generating Scrutiny UUIDs")
for i := range devices {
device := &devices[i]
device.ScrutinyUUID = detect.GenerateScrutinyUUID(device.ModelName, device.SerialNumber, device.WWN)
}
// sqlite doesn't support altering columns
// so we have to create a new one, drop the old one, then rename.
sr.logger.Debug("Creating new devices table")
tx.Table("devices_new").AutoMigrate(&m20260216155600.Device{})
if len(devices) > 0 {
if err := tx.Table("devices_new").Create(&devices).Error; err != nil {
return err
}
}
sr.logger.Debug("Dropping old devices table")
if err := tx.Migrator().DropTable(&m20260216155600.Device{}); err != nil {
return err
}
sr.logger.Debug("Renaming new device table")
if err := tx.Migrator().RenameTable("devices_new", "devices"); err != nil {
return err
}
//
wwnToUUID := make(map[string]string)
for _, device := range devices {
wwnToUUID[device.WWN] = device.ScrutinyUUID.String()
}
err := m20260216155600_ChangeInfluxDBTags(sr, ctx, wwnToUUID)
if ignorePastRetentionPolicyError(err) != nil {
return err
}
return nil
},
},
})
if err := m.Migrate(); err != nil {
@@ -473,6 +522,91 @@ func ignorePastRetentionPolicyError(err error) error {
return err
}
func m20260216155600_ChangeInfluxDBTags(sr *scrutinyRepository, ctx context.Context, wwnToUUID map[string]string) error {
bucket := sr.appConfig.GetString("web.influxdb.bucket")
org := sr.appConfig.GetString("web.influxdb.org")
bucketNames := []string{
bucket,
fmt.Sprintf("%s_weekly", bucket),
fmt.Sprintf("%s_monthly", bucket),
fmt.Sprintf("%s_yearly", bucket),
}
const batchSize = 1000
bucketsAPI := sr.influxClient.BucketsAPI()
for _, bucketName := range bucketNames {
newBucketName := fmt.Sprintf("%s_new", bucketName)
// Step 1: Create the new bucket. Copy retention rules from the original.
sr.logger.Debugf("Creating temporary bucket %s...", newBucketName)
oldBucket, err := bucketsAPI.FindBucketByName(ctx, bucketName)
if err != nil {
return fmt.Errorf("Failed to find bucket %s: %w", bucketName, err)
}
// Delete leftover _new bucket from a previous failed migration attempt.
if existingNew, _ := bucketsAPI.FindBucketByName(ctx, newBucketName); existingNew != nil {
sr.logger.Debugf("Found leftover bucket %s from previous migration, deleting...", newBucketName)
if err := bucketsAPI.DeleteBucket(ctx, existingNew); err != nil {
return fmt.Errorf("Failed to delete leftover bucket %s: %w", newBucketName, err)
}
}
orgObj, err := sr.influxClient.OrganizationsAPI().FindOrganizationByName(ctx, org)
if err != nil {
return fmt.Errorf("failed to find organization %s: %w", org, err)
}
newBucket, err := bucketsAPI.CreateBucketWithName(ctx, orgObj, newBucketName, oldBucket.RetentionRules...)
if err != nil {
return fmt.Errorf("failed to create bucket %s: %w", newBucketName, err)
}
for wwn, scrutinyUUID := range wwnToUUID {
sr.logger.Debugf("Copying points from %s to %s for wwn %s...", bucketName, newBucketName, wwn)
offset := 0
for ; ; offset += batchSize {
queryStr := fmt.Sprintf(`
from(bucket: "%s")
|> range(start: -10y, stop: now())
|> filter(fn: (r) => r["_measurement"] == "smart" or r["_measurement"] == "temp")
|> filter(fn: (r) => r["device_wwn"] == "%s")
|> limit(n: %d, offset: %d)
|> drop(columns: ["device_wwn"])
|> set(key: "scrutiny_uuid", value: "%s")
|> to(bucket: "%s")
`, bucketName, wwn, batchSize, offset, scrutinyUUID, newBucketName)
result, err := sr.influxQueryApi.Query(ctx, queryStr)
if err != nil {
return fmt.Errorf("failed to copy points from %s to %s for wwn %s (offset %d): %w", bucketName, newBucketName, wwn, offset, err)
}
if !result.Next() {
break
}
}
sr.logger.Debugf("Copied approx. %d points for wwn %s", offset, wwn)
}
sr.logger.Debugf("Replacing bucket %s with %s...", bucketName, newBucketName)
if err := bucketsAPI.DeleteBucket(ctx, oldBucket); err != nil {
return fmt.Errorf("Failed to delete old bucket %s: %w", bucketName, err)
}
newBucket.Name = bucketName
if _, err := bucketsAPI.UpdateBucket(ctx, newBucket); err != nil {
return fmt.Errorf("Failed to rename bucket %s to %s: %w", newBucketName, bucketName, err)
}
sr.logger.Debugf("Bucket %s migrated successfully", bucketName)
}
return nil
}
// Deprecated
func m20201107210306_FromPreInfluxDBTempCreatePostInfluxDBTemp(preDevice m20201107210306.Device, preSmartResult m20201107210306.Smart) (error, measurements.SmartTemperature) {
//extract temperature data for every datapoint
webapp/backend/pkg/database/scrutiny_repository_tasks.go
+7 -7
View File
@@ -3,12 +3,13 @@ package database
import (
"context"
"fmt"
"github.com/influxdata/influxdb-client-go/v2/api"
)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Tasks
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) EnsureTasks(ctx context.Context, orgID string) error {
weeklyTaskName := "tsk-weekly-aggr"
weeklyTaskScript := sr.DownsampleScript("weekly", weeklyTaskName, "0 1 * * 0")
@@ -108,7 +109,7 @@ func (sr *scrutinyRepository) DownsampleScript(aggregationType string, name stri
smart_data = from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn", "_field"])
|> group(columns: ["scrutiny_uuid", "_field"])
non_numeric_smart_data = smart_data
|> filter(fn: (r) => types.isType(v: r._value, type: "string") or types.isType(v: r._value, type: "bool"))
@@ -139,20 +140,19 @@ destOrg = "%s"
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn", "_field"])
|> group(columns: ["scrutiny_uuid", "_field"])
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> to(bucket: destBucket, org: destOrg)
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`,
|> to(bucket: destBucket, org: destOrg)`,
name,
cron,
sourceBucket,
webapp/backend/pkg/database/scrutiny_repository_tasks_test.go
+9 -12
View File
@@ -43,20 +43,19 @@ destOrg = "scrutiny"
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn", "_field"])
|> group(columns: ["scrutiny_uuid", "_field"])
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> to(bucket: destBucket, org: destOrg)
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript)
|> to(bucket: destBucket, org: destOrg)`, influxDbScript)
}
func Test_DownsampleScript_Monthly(t *testing.T) {
@@ -94,20 +93,19 @@ destOrg = "scrutiny"
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn", "_field"])
|> group(columns: ["scrutiny_uuid", "_field"])
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> to(bucket: destBucket, org: destOrg)
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript)
|> to(bucket: destBucket, org: destOrg)`, influxDbScript)
}
func Test_DownsampleScript_Yearly(t *testing.T) {
@@ -145,18 +143,17 @@ destOrg = "scrutiny"
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "smart" )
|> group(columns: ["device_wwn", "_field"])
|> group(columns: ["scrutiny_uuid", "_field"])
|> aggregateWindow(every: aggWindow, fn: last, createEmpty: false)
|> to(bucket: destBucket, org: destOrg)
from(bucket: sourceBucket)
|> range(start: rangeStart, stop: rangeEnd)
|> filter(fn: (r) => r["_measurement"] == "temp")
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
|> aggregateWindow(fn: mean, every: aggWindow, createEmpty: false)
|> set(key: "_measurement", value: "temp")
|> set(key: "_field", value: "temp")
|> to(bucket: destBucket, org: destOrg)
`, influxDbScript)
|> to(bucket: destBucket, org: destOrg)`, influxDbScript)
}
webapp/backend/pkg/database/scrutiny_repository_temperature.go
+20 -19
View File
@@ -8,13 +8,14 @@ import (
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
"github.com/gofrs/uuid/v5"
influxdb2 "github.com/influxdata/influxdb-client-go/v2"
)
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Temperature Data
// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, scrutiny_uuid uuid.UUID, deviceProtocol string, collectorSmartData collector.SmartInfo, discardSCTTempHistory bool) error {
if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 && !discardSCTTempHistory {
for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table {
@@ -24,15 +25,15 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
}
intervalSec := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * 60
datapointTime := collectorSmartData.LocalTime.TimeT - int64(ndx) * intervalSec
alignedDatapointTime := datapointTime - datapointTime % intervalSec
datapointTime := collectorSmartData.LocalTime.TimeT - int64(ndx)*intervalSec
alignedDatapointTime := datapointTime - datapointTime%intervalSec
smartTemp := measurements.SmartTemperature{
Date: time.Unix(alignedDatapointTime, 0),
Temp: temp,
}
tags, fields := smartTemp.Flatten()
tags["device_wwn"] = wwn
tags["scrutiny_uuid"] = scrutiny_uuid.String()
p := influxdb2.NewPoint("temp",
tags,
fields,
@@ -44,7 +45,6 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
}
}
// Even if ata_sct_temperature_history is present, also add current temperature. See #824
smartTemp := measurements.SmartTemperature{
Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0),
@@ -52,7 +52,7 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
}
tags, fields := smartTemp.Flatten()
tags["device_wwn"] = wwn
tags["scrutiny_uuid"] = scrutiny_uuid.String()
p := influxdb2.NewPoint("temp",
tags,
fields,
@@ -60,10 +60,10 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri
return sr.influxWriteApi.WritePoint(ctx, p)
}
func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) {
func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[uuid.UUID][]measurements.SmartTemperature, error) {
//we can get temp history for "week", "month", DURATION_KEY_YEAR, "forever"
deviceTempHistory := map[string][]measurements.SmartTemperature{}
deviceTempHistory := map[uuid.UUID][]measurements.SmartTemperature{}
//TODO: change the query range to a variable.
queryStr := sr.aggregateTempQuery(durationKey)
@@ -73,14 +73,15 @@ func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, du
// Use Next() to iterate over query result lines
for result.Next() {
if deviceWWN, ok := result.Record().Values()["device_wwn"]; ok {
if scrutinyUUIDString, ok := result.Record().Values()["scrutiny_uuid"]; ok {
scrutinyUUID := uuid.Must(uuid.FromString(scrutinyUUIDString.(string)))
//check if deviceWWN has been seen and initialized already
if _, ok := deviceTempHistory[deviceWWN.(string)]; !ok {
deviceTempHistory[deviceWWN.(string)] = []measurements.SmartTemperature{}
//check if scrutinyUUID has been seen and initialized already
if _, ok := deviceTempHistory[scrutinyUUID]; !ok {
deviceTempHistory[scrutinyUUID] = []measurements.SmartTemperature{}
}
currentTempHistory := deviceTempHistory[deviceWWN.(string)]
currentTempHistory := deviceTempHistory[scrutinyUUID]
smartTemp := measurements.SmartTemperature{}
for key, val := range result.Record().Values() {
@@ -88,7 +89,7 @@ func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, du
}
smartTemp.Date = result.Record().Values()["_time"].(time.Time)
currentTempHistory = append(currentTempHistory, smartTemp)
deviceTempHistory[deviceWWN.(string)] = currentTempHistory
deviceTempHistory[scrutinyUUID] = currentTempHistory
}
}
if result.Err() != nil {
@@ -113,18 +114,18 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
monthData = from(bucket: "metrics_weekly")
|> range(start: -1mo, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
union(tables: [weekData, monthData])
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> sort(columns: ["_time"], desc: false)
|> schema.fieldsAsCols()
@@ -148,7 +149,7 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]),
`|> filter(fn: (r) => r["_measurement"] == "temp" )`,
fmt.Sprintf(`|> aggregateWindow(every: %s, fn: mean, createEmpty: false)`, durationResolution),
`|> group(columns: ["device_wwn"])`,
`|> group(columns: ["scrutiny_uuid"])`,
`|> toInt()`,
"",
}...)
@@ -164,7 +165,7 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string {
} else {
partialQueryStr = append(partialQueryStr, []string{
fmt.Sprintf("union(tables: [%s])", strings.Join(subQueryNames, ", ")),
`|> group(columns: ["device_wwn"])`,
`|> group(columns: ["scrutiny_uuid"])`,
`|> sort(columns: ["_time"], desc: false)`,
"|> schema.fieldsAsCols()",
}...)
webapp/backend/pkg/database/scrutiny_repository_temperature_test.go
+13 -13
View File
@@ -32,7 +32,7 @@ weekData = from(bucket: "metrics")
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
weekData
@@ -64,18 +64,18 @@ weekData = from(bucket: "metrics")
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
monthData = from(bucket: "metrics_weekly")
|> range(start: -1mo, stop: -1w)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
union(tables: [weekData, monthData])
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> sort(columns: ["_time"], desc: false)
|> schema.fieldsAsCols()`, influxDbScript)
}
@@ -104,25 +104,25 @@ weekData = from(bucket: "metrics")
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
monthData = from(bucket: "metrics_weekly")
|> range(start: -1mo, stop: -1w)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
yearData = from(bucket: "metrics_monthly")
|> range(start: -1y, stop: -1mo)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
union(tables: [weekData, monthData, yearData])
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> sort(columns: ["_time"], desc: false)
|> schema.fieldsAsCols()`, influxDbScript)
}
@@ -151,32 +151,32 @@ weekData = from(bucket: "metrics")
|> range(start: -1w, stop: now())
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
monthData = from(bucket: "metrics_weekly")
|> range(start: -1mo, stop: -1w)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
yearData = from(bucket: "metrics_monthly")
|> range(start: -1y, stop: -1mo)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
foreverData = from(bucket: "metrics_yearly")
|> range(start: -10y, stop: -1y)
|> filter(fn: (r) => r["_measurement"] == "temp" )
|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> toInt()
union(tables: [weekData, monthData, yearData, foreverData])
|> group(columns: ["device_wwn"])
|> group(columns: ["scrutiny_uuid"])
|> sort(columns: ["_time"], desc: false)
|> schema.fieldsAsCols()`, influxDbScript)
}