gilles/scrutiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding support for a collecto config file.

Browse Source 
/scrutiny/config/collector.yaml

Adding ability to specify host identifier (label), that is updated on every collector run.
Can be specified by `host-id` CLI or `COLLECTOR_HOST_ID` env var.

Created a config class, interface and associated tests.

Created a "TransformDetectedDrives" function, that will allow users to insert drives not detected by Smarctl --scan, ignore drives that they dont want, and override smartctl device type.

Added Upsert functionality when registering devices.

Replaced "github.com/jinzhu/gorm" with "gorm.io/gorm" (ORM location moved, was using incorrect lib url)
Removed machineid library.
This commit is contained in:
Jason Kulatunga
2020-10-07 21:54:29 -06:00
parent 32e7044c67
commit b44ef5cb9c
33 changed files with 851 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
collector/pkg/detect/detect.go
+63 -16
View File
@@ -4,9 +4,9 @@ import (
"encoding/json"
"fmt"
"github.com/analogj/scrutiny/collector/pkg/common"
"github.com/analogj/scrutiny/collector/pkg/config"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
"github.com/denisbrodbeck/machineid"
"github.com/sirupsen/logrus"
"os"
"strings"
@@ -14,6 +14,7 @@ import (
type Detect struct {
Logger *logrus.Entry
Config config.Interface
}
//private/common functions
@@ -24,7 +25,7 @@ type Detect struct {
//
// To handle these issues, we have OS specific wrapper functions that update/modify these detected devices.
// models.Device returned from this function only contain the minimum data for smartctl to execute: device type and device name (device file).
func (d *Detect) smartctlScan() ([]models.Device, error) {
func (d *Detect) SmartctlScan() ([]models.Device, error) {
//we use smartctl to detect all the drives available.
detectedDeviceConnJson, err := common.ExecCmd(d.Logger, "smartctl", []string{"--scan", "-j"}, "", os.Environ())
if err != nil {
@@ -39,14 +40,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
return nil, err
}
detectedDevices := []models.Device{}
for _, detectedDevice := range detectedDeviceConns.Devices {
detectedDevices = append(detectedDevices, models.Device{
DeviceType: detectedDevice.Type,
DeviceName: strings.TrimPrefix(detectedDevice.Name, DevicePrefix()),
})
}
detectedDevices := d.TransformDetectedDevices(detectedDeviceConns)
return detectedDevices, nil
}
@@ -55,7 +49,7 @@ func (d *Detect) smartctlScan() ([]models.Device, error) {
// It has a couple of issues however:
// - WWN is provided as component data, rather than a "string". We'll have to generate the WWN value ourselves
// - WWN from smartctl only provided for ATA protocol drives, NVMe and SCSI drives do not include WWN.
func (d *Detect) smartCtlInfo(device *models.Device) error {
func (d *Detect) SmartCtlInfo(device *models.Device) error {
args := []string{"--info", "-j"}
//only include the device type if its a non-standard one. In some cases ata drives are detected as scsi in docker, and metadata is lost.
@@ -77,8 +71,8 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
return err
}
//DeviceType and DeviceName are already populated.
//WWN
//WWN: this is a serial number/world-wide number that will not change.
//DeviceType and DeviceName are already populated, however may change between collector runs (eg. config/host restart)
//InterfaceType:
device.ModelName = availableDeviceInfo.ModelName
device.InterfaceSpeed = availableDeviceInfo.InterfaceSpeed.Current.String
@@ -110,7 +104,60 @@ func (d *Detect) smartCtlInfo(device *models.Device) error {
return nil
}
//uses https://github.com/denisbrodbeck/machineid to get a OS specific unique machine ID.
func (d *Detect) getMachineId() (string, error) {
return machineid.ProtectedID("scrutiny")
// function will remove devices that are marked for "ignore" in config file
// will also add devices that are specified in config file, but "missing" from smartctl --scan
// this function will also update the deviceType to the option specified in config.
func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []models.Device {
groupedDevices := map[string][]models.Device{}
for _, scannedDevice := range detectedDeviceConns.Devices {
deviceFile := strings.ToLower(scannedDevice.Name)
detectedDevice := models.Device{
HostId: d.Config.GetString("host.id"),
DeviceType: scannedDevice.Type,
DeviceName: strings.TrimPrefix(deviceFile, DevicePrefix()),
}
//find (or create) a slice to contain the devices in this group
if groupedDevices[deviceFile] == nil {
groupedDevices[deviceFile] = []models.Device{}
}
// add this scanned device to the group
groupedDevices[deviceFile] = append(groupedDevices[deviceFile], detectedDevice)
}
//now tha we've "grouped" all the devices, lets override any groups specified in the config file.
for _, overrideDevice := range d.Config.GetScanOverrides() {
overrideDeviceFile := strings.ToLower(overrideDevice.Device)
if overrideDevice.Ignore {
// this device file should be deleted if it exists
delete(groupedDevices, overrideDeviceFile)
} else {
//create a new device group, and replace the one generated by smartctl --scan
overrideDeviceGroup := []models.Device{}
for _, overrideDeviceType := range overrideDevice.DeviceType {
overrideDeviceGroup = append(overrideDeviceGroup, models.Device{
HostId: d.Config.GetString("host.id"),
DeviceType: overrideDeviceType,
DeviceName: strings.TrimPrefix(overrideDeviceFile, DevicePrefix()),
})
}
groupedDevices[overrideDeviceFile] = overrideDeviceGroup
}
}
//flatten map
detectedDevices := []models.Device{}
for _, group := range groupedDevices {
detectedDevices = append(detectedDevices, group...)
}
return detectedDevices
}
collector/pkg/detect/detect_test.go
+138
View File
@@ -0,0 +1,138 @@
package detect_test
import (
mock_config "github.com/analogj/scrutiny/collector/pkg/config/mock"
"github.com/analogj/scrutiny/collector/pkg/detect"
"github.com/analogj/scrutiny/collector/pkg/models"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/require"
"testing"
)
func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, "sda", transformedDevices[0].DeviceName)
require.Equal(t, "scsi", transformedDevices[0].DeviceType)
}
func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, []models.Device{}, transformedDevices)
}
func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{
{
Device: "/dev/bus/0",
DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
Ignore: false,
},
{
Device: "/dev/twa0",
DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
Ignore: false,
}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/bus/0",
InfoName: "/dev/bus/0",
Protocol: "scsi",
Type: "scsi",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, 12, len(transformedDevices))
}
func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
//setup
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
fakeConfig := mock_config.NewMockInterface(mockCtrl)
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
fakeConfig.EXPECT().GetScanOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
detectedDevices := models.Scan{
Devices: []models.ScanDevice{
{
Name: "/dev/sda",
InfoName: "/dev/sda",
Protocol: "ata",
Type: "ata",
},
},
}
d := detect.Detect{
Config: fakeConfig,
}
//test
transformedDevices := d.TransformDetectedDevices(detectedDevices)
//assert
require.Equal(t, 1, len(transformedDevices))
require.Equal(t, "sat+megaraid", transformedDevices[0].DeviceType)
}
collector/pkg/detect/devices_darwin.go
+2 -2
View File
@@ -12,7 +12,7 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devicess
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
@@ -25,7 +25,7 @@ func (d *Detect) Start() ([]models.Device, error) {
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_freebsd.go
+2 -2
View File
@@ -12,14 +12,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_linux.go
+2 -2
View File
@@ -12,14 +12,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil
collector/pkg/detect/devices_windows.go
+2 -2
View File
@@ -11,14 +11,14 @@ func DevicePrefix() string {
func (d *Detect) Start() ([]models.Device, error) {
// call the base/common functionality to get a list of devices
detectedDevices, err := d.smartctlScan()
detectedDevices, err := d.SmartctlScan()
if err != nil {
return nil, err
}
//inflate device info for detected devices.
for ndx, _ := range detectedDevices {
d.smartCtlInfo(&detectedDevices[ndx]) //ignore errors.
d.SmartCtlInfo(&detectedDevices[ndx]) //ignore errors.
}
return detectedDevices, nil