overflow_backup_2018_0455/ssh_crawler
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ing

Browse Source
This commit is contained in:
insanity 2017-10-24 18:23:56 +09:00
parent 806f406ee0
commit 3134350172
9 changed files with 273 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
crawler/crawler.go
View File

@ -2,7 +2,6 @@ package crawler
import (
"bytes"
"encoding/json"
"git.loafle.net/overflow/ssh_crawler/ssh"
"git.loafle.net/overflow/ssh_crawler/stat"
)
@ -41,11 +40,7 @@ func (c *Crawler) CPUStat(ch chan interface{}, keys []string) {
if err != nil {
ch <- err
}
jsonData, err := json.Marshal(res)
if err != nil {
ch <- err
}
ch <- string(jsonData)
ch <- res
close(ch)
}()
}
@ -61,16 +56,12 @@ func (c *Crawler) MemStat(ch chan interface{}, keys []string) {
if err != nil {
ch <- err
}
jsonData, err := json.Marshal(res)
if err != nil {
ch <- err
}
ch <- string(jsonData)
ch <- res
close(ch)
}()
}
func (c *Crawler) DiskIOStat(ch chan interface{}) {
func (c *Crawler) DiskIOStat(ch chan interface{}, keys []string) {
go func() {
diskio := &stat.DiskIOStat{}
@ -79,7 +70,7 @@ func (c *Crawler) DiskIOStat(ch chan interface{}) {
ch <- err
}
res, err := diskio.Parse(bytes.NewReader(b))
res, err := diskio.Read(bytes.NewReader(b), keys)
if err != nil {
ch <- err
}
@ -88,14 +79,14 @@ func (c *Crawler) DiskIOStat(ch chan interface{}) {
}()
}
func (c *Crawler) DiskFreeStat(ch chan interface{}) {
func (c *Crawler) DiskFreeStat(ch chan interface{}, keys []string) {
go func() {
diskFree := &stat.DiskFreeStat{}
b, err := c.sshCli.RunCommand(diskFree.Command())
if err != nil {
ch <- err
}
res, err := diskFree.Parse(bytes.NewReader(b))
res, err := diskFree.Read(bytes.NewReader(b), keys)
if err != nil {
ch <- err
}
@ -112,7 +103,7 @@ func (c *Crawler) LoadAvgStat(ch chan interface{}) {
if err != nil {
ch <- err
}
res, err := load.Parse(bytes.NewReader(b))
res, err := load.Read(bytes.NewReader(b))
if err != nil {
ch <- err
}
@ -120,14 +111,14 @@ func (c *Crawler) LoadAvgStat(ch chan interface{}) {
}()
}
func (c *Crawler) NetworkStat(ch chan interface{}) {
func (c *Crawler) NetworkStat(ch chan interface{}, keys []string) {
go func() {
net := &stat.NetDevStat{}
b, err := c.sshCli.RunCommand(net.Command())
if err != nil {
ch <- err
}
res, err := net.Parse(bytes.NewReader(b))
res, err := net.Read(bytes.NewReader(b), keys)
if err != nil {
ch <- err
}
@ -142,7 +133,7 @@ func (c *Crawler) ProcessStat(ch chan interface{}) {
if err != nil {
ch <- err
}
res, err := ps.Parse(bytes.NewReader(b))
res, err := ps.Read(bytes.NewReader(b))
if err != nil {
ch <- err
}

126
main.go
View File

@ -3,6 +3,7 @@ package main
import (
"fmt"
"git.loafle.net/overflow/ssh_crawler/crawler"
"encoding/json"
)
func main() {
@ -17,7 +18,6 @@ func main() {
fmt.Println(err)
}
//사용자가 cpu.usage.total을 선택했을 때의 수집항목들 (..출력 항목도 포함시켜서 계산결과 리턴해야하나?)
cpukeys := []string{
"cpu.usage.sum",
"cpu.usage.user",
@ -29,11 +29,11 @@ func main() {
"cpu.usage.softirq",
"cpu.usage.steal",
"cpu.usage.guest",
"cpu.usage.guestNice",
"cpu.usage.gnice",
}
cpuCh := make(chan interface{})
cr.CPUStat(cpuCh, cpukeys)
fmt.Println(<-cpuCh)
print(<-cpuCh)
//Memory
memkeys := []string {
@ -48,39 +48,93 @@ func main() {
}
memCh := make(chan interface{})
cr.MemStat(memCh, memkeys)
fmt.Println(<-memCh)
print(<-memCh)
//memCh := make(chan interface{})
//cr.MemStat(memCh, keys)
//fmt.Println(<-memCh)
//Disk IO
ioKeys := []string {
"disk[0].io.device",
"disk[0].io.readcnt",
"disk[0].io.merged_readcnt",
"disk[0].io.readbytes",
"disk[0].io.readtime",
"disk[0].io.writecnt",
"disk[0].io.merged_writecnt",
"disk[0].io.writebytes",
"disk[0].io.writetime",
"disk[0].io.iotime",
"disk[0].io.weighted",
////Memory
//memCh := make(chan interface{})
//cr.MemStat(memCh)
//fmt.Println(<-memCh)
//
////Disk IO
//diskioCh := make(chan interface{})
//cr.DiskIOStat(diskioCh)
//fmt.Println(<-diskioCh)
//
////Disk Free
//diskFreeCh := make(chan interface{})
//cr.DiskFreeStat(diskFreeCh)
//fmt.Println(<-diskFreeCh)
//
////Network
//netCh := make(chan interface{})
//cr.NetworkStat(netCh)
//fmt.Println(<-netCh)
//
////Load Avg.
//loadCh := make(chan interface{})
//cr.LoadAvgStat(loadCh)
//fmt.Println(<-loadCh)
//
////Process
//psCh := make(chan interface{})
//cr.ProcessStat(psCh)
//fmt.Println(<-psCh)
"disk[1].io.readcnt",
"disk[2].io.merged_readcnt",
"disk[3].io.readbytes",
"disk[4].io.readtime",
}
diskioCh := make(chan interface{})
cr.DiskIOStat(diskioCh, ioKeys)
print(<-diskioCh)
//Disk Free
diskFreekeys := []string {
"disk[0].usage.fs",
"disk[0].usage.used",
"disk[0].usage.available",
"disk[0].usage.usedperc",
"disk[0].usage.mounted",
"disk[1].usage.fs",
"disk[2].usage.fs",
}
diskFreeCh := make(chan interface{})
cr.DiskFreeStat(diskFreeCh, diskFreekeys)
print(<-diskFreeCh)
//Network
netKeys := []string {
"net[14].transfer.iface",
"net[14].transfer.recv_byte",
"net[14].transfer.recv_packet",
"net[14].transfer.recv_err",
"net[14].transfer.recv_drop",
"net[14].transfer.recv_fifo",
"net[14].transfer.recv_frame",
"net[14].transfer.recv_compressed",
"net[14].transfer.recv_multicast",
"net[14].transfer.send_byte",
"net[14].transfer.send_packet",
"net[14].transfer.send_err",
"net[14].transfer.send_drop",
"net[14].transfer.send_fifo",
"net[14].transfer.send_frame",
"net[14].transfer.send_compressed",
"net[14].transfer.send_multicast",
"net[1].transfer.recv_packet",
"net[2].transfer.send_compressed",
"net[3].transfer.send_packet",
}
netCh := make(chan interface{})
cr.NetworkStat(netCh, netKeys)
print(<-netCh)
//Load Avg.
loadCh := make(chan interface{})
cr.LoadAvgStat(loadCh)
print(<-loadCh)
//Process
psCh := make(chan interface{})
cr.ProcessStat(psCh)
print(<-psCh)
}
func print(data interface{}) {
jsonData, err := json.Marshal(data)
if err != nil {
}
fmt.Println(string(jsonData))
}

41
stat/disk_free.go
View File

@ -4,6 +4,8 @@ import (
"bufio"
"io"
"strings"
"strconv"
"git.loafle.net/overflow/ssh_crawler/util"
)
type DiskFreeStat struct {
@ -19,7 +21,7 @@ func (diskFree DiskFreeStat) Command() string {
return "df -k"
}
func (diskio DiskFreeStat) Parse(r io.Reader) (*[]DiskFreeStat, error) {
func (diskio DiskFreeStat) Read(r io.Reader, keys []string) (*map[string]string, error) {
var scanner = bufio.NewScanner(r)
var stats = []DiskFreeStat{}
@ -37,7 +39,12 @@ func (diskio DiskFreeStat) Parse(r io.Reader) (*[]DiskFreeStat, error) {
})
}
return &stats, scanner.Err()
res, err := diskio.parse(keys, stats)
if err != nil {
return nil, err
}
return &res, scanner.Err()
}
func removePercUnit(str string) string {
@ -47,3 +54,33 @@ func removePercUnit(str string) string {
str = str[:len(str)-1]
return str
}
func (diskio DiskFreeStat) parse(keys []string, data []DiskFreeStat) (map[string]string, error) {
resMap := make(map[string]string)
for _, key := range keys {
t := strings.Split(key, ".")
suffix := t[len(t)-1]
ext := util.ExtractInBracket(key)
idx, _ := strconv.Atoi(ext)
switch suffix {
case "fs":
resMap[key] = data[idx].Filesystem
case "used":
resMap[key] = data[idx].Used
case "available":
resMap[key] = data[idx].Available
case "usedperc":
resMap[key] = data[idx].UsePerc
case "mounted":
resMap[key] = data[idx].MountedOn
default:
}
}
return resMap, nil
}

52
stat/disk_io.go
View File

@ -5,6 +5,8 @@ import (
"fmt"
"io"
"strings"
"git.loafle.net/overflow/ssh_crawler/util"
"strconv"
)
type DiskIOStat struct {
@ -26,7 +28,7 @@ func (diskio DiskIOStat) Command() string {
return "cat /proc/diskstats"
}
func (diskio DiskIOStat) Parse(r io.Reader) (*[]DiskIOStat, error) {
func (diskio DiskIOStat) Read(r io.Reader, keys []string) (*map[string]string, error) {
var (
stats = []DiskIOStat{}
scanner = bufio.NewScanner(r)
@ -55,8 +57,52 @@ func (diskio DiskIOStat) Parse(r io.Reader) (*[]DiskIOStat, error) {
TimeSpentIO: parts[12],
WeightedTimeSpentIO: parts[13],
})
}
return &stats, scanner.Err()
res, err :=diskio.parse(keys, stats)
if err != nil {
return nil, err
}
return &res, scanner.Err()
}
func (diskio DiskIOStat) parse(keys []string, data []DiskIOStat) (map[string]string, error) {
resMap := make(map[string]string)
for _, key := range keys {
t := strings.Split(key, ".")
suffix := t[len(t)-1]
ext := util.ExtractInBracket(key)
idx, _ := strconv.Atoi(ext)
switch suffix {
case "device":
resMap[key] = data[idx].Device
case "readcnt":
resMap[key] = data[idx].ReadCompleted
case "merged_readcnt":
resMap[key] = data[idx].ReadMerged
case "readbytes":
resMap[key] = util.Float64ToString(util.StringToFloat64(data[idx].SectorRead) * 512)
case "readtime":
resMap[key] = data[idx].TimeSpentReading
case "writecnt":
resMap[key] = data[idx].WritesCompleted
case "merged_writecnt":
resMap[key] = data[idx].WritesMerged
case "writebytes":
resMap[key] = util.Float64ToString(util.StringToFloat64(data[idx].SectorsWritten) * 512)
case "writetime":
resMap[key] = data[idx].TimeSpentWriting
case "iotime":
resMap[key] = data[idx].TimeSpentIO
case "ioweighted":
resMap[key] = data[idx].WeightedTimeSpentIO
default:
}
}
return resMap, nil
}

16
stat/loadavg.go
View File

@ -7,27 +7,25 @@ import (
)
type LoadAvg struct {
Min1,
Min5,
Min15 string
}
func (loadavg LoadAvg) Command() string {
return "cat /proc/loadavg"
}
func (loadavg LoadAvg) Parse(r io.Reader) (*LoadAvg, error) {
func (loadavg LoadAvg) Read(r io.Reader) (*map[string]string, error) {
var (
scanner = bufio.NewScanner(r)
resMap = make(map[string]string)
)
load := LoadAvg{}
for scanner.Scan() {
line := scanner.Text()
parts := strings.Fields(line)
load.Min1 = parts[0]
load.Min5 = parts[1]
load.Min15 = parts[2]
resMap["cpu.load.avg1"] = parts[0]
resMap["cpu.load.avg5"]= parts[1]
resMap["cpu.load.avg15"] = parts[2]
}
return &load, scanner.Err()
return &resMap, scanner.Err()
}

68
stat/network.go
View File

@ -6,6 +6,8 @@ import (
"io"
"regexp"
"strings"
"strconv"
"git.loafle.net/overflow/ssh_crawler/util"
)
type NetDevStat struct {
@ -32,11 +34,11 @@ func (net *NetDevStat) Command() string {
return "cat /proc/net/dev"
}
func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) {
func (net *NetDevStat) Read(r io.Reader, keys []string) (*map[string]string, error) {
var (
scanner = bufio.NewScanner(r)
procNetDevFieldSep = regexp.MustCompile("[ :] *")
netDevStats = []NetDevStat{}
stats = []NetDevStat{}
)
scanner.Scan()
scanner.Scan()
@ -55,7 +57,7 @@ func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) {
}
dev := parts[0][:len(parts[0])]
netDevStats = append(netDevStats, NetDevStat{
stats = append(stats, NetDevStat{
Iface: dev,
RecvByte: parts[1],
RecvPacket: parts[2],
@ -75,5 +77,63 @@ func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) {
TransMulticast: parts[16],
})
}
return &netDevStats, scanner.Err()
res, err := net.parse(keys, stats)
if err != nil {
return nil, err
}
return &res, scanner.Err()
}
func (net *NetDevStat) parse(keys []string, data []NetDevStat) (map[string]string, error) {
resMap := make(map[string]string)
for _, key := range keys {
t := strings.Split(key, ".")
suffix := t[len(t)-1]
ext := util.ExtractInBracket(key)
idx, _ := strconv.Atoi(ext)
switch suffix {
case "iface":
resMap[key] = data[idx].Iface
case "recv_byte":
resMap[key] = data[idx].RecvByte
case "recv_packet":
resMap[key] = data[idx].RecvPacket
case "recv_err":
resMap[key] = data[idx].RecvErr
case "recv_drop":
resMap[key] = data[idx].RecvDrop
case "recv_fifo":
resMap[key] = data[idx].RecvFifo
case "recv_frame":
resMap[key] = data[idx].RecvFrame
case "recv_compressed":
resMap[key] = data[idx].RecvCompressed
case "recv_multicast":
resMap[key] = data[idx].RecvMulticast
case "send_byte":
resMap[key] = data[idx].TransByte
case "send_packet":
resMap[key] = data[idx].TransPacket
case "send_err":
resMap[key] = data[idx].TransErr
case "send_drop":
resMap[key] = data[idx].TransDrop
case "send_fifo":
resMap[key] = data[idx].TransFifo
case "send_frame":
resMap[key] = data[idx].TransFrame
case "send_compressed":
resMap[key] = data[idx].TransCompressed
case "send_multicast":
resMap[key] = data[idx].TransMulticast
default:
}
}
return resMap, nil
}

14
stat/process.go
View File

@ -9,17 +9,16 @@ import (
type ProcessStat struct {
User,
Pid,
Size,
Pcpu,
Pmem,
Cmd string
}
func (ps *ProcessStat) Command() string {
return "ps -eo user,pid,size,pcpu,pmem,comm --sort=-pmem c"
return "ps -eo user,pid,pcpu,pmem,comm --sort=-pmem c"
}
func (ps *ProcessStat) Parse(r io.Reader) (*[]ProcessStat, error) {
func (ps *ProcessStat) Read(r io.Reader) (*[]ProcessStat, error) {
var (
psStats = []ProcessStat{}
scanner = bufio.NewScanner(r)
@ -31,11 +30,12 @@ func (ps *ProcessStat) Parse(r io.Reader) (*[]ProcessStat, error) {
psStats = append(psStats, ProcessStat{
User: parts[0],
Pid: parts[1],
Size: parts[2],
Pcpu: parts[3],
Pmem: parts[4],
Cmd: parts[5],
Pcpu: parts[2],
Pmem: parts[3],
Cmd: parts[4],
})
}
return &psStats, scanner.Err()
}

6
util/util.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"math"
"strconv"
"regexp"
)
func StringToFloat64(str string) float64 {
@ -24,3 +25,8 @@ func toFixed(num float64, precision int) float64 {
output := math.Pow(10, float64(precision))
return float64(round(num*output)) / output
}
func ExtractInBracket(str string) string {
regex := regexp.MustCompile(`\[(.*?)\]`)
return regex.FindStringSubmatch(str)[1]
}