From 31343501721921f8a01e96aefbec3123bc7513b6 Mon Sep 17 00:00:00 2001 From: insanity Date: Tue, 24 Oct 2017 18:23:56 +0900 Subject: [PATCH] ing --- crawler/crawler.go | 29 ++++------- main.go | 126 ++++++++++++++++++++++++++++++++------------- stat/cpu.go | 2 +- stat/disk_free.go | 41 ++++++++++++++- stat/disk_io.go | 52 +++++++++++++++++-- stat/loadavg.go | 16 +++--- stat/network.go | 68 ++++++++++++++++++++++-- stat/process.go | 14 ++--- util/util.go | 6 +++ 9 files changed, 273 insertions(+), 81 deletions(-) diff --git a/crawler/crawler.go b/crawler/crawler.go index 74508d0..9daba60 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -2,7 +2,6 @@ package crawler import ( "bytes" - "encoding/json" "git.loafle.net/overflow/ssh_crawler/ssh" "git.loafle.net/overflow/ssh_crawler/stat" ) @@ -41,11 +40,7 @@ func (c *Crawler) CPUStat(ch chan interface{}, keys []string) { if err != nil { ch <- err } - jsonData, err := json.Marshal(res) - if err != nil { - ch <- err - } - ch <- string(jsonData) + ch <- res close(ch) }() } @@ -61,16 +56,12 @@ func (c *Crawler) MemStat(ch chan interface{}, keys []string) { if err != nil { ch <- err } - jsonData, err := json.Marshal(res) - if err != nil { - ch <- err - } - ch <- string(jsonData) + ch <- res close(ch) }() } -func (c *Crawler) DiskIOStat(ch chan interface{}) { +func (c *Crawler) DiskIOStat(ch chan interface{}, keys []string) { go func() { diskio := &stat.DiskIOStat{} @@ -79,7 +70,7 @@ func (c *Crawler) DiskIOStat(ch chan interface{}) { ch <- err } - res, err := diskio.Parse(bytes.NewReader(b)) + res, err := diskio.Read(bytes.NewReader(b), keys) if err != nil { ch <- err } @@ -88,14 +79,14 @@ func (c *Crawler) DiskIOStat(ch chan interface{}) { }() } -func (c *Crawler) DiskFreeStat(ch chan interface{}) { +func (c *Crawler) DiskFreeStat(ch chan interface{}, keys []string) { go func() { diskFree := &stat.DiskFreeStat{} b, err := c.sshCli.RunCommand(diskFree.Command()) if err != nil { ch <- err } - res, err := diskFree.Parse(bytes.NewReader(b)) + res, err := diskFree.Read(bytes.NewReader(b), keys) if err != nil { ch <- err } @@ -112,7 +103,7 @@ func (c *Crawler) LoadAvgStat(ch chan interface{}) { if err != nil { ch <- err } - res, err := load.Parse(bytes.NewReader(b)) + res, err := load.Read(bytes.NewReader(b)) if err != nil { ch <- err } @@ -120,14 +111,14 @@ func (c *Crawler) LoadAvgStat(ch chan interface{}) { }() } -func (c *Crawler) NetworkStat(ch chan interface{}) { +func (c *Crawler) NetworkStat(ch chan interface{}, keys []string) { go func() { net := &stat.NetDevStat{} b, err := c.sshCli.RunCommand(net.Command()) if err != nil { ch <- err } - res, err := net.Parse(bytes.NewReader(b)) + res, err := net.Read(bytes.NewReader(b), keys) if err != nil { ch <- err } @@ -142,7 +133,7 @@ func (c *Crawler) ProcessStat(ch chan interface{}) { if err != nil { ch <- err } - res, err := ps.Parse(bytes.NewReader(b)) + res, err := ps.Read(bytes.NewReader(b)) if err != nil { ch <- err } diff --git a/main.go b/main.go index 5e55c5b..51e0cb7 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "git.loafle.net/overflow/ssh_crawler/crawler" + "encoding/json" ) func main() { @@ -17,7 +18,6 @@ func main() { fmt.Println(err) } - //사용자가 cpu.usage.total을 선택했을 때의 수집항목들 (..출력 항목도 포함시켜서 계산결과 리턴해야하나?) cpukeys := []string{ "cpu.usage.sum", "cpu.usage.user", @@ -29,11 +29,11 @@ func main() { "cpu.usage.softirq", "cpu.usage.steal", "cpu.usage.guest", - "cpu.usage.guestNice", + "cpu.usage.gnice", } cpuCh := make(chan interface{}) cr.CPUStat(cpuCh, cpukeys) - fmt.Println(<-cpuCh) + print(<-cpuCh) //Memory memkeys := []string { @@ -48,39 +48,93 @@ func main() { } memCh := make(chan interface{}) cr.MemStat(memCh, memkeys) - fmt.Println(<-memCh) + print(<-memCh) - //memCh := make(chan interface{}) - //cr.MemStat(memCh, keys) - //fmt.Println(<-memCh) + //Disk IO + ioKeys := []string { + "disk[0].io.device", + "disk[0].io.readcnt", + "disk[0].io.merged_readcnt", + "disk[0].io.readbytes", + "disk[0].io.readtime", + "disk[0].io.writecnt", + "disk[0].io.merged_writecnt", + "disk[0].io.writebytes", + "disk[0].io.writetime", + "disk[0].io.iotime", + "disk[0].io.weighted", - ////Memory - //memCh := make(chan interface{}) - //cr.MemStat(memCh) - //fmt.Println(<-memCh) - // - ////Disk IO - //diskioCh := make(chan interface{}) - //cr.DiskIOStat(diskioCh) - //fmt.Println(<-diskioCh) - // - ////Disk Free - //diskFreeCh := make(chan interface{}) - //cr.DiskFreeStat(diskFreeCh) - //fmt.Println(<-diskFreeCh) - // - ////Network - //netCh := make(chan interface{}) - //cr.NetworkStat(netCh) - //fmt.Println(<-netCh) - // - ////Load Avg. - //loadCh := make(chan interface{}) - //cr.LoadAvgStat(loadCh) - //fmt.Println(<-loadCh) - // - ////Process - //psCh := make(chan interface{}) - //cr.ProcessStat(psCh) - //fmt.Println(<-psCh) + "disk[1].io.readcnt", + "disk[2].io.merged_readcnt", + "disk[3].io.readbytes", + "disk[4].io.readtime", + } + diskioCh := make(chan interface{}) + cr.DiskIOStat(diskioCh, ioKeys) + print(<-diskioCh) + + + //Disk Free + diskFreekeys := []string { + "disk[0].usage.fs", + "disk[0].usage.used", + "disk[0].usage.available", + "disk[0].usage.usedperc", + "disk[0].usage.mounted", + + "disk[1].usage.fs", + "disk[2].usage.fs", + } + diskFreeCh := make(chan interface{}) + cr.DiskFreeStat(diskFreeCh, diskFreekeys) + print(<-diskFreeCh) + + + + //Network + netKeys := []string { + "net[14].transfer.iface", + "net[14].transfer.recv_byte", + "net[14].transfer.recv_packet", + "net[14].transfer.recv_err", + "net[14].transfer.recv_drop", + "net[14].transfer.recv_fifo", + "net[14].transfer.recv_frame", + "net[14].transfer.recv_compressed", + "net[14].transfer.recv_multicast", + "net[14].transfer.send_byte", + "net[14].transfer.send_packet", + "net[14].transfer.send_err", + "net[14].transfer.send_drop", + "net[14].transfer.send_fifo", + "net[14].transfer.send_frame", + "net[14].transfer.send_compressed", + "net[14].transfer.send_multicast", + + "net[1].transfer.recv_packet", + "net[2].transfer.send_compressed", + "net[3].transfer.send_packet", + } + netCh := make(chan interface{}) + cr.NetworkStat(netCh, netKeys) + print(<-netCh) + + + //Load Avg. + loadCh := make(chan interface{}) + cr.LoadAvgStat(loadCh) + print(<-loadCh) + + + //Process + psCh := make(chan interface{}) + cr.ProcessStat(psCh) + print(<-psCh) +} + +func print(data interface{}) { + jsonData, err := json.Marshal(data) + if err != nil { + } + fmt.Println(string(jsonData)) } diff --git a/stat/cpu.go b/stat/cpu.go index aaf9fe8..76a01eb 100644 --- a/stat/cpu.go +++ b/stat/cpu.go @@ -65,7 +65,7 @@ func (cpu CPUStat) Read(r io.Reader, keys []string) (*map[string]string, error) }) } - res, err :=cpu.parse(keys, stats) + res, err := cpu.parse(keys, stats) if err != nil { return nil, err } diff --git a/stat/disk_free.go b/stat/disk_free.go index 1ceaa57..28953e4 100644 --- a/stat/disk_free.go +++ b/stat/disk_free.go @@ -4,6 +4,8 @@ import ( "bufio" "io" "strings" + "strconv" + "git.loafle.net/overflow/ssh_crawler/util" ) type DiskFreeStat struct { @@ -19,7 +21,7 @@ func (diskFree DiskFreeStat) Command() string { return "df -k" } -func (diskio DiskFreeStat) Parse(r io.Reader) (*[]DiskFreeStat, error) { +func (diskio DiskFreeStat) Read(r io.Reader, keys []string) (*map[string]string, error) { var scanner = bufio.NewScanner(r) var stats = []DiskFreeStat{} @@ -37,7 +39,12 @@ func (diskio DiskFreeStat) Parse(r io.Reader) (*[]DiskFreeStat, error) { }) } - return &stats, scanner.Err() + res, err := diskio.parse(keys, stats) + if err != nil { + return nil, err + } + + return &res, scanner.Err() } func removePercUnit(str string) string { @@ -47,3 +54,33 @@ func removePercUnit(str string) string { str = str[:len(str)-1] return str } + + +func (diskio DiskFreeStat) parse(keys []string, data []DiskFreeStat) (map[string]string, error) { + resMap := make(map[string]string) + + for _, key := range keys { + t := strings.Split(key, ".") + suffix := t[len(t)-1] + ext := util.ExtractInBracket(key) + idx, _ := strconv.Atoi(ext) + + switch suffix { + case "fs": + resMap[key] = data[idx].Filesystem + case "used": + resMap[key] = data[idx].Used + case "available": + resMap[key] = data[idx].Available + case "usedperc": + resMap[key] = data[idx].UsePerc + case "mounted": + resMap[key] = data[idx].MountedOn + + default: + + } + } + + return resMap, nil +} \ No newline at end of file diff --git a/stat/disk_io.go b/stat/disk_io.go index 4e8268c..e1acb0c 100644 --- a/stat/disk_io.go +++ b/stat/disk_io.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "strings" + "git.loafle.net/overflow/ssh_crawler/util" + "strconv" ) type DiskIOStat struct { @@ -26,7 +28,7 @@ func (diskio DiskIOStat) Command() string { return "cat /proc/diskstats" } -func (diskio DiskIOStat) Parse(r io.Reader) (*[]DiskIOStat, error) { +func (diskio DiskIOStat) Read(r io.Reader, keys []string) (*map[string]string, error) { var ( stats = []DiskIOStat{} scanner = bufio.NewScanner(r) @@ -55,8 +57,52 @@ func (diskio DiskIOStat) Parse(r io.Reader) (*[]DiskIOStat, error) { TimeSpentIO: parts[12], WeightedTimeSpentIO: parts[13], }) - } - return &stats, scanner.Err() + res, err :=diskio.parse(keys, stats) + if err != nil { + return nil, err + } + + + return &res, scanner.Err() +} + +func (diskio DiskIOStat) parse(keys []string, data []DiskIOStat) (map[string]string, error) { + resMap := make(map[string]string) + + for _, key := range keys { + t := strings.Split(key, ".") + suffix := t[len(t)-1] + ext := util.ExtractInBracket(key) + idx, _ := strconv.Atoi(ext) + switch suffix { + case "device": + resMap[key] = data[idx].Device + case "readcnt": + resMap[key] = data[idx].ReadCompleted + case "merged_readcnt": + resMap[key] = data[idx].ReadMerged + case "readbytes": + resMap[key] = util.Float64ToString(util.StringToFloat64(data[idx].SectorRead) * 512) + case "readtime": + resMap[key] = data[idx].TimeSpentReading + case "writecnt": + resMap[key] = data[idx].WritesCompleted + case "merged_writecnt": + resMap[key] = data[idx].WritesMerged + case "writebytes": + resMap[key] = util.Float64ToString(util.StringToFloat64(data[idx].SectorsWritten) * 512) + case "writetime": + resMap[key] = data[idx].TimeSpentWriting + case "iotime": + resMap[key] = data[idx].TimeSpentIO + case "ioweighted": + resMap[key] = data[idx].WeightedTimeSpentIO + default: + + } + } + + return resMap, nil } diff --git a/stat/loadavg.go b/stat/loadavg.go index d415aff..21e6081 100644 --- a/stat/loadavg.go +++ b/stat/loadavg.go @@ -7,27 +7,25 @@ import ( ) type LoadAvg struct { - Min1, - Min5, - Min15 string } func (loadavg LoadAvg) Command() string { return "cat /proc/loadavg" } -func (loadavg LoadAvg) Parse(r io.Reader) (*LoadAvg, error) { +func (loadavg LoadAvg) Read(r io.Reader) (*map[string]string, error) { var ( scanner = bufio.NewScanner(r) + resMap = make(map[string]string) ) - load := LoadAvg{} for scanner.Scan() { line := scanner.Text() parts := strings.Fields(line) - load.Min1 = parts[0] - load.Min5 = parts[1] - load.Min15 = parts[2] + resMap["cpu.load.avg1"] = parts[0] + resMap["cpu.load.avg5"]= parts[1] + resMap["cpu.load.avg15"] = parts[2] } - return &load, scanner.Err() + + return &resMap, scanner.Err() } diff --git a/stat/network.go b/stat/network.go index 7657a8a..8e6a830 100644 --- a/stat/network.go +++ b/stat/network.go @@ -6,6 +6,8 @@ import ( "io" "regexp" "strings" + "strconv" + "git.loafle.net/overflow/ssh_crawler/util" ) type NetDevStat struct { @@ -32,11 +34,11 @@ func (net *NetDevStat) Command() string { return "cat /proc/net/dev" } -func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) { +func (net *NetDevStat) Read(r io.Reader, keys []string) (*map[string]string, error) { var ( scanner = bufio.NewScanner(r) procNetDevFieldSep = regexp.MustCompile("[ :] *") - netDevStats = []NetDevStat{} + stats = []NetDevStat{} ) scanner.Scan() scanner.Scan() @@ -55,7 +57,7 @@ func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) { } dev := parts[0][:len(parts[0])] - netDevStats = append(netDevStats, NetDevStat{ + stats = append(stats, NetDevStat{ Iface: dev, RecvByte: parts[1], RecvPacket: parts[2], @@ -75,5 +77,63 @@ func (net *NetDevStat) Parse(r io.Reader) (*[]NetDevStat, error) { TransMulticast: parts[16], }) } - return &netDevStats, scanner.Err() + + res, err := net.parse(keys, stats) + if err != nil { + return nil, err + } + + return &res, scanner.Err() } + +func (net *NetDevStat) parse(keys []string, data []NetDevStat) (map[string]string, error) { + resMap := make(map[string]string) + + for _, key := range keys { + t := strings.Split(key, ".") + suffix := t[len(t)-1] + ext := util.ExtractInBracket(key) + idx, _ := strconv.Atoi(ext) + switch suffix { + case "iface": + resMap[key] = data[idx].Iface + case "recv_byte": + resMap[key] = data[idx].RecvByte + case "recv_packet": + resMap[key] = data[idx].RecvPacket + case "recv_err": + resMap[key] = data[idx].RecvErr + case "recv_drop": + resMap[key] = data[idx].RecvDrop + case "recv_fifo": + resMap[key] = data[idx].RecvFifo + case "recv_frame": + resMap[key] = data[idx].RecvFrame + case "recv_compressed": + resMap[key] = data[idx].RecvCompressed + case "recv_multicast": + resMap[key] = data[idx].RecvMulticast + + case "send_byte": + resMap[key] = data[idx].TransByte + case "send_packet": + resMap[key] = data[idx].TransPacket + case "send_err": + resMap[key] = data[idx].TransErr + case "send_drop": + resMap[key] = data[idx].TransDrop + case "send_fifo": + resMap[key] = data[idx].TransFifo + case "send_frame": + resMap[key] = data[idx].TransFrame + case "send_compressed": + resMap[key] = data[idx].TransCompressed + case "send_multicast": + resMap[key] = data[idx].TransMulticast + default: + + } + } + + return resMap, nil +} \ No newline at end of file diff --git a/stat/process.go b/stat/process.go index 5ecb067..5c45944 100644 --- a/stat/process.go +++ b/stat/process.go @@ -9,17 +9,16 @@ import ( type ProcessStat struct { User, Pid, - Size, Pcpu, Pmem, Cmd string } func (ps *ProcessStat) Command() string { - return "ps -eo user,pid,size,pcpu,pmem,comm --sort=-pmem c" + return "ps -eo user,pid,pcpu,pmem,comm --sort=-pmem c" } -func (ps *ProcessStat) Parse(r io.Reader) (*[]ProcessStat, error) { +func (ps *ProcessStat) Read(r io.Reader) (*[]ProcessStat, error) { var ( psStats = []ProcessStat{} scanner = bufio.NewScanner(r) @@ -31,11 +30,12 @@ func (ps *ProcessStat) Parse(r io.Reader) (*[]ProcessStat, error) { psStats = append(psStats, ProcessStat{ User: parts[0], Pid: parts[1], - Size: parts[2], - Pcpu: parts[3], - Pmem: parts[4], - Cmd: parts[5], + Pcpu: parts[2], + Pmem: parts[3], + Cmd: parts[4], }) } + + return &psStats, scanner.Err() } diff --git a/util/util.go b/util/util.go index 8441def..e5906ea 100644 --- a/util/util.go +++ b/util/util.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "strconv" + "regexp" ) func StringToFloat64(str string) float64 { @@ -24,3 +25,8 @@ func toFixed(num float64, precision int) float64 { output := math.Pow(10, float64(precision)) return float64(round(num*output)) / output } + +func ExtractInBracket(str string) string { + regex := regexp.MustCompile(`\[(.*?)\]`) + return regex.FindStringSubmatch(str)[1] +}