From 4d8352994348d52a43ba936e41e44a254f54a5a6 Mon Sep 17 00:00:00 2001 From: crusader Date: Thu, 14 Dec 2017 14:36:17 +0900 Subject: [PATCH] ing --- crawler/impl/ssh/SSHCrawler.go | 94 +++++++++-------- crawler/impl/ssh/client/client.go | 125 +++++++++++++++++++++++ crawler/impl/ssh/parser/cpu_parser.go | 100 ++++++++++++++++++ crawler/impl/ssh/parser/memory_parser.go | 34 ++++++ crawler/impl/ssh/parser/parser.go | 29 ++++++ 5 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 crawler/impl/ssh/client/client.go create mode 100644 crawler/impl/ssh/parser/cpu_parser.go create mode 100644 crawler/impl/ssh/parser/memory_parser.go create mode 100644 crawler/impl/ssh/parser/parser.go diff --git a/crawler/impl/ssh/SSHCrawler.go b/crawler/impl/ssh/SSHCrawler.go index 3510d73..abf2f7e 100644 --- a/crawler/impl/ssh/SSHCrawler.go +++ b/crawler/impl/ssh/SSHCrawler.go @@ -1,11 +1,15 @@ package ssh import ( + "bufio" + "bytes" "fmt" - "reflect" configM "git.loafle.net/overflow/overflow_commons_go/modules/config/model" "git.loafle.net/overflow/overflow_probe_container_network/crawler" + "git.loafle.net/overflow/overflow_probe_container_network/crawler/impl/ssh/client" + "git.loafle.net/overflow/overflow_probe_container_network/crawler/impl/ssh/parser" + "github.com/satori/go.uuid" ) type SSHCrawler struct { @@ -17,65 +21,65 @@ func (c *SSHCrawler) Name() string { } func (c *SSHCrawler) InternalGet(config *configM.Config) (map[string]string, error) { - - return c.start(config) -} - -func (c *SSHCrawler) start(config *configM.Config) (map[string]string, error) { - ip := config.Target.Connection.IP - port := config.Target.Connection.Port - user := config.Target.Auth["id"].(string) - pw := config.Target.Auth["pw"].(string) - keyFilePathObj := config.Target.Auth["keyFilePath"] - keyFilePath := "" - - if keyFilePathObj != nil { - keyFilePath = keyFilePathObj.(string) + sshClient, err := client.New(config.Target) + if nil != err { + return nil, err } - cr, err := crawler.New(ip, port, user, pw, keyFilePath) + itemCount := len(config.Items) + results := make(map[string]string, 0) + boundary := uuid.NewV4().String() + commands := "" - if err != nil { - fmt.Println(err) + for i := 0; i < itemCount; i++ { + if "" != commands { + commands = fmt.Sprintf("%s;echo \"--%s\";", commands, boundary) + } + commands = fmt.Sprintf("%s;%s;", commands, config.Items[i].QueryInfo.Query) } + commands = fmt.Sprintf("%s;echo \"--%s--\";", commands, boundary) - var inter crawler.SSHCrawlerModuler - var resultMap map[string]string = make(map[string]string) - var tempMap map[string]string = nil - var temp interface{} = nil - for _, item := range c.Items { + buf, err := sshClient.RunCommand(commands) + if nil != err { + return nil, err + } + r := bytes.NewReader(buf) + scanner := bufio.NewScanner(r) + +ParseLoop: + for i := 0; i < itemCount; i++ { + item := config.Items[i] mode := item.QueryInfo.Extend["mode"].(string) - - switch mode { - case "cpu": - inter = stat.CPUStat{} - break - case "mem": - inter = stat.MemStat{} - break - default: - continue + p := parser.GetParser(mode) + if nil == p { + return nil, fmt.Errorf("Container: Parser[%s] is not exist", mode) } - ch := make(chan interface{}) - cr.Process(inter, ch, item) - temp = <-ch + rm, err := p.Parse(scanner) + if nil != err { + return nil, err + } - if reflect.TypeOf(temp).String() == "map[string]string" { - tempMap = temp.(map[string]string) - for k, v := range tempMap { - resultMap[k] = v + if nil != rm { + for k, v := range rm { + results[k] = v } - } else { - var errr error = temp.(error) + } - return nil, errr + for scanner.Scan() { + line := scanner.Text() + if line == "--"+boundary+"--" { + break ParseLoop + } + + if line == "--"+boundary { + break + } } } - return &resultMap, nil - + return results, nil } func NewCrawler() crawler.Crawler { diff --git a/crawler/impl/ssh/client/client.go b/crawler/impl/ssh/client/client.go new file mode 100644 index 0000000..fb6bb65 --- /dev/null +++ b/crawler/impl/ssh/client/client.go @@ -0,0 +1,125 @@ +package client + +import ( + "bytes" + "fmt" + "io/ioutil" + "time" + + cuej "git.loafle.net/commons_go/util/encoding/json" + configM "git.loafle.net/overflow/overflow_commons_go/modules/config/model" + "golang.org/x/crypto/ssh" +) + +type SSHConfig struct { + User string + Auth []ssh.AuthMethod + Host string + Port int +} + +type SSHClient struct { + conf *SSHConfig +} + +func (cli *SSHClient) Session() (*ssh.Session, error) { + sshConfig := &ssh.ClientConfig{ + User: cli.conf.User, + Auth: cli.conf.Auth, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: time.Second * 10, + } + + addr := fmt.Sprintf("%s:%d", cli.conf.Host, cli.conf.Port) + + connection, err := ssh.Dial("tcp", addr, sshConfig) + if err != nil { + return nil, err + } + session, err := connection.NewSession() + if err != nil { + return nil, err + } + + modes := ssh.TerminalModes{ + ssh.ECHO: 0, // disable echoing + ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud + ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud + } + + if err := session.RequestPty("xterm", 80, 40, modes); err != nil { + session.Close() + return nil, err + } + + return session, nil +} + +func (cli *SSHClient) RunCommand(command string) ([]byte, error) { + + session, err := cli.Session() + if nil != err { + return nil, err + } + defer func() { + session.Close() + }() + + var b bytes.Buffer + session.Stdout = &b + + if err := session.Run(command); nil != err { + return nil, err + } + + return b.Bytes(), nil +} + +func parsePrivateKey(keyPath, pw string) (ssh.Signer, error) { + buff, err := ioutil.ReadFile(keyPath) + if err != nil { + return nil, err + } + if pw == "" { + return ssh.ParsePrivateKey(buff) + } + return ssh.ParsePrivateKeyWithPassphrase(buff, []byte(pw)) +} + +func New(target *configM.Target) (*SSHClient, error) { + connection := target.Connection + auth := target.Auth + + ip := connection.IP + port, _ := cuej.NumberToInt(connection.Port) + user := auth["id"].(string) + pw := auth["pw"].(string) + keyFilePathObj := auth["keyFilePath"] + keyFilePath := "" + + sshAuth := make([]ssh.AuthMethod, 0) + + if keyFilePathObj != nil { + keyFilePath = keyFilePathObj.(string) + if "" != keyFilePath { + key, err := parsePrivateKey(keyFilePath, pw) + if err != nil { + return nil, err + } + sshAuth = append(sshAuth, ssh.PublicKeys(key)) + + } + } + sshAuth = append(sshAuth, ssh.Password(pw)) + + sshConf := &SSHConfig{ + User: user, + Auth: sshAuth, + Host: ip, + Port: port, + } + + return &SSHClient{ + conf: sshConf, + }, nil +} diff --git a/crawler/impl/ssh/parser/cpu_parser.go b/crawler/impl/ssh/parser/cpu_parser.go new file mode 100644 index 0000000..d42e3af --- /dev/null +++ b/crawler/impl/ssh/parser/cpu_parser.go @@ -0,0 +1,100 @@ +package parser + +import ( + "bufio" + "strconv" + "strings" +) + +type CPUParser struct { +} + +func (p *CPUParser) Name() string { + return "cpu" +} + +func (p *CPUParser) Parse(scanner *bufio.Scanner) (map[string]string, error) { + resMap := make(map[string]string, 0) + + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + + if !strings.HasPrefix(parts[0], "cpu") { + continue + } + + //var steal, guest, guestNice int64 + //if len(parts) > 8 { + // steal = util.StringToInt64(parts[8]) + //} + //if len(parts) > 9 { + // guest = util.StringToInt64(parts[9]) + //} + if 10 >= len(parts) { + //guestNice = util.StringToInt64(parts[10]) + continue + } + + //stats = append(stats, CPUStat{ + //Device := util.StringToInt64(parts[0]) + // User := util.StringToInt64(parts[1]) + // Nice := util.StringToInt64(parts[2]) + // System := util.StringToInt64(parts[3]) + // Idle := util.StringToInt64(parts[4]) + // Iowait := util.StringToInt64(parts[5]) + // Irq := util.StringToInt64(parts[6]) + // SoftIrq := util.StringToInt64(parts[7]) + // Steal := util.StringToInt64(parts[8]) + // Guest := util.StringToInt64(parts[9]) + // GuestNice := util.StringToInt64(parts[10]) + //}) + + // sum := User + Nice + System + Idle + Iowait + Irq + SoftIrq + Steal + Guest + GuestNice + sum := sumFromStrings(parts, 1, 11, nil) + + resMap["sum"] = strconv.FormatInt(sum, 10) + resMap["user"] = parts[1] + resMap["nice"] = parts[2] + resMap["system"] = parts[3] + resMap["idle"] = parts[4] + resMap["iowait"] = parts[5] + resMap["irq"] = parts[6] + resMap["softIrq"] = parts[7] + resMap["steal"] = parts[8] + resMap["guest"] = parts[9] + resMap["gnice"] = parts[10] + + break // first line only --- cpu + } + + //res, err := cpu.parse(keys, stats) + //if err != nil { + // return nil, err + //} + + return resMap, scanner.Err() + +} + +func sumFromStrings(ss []string, startIndex, endIndex int, exclude []int) int64 { + if nil == ss { + return 0 + } + + var result int64 +Loop: + for i := startIndex; i < endIndex; i++ { + if nil != exclude { + for j := 0; j < len(exclude); j++ { + if exclude[j] == i { + continue Loop + } + } + } + i64, _ := strconv.ParseInt(ss[i], 10, 64) + result += i64 + } + + return result +} diff --git a/crawler/impl/ssh/parser/memory_parser.go b/crawler/impl/ssh/parser/memory_parser.go new file mode 100644 index 0000000..afdc21f --- /dev/null +++ b/crawler/impl/ssh/parser/memory_parser.go @@ -0,0 +1,34 @@ +package parser + +import ( + "bufio" + "strings" +) + +type MemoryParser struct { +} + +func (p *MemoryParser) Name() string { + return "mem" +} + +func (p *MemoryParser) Parse(scanner *bufio.Scanner) (map[string]string, error) { + var ( + stats = map[string]string{} + ) + + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + key := parts[0][:len(parts[0])-1] + + stats[key] = parts[1] + } + + //res, err := mem.parse(keys, stats) + //if err != nil { + // return nil, err + //} + + return stats, scanner.Err() +} diff --git a/crawler/impl/ssh/parser/parser.go b/crawler/impl/ssh/parser/parser.go new file mode 100644 index 0000000..3733c9d --- /dev/null +++ b/crawler/impl/ssh/parser/parser.go @@ -0,0 +1,29 @@ +package parser + +import "bufio" + +var parsers map[string]Parser + +func init() { + parsers = make(map[string]Parser, 0) + + addParser(&CPUParser{}) + addParser(&MemoryParser{}) +} + +func addParser(p Parser) { + parsers[p.Name()] = p +} + +func GetParser(name string) Parser { + p, ok := parsers[name] + if !ok { + return nil + } + return p +} + +type Parser interface { + Name() string + Parse(scanner *bufio.Scanner) (map[string]string, error) +}