commit fe424d070d9ad5947d00b4744fae9201df6cf131 Author: snoop Date: Thu Aug 3 19:08:34 2017 +0900 first probe diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98d2148 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +# Created by .ignore support plugin (hsz.mobi) +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### Go template +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ +.idea/ +*.iml + +vendor/ +glide.lock +.DS_Store +dist/ +debug \ No newline at end of file diff --git a/agent_api/config_manager/config.go b/agent_api/config_manager/config.go new file mode 100644 index 0000000..72e3902 --- /dev/null +++ b/agent_api/config_manager/config.go @@ -0,0 +1,53 @@ +package config_manager + +type Connection struct { + Ip string `json:"ip"` + Port string `json:"port"` + PortType string `json:"portType"` + SSL bool `json:"ssl"` +} + +type Target struct { + Auth map[string]interface{} `json:"auth"` + Connection Connection `json:"connection"` +} + +type Schedule struct { + Interval string `json:"interval"` +} + +type Keys struct { + Metric string `json:"metric"` + Key string `json:"key"` +} + +type QueryInfo struct { + Query string `json:"query"` + Extend map[string]interface{} `json:"extend"` +} + +type MappingInfo struct { + ParseDirection string `json:"parseDirection"` + ArrayColumns []string `json:"arrayColumns"` + KeyColumns []string `json:"keyColumns"` + ValueColumn string `json:"valueColumn"` +} + +type Item struct { + Keys []Keys `json:"keys"` + QueryInfo QueryInfo `json:"queryInfo"` + MappingInfo MappingInfo `json:"mappingInfo"` +} + +type Crawler struct { + Name string `json:"name"` + Container string `json:"container"` +} + +type Config struct { + Id string `json:"id"` + Target Target `json:"target"` + Schedule Schedule `json:"schedule"` + Crawler Crawler `json:"crawler"` + Items []Item `json:"items"` +} diff --git a/agent_api/config_manager/config_manager.go b/agent_api/config_manager/config_manager.go new file mode 100644 index 0000000..0ca17e5 --- /dev/null +++ b/agent_api/config_manager/config_manager.go @@ -0,0 +1,7 @@ +package config_manager + +type ConfigManager interface { + GetGlobalConfig() *GlobalConfig + GetSensorById(id string) *Config + GetSensors() map[string]*Config +} diff --git a/agent_api/config_manager/config_test.go b/agent_api/config_manager/config_test.go new file mode 100644 index 0000000..0b007c5 --- /dev/null +++ b/agent_api/config_manager/config_test.go @@ -0,0 +1,54 @@ +package config_manager + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "io/ioutil" + "log" + "testing" +) + +func TestConfigRead(t *testing.T) { + + b, err := ioutil.ReadFile("/root/gowork/src/loafle.com/overflow/agent_api/config_manager/test.json") + if err != nil { + log.Panic(err) + } + var m = Config{} + json.Unmarshal(b, &m) + + assert.Equal(t, m.Id, "123980918237") + assert.Equal(t, m.Target.Connection.Ip, "192.168.1.103") + assert.Equal(t, m.Target.Connection.Port, "1433") + assert.Equal(t, m.Target.Connection.SSL, false) + assert.Equal(t, m.Target.Connection.PortType, "tcp") + + assert.Equal(t, m.Target.Auth["url"], "jdbc:sqlserver://192.168.1.106:1433;") + assert.Equal(t, m.Target.Auth["id"], "sa") + assert.Equal(t, m.Target.Auth["pw"], "qwe123") + assert.Equal(t, m.Target.Auth["query"], "select * from master.dbo.sysprocesses") + + assert.Equal(t, m.Schedule.Interval, "10") + + assert.Equal(t, m.Crawler.Name, "health_activedirectory") + assert.Equal(t, m.Crawler.Container, "network_proxy") + + item := m.Items[0] + + assert.Equal(t, item.Keys[0].Metric, "object[$0].db[$1].datafile_size") + assert.Equal(t, item.Keys[0].Key, "Data File(s) Size (KB)") + + assert.Equal(t, item.Keys[1].Metric, "object[$0].db[$1].logfile_size") + assert.Equal(t, item.Keys[1].Key, "Log File(s) Size (KB)") + + assert.Equal(t, item.QueryInfo.Query, "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'") + assert.Equal(t, item.QueryInfo.Extend["test"], "test") + + assert.Equal(t, item.MappingInfo.ParseDirection, "row") + assert.Equal(t, item.MappingInfo.ArrayColumns[0], "object_name") + assert.Equal(t, item.MappingInfo.ArrayColumns[1], "instance_name") + + assert.Equal(t, item.MappingInfo.KeyColumns[0], "counter_name") + assert.Equal(t, item.MappingInfo.ValueColumn, "cntr_value") + +} diff --git a/agent_api/config_manager/global_config.go b/agent_api/config_manager/global_config.go new file mode 100644 index 0000000..ab5f63c --- /dev/null +++ b/agent_api/config_manager/global_config.go @@ -0,0 +1,31 @@ +package config_manager + +/* global config 예제 +central: + address: "http://localhost:9090" + port: 443 +log_Path: "./bin/log.xml" +paths: + rootFolder : "/home/cm2/" + configFolder : "config/" + binaryFolder : "container/" + pidFolder : "pids/" + scriptFile : "start" +intervalSecond: 10 +*/ + +type GlobalConfig struct { + Central struct { + Address string `yaml:"address"` + Port int `yaml:"port"` + } + LogPath string `yaml:"logPath"` + Paths struct { + RootFolder string `yaml:"rootFolder"` + ConfigFolder string `yaml:"configFolder"` + BinaryFolder string `yaml:"binaryFolder"` + PidFolder string `yaml:"pidFolder"` + ScriptFile string `yaml:"scriptFile"` + } + IntervalSecond int `yaml:"intervalSecond"` +} diff --git a/agent_api/config_manager/test.json b/agent_api/config_manager/test.json new file mode 100644 index 0000000..3bce50f --- /dev/null +++ b/agent_api/config_manager/test.json @@ -0,0 +1,50 @@ +{ + "id" : "123980918237", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "1433", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + "url":"jdbc:sqlserver://192.168.1.106:1433;", + "id":"sa", + "pw":"qwe123", + "query" : "select * from master.dbo.sysprocesses" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"health_activedirectory", + "container":"network_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric" : "object[$0].db[$1].datafile_size", + "key" : "Data File(s) Size (KB)" + }, + { + "metric" : "object[$0].db[$1].logfile_size", + "key" : "Log File(s) Size (KB)" + } + ], + "queryInfo" : { + "query": "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'", + "extend" : { + "test":"test" + } + }, + "mappingInfo" : { + "parseDirection" : "row", + "arrayColumns" : [ "object_name","instance_name"], + "keyColumns" : ["counter_name"], + "valueColumn" : "cntr_value" + } + } + ] +} \ No newline at end of file diff --git a/agent_api/crawler_manager/grpc_clientpool/grpc_clientpool.go b/agent_api/crawler_manager/grpc_clientpool/grpc_clientpool.go new file mode 100644 index 0000000..9abf6fd --- /dev/null +++ b/agent_api/crawler_manager/grpc_clientpool/grpc_clientpool.go @@ -0,0 +1,7 @@ +package grpc_clientpool + +import "google.golang.org/grpc" + +type grpc_clientpool interface { + GetClient(container string) (*grpc.ClientConn, error) +} diff --git a/agent_api/messages/const.go b/agent_api/messages/const.go new file mode 100644 index 0000000..882bb58 --- /dev/null +++ b/agent_api/messages/const.go @@ -0,0 +1,31 @@ +package messages + +const ( + INIT_AGT_STARTING = "INIT_AGT_STARTING" + + EVENT_AGT_START = "EVENT_AGT_START" + EVENT_AGT_STOP = "EVENT_AGT_STOP" + EVENT_AGT_ERROR = "EVENT_AGT_ERROR" + + TASK_SENSOR_START = "TASK_SENSOR_START" + TASK_SENSOR_STOP = "TASK_SENSOR_STOP" + TASK_SENSOR_ADD = "TASK_SENSOR_ADD" + TASK_SENSOR_REMOVE = "TASK_SENSOR_REMOVE" + TASK_SENSOR_UPDATE = "TASK_SENSOR_UPDATE" + TASK_CRAWLER_UPDATE = "TASK_CRAWLER_UPDATE" + TASK_AGENT_UPDATE = "TASK_AGENT_UPDATE" + TASK_LOG_SEND = "TASK_LOG_SEND" + TASK_POL_INTERVAL_UPDATE = "TASK_POLLER_INTERVAL_UPDATE" + + DISCOVERY_START = "DISCOVERY_START" + DISCOVERY_HOST_START = "DISCOVERY_HOST_START" + DISCOVERY_HOST_FOUND = "DISCOVERY_HOST_FOUND" + DISCOVERY_HOST_DONE = "DISCOVERY_HOST_DONE" + DISCOVERY_PORT_START = "DISCOVERY_PORT_START" + DISCOVERY_PORT_FOUND = "DISCOVERY_PORT_FOUND" + DISCOVERY_PORT_DONE = "DISCOVERY_PORT_DONE" + DISCOVERY_SERVICE_START = "DISCOVERY_SERVICE_START" + DISCOVERY_SERVICE_FOUND = "DISCOVERY_SERVICE_FOUND" + DISCOVERY_SERVICE_DONE = "DISCOVERY_SERVICE_DONE" + DISCOVERY_DONE = "DISCOVERY_DONE" +) diff --git a/agent_api/messages/data.go b/agent_api/messages/data.go new file mode 100644 index 0000000..b2cf1ff --- /dev/null +++ b/agent_api/messages/data.go @@ -0,0 +1,9 @@ +package messages + +type Data struct { + AgentId string + SensorId string + Data []byte + StartedAt uint64 + FinishedAt uint64 +} diff --git a/agent_api/messages/event.go b/agent_api/messages/event.go new file mode 100644 index 0000000..aeb8289 --- /dev/null +++ b/agent_api/messages/event.go @@ -0,0 +1,12 @@ +package messages + +type Event struct { + Type int + Id string + Data []byte +} + +const ( + EVT_TYPE_TASK = 1 + EVT_TYPE_NONE = 2 +) diff --git a/agent_api/messages/task.go b/agent_api/messages/task.go new file mode 100644 index 0000000..daf7773 --- /dev/null +++ b/agent_api/messages/task.go @@ -0,0 +1,6 @@ +package messages + +type AgentTask struct { + Command string + Params map[string]string +} diff --git a/agent_api/observer/observer.go b/agent_api/observer/observer.go new file mode 100644 index 0000000..c2f38fd --- /dev/null +++ b/agent_api/observer/observer.go @@ -0,0 +1,100 @@ +package observer + +import ( + "errors" + "sync" +) + +type Observer interface { + Add(id string, ch chan interface{}) error + Remove(id string, rch chan interface{}) error + Notify(id string, arg interface{}) error +} + +type observer struct { + mtx sync.Mutex + events map[string][]chan interface{} +} + +func (o *observer) Add(id string, ch chan interface{}) error { + + o.mtx.Lock() + defer o.mtx.Unlock() + + _, ok := o.events[id] + if ok { + o.events[id] = append(o.events[id], ch) + } else { + arr := make([]chan interface{}, 0) + arr = append(arr, ch) + o.events[id] = arr + } + + return nil +} +func (o *observer) Remove(id string, rch chan interface{}) error { + o.mtx.Lock() + defer o.mtx.Unlock() + + newArr := make([]chan interface{}, 0) + chans, ok := o.events[id] + if !ok { + return errors.New("event not found") + } + + for _, ch := range chans { + if ch != rch { + newArr = append(newArr, ch) + } else { + close(ch) + ch = nil + } + } + if len(newArr) == 0 { + delete(o.events, id) + } else { + o.events[id] = newArr + } + return nil +} +func (o *observer) Notify(id string, arg interface{}) error { + o.mtx.Lock() + defer o.mtx.Unlock() + + chans, ok := o.events[id] + if !ok { + return errors.New("event not found") + } + + for _, ch := range chans { + ch <- arg + } + return nil +} + +var _observer *observer +var once sync.Once + +func init() { + once.Do(func() { + _observer = &observer{ + events: make(map[string][]chan interface{}), + } + }) +} + +// interface + +func GetInstance() Observer { + return _observer +} + +func Add(id string, ch chan interface{}) error { + return _observer.Add(id, ch) +} +func Remove(id string, rch chan interface{}) error { + return _observer.Remove(id, rch) +} +func Notify(id string, arg interface{}) error { + return _observer.Notify(id, arg) +} diff --git a/agent_api/observer/observer_test.go b/agent_api/observer/observer_test.go new file mode 100644 index 0000000..48ecf60 --- /dev/null +++ b/agent_api/observer/observer_test.go @@ -0,0 +1,113 @@ +package observer + +import ( + "fmt" + "git.loafle.net/overflow/agent_api/messages" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +type Test struct { + Id string +} + +func TestAddNotifyObserver(t *testing.T) { + + // others package notify call + go func() { + time.Sleep(1 * time.Second) + Notify("test", Test{Id: "test"}) + }() + + ch := make(chan interface{}, 0) + Add("test", ch) + + dd := <-ch + bb := dd.(Test) + assert.Equal(t, "test", bb.Id) +} + +func TestRemoveObserver(t *testing.T) { + + ch := make(chan interface{}, 0) + Add("test", ch) + + err := Remove("test", ch) + assert.Equal(t, nil, err) +} + +// +build ignore +func TestMultiAddObserver(t *testing.T) { + ch1 := make(chan interface{}, 0) + ch2 := make(chan interface{}, 0) + ch3 := make(chan interface{}, 0) + ch4 := make(chan interface{}, 0) + ch5 := make(chan interface{}, 0) + + go func() { + d := <-ch1 + dd := d.(Test) + assert.Equal(t, dd.Id, "test") + }() + + go func() { + d := <-ch2 + dd := d.(Test) + assert.Equal(t, dd.Id, "test") + }() + + go func() { + d := <-ch3 + dd := d.(Test) + assert.Equal(t, dd.Id, "test") + }() + + go func() { + d := <-ch4 + dd := d.(Test) + assert.Equal(t, dd.Id, "test") + }() + + go func() { + d := <-ch4 + dd := d.(Test) + assert.Equal(t, dd.Id, "test") + }() + + Add("test", ch1) + Add("test", ch2) + Add("test", ch3) + Add("test", ch4) + Add("test", ch5) + + // others package notify call + go func() { + time.Sleep(1 * time.Second) + Notify("test", Test{Id: "test"}) + + time.Sleep(2 * time.Second) + Remove(messages.CRAWLER_ADD, ch1) + Remove("test", ch2) + Remove("test", ch3) + Remove("test", ch4) + Remove("test", ch5) + }() + + time.Sleep(1 * time.Second) + +} + +func TestStringNotify(t *testing.T) { + ch := make(chan interface{}, 0) + Add("test", ch) + + go func() { + data := <-ch + str := data.(string) + fmt.Println(str) + }() + + Notify("test", "testsetasetaset") + time.Sleep(1 * time.Second) +} diff --git a/bootstrap/shell.go b/bootstrap/shell.go new file mode 100644 index 0000000..b208962 --- /dev/null +++ b/bootstrap/shell.go @@ -0,0 +1,77 @@ +package bootstrap + +import ( + log "github.com/cihub/seelog" + "net" + "os" + "strings" +) + +const FILE_SERVER string = "/var/run/of_server" + +var fd net.Conn + +func HandleShell(stop chan bool) { + go func() { + //os.Stdout.Close() + //os.Stderr.Close() + //os.Stdin.Close() + + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: FILE_SERVER, Net: "unix"}) + if err != nil { + log.Error(err.Error()) + return + } + defer func() { + l.Close() + os.Remove(FILE_SERVER) + }() + + for { + fd, err = l.Accept() + if err != nil { + log.Error(err) + return + } + + buf := make([]byte, 1024) + nr, err := fd.Read(buf) + if err != nil { + return + } + + data := string(buf[0:nr]) + switch strings.ToUpper(data) { + case "STOP": + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + stop <- true + case "STATUS": + fd.Write(status()) + default: + } + + fd.Close() + } + }() +} + +func StopHandleShell() { + log.Info("Shell Handler stopped.") + + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + if fd != nil { + fd.Close() + } +} + +func status() []byte { + return []byte("STATUS OK\n") +} diff --git a/bootstrap/signal.go b/bootstrap/signal.go new file mode 100644 index 0000000..2ff95d7 --- /dev/null +++ b/bootstrap/signal.go @@ -0,0 +1,53 @@ +package bootstrap + +import ( + log "github.com/cihub/seelog" + "os" + "os/signal" + "syscall" +) + +func HandleSignal(stop chan bool) { + sigs := make(chan os.Signal, 1) + + signal.Notify(sigs, + os.Kill, + os.Interrupt, + syscall.SIGKILL, + syscall.SIGSTOP, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + + go func() { + for { + s := <-sigs + switch s { + case os.Kill, os.Interrupt, syscall.SIGSTOP, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: + log.Infof("Signal received. [%s]", s) + stop <- true + default: + log.Infof("Signal received. [%s]", s) + stop <- true + } + } + }() +} + +/* + 1) SIGHUP 2) SIGINT 3) SIGQUIT 4) SIGILL 5) SIGTRAP + 6) SIGABRT 7) SIGBUS 8) SIGFPE 9) SIGKILL 10) SIGUSR1 +11) SIGSEGV 12) SIGUSR2 13) SIGPIPE 14) SIGALRM 15) SIGTERM +16) SIGSTKFLT 17) SIGCHLD 18) SIGCONT 19) SIGSTOP 20) SIGTSTP +21) SIGTTIN 22) SIGTTOU 23) SIGURG 24) SIGXCPU 25) SIGXFSZ +26) SIGVTALRM 27) SIGPROF 28) SIGWINCH 29) SIGIO 30) SIGPWR +31) SIGSYS 34) SIGRTMIN 35) SIGRTMIN+1 36) SIGRTMIN+2 37) SIGRTMIN+3 +38) SIGRTMIN+4 39) SIGRTMIN+5 40) SIGRTMIN+6 41) SIGRTMIN+7 42) SIGRTMIN+8 +43) SIGRTMIN+9 44) SIGRTMIN+10 45) SIGRTMIN+11 46) SIGRTMIN+12 47) SIGRTMIN+13 +48) SIGRTMIN+14 49) SIGRTMIN+15 50) SIGRTMAX-14 51) SIGRTMAX-13 52) SIGRTMAX-12 +53) SIGRTMAX-11 54) SIGRTMAX-10 55) SIGRTMAX-9 56) SIGRTMAX-8 57) SIGRTMAX-7 +58) SIGRTMAX-6 59) SIGRTMAX-5 60) SIGRTMAX-4 61) SIGRTMAX-3 62) SIGRTMAX-2 +63) SIGRTMAX-1 64) SIGRTMAX +*/ diff --git a/collector/collector.go b/collector/collector.go new file mode 100644 index 0000000..dfdea9e --- /dev/null +++ b/collector/collector.go @@ -0,0 +1,136 @@ +package collector_go + +import ( + "context" + cm "git.loafle.net/overflow/agent_api/config_manager" + "git.loafle.net/overflow/crawler_go/grpc" + crm "git.loafle.net/overflow/crawler_manager_go" + s "git.loafle.net/overflow/scheduler_go" + "log" + "strconv" + "sync" + "time" +) + +var ( + instance *Collector + once sync.Once +) + +type Collector struct { + scheduler *s.Scheduler + cm cm.ConfigManager + dataCh chan interface{} +} + +func Start(started chan bool, dataCh chan interface{}, conf cm.ConfigManager) { + c := GetInstance() + c.dataCh = dataCh + c.start(started, conf) +} + +func Stop(stopped chan bool) { + c := GetInstance() + c.stop() + stopped <- true +} + +func GetInstance() *Collector { + once.Do(func() { + instance = &Collector{} + }) + return instance +} + +func (c *Collector) start(started chan bool, conf cm.ConfigManager) { + go func() { + c.cm = conf + c.scheduler = &s.Scheduler{} + c.scheduler.Start() + + for _, conf := range c.cm.GetSensors() { + if err := c.addSensor(conf.Id); err != nil { + log.Println(err) + } + } + started <- true + }() +} + +func (c *Collector) stop() { + c.scheduler.RemoveAllSchedule() + c.scheduler.Stop() +} + +func (c *Collector) collect(id string) { + go func() { + conf := c.cm.GetSensorById(id) + conn, err := crm.GetInstance().GetClient(conf.Crawler.Container) + if err != nil { + log.Println(err) + } + defer conn.Close() + + dc := grpc.NewDataClient(conn) + in := &grpc.Input{} + + in.Id = id + in.Name = grpc.Crawlers(grpc.Crawlers_value[conf.Crawler.Name]) + + startAt := time.Now().Unix() + out, err := dc.Get(context.Background(), in) + finishedAt := time.Now().Unix() + if err != nil { + log.Printf("Cannot collect [ID: %s] [Crawler : %s] - [Err : %s]\n", conf.Id, conf.Crawler.Name, err.Error()) + return + } + out.StartDate = startAt + out.EndDate = finishedAt + log.Printf("COLLECTED. [ID: %s] [Crawler : %s] [Result : %s]\n", conf.Id, conf.Crawler.Name, out.GetData()) + c.dataCh <- out + }() +} + +func (c *Collector) addSensor(sensorId string) error { + sensor := c.cm.GetSensorById(sensorId) + interval, err := strconv.Atoi(sensor.Schedule.Interval) + if err != nil { + return err + } + return c.scheduler.NewSchedule(sensorId, uint64(interval), c.collect) +} + +func (c *Collector) removeSensor(id string) error { + if err := c.scheduler.RemoveSchedule(id); err != nil { + return err + } + return nil +} + +func (c *Collector) updateSensor(id string) error { + err := c.removeSensor(id) + if err != nil { + return err + } + return c.addSensor(id) +} + +func AddSensor(id string) error { + return GetInstance().addSensor(id) +} + +func RemSensor(id string) error { + return GetInstance().removeSensor(id) +} + +func UpdateSensor(id string) error { + return GetInstance().updateSensor(id) +} + +func StartSensor(id string) error { + return GetInstance().scheduler.StartSchedule(id) +} + +func StopSensor(id string) error { + return GetInstance().scheduler.StopSchedule(id) +} diff --git a/collector/collector_test.go b/collector/collector_test.go new file mode 100644 index 0000000..abe5734 --- /dev/null +++ b/collector/collector_test.go @@ -0,0 +1,7 @@ +package collector_go + +import "testing" + +func TestCollect(t *testing.T) { + +} diff --git a/config_manager/config_manager.go b/config_manager/config_manager.go new file mode 100644 index 0000000..6b71764 --- /dev/null +++ b/config_manager/config_manager.go @@ -0,0 +1,188 @@ +package config_manager_go + +import ( + "encoding/json" + "git.loafle.net/overflow/agent_api/config_manager" + "gopkg.in/yaml.v2" + "io/ioutil" + "log" + "os" + "strings" + "sync" +) + +type configManager struct { + config_manager.ConfigManager // interface implements + configs map[string]*config_manager.Config + globalConfig config_manager.GlobalConfig +} + +var _configManager *configManager + +func GetInstance() *configManager { + var once sync.Once + once.Do(func() { + _configManager = &configManager{ + configs: make(map[string]*config_manager.Config, 0), + } + }) + + return _configManager + +} + +func (c *configManager) stop() {} + +func (c *configManager) getConfigPath() string { + return c.globalConfig.Paths.RootFolder + c.globalConfig.Paths.ConfigFolder +} + +func (c *configManager) getContainerPath() string { + return c.getConfigPath() + "container/" +} + +func (c *configManager) getConfigFilePath(config *config_manager.Config) string { + return c.getContainerPath() + appendSeperator(config.Crawler.Container) + appendSeperator(config.Crawler.Name) + config.Id +} + +func appendSeperator(str string) string { + if strings.LastIndex(str, "/") != len(str)-1 { + return str + "/" + } + return str +} + +func (c *configManager) loadGlobalConfig(path string) { + data, err := ioutil.ReadFile(path) + if err != nil { + } + err = yaml.Unmarshal(data, &c.globalConfig) + if err != nil { + } +} + +func (c *configManager) loadCrawlerConfigAll() error { + + root := c.getConfigPath() + files, err := ioutil.ReadDir(root) + if err != nil { + log.Panic(err) + } + for _, file := range files { + if file.IsDir() == true { + c.loadCrawlerConfig(root, file.Name()) + } + } + return nil +} + +func (c *configManager) loadCrawlerConfig(root string, dir string) { + + separator := "" + if strings.LastIndex(root, "/") != len(root)-1 { + separator = "/" + } + currentDir := root + separator + dir + files, err := ioutil.ReadDir(currentDir) + if err != nil { + log.Panic(err) + } + + for _, file := range files { + + // 디렉터리라면 재귀 / 파일이라면 설정 로드 + if file.IsDir() == true { + c.loadCrawlerConfig(currentDir, file.Name()) + } else { + b, err := ioutil.ReadFile(currentDir + separator + file.Name()) + if err != nil { + log.Panic(err) + } + var m = config_manager.Config{} + json.Unmarshal(b, &m) + c.configs[file.Name()] = &m + } + } + +} + +func (c *configManager) addConfig(tmp string) { + //for data := range c.addCh { + + path := tmp + b, err := ioutil.ReadFile(path) + if err != nil { + // error process + log.Panic(err) + } + + // 임시파일을 로드 , Config로 변환 + var m = config_manager.Config{} + err = json.Unmarshal(b, &m) + if err != nil { + // error process + log.Panic(err) + } + + // agent 폴더 / config / crawler / .. / .. / .. / 에 해당하는 파일이 있는지 확인, 있다면 삭제 + // Config 파일로 저장 + savePath := c.getConfigFilePath(&m) + ioutil.WriteFile(savePath, b, 0644) + + // tempfile remove + err = os.Remove(path) + if err != nil { + // error process + log.Panic(err) + } + + // Config 맵에 저장 + c.configs[m.Id] = &m + + // notify id + //err = observer.Notify(messages.SCF_SENSOR_ADD_DONE, m.Id) + //if err != nil { + // continue + //} + //} +} + +func (c *configManager) removeConfig(id string) { + //for data := range c.removeCh { + removeid := id + + // check exists + config, ok := c.configs[removeid] + if !ok { + return + } + + // 해당 파일 삭제 + path := c.getConfigFilePath(config) + err := os.Remove(path) + if err != nil { + // error check + log.Panic(err) + } + + // 해당 id 삭제 + delete(c.configs, removeid) + + // notify id + //err = observer.Notify("TASK_DONE", removeid) + //if err != nil { + // continue + //} + //} +} + +// implements methods +func (c *configManager) GetGlobalConfig() *config_manager.GlobalConfig { + return &c.globalConfig +} +func (c *configManager) GetSensorById(id string) *config_manager.Config { + return c.configs[id] +} +func (c *configManager) GetSensors() map[string]*config_manager.Config { + return c.configs +} diff --git a/config_manager/config_manager_test.go b/config_manager/config_manager_test.go new file mode 100644 index 0000000..98f1647 --- /dev/null +++ b/config_manager/config_manager_test.go @@ -0,0 +1,68 @@ +package config_manager_go + +import ( + "testing" + //"github.com/stretchr/testify/assert" + "encoding/json" + "git.loafle.net/overflow/agent_api/config_manager" + "git.loafle.net/overflow/agent_api/messages" + "git.loafle.net/overflow/agent_api/observer" + "github.com/google/uuid" + "io/ioutil" + "log" + "time" +) + +func TestLoadConfig(t *testing.T) { + + // notify temp channel + //ch := make(chan interface{},0) + //observer.Add(messages.CFG_LOADED,ch) + //go func() { + // data :=<- ch + // c := data.(config_manager.ConfigManager) + // cc := c.GetSensors() + // assert.NotEqual(t, len(cc),0) + //}() + + // make config manager after to load + //c := NewConfigManager() + //c.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml") + //c.loadCrawlerConfigAll() + // + //assert.NotEqual(t, len(c.configs),0) + //observer.Notify(messages.CFG_LOADED,c) + //time.Sleep(1 * time.Second) +} + +func TestAddConfig(t *testing.T) { + _configManager.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml") + + // 원본 테스트 설정 파일 로드 + b, err := ioutil.ReadFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/test.json") + if err != nil { + log.Panic(err) + } + var m = config_manager.Config{} + json.Unmarshal(b, &m) + + // uuid로 원본 파일 복제 + rid, _ := uuid.NewRandom() + m.Id = rid.String() + b, err = json.Marshal(&m) + ioutil.WriteFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/"+m.Id, b, 0644) + + // add test + observer.Notify(messages.TASK_SENSOR_ADD, "/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/"+m.Id) + time.Sleep(1 * time.Second) +} + +func TestRemoveConfig(t *testing.T) { + GetInstance() + _configManager.loadGlobalConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml") + _configManager.loadCrawlerConfigAll() + + // remove test + //observer.Notify(messages.TASK_SENSOR_REMOVE,"d0fcc7b1-43a7-4acd-a7bf-c9572a9d4c9e") + time.Sleep(1 * time.Second) +} diff --git a/config_manager/init_method.go b/config_manager/init_method.go new file mode 100644 index 0000000..c2710c2 --- /dev/null +++ b/config_manager/init_method.go @@ -0,0 +1,115 @@ +package config_manager_go + +import ( + "loafle.com/overflow/agent_api/config_manager" +) + +//import ( +// "loafle.com/overflow/agent_api/observer" +// "loafle.com/overflow/agent_api/messages" +//) +// + +func Start(ch chan *config_manager.GlobalConfig, path string) error { + _configManager = GetInstance() + _configManager.loadGlobalConfig(path + "/" + "global.yaml") + ch <- _configManager.GetGlobalConfig() + return nil +} + +func StartSensorConfig(ch chan config_manager.ConfigManager, authKey string) error { + if err := _configManager.loadCrawlerConfigAll(); err != nil { + return err + } + ch <- _configManager + return nil +} + +func Stop(ch chan bool) { + GetInstance().stop() + ch <- true +} + +func AddSensor(tmpFilePath string) { + + GetInstance().addConfig(tmpFilePath) + + ////move file + //moveFile := tmpFilePath + ////load file + // + //file, err := os.Open(moveFile) + // + //if err != nil { + // log.Panic(err) + //} + // + //b, err := ioutil.ReadFile(moveFile) + //if err != nil { + // log.Panic(err) + //} + //var m = config_manager.Config{} + //json.Unmarshal(b, &m) + //GetInstance().configs[file.Name()] = &m + +} + +func RemoveSensor(id string) { + + //remove object + // remove file + + GetInstance().removeConfig(id) + +} + +func UpdateSensor(tmpFile string) { + + //GetInstance(). + + //overwrite file + // reload file + + GetInstance().addConfig(tmpFile) + +} + +//func agentStartHandler() { +// agentStart := make(chan interface{}, 0) +// observer.Add(messages.AGT_STARTING, agentStart) +// go func() { +// data := <-agentStart +// path := data.(string) +// // load global config +// _configManager.loadGlobalConfig(path + "/" + "global.yaml") +// +// // load all crawler configs +// if err := _configManager.loadCrawlerConfigAll(); err != nil { +// // error process +// } +// +// observer.Remove(messages.AGT_STARTING, agentStart) +// observer.Notify(messages.CFG_LOADED, _configManager) +// }() +//} +// +//func agentEndHandler() { +// agentEnd := make(chan interface{}, 0) +// observer.Add(messages.AGT_WILL_STOP, agentEnd) +// go func() { +// _ = <-agentEnd +// observer.Remove(messages.AGT_WILL_STOP, agentEnd) +// observer.Remove(messages.TASK_SENSOR_ADD,_configManager.addCh) +// observer.Remove(messages.TASK_SENSOR_REMOVE,_configManager.removeCh) +// }() +//} +// +//func addSensorHandler() { +// observer.Add(messages.TASK_SENSOR_ADD,_configManager.addCh) +// go _configManager.addConfig() +//} +// +//func removeSensorHandler() { +// observer.Add(messages.TASK_SENSOR_REMOVE,_configManager.removeCh) +// go _configManager.removeConfig() +//} diff --git a/config_manager/test_agent/config/container/java_proxy/jmx/747788 b/config_manager/test_agent/config/container/java_proxy/jmx/747788 new file mode 100644 index 0000000..19b7e68 --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/jmx/747788 @@ -0,0 +1,44 @@ +{ + "id" : "747788", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "9840", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"JMX", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + {"metric" : "net.connector[$0].localPort", "key" : "localPort"}, + {"metric" : "net.connector[$0].port", "key" : "port"}, + {"metric" : "net.connector[$0].protocol", "key" : "protocol"} + ], + "queryInfo" : { + "query": "*Catalina:type=Connector,*", + "extend" : { + "aliases" : [ + { + "key": "port", + "index":0 + } + ] + } + }, + "mappingInfo" : { + "arrayColumns" : [ "portName" ] + } + } + ] +} diff --git a/config_manager/test_agent/config/container/java_proxy/mongodb/26021802 b/config_manager/test_agent/config/container/java_proxy/mongodb/26021802 new file mode 100644 index 0000000..e111958 --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/mongodb/26021802 @@ -0,0 +1,43 @@ +{ + "id" : "26021802", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "27017", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"MONGODB", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + {"metric":"memory.usage.bits", "key":"bits"}, + {"metric":"memory.usage.rss", "key":"resident"}, + {"metric":"memory.usage.vmem", "key":"virtual"}, + {"metric":"memory.usage.supported", "key":"supported"}, + {"metric":"memory.usage.mapped", "key":"mapped"}, + {"metric":"memory.usage.mappedWithJournal", "key":"mappedWithJournal"} + ], + "queryInfo" : { + "query": "mem", + "extend" : { + "dataBaseName":"admin", + "statusCommand": "serverStatus" + } + }, + "mappingInfo" : { + "parseDirection" : "col" + } + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/redis/736387363 b/config_manager/test_agent/config/container/java_proxy/redis/736387363 new file mode 100644 index 0000000..9d971cb --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/redis/736387363 @@ -0,0 +1,67 @@ +{ + "id" : "736387363", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "6379", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"REDIS", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric":"cpu.usage.system", + "key":"used_cpu_sys" + }, + { + "metric":"cpu.usage.user", + "key":"used_cpu_user" + }, + { + "metric":"cpu.usage.system_children", + "key":"used_cpu_sys_children" + }, + { + "metric":"cpu.usage.user_children", + "key":"used_cpu_user_children" + } + ], + "queryInfo" : { + "query" : "CPU" + }, + "mappingInfo" : {} + }, + { + "keys" : [ + { + "metric":"memory.usage.used", + "key":"used_memory" + }, + { + "metric":"memory.usage.rss", + "key":"used_memory_rss" + }, + { + "metric":"memory.usage.peak", + "key":"used_memory_peak" + } + ], + "queryInfo" : { + "query" : "Memory" + }, + "mappingInfo" : {} + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/snmp/68686868 b/config_manager/test_agent/config/container/java_proxy/snmp/68686868 new file mode 100644 index 0000000..a61e6ed --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/snmp/68686868 @@ -0,0 +1,42 @@ +{ + "id" : "68686868", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "161", + "ssl" : false, + "portType" : "udp" + }, + "auth" : { + "version" : "v3", + "user" : "loafle", + "authType" : "MD5", + "authPass" : "qwer5795", + "privType" : "DES", + "privPass" : "qweqwe123" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"SNMP", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + {"metric":"system.uptime", "key":"1.3.6.1.2.1.1.3.0"} + ], + "queryInfo" : { + "query": "mem", + "extend" : { + "method": "get" + } + }, + "mappingInfo" : { + "parseDirection" : "col" + } + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/snmp/86868686 b/config_manager/test_agent/config/container/java_proxy/snmp/86868686 new file mode 100644 index 0000000..74eb2df --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/snmp/86868686 @@ -0,0 +1,38 @@ +{ + "id" : "86868686", + "target" : { + "connection" : { + "ip" : "192.168.1.254", + "port" : "161", + "ssl" : false, + "portType" : "udp" + }, + "auth" : { + "version":"v2c", + "community" : "loafle" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"SNMP", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + {"metric":"system.uptime", "key":"1.3.6.1.2.1.1.3.0"} + ], + "queryInfo" : { + "query": "mem", + "extend" : { + "method": "get" + } + }, + "mappingInfo" : { + "parseDirection" : "col" + } + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/sql/444441122312 b/config_manager/test_agent/config/container/java_proxy/sql/444441122312 new file mode 100644 index 0000000..e2f7901 --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/sql/444441122312 @@ -0,0 +1,44 @@ +{ + "id" : "444441122312", + "target" : { + "connection" : { + "ip" : "192.168.1.104", + "port" : "6379", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + "url":"jdbc:mysql://192.168.1.215:3306", + "id":"root", + "pw":"qwe123" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"SQL", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric":"net.mysql.connection_count", + "key":"Connections" + } + + ], + "queryInfo" : { + "query":"show status where variable_name = 'Connections'" + }, + "mappingInfo" : { + "parseDirection" : "row", + "valueColumn" : "Value", + "keyColumns" : [ + "Variable_name" + ] + } + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/sql/777999444 b/config_manager/test_agent/config/container/java_proxy/sql/777999444 new file mode 100644 index 0000000..ab3eebb --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/sql/777999444 @@ -0,0 +1,38 @@ +{ + "id" : "777999444", + "target" : { + "connection" : { + "ip" : "192.168.1.106", + "port" : "1433", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + "url":"jdbc:sqlserver://192.168.1.106:1433;", + "id":"sa", + "pw":"qwe123" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"SQL", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric" :"net.sqlserver.connection_count", + "key" : "connection_count" + } + ], + "queryInfo" : { + "query": "select count('session_id') as 'connection_count' from sys.dm_exec_connections where session_id = @@SPID" + }, + "mappingInfo" : {} + } + + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/java_proxy/sql/989238744 b/config_manager/test_agent/config/container/java_proxy/sql/989238744 new file mode 100644 index 0000000..cbdb3e0 --- /dev/null +++ b/config_manager/test_agent/config/container/java_proxy/sql/989238744 @@ -0,0 +1,37 @@ +{ + "id" : "989238744", + "target" : { + "connection" : { + "ip" : "192.168.1.106", + "port" : "5432", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + "url":"jdbc:postgresql://192.168.1.106:5432/postgres", + "id":"vertx", + "pw":"qwe123" + } + }, + "schedule" : { + "interval" : "3" + }, + "crawler" : { + "name":"SQL", + "container":"java_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric":"net.pgsql.connection_count", + "key" : "connection_count" + } + ], + "queryInfo":{ + "query" : "select count(pid) as connection_count from pg_catalog.pg_stat_activity where state <> 'idle';" + }, + "mappingInfo" : {} + } + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_activedirectory/123980918237 b/config_manager/test_agent/config/container/network_proxy/health_activedirectory/123980918237 new file mode 100644 index 0000000..613e98e --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_activedirectory/123980918237 @@ -0,0 +1,23 @@ +{ + "id" : "123980918237", + "target" : { + "connection" : { + "ip" : "192.168.1.1", + "port" : "389", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"HEALTH_ACTIVEDIRECTORY", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_cassandra/292513 b/config_manager/test_agent/config/container/network_proxy/health_cassandra/292513 new file mode 100644 index 0000000..97ec650 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_cassandra/292513 @@ -0,0 +1,23 @@ +{ + "id" : "292513", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "165", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_CASSANDRA", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_ftp/1352231 b/config_manager/test_agent/config/container/network_proxy/health_ftp/1352231 new file mode 100644 index 0000000..f5855c1 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_ftp/1352231 @@ -0,0 +1,23 @@ +{ + "id" : "1352231", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "21", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_FTP", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_ftps/135223179 b/config_manager/test_agent/config/container/network_proxy/health_ftps/135223179 new file mode 100644 index 0000000..bf68a0d --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_ftps/135223179 @@ -0,0 +1,23 @@ +{ + "id" : "135223179", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "2121", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_FTPS", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_http/104116116112 b/config_manager/test_agent/config/container/network_proxy/health_http/104116116112 new file mode 100644 index 0000000..701ddbe --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_http/104116116112 @@ -0,0 +1,23 @@ +{ + "id" : "104116116112", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "80", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_HTTP", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_https/104116116112115 b/config_manager/test_agent/config/container/network_proxy/health_https/104116116112115 new file mode 100644 index 0000000..1ee2942 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_https/104116116112115 @@ -0,0 +1,23 @@ +{ + "id" : "104116116112115", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "443", + "ssl" : true, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_HTTPS", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_imap/10510997112 b/config_manager/test_agent/config/container/network_proxy/health_imap/10510997112 new file mode 100644 index 0000000..146bdd0 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_imap/10510997112 @@ -0,0 +1,23 @@ +{ + "id" : "10510997112", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "993", + "ssl" : true, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_IMAP", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_ldap/10810097112 b/config_manager/test_agent/config/container/network_proxy/health_ldap/10810097112 new file mode 100644 index 0000000..ea502ef --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_ldap/10810097112 @@ -0,0 +1,24 @@ +{ + "id" : "10810097112", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "993", + "ssl" : true, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_LDAP", + "container":"network_proxy" + }, + "items" : [ + ] +} + diff --git a/config_manager/test_agent/config/container/network_proxy/health_mongodb/10911111010311110098 b/config_manager/test_agent/config/container/network_proxy/health_mongodb/10911111010311110098 new file mode 100644 index 0000000..54078bb --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_mongodb/10911111010311110098 @@ -0,0 +1,23 @@ +{ + "id" : "10911111010311110098", + "target" : { + "connection" : { + "ip" : "192.168.1.104", + "port" : "27017", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_MONGODB", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_mssql/109115115113108 b/config_manager/test_agent/config/container/network_proxy/health_mssql/109115115113108 new file mode 100644 index 0000000..e5a56f0 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_mssql/109115115113108 @@ -0,0 +1,24 @@ +{ + "id" : "109115115113108", + "target" : { + "connection" : { + "ip" : "192.168.1.106", + "port" : "1433", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_MSSQL", + "container":"network_proxy" + }, + "items" : [ + ] +} + diff --git a/config_manager/test_agent/config/container/network_proxy/health_mysql/109121115113108 b/config_manager/test_agent/config/container/network_proxy/health_mysql/109121115113108 new file mode 100644 index 0000000..1b46c67 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_mysql/109121115113108 @@ -0,0 +1,23 @@ +{ + "id" : "109121115113108", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "3306", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_MYSQL", + "container":"network_proxy" + }, + "items" : [ + ] +} diff --git a/config_manager/test_agent/config/container/network_proxy/health_netbios/11010111698105111115 b/config_manager/test_agent/config/container/network_proxy/health_netbios/11010111698105111115 new file mode 100644 index 0000000..af82a20 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_netbios/11010111698105111115 @@ -0,0 +1,25 @@ +{ + "id" : "11010111698105111115", + "target" : { + "connection" : { + "ip" : "192.168.1.106", + "port" : "139", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_NETBIOS", + "container":"network_proxy" + }, + "items" : [ + ] +} + + diff --git a/config_manager/test_agent/config/container/network_proxy/health_oracle/1111149799108101 b/config_manager/test_agent/config/container/network_proxy/health_oracle/1111149799108101 new file mode 100644 index 0000000..e6033d9 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_oracle/1111149799108101 @@ -0,0 +1,25 @@ +{ + "id" : "1111149799108101", + "target" : { + "connection" : { + "ip" : "192.168.1.30", + "port" : "1521", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_ORACLE", + "container":"network_proxy" + }, + "items" : [ + ] +} + + diff --git a/config_manager/test_agent/config/container/network_proxy/health_pgsql/112103115113108 b/config_manager/test_agent/config/container/network_proxy/health_pgsql/112103115113108 new file mode 100644 index 0000000..dd73905 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_pgsql/112103115113108 @@ -0,0 +1,23 @@ +{ + "id" : "112103115113108", + "target" : { + "connection" : { + "ip" : "192.168.1.107", + "port" : "5432", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_PGSQL", + "container":"network_proxy" + }, + "items" : [ + ] +} diff --git a/config_manager/test_agent/config/container/network_proxy/health_pop3/11211111251 b/config_manager/test_agent/config/container/network_proxy/health_pop3/11211111251 new file mode 100644 index 0000000..ca4f4eb --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_pop3/11211111251 @@ -0,0 +1,23 @@ +{ + "id" : "11211111251", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "110", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_POP3", + "container":"network_proxy" + }, + "items" : [ + ] +} diff --git a/config_manager/test_agent/config/container/network_proxy/health_rmi/114109105 b/config_manager/test_agent/config/container/network_proxy/health_rmi/114109105 new file mode 100644 index 0000000..1145c25 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_rmi/114109105 @@ -0,0 +1,25 @@ +{ + "id" : "114109105", + "target" : { + "connection" : { + "ip" : "192.168.1.103", + "port" : "9840", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_RMI", + "container":"network_proxy" + }, + "items" : [ + ] +} + + diff --git a/config_manager/test_agent/config/container/network_proxy/health_snmpv2c/684845 b/config_manager/test_agent/config/container/network_proxy/health_snmpv2c/684845 new file mode 100644 index 0000000..d319563 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_snmpv2c/684845 @@ -0,0 +1,23 @@ +{ + "id" : "684845", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "161", + "ssl" : false, + "portType" : "udp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "5" + }, + "crawler" : { + "name":"HEALTH_SNMPV2C", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_snmpv3/59797987 b/config_manager/test_agent/config/container/network_proxy/health_snmpv3/59797987 new file mode 100644 index 0000000..6b6f79e --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_snmpv3/59797987 @@ -0,0 +1,23 @@ +{ + "id" : "59797987", + "target" : { + "connection" : { + "ip" : "192.168.1.107", + "port" : "161", + "ssl" : false, + "portType" : "udp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "5" + }, + "crawler" : { + "name":"HEALTH_SNMPV3", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_ssh/115115104 b/config_manager/test_agent/config/container/network_proxy/health_ssh/115115104 new file mode 100644 index 0000000..238c750 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_ssh/115115104 @@ -0,0 +1,23 @@ +{ + "id" : "115115104", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "22", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_SSH", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_telnet/116101108110101116 b/config_manager/test_agent/config/container/network_proxy/health_telnet/116101108110101116 new file mode 100644 index 0000000..4913bd9 --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_telnet/116101108110101116 @@ -0,0 +1,23 @@ +{ + "id" : "116101108110101116", + "target" : { + "connection" : { + "ip" : "192.168.1.215", + "port" : "23", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_TELNET", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/config/container/network_proxy/health_wmi/119109105 b/config_manager/test_agent/config/container/network_proxy/health_wmi/119109105 new file mode 100644 index 0000000..6f3480b --- /dev/null +++ b/config_manager/test_agent/config/container/network_proxy/health_wmi/119109105 @@ -0,0 +1,23 @@ +{ + "id" : "119109105", + "target" : { + "connection" : { + "ip" : "192.168.1.1", + "port" : "135", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "7" + }, + "crawler" : { + "name":"HEALTH_WMI", + "container":"network_proxy" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/config_manager/test_agent/container/java_proxy/grpc_java.jar b/config_manager/test_agent/container/java_proxy/grpc_java.jar new file mode 100644 index 0000000..957a264 Binary files /dev/null and b/config_manager/test_agent/container/java_proxy/grpc_java.jar differ diff --git a/config_manager/test_agent/container/java_proxy/start b/config_manager/test_agent/container/java_proxy/start new file mode 100755 index 0000000..990f046 --- /dev/null +++ b/config_manager/test_agent/container/java_proxy/start @@ -0,0 +1,3 @@ +#!/bin/bash +script_dir=$(cd "$(dirname "$0")" && pwd) +java -jar $script_dir'/grpc_java.jar' $1 \ No newline at end of file diff --git a/config_manager/test_agent/container/network_proxy/ncr b/config_manager/test_agent/container/network_proxy/ncr new file mode 100755 index 0000000..b1343c5 Binary files /dev/null and b/config_manager/test_agent/container/network_proxy/ncr differ diff --git a/config_manager/test_agent/container/network_proxy/start b/config_manager/test_agent/container/network_proxy/start new file mode 100755 index 0000000..68ebb7b --- /dev/null +++ b/config_manager/test_agent/container/network_proxy/start @@ -0,0 +1,4 @@ +#!/bin/bash +script_dir=$(cd "$(dirname "$0")" && pwd) +$script_dir'/ncr' -Port=$1 + diff --git a/config_manager/test_agent/global.yaml b/config_manager/test_agent/global.yaml new file mode 100644 index 0000000..3c33324 --- /dev/null +++ b/config_manager/test_agent/global.yaml @@ -0,0 +1,11 @@ +central: + address: "http://localhost:9090" + port: 443 +logPath: "./bin/log.xml" +paths: + rootFolder : "/home/insanity/Develop/gopath/src/loafle.com/overflow/config_manager_go/test_agent/" + configFolder : "config/" + binaryFolder : "container/" + pidFolder : "pids/" + scriptFile : "start" +intervalSecond: 10 diff --git a/config_manager/test_agent/test.json b/config_manager/test_agent/test.json new file mode 100644 index 0000000..62402ab --- /dev/null +++ b/config_manager/test_agent/test.json @@ -0,0 +1,50 @@ +{ + "id" : "123980918237", + "target" : { + "connection" : { + "ip" : "192.168.1.1", + "port" : "389", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + "url":"jdbc:sqlserver://192.168.1.106:1433;", + "id":"sa", + "pw":"qwe123", + "query" : "select * from master.dbo.sysprocesses" + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"health_activedirectory", + "container":"network_proxy" + }, + "items" : [ + { + "keys" : [ + { + "metric" : "object[$0].db[$1].datafile_size", + "key" : "Data File(s) Size (KB)" + }, + { + "metric" : "object[$0].db[$1].logfile_size", + "key" : "Log File(s) Size (KB)" + } + ], + "queryInfo" : { + "query": "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'", + "extend" : { + "test":"test" + } + }, + "mappingInfo" : { + "parseDirection" : "row", + "arrayColumns" : [ "object_name","instance_name"], + "keyColumns" : ["counter_name"], + "valueColumn" : "cntr_value" + } + } + ] +} \ No newline at end of file diff --git a/crawler_manager/config/example.json b/crawler_manager/config/example.json new file mode 100644 index 0000000..b5158c4 --- /dev/null +++ b/crawler_manager/config/example.json @@ -0,0 +1,23 @@ +{ + "id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734", + "target" : { + "connection" : { + "ip" : "192.168.1.104", + "port" : "6379", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"HEALTH_REDIS", + "container":"network_crawler" + }, + "items" : [ + ] +} \ No newline at end of file diff --git a/crawler_manager/config/ttnc b/crawler_manager/config/ttnc new file mode 100755 index 0000000..522cc82 Binary files /dev/null and b/crawler_manager/config/ttnc differ diff --git a/crawler_manager/crawler_communicator.go b/crawler_manager/crawler_communicator.go new file mode 100644 index 0000000..4b95225 --- /dev/null +++ b/crawler_manager/crawler_communicator.go @@ -0,0 +1,196 @@ +package crawler_manager + +import ( + "context" + g "git.loafle.net/overflow/crawler_go/grpc" + "google.golang.org/grpc" + "log" + + "encoding/json" + "git.loafle.net/overflow/agent_api/config_manager" +) + +func callAdd(container *string, conf *config_manager.Config) bool { + + port := GetInstance().portMap[*container] + + conn, err := grpc.Dial(address+port, grpc.WithInsecure()) + if err != nil { + log.Fatalf("did not connect: %v", err) + return false + } + defer conn.Close() + + cc := g.NewConfigClient(conn) + + in := &g.InputAdd{} + + b, err := json.Marshal(conf) + if err != nil { + return false + } + + in.Data = b + in.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name]) + + out, err := cc.Add(context.Background(), in) + if err != nil { + log.Println(err) + } + log.Println("callAdd:", out) + + return true + +} + +func callInitConfigOne(container *string, conf *config_manager.Config) bool { + + var cl []*config_manager.Config + + cl = append(cl, conf) + + return callInitConfig(container, cl) + +} + +func callInitConfig(container *string, cl []*config_manager.Config) bool { + + port := GetInstance().portMap[*container] + + conn, err := grpc.Dial(address+port, grpc.WithInsecure()) + if err != nil { + log.Fatalf("did not connect: %v", err) + return false + } + defer conn.Close() + + cc := g.NewConfigClient(conn) + + inArr := &g.InputArray{} + + for _, conf := range cl { + + in := &g.Init{} + in.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name]) + + b, err := json.Marshal(conf) + if err != nil { + continue + } + + in.Data = b + inArr.In = append(inArr.In, in) + } + + outInit, errInit := cc.Init(context.Background(), inArr) + if errInit != nil { + log.Println(errInit) + return false + } + log.Println("callInit:", outInit) + + return true + +} + +//func callInit(container *string, paths *[]string) bool { +// +// port := GetInstance().portMap[*container] +// +// return callInitAddress(address + port, paths) +//} +// + +// +//func callInitAddress(address string, paths *[]string) bool { +// conn, err := grpc.Dial(address, grpc.WithInsecure()) +// if err != nil { +// log.Fatalf("did not connect: %v", err) +// return false +// } +// defer conn.Close() +// +// cc := g.NewConfigClient(conn) +// +// inArr := &g.InputArray{} +// base := "HEALTH_" +// for _, path := range *paths { +// +// in := &g.Init{} +// //in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/" +// in.Path = path + "/" +// bcn := filepath.Base(path) +// bcn = strings.ToUpper(bcn) +// +// in.Name = g.Crawlers(g.Crawlers_value[base+bcn]) +// //in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test +// inArr.In = append(inArr.In, in) +// } +// +// outInit, errInit := cc.Init(context.Background(), inArr) +// if errInit != nil { +// log.Println(errInit) +// return false +// } +// log.Println("callInit:",outInit) +// +// return true +//} + +func callRemove(container *string, conf *config_manager.Config) { + + port := GetInstance().portMap[*container] + + conn, err := grpc.Dial(address+port, grpc.WithInsecure()) + + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + cc := g.NewConfigClient(conn) + + inR := &g.Input{} + //inR.Id = *id //FIXME + inR.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name]) + + outRem, errRem := cc.Remove(context.Background(), inR) + if errRem != nil { + log.Println(errRem) + } + log.Println("callRemove:", outRem) + +} + +func callStatus(container *string) bool { + + port := GetInstance().portMap[*container] + + if port == "" { + return false + } + + return callStatusAddress(address + port) + +} + +func callStatusAddress(addr string) bool { + + conn, err := grpc.Dial(addr, grpc.WithInsecure()) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := g.NewStatusClient(conn) + + e := &g.Empty{} + out, err := c.Status(context.Background(), e) + + if err != nil { + log.Fatalf("could not greet: %v", err) + } + + return out.Check + +} diff --git a/crawler_manager/crawler_manager.go b/crawler_manager/crawler_manager.go new file mode 100644 index 0000000..67920b6 --- /dev/null +++ b/crawler_manager/crawler_manager.go @@ -0,0 +1,327 @@ +package crawler_manager + +import ( + //"git.loafle.net/overflow/crawler_go/config" + //"encoding/json" + "google.golang.org/grpc" + "io/ioutil" + "log" + "os" + "os/exec" + "strconv" + "time" + + "errors" + "git.loafle.net/overflow/agent_api/config_manager" +) + +const ( + address = "localhost:" + portArgsName = "-Port=" + defaultPort = 50000 + + //rootFolder = "/home/cm2/" + //ConfigFolder = rootFolder + "/config/container/" + //BinaryFolder = rootFolder + "/container/" + //PidFolder = rootFolder + "/pids/" + //runFile = "ttnc" +) + +var g_CrawlerMananger *CrawlerManager = nil + +type CrawlerManager struct { + currentPort int + portMap map[string]string + pidMap map[string]string + ConfigMgr config_manager.ConfigManager +} + +func init() { + + GetInstance() + + listenEvent() + + //g_CrawlerMananger.init() +} + +func SettingPath() { + + //test + //GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder = "/home/cm2/" + + GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder + +} + +func GetInstance() *CrawlerManager { + + if g_CrawlerMananger == nil { + g_CrawlerMananger = &CrawlerManager{portMap: make(map[string]string), pidMap: make(map[string]string), currentPort: defaultPort} + } + + return g_CrawlerMananger +} + +func (c *CrawlerManager) GetClient(container string) (*grpc.ClientConn, error) { + + b := c.checkContainer(&container) + + if b == false { + err := c.runAndInitContainerOne(&container) + if err != nil { + return nil, err + } + } + gport := c.portMap[container] + + return grpc.Dial(address+gport, grpc.WithInsecure()) +} + +func (c *CrawlerManager) init() error { + + c.checkPid() + + cmap := c.ConfigMgr.GetSensors() + scm := sortContainer(cmap) + + for ctn := range scm { + err := c.runAndInitContainer(&ctn, scm[ctn]) + if err != nil { + return err + } + } + + return nil +} + +func (c *CrawlerManager) checkContainer(container *string) bool { + return callStatus(container) +} + +func (c *CrawlerManager) checkAndRunContainer(container *string) bool { + + b := callStatus(container) + + if b == false { + err := c.runAndInitContainerOne(container) + if err != nil { + return false + } + } + + return true +} + +func (c *CrawlerManager) checkPid() { + + files, err := ioutil.ReadDir(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder) + + if err != nil { + log.Println(err) + return + } + + for _, file := range files { + + if file.IsDir() { + continue + } + + str := file.Name() + + c.stopProcess(&str) + c.removeProcessFile(&str) + + } + +} + +func (c *CrawlerManager) runAndInitContainerOne(container *string) error { + + err := c.runContainer(container) + + if err != nil { + return err + } + + cmap := c.ConfigMgr.GetSensors() + scm := sortContainer(cmap) + + b := callInitConfig(container, scm[*container]) + + if b == false { + return errors.New("call init failed") + } + + return nil + +} + +func (c *CrawlerManager) runAndInitContainer(container *string, cl []*config_manager.Config) error { + + err := c.runContainer(container) + + if err != nil { + return err + } + + b := callInitConfig(container, cl) + + if b == false { + return errors.New("call init failed") + } + + return nil +} + +func sortContainer(cm map[string]*config_manager.Config) map[string][]*config_manager.Config { + + m := make(map[string][]*config_manager.Config) + + var cn string + for key := range cm { + cn = cm[key].Crawler.Container + m[cn] = append(m[cn], cm[key]) + } + + return m +} + +func (c *CrawlerManager) runContainer(container *string) error { + + b := c.checkContainer(container) + if b { + return nil + } + + cmdStr := getRunCommand(container) + + for { + pArg := strconv.Itoa(c.currentPort) + cmd := exec.Command(cmdStr, pArg) + + err := cmd.Start() + if err != nil { + //run error break; + log.Println(err) + return err + } + + time.Sleep(time.Duration(time.Second * 2)) + + b := callStatusAddress(address + strconv.Itoa(c.currentPort)) + + if b == false { + log.Println("false " + strconv.Itoa(c.currentPort)) + c.currentPort++ + continue + } + log.Println(*container+" run success port:", c.currentPort, "pid:", cmd.Process.Pid) + + writePid(cmd.Process.Pid) + + c.portMap[*container] = strconv.Itoa(c.currentPort) + c.pidMap[*container] = strconv.Itoa(cmd.Process.Pid) + + c.currentPort++ + + log.Println(*container + "started") + + break + } + + return nil +} + +func (c *CrawlerManager) addSensor(id string) error { + + conf := c.ConfigMgr.GetSensorById(id) + + b := c.checkAndRunContainer(&conf.Crawler.Name) + + if b == false { + return errors.New("run container error") + } + + b = callAdd(&conf.Crawler.Container, conf) + + if b == false { + return errors.New("Call Add Fail") + } + + return nil +} + +func (c *CrawlerManager) removeSensor(id string) { + + conf := c.ConfigMgr.GetSensorById(id) + + callRemove(&conf.Crawler.Name, conf) + + //remove and stop + +} + +func (c *CrawlerManager) updateSensor(id string) error { + + conf := c.ConfigMgr.GetSensorById(id) + + b := callInitConfigOne(&conf.Crawler.Container, conf) + + if b == false { + return errors.New("update sensor error") + } + + return nil + +} + +func (c *CrawlerManager) stopContainerAll() { + + for k, _ := range c.pidMap { + c.stopContainer(&k) + } + +} + +func (c *CrawlerManager) stopContainer(container *string) { + + pid := c.pidMap[*container] + + if len(pid) <= 0 { + return + } + + c.stopProcess(&pid) + c.removeProcessFile(&pid) + + delete(c.pidMap, *container) + delete(c.portMap, *container) + +} + +func (c *CrawlerManager) stopProcess(pid *string) { + + pidi, err := strconv.Atoi(*pid) + + if err != nil { + log.Println(err) + } + + p, err := os.FindProcess(pidi) + if err != nil { + log.Println(err) + } + p.Kill() + +} + +func (c *CrawlerManager) removeProcessFile(pid *string) { + err := os.Remove(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder + "/" + *pid) + if err != nil { + log.Println(err) + } +} diff --git a/crawler_manager/crawler_manager_event.go b/crawler_manager/crawler_manager_event.go new file mode 100644 index 0000000..8d553a9 --- /dev/null +++ b/crawler_manager/crawler_manager_event.go @@ -0,0 +1,195 @@ +package crawler_manager + +import ( + //ooo "git.loafle.net/overflow/agent_api/observer" + //"git.loafle.net/overflow/agent_api/config_manager" + //"git.loafle.net/overflow/agent_api/messages" + // + //"fmt" + "git.loafle.net/overflow/agent_api/config_manager" +) + +func Start(res chan bool, conMgr config_manager.ConfigManager) { + + GetInstance().ConfigMgr = conMgr + + SettingPath() + + err := GetInstance().init() + if err != nil { + res <- false + return + } + + res <- true +} + +func Stop(res chan bool) { + + GetInstance().stopContainerAll() + + res <- true +} + +func AddSensor(id string) error { + + err := GetInstance().addSensor(id) + + if err != nil { + return err + } + + return nil + +} + +func RemoveSensor(id string) { + + GetInstance().removeSensor(id) + + //FIXME:: error return + +} + +func UpdateSensor(id string) error { + + err := GetInstance().updateSensor(id) + + if err != nil { + return err + } + + return nil +} + +func UpdateCRM() { + + //FIXME:: update crawler + +} + +func listenEvent() { + + //go listenConfigLoaded(); + //go listenAgentStop() + // + //go listenAddSensor(); + //go listenRemoveSensor(); + //go listenUpdateSensor(); + // + //go listenUpdateCrawler() +} + +// +//func listenConfigLoaded() { +// +// ch := make(chan interface{}, 0) +// observer.Add(messages.CFG_LOADED, ch) +// +// o := <-ch +// +// cm := o.(config_manager.ConfigManager) +// +// GetInstance().ConfigMgr = cm; +// +// SettingPath() +// +// err := GetInstance().init(); +// if err != nil { +// //FIXME:: noti err +// return; +// } +// +// +// observer.Notify(messages.CRM_READY, cm) +// +//} +// +//func listenAgentStop() { +// +// ch := make(chan interface{}, 0) +// observer.Add(messages.CLT_STOPPED, ch) +// +// o := <-ch +// +// fmt.Println(o) +// +// GetInstance().stopContainerAll() +// +// observer.Notify(messages.CRM_STOPPED, nil) +// +// +//} +// +//func listenAddSensor() { +// ch := make(chan interface{}, 0) +// observer.Add(messages.SCF_SENSOR_ADD_DONE, ch); +// +// o := <-ch +// +// str := o.(string) +// +// fmt.Println(str) +// +// err := GetInstance().addSensor(str) +// +// if err != nil { +// //FIXME:: noti err +// return +// } +// +// observer.Notify(messages.CRM_SENSOR_ADD_DONE, nil) +// +//} +// +//func listenRemoveSensor() { +// +// ch := make(chan interface{}, 0) +// observer.Add(messages.CLT_SENSOR_REMOVE_DONE, ch); +// +// o := <-ch +// +// str := o.(string) +// +// GetInstance().removeSensor(str) +// +// +// observer.Notify(messages.CRM_SENSOR_REMOVE_DONE, nil) +// +//} +// +//func listenUpdateSensor() { +// +// ch := make(chan interface{}, 0) +// observer.Add(messages.SCF_SENSOR_UPDATE_DONE, ch); +// +// o := <-ch +// +// container := o.(string) +// +// err := GetInstance().runAndInitContainerOne(&container) +// +// if err != nil { +// //FIXME:: noti err +// return +// } +// +// observer.Notify(messages.CRM_SENSOR_UPDATE_DONE, nil) +// +//} +// +//func listenUpdateCrawler() { +// +// ch := make(chan interface{}, 0) +// observer.Add(messages.CLT_CRM_UPDATE_DONE, ch); +// +// o := <-ch +// +// fmt.Println(o) +// //FIXME::update crawler +// +// +// observer.Notify(messages.CRM_UPDATE_DONE, nil) +// +//} +// diff --git a/crawler_manager/crawler_manager_test.go b/crawler_manager/crawler_manager_test.go new file mode 100644 index 0000000..9e2e8fb --- /dev/null +++ b/crawler_manager/crawler_manager_test.go @@ -0,0 +1,409 @@ +package crawler_manager + +import ( + "testing" + //grpc1 "google.golang.org/grpc" + "os" + + "os/exec" + + "path/filepath" + + "io/ioutil" + "strings" + + "encoding/json" + "fmt" + "git.loafle.net/overflow/agent_api/config_manager" +) + +func TestCallInit(t *testing.T) { + //CallInit("") +} + +func TestPid(t *testing.T) { + + pp, err := os.FindProcess(12314) + if err != nil { + t.Log("err : ", err) + } + + t.Log(pp.Pid) + +} + +func TestPr(t *testing.T) { + //ps -aux | awk '{print $2}' | grep 15538 + bytes, _ := exec.Command("ps", "-aux", "awk", "{print $2}").Output() + + t.Log(string(bytes)) + +} + +func TestExe(t *testing.T) { + ps := exec.Command("ps", "-aux") + awk := exec.Command("awk", "{print $2}") + grep := exec.Command("grep", "22373") + + awk.Stdin, _ = ps.StdoutPipe() + grep.Stdin, _ = awk.StdoutPipe() + + ps.Start() + awk.Start() + + byt, _ := grep.Output() + + t.Log(len(byt)) +} + +func TestExeState(t *testing.T) { + ps := exec.Command("ps", "-aux") + awk := exec.Command("awk", "{print $2, $7}") + grep := exec.Command("grep", "22373") + awk2 := exec.Command("awk", "{print $2}") + + awk.Stdin, _ = ps.StdoutPipe() + grep.Stdin, _ = awk.StdoutPipe() + awk2.Stdin, _ = grep.StdoutPipe() + + ps.Start() + awk.Start() + grep.Start() + + byt, _ := awk2.Output() + + t.Log(len(byt)) + t.Log(string(byt)) + t.Log(len(strings.TrimSpace(string(byt)))) +} + +func TestPipe(t *testing.T) { + c1 := exec.Command("ls") + c2 := exec.Command("wc", "-l") + c2.Stdin, _ = c1.StdoutPipe() + //c2.Stdout = os.Stdout + + _ = c1.Start() + aa, _ := c2.Output() + + t.Log(string(aa)) +} + +func TestCom(t *testing.T) { + str := "Z" + + if str == "Z" { + t.Log("aaa") + } +} + +func TestReadConfig(t *testing.T) { + + //c := ReadConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json") + //t.Log(c) + +} + +func TestDir(t *testing.T) { + a := filepath.Dir("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json") + + t.Log(a) +} + +func TestMyPid(t *testing.T) { + t.Log(os.Getpid()) +} + +func TestDir22(t *testing.T) { + + //cs := IsStartContainer() + + //t.Log(cs) + +} + +func TestCrateDir(t *testing.T) { + + cp := "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/" + + var configPath []string + + rootFolder := "/home/cm2/" + ConfigFolder := rootFolder + "/config/container/" + BinaryFolder := rootFolder + "/container/" + PidFolder := rootFolder + "/pids/" + //runFile := "ttnc" + + configPath = append(configPath, ConfigFolder+"/java/oracle/") + + configPath = append(configPath, ConfigFolder+"/go/test/") + + configPath = append(configPath, ConfigFolder+"/java/mysql/") + configPath = append(configPath, ConfigFolder+"/network/http/") + configPath = append(configPath, ConfigFolder+"/network/redis/") + configPath = append(configPath, ConfigFolder+"/network/ldap/") + + configPath = append(configPath, PidFolder) + + var containerPath []string + + containerPath = append(containerPath, BinaryFolder+"/java/") + containerPath = append(containerPath, BinaryFolder+"/go/") + containerPath = append(containerPath, BinaryFolder+"/network/") + + b, _ := ioutil.ReadFile(cp + "config/example.json") + + for i, p := range configPath { + pp := filepath.Join(p) + os.MkdirAll(pp, os.ModePerm) + + if i%2 == 0 { + ioutil.WriteFile(pp+"/id.json", b, os.ModePerm) + } + + } + + bbb, _ := ioutil.ReadFile(cp + "config/ttnc") + + for _, p := range containerPath { + pp := filepath.Join(p) + os.MkdirAll(pp, os.ModePerm) + + pp += "/" + pp += "/ttnc" + ioutil.WriteFile(pp, bbb, os.ModePerm) + + } + +} + +func TestInit(t *testing.T) { + //InitContainer() +} + +func TestDirs(t *testing.T) { + + //cs := IsStartContainer() + // + //for _, c := range cs { + // var ccl []string + // ExistConfigFileDir(ConfigFolder, c, &ccl) + // t.Log(ccl) + // + //} + +} + +func TestSplitPath(t *testing.T) { + + ff := "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff" + a := filepath.Dir(ff) + t.Log(a) + a = filepath.Clean(a) + t.Log(a) + a = filepath.Base(a) + + t.Log(a) + +} + +func TestInitCM(t *testing.T) { + GetInstance().init() + + GetInstance().stopContainerAll() +} + +func TestCallStatus(t *testing.T) { + //c := "java" + //GetInstance().runAndInitContainer(&c) + // + //callStatus(&c) + // + //GetInstance().stopContainer(&c) +} + +func TestCallAdd(t *testing.T) { + //// + //c := "java" + //cn := "HEALTH_"+"MYSQL" + //id := "id.json" + // + //GetInstance().runAndInitContainer(&c) + // + //callAdd(&c, &cn, &id) + // + //GetInstance().stopContainer(&c) + +} + +func TestCallRemove22(t *testing.T) { + + //c := "java" + //cn := "HEALTH_"+"MYSQL" + //id := "id.json" + // + //GetInstance().runAndInitContainer(&c) + // + //callRemove(&c, &cn, &id) + // + //GetInstance().stopContainer(&c) + +} + +func TestRunRun(t *testing.T) { + c := "java" + GetInstance().runContainer(&c) + GetInstance().runContainer(&c) + GetInstance().runContainer(&c) + + GetInstance().stopContainer(&c) + +} + +func TestActiveCrawler(t *testing.T) { + + //c := "java" + //GetInstance().runContainer(&c) + // + //aa := GetInstance().activeCrawler(&c) + // + //t.Log(aa) + // + //GetInstance().stopContainer(&c) + +} + +func TestErer(t *testing.T) { + + var asdf []string + + asdf = nil + + t.Log(len(asdf)) + +} + +// +//func TestStartContainer(t *testing.T) { +// +// cs := GetStartContainer() +// +// t.Log(cs) +// +//} + +// +//func GetStartContainer() []string { +// +// files, _ := ioutil.ReadDir(ConfigFolder) +// +// var cs []string +// +// for _,file := range files { +// +// if file.IsDir() { +// b := existConfigFile(ConfigFolder, file.Name()) +// if b { +// cs = append(cs, file.Name()) +// } +// } +// +// } +// return cs +//} + +func TestJson(t *testing.T) { + + c := config_manager.Config{} + + c.Crawler.Name = "wmi_crawler" + c.Crawler.Container = "java_proxy" + + c.Id = "WMI_TEST_ID_001" + + b, _ := json.Marshal(c) + + fmt.Println(string(b)) + +} + +func TestJsonMa(t *testing.T) { + + b, _ := ioutil.ReadFile("./test/example.json") + + c := config_manager.Config{} + + json.Unmarshal(b, &c) + + t.Log(c) + +} + +func TestMock(t *testing.T) { + + //GetInstance().ConfigMgr = &ConfigManagerTest{} + // + //GetInstance().init() + +} + +func TestRPC(t *testing.T) { + + //c := conMgr.NewConfigManager() + //c.LoadGlobalConfigTTT("/home/snoop/develop/path/go/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml") + //c.LoadCrawlerConfigAllTTT() + // + //observer.Notify(messages.CONFIGMANAGER_LOADED,c) + // + // + //wg := sync.WaitGroup{} + // + //wg.Add(1) + // + //go func() { + // time.Sleep(20 * time.Second) + // wg.Done() + //}() + // + // + //wg.Wait() + +} + +func TestGRPC(t *testing.T) { + b := callStatusAddress("localhost:50052") + + t.Log(b) +} + +type ConfigManagerTest struct { +} + +func (c *ConfigManagerTest) GetGlobalConfig() *config_manager.GlobalConfig { + + g := config_manager.GlobalConfig{} + + rootFolder := "/home/cm2/" + g.Paths.ConfigFolder = rootFolder + "/config/container/" + g.Paths.BinaryFolder = rootFolder + "/container/" + g.Paths.PidFolder = rootFolder + "/pids/" + + return &g +} +func (c *ConfigManagerTest) GetCrawlerById(id string) *config_manager.Config { + return nil +} +func (c *ConfigManagerTest) GetCrawlers() map[string]*config_manager.Config { + + var m map[string]*config_manager.Config + + b, _ := ioutil.ReadFile("./test/example.json") + + ccc := config_manager.Config{} + + json.Unmarshal(b, &ccc) + + m["wmi_crawler"] = &ccc + + return m +} diff --git a/crawler_manager/crawler_util.go b/crawler_manager/crawler_util.go new file mode 100644 index 0000000..52addcd --- /dev/null +++ b/crawler_manager/crawler_util.go @@ -0,0 +1,58 @@ +package crawler_manager + +import ( + "io/ioutil" + "os" + "strconv" +) + +func existConfigFile(prePath string, dir string) bool { + + files, _ := ioutil.ReadDir(prePath + "/" + dir) + + for _, file := range files { + + if file.IsDir() { + retB := existConfigFile(prePath+"/"+dir, file.Name()) + if retB { + return true + } + } else { + return true + } + + } + return false +} + +func existConfigFileDir(prePath string, dir string, configCrawler *[]string) { + + files, _ := ioutil.ReadDir(prePath + "/" + dir) + + for _, file := range files { + + if file.IsDir() { + existConfigFileDir(prePath+"/"+dir, file.Name(), configCrawler) + } else { + *configCrawler = append(*configCrawler, prePath+"/"+dir) + return + } + + } +} + +func getRunCommand(container *string) string { + return GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder + "/" + *container + "/" + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ScriptFile +} + +func writePid(pid int) { + ioutil.WriteFile(GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder+strconv.Itoa(pid), []byte(""), os.ModePerm) +} + +func getConfigPaths(container *string) *[]string { + + var dirs []string + existConfigFileDir(GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder, *container, &dirs) + + return &dirs +} diff --git a/data_sender/data_sender.go b/data_sender/data_sender.go new file mode 100644 index 0000000..bd11525 --- /dev/null +++ b/data_sender/data_sender.go @@ -0,0 +1,187 @@ +package data_sender_go + +import ( + "encoding/json" + cm "git.loafle.net/overflow/agent_api/config_manager" + "git.loafle.net/overflow/agent_api/messages" + pb "git.loafle.net/overflow/crawler_go/grpc" + q "git.loafle.net/overflow/queue_go" + "io/ioutil" + "log" + "os" + "sync" +) + +const ( + FILE_PATH = "/overflow/tmp/data.tmp" + DEFAULT_INTERVAL = 10 +) + +var ( + instance *DataSender + once sync.Once +) + +type DataSender struct { + once sync.Once + queue *q.LoafleQueue + gconf *cm.GlobalConfig +} + +func Start(ch chan bool, conf *cm.GlobalConfig) { + d := GetInstance() + d.start(conf) + ch <- true +} + +func Stop(ch chan bool) { + GetInstance().stop() + ch <- true +} + +func GetInstance() *DataSender { + once.Do(func() { + instance = &DataSender{} + }) + return instance +} + +func (ds *DataSender) start(conf *cm.GlobalConfig) { + qc := make(chan interface{}) + ds.queue = q.NewQueue(DEFAULT_INTERVAL, qc) + ds.gconf = conf + go ds.handleData(qc) +} + +func (ds *DataSender) stop() {} + +func (ds *DataSender) handleData(qc chan interface{}) { + for { + select { + case items := <-qc: + result := make([]*messages.Data, 0) + for _, item := range items.([]*q.Item) { + collectedData := item.Value.(*pb.Output) + d := &messages.Data{} + d.Data = collectedData.Data + d.AgentId = agentIdentifier() + result = append(result, d) + } + ds.send(result) + } + } +} + +func AddData(data interface{}) { + ds := GetInstance() + ds.queue.PushItem(data) +} + +func (ds *DataSender) send(data []*messages.Data) { + //for _, v := range data { + // //log.Printf("SEND SENSOR RESULT : %s - %s", v.SensorId, v.Data) + //} + + //ds.addFailedData(data) + //addr := ds.gconf.Central.Address + ":" + string(ds.gconf.Central.Port) + //conn, err := grpc.Dial(addr, grpc.WithInsecure()) + //if err != nil { + // ds.saveFailedData(data) + // return + //} + //defer conn.Close() + + //temporary + //client := pb.NewStatusClient(conn) + //out, err := client.Status(context.Background(), &pb.Empty{}) + //if err != nil { + // ds.saveFailedData(data) + // return + //} + //ds.removeFailed() + //log.Print(out) +} + +func (ds *DataSender) addFailedData(data []*messages.Data) { + bytes := ds.getFailedData() + if bytes != nil { + failed := messages.Data{} + err := json.Unmarshal(bytes, &failed) + if err != nil { + log.Println(err) + } + data = append([]*messages.Data{&failed}, data...) //prepend + } +} + +func (ds *DataSender) getFailedData() []byte { + b, err := ioutil.ReadFile(FILE_PATH) + if err != nil { + log.Println(err) + return nil + } + return b +} + +func (ds *DataSender) removeFailed() { + err := os.Remove(FILE_PATH) + if err != nil { + log.Println(err) + } +} + +func (ds *DataSender) saveFailedData(datas []*messages.Data) { + + file, err := tempFile() + if err != nil { + log.Println(err) + } + defer func() { + if err := file.Close(); err != nil { + log.Println(err) + return + } + }() + + for _, data := range datas { + bytes, err := json.Marshal(&data) + if err != nil { + log.Println(err) + } + if bytes != nil { + log.Println("write : ", string(bytes)) + _, err = file.Write(bytes) + if err != nil { + log.Println(err) + } + } + } +} + +func tempFile() (*os.File, error) { + + var file *os.File + var fileInfo os.FileInfo + var err error + if fileInfo, err = os.Stat(FILE_PATH); err != nil { + if os.IsNotExist(err) { + file, err = os.Create(FILE_PATH) + if err != nil { + return nil, err + } + } + } else { + if fileInfo != nil { + file, err = os.OpenFile(FILE_PATH, os.O_RDWR|os.O_APPEND, 0660) + if err != nil { + return nil, err + } + } + } + + return file, nil +} + +func agentIdentifier() string { + return "agentID_000000001" +} diff --git a/data_sender/data_sender_test.go b/data_sender/data_sender_test.go new file mode 100644 index 0000000..dd0c8e0 --- /dev/null +++ b/data_sender/data_sender_test.go @@ -0,0 +1,64 @@ +package data_sender_go + +import ( + "git.loafle.net/overflow/agent_api/observer" + "strconv" + "testing" + "time" +) + +type Result struct { + Data []byte +} + +func TestTotal(t *testing.T) { + + time.Sleep(time.Second * 5) + + observer.Notify("CONFIGMANAGER_LOADED", nil) + + time.Sleep(time.Second * 5) + + for i := 0; i < 20; i++ { + testNotify(strconv.Itoa(i)) + } + + time.Sleep(time.Second * 12) + for i := 20; i < 30; i++ { + testNotify(strconv.Itoa(i)) + } + + time.Sleep(time.Second * 100) +} + +func TestSend(t *testing.T) { + + ds := &DataSender{} + ds.start() + + for i := 0; i < 20; i++ { + testNotify(strconv.Itoa(i)) + } + + time.Sleep(time.Second * 12) + for i := 20; i < 30; i++ { + testNotify(strconv.Itoa(i)) + } + + time.Sleep(time.Second * 100) +} + +func testNotify(val string) { + + result := make(map[string]string) + result["a"] = val + result["b"] = val + result["c"] = val + + cd := &Data{ + SensorId: "insanity", + Data: result, + } + + observer.Notify(messages.QUEUE_DATA, cd) +} diff --git a/discovery/.gitignore b/discovery/.gitignore new file mode 100644 index 0000000..d786fbf --- /dev/null +++ b/discovery/.gitignore @@ -0,0 +1,21 @@ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +.idea/ +*.iml + +bin/logs \ No newline at end of file diff --git a/discovery/bin/bridge_conf.xml b/discovery/bin/bridge_conf.xml new file mode 100644 index 0000000..f08c072 --- /dev/null +++ b/discovery/bin/bridge_conf.xml @@ -0,0 +1,6 @@ + + http://localhost:8080/discoveryHistory + sa + qwe123 + application/json + \ No newline at end of file diff --git a/discovery/bin/collector.yaml b/discovery/bin/collector.yaml new file mode 100644 index 0000000..fe405f5 --- /dev/null +++ b/discovery/bin/collector.yaml @@ -0,0 +1,4 @@ +central: + address: "http://localhost:9090" + port: 443 +log_path: "./bin/log.xml" diff --git a/discovery/bin/log.xml b/discovery/bin/log.xml new file mode 100644 index 0000000..f8c9647 --- /dev/null +++ b/discovery/bin/log.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/discovery/bin/scripts/ubuntu/of_collector b/discovery/bin/scripts/ubuntu/of_collector new file mode 100644 index 0000000..33cd6b9 --- /dev/null +++ b/discovery/bin/scripts/ubuntu/of_collector @@ -0,0 +1,71 @@ +### BEGIN INIT INFO +# Provides: overFlow +# Required-Start: $network +# Required-Stop: $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: of-collector agent +# Description: overFlow Collector Agent +### END INIT INFO + +PROG="ofcollector" +PROG_PATH="/home/insanity/develop/overflow/overflow.collector/bin/" +PID_PATH="/var/run/" +FILE_SERVER=/var/run/of_server + +start() { + if [ -e "$PID_PATH/$PROG.pid" ]; then + ## Program is running, exit with error. + echo "$PROG is currently running!" 1>&2 + exit 1 + else + $PROG_PATH/$PROG 2>&1 > /dev/null & + echo "$PROG started" + touch "$PID_PATH/$PROG.pid" + fi +} + +stop() { + sudo echo -n 'STOP' | netcat -U $FILE_SERVER + + if [ -e "$PID_PATH/$PROG.pid" ]; then + ## Program is running, so stop it + #killall $PROG + rm "$PID_PATH/$PROG.pid" + echo "$PROG stopped" + else + ## Program is not running, exit with error. + echo "Error! $PROG not started!" 1>&2 + exit 1 + fi +} + +status() { + sudo echo -n $1 | netcat -U $FILE_SERVER +} + + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +case "$1" in + start) + start + ;; + stop) + stop + ;; + reload|restart|force-reload) + stop + start + ;; + status) + status 'STATUS' + ;; + **) + echo "Usage: $0 {start|stop|restart|status}" 1>&2 + exit 1 + ;; +esac \ No newline at end of file diff --git a/discovery/bootstrap/bootstrap.go b/discovery/bootstrap/bootstrap.go new file mode 100644 index 0000000..749e562 --- /dev/null +++ b/discovery/bootstrap/bootstrap.go @@ -0,0 +1,51 @@ +package main + +import ( + "flag" + "fmt" + "git.loafle.net/overflow/discovery" + "git.loafle.net/overflow/discovery/conf" + log "github.com/cihub/seelog" + "os" + "runtime" +) + +func init() { + paramA := flag.String("conf", "/overflow/collector", "Config file path") + //paramB := flag.Int("b", 1, "int type param") + //paramC := flag.Bool("c", false, "bool type param") + flag.Parse() + //fmt.Println("Args: ", *paramA, *paramB, *paramC) + fmt.Println("Args: ", *paramA) + //if len(os.Args) < /* paramcount */ { + // flag.Usage() + // os.Exit(1) + //} + + if err := conf.LoadConfig(*paramA); err != nil { + fmt.Printf("Error: %s\n", err) + os.Exit(1) + } +} + +func main() { + + runtime.GOMAXPROCS(runtime.NumCPU()) + + defer log.Flush() + + stop := make(chan bool, 1) + go handleShell(stop) + handleSignal(stop) + + collector.Start() + + if <-stop { + // Comes from shell cmd 'stop' or quit signals + stopHandleShell() + collector.Stop() + close(stop) + log.Flush() + os.Exit(0) + } +} diff --git a/discovery/bootstrap/shell.go b/discovery/bootstrap/shell.go new file mode 100644 index 0000000..fef23b3 --- /dev/null +++ b/discovery/bootstrap/shell.go @@ -0,0 +1,76 @@ +package main + +import ( + log "github.com/cihub/seelog" + "net" + "os" + "strings" +) + +const FILE_SERVER string = "/var/run/of_server" + +var fd net.Conn + +func handleShell(stop chan bool) { + + //os.Stdout.Close() + //os.Stderr.Close() + //os.Stdin.Close() + + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: FILE_SERVER, Net: "unix"}) + if err != nil { + log.Error(err.Error()) + return + } + defer func() { + l.Close() + os.Remove(FILE_SERVER) + }() + + for { + fd, err = l.Accept() + if err != nil { + log.Error(err) + return + } + + buf := make([]byte, 1024) + nr, err := fd.Read(buf) + if err != nil { + return + } + + data := string(buf[0:nr]) + switch strings.ToUpper(data) { + case "STOP": + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + stop <- true + case "STATUS": + fd.Write(status()) + default: + } + + fd.Close() + } +} + +func stopHandleShell() { + log.Info("Shell Handler stopped.") + + if fi, _ := os.Stat(FILE_SERVER); fi != nil { + os.Remove(FILE_SERVER) + } + if fd != nil { + fd.Close() + } +} + +func status() []byte { + return []byte("STATUS OK\n") +} diff --git a/discovery/bootstrap/signal.go b/discovery/bootstrap/signal.go new file mode 100644 index 0000000..ce163bf --- /dev/null +++ b/discovery/bootstrap/signal.go @@ -0,0 +1,53 @@ +package main + +import ( + log "github.com/cihub/seelog" + "os" + "os/signal" + "syscall" +) + +func handleSignal(stop chan bool) { + sigs := make(chan os.Signal, 1) + + signal.Notify(sigs, + os.Kill, + os.Interrupt, + syscall.SIGKILL, + syscall.SIGSTOP, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + + go func() { + for { + s := <-sigs + switch s { + case os.Kill, os.Interrupt, syscall.SIGSTOP, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: + log.Infof("Signal received. [%s]", s) + stop <- true + default: + log.Infof("Signal received. [%s]", s) + stop <- true + } + } + }() +} + +/* + 1) SIGHUP 2) SIGINT 3) SIGQUIT 4) SIGILL 5) SIGTRAP + 6) SIGABRT 7) SIGBUS 8) SIGFPE 9) SIGKILL 10) SIGUSR1 +11) SIGSEGV 12) SIGUSR2 13) SIGPIPE 14) SIGALRM 15) SIGTERM +16) SIGSTKFLT 17) SIGCHLD 18) SIGCONT 19) SIGSTOP 20) SIGTSTP +21) SIGTTIN 22) SIGTTOU 23) SIGURG 24) SIGXCPU 25) SIGXFSZ +26) SIGVTALRM 27) SIGPROF 28) SIGWINCH 29) SIGIO 30) SIGPWR +31) SIGSYS 34) SIGRTMIN 35) SIGRTMIN+1 36) SIGRTMIN+2 37) SIGRTMIN+3 +38) SIGRTMIN+4 39) SIGRTMIN+5 40) SIGRTMIN+6 41) SIGRTMIN+7 42) SIGRTMIN+8 +43) SIGRTMIN+9 44) SIGRTMIN+10 45) SIGRTMIN+11 46) SIGRTMIN+12 47) SIGRTMIN+13 +48) SIGRTMIN+14 49) SIGRTMIN+15 50) SIGRTMAX-14 51) SIGRTMAX-13 52) SIGRTMAX-12 +53) SIGRTMAX-11 54) SIGRTMAX-10 55) SIGRTMAX-9 56) SIGRTMAX-8 57) SIGRTMAX-7 +58) SIGRTMAX-6 59) SIGRTMAX-5 60) SIGRTMAX-4 61) SIGRTMAX-3 62) SIGRTMAX-2 +63) SIGRTMAX-1 64) SIGRTMAX +*/ diff --git a/discovery/collector.go b/discovery/collector.go new file mode 100644 index 0000000..d6aab60 --- /dev/null +++ b/discovery/collector.go @@ -0,0 +1,68 @@ +package collector + +import ( + "git.loafle.net/overflow/discovery/discovery" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + log "github.com/cihub/seelog" + "os/exec" + "sync" +) + +type Collector struct { + stop chan bool `json:"-"` + + ID int `json:"id,omitempty"` + ProductId string `json:"productId"` + Version string `json:"version"` + ConfigPath string `json:"configPath"` + InstallDate timestamp.Timestamp `json:"installDate"` + UpdateDate timestamp.Timestamp `json:"updateDate"` +} + +var coll *Collector +var once sync.Once + +func Start() *Collector { + + once.Do(func() { + coll = &Collector{} + coll.info() + }) + coll.start() + + return coll +} + +func Stop() { + log.Info("Collector has stopped.") +} + +func (c *Collector) start() { + go coll.run() +} + +func (c *Collector) run() { + log.Info("Collector is now running.") + + fin := make(chan bool, 1) + c.stop = make(chan bool) + + discovery.Discover(fin) + + if <-fin { + log.Info("Discovery has finished.") + } +} + +func (c *Collector) info() { + + uuid, err := exec.Command("uuidgen").Output() + if err != nil { + log.Error(err) + } + c.ProductId = string(uuid) + c.Version = "1.0" + c.ConfigPath = "/root" + c.InstallDate = timestamp.Now() + c.UpdateDate = timestamp.Now() +} diff --git a/discovery/collector_test.go b/discovery/collector_test.go new file mode 100644 index 0000000..23d2013 --- /dev/null +++ b/discovery/collector_test.go @@ -0,0 +1,72 @@ +package collector + +import ( + "bytes" + "encoding/json" + "fmt" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "io/ioutil" + "net/http" + "strconv" + "testing" + "time" +) + +func TestTimeStampe(t *testing.T) { + + aa := "1479186033399" + + i, err := strconv.ParseInt(aa, 10, 64) + if err != nil { + panic(err) + } + tm := time.Unix(i/1000, 0) + fmt.Println(tm) + +} + +func TestJPAInsert(t *testing.T) { + + c := &Collector{} + c.Version = "2.0" + c.ConfigPath = "/root" + c.LicenseType = LICENSE1 + c.ProductId = "6666" + c.InstallDate = timestamp.Now() + c.UpdateDate = timestamp.Now() + c.LicenseDueDate = timestamp.Date(2017, 12, 25) + + b, err := json.Marshal(c) + if err != nil { + fmt.Println(err) + } + + http.Post("http://localhost:8080/collector", "application/x-spring-data-verbose+json", bytes.NewBuffer(b)) +} + +func TestJPASelect(t *testing.T) { + //res, err := http.Get("http://localhost:8080/collector/1") + res, err := http.Get("http://localhost:8080/collector/p/6666") + if err != nil { + fmt.Println(err) + } + + body, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if len(body) <= 0 { + t.Error("not found.") + return + } + + if err != nil { + fmt.Println(err) + } + + var result Collector + json.Unmarshal(body, &result) + t.Log(result) + + t.Log("ID : ", result.ID) + t.Log("productID : ", result.ProductId) + t.Log("version : ", result.Version) +} diff --git a/discovery/communicate/communicate.go b/discovery/communicate/communicate.go new file mode 100644 index 0000000..a2bb6d2 --- /dev/null +++ b/discovery/communicate/communicate.go @@ -0,0 +1,26 @@ +package communicate + +import ( + "fmt" + //"git.loafle.net/overflow/central/client/events" +) + +var _c *communicator = nil + +func init() { + fmt.Println("init communicator") + //_c = NewCommunicator() + //_c.start() +} + +func SetRootURL(url string) { + _c.RootURL = url +} + +func GetRootURL() string { + return _c.RootURL +} + +//func Send(e *events.Event) { +// _c.addEvent(e) +//} diff --git a/discovery/communicate/communicate_test.go b/discovery/communicate/communicate_test.go new file mode 100644 index 0000000..3e5ccd0 --- /dev/null +++ b/discovery/communicate/communicate_test.go @@ -0,0 +1,83 @@ +package communicate + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/stretchr/testify/assert" + "gopkg.in/gin-gonic/gin.v1" + "io/ioutil" + + "git.loafle.net/overflow/central/client/events" + "net/http" + "net/http/httptest" + "testing" +) + +func TestCommunicatorInit(t *testing.T) { + assert.NotNil(t, _c) +} + +func makeGin() *gin.Engine { + r := gin.New() + api := r.Group("/_api") + { + collector := api.Group("/collector") + { + event := collector.Group("/event") + { + { + types := event.Group("/status") + { + types.POST("/:type", func(c *gin.Context) { + fmt.Println("called /_api/collector/event/status/:type") + var j events.Event + c.BindJSON(&j) + fmt.Println(j) + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + }) + + } + } + } + } + } + return r +} + +func TestSend(t *testing.T) { + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + defer ts.Close() + for i := 0; i < 10; i++ { + e := events.NewEvent(events.CENTRAL_EVENT, events.CollectorInstallEvent{Version: "Test"}) + Send(e) + // t.Log(e.GetResult()) + } +} + +func TestRealSendByGin(t *testing.T) { + + SetRootURL("http://localhost:8080") + + e := events.NewEvent(events.CENTRAL_EVENT, events.NewInstallEvent("TestInstallEvent")) + data, _ := json.Marshal(&e) + + var u events.URLMaker + u = e.Data.(events.URLMaker) + t.Log(GetRootURL() + u.GetUrl()) + req := httptest.NewRequest("POST", u.GetUrl(), bytes.NewReader(data)) + w := httptest.NewRecorder() + + g := makeGin() + g.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("Home page didn't return %v", http.StatusOK) + } else { + t.Log("OKOKOK") + data, _ := ioutil.ReadAll(w.Body) + t.Log(string(data)) + } +} diff --git a/discovery/communicate/communicator.go b/discovery/communicate/communicator.go new file mode 100644 index 0000000..6d4f5ec --- /dev/null +++ b/discovery/communicate/communicator.go @@ -0,0 +1,53 @@ +package communicate + +import ( +//"bytes" +//"compress/gzip" +//"git.loafle.net/overflow/discovery/communicate/events" +//"git.loafle.net/overflow/central/client/events" +) + +type communicator struct { + //Queue chan *events.Event + RootURL string +} + +// +//func NewCommunicator() *communicator { +// return &communicator{Queue: make(chan *events.Event, 10)} +//} +// +//func (c *communicator) addEvent(e *events.Event) { +// c.Queue <- e +//} +// +//func CompressDataGzip(data []byte) []byte { +// var b bytes.Buffer +// w := gzip.NewWriter(&b) +// w.Write(data) +// w.Close() +// return b.Bytes() +//} +// +//func (c *communicator) start() { +// go func() { +// for e := range c.Queue { +// go func(event *events.Event) { +// // m := event.Data.(events.URLMaker) +// // data, _ := json.Marshal(event) +// // +// // // compress , accept-encoding : gzip +// // //res, err := http.Post(GetRootURL()+m.GetUrl(), "application/json", bytes.NewBuffer(CompressDataGzip(data))) +// // //res, err := http.Post(GetRootURL()+m.GetUrl(), "application/json", bytes.NewBuffer(data)) +// // +// // // todo timeout,error +// // if err != nil { +// // return +// // } +// // if res.StatusCode != 200 && res.StatusCode != 201 { +// // +// // } +// }(e) +// } +// }() +//} diff --git a/discovery/communicate/communicator_test.go b/discovery/communicate/communicator_test.go new file mode 100644 index 0000000..1edf2c7 --- /dev/null +++ b/discovery/communicate/communicator_test.go @@ -0,0 +1,26 @@ +package communicate + +import ( + "bytes" + "encoding/json" + + "git.loafle.net/overflow/central/client/events" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "net/http" + "testing" +) + +func TestEventCollector(t *testing.T) { + ev := events.Event{} + + ev.EventType = "install" + ev.Time = timestamp.Now() + ev.Collector_id = "1111111" + ev.Data = events.CollectorInstallEvent{Version: "2.2"} + + bb, _ := json.Marshal(ev) + + //com := CompressTest(CompressDataGzip, UnCompressDataGzip, bb) + + http.Post("http://localhost:9090/_api/collector/event/status/install", "application/json", bytes.NewBuffer(bb)) +} diff --git a/discovery/conf/conf.go b/discovery/conf/conf.go new file mode 100644 index 0000000..7151840 --- /dev/null +++ b/discovery/conf/conf.go @@ -0,0 +1,57 @@ +package conf + +import ( + "errors" + "fmt" + "github.com/cihub/seelog" + "gopkg.in/yaml.v2" + "io/ioutil" + //"git.loafle.net/overflow/discovery/communicate" + //"git.loafle.net/overflow/overflow.collector_backup2/src/git.loafle.net/overflow/discovery/communicate" +) + +type Config struct { + Central struct { + Address string + Port int + } + Log_Path string +} + +var c Config + +func LoadConfig(f string) error { + if len(f) <= 0 { + return errors.New("conf file path is nil") + } + c = Config{} + data, err := ioutil.ReadFile(f) + if err != nil { + return err + } + err = yaml.Unmarshal([]byte(data), &c) + if err != nil { + return err + } + + err = loadLogConfig(c.Log_Path) + if err != nil { + fmt.Println("Log Config Load Error : ", err) + return err + } + fmt.Println(c) + + //communicate.SetRootURL(c.Central.Address) + return nil +} + +func loadLogConfig(path string) error { + + l, err := seelog.LoggerFromConfigAsFile(path) + if err != nil { + fmt.Println("Error : ", err) + return err + } + seelog.ReplaceLogger(l) + return err +} diff --git a/discovery/core/device/hw/cpu.go b/discovery/core/device/hw/cpu.go new file mode 100644 index 0000000..624a52d --- /dev/null +++ b/discovery/core/device/hw/cpu.go @@ -0,0 +1 @@ +package hw diff --git a/discovery/core/device/hw/hdd.go b/discovery/core/device/hw/hdd.go new file mode 100644 index 0000000..624a52d --- /dev/null +++ b/discovery/core/device/hw/hdd.go @@ -0,0 +1 @@ +package hw diff --git a/discovery/core/device/hw/memory.go b/discovery/core/device/hw/memory.go new file mode 100644 index 0000000..624a52d --- /dev/null +++ b/discovery/core/device/hw/memory.go @@ -0,0 +1 @@ +package hw diff --git a/discovery/core/device/hw/nic.go b/discovery/core/device/hw/nic.go new file mode 100644 index 0000000..5dcd956 --- /dev/null +++ b/discovery/core/device/hw/nic.go @@ -0,0 +1,10 @@ +package hw + +type NIC struct { + Cidr int32 + Dns [2]int64 + Gateway int64 + Iface string + Ip int64 + Mac int64 +} diff --git a/discovery/core/device/sw/application.go b/discovery/core/device/sw/application.go new file mode 100644 index 0000000..7e78bc9 --- /dev/null +++ b/discovery/core/device/sw/application.go @@ -0,0 +1,6 @@ +package sw + +type Application struct { + Name string + Version string +} diff --git a/discovery/core/device/sw/os.go b/discovery/core/device/sw/os.go new file mode 100644 index 0000000..e0f487d --- /dev/null +++ b/discovery/core/device/sw/os.go @@ -0,0 +1,7 @@ +package sw + +type OS struct { + Name string + Version string + DisplayName string +} diff --git a/discovery/core/device/sw/process.go b/discovery/core/device/sw/process.go new file mode 100644 index 0000000..523dba1 --- /dev/null +++ b/discovery/core/device/sw/process.go @@ -0,0 +1,7 @@ +package sw + +type Process struct { + Command string + Pid int + User string +} diff --git a/discovery/core/net/host.go b/discovery/core/net/host.go new file mode 100644 index 0000000..eb03dd9 --- /dev/null +++ b/discovery/core/net/host.go @@ -0,0 +1,5 @@ +package net + +type Host struct { + Name string +} diff --git a/discovery/core/net/port.go b/discovery/core/net/port.go new file mode 100644 index 0000000..9290705 --- /dev/null +++ b/discovery/core/net/port.go @@ -0,0 +1,6 @@ +package net + +type Port struct { + Number int16 + PortType string +} diff --git a/discovery/core/net/service.go b/discovery/core/net/service.go new file mode 100644 index 0000000..b7ddd2e --- /dev/null +++ b/discovery/core/net/service.go @@ -0,0 +1,6 @@ +package net + +type Service struct { + CryptoType string + Name string +} diff --git a/discovery/core/net/zone.go b/discovery/core/net/zone.go new file mode 100644 index 0000000..4c0fc1f --- /dev/null +++ b/discovery/core/net/zone.go @@ -0,0 +1,6 @@ +package net + +type Zone struct { + Cidr int32 + Ip int64 +} diff --git a/discovery/core/pcapwrapper/pcap.go b/discovery/core/pcapwrapper/pcap.go new file mode 100644 index 0000000..708f1f6 --- /dev/null +++ b/discovery/core/pcapwrapper/pcap.go @@ -0,0 +1,64 @@ +package pcap + +import ( + "git.loafle.net/overflow/discovery/discovery/types" + "sync" +) + +var pcaps map[string]*PcapWrapper = nil + +var m *sync.Mutex + +func init() { + pcaps = make(map[string]*PcapWrapper, 0) + m = new(sync.Mutex) +} + +func GetInstance(zone *types.DiscoveryZone) (*PcapWrapper, error) { + m.Lock() + // + var p *PcapWrapper + var ok bool + p, ok = pcaps[zone.CidrStr()] + if !ok { + p, err := newPcapWrapper(zone) + if err != nil { + m.Unlock() + return nil, err + } + pcaps[zone.CidrStr()] = p + m.Unlock() + return p, nil + } + m.Unlock() + return p, nil +} + +func Release(zone *types.DiscoveryZone) { + + m.Lock() + + p, ok := pcaps[zone.CidrStr()] + if ok == true { + + if d := p.release(); d == true { + + delete(pcaps, zone.CidrStr()) + } + + } + m.Unlock() +} + +func RemoveAll() { + m.Lock() + if pcaps == nil { + m.Unlock() + return + } + for k, v := range pcaps { + v.destroy() + delete(pcaps, k) + } + m.Unlock() +} diff --git a/discovery/core/pcapwrapper/pcap_test.go b/discovery/core/pcapwrapper/pcap_test.go new file mode 100644 index 0000000..856b5a5 --- /dev/null +++ b/discovery/core/pcapwrapper/pcap_test.go @@ -0,0 +1,45 @@ +package pcap + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/zone" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestMapRemove(t *testing.T) { + + m := make(map[string]string, 0) + + m["1"] = "1" + m["2"] = "2" + m["3"] = "3" + assert.Equal(t, 3, len(m)) + + delete(m, "1") + assert.Equal(t, 2, len(m)) +} + +func TestPcapNewRemove(t *testing.T) { + + z := zone.NewZone() + + GetInstance(z[0]) + assert.Equal(t, 1, len(pcaps)) + + Release(z[0]) + assert.Equal(t, 0, len(pcaps)) +} + +func TestRemoveAll(t *testing.T) { + + z := zone.NewZone() + + GetInstance(z[0]) + fmt.Println("a9se8rypasd") + assert.Equal(t, 1, len(pcaps)) + + RemoveAll() + fmt.Println("a9se8rypasd") + assert.Equal(t, 0, len(pcaps)) +} diff --git a/discovery/core/pcapwrapper/pcapwrapper.go b/discovery/core/pcapwrapper/pcapwrapper.go new file mode 100644 index 0000000..796cc02 --- /dev/null +++ b/discovery/core/pcapwrapper/pcapwrapper.go @@ -0,0 +1,272 @@ +package pcap + +import ( + "git.loafle.net/overflow/discovery/discovery/types" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" + "sync" +) + +type PcapWrapper struct { + m *sync.RWMutex + HND *pcap.Handle + refCount int + stop chan bool + arps []chan *layers.ARP + tcps map[string][]chan *layers.TCP + udps map[string][]chan gopacket.Packet +} + +func newPcapWrapper(zone *types.DiscoveryZone) (*PcapWrapper, error) { + + // new pcap handle + handle, err := pcap.OpenLive(zone.Iface, 65536, true, pcap.BlockForever) + if err != nil { + return nil, err + } + // set filter + // todo add tcp, udp filter + err = handle.SetBPFFilter("arp and src net " + zone.CidrStr() + " or (((tcp[tcpflags] & (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] & (tcp-rst) != 0)) and port 60000) or udp ") + if err != nil { + return nil, err + } + // make pcap wrapper + w := &PcapWrapper{ + HND: handle, + arps: make([]chan *layers.ARP, 0), + tcps: make(map[string][]chan *layers.TCP, 0), + udps: make(map[string][]chan gopacket.Packet, 0), + stop: make(chan bool), + refCount: 1, + m: new(sync.RWMutex), + } + + // start recv goroutine + go w.recvPacket() + return w, err +} + +// arp channel funcs +func (p *PcapWrapper) OpenARPChannel() chan *layers.ARP { + c := make(chan *layers.ARP, 0) + p.arps = append(p.arps, c) + return c +} + +func (p *PcapWrapper) CloseARPChannel(ch chan *layers.ARP) { + + var n int = -1 + for index, value := range p.arps { + if ch == value { + close(ch) + n = index + break + } + } + if n != -1 { + p.arps = append(p.arps[:n], p.arps[n+1:]...) + } +} + +// tcp channel funcs +func (p *PcapWrapper) OpenTCPChannel(host *types.DiscoveryHost) chan *layers.TCP { + p.m.Lock() + defer p.m.Unlock() + _, ok := p.tcps[host.Ip_] + if !ok { + p.tcps[host.Ip_] = make([]chan *layers.TCP, 0) + } + + c := make(chan *layers.TCP, 0) + p.tcps[host.Ip_] = append(p.tcps[host.Ip_], c) + + return c +} +func (p *PcapWrapper) CloseTCPChannel(host *types.DiscoveryHost, ch chan *layers.TCP) { + p.m.Lock() + defer p.m.Unlock() + + _, ok := p.tcps[host.Ip_] + if ok { + var n int = -1 + for index, value := range p.tcps[host.Ip_] { + if ch == value { + close(ch) + n = index + break + } + } + if n != -1 { + p.tcps[host.Ip_] = append(p.tcps[host.Ip_][:n], p.tcps[host.Ip_][n+1:]...) + } + } +} + +// udp channel funcs +func (p *PcapWrapper) OpenUDPChannel(host *types.DiscoveryHost) chan gopacket.Packet { + p.m.Lock() + defer p.m.Unlock() + + _, ok := p.udps[host.Ip_] + if !ok { + p.udps[host.Ip_] = make([]chan gopacket.Packet, 0) + } + + c := make(chan gopacket.Packet, 0) + p.udps[host.Ip_] = append(p.udps[host.Ip_], c) + + return c +} + +func (p *PcapWrapper) CloseUDPChannel(host *types.DiscoveryHost, ch chan gopacket.Packet) { + p.m.Lock() + defer p.m.Unlock() + + _, ok := p.udps[host.Ip_] + if ok { + var n int = -1 + for index, value := range p.udps[host.Ip_] { + if ch == value { + close(ch) + n = index + break + } + } + if n != -1 { + p.udps[host.Ip_] = append(p.udps[host.Ip_][:n], p.udps[host.Ip_][n+1:]...) + } + } +} + +func (p *PcapWrapper) incRefCount() { + p.refCount++ +} + +func (p *PcapWrapper) release() bool { + p.refCount-- + if p.refCount == 0 { + p.destroy() + return true + } + return false +} + +func (p *PcapWrapper) closeChannels() { + + // close stop + if p.stop != nil { + close(p.stop) + p.stop = nil + } + // close tcp channels + for k, v := range p.tcps { + for _, ch := range v { + close(ch) + } + v = v[:0] + delete(p.tcps, k) + } + // close udp channels + for k, v := range p.udps { + for _, ch := range v { + close(ch) + } + v = v[:0] + delete(p.udps, k) + } + // close arp channels + for _, v := range p.arps { + close(v) + } + p.arps = p.arps[:0] +} + +func (p *PcapWrapper) destroy() { + p.m.Lock() + defer p.m.Unlock() + p.closeChannels() + go p.HND.Close() +} + +const arptype = 0 +const tcptype = 1 +const udptype = 2 + +func (p *PcapWrapper) recvPacket() { + + src := gopacket.NewPacketSource(p.HND, layers.LayerTypeEthernet) + in := src.Packets() + + for { + var packet gopacket.Packet + select { + case <-p.stop: + return + case packet = <-in: + ptype := CheckProtocol(packet) + + if ptype == arptype { + if p.arps != nil { + arpLayer := packet.Layer(layers.LayerTypeARP) + arp := arpLayer.(*layers.ARP) + for _, c := range p.arps { + c <- arp + } + } + } else if ptype == tcptype { + ipLayer := packet.Layer(layers.LayerTypeIPv4) + ip := ipLayer.(*layers.IPv4).SrcIP.String() + p.m.RLock() + chans, ok := p.tcps[ip] + if ok { + layer := packet.Layer(layers.LayerTypeTCP) + tcp, _ := layer.(*layers.TCP) + for _, c := range chans { + c <- tcp + } + } + p.m.RUnlock() + } else if ptype == udptype { + ipLayer := packet.Layer(layers.LayerTypeIPv4) + if ipLayer != nil { + ip := ipLayer.(*layers.IPv4).SrcIP.String() + p.m.RLock() + chans, ok := p.udps[ip] + + if ok { + for _, c := range chans { + c <- packet + } + } + + p.m.RUnlock() + } + } + } + } +} +func CheckProtocol(packet gopacket.Packet) int { + if packet == nil { + return -1 + } + layer := packet.Layer(layers.LayerTypeARP) + if layer != nil { + return arptype + } + + layer = packet.Layer(layers.LayerTypeTCP) + if layer != nil { + if _, ok := layer.(*layers.TCP); ok { + return tcptype + } + } + + layer = packet.Layer(layers.LayerTypeUDP) + if layer != nil { + if _, ok := layer.(*layers.UDP); ok { + return udptype + } + } + return -1 +} diff --git a/discovery/core/pcapwrapper/pcapwrapper_test.go b/discovery/core/pcapwrapper/pcapwrapper_test.go new file mode 100644 index 0000000..01b45d3 --- /dev/null +++ b/discovery/core/pcapwrapper/pcapwrapper_test.go @@ -0,0 +1,76 @@ +package pcap + +import ( + "fmt" + "github.com/google/gopacket/layers" + "testing" +) + +func TestChanKey(t *testing.T) { + + ch := make([]chan struct{}, 10) + + for i := 0; i < 10; i++ { + ch[i] = make(chan struct{}) + } + + c := ch[5] + + for n, v := range ch { + if c == v { + fmt.Println(n) + } + } + + for i := 0; i < 10; i++ { + close(ch[i]) + } +} + +func TestDeleteSlice(t *testing.T) { + + a := make([]int, 0) + + for i := 0; i < 10; i++ { + a = append(a, i) + } + + t.Log(a) + + i := 4 + + a = append(a[:i], a[i+1:]...) + t.Log(a) + + a = a[:0] + t.Log(len(a)) + +} + +func TestMapForRange(t *testing.T) { + a := make(map[int]int, 0) + + for i := 0; i < 10; i++ { + a[i] = i + 1000 + } + + for v, vv := range a { + t.Log(v) + t.Log(vv) + } +} + +const arptest = 0 + +func closechan(s interface{}, t int) { + if t == arptest { + close(s.(chan *layers.ARP)) + } + +} + +func TestCloseChannels(t *testing.T) { + a := make(chan *layers.ARP, 0) + closechan(a, arptest) + +} diff --git a/discovery/core/scan/host/host.go b/discovery/core/scan/host/host.go new file mode 100644 index 0000000..aa4286b --- /dev/null +++ b/discovery/core/scan/host/host.go @@ -0,0 +1,116 @@ +package host + +import ( + p "git.loafle.net/overflow/discovery/core/pcapwrapper" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/util/converter" + log "github.com/cihub/seelog" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" + "net" + "time" +) + +type ArpRecvHandler func(arp *layers.ARP) +type HistoryHandler func(ip net.IP) + +// discovery +func Scan(zone *types.DiscoveryZone, recvhandler ArpRecvHandler, historyhandler HistoryHandler) error { + // get channel by zone + w, err := p.GetInstance(zone) + if err != nil { + } + + //// save singleton channel + ch := w.OpenARPChannel() + // + //// read channel + go func() { + for data := range ch { + recvhandler(data) + } + }() + rangeHost := MakeTargetHostRange(zone) + //// send arp + err = sendARP(w.HND, rangeHost, zone, historyhandler) + if err != nil { + } + // + //// wait + time.Sleep(10 * time.Second) + // + //// release arp + w.CloseARPChannel(ch) + p.Release(zone) + + return nil +} + +func sendARP(handle *pcap.Handle, rangeHost []net.IP, zone *types.DiscoveryZone, hh HistoryHandler) error { + + hw := zone.Mac + localIp := converter.IntToIP4(zone.Ip) + + eth := makeEthernetPacket(converter.IntToMac(hw)) + arp := makeArpPacket(converter.IntToMac(hw), localIp.To4()) + opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true} + + buf := gopacket.NewSerializeBuffer() + + for _, targetHost := range rangeHost { + if hh != nil { + hh(targetHost) + } + arp.DstProtAddress = []byte(targetHost) + gopacket.SerializeLayers(buf, opts, ð, &arp) + if err := handle.WritePacketData(buf.Bytes()); err != nil { + log.Critical("ARP Send Error : ", err) + return err + } + time.Sleep(time.Microsecond * 500) + } + return nil +} + +func MakeTargetHostRange(zone *types.DiscoveryZone) []net.IP { + + //returnSlice := make([]net.IP, 0) + // + //for i := zone.FirstScanRange; i <= zone.LastScanRange; i++ { + // cIp := converter.IntToIP4(i) + // returnSlice = append(returnSlice, cIp) + //} + // + //return returnSlice + + minIp := converter.IntToIP4(zone.FirstScanRange).To4() + maxIp := converter.IntToIP4(zone.LastScanRange).To4() + + returnSlice := make([]net.IP, 0) + for i := minIp[3]; i <= maxIp[3]; i++ { + returnSlice = append(returnSlice, []byte{minIp[0], minIp[1], minIp[2], i}) + } + return returnSlice +} + +func makeArpPacket(hw net.HardwareAddr, ip net.IP) layers.ARP { + return layers.ARP{ + AddrType: layers.LinkTypeEthernet, + Protocol: layers.EthernetTypeIPv4, + HwAddressSize: 6, + ProtAddressSize: 4, + Operation: layers.ARPRequest, + SourceHwAddress: []byte(hw), + SourceProtAddress: []byte(ip), + DstHwAddress: []byte{0, 0, 0, 0, 0, 0}, + } +} + +func makeEthernetPacket(hw net.HardwareAddr) layers.Ethernet { + return layers.Ethernet{ + SrcMAC: hw, + DstMAC: net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + EthernetType: layers.EthernetTypeARP, + } +} diff --git a/discovery/core/scan/port/port.go b/discovery/core/scan/port/port.go new file mode 100644 index 0000000..53ccc8d --- /dev/null +++ b/discovery/core/scan/port/port.go @@ -0,0 +1 @@ +package port diff --git a/discovery/core/scan/port/port_test.go b/discovery/core/scan/port/port_test.go new file mode 100644 index 0000000..53ccc8d --- /dev/null +++ b/discovery/core/scan/port/port_test.go @@ -0,0 +1 @@ +package port diff --git a/discovery/core/scan/port/tcp/tcp.go b/discovery/core/scan/port/tcp/tcp.go new file mode 100644 index 0000000..88c6495 --- /dev/null +++ b/discovery/core/scan/port/tcp/tcp.go @@ -0,0 +1,135 @@ +package tcp + +import ( + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "time" + + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + log "github.com/cihub/seelog" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + + //"git.loafle.net/overflow/central/client/events" + //"git.loafle.net/overflow/discovery/communicate" + pw "git.loafle.net/overflow/discovery/core/pcapwrapper" + "git.loafle.net/overflow/discovery/util/converter" +) + +type tcpRecvCallback func(tcp *layers.TCP) + +func Scan(host *types.DiscoveryHost, tcpCB tcpRecvCallback) { + + //Recv(host.Zone, PcapTcpHandler{}) + + w, err := pw.GetInstance(host.Zone) + if err != nil { + log.Error(err) + } + + ch := w.OpenTCPChannel(host) + + //read channel + go func() { + for data := range ch { + tcpCB(data) + } + }() + + Send(host, nil) + + time.Sleep(time.Second * 3) + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortEndEvent(host.Zone.CidrInt64(), host.Ip, host.Histories, types.TYPE_TCP))) + + w.CloseTCPChannel(host, ch) + pw.Release(host.Zone) + +} + +func Check(host *types.DiscoveryHost, ports []*types.DiscoveryPort) { + + //Recv(host.Zone, PcapTcpHandler{}) + Send(host, ports) + + time.Sleep(time.Second * 3) +} + +func Send(host *types.DiscoveryHost, ports []*types.DiscoveryPort) { + + portPacket := CreatePortPacket(host.Zone, host) + + defer portPacket.PacketConn.Close() + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortStartEvent(host.Zone.CidrInt64(), host.Ip, 10000, types.TYPE_TCP))) + host.PortDiscoveryTime = timestamp.Now() + + if ports == nil { + for portNumber := host.FirstScanRange; portNumber <= host.LastScanRange; portNumber++ { + SendPortPacket(portNumber, portPacket, host) + } + } else { + for _, port := range ports { + SendPortPacket(port.Number, portPacket, host) + } + } + +} + +type PortPacket struct { + Ip *layers.IPv4 + Tcp *layers.TCP + Opts gopacket.SerializeOptions + PacketConn net.PacketConn +} + +func CreatePortPacket(zone *types.DiscoveryZone, host *types.DiscoveryHost) *PortPacket { + + pp := &PortPacket{} + + pp.Ip = &layers.IPv4{ + SrcIP: converter.IntToIP4(zone.Ip), + DstIP: net.ParseIP(host.Ip_), + Version: 4, + TTL: 64, + Protocol: layers.IPProtocolTCP, + } + pp.Tcp = &layers.TCP{ + SrcPort: 60000, + DstPort: 0, // will be incremented during the scan + SYN: true, + Seq: 0, + } + pp.Opts = gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + } + + conn, err := net.ListenPacket("ip4:tcp", "0.0.0.0") + if err != nil { + log.Error("SYN create socket error : " + err.Error()) + return nil + } + + pp.PacketConn = conn + + return pp +} + +func SendPortPacket(port uint16, pp *PortPacket, host *types.DiscoveryHost) { + + pp.Tcp.DstPort = layers.TCPPort(port) + pp.Tcp.SetNetworkLayerForChecksum(pp.Ip) + + buf := gopacket.NewSerializeBuffer() + if err := gopacket.SerializeLayers(buf, pp.Opts, pp.Tcp); err != nil { + log.Error(err) + } + + if _, err := pp.PacketConn.WriteTo(buf.Bytes(), &net.IPAddr{IP: net.ParseIP(host.Ip_)}); err != nil { + log.Error(err) + } + + types.NewPortScanHistory(host, port, types.TYPE_TCP, types.SEND, "") + time.Sleep(time.Microsecond * 200) +} diff --git a/discovery/core/scan/port/tcp/tcp_test.go b/discovery/core/scan/port/tcp/tcp_test.go new file mode 100644 index 0000000..89f0ee9 --- /dev/null +++ b/discovery/core/scan/port/tcp/tcp_test.go @@ -0,0 +1,61 @@ +package tcp + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "testing" + + "github.com/google/gopacket/layers" +) + +func TestPortScan(t *testing.T) { + + zone := types.NewZones()[0] + + hh := &types.DiscoveryHost{ + Ip_: "192.168.1.215", + Ports_: make(map[string]*types.DiscoveryPort, 100), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 1, + LastScanRange: 100, + } + + Scan(hh, func(tcp *layers.TCP) { + fmt.Println(tcp) + }) + +} + +func TestPortCheck(t *testing.T) { + + zone := types.NewZones()[0] + + hh := &types.DiscoveryHost{ + Ip_: "192.168.1.215", + Ports_: make(map[string]*types.DiscoveryPort, 100), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 1, + LastScanRange: 100, + } + + var ports []*types.DiscoveryPort + + ports = append(ports, &types.DiscoveryPort{Number: 1}) + ports = append(ports, &types.DiscoveryPort{Number: 3}) + ports = append(ports, &types.DiscoveryPort{Number: 5}) + ports = append(ports, &types.DiscoveryPort{Number: 7}) + ports = append(ports, &types.DiscoveryPort{Number: 9}) + ports = append(ports, &types.DiscoveryPort{Number: 11}) + + Check(hh, ports) + +} diff --git a/discovery/core/scan/port/udp/udp.go b/discovery/core/scan/port/udp/udp.go new file mode 100644 index 0000000..5491cf4 --- /dev/null +++ b/discovery/core/scan/port/udp/udp.go @@ -0,0 +1,105 @@ +package udp + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + log "github.com/cihub/seelog" + "net" + "time" + + "github.com/google/gopacket" + + //"git.loafle.net/overflow/central/client/events" + //"git.loafle.net/overflow/discovery/communicate" + pw "git.loafle.net/overflow/discovery/core/pcapwrapper" +) + +type udpRecvCallback func(packet gopacket.Packet) + +func Scan(host *types.DiscoveryHost, udpCB udpRecvCallback) { + + //Recv(host.Zone, PcapTcpHandler{}) + + w, err := pw.GetInstance(host.Zone) + if err != nil { + log.Error(err) + } + + ch := w.OpenUDPChannel(host) + + //read channel + go func() { + for data := range ch { + udpCB(data) + } + }() + + Send(host, nil) + + time.Sleep(time.Second * 3) + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortEndEvent(host.Zone.CidrInt64(), host.Ip, host.Histories, types.TYPE_UDP))) + + //good game + w.CloseUDPChannel(host, ch) + pw.Release(host.Zone) + +} + +func Check(host *types.DiscoveryHost, ports []*types.DiscoveryPort) { + + //Recv(host.Zone, PcapTcpHandler{}) + Send(host, ports) +} + +func Send(host *types.DiscoveryHost, ports []*types.DiscoveryPort) { + + mats := matcher.GetUdpMatchers() + + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + if err != nil { + log.Error(err) + } + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortStartEvent(host.Zone.CidrInt64(), host.Ip, 10000, types.TYPE_UDP))) + + for indexI := 0; indexI < len(mats); indexI++ { + + mat := mats[indexI] + + if ports == nil { + for portNumber := host.FirstScanRange; portNumber <= host.LastScanRange; portNumber++ { + SendUDPPortPacket(host, conn, mat, portNumber) + } + } else { + for _, port := range ports { + SendUDPPortPacket(host, conn, mat, port.Number) + } + } + + } + + conn.Close() + +} + +func SendUDPPortPacket(host *types.DiscoveryHost, conn *net.UDPConn, mat matcher.UDPMatcher, pn uint16) { + + if mat.IsSend(int(pn)) != true { + return + } + + addr := &net.UDPAddr{IP: net.ParseIP(host.Ip_), Port: int(pn)} + for i := 0; i < mat.PacketCount(); i++ { + var p *packet.Packet = mat.Packet(i) + if _, err := conn.WriteToUDP(p.Buffer, addr); err != nil { + log.Error(err) + } + } + + ph := types.NewPortScanHistory(host, pn, types.TYPE_UDP, types.SEND, "") + host.AddHistory(ph) + + time.Sleep(time.Microsecond * 200) +} diff --git a/discovery/core/scan/port/udp/udp_test.go b/discovery/core/scan/port/udp/udp_test.go new file mode 100644 index 0000000..335d007 --- /dev/null +++ b/discovery/core/scan/port/udp/udp_test.go @@ -0,0 +1,31 @@ +package udp + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "github.com/google/gopacket" + "testing" +) + +func TestUDPPortScan(t *testing.T) { + + zone := types.NewZones()[0] + + hh := &types.DiscoveryHost{ + Ip_: "192.168.1.215", + Ports_: make(map[string]*types.DiscoveryPort, 100), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 1, + LastScanRange: 100, + } + + Scan(hh, func(packet gopacket.Packet) { + fmt.Println(packet) + }) + +} diff --git a/discovery/core/scan/service/connfactory.go b/discovery/core/scan/service/connfactory.go new file mode 100644 index 0000000..0ec9c9d --- /dev/null +++ b/discovery/core/scan/service/connfactory.go @@ -0,0 +1,67 @@ +package service + +import ( + "crypto/tls" + "net" + "time" +) + +type ConnFactory interface { + createSocket(ip string, port string) (net.Conn, error) + Type() string +} + +type ConnFactoryNormal struct { + t string +} + +func (n ConnFactoryNormal) Type() string { + return n.t +} + +func (n ConnFactoryNormal) createSocket(ip string, port string) (net.Conn, error) { + netinfo := ip + ":" + port + conn, err := net.DialTimeout("tcp", netinfo, time.Duration(3)*time.Second) + if err != nil { + return nil, err + } + + err = conn.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + return nil, err + } + return conn, err +} + +type ConnFactoryTLS struct { + t string +} + +func (n ConnFactoryTLS) Type() string { + return n.t +} + +func (n ConnFactoryTLS) createSocket(ip string, port string) (net.Conn, error) { + netinfo := ip + ":" + port + dialer := &net.Dialer{ + Timeout: 3 * time.Second, + } + conn, err := tls.DialWithDialer( + dialer, + "tcp", + netinfo, + &tls.Config{ + InsecureSkipVerify: true, + ServerName: ip, + }, + ) + if err != nil { + return nil, err + } + + err = conn.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + return nil, err + } + return conn, err +} diff --git a/discovery/core/scan/service/matcher/activedirectory/activedirectory.go b/discovery/core/scan/service/matcher/activedirectory/activedirectory.go new file mode 100644 index 0000000..d9e9b15 --- /dev/null +++ b/discovery/core/scan/service/matcher/activedirectory/activedirectory.go @@ -0,0 +1,342 @@ +package activedirectory + +import ( + "bytes" + "encoding/binary" + + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + AD_MESSAGE_ID = 0x99 + AD_MESSAGE_ID_QUIT = 0x89 + + LDAP_VERSION3 = 3 + + LDAP_SUCCESS = 0x00 + + LDAP_REQ_BIND = 0x60 + LDAP_RES_SEARCH_ENTRY = 0x64 + + LDAP_REQ_UNBIND = 0x42 + LDAP_REQ_SEARCH = 0x63 + + LDAP_SCOPE_BASE = 0x00 + LDAP_DEREF_NEVER = 0x00 + LDAP_FILTER_PRESENT = 0x87 + + LDAP_RES_BIND = 0x61 + + LDAP_AUTH_SIMPLE = 0x80 + + AD_TYPE_STR = "supportedCapabilities" +) + +type AD_SENDaaa struct { + DefaultCode uint8 + PackLenFlag uint8 + PacketLen uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint32 + + ProtocolOp uint8 + PtLenFlag uint8 + PtPacketLen uint32 + NextType2 uint8 + NextTypeLength2 uint8 + + Version uint8 + + NextType3 uint8 + NextTypeLength3 uint8 + Auth uint8 + AuthLength uint8 +} + +type AD_SEND struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLen uint32 + + NextType1 uint8 + NextType1Len uint8 + + MessageId uint32 + + ProtocolOp uint8 + + PtPackLenFlag uint8 + PtPacketLen uint32 + + NextType2 uint8 + NextType2Len uint8 + NextType3 uint8 + NextType3Len uint8 + + Scope uint8 + + NextType4 uint8 + NextType4Len uint8 + + DerefAliases uint8 + + NextType5 uint8 + NextType5Len uint8 + + SizeLimit uint8 + + NextType6 uint8 + NextType6Len uint8 + + TimeLimit uint8 + + NextType7 uint8 + NextType7Len uint8 + + TypesOnly uint8 + + Filter1 uint8 + PresentLen uint8 + + Present [11]byte + + DefaultCode2 uint8 + Pack2LenFlag uint8 + + Packet2Len uint32 + + UnknwonCode8 uint8 + ItemLength uint8 + + AttributeDescription [21]byte +} + +type AD_QUIT struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLength uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint32 + + ProtocolOp uint8 + + PtLenFlag uint8 + PtPacketLen uint32 +} + +type AD_RECV struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLength uint32 + + NextType1 uint8 + NextType1Len uint8 + + MessageId uint16 + + ProtocolOp uint8 + + PtPackLenFlag uint8 + PtPacketLen uint32 + + NextType2 uint8 + NextType2Len uint8 + + UnknwonCode21 uint8 + UnknwonCode22 uint8 + UnknwonCode23 uint8 + UnknwonCode24 uint8 + UnknwonCode25 uint8 + UnknwonCode26 uint8 + + UnknwonCode31 uint8 + UnknwonCode32 uint8 + UnknwonCode33 uint8 + UnknwonCode34 uint8 + UnknwonCode35 uint8 + UnknwonCode36 uint8 + UnknwonCode37 uint8 + + TypeLength uint8 +} + +type ActiveDirectoryMatcher struct { + sendPackets []*packet.Packet +} + +func (ad *ActiveDirectoryMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + adRecv := AD_RECV{} + + binary.Read(buf, binary.BigEndian, &adRecv) + + if adRecv.MessageId != AD_MESSAGE_ID { + return false + } + + if adRecv.ProtocolOp != LDAP_RES_SEARCH_ENTRY { + return false + } + + ///AD_TYPE_STR + + // + //if(packet->readCount_ < sizeof(AD_RECV) + recv->typeLength) { + // return false; + //} + + //char* type = new char[recv->typeLength]; + //memcpy(type, packet->buffer_+sizeof(AD_RECV), recv->typeLength); + //std::string typeStr = type; + // + //delete[] type; + //if(typeStr.compare(AD_TYPE_STR) != 0) { + //return false; + //} + + return true +} + +func (ad *ActiveDirectoryMatcher) PacketCount() int { + return len(ad.sendPackets) +} +func (ad *ActiveDirectoryMatcher) Packet(index int) *packet.Packet { + return ad.sendPackets[index] +} +func (ad *ActiveDirectoryMatcher) ServiceName() string { + return "ActiveDirectory" +} + +func (ad *ActiveDirectoryMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (ad *ActiveDirectoryMatcher) IsNoResponse(index int) bool { + + if index == 1 { + return true + } + + return false +} + +func (ad *ActiveDirectoryMatcher) IsPrePacket() bool { + return false +} + +func NewActiveDirectoryMatcher() *ActiveDirectoryMatcher { + + ls := AD_SEND{ + DefaultCode: 0x30, + PackLenFlag: 0x84, + + PacketLen: 0x47, + + NextType1: 0x02, + NextType1Len: 0x04, + + MessageId: AD_MESSAGE_ID, + + ProtocolOp: LDAP_REQ_SEARCH, + + PtPackLenFlag: 0x84, + PtPacketLen: 0x3b, + + NextType2: 0x04, + NextType2Len: 0x00, + NextType3: 0x0a, + NextType3Len: 0x01, + + Scope: LDAP_SCOPE_BASE, + + NextType4: 0x0a, + NextType4Len: 0x01, + + DerefAliases: LDAP_DEREF_NEVER, + + NextType5: 0x02, + NextType5Len: 0x01, + + SizeLimit: 0, + + NextType6: 0x02, + NextType6Len: 0x01, + + TimeLimit: 0x78, + + NextType7: 0x01, + NextType7Len: 0x01, + + TypesOnly: 0, + + Filter1: LDAP_FILTER_PRESENT, + PresentLen: 0x0b, + + //Present :0000, + + DefaultCode2: 0x30, + Pack2LenFlag: 0x84, + + Packet2Len: 0x17, + + UnknwonCode8: 0x04, + ItemLength: 0x15, + + //AttributeDescription:, + } + + copy(ls.Present[:], "objectclass") + copy(ls.AttributeDescription[:], AD_TYPE_STR) + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.BigEndian, ls) + + sendByte1 := mCache.Bytes() + + adm := ActiveDirectoryMatcher{ + //sendPackets: make([][]byte, 2), + } + + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + adm.sendPackets = append(adm.sendPackets, pp) + + aq := AD_QUIT{ + DefaultCode: 0x30, + PackLenFlag: 0x84, + PacketLength: 0x0c, + NextType1: 0x02, + NextTypeLength1: 0x04, + MessageId: AD_MESSAGE_ID_QUIT, + ProtocolOp: LDAP_REQ_UNBIND, + PtLenFlag: 0x84, + PtPacketLen: 0x00, + } + + lqBuffer := new(bytes.Buffer) + binary.Write(lqBuffer, binary.BigEndian, aq) + + quBytes := lqBuffer.Bytes() + + pp2 := packet.NewPacket(quBytes, len(quBytes)) + + adm.sendPackets = append(adm.sendPackets, pp2) + + return &adm +} diff --git a/discovery/core/scan/service/matcher/activedirectory/activedirectory_test.go b/discovery/core/scan/service/matcher/activedirectory/activedirectory_test.go new file mode 100644 index 0000000..0c6204c --- /dev/null +++ b/discovery/core/scan/service/matcher/activedirectory/activedirectory_test.go @@ -0,0 +1,80 @@ +package activedirectory + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" + + "crypto/tls" +) + +func TestADNor(t *testing.T) { + client, err := net.Dial("tcp", "192.168.1.1:389") + + if err != nil { + t.Log(err) + } + + defer client.Close() + + dDRun(client, t) +} + +func TestADTLS(t *testing.T) { + conn, err := tls.Dial( + "tcp", + "192.168.1.1:636", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.1", + }, + ) + + if err != nil { + t.Log(err) + return + } + + defer conn.Close() + + dDRun(conn, t) +} + +func dDRun(client net.Conn, t *testing.T) { + + lm := NewActiveDirectoryMatcher() + + port := types.NewPort("389", types.NewHost("192.168.1.1"), types.TYPE_TCP) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + read, _ := client.Read(bytes) + + fmt.Println(bytes) + + b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) + + if b { + fmt.Println("Good") + } + + } + + t.Log(scanInfo) +} diff --git a/discovery/core/scan/service/matcher/cassandra/cassandra.go b/discovery/core/scan/service/matcher/cassandra/cassandra.go new file mode 100644 index 0000000..adc33d6 --- /dev/null +++ b/discovery/core/scan/service/matcher/cassandra/cassandra.go @@ -0,0 +1,105 @@ +package cassandra + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + COMPRESSION = "COMPRESSION" + CQL_VERSION = "CQL_VERSION" +) + +type cassandra struct { + Version uint8 + Flags uint8 + Stream uint16 + Opcode uint8 + Length uint32 +} + +type CassandraMatcher struct { + packets []*packet.Packet +} + +func NewCassandraMatcher() *CassandraMatcher { + + cassMatcher := &CassandraMatcher{} + c := cassandra{ + Version: 4, + Flags: 0, + Stream: 0, + Opcode: 5, + Length: 0, + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, c) + + cassMatcher.packets = append(cassMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return cassMatcher +} + +func (t *CassandraMatcher) ServiceName() string { + return "Cassandra" +} + +func (t *CassandraMatcher) PacketCount() int { + return len(t.packets) +} +func (t *CassandraMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *CassandraMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *CassandraMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *CassandraMatcher) IsPrePacket() bool { + return false +} + +func (t *CassandraMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + c := cassandra{} + if err := binary.Read(reader, binary.BigEndian, &c); err != nil { + return false + } + if c.Version != 0x84 { + return false + } + if c.Flags != 0x00 { + return false + } + if c.Stream != 0x00 { + return false + } + if c.Opcode != 0x06 { + return false + } + var itemcount uint16 + + if binary.Read(reader, binary.BigEndian, &itemcount) != nil { + return false + } + + if itemcount != 0 && itemcount != 2 { + return false + } + + return true + +} diff --git a/discovery/core/scan/service/matcher/cassandra/cassandra_test.go b/discovery/core/scan/service/matcher/cassandra/cassandra_test.go new file mode 100644 index 0000000..461cb1a --- /dev/null +++ b/discovery/core/scan/service/matcher/cassandra/cassandra_test.go @@ -0,0 +1,73 @@ +package cassandra + +import ( + "crypto/tls" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestCassandra(t *testing.T) { + + m := NewCassandraMatcher() + + conn, err := net.Dial("tcp", "192.168.1.104:9042") + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("Cassandra found") + return + } + + t.Error("Cassandra not found") + } + +} + +func TestCassandraTLS(t *testing.T) { + + m := NewCassandraMatcher() + + conn, err := tls.Dial( + "tcp", + "192.168.1.104:9042", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.104", + }, + ) + if err != nil { + t.Fatal(err) + } + + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("Cassandra found") + return + } + + t.Error("Cassandra not found") + } + +} diff --git a/discovery/core/scan/service/matcher/dhcp/dhcp.go b/discovery/core/scan/service/matcher/dhcp/dhcp.go new file mode 100644 index 0000000..1cac9d6 --- /dev/null +++ b/discovery/core/scan/service/matcher/dhcp/dhcp.go @@ -0,0 +1,188 @@ +package dhcp + +import ( + "bytes" + "encoding/binary" + "fmt" + "git.loafle.net/overflow/discovery/discovery/types" + log "github.com/cihub/seelog" + "net" + "time" +) + +const ( + MAGIC_COOKIE uint32 = 0x63825363 + OPT_CODE_SERVER_IDENTIFIER uint8 = 54 + OPT_CODE_SUBNET_MASK uint8 = 1 + OPT_CODE_ROUTER uint8 = 3 + OPT_CODE_DNS uint8 = 6 +) + +type dhcpDiscover struct { + MsgType byte + HwType byte + HwAddrLen byte + Hops byte + Xid uint32 + Secs uint16 + BootpFlags uint16 + ClientIp uint32 + YourIp uint32 + NextServerIp uint32 + RelayAgentIp uint32 + ClientMacAddr [6]byte + ClientHwAddrPadding [10]byte + ServerHostName [64]byte + BootFileName [128]byte + MagicCookie uint32 + Mtype byte + MtypeLen byte + MtypeVal byte + Opts [200]byte + End byte + Padding [16]byte +} + +func DiscoverDHCP(zone *types.DiscoveryZone) { + err := sendDHCPDiscovery() + if err != nil { + log.Error(err) + return + } + recvDHCPOffer(zone) +} + +func sendDHCPDiscovery() error { + dhcp := dhcpDiscover{ + MsgType: 0x01, + HwType: 0x01, + HwAddrLen: 0x06, + Hops: 0x00, + Xid: 0x00000000, + Secs: 0x0000, + ClientIp: 0x00000000, + YourIp: 0x00000000, + NextServerIp: 0x00000000, + RelayAgentIp: 0x00000000, + MagicCookie: MAGIC_COOKIE, + Mtype: 0x35, + MtypeLen: 0x01, + MtypeVal: 0x01, + End: 0xff, + } + + var flag uint16 = 0 + dhcp.BootpFlags = ^flag // flag = unicast , ^flag = broadcast + + //TODO : getting mac addr from zone + dhcp.ClientMacAddr[0] = 0x50 + dhcp.ClientMacAddr[1] = 0xe5 + dhcp.ClientMacAddr[2] = 0x49 + dhcp.ClientMacAddr[3] = 0x46 + dhcp.ClientMacAddr[4] = 0x93 + dhcp.ClientMacAddr[5] = 0x28 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.BigEndian, dhcp) + conn, err := net.Dial("udp", "255.255.255.255:67") + if err != nil { + return err + } + conn.Write(writer.Bytes()) + defer conn.Close() + + return nil +} + +func recvDHCPOffer(zone *types.DiscoveryZone) { + + socket, err := net.ListenUDP("udp4", &net.UDPAddr{ + IP: net.IPv4(255, 255, 255, 255), + Port: 68, + }) + if err != nil { + log.Error(err) + return + } + err = socket.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + log.Error(err) + return + } + + buf := make([]byte, 4096) + n, _, err := socket.ReadFromUDP(buf) + if err != nil { + log.Error(err) + return + } + if n <= 0 { + log.Error("No DHCP offer.") + return + } + + offer := dhcpDiscover{} + + reader := new(bytes.Buffer) + reader.Write(buf) + if err := binary.Read(reader, binary.BigEndian, &offer); err != nil { + log.Error(err) + return + } + if offer.MagicCookie != MAGIC_COOKIE { + log.Error("Not a DHCP packet.") + return + } + + //option searching + r := new(bytes.Buffer) + r.Write(offer.Opts[:]) + + for i := 0; i < r.Len(); i++ { + v := r.Next(1)[0] + + if v == OPT_CODE_SUBNET_MASK && r.Next(1)[0] == 4 { + ipStr := byteToIpString(r.Next(4)) + log.Infof("SUBNET MASK: %s", ipStr) + } + + if v == OPT_CODE_ROUTER && r.Next(1)[0] == 4 { + ipStr := byteToIpString(r.Next(4)) + log.Infof("ROUTER: %s", ipStr) + } + + if v == OPT_CODE_DNS { + len := r.Next(1)[0] + var dns []string = make([]string, 0) + var ipStr string + ipStr = byteToIpString(r.Next(4)) + dns = append(dns, ipStr) + if len == 8 { + ipStr = byteToIpString(r.Next(4)) + dns = append(dns, ipStr) + } + log.Info("DNS: ", dns) + } + + if v == OPT_CODE_SERVER_IDENTIFIER && r.Next(1)[0] == 4 { + ipStr := byteToIpString(r.Next(4)) + log.Infof("DHCP SERVER: %s", ipStr) + } + } + log.Flush() + +} + +func byteToIpString(b []byte) string { + var ipStr string + len := len(b) + for i := 0; i < len; i++ { + v := b[i] + ipStr += fmt.Sprintf("%d", v) + if i < len-1 { + ipStr += "." + } + } + + return ipStr +} diff --git a/discovery/core/scan/service/matcher/dhcp/dhcp_test.go b/discovery/core/scan/service/matcher/dhcp/dhcp_test.go new file mode 100644 index 0000000..4831c66 --- /dev/null +++ b/discovery/core/scan/service/matcher/dhcp/dhcp_test.go @@ -0,0 +1,11 @@ +package dhcp + +import ( + "git.loafle.net/overflow/discovery/core/scan/zone" + "testing" +) + +func TestDHCP(t *testing.T) { + zone := zone.NewZone() + DiscoverDHCP(zone) +} diff --git a/discovery/core/scan/service/matcher/dns/dns.go b/discovery/core/scan/service/matcher/dns/dns.go new file mode 100644 index 0000000..8cef006 --- /dev/null +++ b/discovery/core/scan/service/matcher/dns/dns.go @@ -0,0 +1,151 @@ +package dns + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +type Dns_frame_header struct { + Transaction_id uint16 + Flags uint16 + Questions uint16 + Answer_rrs uint16 + Authority_rrs uint16 + Additional_rrs uint16 +} + +type Dns_query_section struct { + Name uint8 + Query_type uint16 + Class_type uint16 +} + +type Dns_authority_section struct { + Name uint8 + Auth_type uint16 + Class_type uint16 + Time_to_live uint32 + Data_length uint16 + Primary_name_server [20]uint8 + Responsible_authority_mailbox [24]uint8 + Serial_number uint32 + Refresh_interval uint32 + Retry_interval uint32 + Expire_limit uint32 + Minium_ttl uint32 +} + +type DNSMatcher struct { + packets []*packet.Packet +} + +func NewDnsMatcher() *DNSMatcher { + + m := &DNSMatcher{} + + header := Dns_frame_header{ + Transaction_id: 0x2a88, + Flags: 0x0100, + Questions: 1, + Answer_rrs: 0, + Authority_rrs: 0, + Additional_rrs: 0, + } + + query := Dns_query_section{ + Name: 0, + Query_type: 1, + Class_type: 1, + } + + buf := new(bytes.Buffer) + binary.Write(buf, binary.BigEndian, header) + binary.Write(buf, binary.BigEndian, query) + + m.packets = append(m.packets, packet.NewPacket(buf.Bytes(), buf.Len())) + + return m +} + +func (t *DNSMatcher) ServiceName() string { + return "DNS" +} + +func (t *DNSMatcher) PacketCount() int { + return len(t.packets) +} +func (t *DNSMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *DNSMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *DNSMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *DNSMatcher) IsPrePacket() bool { + return true +} + +func (t *DNSMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + if packet == nil { + return false + } + if packet.Len <= 0 { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + h := Dns_frame_header{} + if err := binary.Read(reader, binary.BigEndian, &h); err != nil { + return false + } + + if h.Transaction_id != 0x2a88 { + return false + } + if h.Flags != 0x8180 && h.Flags != 0x8182 { + return false + } + if h.Questions != 1 { + return false + } + if h.Answer_rrs != 0 { + return false + } + if h.Authority_rrs != 0 && h.Authority_rrs != 1 { + return false + } + if h.Additional_rrs != 0 && h.Additional_rrs != 1 { + return false + } + + q := Dns_query_section{} + if err := binary.Read(reader, binary.BigEndian, &q); err != nil { + return false + } + if q.Name != 0 { + return false + } + if q.Query_type != 1 { + return false + } + if q.Class_type != 1 { + return false + } + + return true +} +func (t *DNSMatcher) IsSend(port int) bool { + if port == 53 { + return true + } + return false +} diff --git a/discovery/core/scan/service/matcher/dns/dns_test.go b/discovery/core/scan/service/matcher/dns/dns_test.go new file mode 100644 index 0000000..e9de4ec --- /dev/null +++ b/discovery/core/scan/service/matcher/dns/dns_test.go @@ -0,0 +1,33 @@ +package dns + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestDns(t *testing.T) { + m := NewDnsMatcher() + + conn, _ := net.Dial("udp", "192.168.1.215:53") + + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + if m.IsSend(53) != true { + t.Error("not port") + } + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("dns found") + return + } + + t.Error("dns not found") + } +} diff --git a/discovery/core/scan/service/matcher/ftp/ftp.go b/discovery/core/scan/service/matcher/ftp/ftp.go new file mode 100644 index 0000000..cdefdb2 --- /dev/null +++ b/discovery/core/scan/service/matcher/ftp/ftp.go @@ -0,0 +1,142 @@ +package ftp + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + log "github.com/cihub/seelog" +) + +// FTP Status codes, defined in RFC 959 +const ( + statusReadyServer = "120" + statusOK = "200" + statusNewConnectOK = "220" + statusSystemNameOK = "215" + statusCloseConnect = "221" + statusUnkownCMD = "202" + statusTlsUseOK = "234" + statusCloseControlConnect = "421" + statusSyntaxErr = "500" + statusParamSyntaxErr = "501" + statusNotUseCMD = "502" + statusIncorrectCMD = "503" + statusTlsNotUse = "534" + statusNeedUserId = "332" +) + +type FTPMatcher struct { + sendPackets []*packet.Packet + isFtps bool +} + +func (ftp *FTPMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + log.Error("Packet nil") + return result + } + + str := string(packet.Buffer) + //fmt.Println(str) + code := str[:3] + + if index == 0 { + switch code { + case statusNewConnectOK, statusReadyServer: + //fmt.Println(code) + result = true + break + } + } else if index == 1 { + switch code { + case statusSystemNameOK, statusSyntaxErr, statusParamSyntaxErr, statusNotUseCMD: + //fmt.Println(code) + result = true + break + } + } else if index == 2 { + switch code { + case statusIncorrectCMD, statusParamSyntaxErr, statusNotUseCMD, statusNeedUserId: + //fmt.Println(code) + result = true + break + } + } else if index == 3 { + switch code { + case statusCloseConnect, statusSyntaxErr: + //fmt.Println(code) + result = true + break + } + } + + if index == 3 && result == true { + var err error + var isfs bool + + //fmt.Println(info.Port.Host.Ip, info.Port.Port) + + isfs, err = StartCheckFTPS(info.Port.Host.Ip_, info.Port.Port_) + + if isfs && err == nil { + ftp.isFtps = isfs + } else if err != nil { + log.Warn("FTPS Check Error : ", err.Error()) + } + } + + return result +} + +func (ftp *FTPMatcher) PacketCount() int { + return len(ftp.sendPackets) +} + +func (ftp *FTPMatcher) Packet(index int) *packet.Packet { + return ftp.sendPackets[index] +} + +func (ftp *FTPMatcher) ServiceName() string { + re := "" + if ftp.isFtps { + re = "FTPS" + } else { + re = "FTP" + } + return re +} + +func (ftp *FTPMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (ftp *FTPMatcher) IsNoResponse(index int) bool { + return false +} + +func (ftp *FTPMatcher) IsPrePacket() bool { + return true +} + +func NewFTPMatcher() *FTPMatcher { + + ftm := FTPMatcher{} + + sysStr := "SYST\r\n" + systByte := make([]byte, len(sysStr)) + copy(systByte[:], sysStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(systByte, len(sysStr))) + + passStr := "PASS \r\n" + passByte := make([]byte, len(passStr)) + copy(passByte[:], passStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(passByte, len(passStr))) + + quitStr := "QUIT\r\n" + quitByte := make([]byte, len(quitStr)) + copy(quitByte[:], quitStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(quitByte, len(quitStr))) + + return &ftm +} diff --git a/discovery/core/scan/service/matcher/ftp/ftp_test.go b/discovery/core/scan/service/matcher/ftp/ftp_test.go new file mode 100644 index 0000000..5774889 --- /dev/null +++ b/discovery/core/scan/service/matcher/ftp/ftp_test.go @@ -0,0 +1,235 @@ +package ftp + +import ( + "fmt" + "testing" + + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + log "github.com/cihub/seelog" + "net" +) + +//type FTP struct { +// conn net.Conn +// addr string +// +// reader *bufio.Reader +// writer *bufio.Writer +//} +// +//func (ftp *FTP) Close() { +// ftp.conn.Close() +//} +// +//func Connect(addr string) (*FTP, error) { +// var err error +// var conn net.Conn +// +// if conn, err = net.Dial("tcp", addr); err != nil { +// return nil, err +// } +// +// writer := bufio.NewWriter(conn) +// reader := bufio.NewReader(conn) +// +// obj := &FTP{ +// conn:conn, +// addr:addr, +// reader:reader, +// writer:writer, +// } +// recv, _ := obj.receive() +// +// fmt.Println(recv) +// +// return obj, nil +// +//} +// +//func (ftp *FTP) receive() (string, error) { +// line, err := ftp.receiveLine() +// +// if err != nil { +// return line, err +// } +// +// fmt.Println("len : ", len(line)) +// fmt.Println("line[3] :", line[3]) +// // +// //if (len(line) >= 4) && (line[3] == '-') { +// // closingCode := line[:3] + " " +// // +// // for { +// // str, err := ftp.receiveLine() +// // fmt.Println("str pre: ", str) +// // line = line + str +// // fmt.Println("str after: ", line) +// // if err != nil { +// // return line, err +// // } +// // +// // if len(str) < 4 { +// // fmt.Println("Uncorrectly terminated response") +// // }else { +// // if str[:4] == closingCode { +// // break +// // } +// // } +// // } +// //} +// +// ftp.ReadAndDiscard() +// +// fmt.Println("receive line: ", line) +// return line, err +//} +// +//func (ftp *FTP) ReadAndDiscard() (int, error) { +// var i int +// bufferSize := ftp.reader.Buffered() +// +// for i = 0; i < bufferSize ; i++ { +// if _, err := ftp.reader.ReadByte(); err != nil { +// return i, err +// } +// } +// +// return i, nil +//} +// +//func (ftp *FTP) send(command string, arguments ...interface{}) error { +// +// command = fmt.Sprintf(command) +// command += "\r\n" +// +// if _, err := ftp.writer.WriteString(command); err != nil { +// return err +// } +// +// if err := ftp.writer.Flush(); err != nil { +// return err +// } +// +// return nil +//} +// +//func (ftp *FTP) cmd(expects string, command string, args ...interface{}) (line string, err error) { +// +// if err = ftp.send(command, args); err != nil { +// return +// } +// +// if line, err = ftp.receive(); err != nil { +// return +// } +// +// +// if !strings.HasPrefix(line, expects) { +// err = errors.New(line) +// return +// } +// +// return +//} +// +//func (ftp *FTP) receiveLine() (string, error) { +// line, err := ftp.reader.ReadString('\n') +// +// log.Printf("< %s", line) +// +// return line, err +//} +// +//func (ftp *FTP) Syst() (line string, err error) { +// if err := ftp.send("SYST"); err != nil { +// return "", err +// } +// +// if line, err = ftp.receive(); err != nil { +// return +// } +// +// if !strings.HasPrefix(line, "215") { +// err = errors.New(line) +// return +// } +// +// return strings.SplitN(strings.TrimSpace(line), " ", 2)[1], nil +//} + +//func TestFtp(t *testing.T) { +// var err error +// var ftp *FTP +// //var f *FTPMatcher +// +// if ftp, err = Connect("192.168.1.202:21"); err != nil { +// panic(err) +// } +// +// //f.Match(0, nil,nil) +// ftp.Syst() +// ftp.cmd("503","PASS ") +// ftp.cmd("221","QUIT") +// defer ftp.Close() +//} + +func TestMatchFTP(t *testing.T) { + ftm := NewFTPMatcher() + //fmt.Println(ftm) + //fmt.Println(ftm.sendPackets[0]) + + //log.LoadLogConfig("../../../../../../../../bin/log.xml") + //defer log.Flush() + + port := types.NewPort("21", types.NewHost("192.168.1.202"), types.TYPE_TCP) + info := scaninfo.NewServiceScanInfo(port) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + log.Debug(ipport) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + //reader := bufio.NewReader(client) + //writer := bufio.NewWriter(client) + + fmt.Println(ftm.PacketCount()) + //fmt.Println(reader.ReadString('\n')) + + bytes := make([]byte, 512) + + le, _ := client.Read(bytes) + + fmt.Println(bytes) + + b := ftm.Match(0, packet.NewPacket(bytes, le), nil) + + fmt.Println(b) + + for ii := 0; ii < ftm.PacketCount(); ii++ { + pack := ftm.Packet(ii) + + fmt.Println(pack) + + //writer.WriteString(pack) + client.Write(pack.Buffer) + //fmt.Println(reader.ReadString('\n')) + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + //fmt.Println(bytes) + + b := ftm.Match(ii+1, packet.NewPacket(bytes, l), info) + + fmt.Println(b) + + } + + fmt.Println("Service Name : ", ftm.ServiceName()) +} diff --git a/discovery/core/scan/service/matcher/ftp/ftps.go b/discovery/core/scan/service/matcher/ftp/ftps.go new file mode 100644 index 0000000..ced39ab --- /dev/null +++ b/discovery/core/scan/service/matcher/ftp/ftps.go @@ -0,0 +1,191 @@ +package ftp + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + log "github.com/cihub/seelog" + "net" + "strings" + "time" +) + +// FTP is a Session for file Transfer Protocol +type FTPS struct { + conn net.Conn + addr string + tlsconfig *tls.Config + + reader *bufio.Reader + writer *bufio.Writer + + isFtps bool +} + +func (fs *FTPS) close() { + fs.conn.Close() +} + +func (fs *FTPS) quit() (err error) { + if _, err := fs.cmd(statusCloseConnect, "QUIT"); err != nil { + return err + } + fs.conn.Close() + fs.conn = nil + return nil +} + +func (fs *FTPS) cmd(expects string, cmd string) (line string, err error) { + if err = fs.send(cmd); err != nil { + log.Error("ftps.go cmd send error: ", err) + return "", err + } + + if line, err = fs.receive(); err != nil { + log.Error("ftps.go cmd receive error: ", err) + return line, err + } + + if !strings.HasPrefix(line, expects) { + err = errors.New(line) + return line, err + } + + return line, err +} + +func (fs *FTPS) readAndDiscard() (int, error) { + var i int + bufferSize := fs.reader.Buffered() + + for i = 0; i < bufferSize; i++ { + if _, err := fs.reader.ReadByte(); err != nil { + return i, err + } + } + + return i, nil +} + +func (fs *FTPS) receive() (string, error) { + line, err := fs.reader.ReadString('\n') + //log.Debug("< %s", line) + + if err != nil { + return line, err + } + + fs.readAndDiscard() + //fmt.Println(line) + return line, err +} + +func (fs *FTPS) send(cmd string) (err error) { + + if len(cmd) == 0 { + err = errors.New("command length 0") + } + + cmd = fmt.Sprintf(cmd) + cmd += "\r\n" + + if _, err := fs.writer.WriteString(cmd); err != nil { + return err + } + + if err := fs.writer.Flush(); err != nil { + return err + } + return nil +} + +func (fs *FTPS) authTls(config *tls.Config) error { + if _, err := fs.cmd(statusTlsUseOK, "AUTH TLS"); err != nil { + return err + } + + fs.tlsconfig = config + + fs.conn = tls.Client(fs.conn, config) + fs.writer = bufio.NewWriter(fs.conn) + fs.reader = bufio.NewReader(fs.conn) + + _, err := fs.cmd(statusOK, "PBSZ 0") + + if err != nil { + return err + } + + _, err = fs.cmd(statusOK, "PROT P") + + if err != nil { + return err + } + + return nil +} +func (fs *FTPS) NewFTPSConnect(addr string) (*FTPS, error) { + var err error + var conn net.Conn + + if conn, err = net.Dial("tcp", addr); err != nil { + log.Error("FTPS Socket Fail: ", err.Error()) + return nil, err + } + + err = conn.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + //log.Error("FTPS Socket Fail: ", err.Error()) + return nil, err + } + + writer := bufio.NewWriter(conn) + reader := bufio.NewReader(conn) + + var line string + + obj := &FTPS{ + conn: conn, + addr: addr, + reader: reader, + writer: writer, + } + + line, err = obj.receive() + + if !strings.HasPrefix(line, "220") { + err = errors.New(line) + return nil, err + } + + //log.Debug(line) + + return obj, err +} + +func StartCheckFTPS(ip string, port string) (bool, error) { + + var err error + var fs *FTPS + + addr := ip + ":" + port + //log.Debug("address : " + addr) + + if fs, err = fs.NewFTPSConnect(addr); err != nil { + return false, err + } + + defer fs.close() + + config := &tls.Config{ + InsecureSkipVerify: true, + ClientAuth: tls.RequestClientCert, + } + + if err = fs.authTls(config); err != nil { + return false, err + } + + return true, err +} diff --git a/discovery/core/scan/service/matcher/ftp/ftps_test.go b/discovery/core/scan/service/matcher/ftp/ftps_test.go new file mode 100644 index 0000000..6771896 --- /dev/null +++ b/discovery/core/scan/service/matcher/ftp/ftps_test.go @@ -0,0 +1,13 @@ +package ftp + +import ( + "fmt" + "testing" +) + +func TestStartCheckFTPS(t *testing.T) { + + isFtps, err := StartCheckFTPS("192.168.1.202", "80") + fmt.Println("Result : ", isFtps) + fmt.Println("Error : ", err) +} diff --git a/discovery/core/scan/service/matcher/http/http.go b/discovery/core/scan/service/matcher/http/http.go new file mode 100644 index 0000000..a2ea5b6 --- /dev/null +++ b/discovery/core/scan/service/matcher/http/http.go @@ -0,0 +1,90 @@ +package http + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strings" +) + +type HTTPMatcher struct { + sendPackets []*packet.Packet +} + +func (h *HTTPMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + str := string(packet.Buffer) + //fmt.Println(str) + elems := strings.Split(str, "\r\n") + + if len(elems) <= 0 { + return result + } + + protocol := (elems[0])[:8] + + httpv0 := strings.Compare(protocol, "HTTP/1.0") + httpv1 := strings.Compare(protocol, "HTTP/1.1") + httpv2 := strings.Compare(protocol, "HTTP/1.2") + + if 0 == httpv0 || 0 == httpv1 || 0 == httpv2 { + result = true + } + + serverName := "Unknown Server" + + for _, valueStr := range elems { + tempElems := strings.Split(valueStr, ":") + + if 0 == strings.Compare(tempElems[0], "Server") { + serverName = tempElems[1] + break + } + } + + strings.Compare(serverName, "Unknown") + //fmt.Println("HTTP Server Name: ", serverName) + + return result +} + +func (h *HTTPMatcher) PacketCount() int { + return len(h.sendPackets) +} + +func (h *HTTPMatcher) Packet(index int) *packet.Packet { + + return h.sendPackets[index] +} + +func (h *HTTPMatcher) ServiceName() string { + return "HTTP" +} + +func (h *HTTPMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (h *HTTPMatcher) IsNoResponse(index int) bool { + return false +} + +func (h *HTTPMatcher) IsPrePacket() bool { + return false +} + +func NewHTTPMatcher() *HTTPMatcher { + + h := HTTPMatcher{} + reqStr := "GET / HTTP/1.1\r\n\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + + h.sendPackets = append(h.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &h +} diff --git a/discovery/core/scan/service/matcher/http/http_test.go b/discovery/core/scan/service/matcher/http/http_test.go new file mode 100644 index 0000000..bf6d4e2 --- /dev/null +++ b/discovery/core/scan/service/matcher/http/http_test.go @@ -0,0 +1,49 @@ +package http + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestHTTPMatcher_Packet(t *testing.T) { + hm := NewHTTPMatcher() + fmt.Println(hm) + fmt.Println(hm.sendPackets[0]) + +} + +func TestHTTPMatcher_Match(t *testing.T) { + fmt.Println("Match") + + hm := NewHTTPMatcher() + + port := types.NewPort("80", types.NewHost("192.168.1.103"), types.TYPE_TCP) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + //fmt.Println(ipport) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + pack := hm.Packet(0) + + //fmt.Println(pack) + + //writer.WriteString(pack) + client.Write(pack.Buffer) + + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + //fmt.Println(bytes) + + hm.Match(0, packet.NewPacket(bytes, l), nil) + +} diff --git a/discovery/core/scan/service/matcher/http/https_test.go b/discovery/core/scan/service/matcher/http/https_test.go new file mode 100644 index 0000000..ca78b9c --- /dev/null +++ b/discovery/core/scan/service/matcher/http/https_test.go @@ -0,0 +1,55 @@ +package http + +import ( + "crypto/tls" + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "log" + "net" + "testing" + "time" +) + +func TestHTTPSMatcher_Match(t *testing.T) { + netinfo := "192.168.1.1:443" + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + } + + conn, err := tls.DialWithDialer( + dialer, + "tcp", + netinfo, + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.103", + }, + ) + + if err != nil { + log.Println(err) + return + } + defer conn.Close() + + //fmt.Println(conn) + h := NewHTTPMatcher() + + pac := h.Packet(0) + + //fmt.Println(pac) + //fmt.Println(pac.Buffer) + + //bytes := make([]byte, 1024) + + l, _ := conn.Write(pac.Buffer) + + buf := make([]byte, 1024) + l, _ = conn.Read(buf) + + fmt.Println(string(buf)) + fmt.Println(l) + is := h.Match(0, packet.NewPacket(buf, l), nil) + fmt.Println(is) + +} diff --git a/discovery/core/scan/service/matcher/imap/imap.go b/discovery/core/scan/service/matcher/imap/imap.go new file mode 100644 index 0000000..a1ea546 --- /dev/null +++ b/discovery/core/scan/service/matcher/imap/imap.go @@ -0,0 +1,87 @@ +package imap + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + PRE_COMPARE_STR = "* OK" + SEND_COMPARE_STR = "* BYE" +) + +type IMAPMatcher struct { + sendPackets []*packet.Packet +} + +func (i *IMAPMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + switch index { + case 0: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 3 { + return false + } + + compareStr := recvStr[0:4] + + if compareStr == PRE_COMPARE_STR { + return true + } + + case 1: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 5 { + return false + } + + compareStr := recvStr[0:5] + + if compareStr == SEND_COMPARE_STR { + return true + } + + } + + return false +} + +func (i *IMAPMatcher) PacketCount() int { + return len(i.sendPackets) +} +func (i *IMAPMatcher) Packet(index int) *packet.Packet { + return i.sendPackets[index] +} +func (i *IMAPMatcher) ServiceName() string { + return "IMAP" +} + +func (i *IMAPMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (i *IMAPMatcher) IsNoResponse(index int) bool { + + return false +} + +func (i *IMAPMatcher) IsPrePacket() bool { + return true +} + +func NewIMAPMatcher() *IMAPMatcher { + + im := IMAPMatcher{} + + reqStr := "A0001 LOGOUT\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + + im.sendPackets = append(im.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &im + +} diff --git a/discovery/core/scan/service/matcher/imap/imap_test.go b/discovery/core/scan/service/matcher/imap/imap_test.go new file mode 100644 index 0000000..d3b7b0c --- /dev/null +++ b/discovery/core/scan/service/matcher/imap/imap_test.go @@ -0,0 +1,147 @@ +package imap + +import ( + "crypto/tls" + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func ImapRun(client net.Conn, t *testing.T) { + + lm := NewIMAPMatcher() + + port := types.NewPort("143", types.NewHost("192.168.1.215"), types.TYPE_TCP) + + scanInfo := scaninfo.NewServiceScanInfo(port) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + //client, _ := net.Dial("tcp", ipport) + + //defer client.Close() + + bytett := make([]byte, 1024) + + rr, _ := client.Read(bytett) + + bb := lm.Match(0, packet.NewPacket(bytett, rr), scanInfo) + + if bb { + t.Log("good!") + } + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + //fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + read, _ := client.Read(bytes) + + fmt.Println(cap(bytes)) + + //fmt.Println(bytes) + + b := lm.Match(ii+1, packet.NewPacket(bytes, read), scanInfo) + + if b { + t.Log("send Good!") + } + + } + t.Log(scanInfo) + +} + +func TestIMapTls(t *testing.T) { + + conn, _ := tls.Dial( + "tcp", + "192.168.1.215:993", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.215", + }, + ) + + defer conn.Close() + + ImapRun(conn, t) + +} + +func TestIMapNormal(t *testing.T) { + + client, _ := net.Dial("tcp", "192.168.1.215:143") + + defer client.Close() + + ImapRun(client, t) + +} + +func TestImap(t *testing.T) { + + lm := NewIMAPMatcher() + + port := types.NewPort("143", types.NewHost("192.168.1.215"), types.TYPE_TCP) + + scanInfo := scaninfo.NewServiceScanInfo(port) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + bytett := make([]byte, 1024) + + rr, _ := client.Read(bytett) + + bb := lm.Match(0, packet.NewPacket(bytett, rr), scanInfo) + + if bb { + t.Log("good!") + } + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + //fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + read, _ := client.Read(bytes) + + fmt.Println(cap(bytes)) + + //fmt.Println(bytes) + + b := lm.Match(ii+1, packet.NewPacket(bytes, read), scanInfo) + + if b { + t.Log("send Good!") + } + + } + t.Log(scanInfo) +} diff --git a/discovery/core/scan/service/matcher/ldap/ldap.go b/discovery/core/scan/service/matcher/ldap/ldap.go new file mode 100644 index 0000000..18c3b63 --- /dev/null +++ b/discovery/core/scan/service/matcher/ldap/ldap.go @@ -0,0 +1,198 @@ +package ldap + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +type LDAPMatcher struct { + sendPackets []*packet.Packet +} + +func (l *LDAPMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + ldapRecv := LDAP_RECV{} + + binary.Read(buf, binary.LittleEndian, &ldapRecv) + + if ldapRecv.MessageId != LDAP_MESSAGE_ID { + return false + } + + if ldapRecv.ProtocolOp != LDAP_RES_BIND { + return false + } + + if ldapRecv.ResultCode != LDAP_SUCCESS { + return false + } + + return true +} + +func (l *LDAPMatcher) PacketCount() int { + return len(l.sendPackets) +} +func (l *LDAPMatcher) Packet(index int) *packet.Packet { + return l.sendPackets[index] +} +func (l *LDAPMatcher) ServiceName() string { + return "LDAP" +} + +func (l *LDAPMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (l *LDAPMatcher) IsNoResponse(index int) bool { + + if index == 1 { + return true + } + + return false +} + +func (l *LDAPMatcher) IsPrePacket() bool { + return false +} + +func NewLDAPMatcher() *LDAPMatcher { + + ls := LDAP_SEND{ + DefaultCode: 0x30, + PacketLength: 0x0c, // size -2 + NextType1: 0x02, + NextTypeLength1: 0x01, + MessageId: LDAP_MESSAGE_ID, + ProtocolOp: LDAP_REQ_BIND, + ProtocolOpLength: 0x07, + NextType2: 0x02, + NextTypeLength2: 0x01, + Version: LDAP_VERSION3, + NextType3: 0x04, + NextTypeLength3: 0x00, + Auth: LDAP_AUTH_SIMPLE, + AuthLength: 0x00, + } + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.LittleEndian, ls) + + sendByte1 := mCache.Bytes() + + lm := LDAPMatcher{ + //sendPackets: make([][]byte, 2), + } + + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + lm.sendPackets = append(lm.sendPackets, pp) + + lq := LDAP_QUIT{ + DefaultCode: 0x30, + UnknwonCode1: 0x84, + PacketLength: 0x05, + NextType1: 0x02, + NextTypeLength1: 0x01, + MessageId: LDAP_MESSAGE_ID_QUIT, + ProtocolOp: LDAP_REQ_UNBIND, + protocolOpLength: 0x00, + } + + lqBuffer := new(bytes.Buffer) + binary.Write(lqBuffer, binary.BigEndian, lq) + + sendByte2 := lqBuffer.Bytes() + + pp2 := packet.NewPacket(sendByte2, len(sendByte2)) + + lm.sendPackets = append(lm.sendPackets, pp2) + + return &lm + +} + +type LDAP_SEND struct { + DefaultCode uint8 + PacketLength uint8 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint8 + + ProtocolOp uint8 + ProtocolOpLength uint8 + NextType2 uint8 + NextTypeLength2 uint8 + + Version uint8 + + NextType3 uint8 + NextTypeLength3 uint8 + Auth uint8 + AuthLength uint8 +} + +type LDAP_RECV struct { + DefaultCode uint8 + UnknwonCode1 uint8 + EndCode11 uint8 + EndCode12 uint8 + + MessageId uint8 + + ProtocolOp uint8 + UnknwonCode2 uint8 + EndCode21 uint8 + EndCode22 uint8 + + ResultCode uint8 + + UnknwonCode3 uint8 + UnknwonCode4 uint8 + Auth uint8 + UnknwonCode5 uint8 +} + +type LDAP_QUIT struct { + DefaultCode uint8 + UnknwonCode1 uint8 + + PacketLength uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint8 + + ProtocolOp uint8 + + protocolOpLength uint8 +} + +const ( + LDAP_MESSAGE_ID = 0x99 + LDAP_MESSAGE_ID_QUIT = 0x89 + + LDAP_VERSION3 = 3 + + LDAP_SUCCESS = 0x00 + + LDAP_REQ_BIND = 0x60 + + LDAP_REQ_UNBIND = 0x42 + + LDAP_RES_BIND = 0x61 + + LDAP_AUTH_SIMPLE = 0x80 +) diff --git a/discovery/core/scan/service/matcher/ldap/ldap_test.go b/discovery/core/scan/service/matcher/ldap/ldap_test.go new file mode 100644 index 0000000..9b970bb --- /dev/null +++ b/discovery/core/scan/service/matcher/ldap/ldap_test.go @@ -0,0 +1,109 @@ +package ldap + +import ( + "crypto/tls" + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +//func SetUp() { +// fmt.Println("SetUp") +//} +// +//func TearDown() { +// fmt.Println("TearDown") +//} + +//func TestMain(m *testing.M) { +// SetUp() +// m.Run() +// TearDown() +//} + +func TestAAAA(t *testing.T) { + ///animals := []Animal{Dog{}, Cat{}, Llama{}, JavaProgrammer{}} + + var ttt [][]int = make([][]int, 10) + + var aaa []int + aaa = append(aaa, 111) + + ttt = append(ttt, aaa) + + fmt.Println(cap(ttt)) + +} + +func ldapRun(client net.Conn, t *testing.T) { + lm := NewLDAPMatcher() + + port := types.NewPort("389", types.NewHost("192.168.1.215"), types.TYPE_TCP) + scanInfo := scaninfo.NewServiceScanInfo(port) + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + //client, _ := net.Dial("tcp", ipport) + //defer client.Close() + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + bytes := make([]byte, 1024) + + client.Write(pack.Buffer) + + read, _ := client.Read(bytes) + + if read <= 0 { + bb := lm.IsNoResponse(ii) + if bb { + + t.Log("IsNoResponse good") + break + } + + } + + fmt.Println(bytes) + + b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) + + if b { + t.Log("Good") + } + + } + + t.Log(scanInfo) +} + +func TestLdapTls(t *testing.T) { + conn, _ := tls.Dial( + "tcp", + "192.168.1.215:636", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.215", + }, + ) + + defer conn.Close() + + ldapRun(conn, t) +} + +func TestLdapNormal(t *testing.T) { + client, _ := net.Dial("tcp", "192.168.1.215:389") + + defer client.Close() + + ldapRun(client, t) +} diff --git a/discovery/core/scan/service/matcher/matcher.go b/discovery/core/scan/service/matcher/matcher.go new file mode 100644 index 0000000..520fbef --- /dev/null +++ b/discovery/core/scan/service/matcher/matcher.go @@ -0,0 +1,115 @@ +package matcher + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/activedirectory" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/cassandra" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/dns" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/ftp" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/http" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/imap" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/ldap" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/mongodb" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/mssql" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/mysql" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/netbios" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/oracle" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/pop" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/redis" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/rmi" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/smb" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/smtp" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/snmp" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/ssh" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/telnet" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/wmi" + "git.loafle.net/overflow/discovery/discovery/types" +) + +var ( + TcpMatchers []Matcher + UdpMatchers []UDPMatcher +) + +func init() { + //TCP + TcpMatchers = append(TcpMatchers, smtp.NewSmtpMatcher()) + TcpMatchers = append(TcpMatchers, ldap.NewLDAPMatcher()) + TcpMatchers = append(TcpMatchers, activedirectory.NewActiveDirectoryMatcher()) + TcpMatchers = append(TcpMatchers, mongodb.NewMongoDBMatcher()) + TcpMatchers = append(TcpMatchers, mysql.NewMySqlMatcher()) + TcpMatchers = append(TcpMatchers, mssql.NewMSSqlMatcher()) + TcpMatchers = append(TcpMatchers, redis.NewRedisMatcher()) + TcpMatchers = append(TcpMatchers, redis.NewRedisProtectedMatcher()) + TcpMatchers = append(TcpMatchers, netbios.NewNetBiosMatcher()) + TcpMatchers = append(TcpMatchers, smb.NewSMBMatcher()) + TcpMatchers = append(TcpMatchers, cassandra.NewCassandraMatcher()) + TcpMatchers = append(TcpMatchers, imap.NewIMAPMatcher()) + TcpMatchers = append(TcpMatchers, oracle.NewOracleMatcher()) + TcpMatchers = append(TcpMatchers, pop.NewPOPMatcher()) + TcpMatchers = append(TcpMatchers, wmi.NewWMIMatcher()) + TcpMatchers = append(TcpMatchers, ftp.NewFTPMatcher()) + TcpMatchers = append(TcpMatchers, http.NewHTTPMatcher()) + TcpMatchers = append(TcpMatchers, rmi.NewRMIMatcher()) + TcpMatchers = append(TcpMatchers, ssh.NewSSHMatcher()) + TcpMatchers = append(TcpMatchers, telnet.NewTelnetMatcher()) + UdpMatchers = append(UdpMatchers, dns.NewDnsMatcher()) + UdpMatchers = append(UdpMatchers, snmp.NewSNMPv2Matcher()) + UdpMatchers = append(UdpMatchers, snmp.NewSNMPv3Matcher()) +} + +type Matcher interface { + Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool + + PacketCount() int + Packet(index int) *packet.Packet + ServiceName() string + + IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool + IsNoResponse(index int) bool + IsPrePacket() bool +} + +type UDPMatcher interface { + Matcher + IsSend(port int) bool +} + +func GetTcpMatchers(ispre bool) []Matcher { + + retMatchers := make([]Matcher, 0) + l := len(TcpMatchers) + for i := 0; i < l; i++ { + c := TcpMatchers[i].IsPrePacket() + if c == ispre { + retMatchers = append(retMatchers, TcpMatchers[i]) + } + } + + return retMatchers +} + +func GetUdpMatchers() []UDPMatcher { + + retMatchers := make([]UDPMatcher, 0) + l := len(UdpMatchers) + for i := 0; i < l; i++ { + retMatchers = append(retMatchers, UdpMatchers[i]) + } + + return retMatchers +} + +func GetMatcherByName(serName string) Matcher { + for _, m := range TcpMatchers { + if m.ServiceName() == serName { + return m + } + } + for _, m := range UdpMatchers { + if m.ServiceName() == serName { + return m + } + } + return nil +} diff --git a/discovery/core/scan/service/matcher/matcher_test.go b/discovery/core/scan/service/matcher/matcher_test.go new file mode 100644 index 0000000..c1a567b --- /dev/null +++ b/discovery/core/scan/service/matcher/matcher_test.go @@ -0,0 +1,91 @@ +package matcher + +import ( + "fmt" + "testing" +) + +type TestMatcher struct { +} + +func (t *TestMatcher) ServiceName() string { + return "TestMatcher" +} + +func (t *TestMatcher) Match(index int, packet []byte, info *ServiceScanInfo) bool { + return true +} + +func (t *TestMatcher) PacketCount() int { + return 1 +} +func (t *TestMatcher) Packet(index int) []byte { + return nil +} + +func (t *TestMatcher) IsError(index int, packet []byte, info *ServiceScanInfo) bool { + return true +} +func (t *TestMatcher) IsNoResponse(index int) bool { + return true +} +func (t *TestMatcher) IsPrePacket() bool { + return true +} + +type Animal interface { + Speak() string +} + +type Dog struct { +} + +func (d Dog) Speak() string { + return "Woof!" +} + +type Cat struct { +} + +func (c Cat) Speak() string { + return "Meow!" +} + +type Llama struct { +} + +func (l Llama) Speak() string { + return "?????" +} + +type JavaProgrammer struct { +} + +func (j JavaProgrammer) Speak() string { + return "Design patterns!" +} + +func TestMatcherTTT(t *testing.T) { + + //animals := []Animal{Dog{}, Cat{}, Llama{}, JavaProgrammer{}} + + var animals []Animal + + animals = append(animals, &Dog{}) + animals = append(animals, &Cat{}) + animals = append(animals, &Llama{}) + animals = append(animals, &JavaProgrammer{}) + + for _, a := range animals { + fmt.Println(a.Speak()) + } + +} + +func TestMatchersInit(t *testing.T) { + Matchers = append(Matchers, &TestMatcher{}) + + for _, m := range Matchers { + fmt.Println(m.ServiceName()) + } +} diff --git a/discovery/core/scan/service/matcher/mongodb/mongodb.go b/discovery/core/scan/service/matcher/mongodb/mongodb.go new file mode 100644 index 0000000..248a2c5 --- /dev/null +++ b/discovery/core/scan/service/matcher/mongodb/mongodb.go @@ -0,0 +1,122 @@ +package mongodb + +import ( + "bytes" + "encoding/binary" + "math/rand" + + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + MONGO_OP_REQUEST uint32 = 2004 + MONGO_OP_REPLY uint32 = 1 + MONGO_FCNAME string = "admin.$cmd" + MONGO_ELEMENT string = "ismaster" +) + +var MONGO_REQUEST_ID uint32 + +type mongo struct { + MessageLength uint32 + RequestId uint32 + ResponseTo uint32 + OpCode uint32 + Flags uint32 + FullCollectionName [11]byte + NumberToSkip uint32 + NumberToReturn int32 + DocumentLength uint32 + Type_ uint8 + Element [9]byte + Value uint8 + _ uint8 +} + +type MongoDBMatcher struct { + packets []*packet.Packet +} + +func NewMongoDBMatcher() *MongoDBMatcher { + + mongoMatcher := &MongoDBMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, mongo{}) + + var fcn [11]byte + copy(fcn[:], MONGO_FCNAME) + + var elem [9]byte + copy(elem[:], MONGO_ELEMENT) + + MONGO_REQUEST_ID = rand.Uint32() + m := mongo{ + MessageLength: uint32(len(tempBuf.Bytes())), + RequestId: MONGO_REQUEST_ID, + ResponseTo: 0, + OpCode: MONGO_OP_REQUEST, + Flags: 0, + FullCollectionName: fcn, + NumberToSkip: 0, + NumberToReturn: -1, + DocumentLength: 16, + Type_: 0x08, + Element: elem, + Value: 1, + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, m) + + mongoMatcher.packets = append(mongoMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return mongoMatcher +} + +func (t *MongoDBMatcher) ServiceName() string { + return "MongoDB" +} + +func (t *MongoDBMatcher) PacketCount() int { + return len(t.packets) +} +func (t *MongoDBMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MongoDBMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *MongoDBMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MongoDBMatcher) IsPrePacket() bool { + return false +} + +func (t *MongoDBMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + m := mongo{} + if err := binary.Read(reader, binary.LittleEndian, &m); err != nil { + return false + } + + if uint32(packet.Len) != m.MessageLength || + m.ResponseTo != MONGO_REQUEST_ID || + m.OpCode != MONGO_OP_REPLY { + return false + } + + return true + +} diff --git a/discovery/core/scan/service/matcher/mongodb/mongodb_test.go b/discovery/core/scan/service/matcher/mongodb/mongodb_test.go new file mode 100644 index 0000000..bf20e03 --- /dev/null +++ b/discovery/core/scan/service/matcher/mongodb/mongodb_test.go @@ -0,0 +1,54 @@ +package mongodb + +import ( + "crypto/tls" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestMongoNor(t *testing.T) { + + conn, _ := net.Dial("tcp", "192.168.1.105:27017") + + defer conn.Close() + + MongoRun(conn, t) + +} +func TestMongoTLS(t *testing.T) { + conn, _ := tls.Dial( + "tcp", + "192.168.1.105:27017", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.105", + }, + ) + + defer conn.Close() + + MongoRun(conn, t) +} + +func MongoRun(conn net.Conn, t *testing.T) { + + m := NewMongoDBMatcher() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("MongoDB found") + return + } + + t.Error("MongoDB not found") + } + +} diff --git a/discovery/core/scan/service/matcher/mssql/mssql.go b/discovery/core/scan/service/matcher/mssql/mssql.go new file mode 100644 index 0000000..6c1dba1 --- /dev/null +++ b/discovery/core/scan/service/matcher/mssql/mssql.go @@ -0,0 +1,175 @@ +package mssql + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + HEADER_TYPE_PRELOGIN uint8 = 0x12 + HEADER_TYPE_RESPONSE uint8 = 0x4 + PL_OPTION_TOKEN_VERSION uint8 = 0x00 + PL_OPTION_TOKEN_ENCRYPTION uint8 = 0x01 + PL_OPTION_TOKEN_TRACEID uint8 = 0x05 + PL_OPTION_TOKEN_TERMINATOR uint8 = 0xff + + /* + ENCRYPT_OFF string = "Encryption is available but off." + ENCRYPT_ON string = "Encryption is available and on." + ENCRYPT_NOT_SUP string = "Encryption is not available." + ENCRYPT_REQ string = "Encryption is required." + */ +) + +type PreloginMsg struct { + VersionToken uint8 + VersionOffset uint16 + VersionLength uint16 + + EncryptionToken uint8 + EncryptionOffset uint16 + EncryptionLength uint16 + + TraceIdToken uint8 + TraceIdOffset uint16 + TraceIdLength uint16 + + Terminator uint8 + Options [7]uint8 + TraceId [36]uint8 +} + +type mssql struct { + Type_ uint8 + Status uint8 + Length uint16 + Channel uint16 + PacketNum uint8 + Window uint8 + Prelogin PreloginMsg +} + +type PreloginResponse struct { + Msg [256]uint8 +} + +type mssqlResponse struct { + Type_ uint8 + Status uint8 + Length uint16 + Channel uint16 + PacketNum uint8 + Window uint8 + PreLoginResp PreloginResponse +} + +type MSSqlMatcher struct { + packets []*packet.Packet + isSSL bool +} + +func NewMSSqlMatcher() *MSSqlMatcher { + + mssqlMatcher := &MSSqlMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, mssql{}) + + m := mssql{ + Type_: HEADER_TYPE_PRELOGIN, + Status: 0x01, + Length: uint16(len(tempBuf.Bytes())), + Channel: 0, + PacketNum: 0, + Window: 0, + Prelogin: PreloginMsg{ + VersionToken: PL_OPTION_TOKEN_VERSION, + VersionOffset: 0x0010, + VersionLength: 0x0006, + EncryptionToken: PL_OPTION_TOKEN_ENCRYPTION, + EncryptionOffset: 0x0016, + EncryptionLength: 0x0001, + TraceIdToken: PL_OPTION_TOKEN_TRACEID, + TraceIdOffset: 0x0017, + TraceIdLength: 0x0024, + Terminator: PL_OPTION_TOKEN_TERMINATOR, + }, + } + + writer := new(bytes.Buffer) + binary.Write(writer, binary.BigEndian, m) + + mssqlMatcher.packets = append(mssqlMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return mssqlMatcher +} + +func (t *MSSqlMatcher) ServiceName() string { + if t.isSSL { + return "SQL Server (SSL)" + } + return "SQL Server" +} + +func (t *MSSqlMatcher) PacketCount() int { + return len(t.packets) +} +func (t *MSSqlMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MSSqlMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *MSSqlMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MSSqlMatcher) IsPrePacket() bool { + return false +} + +func (t *MSSqlMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + m := mssqlResponse{} + + if err := binary.Read(reader, binary.BigEndian, &m); err != nil { + return false + } + + if m.Type_ != HEADER_TYPE_RESPONSE { + return false + } + + if m.Length != uint16(packet.Len) { + return false + } + + switch m.PreLoginResp.Msg[m.Length-9 : m.Length-8][0] { + case 0: + return true + case 1: + t.isSSL = true + return true + case 2: + return true + case 3: + t.isSSL = true + return true + default: + return false + } + + return false + +} diff --git a/discovery/core/scan/service/matcher/mssql/mssql_test.go b/discovery/core/scan/service/matcher/mssql/mssql_test.go new file mode 100644 index 0000000..dce94e6 --- /dev/null +++ b/discovery/core/scan/service/matcher/mssql/mssql_test.go @@ -0,0 +1,63 @@ +package mssql + +import ( + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "net" + "testing" +) + +/* +192.168.1.106:1433 - normal +192.168.1.103:1433 - ssl +*/ +func TestSqlNor(t *testing.T) { + + conn, _ := net.Dial("tcp", "192.168.1.103:1433") + + defer conn.Close() + + sqlServerRun(conn, t) + +} + +//func TestSqlTLS(t *testing.T) { +// conn, err := tls.Dial( +// "tcp", +// "192.168.1.103:7680", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.103", +// }, +// ) +// +// if err != nil { +// t.Log(err) +// return +// } +// +// defer conn.Close() +// +// sqlServerRun(conn, t) +//} + +func sqlServerRun(conn net.Conn, t *testing.T) { + + m := NewMSSqlMatcher() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log(m.ServiceName()) + return + } + + t.Error("MSSQL not found") + } + +} diff --git a/discovery/core/scan/service/matcher/mysql/mysql.go b/discovery/core/scan/service/matcher/mysql/mysql.go new file mode 100644 index 0000000..fee16d1 --- /dev/null +++ b/discovery/core/scan/service/matcher/mysql/mysql.go @@ -0,0 +1,151 @@ +package mysql + +import ( + "bytes" + "encoding/binary" + "fmt" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strconv" + "strings" +) + +type PacketSize struct { + PacketLength [3]byte + PacketNumber byte +} + +type mySql struct { + Payload PacketSize + Protocol byte + Version [256]byte + + TreadId uint32 + Salt1 [9]byte + ServerCapa uint16 + ServerLang uint8 + ServerStat uint16 + ExtServerCapa uint16 + AuthPlugLen uint8 + _ [10]uint8 + Salt2 [13]uint8 + AuthPlugName [64]uint8 +} + +type MySqlMatcher struct { + packets []*packet.Packet + version string + isErrResp bool + errCode int + errMsg string + isSSL bool +} + +func NewMySqlMatcher() *MySqlMatcher { + return &MySqlMatcher{} +} + +func (t *MySqlMatcher) ServiceName() string { + if t.isErrResp { + return "MySQL" + "(Err-" + strconv.Itoa(t.errCode) + " : " + t.errMsg + ")" + } + if t.isSSL { + return "MySQL" + "-" + t.version + "(SSL)" + } + return "MySQL" + "(" + t.version + ")" +} + +func (t *MySqlMatcher) PacketCount() int { + return len(t.packets) +} + +func (t *MySqlMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MySqlMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *MySqlMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MySqlMatcher) IsPrePacket() bool { + return true +} + +func (t *MySqlMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil || len(packet.Buffer) <= 0 { + return false + } + + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + m := mySql{} + if err := binary.Read(r, binary.LittleEndian, &m); err != nil { + return false + } + + buf := bytes.NewBuffer(m.Payload.PacketLength[:]) + packetLen, _ := binary.ReadUvarint(buf) + + if packetLen != uint64(packet.Len-4) { + return false + } + + if m.Protocol == 0xff { + //MySQL error response + var code [2]uint8 + copy(code[:], m.Version[:2]) + + var msg [256]uint8 + copy(msg[:], m.Version[2:]) + + errCode := binary.LittleEndian.Uint16(code[:]) + if errCode < 1000 || errCode > 1727 { + return false + } + errMsg := bytes.Trim(msg[:], "\x00") + + t.isErrResp = true + t.errCode = int(errCode) + t.errMsg = string(errMsg) + + return true + } + + if m.Protocol != 10 && m.Protocol != 9 { + return false + } + + t.checkSSL(packet) + + return true +} + +func (t *MySqlMatcher) checkSSL(packet *packet.Packet) { + + temp := make([]byte, packet.Len) + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + if err := binary.Read(r, binary.LittleEndian, &temp); err != nil { + return + } + + t.version = strings.Split(string(packet.Buffer)[5:packet.Len], "\x00")[0] + versionLen := len(t.version) + 1 + data := binary.LittleEndian.Uint16(temp[18+versionLen : 20+versionLen]) + s := fmt.Sprintf("%b", data) + + for i, b := range s { + if i == 4 { + if b == 49 { + t.isSSL = true + } + } + } +} diff --git a/discovery/core/scan/service/matcher/mysql/mysql_test.go b/discovery/core/scan/service/matcher/mysql/mysql_test.go new file mode 100644 index 0000000..b94be2b --- /dev/null +++ b/discovery/core/scan/service/matcher/mysql/mysql_test.go @@ -0,0 +1,70 @@ +package mysql + +import ( + "crypto/tls" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestMySql(t *testing.T) { + + m := NewMySqlMatcher() + + /* + 192.168.1.103:3306 - normal + 192.168.1.105:8306 - ssl + 192.168.1.203:3306 - mysql with error code + */ + conn, _ := net.Dial("tcp", "192.168.1.215:3306") + + defer conn.Close() + + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(0, p, nil) { + t.Log(m.ServiceName()) + return + } + + t.Error("MySQL not found") + +} + +func TestCassandraTLS(t *testing.T) { + + m := NewMySqlMatcher() + + conn, err := tls.Dial( + "tcp", + "192.168.1.105:8306", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.105", + }, + ) + if err != nil { + t.Fatal(err) + } + + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log(m.ServiceName()) + return + } + + t.Error("MySQL not found") + } + +} diff --git a/discovery/core/scan/service/matcher/netbios/netbios.go b/discovery/core/scan/service/matcher/netbios/netbios.go new file mode 100644 index 0000000..bb21428 --- /dev/null +++ b/discovery/core/scan/service/matcher/netbios/netbios.go @@ -0,0 +1,109 @@ +package netbios + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + NBSS_SESSION_REQUEST uint8 = 0x81 + NBSS_POSITIVE_SESSION_RESPONSE uint8 = 0x82 + NBSS_NEGATIVE_SESSION_RESPONSE uint8 = 0x83 + ADDR string = "192.168.1.202:139" +) + +type netBios struct { + MsgType uint8 + Flags uint8 //0-6 : Reserved, must be zero. 7 : Length extension. + Length uint16 + CalledNameLen uint8 + CalledName [16]uint16 + _ uint8 + CallingNameLen uint8 + CallingName [16]uint16 + _ uint8 +} + +type NetBiosMatcher struct { + packets []*packet.Packet +} + +func NewNetBiosMatcher() *NetBiosMatcher { + + nbssMatcher := &NetBiosMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, netBios{}) + + query := netBios{ + MsgType: NBSS_SESSION_REQUEST, + Flags: 0x00, + Length: 0x4400, + CalledNameLen: 0x20, + CallingNameLen: 0x20, + } + + query.CalledName[0] = 0x4D45 // L + query.CalledName[1] = 0x4745 // F + + query.CallingName[0] = 0x4D45 + query.CallingName[1] = 0x4745 + + for i := 2; i < 16; i++ { + query.CalledName[i] = 0x4143 //Space + query.CallingName[i] = 0x4143 + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, query) + + nbssMatcher.packets = append(nbssMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return nbssMatcher +} + +func (t *NetBiosMatcher) ServiceName() string { + return "NBSS" +} + +func (t *NetBiosMatcher) PacketCount() int { + return len(t.packets) +} +func (t *NetBiosMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *NetBiosMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *NetBiosMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *NetBiosMatcher) IsPrePacket() bool { + return false +} + +func (t *NetBiosMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + n := netBios{} + if err := binary.Read(reader, binary.LittleEndian, &n); err != nil { + return false + } + + if NBSS_NEGATIVE_SESSION_RESPONSE != n.MsgType { + return false + } + + return true + +} diff --git a/discovery/core/scan/service/matcher/netbios/netbios_test.go b/discovery/core/scan/service/matcher/netbios/netbios_test.go new file mode 100644 index 0000000..587d73e --- /dev/null +++ b/discovery/core/scan/service/matcher/netbios/netbios_test.go @@ -0,0 +1,33 @@ +package netbios + +import ( + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "net" + "testing" +) + +func TestNBSS(t *testing.T) { + + m := NewNetBiosMatcher() + + conn, _ := net.Dial("tcp", "192.168.1.202:139") + + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("NBSS found") + return + } + + t.Error("NBSS not found") + } + +} diff --git a/discovery/core/scan/service/matcher/oracle/oracle.go b/discovery/core/scan/service/matcher/oracle/oracle.go new file mode 100644 index 0000000..c4b7021 --- /dev/null +++ b/discovery/core/scan/service/matcher/oracle/oracle.go @@ -0,0 +1,187 @@ +package oracle + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +type OracleMatcher struct { + sendPackets []*packet.Packet +} + +func (o *OracleMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + header := header_packet{} + refuse := body_refuse{} + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + binary.Read(buf, binary.BigEndian, &header) + binary.Read(buf, binary.BigEndian, &refuse) + + //fmt.Println(header) + //fmt.Println(refuse) + + if header.Check_sum != 0 { + return false + } + if header.Types != 4 { + return false + } + if header.Reserved_byte != 0 { + return false + } + if header.Header_sum != 0 { + return false + } + if refuse.Reason_user != 34 { + return false + } + if refuse.Reason_system != 0 { + return false + } + + var dataLen int = int(refuse.Data_len) + if dataLen != packet.Len-12 { + return false + } + + return true +} + +func (o *OracleMatcher) PacketCount() int { + return len(o.sendPackets) +} +func (o *OracleMatcher) Packet(index int) *packet.Packet { + return o.sendPackets[index] +} +func (o *OracleMatcher) ServiceName() string { + return "POP3Matcher" +} + +func (o *OracleMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (o *OracleMatcher) IsNoResponse(index int) bool { + + return false +} + +func (o *OracleMatcher) IsPrePacket() bool { + return true +} + +func NewOracleMatcher() *OracleMatcher { + + pm := OracleMatcher{} + + hp := header_packet{ + Length: 247, + Check_sum: 0, + Types: 1, + Reserved_byte: 0, + Header_sum: 0, + } + + bc := body_connect{ + Version: 315, + Version_compatible: 300, + //Service_options: + Session_unit_size: 8192, + Maxumum_trans_data_unit_size: 65535, + //Nt_protocol_characteristics: + Line_turnaround_value: 0, + Value_of_1_in_hardware: 1, + Length_of_connect_data: 177, + Offset_to_connect_data: 70, + Maximum_receivable_connect_data: 0, + //Connect_flag0: + //Connect_flag1: + Trace_cross_facility_item_1: 0, + Trace_cross_facility_item_2: 0, + Trace_unique_connection_id: 0, + //Unknown_data: + //Connect_data: + + } + + bc.Service_options[0] = 0x0c + bc.Service_options[1] = 0x41 + + bc.Nt_protocol_characteristics[0] = 0x4f + bc.Nt_protocol_characteristics[1] = 0x98 + + bc.Connect_flag0 = 0x81 + bc.Connect_flag1 = 0x81 + + bc.Unknown_data[10] = 0x20 + bc.Unknown_data[13] = 0x20 + + conDataStr := "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=192.168.1.30)(PORT=1521))(CONNECT_DATA=(CID=(PROGRAM=JDBC Thin Client)(HOST=__jdbc__)(USER=Jackdaw))(SERVICE_NAME=oracle.loafle.co1m)))" + connect_data := make([]byte, len(conDataStr)) + copy(connect_data, conDataStr) + + hpBuf := new(bytes.Buffer) + binary.Write(hpBuf, binary.BigEndian, hp) + + hpBt := hpBuf.Bytes() + + bcBuf := new(bytes.Buffer) + binary.Write(bcBuf, binary.BigEndian, bc) + bcBt := bcBuf.Bytes() + + byteSize := len(hpBt) + len(bcBt) + len(conDataStr) + sendByte := make([]byte, byteSize) + + copy(sendByte[0:], hpBt) + copy(sendByte[len(hpBt):], bcBt) + copy(sendByte[len(hpBt)+len(bcBt):], connect_data) + + pm.sendPackets = append(pm.sendPackets, packet.NewPacket(sendByte, byteSize)) + + return &pm + +} + +type header_packet struct { + Length uint16 + Check_sum uint16 + Types byte + Reserved_byte byte + Header_sum uint16 +} + +type body_connect struct { + Version uint16 + Version_compatible uint16 + Service_options [2]byte + Session_unit_size uint16 + Maxumum_trans_data_unit_size uint16 + Nt_protocol_characteristics [2]byte + Line_turnaround_value uint16 + Value_of_1_in_hardware uint16 + Length_of_connect_data uint16 + Offset_to_connect_data uint16 + Maximum_receivable_connect_data uint32 + Connect_flag0 byte + Connect_flag1 byte + Trace_cross_facility_item_1 uint32 + Trace_cross_facility_item_2 uint32 + Trace_unique_connection_id uint64 + Unknown_data [20]byte + //Connect_data []byte +} + +type body_refuse struct { + Reason_user byte + Reason_system byte + Data_len uint16 + //Data []byte +} diff --git a/discovery/core/scan/service/matcher/oracle/oracle_test.go b/discovery/core/scan/service/matcher/oracle/oracle_test.go new file mode 100644 index 0000000..27ad8ac --- /dev/null +++ b/discovery/core/scan/service/matcher/oracle/oracle_test.go @@ -0,0 +1,52 @@ +package oracle + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestOracle(t *testing.T) { + + lm := NewOracleMatcher() + + port := types.NewPort("1521", types.NewHost("192.168.1.30"), types.TYPE_TCP) + scanInfo := scaninfo.NewServiceScanInfo(port) + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + read, _ := client.Read(bytes) + + fmt.Println(bytes) + + b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) + + if b { + t.Log("Good") + } + + } + + t.Log(scanInfo) + +} diff --git a/discovery/core/scan/service/matcher/packet/packet.go b/discovery/core/scan/service/matcher/packet/packet.go new file mode 100644 index 0000000..9c7c1af --- /dev/null +++ b/discovery/core/scan/service/matcher/packet/packet.go @@ -0,0 +1,13 @@ +package packet + +type Packet struct { + Buffer []byte + Len int +} + +func NewPacket(buf []byte, len int) *Packet { + return &Packet{ + Buffer: buf, + Len: len, + } +} diff --git a/discovery/core/scan/service/matcher/pop/pop.go b/discovery/core/scan/service/matcher/pop/pop.go new file mode 100644 index 0000000..006ff3a --- /dev/null +++ b/discovery/core/scan/service/matcher/pop/pop.go @@ -0,0 +1,73 @@ +package pop + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + COMPARE_STR = "+OK" +) + +type POPMatcher struct { + sendPackets []*packet.Packet +} + +func (p *POPMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + switch index { + case 0: + fallthrough + case 1: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 3 { + return false + } + + compareStr := recvStr[0:3] + + if compareStr == COMPARE_STR { + return true + } + + } + + return false +} + +func (p *POPMatcher) PacketCount() int { + return len(p.sendPackets) +} +func (p *POPMatcher) Packet(index int) *packet.Packet { + return p.sendPackets[index] +} +func (p *POPMatcher) ServiceName() string { + return "POPMatcher" +} + +func (p *POPMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (p *POPMatcher) IsNoResponse(index int) bool { + + return false +} + +func (p *POPMatcher) IsPrePacket() bool { + return true +} + +func NewPOPMatcher() *POPMatcher { + + pm := POPMatcher{} + + reqStr := "QUIT\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + pm.sendPackets = append(pm.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &pm + +} diff --git a/discovery/core/scan/service/matcher/pop/pop_test.go b/discovery/core/scan/service/matcher/pop/pop_test.go new file mode 100644 index 0000000..de8e4f5 --- /dev/null +++ b/discovery/core/scan/service/matcher/pop/pop_test.go @@ -0,0 +1,85 @@ +package pop + +import ( + "crypto/tls" + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestPopTLS(t *testing.T) { + conn, _ := tls.Dial( + "tcp", + "192.168.1.215:995", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.215", + }, + ) + + defer conn.Close() + + pop3Run(conn, t) +} + +func TestPopNor(t *testing.T) { + + client, _ := net.Dial("tcp", "192.168.1.215:110") + + defer client.Close() + + pop3Run(client, t) + +} + +func pop3Run(client net.Conn, t *testing.T) { + + lm := NewPOPMatcher() + + port := types.NewPort("110", types.NewHost("192.168.1.215"), types.TYPE_TCP) + scanInfo := scaninfo.NewServiceScanInfo(port) + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + + bytett := make([]byte, 1024) + + read, _ := client.Read(bytett) + + bb := lm.Match(0, packet.NewPacket(bytett, read), scanInfo) + + if bb { + t.Log("good!") + } + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + //fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + rr, _ := client.Read(bytes) + + //fmt.Println(bytes) + + b := lm.Match(ii+1, packet.NewPacket(bytes, rr), scanInfo) + + if b { + t.Log("send Good!") + } + + } + + t.Log(scanInfo) + +} diff --git a/discovery/core/scan/service/matcher/postgresql/postgresql.go b/discovery/core/scan/service/matcher/postgresql/postgresql.go new file mode 100644 index 0000000..3685ef0 --- /dev/null +++ b/discovery/core/scan/service/matcher/postgresql/postgresql.go @@ -0,0 +1,155 @@ +package postgresql + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + RESPONSE_TYPE_ERR uint8 = 0x45 +) + +type pgsql struct { + Len uint32 + MessageType uint16 + _ uint16 + Name [5]byte + NameValue byte + Db [9]byte + DBValue byte + Encoding [16]byte + EncodingValue [5]byte + DateStyle [10]byte + DateStyleValue [4]byte + TimeZone [9]byte + TimeZoneValue [11]byte + ExtraDigits [19]byte + ExtraDigitsValue uint16 + End byte +} + +type pgsqlErrResponse struct { + ResponseType uint8 + Len [4]byte + Severity [6]byte + _ byte + Code [6]byte + _ byte + Message [53]byte +} + +type PostgreSQLMatcher struct { + packets []*packet.Packet +} + +func NewPostgreSQLMatcher() *PostgreSQLMatcher { + + pgSqlMatcher := &PostgreSQLMatcher{} + + pg := pgsql{} + + pg.Len = 0x00000065 + pg.MessageType = 0x0003 + + var name [5]byte + copy(name[:], "user") + pg.Name = name + pg.NameValue = 0x00 + + var db [9]byte + copy(db[:], "database") + pg.Db = db + pg.DBValue = 0x00 + + var encoding [16]byte + copy(encoding[:], "client_encoding") + pg.Encoding = encoding + + var encodingValue [5]byte + copy(encodingValue[:], "UTF8") + pg.EncodingValue = encodingValue + + var dateStyle [10]byte + copy(dateStyle[:], "DateStyle") + pg.DateStyle = dateStyle + + var dateStyleValue [4]byte + copy(dateStyleValue[:], "ISO") + pg.DateStyleValue = dateStyleValue + + var timeZone [9]byte + copy(timeZone[:], "TimeZone") + pg.TimeZone = timeZone + + var timeZoneValue [11]byte + copy(timeZoneValue[:], "Asia/Seoul") + pg.TimeZoneValue = timeZoneValue + + var extraDigit [19]byte + copy(extraDigit[:], "extra_float_digits") + pg.ExtraDigits = extraDigit + + pg.ExtraDigitsValue = 0x3200 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.BigEndian, pg) + + pgSqlMatcher.packets = append(pgSqlMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return pgSqlMatcher +} + +func (t *PostgreSQLMatcher) ServiceName() string { + return "PostgreSQL" +} + +func (t *PostgreSQLMatcher) PacketCount() int { + return len(t.packets) +} +func (t *PostgreSQLMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *PostgreSQLMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *PostgreSQLMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *PostgreSQLMatcher) IsPrePacket() bool { + return false +} + +func (t *PostgreSQLMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + pg := pgsqlErrResponse{} + if err := binary.Read(reader, binary.BigEndian, &pg); err != nil { + return false + } + + if pg.ResponseType != RESPONSE_TYPE_ERR { + return false + } + + if string(pg.Severity[:]) != "SFATAL" { + return false + } + + if string(pg.Code[:]) != "C28000" { + return false + } + + return true + +} diff --git a/discovery/core/scan/service/matcher/postgresql/postgresql_test.go b/discovery/core/scan/service/matcher/postgresql/postgresql_test.go new file mode 100644 index 0000000..0d927d3 --- /dev/null +++ b/discovery/core/scan/service/matcher/postgresql/postgresql_test.go @@ -0,0 +1,34 @@ +package postgresql + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestPG(t *testing.T) { + m := NewPostgreSQLMatcher() + + conn, err := net.Dial("tcp", "192.168.1.88:5432") + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("PostgreSQL found") + return + } + + t.Error("PostgreSQL not found") + } +} diff --git a/discovery/core/scan/service/matcher/redis/redis.go b/discovery/core/scan/service/matcher/redis/redis.go new file mode 100644 index 0000000..8acadc9 --- /dev/null +++ b/discovery/core/scan/service/matcher/redis/redis.go @@ -0,0 +1,76 @@ +package redis + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strings" +) + +const REDIS_PING string = "*1\r\n$4\r\nping\r\n" + +type RedisMatcher struct { + packets []*packet.Packet +} + +func NewRedisMatcher() *RedisMatcher { + + redisMatcher := &RedisMatcher{} + + redisMatcher.packets = append(redisMatcher.packets, packet.NewPacket([]byte(REDIS_PING), len(REDIS_PING))) + + return redisMatcher +} + +func (t *RedisMatcher) ServiceName() string { + return "Redis" +} + +func (t *RedisMatcher) PacketCount() int { + return len(t.packets) +} +func (t *RedisMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *RedisMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *RedisMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *RedisMatcher) IsPrePacket() bool { + return false +} + +func (t *RedisMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + resp := strings.Split(string(packet.Buffer), "\r\n")[0] + if len(resp) <= 0 { + return false + } + + sign := string([]rune(resp)[0]) + if len(sign) <= 0 { + return false + } + + if sign == "+" { + if resp == "+PONG" || resp == "+OK" { + return true + } + } + if sign == "-" { + if resp == "-NOAUTH" || resp == "-ERR" { + return true + } + } + + return false + +} diff --git a/discovery/core/scan/service/matcher/redis/redisProtected.go b/discovery/core/scan/service/matcher/redis/redisProtected.go new file mode 100644 index 0000000..9cf3a8d --- /dev/null +++ b/discovery/core/scan/service/matcher/redis/redisProtected.go @@ -0,0 +1,78 @@ +package redis + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + COMPARE_STR_1 = "-" + COMPARE_STR_2 = "DENIED" +) + +type RedisProtectedMatcher struct { +} + +func NewRedisProtectedMatcher() *RedisProtectedMatcher { + + redisMatcher := &RedisProtectedMatcher{} + + return redisMatcher +} + +func (r *RedisProtectedMatcher) ServiceName() string { + return "RedisProtectedMatcher" +} + +func (r *RedisProtectedMatcher) PacketCount() int { + return 0 +} +func (r *RedisProtectedMatcher) Packet(index int) *packet.Packet { + return nil +} + +func (r *RedisProtectedMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (r *RedisProtectedMatcher) IsNoResponse(index int) bool { + return false +} + +func (r *RedisProtectedMatcher) IsPrePacket() bool { + return true +} + +func (r *RedisProtectedMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + switch index { + case 0: + str := string(packet.Buffer[:packet.Len]) + + if str == "" { + return false + } + if len(str) <= 0 { + return false + } + + firstCompare := str[0:1] + seconcdCompare := str[1 : len(COMPARE_STR_2)+1] + + if firstCompare != COMPARE_STR_1 { + return false + } + if seconcdCompare != COMPARE_STR_2 { + return false + } + + return true + } + + return false + +} diff --git a/discovery/core/scan/service/matcher/redis/redisProtected_test.go b/discovery/core/scan/service/matcher/redis/redisProtected_test.go new file mode 100644 index 0000000..fda9cca --- /dev/null +++ b/discovery/core/scan/service/matcher/redis/redisProtected_test.go @@ -0,0 +1,33 @@ +package redis + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestRedisProtected(t *testing.T) { + + m := NewRedisProtectedMatcher() + + conn, err := net.Dial("tcp", "192.168.1.215:8379") + + if err != nil { + t.Log(err) + return + } + + defer conn.Close() + + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + + //fmt.Println(string(bytes[:n])) + + b := m.Match(0, packet.NewPacket(bytes, n), nil) + + if b { + t.Log("good!") + } + +} diff --git a/discovery/core/scan/service/matcher/redis/redis_test.go b/discovery/core/scan/service/matcher/redis/redis_test.go new file mode 100644 index 0000000..cfee834 --- /dev/null +++ b/discovery/core/scan/service/matcher/redis/redis_test.go @@ -0,0 +1,37 @@ +package redis + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +const ( + ADDR string = "192.168.1.215:6379" +) + +func TestRedisMatcher(t *testing.T) { + + m := NewRedisMatcher() + + conn, _ := net.Dial("tcp", ADDR) + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("Redis found.") + return + } + + t.Error("Redis not found") + } + +} diff --git a/discovery/core/scan/service/matcher/rmi/rmi.go b/discovery/core/scan/service/matcher/rmi/rmi.go new file mode 100644 index 0000000..656816e --- /dev/null +++ b/discovery/core/scan/service/matcher/rmi/rmi.go @@ -0,0 +1,111 @@ +package rmi + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + MAGIC_NUMBER = 0x4a524d49 + STREAM_PROTOCOL = 0x4b + VERSION = 0x0002 + ACK_PROTOCOL = 0x4e +) + +type RMI_SEND_MESSAGE struct { + magic uint32 + version uint16 + protocol uint8 +} + +type RMI_RECV_MESSAGE struct { + streamMessage uint8 + packetLen uint16 + host []byte + port [2]byte +} +type RMIMatcher struct { + sendPackets []*packet.Packet +} + +func (r *RMIMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + //fmt.Println("packet :", packet) + rmiRecv := RMI_RECV_MESSAGE{} + + buf := bytes.NewReader(packet.Buffer) + binary.Read(buf, binary.BigEndian, &rmiRecv.streamMessage) + binary.Read(buf, binary.BigEndian, &rmiRecv.packetLen) + + lenInt := int(rmiRecv.packetLen) + + var tempHost = make([]byte, lenInt, lenInt) + copy(rmiRecv.host, tempHost) + + rmiRecv.host = tempHost + + binary.Read(buf, binary.BigEndian, &rmiRecv.host) + binary.Read(buf, binary.BigEndian, &rmiRecv.port) + + hostIp := string(rmiRecv.host[:lenInt]) + //fmt.Println(hostIp) + + //hostPort := binary.BigEndian.Uint16(rmiRecv.port[:2]) + + if rmiRecv.streamMessage == ACK_PROTOCOL && lenInt == len(hostIp) { + result = true + } + + return result +} + +func (r *RMIMatcher) PacketCount() int { + return len(r.sendPackets) +} + +func (r *RMIMatcher) Packet(index int) *packet.Packet { + return r.sendPackets[index] +} + +func (r *RMIMatcher) ServiceName() string { + return "RMI" +} + +func (r *RMIMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (r *RMIMatcher) IsNoResponse(index int) bool { + return false +} + +func (r *RMIMatcher) IsPrePacket() bool { + return false +} + +func NewRMIMatcher() *RMIMatcher { + + r := RMIMatcher{} + rsm := RMI_SEND_MESSAGE{ + magic: MAGIC_NUMBER, + version: VERSION, + protocol: STREAM_PROTOCOL, + } + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.BigEndian, rsm) + + sendByte1 := mCache.Bytes() + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + r.sendPackets = append(r.sendPackets, pp) + + return &r +} diff --git a/discovery/core/scan/service/matcher/rmi/rmi_test.go b/discovery/core/scan/service/matcher/rmi/rmi_test.go new file mode 100644 index 0000000..146d495 --- /dev/null +++ b/discovery/core/scan/service/matcher/rmi/rmi_test.go @@ -0,0 +1,48 @@ +package rmi + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestNew(t *testing.T) { + r := NewRMIMatcher() + fmt.Println("TestNew: ", r) +} +func TestRMIMatcher_Match(t *testing.T) { + + fmt.Println("Match") + + hm := NewRMIMatcher() + + port := types.NewPort("9840", types.NewHost("192.168.1.101"), types.TYPE_TCP) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + //fmt.Println(ipport) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + pack := hm.Packet(0) + + fmt.Println(pack.Buffer) + + //writer.WriteString(pack) + client.Write(pack.Buffer) + + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + //fmt.Println(bytes) + + t1 := hm.Match(0, packet.NewPacket(bytes, l), nil) + + fmt.Println(t1) +} diff --git a/discovery/core/scan/service/matcher/smb/smb.go b/discovery/core/scan/service/matcher/smb/smb.go new file mode 100644 index 0000000..c258c10 --- /dev/null +++ b/discovery/core/scan/service/matcher/smb/smb.go @@ -0,0 +1,161 @@ +package smb + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strings" +) + +const ( + SMB_COM_NEGOTIATE uint8 = 0x72 + SMB_SUCCESS uint8 = 0x00 +) + +type netBIOS struct { + MsgType byte + MsgLength [3]uint8 +} + +type smb struct { + NetBios netBIOS + Component [4]uint8 + SmbCommand uint8 + NtStatus [4]uint8 + Flags uint8 + Flags2 [2]uint8 + ProcessId uint16 + Signature uint64 + Reserved uint16 + Tid uint16 + Pid uint16 + Uid uint16 + Mid uint16 + Wct uint8 + Bcc uint16 + Bf1 uint8 + Name1 [23]uint8 + Bf2 uint8 + Name2 [10]uint8 + Bf3 uint8 + Name3 [28]uint8 + Bf4 uint8 + Name4 [10]uint8 + Bf5 uint8 + Name5 [10]uint8 + Bf6 uint8 + Name6 [11]uint8 +} + +type SMBMatcher struct { + packets []*packet.Packet +} + +func NewSMBMatcher() *SMBMatcher { + + nbssMatcher := &SMBMatcher{} + + query := smb{} + query.NetBios.MsgType = 0x00 + query.NetBios.MsgLength[2] = 0x85 + + query.Component[0] = 0xff + query.Component[1] = 'S' + query.Component[2] = 'M' + query.Component[3] = 'B' + + query.SmbCommand = SMB_COM_NEGOTIATE + query.NtStatus[3] = SMB_SUCCESS + query.Flags = 0x18 + query.Flags2[0] = 0x53 + query.Flags2[1] = 0xC8 + + query.ProcessId = 0x00 + query.Signature = 0x00 + query.Reserved = 0 + query.Tid = 0 + query.Pid = 0xfeff + query.Uid = 0 + query.Mid = 0 + query.Wct = 0 + query.Bcc = 0x0062 + + query.Bf1 = 0x02 + copy(query.Name1[:], "PC NETWORK PROGRAM 1.0") + + query.Bf2 = 0x02 + copy(query.Name2[:], "LANMAN1.0") + + query.Bf3 = 0x02 + copy(query.Name3[:], "Windows for Workgroups 3.1a") + + query.Bf4 = 0x02 + copy(query.Name4[:], "LM1.2X002") + + query.Bf5 = 0x02 + copy(query.Name5[:], "LANMAN2.1") + + query.Bf6 = 0x02 + copy(query.Name6[:], "NT LM 0.12") + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, query) + + nbssMatcher.packets = append(nbssMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return nbssMatcher +} + +func (t *SMBMatcher) ServiceName() string { + return "SMB" +} + +func (t *SMBMatcher) PacketCount() int { + return len(t.packets) +} +func (t *SMBMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SMBMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *SMBMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *SMBMatcher) IsPrePacket() bool { + return false +} + +func (t *SMBMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + s := smb{} + if err := binary.Read(reader, binary.BigEndian, &s); err != nil { + return false + } + + var des [4]byte + copy(des[1:], s.NetBios.MsgLength[:]) + packetLen := binary.BigEndian.Uint32(des[:]) + + if packetLen != uint32(packet.Len-4) { + return false + } + + if !strings.Contains(string(s.Component[:]), "SMB") { + return false + } + + return true + +} diff --git a/discovery/core/scan/service/matcher/smb/smb_test.go b/discovery/core/scan/service/matcher/smb/smb_test.go new file mode 100644 index 0000000..60781d1 --- /dev/null +++ b/discovery/core/scan/service/matcher/smb/smb_test.go @@ -0,0 +1,37 @@ +package smb + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +const ( + ADDR string = "192.168.1.104:139" +) + +func TestSMBMatcher(t *testing.T) { + + m := NewSMBMatcher() + + conn, _ := net.Dial("tcp", ADDR) + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("SMB found.") + return + } + + t.Error("SMB not found") + } + +} diff --git a/discovery/core/scan/service/matcher/smtp/smtp.go b/discovery/core/scan/service/matcher/smtp/smtp.go new file mode 100644 index 0000000..7fd0681 --- /dev/null +++ b/discovery/core/scan/service/matcher/smtp/smtp.go @@ -0,0 +1,71 @@ +package smtp + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strings" +) + +type SmtpMatcher struct { + packets []*packet.Packet +} + +func NewSmtpMatcher() *SmtpMatcher { + + m := &SmtpMatcher{} + b := []byte("helo test\r\n") + m.packets = append(m.packets, packet.NewPacket(b, len(b))) + b = []byte("quit\r\n") + m.packets = append(m.packets, packet.NewPacket(b, len(b))) + return m +} + +func (t *SmtpMatcher) ServiceName() string { + return "SMTP" +} + +func (t *SmtpMatcher) PacketCount() int { + return len(t.packets) +} +func (t *SmtpMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SmtpMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *SmtpMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *SmtpMatcher) IsPrePacket() bool { + return true +} + +func (t *SmtpMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + if packet == nil { + return false + } + buf := string(packet.Buffer) + if len(buf) == 0 || len(buf) < 5 { + return false + } + + splits := strings.Split(buf, "\r\n") + splits = strings.Split(buf, " ") + if index == 0 { + if splits[0] == "220" { + return true + } + } else if index == 1 { + if splits[0] == "250" { + return true + } + } else if index == 2 { + if splits[0] == "221" { + return true + } + } + return false +} diff --git a/discovery/core/scan/service/matcher/smtp/smtp_test.go b/discovery/core/scan/service/matcher/smtp/smtp_test.go new file mode 100644 index 0000000..8d4d5b4 --- /dev/null +++ b/discovery/core/scan/service/matcher/smtp/smtp_test.go @@ -0,0 +1,67 @@ +package smtp + +import ( + "crypto/tls" + "fmt" + "github.com/stretchr/testify/assert" + "net" + "strings" + "testing" +) + +func TestSMTPTLS(t *testing.T) { + + conn, err := tls.Dial("tcp", + "192.168.1.215:465", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.215", + }, + ) + + if err != nil { + t.Log(err) + return + } + + b := make([]byte, 1024) + + check(t, b, conn, "", "220") + check(t, b, conn, "helo test\r\n", "250") + check(t, b, conn, "quit\r\n", "221") + + conn.Close() +} + +func TestSMTP(t *testing.T) { + + conn, _ := net.Dial("tcp", "192.168.1.215:25") + + b := make([]byte, 1024) + + check(t, b, conn, "", "220") + check(t, b, conn, "helo test\r\n", "250") + check(t, b, conn, "quit\r\n", "221") + + conn.Close() + +} +func check(t *testing.T, b []byte, conn net.Conn, cmd string, compare string) { + + if cmd != "" { + wlen, _ := conn.Write([]byte(cmd)) + assert.Equal(t, wlen, len(cmd)) + } + + rlen, _ := conn.Read(b) + + fmt.Println(rlen) + fmt.Println(len(b)) + + data := string(b[:rlen]) + fmt.Println(data) + assert.Equal(t, true, rlen > 4) + splits := strings.Split(data, " ") + assert.Equal(t, compare, splits[0]) + +} diff --git a/discovery/core/scan/service/matcher/snmp/snmpv2.go b/discovery/core/scan/service/matcher/snmp/snmpv2.go new file mode 100644 index 0000000..c3c233b --- /dev/null +++ b/discovery/core/scan/service/matcher/snmp/snmpv2.go @@ -0,0 +1,200 @@ +package snmp + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + SNMP_START_SEQUENCE uint8 = 0X30 + SNMP_TYPE_INTEGER uint8 = 0X02 + SNMP_TYPE_STRING uint8 = 0X04 + SNMP_TYPE_NULL uint8 = 0X05 + SNMP_TYPE_OBJECT uint8 = 0X06 + SNMP_GET_REQUEST uint8 = 0XA0 + //SNMP_RESPONSE uint8 = 0XA2 + SNMP_NO_DESC uint16 = 0X0004 + SNMP_END_SEQUENCE uint8 = 0X30 + SNMP_PROTOCOL_VERSION_3 uint8 = 0X03 + SNMP_PROTOCOL_VERSION_2c uint8 = 0X01 + SNMP_MSG_ID_MAX_VALUE uint32 = 0xFFFFFF7F +) + +type snmpv2VarBinding struct { + VarBindindStart uint8 + VarBindLen uint8 + ObjectStart uint8 + ObjectLen uint8 + ValueType uint8 + ValueLen uint8 + ObjectValue uint64 + NullValue uint8 + EndIndicator uint8 +} + +type snmpv2Data struct { + DataType uint8 + DataLen uint8 + RequestIdType uint8 + RequestIdLen uint8 + RequestId uint8 + ErrorStatusType uint8 + ErrorStatusLen uint8 + ErrorStatus uint8 + ErrorIndexType uint8 + ErrorIndexLen uint8 + ErrorIndex uint8 + + VarBinding snmpv2VarBinding +} + +type snmpv2 struct { + StartSeq uint8 + SeqLen uint8 + SNMPVersionType uint8 + SNMPVersionLen uint8 + SNMPVersion uint8 + CommunityVersionType uint8 + CommunityVersionLen uint8 + Community [6]byte + + Data snmpv2Data +} + +type SNMPv2Matcher struct { + packets []*packet.Packet +} + +func NewSNMPv2Matcher() *SNMPv2Matcher { + + snmpMatcher := &SNMPv2Matcher{} + + snmpTempBuf := new(bytes.Buffer) + binary.Write(snmpTempBuf, binary.BigEndian, snmpv2{}) //For getting the struct size + + snmpDataTempBuf := new(bytes.Buffer) + binary.Write(snmpDataTempBuf, binary.BigEndian, snmpv2Data{}) //For getting the struct size + + snmpVarTempBuf := new(bytes.Buffer) + binary.Write(snmpVarTempBuf, binary.BigEndian, snmpv2VarBinding{}) //For getting the struct size + + q := snmpv2{} + q.StartSeq = SNMP_START_SEQUENCE + q.SeqLen = uint8(len(snmpTempBuf.Bytes())) - 2 + q.SNMPVersionType = SNMP_TYPE_INTEGER + q.SNMPVersionLen = 0x01 + q.SNMPVersion = SNMP_PROTOCOL_VERSION_2c + q.CommunityVersionType = SNMP_TYPE_STRING + q.CommunityVersionLen = 0x06 + var community [6]byte + copy(community[:], "public") + q.Community = community + q.Data.DataType = SNMP_GET_REQUEST + q.Data.DataLen = uint8(len(snmpDataTempBuf.Bytes())) - 2 + q.Data.RequestIdType = SNMP_TYPE_INTEGER + q.Data.RequestIdLen = 0x01 + q.Data.RequestId = 0x01 + q.Data.ErrorStatusType = SNMP_TYPE_INTEGER + q.Data.ErrorStatusLen = 0x01 + q.Data.ErrorStatus = 0x00 + q.Data.ErrorIndexType = SNMP_TYPE_INTEGER + q.Data.ErrorIndexLen = 0x01 + q.Data.ErrorIndex = 0x00 + q.Data.VarBinding.VarBindindStart = SNMP_START_SEQUENCE + q.Data.VarBinding.VarBindLen = uint8(len(snmpVarTempBuf.Bytes())) - 2 + q.Data.VarBinding.ObjectStart = SNMP_START_SEQUENCE + q.Data.VarBinding.ObjectLen = uint8(len(snmpVarTempBuf.Bytes())) - 4 + q.Data.VarBinding.ValueType = SNMP_TYPE_OBJECT + q.Data.VarBinding.ValueLen = 0x08 + q.Data.VarBinding.ObjectValue = 0x000001010201062b + q.Data.VarBinding.NullValue = SNMP_TYPE_NULL + q.Data.VarBinding.EndIndicator = 0x00 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, q) + + snmpMatcher.packets = append(snmpMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return snmpMatcher +} + +func (t *SNMPv2Matcher) ServiceName() string { + return "SNMP" +} + +func (t *SNMPv2Matcher) PacketCount() int { + return len(t.packets) +} +func (t *SNMPv2Matcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SNMPv2Matcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (t *SNMPv2Matcher) IsNoResponse(index int) bool { + return false +} + +func (t *SNMPv2Matcher) IsPrePacket() bool { + return false +} + +func (t *SNMPv2Matcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + s := snmpv2{} + if err := binary.Read(r, binary.BigEndian, &s); err != nil { + return false + } + + if s.StartSeq != SNMP_START_SEQUENCE { + return false + } + + var p uint8 + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + for idx := 0; idx < 5; idx++ { + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + if p == SNMP_TYPE_INTEGER { + break + } + p++ + } + + //finding protocol version type : 0x02 0x01 0x01 + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + + if p == 0x01 { + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + if p == 0x01 { + return true + } + } + + return false + +} + +func (t *SNMPv2Matcher) IsSend(port int) bool { + if port == 161 { + return true + } + return false +} diff --git a/discovery/core/scan/service/matcher/snmp/snmpv2_test.go b/discovery/core/scan/service/matcher/snmp/snmpv2_test.go new file mode 100644 index 0000000..f575275 --- /dev/null +++ b/discovery/core/scan/service/matcher/snmp/snmpv2_test.go @@ -0,0 +1,36 @@ +package snmp + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestSNMP2(t *testing.T) { + + m := NewSNMPv2Matcher() + + conn, err := net.Dial("udp", "192.168.1.215:161") + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("SNMP found") + return + } + + t.Error("SNMP not found") + } + +} diff --git a/discovery/core/scan/service/matcher/snmp/snmpv3.go b/discovery/core/scan/service/matcher/snmp/snmpv3.go new file mode 100644 index 0000000..eb9d6f0 --- /dev/null +++ b/discovery/core/scan/service/matcher/snmp/snmpv3.go @@ -0,0 +1,218 @@ +package snmp + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +type snmpv3GlobalData struct { + GlobalDataStartSeq uint8 + GlobalDataLen uint8 + MsgIdType uint8 + MsgIdLen uint8 + MsgId uint32 + MsgMaxSizeType uint8 + MsgMaxSizeLen uint8 + MsgMaxSize [3]uint8 + MsgFlagsType uint8 + MsgFlagsTypeLen uint8 + MsgFlags uint8 + MsgSecurityModelType uint8 + MsgSecurityModelLen uint8 + MsgSecurityModel uint8 +} + +type snmpv3MsgData struct { + MsgDataStartSeq uint8 + MsgDataLen uint8 + ContextEngineId uint16 + ContextEngineName uint16 + SnmpType uint8 + Len uint8 + RequestIdType uint8 + RequestIdLen uint8 + RequestId uint32 + ErrorStatusType uint8 + ErrorStatusLen uint8 + ErrorStatus uint8 + ErrorIndexType uint8 + ErrorIndexLen uint8 + ErrorIndex uint8 + EndSeq uint8 + EndIndicator uint8 +} + +type snmpv3 struct { + StartSeq uint8 + SeqLen uint8 + SNMPVersionType uint8 + SNMPVersionLen uint8 + SNMPVersion uint8 + MsgGlobalData snmpv3GlobalData + Unk1 uint16 + Unk2 uint16 + MsgAuthoritativeEngineId uint16 + + MsgAuthoritativeEngineBootsType uint8 + MsgAuthoritativeEngineBootsLen uint8 + MsgAuthoritativeEngineBoots uint8 + MsgAuthoritativeEngineTimeType uint8 + MsgAuthoritativeEngineTimeLen uint8 + MsgAuthoritativeEngineTime uint8 + MsgUserName uint16 + MsgAuthenticationParam uint16 + MsgPrivacyParam uint16 + MsgData snmpv3MsgData +} + +type SNMPv3Matcher struct { + packets []*packet.Packet +} + +func NewSNMPv3Matcher() *SNMPv3Matcher { + + snmpMatcher := &SNMPv3Matcher{} + + snmpTempBuf := new(bytes.Buffer) + binary.Write(snmpTempBuf, binary.BigEndian, snmpv3{}) //For getting the struct size + + snmpMsgDataTempBuf := new(bytes.Buffer) + binary.Write(snmpMsgDataTempBuf, binary.BigEndian, snmpv3MsgData{}) //For getting the struct size + + snmpGlobalTempBuf := new(bytes.Buffer) + binary.Write(snmpGlobalTempBuf, binary.BigEndian, snmpv3GlobalData{}) //For getting the struct size + + q := snmpv3{} + q.StartSeq = SNMP_START_SEQUENCE + q.SeqLen = uint8(len(snmpTempBuf.Bytes())) - 2 + q.SNMPVersionType = SNMP_TYPE_INTEGER + q.SNMPVersionLen = 0x01 + q.SNMPVersion = SNMP_PROTOCOL_VERSION_3 + q.MsgGlobalData.GlobalDataStartSeq = SNMP_START_SEQUENCE + q.MsgGlobalData.GlobalDataLen = uint8(len(snmpGlobalTempBuf.Bytes())) - 2 + q.MsgGlobalData.MsgIdType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgIdLen = 0x04 + q.MsgGlobalData.MsgId = SNMP_MSG_ID_MAX_VALUE + q.MsgGlobalData.MsgMaxSizeType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgMaxSizeLen = 0x03 + q.MsgGlobalData.MsgMaxSize[2] = 0xe3 + q.MsgGlobalData.MsgMaxSize[1] = 0xff + q.MsgGlobalData.MsgMaxSize[0] = 0x00 + q.MsgGlobalData.MsgFlagsType = SNMP_TYPE_STRING + q.MsgGlobalData.MsgFlagsTypeLen = 0x01 + q.MsgGlobalData.MsgFlags = 0x04 + q.MsgGlobalData.MsgSecurityModelType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgSecurityModelLen = 0x01 + q.MsgGlobalData.MsgSecurityModel = 0x03 + q.Unk1 = 0x1004 + q.Unk2 = 0x0e30 + q.MsgAuthoritativeEngineId = SNMP_NO_DESC + q.MsgAuthoritativeEngineBootsType = SNMP_TYPE_INTEGER + q.MsgAuthoritativeEngineBootsLen = 0x01 + q.MsgAuthoritativeEngineBoots = 0x00 + q.MsgAuthoritativeEngineTimeType = SNMP_TYPE_INTEGER + q.MsgAuthoritativeEngineTimeLen = 0x01 + q.MsgAuthoritativeEngineTime = 0x00 + q.MsgUserName = SNMP_NO_DESC + q.MsgAuthenticationParam = SNMP_NO_DESC + q.MsgPrivacyParam = SNMP_NO_DESC + + q.MsgData.MsgDataStartSeq = SNMP_START_SEQUENCE + q.MsgData.MsgDataLen = uint8(len(snmpMsgDataTempBuf.Bytes())) - 2 + q.MsgData.ContextEngineId = SNMP_NO_DESC + q.MsgData.ContextEngineName = SNMP_NO_DESC + q.MsgData.SnmpType = SNMP_GET_REQUEST + q.MsgData.Len = 0x0E + q.MsgData.RequestIdType = SNMP_TYPE_INTEGER + q.MsgData.RequestIdLen = 0x04 + q.MsgData.RequestId = 0x00 // + q.MsgData.ErrorStatusType = SNMP_TYPE_INTEGER + q.MsgData.ErrorStatusLen = 0x01 + q.MsgData.ErrorStatus = 0x00 + q.MsgData.ErrorIndexType = SNMP_TYPE_INTEGER + q.MsgData.ErrorIndexLen = 0x01 + q.MsgData.ErrorIndex = 0x00 + q.MsgData.EndSeq = SNMP_END_SEQUENCE + q.MsgData.EndIndicator = 0x00 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, q) + + snmpMatcher.packets = append(snmpMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return snmpMatcher +} + +func (t *SNMPv3Matcher) ServiceName() string { + return "SNMP" +} + +func (t *SNMPv3Matcher) PacketCount() int { + return len(t.packets) +} + +func (t *SNMPv3Matcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SNMPv3Matcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (t *SNMPv3Matcher) IsNoResponse(index int) bool { + return false +} + +func (t *SNMPv3Matcher) IsPrePacket() bool { + return false +} + +func (t *SNMPv3Matcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + s := snmpv3{} + if err := binary.Read(reader, binary.LittleEndian, &s); err != nil { + return false + } + + if s.StartSeq != SNMP_START_SEQUENCE { + return false + } + + var p uint8 + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + for { + binary.Read(r, binary.LittleEndian, &p) + + if p == SNMP_TYPE_INTEGER { + break + } + } + + binary.Read(r, binary.BigEndian, &p) + if p == 0x01 { + binary.Read(r, binary.BigEndian, &p) + if p == 0x03 { + return true + } + } + + return false + +} + +func (t *SNMPv3Matcher) IsSend(port int) bool { + if port == 161 { + return true + } + return false +} diff --git a/discovery/core/scan/service/matcher/snmp/snmpv3_test.go b/discovery/core/scan/service/matcher/snmp/snmpv3_test.go new file mode 100644 index 0000000..0b2d59f --- /dev/null +++ b/discovery/core/scan/service/matcher/snmp/snmpv3_test.go @@ -0,0 +1,36 @@ +package snmp + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "net" + "testing" +) + +func TestSNMP3(t *testing.T) { + + m := NewSNMPv3Matcher() + + conn, err := net.Dial("udp", "192.168.1.254:161") + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("SNMP found") + return + } + + t.Error("SNMP not found") + } + +} diff --git a/discovery/core/scan/service/matcher/ssh/ssh.go b/discovery/core/scan/service/matcher/ssh/ssh.go new file mode 100644 index 0000000..156a161 --- /dev/null +++ b/discovery/core/scan/service/matcher/ssh/ssh.go @@ -0,0 +1,66 @@ +package ssh + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "strings" +) + +type SSHMatcher struct { + sendPackets []*packet.Packet +} + +func (ssh *SSHMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + str := string(packet.Buffer) + + //fmt.Println(str) + temps := strings.Split(str, " ") + protocol := strings.Split(temps[0], "-") + //osType := temps[1] + + if 0 == strings.Compare(protocol[0], "SSH") { + majorVersion := protocol[1] + //fmt.Println(majorVersion) + if 0 == strings.Compare(majorVersion, "2.0") || 0 == strings.Compare(majorVersion, "1.0") { + result = true + } + } + + return result +} + +func (ssh *SSHMatcher) PacketCount() int { + return len(ssh.sendPackets) +} + +func (ssh *SSHMatcher) Packet(index int) *packet.Packet { + return ssh.sendPackets[index] +} + +func (ssh *SSHMatcher) ServiceName() string { + return "SSH" +} + +func (ssh *SSHMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (ssh *SSHMatcher) IsNoResponse(index int) bool { + return false +} + +func (ssh *SSHMatcher) IsPrePacket() bool { + return true +} + +func NewSSHMatcher() *SSHMatcher { + r := SSHMatcher{} + + return &r +} diff --git a/discovery/core/scan/service/matcher/ssh/ssh_test.go b/discovery/core/scan/service/matcher/ssh/ssh_test.go new file mode 100644 index 0000000..324f5ff --- /dev/null +++ b/discovery/core/scan/service/matcher/ssh/ssh_test.go @@ -0,0 +1,33 @@ +package ssh + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestSSHMatcher_Match(t *testing.T) { + + port := types.NewPort("22", types.NewHost("192.168.1.103"), types.TYPE_TCP) + ssh := NewSSHMatcher() + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + fmt.Println(bytes) + + b := ssh.Match(0, packet.NewPacket(bytes, l), nil) + + fmt.Println(b) + +} diff --git a/discovery/core/scan/service/matcher/telnet/telnet.go b/discovery/core/scan/service/matcher/telnet/telnet.go new file mode 100644 index 0000000..fba2f38 --- /dev/null +++ b/discovery/core/scan/service/matcher/telnet/telnet.go @@ -0,0 +1,79 @@ +package telnet + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + DO = 0xfd + WONT = 0x4b + WILL = 0xfb + DONT = 0xfe + CMD = 0xff +) + +type TelnetMatcher struct { + sendPackets []*packet.Packet +} + +func (tel *TelnetMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + buf := make([]byte, 0, 0) + count := 0 + + for i := 0; i < len(packet.Buffer); i++ { + if packet.Buffer[i] > 0 { + buf = append(buf, packet.Buffer[i]) + } else if count > 2 { + break + } else { + count++ + } + } + + for idx := 0; idx < len(buf); idx += 3 { + if buf[idx] == CMD && (buf[idx+1] == DO || buf[idx+1] == WONT || buf[idx+1] == WILL || buf[idx+1] == DONT) { + result = true + } else { + result = false + } + } + + return result +} + +func (tel *TelnetMatcher) PacketCount() int { + return len(tel.sendPackets) +} + +func (tel *TelnetMatcher) Packet(index int) *packet.Packet { + return tel.sendPackets[index] +} + +func (tel *TelnetMatcher) ServiceName() string { + return "Telnet" +} + +func (tel *TelnetMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} + +func (tel *TelnetMatcher) IsNoResponse(index int) bool { + return false +} + +func (tel *TelnetMatcher) IsPrePacket() bool { + return true +} + +func NewTelnetMatcher() *TelnetMatcher { + r := TelnetMatcher{} + + return &r +} diff --git a/discovery/core/scan/service/matcher/telnet/telnet_test.go b/discovery/core/scan/service/matcher/telnet/telnet_test.go new file mode 100644 index 0000000..1713822 --- /dev/null +++ b/discovery/core/scan/service/matcher/telnet/telnet_test.go @@ -0,0 +1,33 @@ +package telnet + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestTelnetMatcher_Match(t *testing.T) { + + port := types.NewPort("23", types.NewHost("192.168.1.210"), types.TYPE_TCP) + telnet := NewTelnetMatcher() + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + fmt.Println("length :", l) + fmt.Println(bytes) + + b := telnet.Match(0, packet.NewPacket(bytes, l), nil) + + fmt.Println(b) +} diff --git a/discovery/core/scan/service/matcher/wmi/wmi.go b/discovery/core/scan/service/matcher/wmi/wmi.go new file mode 100644 index 0000000..3318a85 --- /dev/null +++ b/discovery/core/scan/service/matcher/wmi/wmi.go @@ -0,0 +1,250 @@ +package wmi + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" +) + +const ( + PDU_BIND = 11 + PDU_BIND_ACK = 12 + PDU_REQ = 0 + PDU_RESP = 2 + + WMI_CALL_ID_1 = 0x95 + WMI_CALL_ID_2 = 0x96 +) + +type WMIMatcher struct { + sendPackets []*packet.Packet +} + +func (w *WMIMatcher) Match(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + wmiRecv := DCERPC_DEFAULT{} + + binary.Read(buf, binary.LittleEndian, &wmiRecv) + + switch index { + case 0: + if wmiRecv.Call_id != WMI_CALL_ID_1 { + return false + } + + if wmiRecv.Ptype != PDU_BIND_ACK { + return false + } + + return true + case 1: + + if wmiRecv.Call_id != WMI_CALL_ID_2 { + return false + } + + if wmiRecv.Ptype != PDU_RESP { + return false + } + + return true + } + + return false +} + +func (w *WMIMatcher) PacketCount() int { + return len(w.sendPackets) +} +func (w *WMIMatcher) Packet(index int) *packet.Packet { + return w.sendPackets[index] +} +func (w *WMIMatcher) ServiceName() string { + return "WMI" +} + +func (w *WMIMatcher) IsError(index int, packet *packet.Packet, info *types.ServiceScanInfo) bool { + return false +} +func (w *WMIMatcher) IsNoResponse(index int) bool { + + return false +} + +func (w *WMIMatcher) IsPrePacket() bool { + return true +} + +func NewWMIMatcher() *WMIMatcher { + + wm := WMIMatcher{} + + ds1 := DCERPC_DEFAULT{ + Rpc_ver: 5, + Rpc_ver_minor: 0, + Ptype: PDU_BIND, + Flags: 0x03, + Drep: 0x10, + Frag_len: 16 + 56, + Auth_len: 0, + Call_id: WMI_CALL_ID_1, + } + + ds2 := DCERPC_DEFAULT{ + Rpc_ver: 5, + Rpc_ver_minor: 0, + Ptype: PDU_REQ, + Flags: 0x03, + Drep: 0x10, + Frag_len: 16 + 8, + Auth_len: 0, + Call_id: WMI_CALL_ID_2, + } + + ioxidr := DCERPC_IOXIDResolver{ + MaxXmitFrag: 0x16d0, + MaxRecvFrag: 0x16d0, + + AssocGroup: 0, + NumCtxItem: 1, + + ContextId: 0, + NumTransItem: 1, + + //interfaces + + InterfaceVer: 0, + InterfaceVerMinor: 0, + + //transSyntax + + TransSyntaxVer: 2, + } + + ioxidr.Interfaces[0] = 0xc4 + ioxidr.Interfaces[1] = 0xfe + ioxidr.Interfaces[2] = 0xfc + ioxidr.Interfaces[3] = 0x99 + + ioxidr.Interfaces[4] = 0x60 + ioxidr.Interfaces[5] = 0x52 + + ioxidr.Interfaces[6] = 0x1b + ioxidr.Interfaces[7] = 0x10 + + ioxidr.Interfaces[8] = 0xbb + ioxidr.Interfaces[9] = 0xcb + + ioxidr.Interfaces[10] = 0x00 + ioxidr.Interfaces[11] = 0xaa + ioxidr.Interfaces[12] = 0x00 + ioxidr.Interfaces[13] = 0x21 + ioxidr.Interfaces[14] = 0x34 + ioxidr.Interfaces[15] = 0x7a + + ioxidr.TransSyntax[0] = 0x04 + ioxidr.TransSyntax[1] = 0x5d + ioxidr.TransSyntax[2] = 0x88 + ioxidr.TransSyntax[3] = 0x8a + + ioxidr.TransSyntax[4] = 0xeb + ioxidr.TransSyntax[5] = 0x1c + + ioxidr.TransSyntax[6] = 0xc9 + ioxidr.TransSyntax[7] = 0x11 + + ioxidr.TransSyntax[8] = 0x9f + ioxidr.TransSyntax[9] = 0xe8 + + ioxidr.TransSyntax[10] = 0x08 + ioxidr.TransSyntax[11] = 0x00 + ioxidr.TransSyntax[12] = 0x2b + ioxidr.TransSyntax[13] = 0x10 + ioxidr.TransSyntax[14] = 0x48 + ioxidr.TransSyntax[15] = 0x60 + + da := DCERPC_ALIVE{ + AllocHint: 0, + ContextId: 0, + OpNum: 3, + } + + buf1 := new(bytes.Buffer) + + binary.Write(buf1, binary.LittleEndian, ds1) + ds1Bytes := buf1.Bytes() + + buf2 := new(bytes.Buffer) + + binary.Write(buf2, binary.LittleEndian, ds2) + ds2Bytes := buf2.Bytes() + + buf3 := new(bytes.Buffer) + + binary.Write(buf3, binary.LittleEndian, ioxidr) + ioxidrBytes := buf3.Bytes() + + buf4 := new(bytes.Buffer) + + binary.Write(buf4, binary.LittleEndian, da) + daBytes := buf4.Bytes() + + firstByte := make([]byte, len(ds1Bytes)+len(ioxidrBytes)) + + copy(firstByte[0:], ds1Bytes) + copy(firstByte[len(ds1Bytes):], ioxidrBytes) + + secondByte := make([]byte, len(ds2Bytes)+len(daBytes)) + + copy(secondByte[0:], ds2Bytes) + copy(secondByte[len(ds2Bytes):], daBytes) + + wm.sendPackets = append(wm.sendPackets, packet.NewPacket(firstByte, len(ds1Bytes)+len(ioxidrBytes))) + wm.sendPackets = append(wm.sendPackets, packet.NewPacket(secondByte, len(ds2Bytes)+len(daBytes))) + + return &wm + +} + +type DCERPC_DEFAULT struct { + Rpc_ver uint8 + Rpc_ver_minor uint8 + Ptype uint8 + Flags uint8 + Drep uint32 + Frag_len uint16 + Auth_len uint16 + Call_id uint32 +} + +type DCERPC_ALIVE struct { + AllocHint uint32 + ContextId uint16 + OpNum uint16 +} + +type DCERPC_IOXIDResolver struct { + MaxXmitFrag uint16 + MaxRecvFrag uint16 + AssocGroup uint32 + NumCtxItem uint8 + UnknownCode [3]uint8 + + ContextId uint16 + NumTransItem uint8 + UnknownCode2 uint8 + + Interfaces [16]uint8 + InterfaceVer uint16 + InterfaceVerMinor uint16 + TransSyntax [16]uint8 + TransSyntaxVer uint32 +} diff --git a/discovery/core/scan/service/matcher/wmi/wmi_test.go b/discovery/core/scan/service/matcher/wmi/wmi_test.go new file mode 100644 index 0000000..b838484 --- /dev/null +++ b/discovery/core/scan/service/matcher/wmi/wmi_test.go @@ -0,0 +1,51 @@ +package wmi + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "testing" +) + +func TestWMI(t *testing.T) { + + lm := NewWMIMatcher() + + port := types.NewPort("135", types.NewHost("192.168.1.1"), types.TYPE_TCP) + scanInfo := scaninfo.NewServiceScanInfo(port) + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + fmt.Println(ipport) + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + fmt.Println(lm.PacketCount()) + + for ii := 0; ii < lm.PacketCount(); ii++ { + + pack := lm.Packet(ii) + + fmt.Println(pack) + + client.Write(pack.Buffer) + + bytes := make([]byte, 1024) + + read, _ := client.Read(bytes) + + //fmt.Println(bytes) + + b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) + + if b { + fmt.Println("Good") + } + + } + + t.Log(scanInfo) +} diff --git a/discovery/core/scan/service/postpacket.go b/discovery/core/scan/service/postpacket.go new file mode 100644 index 0000000..7aa286e --- /dev/null +++ b/discovery/core/scan/service/postpacket.go @@ -0,0 +1,83 @@ +package service + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + log "github.com/cihub/seelog" +) + +func processPostPacket(mats []matcher.Matcher, info *types.ServiceScanInfo, m ConnFactory) bool { + + var ser *types.DiscoveryService = nil + var curMatcher matcher.Matcher = nil + isFound := false + + for i := 0; i < len(mats); i++ { + + info.History = make([]*types.ServiceScanHistory, 0) + conn, err := m.createSocket(info.Port.Host.Ip_, info.Port.Port_) + if err != nil { + log.Error(m.Type() + " process postpacket create socket error :" + info.Port.Host.Ip_ + " port: " + info.Port.Port_ + " error: " + err.Error()) + break + } + curMatcher = mats[i] + + for indexI := 0; indexI < curMatcher.PacketCount(); indexI++ { + + p := curMatcher.Packet(indexI) + l, err := conn.Write(p.Buffer) + if err != nil { + break + } + if l != p.Len { + break + } + AddServiceScanHistory( + &info.History, + types.NewServiceScanHistory(info.Port, curMatcher.ServiceName(), types.DIRECTION_SEND, p.Buffer)) + + buf := make([]byte, 1024) + l, err = conn.Read(buf) + + AddServiceScanHistory( + &info.History, + types.NewServiceScanHistory(info.Port, curMatcher.ServiceName(), types.DIRECTION_RECV, p.Buffer)) + + if err != nil { + if curMatcher.IsNoResponse(indexI) == true { + ser = types.NewService(m.Type(), curMatcher.ServiceName(), info.Port) + isFound = true + break + } + break + + } + + if curMatcher.Match(indexI, packet.NewPacket(buf, l), info) == true { + if indexI == (curMatcher.PacketCount() - 1) { + ser = types.NewService(m.Type(), curMatcher.ServiceName(), info.Port) + break + } + isFound = true + continue + } else { + isFound = false + break + } + + } + if isFound == true { + conn.Close() + break + } + conn.Close() + } + + AddService(ser, info) + if ser == nil { + return false + } else { + return true + } +} diff --git a/discovery/core/scan/service/prepacket.go b/discovery/core/scan/service/prepacket.go new file mode 100644 index 0000000..099ed26 --- /dev/null +++ b/discovery/core/scan/service/prepacket.go @@ -0,0 +1,76 @@ +package service + +import ( + "git.loafle.net/overflow/discovery/core/scan/service/matcher" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" +) + +func processPrePacket(conn net.Conn, prepacket *packet.Packet, mats []matcher.Matcher, info *types.ServiceScanInfo, m ConnFactory) bool { + + defer conn.Close() + + isFound := false + var ser *types.DiscoveryService = nil + for i := 0; i < len(mats); i++ { + + info.History = make([]*types.ServiceScanHistory, 0) + + AddServiceScanHistory( + &info.History, + types.NewServiceScanHistory(info.Port, mats[i].ServiceName(), types.DIRECTION_RECV, prepacket.Buffer)) + + if mats[i].Match(0, prepacket, info) == true { + packetCount := mats[i].PacketCount() + if packetCount == 0 { + ser = types.NewService(m.Type(), mats[i].ServiceName(), info.Port) + break + } + isFound = false + + for j := 0; j < packetCount; j++ { + + p := mats[i].Packet(j) + l, err := conn.Write(p.Buffer) + if err != nil { + break + } + if l != p.Len { + break + } + AddServiceScanHistory( + &info.History, + types.NewServiceScanHistory(info.Port, mats[i].ServiceName(), types.DIRECTION_SEND, p.Buffer)) + + buffer := make([]byte, 1024) + l, err = conn.Read(buffer) + if err != nil { + break + } + + AddServiceScanHistory( + &info.History, + types.NewServiceScanHistory(info.Port, mats[i].ServiceName(), types.DIRECTION_RECV, buffer)) + + if mats[i].Match(j+1, packet.NewPacket(buffer, l), info) == true { + isFound = true + } else { + isFound = false + break + } + } + if isFound == true { + ser = types.NewService(m.Type(), mats[i].ServiceName(), info.Port) + break + } + } + } + + AddService(ser, info) + if ser == nil { + return false + } else { + return true + } +} diff --git a/discovery/core/scan/service/service.go b/discovery/core/scan/service/service.go new file mode 100644 index 0000000..f934894 --- /dev/null +++ b/discovery/core/scan/service/service.go @@ -0,0 +1,177 @@ +package service + +import ( + "crypto/tls" + "errors" + log "github.com/cihub/seelog" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + //"git.loafle.net/overflow/central/client/events" + //"git.loafle.net/overflow/discovery/communicate" + "git.loafle.net/overflow/discovery/core/scan/service/matcher" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/packet" + "git.loafle.net/overflow/discovery/discovery/types" + "net" + "strconv" + "strings" +) + +func Scan(dp *types.DiscoveryPort) { + ip := dp.Host.Ip_ + port := dp.Port_ + + factories := []ConnFactory{ + ConnFactoryNormal{t: types.TYPE_TCP}, + ConnFactoryTLS{t: types.TYPE_TLS}, + } + info := types.NewServiceScanInfo(dp) + + for i := 0; i < len(factories); i++ { + m := factories[i] + + isFound := false + conn, err := m.createSocket(ip, port) + if err != nil { + log.Error("An error has occurred on " + m.Type() + " " + ip + ":" + port + " - " + err.Error()) + break + } + + buf := make([]byte, 1024) + l, err := conn.Read(buf) + if err != nil { + l = 0 + } + if l != 0 { + isFound = processPrePacket(conn, packet.NewPacket(buf, l), matcher.GetTcpMatchers(true), info, m) + } else { + conn.Close() + isFound = processPostPacket(matcher.GetTcpMatchers(false), info, m) + } + if isFound == true { + break + } + } +} + +func Check(dp *types.DiscoveryPort, services []*types.DiscoveryService) error { + + if len(services) <= 0 { + return errors.New("No service has bound to this port.") + } + for _, service := range services { + + m := matcher.GetMatcherByName(service.ServiceName) + if m == nil { + return errors.New("Not supported service : " + service.ServiceName) + } + + addr := dp.Host.Ip_ + ":" + dp.Port_ + return checkService(m, addr, dp.PortType) + } + + return nil +} + +func checkService(m matcher.Matcher, addr, portType string) error { + conn, err := getConnection(portType, addr) + if err != nil { + return err + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + return nil + } + } + + return errors.New("Fail") +} + +func getConnection(portType, addr string) (net.Conn, error) { + var conn net.Conn + var err error + if portType == types.TYPE_TLS { + conn, err = tls.Dial( + "tcp", + addr, + &tls.Config{ + InsecureSkipVerify: true, + ServerName: addr, + }, + ) + } else { + conn, err = net.Dial(strings.ToLower(portType), addr) + } + + if err != nil { + return nil, err + } + + return conn, nil + +} + +func MatchUDPPacket(zone *types.DiscoveryZone, ipString, portString string, udpLayer gopacket.Layer) { + + host, _ := zone.GetHost(ipString) + port := types.NewPort(portString, host, types.TYPE_UDP) + + mats := matcher.GetUdpMatchers() + info := types.NewServiceScanInfo(port) + + for i := 0; i < len(mats); i++ { + matcher := mats[i] + p := packet.NewPacket(udpLayer.LayerPayload(), len(udpLayer.LayerPayload())) + + if matcher.Match(0, p, info) { + key := portString + ":" + types.TYPE_UDP + host.SetPort(key, port) + + ser := types.NewService(types.TYPE_UDP, matcher.ServiceName(), info.Port) + AddService(ser, info) + } + } +} + +func AddService(ds *types.DiscoveryService, info *types.ServiceScanInfo) { + port := info.Port + var serviceName string + if ds != nil { + port.Services = append(port.Services, ds) + port.SetHistory(info.History) + serviceName = ds.ServiceName + log.Debugf("%s, %s:%s, Service was found : %s ", ds.PortType, port.Host.Ip_, port.Port_, ds.ServiceName) + } else { + i, _ := strconv.Atoi(port.Port_) + p := uint16(i) + expectName := layers.TCPPortNames[layers.TCPPort(p)] + serviceName = "Unknown Service was found. (Perhaps : " + expectName + ")" + log.Debugf("%s:%s. %s", port.Host.Ip_, port.Port_, serviceName) + } + + sendEvent(info, serviceName) +} + +func AddServiceScanHistory(histories *[]*types.ServiceScanHistory, h *types.ServiceScanHistory) { + *histories = append(*histories, h) +} + +func sendEvent(info *types.ServiceScanInfo, serviceName string) { + + //communicate.Send( + // events.NewEvent( + // events.CENTRAL_EVENT, + // events.NewServiceEndEvent( + // info.Port.Host.Zone.CidrInt64(), + // info.Port.Host.Ip, + // info.Port.Number, + // serviceName))) +} diff --git a/discovery/core/scan/zone/traceroute/traceroute.go b/discovery/core/scan/zone/traceroute/traceroute.go new file mode 100644 index 0000000..a4a7852 --- /dev/null +++ b/discovery/core/scan/zone/traceroute/traceroute.go @@ -0,0 +1,129 @@ +package traceroute + +import ( + "fmt" + "net" + "syscall" + "time" +) + +const PORT = 33434 +const MAX_HOPS = 30 +const TIMEOUT_MS = 500 +const RETRIES = 3 +const PACKET_SIZE = 52 + +type Hop struct { + Success bool + Address [4]byte + ElapsedTime time.Duration + TTL int +} + +func Traceroute(dest string, c chan<- Hop) error { + + ttl := 1 + retry := 0 + tv := syscall.NsecToTimeval(1000 * 1000 * TIMEOUT_MS) + + for { + start := time.Now() + + recvSock, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_ICMP) + if err != nil { + return err + } + + sendSock, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP) + if err != nil { + return err + } + syscall.SetsockoptInt(sendSock, 0x0, syscall.IP_TTL, ttl) + syscall.SetsockoptTimeval(recvSock, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv) + + defer syscall.Close(recvSock) + defer syscall.Close(sendSock) + + sockaddr, err := socketAddr() + if err != nil { + return err + } + syscall.Bind(recvSock, &syscall.SockaddrInet4{Port: PORT, Addr: sockaddr}) + + destaddr, err := destAddr(dest) + if err != nil { + return err + } + syscall.Sendto(sendSock, []byte{0x0}, 0, &syscall.SockaddrInet4{Port: PORT, Addr: destaddr}) + + var p = make([]byte, PACKET_SIZE) + _, from, err := syscall.Recvfrom(recvSock, p, 0) + elapsed := time.Since(start) + + if err == nil { + currAddr := from.(*syscall.SockaddrInet4).Addr + + hop := Hop{Success: true, Address: currAddr, ElapsedTime: elapsed, TTL: ttl} + c <- hop + + ttl += 1 + retry = 0 + + if ttl > MAX_HOPS || currAddr == destaddr { + close(c) + return nil + } + + } else { + retry += 1 + if retry > RETRIES { + hop := Hop{Success: false, TTL: ttl} + c <- hop + + ttl += 1 + retry = 0 + } + if ttl > MAX_HOPS { + close(c) + return nil + } + } + + } +} + +func socketAddr() (addr [4]byte, err error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return + } + + for _, a := range addrs { + if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if len(ipnet.IP.To4()) == net.IPv4len { + copy(addr[:], ipnet.IP.To4()) + return + } + } + } + return +} + +func destAddr(dest string) (destAddr [4]byte, err error) { + addrs, err := net.LookupHost(dest) + if err != nil { + return + } + addr := addrs[0] + + ipAddr, err := net.ResolveIPAddr("ip", addr) + if err != nil { + return + } + copy(destAddr[:], ipAddr.IP.To4()) + return +} + +func AddressToString(addr [4]byte) string { + return fmt.Sprintf("%v.%v.%v.%v", addr[0], addr[1], addr[2], addr[3]) +} diff --git a/discovery/core/scan/zone/traceroute/traceroute_test.go b/discovery/core/scan/zone/traceroute/traceroute_test.go new file mode 100644 index 0000000..6bd2040 --- /dev/null +++ b/discovery/core/scan/zone/traceroute/traceroute_test.go @@ -0,0 +1,28 @@ +package traceroute + +import ( + "fmt" + "testing" +) + +func TestTracerouteAsync(t *testing.T) { + + c := make(chan Hop, 0) + + go func() { + for { + hop, ok := <-c + if !ok { + fmt.Println() + return + } + t.Logf("%d. %v - %v\n", hop.TTL, AddressToString(hop.Address), hop.ElapsedTime) + + } + }() + + err := Traceroute("216.58.197.228", c) + if err != nil { + t.Error(err) + } +} diff --git a/discovery/core/scan/zone/zone.go b/discovery/core/scan/zone/zone.go new file mode 100644 index 0000000..3d89fe4 --- /dev/null +++ b/discovery/core/scan/zone/zone.go @@ -0,0 +1,151 @@ +package zone + +import ( + "github.com/cihub/seelog" + "net" + "strings" + + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/util/converter" + "strconv" + "sync" +) + +var checkStr = []string{ + "docker", + "lo", + "br-f23a505183ce", +} + +type FoundZoneHandler func(z *types.DiscoveryZone) + +func Scan(callback FoundZoneHandler) { + startZone(callback) +} + +func ScanAndCheck(zones *[]types.DiscoveryZone) { + +} + +func Check(zones *[]types.DiscoveryZone) { + +} + +func startZone(callback FoundZoneHandler) { + //var zones []*types.DiscoveryZone + + ifaces, _ := net.Interfaces() + + for _, i := range ifaces { + + if checkStrByInterfaces(i.Name) { + continue + } + + addrs, err := i.Addrs() + + if err != nil { + seelog.Critical(err) + continue + } + + for _, addr := range addrs { + + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + + if err != nil { + seelog.Critical(err) + continue + } + + cidrs := strings.Split(addr.String(), "/") + tempNet, err := strconv.Atoi(cidrs[1]) + if err != nil { + seelog.Critical(err) + } + netMask := uint8(tempNet) + f, l := GetMinAndMaxIPRange(addr.String()) + + first := net.ParseIP(f) + last := net.ParseIP(l) + zone := &types.DiscoveryZone{ + M: new(sync.RWMutex), + Netmask: netMask, + Hosts_: make(map[string]*types.DiscoveryHost, 255), + Ip: converter.IP4toInt(ipnet.IP), + FirstScanRange: converter.IP4toInt(first), + LastScanRange: converter.IP4toInt(last), + Mac: converter.MacToInt(i.HardwareAddr), + Iface: i.Name, + } + + if callback != nil { + callback(zone) + } + + } + } + + } + } +} + +// Test +func NewZone() *types.DiscoveryZone { + + var z = &types.DiscoveryZone{} + startZone(func(zone *types.DiscoveryZone) { + if zone != nil { + z = zone + } + }) + return z +} + +func GetMinAndMaxIPRange(ip string) (min string, max string) { + _, ipnet, err := net.ParseCIDR(ip) + + if err != nil { + return + } + + var ips []string + cc := strings.Split(ip, "/") + + cip := net.ParseIP(cc[0]) + for addr := cip.Mask(ipnet.Mask); ipnet.Contains(addr); inc(addr) { + ips = append(ips, addr.String()) + } + + if len(ips) < 3 { + return + } + + min = ips[1] + max = ips[len(ips)-2] + + return + +} + +func checkStrByInterfaces(inter string) bool { + + for _, str := range checkStr { + index := strings.Index(inter, str) + if index >= 0 { + return true + } + } + + return false +} + +func inc(ip net.IP) { + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } +} diff --git a/discovery/core/scan/zone/zone_test.go b/discovery/core/scan/zone/zone_test.go new file mode 100644 index 0000000..8ed8ee3 --- /dev/null +++ b/discovery/core/scan/zone/zone_test.go @@ -0,0 +1,29 @@ +package zone + +import ( + "fmt" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/util/converter" + "testing" +) + +func TestZoneScan(t *testing.T) { + zoneCallback := func(z *types.DiscoveryZone) { + //fmt.Println(z.MinIp_) + //fmt.Println(z.MaxIp_) + //fmt.Println(z.LocalIp_) + //fmt.Println(z.Cidr) + fmt.Println(z) + fmt.Println(z.CidrStr()) + } + + Scan(zoneCallback) +} + +func TestNewZone(t *testing.T) { + zone := NewZone() + fmt.Println(zone) + fmt.Println(zone.CidrInt64()) + fmt.Println(converter.Int64ToCidr(zone.CidrInt64())) + fmt.Println(zone.Iface) +} diff --git a/discovery/discovery/bridge.go b/discovery/discovery/bridge.go new file mode 100644 index 0000000..466a673 --- /dev/null +++ b/discovery/discovery/bridge.go @@ -0,0 +1,101 @@ +package discovery + +import ( + //"bytes" + "encoding/json" + "encoding/xml" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "github.com/cihub/seelog" + "io/ioutil" + //"net/http" + "os" +) + +type Conf struct { + Version string `xml:"version,attr"` + Url string `xml:"url"` + User string `xml:"user"` + Password string `xml:"password"` + BodyType string `xml:"bodyType"` +} + +func sendBridge(dhs []types.DiscoveryHistory) { + + fp, err := os.Open("bin/bridge_conf.xml") + if err != nil { + panic(err) + } + defer fp.Close() + + data, err := ioutil.ReadAll(fp) + + var conf Conf + xmlerr := xml.Unmarshal(data, &conf) + if xmlerr != nil { + panic(xmlerr) + } + + for _, dh := range dhs { + + dh.Result = true + dh.EndDate = timestamp.Now() + //map -> slice + convertMapToSlice(dh.Zone) + + data, _ := json.Marshal(&dh) + seelog.Debug("!!! " + string(data)) + + //for _, z := range dh.Zone.Histories { + // seelog.Debug("Host History : Start ") + // hostH, _ := json.Marshal(&z) + // seelog.Debug("Host" + string(hostH)) + // seelog.Debug("Host History : End ") + //} + // + //for _, h := range dh.Zone.Hosts { + // + // for _, hh := range h.Histories { + // portH, _ := json.Marshal(&hh) + // seelog.Debug("Port History Start ") + // //seelog.Debug("Port Number: " + hh.PortNumber) + // //seelog.Debug("Port Type: "+hh.PortType) + // seelog.Debug(string(portH)) + // seelog.Debug("Port History End ") + // } + // + // for _, p := range h.Ports { + // for _, sh := range p.Histories { + // seelog.Debug("Service History Start") + // serviceH, _ := json.Marshal(&sh) + // //seelog.Debug("Service Name : " + sh.ServiceName) + // seelog.Debug(string(serviceH)) + // seelog.Debug("Service History End") + // } + // + // } + //} + + //resp, _ := http.Post(conf.Url, conf.BodyType, bytes.NewBuffer(data)) + //body, _ := ioutil.ReadAll(resp.Body) + //seelog.Debug(resp.Status) + //seelog.Debug(resp.StatusCode) + //seelog.Debug(string(body)) + //resp.Body.Close() + } + +} + +func convertMapToSlice(zone *types.DiscoveryZone) { + + for _, host := range zone.Hosts_ { + zone.Hosts = append(zone.Hosts, host) + + for _, port := range host.Ports_ { + host.Ports = append(host.Ports, port) + + } + + } + +} diff --git a/discovery/discovery/discovery.go b/discovery/discovery/discovery.go new file mode 100644 index 0000000..5235925 --- /dev/null +++ b/discovery/discovery/discovery.go @@ -0,0 +1,73 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/core/scan/service" + "git.loafle.net/overflow/discovery/discovery/types" + "github.com/cihub/seelog" + "github.com/google/gopacket" + "sync" +) + +type discoverZoneCallback func(host *types.DiscoveryZone) +type discoverHostCallback func(host *types.DiscoveryHost) +type discoverTCPPortCallback func(port *types.DiscoveryPort) +type discoverUDPPortCallback func(zone *types.DiscoveryZone, ipString, portString string, udpLayer gopacket.Layer) + +func Discover(fin chan<- bool) { + + dhs := make([]types.DiscoveryHistory, 0) + var wg sync.WaitGroup + wg.Add(1) + DiscoverZone(&wg, true, &dhs) + wg.Wait() + sendBridge(dhs) + fin <- true +} + +func DiscoverZone(wg *sync.WaitGroup, hasHostScan bool, dhs *[]types.DiscoveryHistory) { + + discoveryZone(func(zone *types.DiscoveryZone) { + *dhs = append(*dhs, types.NewDiscoveryHistory(zone)) + if hasHostScan == true { + wg.Add(1) + go DiscoveryHost(zone, wg, true) + } + }) + wg.Done() +} + +func DiscoveryHost(zone *types.DiscoveryZone, wg *sync.WaitGroup, hasPortScan bool) { + discoveryHost(zone, func(host *types.DiscoveryHost) { + seelog.Debug("find host : " + host.Ip_) + if hasPortScan == true { + wg.Add(1) + go DiscoverPort(host, wg, hasPortScan) + } + }) + wg.Done() +} + +func DiscoverPort(host *types.DiscoveryHost, wg *sync.WaitGroup, hasServiceScan bool) { + discoverPort(host, + func(port *types.DiscoveryPort) { + if hasServiceScan == true { + wg.Add(1) + go DiscoverService(port, wg) + } + }, + func(zone *types.DiscoveryZone, ipString, portString string, udpLayer gopacket.Layer) { + wg.Add(1) + go DiscoverUDPService(zone, wg, ipString, portString, udpLayer) + }) + wg.Done() +} + +func DiscoverService(port *types.DiscoveryPort, wg *sync.WaitGroup) { + discoverService(port) + wg.Done() +} + +func DiscoverUDPService(zone *types.DiscoveryZone, wg *sync.WaitGroup, ipString, portString string, udpLayer gopacket.Layer) { + service.MatchUDPPacket(zone, ipString, portString, udpLayer) + wg.Done() +} diff --git a/discovery/discovery/host.go b/discovery/discovery/host.go new file mode 100644 index 0000000..87d2434 --- /dev/null +++ b/discovery/discovery/host.go @@ -0,0 +1,95 @@ +package discovery + +import ( + "errors" + "github.com/cihub/seelog" + "github.com/google/gopacket/layers" + //"git.loafle.net/overflow/central/client/events" + //"git.loafle.net/overflow/discovery/communicate" + "git.loafle.net/overflow/discovery/core/scan/host" + "git.loafle.net/overflow/discovery/core/scan/service/matcher/dhcp" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "git.loafle.net/overflow/discovery/util/converter" + "net" + "strconv" +) + +func discoveryHost(zone *types.DiscoveryZone, callback discoverHostCallback) { + + _, err := getHostRangeCount(zone) + if err != nil { + return + } + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewHostStartEvent(rangeCount, zone.CidrInt64()))) + + arpCallback := func(packet *layers.ARP) { + + keyIp := net.IP(packet.SourceProtAddress).String() + if _, ok := zone.GetHost(keyIp); ok == true { + return + } + + h := types.NewHost(packet.SourceProtAddress, packet.SourceHwAddress) + h.FirstScanRange = 1 + h.LastScanRange = 10000 + h.Zone = zone + zone.SetHost(keyIp, h) + + updateHostHistoryProperty(h, packet, h) + + //e := events.NewEvent(events.CENTRAL_EVENT, events.NewHostFoundEvent(zone.CidrInt64(), h.Ip)) + //communicate.Send(e) + callback(h) + } + + // arp send history callback // todo 히스토리 콜백이 필요??? + historyCallback := func(targetHost net.IP) { + hh := types.NewHostHistory(targetHost, types.TIMEOUT, "") + zone.AddHistory(hh) + } + + host.Scan(zone, arpCallback, historyCallback) + + //mdns + //nbns + dhcp.DiscoverDHCP(zone) + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewHostEndEvent(zone.CidrInt64(), zone.Histories))) +} +func getHostRangeCount(zone *types.DiscoveryZone) (int, error) { + minIp := converter.IntToIP4(zone.FirstScanRange).To4() + maxIp := converter.IntToIP4(zone.LastScanRange).To4() + if maxIp[3] == 0 || minIp[0] == 0 { + err := errors.New("target host range error") + seelog.Critical(err) + return -1, err + } + return int(maxIp[3]) - (int(minIp[3]) - 1), nil +} + +func updateHostHistoryProperty(host *types.DiscoveryHost, arp *layers.ARP, h *types.DiscoveryHost) { + + idx := findHistoriyByIP(len(host.Zone.Histories), func(i int) bool { + if host.Zone.Histories[i].Ip == h.Ip { + return true + } + return false + }) + + if idx >= 0 { + hh := host.Zone.Histories[idx] + hh.ResultType = types.SUCCESS + hh.ResultDate = timestamp.Now() + hh.Description = strconv.FormatInt(converter.MacToInt(arp.SourceHwAddress), 10) + } +} + +func findHistoriyByIP(limit int, predicate func(i int) bool) int { + for i := 0; i < limit; i++ { + if predicate(i) { + return i + } + } + return -1 +} diff --git a/discovery/discovery/host_test.go b/discovery/discovery/host_test.go new file mode 100644 index 0000000..8dd545b --- /dev/null +++ b/discovery/discovery/host_test.go @@ -0,0 +1,23 @@ +package discovery + +import ( + "fmt" + "git.loafle.net/overflow/discovery/core/scan/zone" + "sync" + "testing" +) + +func TestDiscoverHost(t *testing.T) { + + // todo zone생성 추후 변경 필요 + zone := zone.NewZone() + var wg sync.WaitGroup + DiscoveryHost(zone, &wg, false) + wg.Wait() + + m := zone.Hosts_ + for k := range m { + fmt.Println(k) + } + +} diff --git a/discovery/discovery/port.go b/discovery/discovery/port.go new file mode 100644 index 0000000..6132d2c --- /dev/null +++ b/discovery/discovery/port.go @@ -0,0 +1,90 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/discovery/types" + + "git.loafle.net/overflow/discovery/core/scan/port/tcp" + "git.loafle.net/overflow/discovery/core/scan/port/udp" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + + //"git.loafle.net/overflow/central/client/events" + //"git.loafle.net/overflow/discovery/communicate" + "strconv" + + "git.loafle.net/overflow/discovery/util/converter" + "sync" +) + +func discoverPort(host *types.DiscoveryHost, tcpCB discoverTCPPortCallback, udpCB discoverUDPPortCallback) { + + var wg sync.WaitGroup + wg.Add(1) + go discoverTcp(&wg, host, tcpCB) + wg.Add(1) + go discoverUdp(&wg, host, udpCB) + + wg.Wait() + +} + +func discoverTcp(wg *sync.WaitGroup, host *types.DiscoveryHost, serviceCB discoverTCPPortCallback) { + + recvCallback := func(tcp *layers.TCP) { + + if tcp == nil { + } else if tcp.DstPort != 60000 { + } else if tcp.RST { + types.NewPortScanHistory(host, uint16(tcp.SrcPort), types.TYPE_TCP, types.RECV_RST, "") + } else if tcp.SYN && tcp.ACK { + portString := strconv.Itoa(int(tcp.SrcPort)) + port := types.NewPort(portString, host, types.TYPE_TCP) + + types.NewPortScanHistory(host, uint16(tcp.SrcPort), types.TYPE_TCP, types.RECV, "") + key := portString + ":" + types.TYPE_TCP + host.SetPort(key, port) + + serviceCB(port) + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortFoundEvent(host.Zone.CidrInt64(), host.Ip, uint16(tcp.SrcPort), types.TYPE_TCP))) + } + + } + + tcp.Scan(host, recvCallback) + wg.Done() +} + +func discoverUdp(wg *sync.WaitGroup, host *types.DiscoveryHost, udpCB discoverUDPPortCallback) { + + rcb := func(packet gopacket.Packet) { + + ipLayer := packet.Layer(layers.LayerTypeIPv4) + + if ipLayer.(*layers.IPv4).SrcIP.String() == converter.IntToStr(host.Zone.Ip) { + return + } + + if net := packet.NetworkLayer(); net == nil { + } else if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer == nil { + } else if udp, ok := udpLayer.(*layers.UDP); ok { + + ipString := ipLayer.(*layers.IPv4).SrcIP.String() + portString := strconv.Itoa(int(udp.SrcPort)) + + if _, exists := host.Zone.GetHost(ipString); !exists { + return + } + + ph := types.NewPortScanHistory(host, uint16(udp.SrcPort), types.TYPE_UDP, types.RECV, "") + host.AddHistory(ph) + + //communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortFoundEvent(host.Zone.CidrInt64(), host.Ip, uint16(udp.SrcPort), types.TYPE_UDP))) + + udpCB(host.Zone, ipString, portString, udpLayer) + + } + } + + udp.Scan(host, rcb) + wg.Done() +} diff --git a/discovery/discovery/port_test.go b/discovery/discovery/port_test.go new file mode 100644 index 0000000..4080c9f --- /dev/null +++ b/discovery/discovery/port_test.go @@ -0,0 +1,84 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "testing" + + "fmt" + "github.com/google/gopacket" + + "git.loafle.net/overflow/discovery/core/scan/zone" + "git.loafle.net/overflow/discovery/util/converter" + "sync" +) + +func TestDiscoverTCPPort(t *testing.T) { + + zone := zone.NewZone() + + hh := &types.DiscoveryHost{ + Ip_: "192.168.1.215", + Ports_: make(map[string]*types.DiscoveryPort, 100), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 1, + LastScanRange: 100, + M: new(sync.RWMutex), + } + + wg := sync.WaitGroup{} + wg.Add(1) + discoverTcp(&wg, hh, func(port *types.DiscoveryPort) { + fmt.Println(port.Number) + }) + + wg.Wait() + //DiscoverPort(hh, false) + +} + +func TestDiscoverUDPPort(t *testing.T) { + + //zone := types.DiscoveryZone{ + // Ip:converter.StrToInt("192.168.1.207"), + // Netmask:24, + // Iface:"", + //} + + zone := zone.NewZone() + + hh := &types.DiscoveryHost{ + Ip_: "192.168.1.215", + Ports_: make(map[string]*types.DiscoveryPort, 100), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 53, + LastScanRange: 53, + } + + zone.Hosts_["192.168.1.215"] = hh + + wg := sync.WaitGroup{} + wg.Add(1) + discoverUdp(&wg, hh, func(zone *types.DiscoveryZone, ipString, portString string, udpLayer gopacket.Layer) { + fmt.Println(portString) + }) + + wg.Wait() + + //DiscoverPort(hh, false) + +} + +func TestIp(t *testing.T) { + + fmt.Println(converter.StrToInt("")) + +} diff --git a/discovery/discovery/service.go b/discovery/discovery/service.go new file mode 100644 index 0000000..35da61f --- /dev/null +++ b/discovery/discovery/service.go @@ -0,0 +1,10 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/core/scan/service" + "git.loafle.net/overflow/discovery/discovery/types" +) + +func discoverService(port *types.DiscoveryPort) { + service.Scan(port) +} diff --git a/discovery/discovery/service_test.go b/discovery/discovery/service_test.go new file mode 100644 index 0000000..4e64357 --- /dev/null +++ b/discovery/discovery/service_test.go @@ -0,0 +1,78 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/core/scan/service" + "git.loafle.net/overflow/discovery/core/scan/zone" + "git.loafle.net/overflow/discovery/discovery/types" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "github.com/stretchr/testify/assert" + "sync" + "testing" +) + +func TestServiceScan(t *testing.T) { + + _, p := getDummyPort() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + service.Scan(p) + wg.Done() + }() + wg.Wait() + + if len(p.Services) <= 0 { + t.Error("err") + return + } + + for _, v := range p.Services { + t.Log("service: ", v.ServiceName) + } + + assert.Equal(t, "SMB", p.Services[0].ServiceName) + +} + +func TestServiceCheck(t *testing.T) { + + _, p := getDummyPort() + + var services []*types.DiscoveryService = make([]*types.DiscoveryService, 0) + services = append(services, types.NewService(p.PortType, "SMB", p)) + + err := service.Check(p, services) + if err == nil { + t.Logf("%s is okay.", services[0].ServiceName) + } + + assert.Nil(t, err) +} + +func getDummyPort() (*types.DiscoveryHost, *types.DiscoveryPort) { + zone := zone.NewZone() + + h := &types.DiscoveryHost{ + Ip_: "192.168.1.104", + Ports_: make(map[string]*types.DiscoveryPort, 139), + Ip: 111, + Mac: 222, + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Zone: zone, + FirstScanRange: 1, + LastScanRange: 100, + } + + p := &types.DiscoveryPort{ + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + Host: h, + Number: 139, + Port_: "139", + PortType: types.TYPE_TCP, + } + + return h, p +} diff --git a/discovery/discovery/types/const.go b/discovery/discovery/types/const.go new file mode 100644 index 0000000..a899b90 --- /dev/null +++ b/discovery/discovery/types/const.go @@ -0,0 +1,21 @@ +package types + +const ( + TYPE_TCP = "TCP" + TYPE_UDP = "UDP" + TYPE_TLS = "TLS" + + SEND = "Send" + RECV = "Recv" + TIMEOUT = "Timeout" + RECV_RST = "Closed" + + SUCCESS = "Success" + + TYPE_DISCOVERY = "DISCOVERY" + TYPE_DETECTNETWORK = "DETECTNETWORK" + TYPE_SENSOR = "SENSOR" + + RANGE_TYPE_RANGE = "RANGE" + RANGE_TYPE_TARGET = "TARGET" +) diff --git a/discovery/discovery/types/discovery.go b/discovery/discovery/types/discovery.go new file mode 100644 index 0000000..7615eaa --- /dev/null +++ b/discovery/discovery/types/discovery.go @@ -0,0 +1,27 @@ +package types + +import ( + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "sync" +) + +type DiscoveryHistory struct { + ID int `json:"id,omitempty"` + StartDate timestamp.Timestamp `json:"startDate"` + EndDate timestamp.Timestamp `json:"endDate"` + Result bool `json:"result"` + Zone *DiscoveryZone `json:"zone"` +} + +func NewDiscoveryHistory(zone *DiscoveryZone) DiscoveryHistory { + return DiscoveryHistory{ + Zone: zone, + StartDate: timestamp.Now(), + } +} + +type DiscoveryMgr struct { + Stop chan bool + DiscoveryWg *sync.WaitGroup + Zone *DiscoveryZone +} diff --git a/discovery/discovery/types/discoveryhost.go b/discovery/discovery/types/discoveryhost.go new file mode 100644 index 0000000..e731327 --- /dev/null +++ b/discovery/discovery/types/discoveryhost.go @@ -0,0 +1,93 @@ +package types + +import ( + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "git.loafle.net/overflow/discovery/util/converter" + "net" + "sync" +) + +type DiscoveryHost struct { + M *sync.RWMutex `json:"-"` + Ip_ string `json:"-"` + Ports_ map[string]*DiscoveryPort `json:"-"` + HostName string `json:"-"` + PortDiscoveryTime timestamp.Timestamp `json:"-"` + Zone *DiscoveryZone `json:"-"` + Histories []*PortScanHistory `json:"-"` + + FirstScanRange uint16 `json:"firstScanRange"` + LastScanRange uint16 `json:"lastScanRange"` + Name string `json:"name"` + + ID int `json:"id,omitempty"` + Ip int64 `json:"ip"` + Mac int64 `json:"mac"` + Ports []*DiscoveryPort `json:"ports"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + UpdateDate timestamp.Timestamp `json:"updateDate,omitempty"` + + Os string `json:"os,omitempty"` + Target bool `json:"target,omitempty"` +} + +func (h *DiscoveryHost) SetPort(key string, value *DiscoveryPort) { + h.M.Lock() + defer h.M.Unlock() + h.Ports_[key] = value +} + +func (h *DiscoveryHost) GetPort(key string) *DiscoveryPort { + h.M.RLock() + defer h.M.RUnlock() + return h.Ports_[key] +} + +type HostScanHistory struct { + ID int `json:"id,omitempty"` + Ip int64 `json:"ip"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + SendDate timestamp.Timestamp `json:"sendDate"` + ResultDate timestamp.Timestamp `json:"resultDate"` + ResultType string `json:"resultType"` + Description string `json:"description"` + Zone *DiscoveryZone `json:"-"` +} + +type HostMgr struct { + DiscoveryMgr + ToPort chan *DiscoveryHost +} + +func NewHost(ip net.IP, macAddr net.HardwareAddr) *DiscoveryHost { + host := &DiscoveryHost{ + M: new(sync.RWMutex), + Ip_: ip.String(), + Ports_: make(map[string]*DiscoveryPort, 100), + Ip: converter.IP4toInt(ip), + Mac: converter.MacToInt(macAddr), + CreateDate: timestamp.Now(), + UpdateDate: timestamp.Now(), + } + return host +} + +func NewHostHistory(ip net.IP, resultType string, des string) *HostScanHistory { + hh := &HostScanHistory{ + Ip: converter.IP4toInt(ip), + SendDate: timestamp.Now(), + ResultType: resultType, + Description: des, + } + + return hh +} +func (p *DiscoveryHost) AddHistory(h *PortScanHistory) { + p.Histories = append(p.Histories, h) +} + +type PortRange struct { + Types string + First int + Second int +} diff --git a/discovery/discovery/types/discoveryport.go b/discovery/discovery/types/discoveryport.go new file mode 100644 index 0000000..668190f --- /dev/null +++ b/discovery/discovery/types/discoveryport.go @@ -0,0 +1,64 @@ +package types + +import ( + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + "strconv" +) + +type DiscoveryPort struct { + Port_ string `json:"-"` + + ID int `json:"id,omitempty"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + UpdateDate timestamp.Timestamp `json:"updateDate,omitempty"` + Host *DiscoveryHost `json:"-"` + Services []*DiscoveryService `json:"services"` + Histories []*ServiceScanHistory `json:"-"` + + PortType string `json:"portType"` + Number uint16 `json:"portNumber"` + + Target bool `json:"target,omitempty"` +} + +type PortScanHistory struct { + ID int `json:"id,omitempty"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + Host *DiscoveryHost `json:"-"` + PortNumber uint16 `json:"portNumber"` + PortType string `json:"portType"` + DirectionType string `json:"directionType"` + Description string `json:"description"` + StartDate timestamp.Timestamp `json:"startDate"` + EndDate timestamp.Timestamp `json:"endDate"` +} + +type PortMgr struct { + DiscoveryMgr + FromHost chan *DiscoveryHost + ToService chan *DiscoveryPort +} + +func NewPort(port string, host *DiscoveryHost, portType string) *DiscoveryPort { + rPort := &DiscoveryPort{Port_: port, Host: host, PortType: portType} + i, _ := strconv.ParseUint(port, 10, 16) + rPort.Number = uint16(i) + return rPort +} + +func NewPortScanHistory(host *DiscoveryHost, port uint16, portType string, directionType string, description string) *PortScanHistory { + rPortSH := &PortScanHistory{ + CreateDate: timestamp.Now(), + Host: host, + PortNumber: port, + PortType: portType, + DirectionType: directionType, + Description: description, + } + host.AddHistory(rPortSH) + return rPortSH +} + +func (p *DiscoveryPort) SetHistory(h []*ServiceScanHistory) { + p.Histories = h +} diff --git a/discovery/discovery/types/discoveryservice.go b/discovery/discovery/types/discoveryservice.go new file mode 100644 index 0000000..e28e183 --- /dev/null +++ b/discovery/discovery/types/discoveryservice.go @@ -0,0 +1,45 @@ +package types + +import ( + "git.loafle.net/overflow/discovery/discovery/types/timestamp" +) + +type DiscoveryService struct { + ID int `json:"id,omitempty"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + UpdateDate timestamp.Timestamp `json:"updateDate,omitempty"` + Port *DiscoveryPort `json:"-"` + PortType string `json:"portType"` /*tls or normal*/ + ServiceName string `json:"serviceName"` + + Target bool `json:"target,omitempty"` +} + +func NewService(ptype string, serviceName string, port *DiscoveryPort) *DiscoveryService { + service := &DiscoveryService{PortType: ptype, ServiceName: serviceName, Port: port} + return service +} + +const ( + DIRECTION_SEND = "Send" + DIRECTION_RECV = "Recv" +) + +type ServiceScanHistory struct { + ID int `json:"id,omitempty"` + CreateDate timestamp.Timestamp `json:"createDate,omitempty"` + Port *DiscoveryPort `json:"-"` + ServiceName string `json:"serviceName"` + Direction string `json:"direction"` + Packet []byte `json:"packet"` +} + +func NewServiceScanHistory(port *DiscoveryPort, serviceName string, direction string, packet []byte) *ServiceScanHistory { + return &ServiceScanHistory{ + CreateDate: timestamp.Now(), + Port: port, + ServiceName: serviceName, + Direction: direction, + Packet: packet, + } +} diff --git a/discovery/discovery/types/discoveryzone.go b/discovery/discovery/types/discoveryzone.go new file mode 100644 index 0000000..48dae29 --- /dev/null +++ b/discovery/discovery/types/discoveryzone.go @@ -0,0 +1,62 @@ +package types + +import ( + "git.loafle.net/overflow/discovery/util/converter" + "github.com/cihub/seelog" + "net" + "strconv" + "sync" +) + +type discoverTestZoneCallback func(host *DiscoveryZone) + +type DiscoveryZone struct { + ID int `json:"id,omitempty"` + Ip int64 `json:"ip"` + Netmask uint8 `json:"mask"` + Iface string `json:"iface"` + Mac int64 `json:"mac"` + FirstScanRange int64 `json:"firstScanRange"` + LastScanRange int64 `json:"lastScanRange"` + Histories []*HostScanHistory `json:"-"` + Hosts []*DiscoveryHost `json:"hosts"` + CurrentTypes string `json:"-"` + + Hosts_ map[string]*DiscoveryHost `json:"-"` + M *sync.RWMutex `json:"-"` +} + +func (z *DiscoveryZone) SetHost(ip string, value *DiscoveryHost) { + z.M.Lock() + defer z.M.Unlock() + z.Hosts_[ip] = value +} + +func (z *DiscoveryZone) GetHost(ip string) (*DiscoveryHost, bool) { + z.M.RLock() + defer z.M.RUnlock() + h, ok := z.Hosts_[ip] + return h, ok +} + +func (p *DiscoveryZone) AddHistory(h *HostScanHistory) { + p.Histories = append(p.Histories, h) +} + +func (z *DiscoveryZone) CidrStr() string { + tempIp := converter.IntToIP4(z.Ip) + tempNet := int(z.Netmask) + netmask := strconv.Itoa(tempNet) + str := tempIp.String() + "/" + netmask + + _, ipnet, err := net.ParseCIDR(str) + + if err != nil { + seelog.Critical(err) + } + return ipnet.String() +} + +func (z *DiscoveryZone) CidrInt64() int64 { + return converter.CidrToInt64(z.CidrStr()) +} diff --git a/discovery/discovery/types/network_test.go b/discovery/discovery/types/network_test.go new file mode 100644 index 0000000..48c23c4 --- /dev/null +++ b/discovery/discovery/types/network_test.go @@ -0,0 +1,272 @@ +package types + +import ( + "encoding/json" + "fmt" + "net" + //"strconv" + //"strings" + "testing" + //"bytes" + //"encoding/binary" + //"bytes" + //"io/ioutil" + "git.loafle.net/overflow/discovery/discovery/types/timestamp" + //"net/http" +) + +type DiscoveryZone222 struct { + MinIp string `json:"-"` + MaxIp string `json:"-"` + LocalIp string `json:"ip"` + Cidr string `json:"cidr"` + Inter net.Interface `json:"-"` + //Hosts map[string]*Host `json:"-"` +} + +func TestZone222(t *testing.T) { + + zz := DiscoveryZone222{} + + zz.MaxIp = "192.168.1.254" + zz.MinIp = "192.168.1.1" + zz.LocalIp = "192.168.1.207" + zz.Cidr = "192.168.1.0/24" + + jb, _ := json.Marshal(zz) + + fmt.Println(string(jb)) + +} + +type DiscoveryZone1 struct { + MinIp string `json:"-"` + MaxIp string `json:"-"` + Ip_ string `json:"-"` + Cidr_ string `json:"-"` + Inter_ net.Interface `json:"-"` + //Hosts map[string]*Host `json:"-"` + + Ip int64 `json:"ip"` + Cidr int64 `json:"cidr"` + + HostScanHistories []*DiscoveryHostHistory `json:"hostScanHistories"` +} + +type DiscoveryHostHistory struct { + ID int `json:"id"` + Ip int64 `json:"ip"` + + CreateDate timestamp.Timestamp `json:"createDate"` + SendDate timestamp.Timestamp `json:"sendDate"` + ResultDate timestamp.Timestamp `json:"resultDate"` + ResultType string `json:"resultType"` + Description string `json:"description"` +} + +func TestDiscoveryZone(t *testing.T) { + + zones := NewZones() + + fmt.Println(len(zones)) + fmt.Println(zones[0].LocalIp_) + fmt.Println(zones[0].Cidr) + + var zone DiscoveryZone = DiscoveryZone{ + MinIp_: "192.168.0.1", + MaxIp_: "192.168.0.254", + Cidr: "192.168.0.0/24", + LocalIp_: "192.168.0.203", + Inter_: net.Interface{}, + Hosts_: make(map[string]*DiscoveryHost, 255), + Ip: 233312321, + Cidr_: 123123321, + } + + var zones1 []*DiscoveryZone + zones1 = append(zones1, &zone) + z := NewZonesByDiscoveryZone(zones1) + + fmt.Println(len(z)) + //for _, zone := range zones { + // //a := inet_strton(zone.LocalIp) + // //fmt.Println(a) + // // + // //str := inet_ntoa(a) + // // + // //fmt.Println(str) + // + // //aa := CidrToInt64(zone.Cidr) + // // + // //fmt.Println(aa) + // // + // //str := Int64ToCidr(aa) + // // + // //fmt.Println(str) + // + //} + +} + +//func CreateDHH(ip string) *DiscoveryHostHistory { +// dhh := &DiscoveryHostHistory{} +// +// dhh.CreateDate = timestamp.Now() +// dhh.SendDate = timestamp.Now() +// dhh.ResultDate = timestamp.Now() +// dhh.Description = "testest" +// +// strIp := "192.168.1." +// strIp += ip +// dhh.Ip = inet_strton(strIp) +// +// return dhh +//} +// +//func TestInsertDZone(t *testing.T) { +// +// dz := DiscoveryZone{} +// +// dz.Cidr_ = "192.168.1.0/24" +// dz.Ip_ = "192.168.1.207" +// +// dz.Cidr = CidrToInt64(dz.Cidr_) +// dz.Ip = inet_strton(dz.Ip_) +// +// dz.HostScanHistories = append(dz.HostScanHistories, CreateDHH("103"), CreateDHH("104"), CreateDHH("105"), CreateDHH("106")) +// +// fmt.Println(inet_ntoa(dz.Ip)) +// fmt.Println(Int64ToCidr(dz.Cidr)) +// +// jb, _ := json.Marshal(dz) +// +// fmt.Println(string(jb)) +// +// resp, _ := http.Post("http://192.168.1.207:8080/discoveryZone", "application/json", bytes.NewBuffer(jb)) +// bb, _ := ioutil.ReadAll(resp.Body) +// +// ddzz := DiscoveryZone{} +// +// json.Unmarshal(bb, &ddzz) +// +// fmt.Println(ddzz) +// +// resp.Body.Close() +// +//} +// +//func TestSelectDZone(t *testing.T) { +// +// resp, _ := http.Get("http://192.168.1.207:8080/discoveryZone/1") +// bb, _ := ioutil.ReadAll(resp.Body) +// ddzz := DiscoveryZone{} +// +// fmt.Println(string(bb)) +// json.Unmarshal(bb, &ddzz) +// +// fmt.Println(ddzz) +// +// fmt.Println(Int64ToCidr(ddzz.Cidr)) +// fmt.Println(inet_ntoa(ddzz.Ip)) +// +// resp.Body.Close() +//} +// +//func inet_ntoa(ipnr int64) net.IP { +// var bytes [4]byte +// bytes[0] = byte(ipnr & 0xFF) +// bytes[1] = byte((ipnr >> 8) & 0xFF) +// bytes[2] = byte((ipnr >> 16) & 0xFF) +// bytes[3] = byte((ipnr >> 24) & 0xFF) +// +// return net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0]) +//} +// +//// Convert net.IP to int64 , http://www.outofmemory.cn +//func inet_aton(ipnr net.IP) int64 { +// bits := strings.Split(ipnr.String(), ".") +// +// b0, _ := strconv.Atoi(bits[0]) +// b1, _ := strconv.Atoi(bits[1]) +// b2, _ := strconv.Atoi(bits[2]) +// b3, _ := strconv.Atoi(bits[3]) +// +// var sum int64 +// +// sum += int64(b0) << 24 +// sum += int64(b1) << 16 +// sum += int64(b2) << 8 +// sum += int64(b3) +// +// return sum +//} +// +//func inet_strton(ipnr string) int64 { +// bits := strings.Split(ipnr, ".") +// +// b0, _ := strconv.Atoi(bits[0]) +// b1, _ := strconv.Atoi(bits[1]) +// b2, _ := strconv.Atoi(bits[2]) +// b3, _ := strconv.Atoi(bits[3]) +// +// var sum int64 +// +// sum += int64(b0) << 24 +// sum += int64(b1) << 16 +// sum += int64(b2) << 8 +// sum += int64(b3) +// +// return sum +//} +// +//func CidrToInt64(cidr string) int64 { +// +// bits := strings.Split(cidr, ".") +// +// b0, _ := strconv.Atoi(bits[0]) +// b1, _ := strconv.Atoi(bits[1]) +// b2, _ := strconv.Atoi(bits[2]) +// //b3, _ := strconv.Atoi(bits[3]) +// +// masks := strings.Split(bits[3], "/") +// +// bb3 := masks[0] +// bb4 := masks[1] +// +// b3, _ := strconv.Atoi(bb3) +// b4, _ := strconv.Atoi(bb4) +// +// var sum int64 +// +// sum += int64(b0) << 32 +// sum += int64(b1) << 24 +// sum += int64(b2) << 16 +// sum += int64(b3) << 8 +// sum += int64(b4) +// +// return sum +// +//} +// +//func Int64ToCidr(cidrInt int64) string { +// +// c1 := cidrInt & 0xFF +// c2 := (cidrInt >> 8) & 0xFF +// c3 := (cidrInt >> 16) & 0xFF +// c4 := (cidrInt >> 24) & 0xFF +// c5 := (cidrInt >> 32) & 0xFF +// +// var cidr string +// +// cidr += strconv.FormatInt(c5, 10) +// cidr += "." +// cidr += strconv.FormatInt(c4, 10) +// cidr += "." +// cidr += strconv.FormatInt(c3, 10) +// cidr += "." +// cidr += strconv.FormatInt(c2, 10) +// cidr += "/" +// cidr += strconv.FormatInt(c1, 10) +// +// return cidr +//} diff --git a/discovery/discovery/types/scaninfo.go b/discovery/discovery/types/scaninfo.go new file mode 100644 index 0000000..146f8d9 --- /dev/null +++ b/discovery/discovery/types/scaninfo.go @@ -0,0 +1,12 @@ +package types + +type ServiceScanInfo struct { + History []*ServiceScanHistory + Port *DiscoveryPort +} + +func NewServiceScanInfo(port *DiscoveryPort) *ServiceScanInfo { + s := &ServiceScanInfo{} + s.Port = port + return s +} diff --git a/discovery/discovery/types/timestamp/timestamp.go b/discovery/discovery/types/timestamp/timestamp.go new file mode 100644 index 0000000..6a465d0 --- /dev/null +++ b/discovery/discovery/types/timestamp/timestamp.go @@ -0,0 +1,37 @@ +package timestamp + +import ( + "fmt" + "strconv" + "time" +) + +type Timestamp time.Time + +func (t Timestamp) MarshalJSON() ([]byte, error) { + ts := time.Time(t).Unix() + stamp := fmt.Sprint(ts * 1000) + return []byte(stamp), nil +} + +func (t *Timestamp) UnmarshalJSON(b []byte) error { + ts, err := strconv.Atoi(string(b)) + if err != nil { + return err + } + *t = Timestamp(time.Unix(int64(ts)/1000, 0)) + + return nil +} + +func (t Timestamp) String() string { + return time.Time(t).String() +} + +func Now() Timestamp { + return Timestamp(time.Now()) +} + +func Date(year int, month time.Month, day int) Timestamp { + return Timestamp(time.Date(year, month, day, 0, 0, 0, 0, time.UTC)) +} diff --git a/discovery/discovery/zone.go b/discovery/discovery/zone.go new file mode 100644 index 0000000..81f45be --- /dev/null +++ b/discovery/discovery/zone.go @@ -0,0 +1,17 @@ +package discovery + +import ( + "git.loafle.net/overflow/discovery/core/scan/zone" + "git.loafle.net/overflow/discovery/discovery/types" +) + +func discoveryZone(callback discoverZoneCallback) { + // zone + zoneCallback := func(zone *types.DiscoveryZone) { + // pcap wrapper instance create + // callback call + callback(zone) + + } + zone.Scan(zoneCallback) +} diff --git a/discovery/discovery/zone_test.go b/discovery/discovery/zone_test.go new file mode 100644 index 0000000..adc83f0 --- /dev/null +++ b/discovery/discovery/zone_test.go @@ -0,0 +1,7 @@ +package discovery + +import "testing" + +func TestDiscoverZone(t *testing.T) { + // DiscoverZone() +} diff --git a/discovery/sensor/sensor.go b/discovery/sensor/sensor.go new file mode 100644 index 0000000..75facdc --- /dev/null +++ b/discovery/sensor/sensor.go @@ -0,0 +1 @@ +package sensor diff --git a/discovery/util/converter/cidr.go b/discovery/util/converter/cidr.go new file mode 100644 index 0000000..29c9e22 --- /dev/null +++ b/discovery/util/converter/cidr.go @@ -0,0 +1,58 @@ +package converter + +import ( + "strconv" + "strings" +) + +func CidrToInt64(cidr string) int64 { + + bits := strings.Split(cidr, ".") + + b0, _ := strconv.Atoi(bits[0]) + b1, _ := strconv.Atoi(bits[1]) + b2, _ := strconv.Atoi(bits[2]) + //b3, _ := strconv.Atoi(bits[3]) + + masks := strings.Split(bits[3], "/") + + bb3 := masks[0] + bb4 := masks[1] + + b3, _ := strconv.Atoi(bb3) + b4, _ := strconv.Atoi(bb4) + + var sum int64 + + sum += int64(b0) << 32 + sum += int64(b1) << 24 + sum += int64(b2) << 16 + sum += int64(b3) << 8 + sum += int64(b4) + + return sum + +} + +func Int64ToCidr(cidrInt int64) string { + + c1 := cidrInt & 0xFF + c2 := (cidrInt >> 8) & 0xFF + c3 := (cidrInt >> 16) & 0xFF + c4 := (cidrInt >> 24) & 0xFF + c5 := (cidrInt >> 32) & 0xFF + + var cidr string + + cidr += strconv.FormatInt(c5, 10) + cidr += "." + cidr += strconv.FormatInt(c4, 10) + cidr += "." + cidr += strconv.FormatInt(c3, 10) + cidr += "." + cidr += strconv.FormatInt(c2, 10) + cidr += "/" + cidr += strconv.FormatInt(c1, 10) + + return cidr +} diff --git a/discovery/util/converter/ip.go b/discovery/util/converter/ip.go new file mode 100644 index 0000000..1e24cab --- /dev/null +++ b/discovery/util/converter/ip.go @@ -0,0 +1,46 @@ +package converter + +import ( + "net" + "strconv" + "strings" +) + +func IP4toInt(IPv4Addr net.IP) int64 { + return StrToInt(IPv4Addr.String()) +} + +func StrToInt(strIp string) int64 { + bits := strings.Split(strIp, ".") + + b0, _ := strconv.Atoi(bits[0]) + b1, _ := strconv.Atoi(bits[1]) + b2, _ := strconv.Atoi(bits[2]) + b3, _ := strconv.Atoi(bits[3]) + + var sum int64 + + // left shifting 24,16,8,0 and bitwise OR + + sum += int64(b0) << 24 + sum += int64(b1) << 16 + sum += int64(b2) << 8 + sum += int64(b3) + + return sum +} + +func IntToIP4(ipInt int64) net.IP { + + //return b0 + "." + b1 + "." + b2 + "." + b3 + return net.ParseIP(IntToStr(ipInt)) +} + +func IntToStr(ipInt int64) string { + b0 := strconv.FormatInt((ipInt>>24)&0xff, 10) + b1 := strconv.FormatInt((ipInt>>16)&0xff, 10) + b2 := strconv.FormatInt((ipInt>>8)&0xff, 10) + b3 := strconv.FormatInt((ipInt & 0xff), 10) + + return b0 + "." + b1 + "." + b2 + "." + b3 +} diff --git a/discovery/util/converter/mac.go b/discovery/util/converter/mac.go new file mode 100644 index 0000000..0e60695 --- /dev/null +++ b/discovery/util/converter/mac.go @@ -0,0 +1,68 @@ +package converter + +import ( + log "github.com/cihub/seelog" + "net" + "strconv" + "strings" +) + +func MacToInt(IPv4Addr net.HardwareAddr) int64 { + bits := strings.Split(IPv4Addr.String(), ":") + + b0, _ := strconv.ParseInt(bits[0], 16, 64) + b1, _ := strconv.ParseInt(bits[1], 16, 64) + b2, _ := strconv.ParseInt(bits[2], 16, 64) + b3, _ := strconv.ParseInt(bits[3], 16, 64) + b4, _ := strconv.ParseInt(bits[4], 16, 64) + b5, _ := strconv.ParseInt(bits[5], 16, 64) + + var sum int64 + + //52242420297 + //52242420297 + sum += int64(b0) << 40 + sum += int64(b1) << 32 + sum += int64(b2) << 24 + sum += int64(b3) << 16 + sum += int64(b4) << 8 + sum += int64(b5) + + return sum +} + +func IntToMac(ipInt int64) net.HardwareAddr { + + b0 := strconv.FormatInt((ipInt>>40)&0xff, 16) + b1 := strconv.FormatInt((ipInt>>32)&0xff, 16) + b2 := strconv.FormatInt((ipInt>>24)&0xff, 16) + b3 := strconv.FormatInt((ipInt>>16)&0xff, 16) + b4 := strconv.FormatInt((ipInt>>8)&0xff, 16) + b5 := strconv.FormatInt((ipInt & 0xff), 16) + + if len(b0) == 1 { + b0 = "0" + b0 + } + if len(b1) == 1 { + b1 = "0" + b1 + } + if len(b2) == 1 { + b2 = "0" + b2 + } + if len(b3) == 1 { + b3 = "0" + b3 + } + if len(b4) == 1 { + b4 = "0" + b4 + } + if len(b5) == 1 { + b5 = "0" + b5 + } + + str := b0 + ":" + b1 + ":" + b2 + ":" + b3 + ":" + b4 + ":" + b5 + + log.Debug("mac string: ", str) + nt, _ := net.ParseMAC(str) + + return nt +} diff --git a/discovery/util/converter/mac_test.go b/discovery/util/converter/mac_test.go new file mode 100644 index 0000000..be689d6 --- /dev/null +++ b/discovery/util/converter/mac_test.go @@ -0,0 +1,92 @@ +package converter + +import ( + "fmt" + "strings" + "testing" +) + +func TestMacToInt(t *testing.T) { + +} + +func TestIntToMac(t *testing.T) { + tt := IntToMac(75361038758387) + + //seelog.Debug(tt.String()) + fmt.Println(tt.String()) +} + +func TestIntToIP4(t *testing.T) { + + tt := IntToIP4(3232235877) + fmt.Println(tt.String()) +} + +func TestIpMac(t *testing.T) { + ipMacMap := map[int64]int64{ + 3232235777: 119779280707688, + 3232235787: 159013445376676, + 3232235977: 75361038758611, + 3232235782: 91754660618, + 3232235796: 91754662917, + 3232235820: 91754662924, + 3232235786: 159013445376672, + 3232235798: 91754662914, + 3232235842: 126665265695, + 3232235800: 91754662916, + 3232235879: 75361027394792, + 3232235983: 8796753988883, + 3232236030: 110937283749760, + 3232235984: 237872197994881, + 3232235978: 92378945339098, + 3232235778: 269295397419301, + 3232235881: 88945707094824, + 3232235797: 91754662913, + 3232235882: 75361038711421, + 3232235864: 207120303401094, + 3232235806: 91754662922, + 3232235880: 92378945273005, + 3232235788: 159013445322194, + 3232235865: 207120303401094, + 3232235877: 75361038758387, + } + + //ipMacMap := map[int64]int64{ + // 3232235787: 159013445376676, + // 3232235782: 91754660618, + // 3232235798: 91754662914, + // 3232235842: 126665265695, + // 3232235879: 75361027394792, + // 3232236030: 110937283749760, + // 3232235978: 92378945339098, + // 3232235877: 75361038758387, + // 3232235820: 91754662924, + // 3232235864: 207120303401094, + // 3232235881: 88945707094824, + // 3232235977: 75361038758611, + // 3232235806: 91754662922, + // 3232235777: 119779280707688, + // 3232235983: 8796753988883, + // 3232235796: 91754662917, + // 3232235778: 269295397419301, + // 3232235788: 159013445322194, + // 3232235786: 159013445376672, + // 3232235882: 75361038711421, + // 3232235800: 91754662916, + // 3232235880: 92378945273005, + // 3232235865: 207120303401094, + // 3232235797: 91754662913, + //} + + for k, v := range ipMacMap { + ip := IntToIP4(k) + mac := IntToMac(v) + + if strings.Compare("192.168.1.208", ip.String()) == 0 { + fmt.Println(k) + fmt.Println(v) + } + fmt.Println("IP: ", ip.String(), "\t Mac : ", mac.String()) + } +} diff --git a/encryption/encryption.go b/encryption/encryption.go new file mode 100644 index 0000000..2791120 --- /dev/null +++ b/encryption/encryption.go @@ -0,0 +1,46 @@ +package encryption + +import ( + "crypto/rand" + "encoding/hex" + "golang.org/x/crypto/scrypt" + "io" +) + +const ( + PW_SALT_BYTES = 64 + PW_HASH_BYTES = 64 +) + +func Encrypt(pw string) (string, string, error) { + + salt := make([]byte, PW_SALT_BYTES) + _, err := io.ReadFull(rand.Reader, salt) + if err != nil { + return "", "", err + } + + hash, err := scrypt.Key([]byte(pw), salt, 16384, 8, 1, PW_HASH_BYTES) + if err != nil { + return "", "", err + } + saltStr := hex.EncodeToString(salt) + hashStr := hex.EncodeToString(hash) + + return saltStr, hashStr, nil +} + +func Check(pw, savedSalt, savedDigest string) bool { + salt, err := hex.DecodeString(savedSalt) + + hash, err := scrypt.Key([]byte(pw), salt, 16384, 8, 1, PW_HASH_BYTES) + if err != nil { + return false + } + new := hex.EncodeToString(hash) + + if new == savedDigest { + return true + } + return false +} diff --git a/encryption/encryption_test.go b/encryption/encryption_test.go new file mode 100644 index 0000000..2ddd0a5 --- /dev/null +++ b/encryption/encryption_test.go @@ -0,0 +1,15 @@ +package encryption + +import ( + "testing" +) + +func TestEncrypt(t *testing.T) { + salt, hash, err := Encrypt("MyPassword") + if err != nil { + t.Fatal(err) + } + + b := Check("MyPassword", salt, hash) + t.Log(b) +} diff --git a/event_sender/event_sender.go b/event_sender/event_sender.go new file mode 100644 index 0000000..46d1a0b --- /dev/null +++ b/event_sender/event_sender.go @@ -0,0 +1,155 @@ +package event_sender_go + +import ( + "context" + "google.golang.org/grpc" + //"git.loafle.net/overflow/agent_api/observer/messages" + pb "git.loafle.net/overflow/crawler_go/grpc" + + "git.loafle.net/overflow/agent_api/messages" + q "git.loafle.net/overflow/queue_go" + "log" + "reflect" + "sync" +) + +var ( + instance *EventSender + once sync.Once +) + +const ( + CENTRAL_ADDR = "192.168.1.105:50052" + SENDER_ID = "OVERFLOW_EVENT_SENDER" + DEFAULT_INTERVAL = 1 +) + +type Data struct { + AgentId string + SensorId string + Data map[string]string + StartedAt uint64 + FinishedAt uint64 +} + +func GetInstance() *EventSender { + once.Do(func() { + instance = &EventSender{} + }) + return instance +} + +//func init() { +// ch := make(chan interface{}, 0) +// ch1 := make(chan interface{}, 0) +// observer.Add(messages.CFG_LOADED, ch) +// observer.Add(messages.SCF_STOPPED, ch1) +// handleInit(ch, ch1) +//} +// +//func handleInit(ch chan interface{}, ch1 chan interface{}) { +// es := GetInstance() +// go func() { +// data := <-ch +// log.Println("handleInit", data) +// //ds.gconf = data.(cm.ConfigManager).GetGlobalConfig() +// es.start() +// observer.Remove(messages.CFG_LOADED, ch) +// }() +// +// go func() { +// data := <-ch1 +// observer.Notify(messages.QUEUE_EVENT, data) +// time.Sleep(2 * time.Second) +// es.Stop() +// }() +//} + +func Start(ch chan bool) (err error) { + es := GetInstance() + + es.start() + + ch <- true + return nil +} + +func Stop() (err error) { + es := GetInstance() + es.stop() + return nil +} +func AddEventData(event interface{}) { + es := GetInstance() + es.lq.PushItem(event) +} + +type EventSender struct { + once sync.Once + lq *q.LoafleQueue + sc chan interface{} +} + +func (es *EventSender) start() { + es.sc = make(chan interface{}, 0) + + es.lq = q.NewQueue(DEFAULT_INTERVAL, es.sc) + + go es.checkQueue() +} + +func (es *EventSender) stop() { + if es.sc != nil { + es.lq.Close() + close(es.sc) + es.sc = nil + } +} + +func (es *EventSender) checkQueue() { + + result := make([]*messages.Event, 0) + + for sc := range es.sc { + items := reflect.ValueOf(sc) + if items.Kind() != reflect.Slice { + log.Println("ddddd") + } + + for i := 0; i < items.Len(); i++ { + item := items.Index(i).Elem().Interface() + tempCollectedData := item.(q.Item) + collectedData := tempCollectedData.Value.(messages.Event) + //collectedData.AgentId = agentIdentifier() + result = append(result, &collectedData) + //log.Println("Result Len: ", len(result)) + } + //es.send(result) + result = nil + } + +} + +func (es *EventSender) send(data []*messages.Event) { + log.Println("Send Started") + conn, err := grpc.Dial(CENTRAL_ADDR, grpc.WithInsecure()) + + if err != nil { + log.Fatal("Connection Error :", err.Error()) + return + } + defer conn.Close() + + client := pb.NewStatusClient(conn) + out, err := client.Status(context.Background(), &pb.Empty{}) + + if err != nil { + log.Fatal("client Error:", err.Error()) + return + } + log.Print(out) +} + +func agentIdentifier() string { + return "agentID_000000001" +} diff --git a/event_sender/event_sender_test.go b/event_sender/event_sender_test.go new file mode 100644 index 0000000..a88ab33 --- /dev/null +++ b/event_sender/event_sender_test.go @@ -0,0 +1,50 @@ +package event_sender_go + +import ( + //"git.loafle.net/overflow/agent_api/observer" + //"git.loafle.net/overflow/agent_api/observer/messages" + "log" + "strconv" + "testing" + "time" +) + +func TestEventSender_Start(t *testing.T) { + + //observer.Notify("CONFIGMANAGER_LOADED", nil) + + c := make(chan bool, 1) + err := Start(c) + + <-c + + if err != nil { + log.Println("err: ", err) + } + time.Sleep(time.Second * 5) + + testNotify() + + time.Sleep(time.Second * 5) + +} + +func testNotify() { + //time.Sleep(time.Second * 5) + //es := GetInstance() + + for i := 0; i < 20; i++ { + result := make(map[string]string) + result["ab"] = "123 ( " + strconv.Itoa(i) + ")" + result["cd"] = "456 ( " + strconv.Itoa(i) + ")" + result["ef"] = "789 ( " + strconv.Itoa(i) + ")" + + cd := &Data{ + SensorId: "insanity", + Data: result, + } + AddEventData(cd) + //observer.Notify(messages.QUEUE_EVENT, cd) + } + log.Println("New data Notify") +} diff --git a/file_cipher/file_cipher.go b/file_cipher/file_cipher.go new file mode 100644 index 0000000..86d37ca --- /dev/null +++ b/file_cipher/file_cipher.go @@ -0,0 +1,69 @@ +package file_cipher_go + +import ( + "crypto/aes" + "crypto/cipher" + "io" + "os" +) + +var KEY = []byte("qwerqwerqwerqwer") + +func Encrypt(filePath, destName string) string { + + inFile, err := os.Open(filePath) + if err != nil { + panic(err) + } + defer inFile.Close() + + block, err := aes.NewCipher(KEY) + if err != nil { + panic(err) + } + + var iv [aes.BlockSize]byte + stream := cipher.NewOFB(block, iv[:]) + + outFile, err := os.OpenFile(destName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + panic(err) + } + defer outFile.Close() + + writer := &cipher.StreamWriter{S: stream, W: outFile} + if _, err := io.Copy(writer, inFile); err != nil { + panic(err) + } + + return outFile.Name() +} + +func Decrypt(filePath, destName string) string { + inFile, err := os.Open(filePath) + if err != nil { + panic(err) + } + defer inFile.Close() + + block, err := aes.NewCipher(KEY) + if err != nil { + panic(err) + } + + var iv [aes.BlockSize]byte + stream := cipher.NewOFB(block, iv[:]) + + outFile, err := os.OpenFile(destName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + panic(err) + } + defer outFile.Close() + + reader := &cipher.StreamReader{S: stream, R: inFile} + if _, err := io.Copy(outFile, reader); err != nil { + panic(err) + } + + return outFile.Name() +} diff --git a/file_cipher/file_cipher_test.go b/file_cipher/file_cipher_test.go new file mode 100644 index 0000000..0b6da34 --- /dev/null +++ b/file_cipher/file_cipher_test.go @@ -0,0 +1,11 @@ +package file_cipher_go + +import "testing" + +func TestEncrypt(t *testing.T) { + Encrypt("/home/insanity/addressbook.proto", "/home/insanity/addressbook.enc") +} + +func TestDecrypt(t *testing.T) { + Decrypt("/home/insanity/addressbook.enc", "/home/insanity/addressbook.dec") +} diff --git a/initializer/initializer.go b/initializer/initializer.go new file mode 100644 index 0000000..4c20f14 --- /dev/null +++ b/initializer/initializer.go @@ -0,0 +1,59 @@ +package initializer_go + +import ( + cm "git.loafle.net/overflow/agent_api/config_manager" + "google.golang.org/grpc" + "sync" +) + +var ( + once sync.Once + instance *Initializer +) + +type Initializer struct { + gconf *cm.GlobalConfig +} + +func Start(ch chan string, conf *cm.GlobalConfig) error { + i := GetInstance() + i.gconf = conf + key, err := i.getSecretKey() + if err != nil { + return err + } + ch <- key + return nil +} + +func Stop(res chan bool) { + + GetInstance().stop() + + res <- true +} + +func GetInstance() *Initializer { + once.Do(func() { + instance = &Initializer{} + }) + return instance +} + +func (i *Initializer) getSecretKey() (string, error) { + //Todo. getting secret key from CentralAPI + + addr := i.gconf.Central.Address + ":" + string(i.gconf.Central.Port) + conn, err := grpc.Dial(addr, grpc.WithInsecure()) + if err != nil { + return nil, err + } + defer conn.Close() + + return nil, nil + +} + +func (i *Initializer) stop() { + +} diff --git a/matcher/activedirectory/activedirectory.go b/matcher/activedirectory/activedirectory.go new file mode 100644 index 0000000..5a3146b --- /dev/null +++ b/matcher/activedirectory/activedirectory.go @@ -0,0 +1,342 @@ +package activedirectory + +import ( + "bytes" + "encoding/binary" + + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + AD_MESSAGE_ID = 0x99 + AD_MESSAGE_ID_QUIT = 0x89 + + LDAP_VERSION3 = 3 + + LDAP_SUCCESS = 0x00 + + LDAP_REQ_BIND = 0x60 + LDAP_RES_SEARCH_ENTRY = 0x64 + + LDAP_REQ_UNBIND = 0x42 + LDAP_REQ_SEARCH = 0x63 + + LDAP_SCOPE_BASE = 0x00 + LDAP_DEREF_NEVER = 0x00 + LDAP_FILTER_PRESENT = 0x87 + + LDAP_RES_BIND = 0x61 + + LDAP_AUTH_SIMPLE = 0x80 + + AD_TYPE_STR = "supportedCapabilities" +) + +type AD_SENDaaa struct { + DefaultCode uint8 + PackLenFlag uint8 + PacketLen uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint32 + + ProtocolOp uint8 + PtLenFlag uint8 + PtPacketLen uint32 + NextType2 uint8 + NextTypeLength2 uint8 + + Version uint8 + + NextType3 uint8 + NextTypeLength3 uint8 + Auth uint8 + AuthLength uint8 +} + +type AD_SEND struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLen uint32 + + NextType1 uint8 + NextType1Len uint8 + + MessageId uint32 + + ProtocolOp uint8 + + PtPackLenFlag uint8 + PtPacketLen uint32 + + NextType2 uint8 + NextType2Len uint8 + NextType3 uint8 + NextType3Len uint8 + + Scope uint8 + + NextType4 uint8 + NextType4Len uint8 + + DerefAliases uint8 + + NextType5 uint8 + NextType5Len uint8 + + SizeLimit uint8 + + NextType6 uint8 + NextType6Len uint8 + + TimeLimit uint8 + + NextType7 uint8 + NextType7Len uint8 + + TypesOnly uint8 + + Filter1 uint8 + PresentLen uint8 + + Present [11]byte + + DefaultCode2 uint8 + Pack2LenFlag uint8 + + Packet2Len uint32 + + UnknwonCode8 uint8 + ItemLength uint8 + + AttributeDescription [21]byte +} + +type AD_QUIT struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLength uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint32 + + ProtocolOp uint8 + + PtLenFlag uint8 + PtPacketLen uint32 +} + +type AD_RECV struct { + DefaultCode uint8 + PackLenFlag uint8 + + PacketLength uint32 + + NextType1 uint8 + NextType1Len uint8 + + MessageId uint16 + + ProtocolOp uint8 + + PtPackLenFlag uint8 + PtPacketLen uint32 + + NextType2 uint8 + NextType2Len uint8 + + UnknwonCode21 uint8 + UnknwonCode22 uint8 + UnknwonCode23 uint8 + UnknwonCode24 uint8 + UnknwonCode25 uint8 + UnknwonCode26 uint8 + + UnknwonCode31 uint8 + UnknwonCode32 uint8 + UnknwonCode33 uint8 + UnknwonCode34 uint8 + UnknwonCode35 uint8 + UnknwonCode36 uint8 + UnknwonCode37 uint8 + + TypeLength uint8 +} + +type ActiveDirectoryMatcher struct { + sendPackets []*packet.Packet +} + +func (ad *ActiveDirectoryMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + adRecv := AD_RECV{} + + binary.Read(buf, binary.BigEndian, &adRecv) + + if adRecv.MessageId != AD_MESSAGE_ID { + return false + } + + if adRecv.ProtocolOp != LDAP_RES_SEARCH_ENTRY { + return false + } + + ///AD_TYPE_STR + + // + //if(packet->readCount_ < sizeof(AD_RECV) + recv->typeLength) { + // return false; + //} + + //char* type = new char[recv->typeLength]; + //memcpy(type, packet->buffer_+sizeof(AD_RECV), recv->typeLength); + //std::string typeStr = type; + // + //delete[] type; + //if(typeStr.compare(AD_TYPE_STR) != 0) { + //return false; + //} + + return true +} + +func (ad *ActiveDirectoryMatcher) PacketCount() int { + return len(ad.sendPackets) +} +func (ad *ActiveDirectoryMatcher) Packet(index int) *packet.Packet { + return ad.sendPackets[index] +} +func (ad *ActiveDirectoryMatcher) ServiceName() string { + return "ActiveDirectory" +} + +func (ad *ActiveDirectoryMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (ad *ActiveDirectoryMatcher) IsNoResponse(index int) bool { + + if index == 1 { + return true + } + + return false +} + +func (ad *ActiveDirectoryMatcher) IsPrePacket() bool { + return false +} + +func NewActiveDirectoryMatcher() *ActiveDirectoryMatcher { + + ls := AD_SEND{ + DefaultCode: 0x30, + PackLenFlag: 0x84, + + PacketLen: 0x47, + + NextType1: 0x02, + NextType1Len: 0x04, + + MessageId: AD_MESSAGE_ID, + + ProtocolOp: LDAP_REQ_SEARCH, + + PtPackLenFlag: 0x84, + PtPacketLen: 0x3b, + + NextType2: 0x04, + NextType2Len: 0x00, + NextType3: 0x0a, + NextType3Len: 0x01, + + Scope: LDAP_SCOPE_BASE, + + NextType4: 0x0a, + NextType4Len: 0x01, + + DerefAliases: LDAP_DEREF_NEVER, + + NextType5: 0x02, + NextType5Len: 0x01, + + SizeLimit: 0, + + NextType6: 0x02, + NextType6Len: 0x01, + + TimeLimit: 0x78, + + NextType7: 0x01, + NextType7Len: 0x01, + + TypesOnly: 0, + + Filter1: LDAP_FILTER_PRESENT, + PresentLen: 0x0b, + + //Present :0000, + + DefaultCode2: 0x30, + Pack2LenFlag: 0x84, + + Packet2Len: 0x17, + + UnknwonCode8: 0x04, + ItemLength: 0x15, + + //AttributeDescription:, + } + + copy(ls.Present[:], "objectclass") + copy(ls.AttributeDescription[:], AD_TYPE_STR) + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.BigEndian, ls) + + sendByte1 := mCache.Bytes() + + adm := ActiveDirectoryMatcher{ + //sendPackets: make([][]byte, 2), + } + + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + adm.sendPackets = append(adm.sendPackets, pp) + + aq := AD_QUIT{ + DefaultCode: 0x30, + PackLenFlag: 0x84, + PacketLength: 0x0c, + NextType1: 0x02, + NextTypeLength1: 0x04, + MessageId: AD_MESSAGE_ID_QUIT, + ProtocolOp: LDAP_REQ_UNBIND, + PtLenFlag: 0x84, + PtPacketLen: 0x00, + } + + lqBuffer := new(bytes.Buffer) + binary.Write(lqBuffer, binary.BigEndian, aq) + + quBytes := lqBuffer.Bytes() + + pp2 := packet.NewPacket(quBytes, len(quBytes)) + + adm.sendPackets = append(adm.sendPackets, pp2) + + return &adm +} diff --git a/matcher/activedirectory/activedirectory_test.go b/matcher/activedirectory/activedirectory_test.go new file mode 100644 index 0000000..db318a1 --- /dev/null +++ b/matcher/activedirectory/activedirectory_test.go @@ -0,0 +1,77 @@ +package activedirectory + +import ( + "crypto/tls" + "net" + "testing" +) + +func TestADNor(t *testing.T) { + client, err := net.Dial("tcp", "192.168.1.1:389") + + if err != nil { + t.Log(err) + } + + defer client.Close() + + dDRun(client, t) +} + +func TestADTLS(t *testing.T) { + conn, err := tls.Dial( + "tcp", + "192.168.1.1:636", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.1", + }, + ) + + if err != nil { + t.Log(err) + return + } + + defer conn.Close() + + dDRun(conn, t) +} + +func dDRun(client net.Conn, t *testing.T) { + + //lm := NewActiveDirectoryMatcher() + // + //port := types.NewPort("389", types.NewHost("192.168.1.1"), types.TYPE_TCP) + // + //var ipport string + //ipport = port.Host.Ip + ":" + string(port.Port) + // + //fmt.Println(ipport) + // + //fmt.Println(lm.PacketCount()) + // + //for ii := 0; ii < lm.PacketCount(); ii++ { + // + // pack := lm.Packet(ii) + // + // fmt.Println(pack) + // + // client.Write(pack.Buffer) + // + // bytes := make([]byte, 1024) + // + // read, _ := client.Read(bytes) + // + // fmt.Println(bytes) + // + // b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) + // + // if b { + // fmt.Println("Good") + // } + // + //} + // + //t.Log(scanInfo) +} diff --git a/matcher/cassandra/cassandra.go b/matcher/cassandra/cassandra.go new file mode 100644 index 0000000..83c63eb --- /dev/null +++ b/matcher/cassandra/cassandra.go @@ -0,0 +1,105 @@ +package cassandra + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + COMPRESSION = "COMPRESSION" + CQL_VERSION = "CQL_VERSION" +) + +type cassandra struct { + Version uint8 + Flags uint8 + Stream uint16 + Opcode uint8 + Length uint32 +} + +type CassandraMatcher struct { + packets []*packet.Packet +} + +func NewCassandraMatcher() *CassandraMatcher { + + cassMatcher := &CassandraMatcher{} + c := cassandra{ + Version: 4, + Flags: 0, + Stream: 0, + Opcode: 5, + Length: 0, + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, c) + + cassMatcher.packets = append(cassMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return cassMatcher +} + +func (t *CassandraMatcher) ServiceName() string { + return "Cassandra" +} + +func (t *CassandraMatcher) PacketCount() int { + return len(t.packets) +} +func (t *CassandraMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *CassandraMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *CassandraMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *CassandraMatcher) IsPrePacket() bool { + return false +} + +func (t *CassandraMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + c := cassandra{} + if err := binary.Read(reader, binary.BigEndian, &c); err != nil { + return false + } + if c.Version != 0x84 { + return false + } + if c.Flags != 0x00 { + return false + } + if c.Stream != 0x00 { + return false + } + if c.Opcode != 0x06 { + return false + } + var itemcount uint16 + + if binary.Read(reader, binary.BigEndian, &itemcount) != nil { + return false + } + + if itemcount != 0 && itemcount != 2 { + return false + } + + return true + +} diff --git a/matcher/cassandra/cassandra_test.go b/matcher/cassandra/cassandra_test.go new file mode 100644 index 0000000..52ade00 --- /dev/null +++ b/matcher/cassandra/cassandra_test.go @@ -0,0 +1,74 @@ +package cassandra + +// +//import ( +// "crypto/tls" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestCassandra(t *testing.T) { +// +// m := NewCassandraMatcher() +// +// conn, err := net.Dial("tcp", "192.168.1.104:9042") +// if err != nil { +// t.Error(err) +// return +// } +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("Cassandra found") +// return +// } +// +// t.Error("Cassandra not found") +// } +// +//} +// +//func TestCassandraTLS(t *testing.T) { +// +// m := NewCassandraMatcher() +// +// conn, err := tls.Dial( +// "tcp", +// "192.168.1.104:9042", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.104", +// }, +// ) +// if err != nil { +// t.Fatal(err) +// } +// +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("Cassandra found") +// return +// } +// +// t.Error("Cassandra not found") +// } +// +//} diff --git a/matcher/dhcp/dhcp.go b/matcher/dhcp/dhcp.go new file mode 100644 index 0000000..97a176b --- /dev/null +++ b/matcher/dhcp/dhcp.go @@ -0,0 +1,188 @@ +package dhcp + +// +//import ( +// "bytes" +// "encoding/binary" +// "fmt" +// log "github.com/cihub/seelog" +// "loafle.com/overflow/commons_go/matcher" +// "net" +// "time" +//) +// +//const ( +// MAGIC_COOKIE uint32 = 0x63825363 +// OPT_CODE_SERVER_IDENTIFIER uint8 = 54 +// OPT_CODE_SUBNET_MASK uint8 = 1 +// OPT_CODE_ROUTER uint8 = 3 +// OPT_CODE_DNS uint8 = 6 +//) +// +//type dhcpDiscover struct { +// MsgType byte +// HwType byte +// HwAddrLen byte +// Hops byte +// Xid uint32 +// Secs uint16 +// BootpFlags uint16 +// ClientIp uint32 +// YourIp uint32 +// NextServerIp uint32 +// RelayAgentIp uint32 +// ClientMacAddr [6]byte +// ClientHwAddrPadding [10]byte +// ServerHostName [64]byte +// BootFileName [128]byte +// MagicCookie uint32 +// Mtype byte +// MtypeLen byte +// MtypeVal byte +// Opts [200]byte +// End byte +// Padding [16]byte +//} +// +//func DiscoverDHCP(zone *types.DiscoveryZone) { +// err := sendDHCPDiscovery() +// if err != nil { +// log.Error(err) +// return +// } +// recvDHCPOffer(zone) +//} +// +//func sendDHCPDiscovery() error { +// dhcp := dhcpDiscover{ +// MsgType: 0x01, +// HwType: 0x01, +// HwAddrLen: 0x06, +// Hops: 0x00, +// Xid: 0x00000000, +// Secs: 0x0000, +// ClientIp: 0x00000000, +// YourIp: 0x00000000, +// NextServerIp: 0x00000000, +// RelayAgentIp: 0x00000000, +// MagicCookie: MAGIC_COOKIE, +// Mtype: 0x35, +// MtypeLen: 0x01, +// MtypeVal: 0x01, +// End: 0xff, +// } +// +// var flag uint16 = 0 +// dhcp.BootpFlags = ^flag // flag = unicast , ^flag = broadcast +// +// //TODO : getting mac addr from zone +// dhcp.ClientMacAddr[0] = 0x50 +// dhcp.ClientMacAddr[1] = 0xe5 +// dhcp.ClientMacAddr[2] = 0x49 +// dhcp.ClientMacAddr[3] = 0x46 +// dhcp.ClientMacAddr[4] = 0x93 +// dhcp.ClientMacAddr[5] = 0x28 +// +// writer := new(bytes.Buffer) +// binary.Write(writer, binary.BigEndian, dhcp) +// conn, err := net.Dial("udp", "255.255.255.255:67") +// if err != nil { +// return err +// } +// conn.Write(writer.Bytes()) +// defer conn.Close() +// +// return nil +//} +// +//func recvDHCPOffer(zone *types.DiscoveryZone) { +// +// socket, err := net.ListenUDP("udp4", &net.UDPAddr{ +// IP: net.IPv4(255, 255, 255, 255), +// Port: 68, +// }) +// if err != nil { +// log.Error(err) +// return +// } +// err = socket.SetDeadline(time.Now().Add(3 * time.Second)) +// if err != nil { +// log.Error(err) +// return +// } +// +// buf := make([]byte, 4096) +// n, _, err := socket.ReadFromUDP(buf) +// if err != nil { +// log.Error(err) +// return +// } +// if n <= 0 { +// log.Error("No DHCP offer.") +// return +// } +// +// offer := dhcpDiscover{} +// +// reader := new(bytes.Buffer) +// reader.Write(buf) +// if err := binary.Read(reader, binary.BigEndian, &offer); err != nil { +// log.Error(err) +// return +// } +// if offer.MagicCookie != MAGIC_COOKIE { +// log.Error("Not a DHCP packet.") +// return +// } +// +// //option searching +// r := new(bytes.Buffer) +// r.Write(offer.Opts[:]) +// +// for i := 0; i < r.Len(); i++ { +// v := r.Next(1)[0] +// +// if v == OPT_CODE_SUBNET_MASK && r.Next(1)[0] == 4 { +// ipStr := byteToIpString(r.Next(4)) +// log.Infof("SUBNET MASK: %s", ipStr) +// } +// +// if v == OPT_CODE_ROUTER && r.Next(1)[0] == 4 { +// ipStr := byteToIpString(r.Next(4)) +// log.Infof("ROUTER: %s", ipStr) +// } +// +// if v == OPT_CODE_DNS { +// len := r.Next(1)[0] +// var dns []string = make([]string, 0) +// var ipStr string +// ipStr = byteToIpString(r.Next(4)) +// dns = append(dns, ipStr) +// if len == 8 { +// ipStr = byteToIpString(r.Next(4)) +// dns = append(dns, ipStr) +// } +// log.Infof("DNS", dns) +// } +// +// if v == OPT_CODE_SERVER_IDENTIFIER && r.Next(1)[0] == 4 { +// ipStr := byteToIpString(r.Next(4)) +// log.Infof("DHCP SERVER: %s", ipStr) +// } +// } +// +//} +// +//func byteToIpString(b []byte) string { +// var ipStr string +// len := len(b) +// for i := 0; i < len; i++ { +// v := b[i] +// ipStr += fmt.Sprintf("%d", v) +// if i < len-1 { +// ipStr += "." +// } +// } +// +// return ipStr +//} diff --git a/matcher/dhcp/dhcp_test.go b/matcher/dhcp/dhcp_test.go new file mode 100644 index 0000000..b1ca4cb --- /dev/null +++ b/matcher/dhcp/dhcp_test.go @@ -0,0 +1,12 @@ +package dhcp + +// +//import ( +// "loafle.com/overflow/collector/core/scan/zone" +// "testing" +//) +// +//func TestDHCP(t *testing.T) { +// zone := zone.NewZone() +// DiscoverDHCP(zone) +//} diff --git a/matcher/dns/dns.go b/matcher/dns/dns.go new file mode 100644 index 0000000..dcadd97 --- /dev/null +++ b/matcher/dns/dns.go @@ -0,0 +1,151 @@ +package dns + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +type Dns_frame_header struct { + Transaction_id uint16 + Flags uint16 + Questions uint16 + Answer_rrs uint16 + Authority_rrs uint16 + Additional_rrs uint16 +} + +type Dns_query_section struct { + Name uint8 + Query_type uint16 + Class_type uint16 +} + +type Dns_authority_section struct { + Name uint8 + Auth_type uint16 + Class_type uint16 + Time_to_live uint32 + Data_length uint16 + Primary_name_server [20]uint8 + Responsible_authority_mailbox [24]uint8 + Serial_number uint32 + Refresh_interval uint32 + Retry_interval uint32 + Expire_limit uint32 + Minium_ttl uint32 +} + +type DNSMatcher struct { + packets []*packet.Packet +} + +func NewDnsMatcher() *DNSMatcher { + + m := &DNSMatcher{} + + header := Dns_frame_header{ + Transaction_id: 0x2a88, + Flags: 0x0100, + Questions: 1, + Answer_rrs: 0, + Authority_rrs: 0, + Additional_rrs: 0, + } + + query := Dns_query_section{ + Name: 0, + Query_type: 1, + Class_type: 1, + } + + buf := new(bytes.Buffer) + binary.Write(buf, binary.BigEndian, header) + binary.Write(buf, binary.BigEndian, query) + + m.packets = append(m.packets, packet.NewPacket(buf.Bytes(), buf.Len())) + + return m +} + +func (t *DNSMatcher) ServiceName() string { + return "DNS" +} + +func (t *DNSMatcher) PacketCount() int { + return len(t.packets) +} +func (t *DNSMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *DNSMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *DNSMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *DNSMatcher) IsPrePacket() bool { + return false +} + +func (t *DNSMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + if packet == nil { + return false + } + if packet.Len <= 0 { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + h := Dns_frame_header{} + if err := binary.Read(reader, binary.BigEndian, &h); err != nil { + return false + } + + if h.Transaction_id != 0x2a88 { + return false + } + if h.Flags != 0x8180 && h.Flags != 0x8182 { + return false + } + if h.Questions != 1 { + return false + } + if h.Answer_rrs != 0 { + return false + } + if h.Authority_rrs != 0 && h.Authority_rrs != 1 { + return false + } + if h.Additional_rrs != 0 && h.Additional_rrs != 1 { + return false + } + + q := Dns_query_section{} + if err := binary.Read(reader, binary.BigEndian, &q); err != nil { + return false + } + if q.Name != 0 { + return false + } + if q.Query_type != 1 { + return false + } + if q.Class_type != 1 { + return false + } + + return true +} +func (t *DNSMatcher) IsSend(port int) bool { + if port == 53 { + return true + } + return false +} diff --git a/matcher/dns/dns_test.go b/matcher/dns/dns_test.go new file mode 100644 index 0000000..18cb258 --- /dev/null +++ b/matcher/dns/dns_test.go @@ -0,0 +1,33 @@ +package dns + +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestDns(t *testing.T) { +// m := NewDnsMatcher() +// +// conn, _ := net.Dial("udp", "192.168.1.215:53") +// +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// if m.IsSend(53) != true { +// t.Error("not port") +// } +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("dns found") +// return +// } +// +// t.Error("dns not found") +// } +//} diff --git a/matcher/ftp/ftp.go b/matcher/ftp/ftp.go new file mode 100644 index 0000000..b052892 --- /dev/null +++ b/matcher/ftp/ftp.go @@ -0,0 +1,142 @@ +package ftp + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + log "github.com/cihub/seelog" +) + +// FTP Status codes, defined in RFC 959 +const ( + statusReadyServer = "120" + statusOK = "200" + statusNewConnectOK = "220" + statusSystemNameOK = "215" + statusCloseConnect = "221" + statusUnkownCMD = "202" + statusTlsUseOK = "234" + statusCloseControlConnect = "421" + statusSyntaxErr = "500" + statusParamSyntaxErr = "501" + statusNotUseCMD = "502" + statusIncorrectCMD = "503" + statusTlsNotUse = "534" + statusNeedUserId = "332" +) + +type FTPMatcher struct { + sendPackets []*packet.Packet + isFtps bool +} + +func (ftp *FTPMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + log.Error("Packet nil") + return result + } + + str := string(packet.Buffer) + //fmt.Println(str) + code := str[:3] + + if index == 0 { + switch code { + case statusNewConnectOK, statusReadyServer: + //fmt.Println(code) + result = true + break + } + } else if index == 1 { + switch code { + case statusSystemNameOK, statusSyntaxErr, statusParamSyntaxErr, statusNotUseCMD: + //fmt.Println(code) + result = true + break + } + } else if index == 2 { + switch code { + case statusIncorrectCMD, statusParamSyntaxErr, statusNotUseCMD, statusNeedUserId: + //fmt.Println(code) + result = true + break + } + } else if index == 3 { + switch code { + case statusCloseConnect, statusSyntaxErr: + //fmt.Println(code) + result = true + break + } + } + + if index == 3 && result == true { + var err error + var isfs bool + + //fmt.Println(info.Port.Host.Ip, info.Port.Port) + + isfs, err = StartCheckFTPS(info.GetIP(), info.GetPort()) + + if isfs && err == nil { + ftp.isFtps = isfs + } else if err != nil { + log.Warn("FTPS Check Error : ", err.Error()) + } + } + + return result +} + +func (ftp *FTPMatcher) PacketCount() int { + return len(ftp.sendPackets) +} + +func (ftp *FTPMatcher) Packet(index int) *packet.Packet { + return ftp.sendPackets[index] +} + +func (ftp *FTPMatcher) ServiceName() string { + re := "" + if ftp.isFtps { + re = "FTPS" + } else { + re = "FTP" + } + return re +} + +func (ftp *FTPMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (ftp *FTPMatcher) IsNoResponse(index int) bool { + return false +} + +func (ftp *FTPMatcher) IsPrePacket() bool { + return true +} + +func NewFTPMatcher() *FTPMatcher { + + ftm := FTPMatcher{} + + sysStr := "SYST\r\n" + systByte := make([]byte, len(sysStr)) + copy(systByte[:], sysStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(systByte, len(sysStr))) + + passStr := "PASS \r\n" + passByte := make([]byte, len(passStr)) + copy(passByte[:], passStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(passByte, len(passStr))) + + quitStr := "QUIT\r\n" + quitByte := make([]byte, len(quitStr)) + copy(quitByte[:], quitStr) + ftm.sendPackets = append(ftm.sendPackets, packet.NewPacket(quitByte, len(quitStr))) + + return &ftm +} diff --git a/matcher/ftp/ftp_test.go b/matcher/ftp/ftp_test.go new file mode 100644 index 0000000..414092e --- /dev/null +++ b/matcher/ftp/ftp_test.go @@ -0,0 +1,235 @@ +package ftp + +import ( + "fmt" + "testing" + + "git.loafle.net/overflow/collector/discovery/scan/matcher/packet" + "git.loafle.net/overflow/collector/discovery/scan/matcher/scaninfo" + "git.loafle.net/overflow/collector/discovery/types" + log "github.com/cihub/seelog" + "net" +) + +//type FTP struct { +// conn net.Conn +// addr string +// +// reader *bufio.Reader +// writer *bufio.Writer +//} +// +//func (ftp *FTP) Close() { +// ftp.conn.Close() +//} +// +//func Connect(addr string) (*FTP, error) { +// var err error +// var conn net.Conn +// +// if conn, err = net.Dial("tcp", addr); err != nil { +// return nil, err +// } +// +// writer := bufio.NewWriter(conn) +// reader := bufio.NewReader(conn) +// +// obj := &FTP{ +// conn:conn, +// addr:addr, +// reader:reader, +// writer:writer, +// } +// recv, _ := obj.receive() +// +// fmt.Println(recv) +// +// return obj, nil +// +//} +// +//func (ftp *FTP) receive() (string, error) { +// line, err := ftp.receiveLine() +// +// if err != nil { +// return line, err +// } +// +// fmt.Println("len : ", len(line)) +// fmt.Println("line[3] :", line[3]) +// // +// //if (len(line) >= 4) && (line[3] == '-') { +// // closingCode := line[:3] + " " +// // +// // for { +// // str, err := ftp.receiveLine() +// // fmt.Println("str pre: ", str) +// // line = line + str +// // fmt.Println("str after: ", line) +// // if err != nil { +// // return line, err +// // } +// // +// // if len(str) < 4 { +// // fmt.Println("Uncorrectly terminated response") +// // }else { +// // if str[:4] == closingCode { +// // break +// // } +// // } +// // } +// //} +// +// ftp.ReadAndDiscard() +// +// fmt.Println("receive line: ", line) +// return line, err +//} +// +//func (ftp *FTP) ReadAndDiscard() (int, error) { +// var i int +// bufferSize := ftp.reader.Buffered() +// +// for i = 0; i < bufferSize ; i++ { +// if _, err := ftp.reader.ReadByte(); err != nil { +// return i, err +// } +// } +// +// return i, nil +//} +// +//func (ftp *FTP) send(command string, arguments ...interface{}) error { +// +// command = fmt.Sprintf(command) +// command += "\r\n" +// +// if _, err := ftp.writer.WriteString(command); err != nil { +// return err +// } +// +// if err := ftp.writer.Flush(); err != nil { +// return err +// } +// +// return nil +//} +// +//func (ftp *FTP) cmd(expects string, command string, args ...interface{}) (line string, err error) { +// +// if err = ftp.send(command, args); err != nil { +// return +// } +// +// if line, err = ftp.receive(); err != nil { +// return +// } +// +// +// if !strings.HasPrefix(line, expects) { +// err = errors.New(line) +// return +// } +// +// return +//} +// +//func (ftp *FTP) receiveLine() (string, error) { +// line, err := ftp.reader.ReadString('\n') +// +// log.Printf("< %s", line) +// +// return line, err +//} +// +//func (ftp *FTP) Syst() (line string, err error) { +// if err := ftp.send("SYST"); err != nil { +// return "", err +// } +// +// if line, err = ftp.receive(); err != nil { +// return +// } +// +// if !strings.HasPrefix(line, "215") { +// err = errors.New(line) +// return +// } +// +// return strings.SplitN(strings.TrimSpace(line), " ", 2)[1], nil +//} + +//func TestFtp(t *testing.T) { +// var err error +// var ftp *FTP +// //var f *FTPMatcher +// +// if ftp, err = Connect("192.168.1.202:21"); err != nil { +// panic(err) +// } +// +// //f.Match(0, nil,nil) +// ftp.Syst() +// ftp.cmd("503","PASS ") +// ftp.cmd("221","QUIT") +// defer ftp.Close() +//} + +func TestMatchFTP(t *testing.T) { + ftm := NewFTPMatcher() + //fmt.Println(ftm) + //fmt.Println(ftm.sendPackets[0]) + + //log.LoadLogConfig("../../../../../../../../bin/log.xml") + //defer log.Flush() + + port := types.NewPort("21", types.NewHost("192.168.1.202"), types.TYPE_TCP) + info := scaninfo.NewServiceScanInfo(port) + + var ipport string + ipport = port.Host.Ip + ":" + string(port.Port) + + log.Debug(ipport) + + client, _ := net.Dial("tcp", ipport) + + defer client.Close() + + //reader := bufio.NewReader(client) + //writer := bufio.NewWriter(client) + + fmt.Println(ftm.PacketCount()) + //fmt.Println(reader.ReadString('\n')) + + bytes := make([]byte, 512) + + le, _ := client.Read(bytes) + + fmt.Println(bytes) + + b := ftm.Match(0, packet.NewPacket(bytes, le), nil) + + fmt.Println(b) + + for ii := 0; ii < ftm.PacketCount(); ii++ { + pack := ftm.Packet(ii) + + fmt.Println(pack) + + //writer.WriteString(pack) + client.Write(pack.Buffer) + //fmt.Println(reader.ReadString('\n')) + bytes := make([]byte, 512) + + l, _ := client.Read(bytes) + + //fmt.Println(bytes) + + b := ftm.Match(ii+1, packet.NewPacket(bytes, l), info) + + fmt.Println(b) + + } + + fmt.Println("Service Name : ", ftm.ServiceName()) +} diff --git a/matcher/ftp/ftps.go b/matcher/ftp/ftps.go new file mode 100644 index 0000000..ced39ab --- /dev/null +++ b/matcher/ftp/ftps.go @@ -0,0 +1,191 @@ +package ftp + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + log "github.com/cihub/seelog" + "net" + "strings" + "time" +) + +// FTP is a Session for file Transfer Protocol +type FTPS struct { + conn net.Conn + addr string + tlsconfig *tls.Config + + reader *bufio.Reader + writer *bufio.Writer + + isFtps bool +} + +func (fs *FTPS) close() { + fs.conn.Close() +} + +func (fs *FTPS) quit() (err error) { + if _, err := fs.cmd(statusCloseConnect, "QUIT"); err != nil { + return err + } + fs.conn.Close() + fs.conn = nil + return nil +} + +func (fs *FTPS) cmd(expects string, cmd string) (line string, err error) { + if err = fs.send(cmd); err != nil { + log.Error("ftps.go cmd send error: ", err) + return "", err + } + + if line, err = fs.receive(); err != nil { + log.Error("ftps.go cmd receive error: ", err) + return line, err + } + + if !strings.HasPrefix(line, expects) { + err = errors.New(line) + return line, err + } + + return line, err +} + +func (fs *FTPS) readAndDiscard() (int, error) { + var i int + bufferSize := fs.reader.Buffered() + + for i = 0; i < bufferSize; i++ { + if _, err := fs.reader.ReadByte(); err != nil { + return i, err + } + } + + return i, nil +} + +func (fs *FTPS) receive() (string, error) { + line, err := fs.reader.ReadString('\n') + //log.Debug("< %s", line) + + if err != nil { + return line, err + } + + fs.readAndDiscard() + //fmt.Println(line) + return line, err +} + +func (fs *FTPS) send(cmd string) (err error) { + + if len(cmd) == 0 { + err = errors.New("command length 0") + } + + cmd = fmt.Sprintf(cmd) + cmd += "\r\n" + + if _, err := fs.writer.WriteString(cmd); err != nil { + return err + } + + if err := fs.writer.Flush(); err != nil { + return err + } + return nil +} + +func (fs *FTPS) authTls(config *tls.Config) error { + if _, err := fs.cmd(statusTlsUseOK, "AUTH TLS"); err != nil { + return err + } + + fs.tlsconfig = config + + fs.conn = tls.Client(fs.conn, config) + fs.writer = bufio.NewWriter(fs.conn) + fs.reader = bufio.NewReader(fs.conn) + + _, err := fs.cmd(statusOK, "PBSZ 0") + + if err != nil { + return err + } + + _, err = fs.cmd(statusOK, "PROT P") + + if err != nil { + return err + } + + return nil +} +func (fs *FTPS) NewFTPSConnect(addr string) (*FTPS, error) { + var err error + var conn net.Conn + + if conn, err = net.Dial("tcp", addr); err != nil { + log.Error("FTPS Socket Fail: ", err.Error()) + return nil, err + } + + err = conn.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + //log.Error("FTPS Socket Fail: ", err.Error()) + return nil, err + } + + writer := bufio.NewWriter(conn) + reader := bufio.NewReader(conn) + + var line string + + obj := &FTPS{ + conn: conn, + addr: addr, + reader: reader, + writer: writer, + } + + line, err = obj.receive() + + if !strings.HasPrefix(line, "220") { + err = errors.New(line) + return nil, err + } + + //log.Debug(line) + + return obj, err +} + +func StartCheckFTPS(ip string, port string) (bool, error) { + + var err error + var fs *FTPS + + addr := ip + ":" + port + //log.Debug("address : " + addr) + + if fs, err = fs.NewFTPSConnect(addr); err != nil { + return false, err + } + + defer fs.close() + + config := &tls.Config{ + InsecureSkipVerify: true, + ClientAuth: tls.RequestClientCert, + } + + if err = fs.authTls(config); err != nil { + return false, err + } + + return true, err +} diff --git a/matcher/ftp/ftps_test.go b/matcher/ftp/ftps_test.go new file mode 100644 index 0000000..6771896 --- /dev/null +++ b/matcher/ftp/ftps_test.go @@ -0,0 +1,13 @@ +package ftp + +import ( + "fmt" + "testing" +) + +func TestStartCheckFTPS(t *testing.T) { + + isFtps, err := StartCheckFTPS("192.168.1.202", "80") + fmt.Println("Result : ", isFtps) + fmt.Println("Error : ", err) +} diff --git a/matcher/http/http.go b/matcher/http/http.go new file mode 100644 index 0000000..91d4b26 --- /dev/null +++ b/matcher/http/http.go @@ -0,0 +1,90 @@ +package http + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +type HTTPMatcher struct { + sendPackets []*packet.Packet +} + +func (h *HTTPMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + str := string(packet.Buffer) + //fmt.Println(str) + elems := strings.Split(str, "\r\n") + + if len(elems) <= 0 { + return result + } + + protocol := (elems[0])[:8] + + httpv0 := strings.Compare(protocol, "HTTP/1.0") + httpv1 := strings.Compare(protocol, "HTTP/1.1") + httpv2 := strings.Compare(protocol, "HTTP/1.2") + + if 0 == httpv0 || 0 == httpv1 || 0 == httpv2 { + result = true + } + + serverName := "Unknown Server" + + for _, valueStr := range elems { + tempElems := strings.Split(valueStr, ":") + + if 0 == strings.Compare(tempElems[0], "Server") { + serverName = tempElems[1] + break + } + } + + strings.Compare(serverName, "Unknown") + //fmt.Println("HTTP Server Name: ", serverName) + + return result +} + +func (h *HTTPMatcher) PacketCount() int { + return len(h.sendPackets) +} + +func (h *HTTPMatcher) Packet(index int) *packet.Packet { + + return h.sendPackets[index] +} + +func (h *HTTPMatcher) ServiceName() string { + return "HTTP" +} + +func (h *HTTPMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (h *HTTPMatcher) IsNoResponse(index int) bool { + return false +} + +func (h *HTTPMatcher) IsPrePacket() bool { + return false +} + +func NewHTTPMatcher() *HTTPMatcher { + + h := HTTPMatcher{} + reqStr := "GET / HTTP/1.1\r\n\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + + h.sendPackets = append(h.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &h +} diff --git a/matcher/http/http_test.go b/matcher/http/http_test.go new file mode 100644 index 0000000..9200536 --- /dev/null +++ b/matcher/http/http_test.go @@ -0,0 +1,49 @@ +package http + +//import ( +// "fmt" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestHTTPMatcher_Packet(t *testing.T) { +// hm := NewHTTPMatcher() +// fmt.Println(hm) +// fmt.Println(hm.sendPackets[0]) +// +//} +// +//func TestHTTPMatcher_Match(t *testing.T) { +// fmt.Println("Match") +// +// hm := NewHTTPMatcher() +// +// port := types.NewPort("80", types.NewHost("192.168.1.103"), types.TYPE_TCP) +// +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// //fmt.Println(ipport) +// +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// pack := hm.Packet(0) +// +// //fmt.Println(pack) +// +// //writer.WriteString(pack) +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 512) +// +// l, _ := client.Read(bytes) +// +// //fmt.Println(bytes) +// +// hm.Match(0, packet.NewPacket(bytes, l), nil) +// +//} diff --git a/matcher/http/https_test.go b/matcher/http/https_test.go new file mode 100644 index 0000000..ab4b125 --- /dev/null +++ b/matcher/http/https_test.go @@ -0,0 +1,55 @@ +package http + +//import ( +// "crypto/tls" +// "fmt" +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "log" +// "net" +// "testing" +// "time" +//) +// +//func TestHTTPSMatcher_Match(t *testing.T) { +// netinfo := "192.168.1.1:443" +// dialer := &net.Dialer{ +// Timeout: 5 * time.Second, +// } +// +// conn, err := tls.DialWithDialer( +// dialer, +// "tcp", +// netinfo, +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.103", +// }, +// ) +// +// if err != nil { +// log.Println(err) +// return +// } +// defer conn.Close() +// +// //fmt.Println(conn) +// h := NewHTTPMatcher() +// +// pac := h.Packet(0) +// +// //fmt.Println(pac) +// //fmt.Println(pac.Buffer) +// +// //bytes := make([]byte, 1024) +// +// l, _ := conn.Write(pac.Buffer) +// +// buf := make([]byte, 1024) +// l, _ = conn.Read(buf) +// +// fmt.Println(string(buf)) +// fmt.Println(l) +// is := h.Match(0, packet.NewPacket(buf, l), nil) +// fmt.Println(is) +// +//} diff --git a/matcher/imap/imap.go b/matcher/imap/imap.go new file mode 100644 index 0000000..5102337 --- /dev/null +++ b/matcher/imap/imap.go @@ -0,0 +1,87 @@ +package imap + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + PRE_COMPARE_STR = "* OK" + SEND_COMPARE_STR = "* BYE" +) + +type IMAPMatcher struct { + sendPackets []*packet.Packet +} + +func (i *IMAPMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + switch index { + case 0: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 3 { + return false + } + + compareStr := recvStr[0:4] + + if compareStr == PRE_COMPARE_STR { + return true + } + + case 1: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 5 { + return false + } + + compareStr := recvStr[0:5] + + if compareStr == SEND_COMPARE_STR { + return true + } + + } + + return false +} + +func (i *IMAPMatcher) PacketCount() int { + return len(i.sendPackets) +} +func (i *IMAPMatcher) Packet(index int) *packet.Packet { + return i.sendPackets[index] +} +func (i *IMAPMatcher) ServiceName() string { + return "IMAP" +} + +func (i *IMAPMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (i *IMAPMatcher) IsNoResponse(index int) bool { + + return false +} + +func (i *IMAPMatcher) IsPrePacket() bool { + return true +} + +func NewIMAPMatcher() *IMAPMatcher { + + im := IMAPMatcher{} + + reqStr := "A0001 LOGOUT\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + + im.sendPackets = append(im.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &im + +} diff --git a/matcher/imap/imap_test.go b/matcher/imap/imap_test.go new file mode 100644 index 0000000..c5f8e8c --- /dev/null +++ b/matcher/imap/imap_test.go @@ -0,0 +1,150 @@ +package imap + +// +//import ( +// "crypto/tls" +// "fmt" +// "loafle.com/overflow/collector/core/scan/port" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "loafle.com/overflow/collector/core/scan/service/matcher/scaninfo" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func ImapRun(client net.Conn, t *testing.T) { +// +// lm := NewIMAPMatcher() +// +// port := types.NewPort("143", types.NewHost("192.168.1.215"), types.TYPE_TCP) +// +// scanInfo := types.NewServiceScanInfo(port) +// +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// fmt.Println(ipport) +// //client, _ := net.Dial("tcp", ipport) +// +// //defer client.Close() +// +// bytett := make([]byte, 1024) +// +// rr, _ := client.Read(bytett) +// +// bb := lm.Match(0, packet.NewPacket(bytett, rr), scanInfo) +// +// if bb { +// t.Log("good!") +// } +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// //fmt.Println(pack) +// +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 1024) +// +// read, _ := client.Read(bytes) +// +// fmt.Println(cap(bytes)) +// +// //fmt.Println(bytes) +// +// b := lm.Match(ii+1, packet.NewPacket(bytes, read), scanInfo) +// +// if b { +// t.Log("send Good!") +// } +// +// } +// t.Log(scanInfo) +// +//} +// +//func TestIMapTls(t *testing.T) { +// +// conn, _ := tls.Dial( +// "tcp", +// "192.168.1.215:993", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.215", +// }, +// ) +// +// defer conn.Close() +// +// ImapRun(conn, t) +// +//} +// +//func TestIMapNormal(t *testing.T) { +// +// client, _ := net.Dial("tcp", "192.168.1.215:143") +// +// defer client.Close() +// +// ImapRun(client, t) +// +//} +// +//func TestImap(t *testing.T) { +// +// lm := NewIMAPMatcher() +// +// //port := types.NewPort("143", types.NewHost("192.168.1.215"), types.TYPE_TCP) +// +// //scanInfo := scaninfo.NewServiceScanInfo(port) +// +// var ipport string +// //ipport = port.Host.Ip + ":" + port.Port_ +// +// fmt.Println(ipport) +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// bytett := make([]byte, 1024) +// +// rr, _ := client.Read(bytett) +// +// //bb := lm.Match(0, packet.NewPacket(bytett, rr), scanInfo) +// bb := lm.Match(0, packet.NewPacket(bytett, rr), nil) +// +// if bb { +// t.Log("good!") +// } +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// //fmt.Println(pack) +// +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 1024) +// +// read, _ := client.Read(bytes) +// +// fmt.Println(cap(bytes)) +// +// //fmt.Println(bytes) +// +// b := lm.Match(ii+1, packet.NewPacket(bytes, read), nil) +// +// if b { +// t.Log("send Good!") +// } +// +// } +// //t.Log(scanInfo) +//} diff --git a/matcher/ldap/ldap.go b/matcher/ldap/ldap.go new file mode 100644 index 0000000..4aa246c --- /dev/null +++ b/matcher/ldap/ldap.go @@ -0,0 +1,198 @@ +package ldap + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +type LDAPMatcher struct { + sendPackets []*packet.Packet +} + +func (l *LDAPMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + ldapRecv := LDAP_RECV{} + + binary.Read(buf, binary.LittleEndian, &ldapRecv) + + if ldapRecv.MessageId != LDAP_MESSAGE_ID { + return false + } + + if ldapRecv.ProtocolOp != LDAP_RES_BIND { + return false + } + + if ldapRecv.ResultCode != LDAP_SUCCESS { + return false + } + + return true +} + +func (l *LDAPMatcher) PacketCount() int { + return len(l.sendPackets) +} +func (l *LDAPMatcher) Packet(index int) *packet.Packet { + return l.sendPackets[index] +} +func (l *LDAPMatcher) ServiceName() string { + return "LDAP" +} + +func (l *LDAPMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (l *LDAPMatcher) IsNoResponse(index int) bool { + + if index == 1 { + return true + } + + return false +} + +func (l *LDAPMatcher) IsPrePacket() bool { + return false +} + +func NewLDAPMatcher() *LDAPMatcher { + + ls := LDAP_SEND{ + DefaultCode: 0x30, + PacketLength: 0x0c, // size -2 + NextType1: 0x02, + NextTypeLength1: 0x01, + MessageId: LDAP_MESSAGE_ID, + ProtocolOp: LDAP_REQ_BIND, + ProtocolOpLength: 0x07, + NextType2: 0x02, + NextTypeLength2: 0x01, + Version: LDAP_VERSION3, + NextType3: 0x04, + NextTypeLength3: 0x00, + Auth: LDAP_AUTH_SIMPLE, + AuthLength: 0x00, + } + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.LittleEndian, ls) + + sendByte1 := mCache.Bytes() + + lm := LDAPMatcher{ + //sendPackets: make([][]byte, 2), + } + + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + lm.sendPackets = append(lm.sendPackets, pp) + + lq := LDAP_QUIT{ + DefaultCode: 0x30, + UnknwonCode1: 0x84, + PacketLength: 0x05, + NextType1: 0x02, + NextTypeLength1: 0x01, + MessageId: LDAP_MESSAGE_ID_QUIT, + ProtocolOp: LDAP_REQ_UNBIND, + protocolOpLength: 0x00, + } + + lqBuffer := new(bytes.Buffer) + binary.Write(lqBuffer, binary.BigEndian, lq) + + sendByte2 := lqBuffer.Bytes() + + pp2 := packet.NewPacket(sendByte2, len(sendByte2)) + + lm.sendPackets = append(lm.sendPackets, pp2) + + return &lm + +} + +type LDAP_SEND struct { + DefaultCode uint8 + PacketLength uint8 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint8 + + ProtocolOp uint8 + ProtocolOpLength uint8 + NextType2 uint8 + NextTypeLength2 uint8 + + Version uint8 + + NextType3 uint8 + NextTypeLength3 uint8 + Auth uint8 + AuthLength uint8 +} + +type LDAP_RECV struct { + DefaultCode uint8 + UnknwonCode1 uint8 + EndCode11 uint8 + EndCode12 uint8 + + MessageId uint8 + + ProtocolOp uint8 + UnknwonCode2 uint8 + EndCode21 uint8 + EndCode22 uint8 + + ResultCode uint8 + + UnknwonCode3 uint8 + UnknwonCode4 uint8 + Auth uint8 + UnknwonCode5 uint8 +} + +type LDAP_QUIT struct { + DefaultCode uint8 + UnknwonCode1 uint8 + + PacketLength uint32 + + NextType1 uint8 + NextTypeLength1 uint8 + + MessageId uint8 + + ProtocolOp uint8 + + protocolOpLength uint8 +} + +const ( + LDAP_MESSAGE_ID = 0x99 + LDAP_MESSAGE_ID_QUIT = 0x89 + + LDAP_VERSION3 = 3 + + LDAP_SUCCESS = 0x00 + + LDAP_REQ_BIND = 0x60 + + LDAP_REQ_UNBIND = 0x42 + + LDAP_RES_BIND = 0x61 + + LDAP_AUTH_SIMPLE = 0x80 +) diff --git a/matcher/ldap/ldap_test.go b/matcher/ldap/ldap_test.go new file mode 100644 index 0000000..2682bb6 --- /dev/null +++ b/matcher/ldap/ldap_test.go @@ -0,0 +1,110 @@ +package ldap + +// +//import ( +// "crypto/tls" +// "fmt" +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "loafle.com/overflow/collector/discovery/scan/matcher/scaninfo" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +////func SetUp() { +//// fmt.Println("SetUp") +////} +//// +////func TearDown() { +//// fmt.Println("TearDown") +////} +// +////func TestMain(m *testing.M) { +//// SetUp() +//// m.Run() +//// TearDown() +////} +// +//func TestAAAA(t *testing.T) { +// ///animals := []Animal{Dog{}, Cat{}, Llama{}, JavaProgrammer{}} +// +// var ttt [][]int = make([][]int, 10) +// +// var aaa []int +// aaa = append(aaa, 111) +// +// ttt = append(ttt, aaa) +// +// fmt.Println(cap(ttt)) +// +//} +// +//func ldapRun(client net.Conn, t *testing.T) { +// lm := NewLDAPMatcher() +// +// port := types.NewPort("389", types.NewHost("192.168.1.215"), types.TYPE_TCP) +// scanInfo := scaninfo.NewServiceScanInfo(port) +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// fmt.Println(ipport) +// //client, _ := net.Dial("tcp", ipport) +// //defer client.Close() +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// bytes := make([]byte, 1024) +// +// client.Write(pack.Buffer) +// +// read, _ := client.Read(bytes) +// +// if read <= 0 { +// bb := lm.IsNoResponse(ii) +// if bb { +// +// t.Log("IsNoResponse good") +// break +// } +// +// } +// +// fmt.Println(bytes) +// +// b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) +// +// if b { +// t.Log("Good") +// } +// +// } +// +// t.Log(scanInfo) +//} +// +//func TestLdapTls(t *testing.T) { +// conn, _ := tls.Dial( +// "tcp", +// "192.168.1.215:636", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.215", +// }, +// ) +// +// defer conn.Close() +// +// ldapRun(conn, t) +//} +// +//func TestLdapNormal(t *testing.T) { +// client, _ := net.Dial("tcp", "192.168.1.215:389") +// +// defer client.Close() +// +// ldapRun(client, t) +//} diff --git a/matcher/matcher.go b/matcher/matcher.go new file mode 100644 index 0000000..afbc884 --- /dev/null +++ b/matcher/matcher.go @@ -0,0 +1,115 @@ +package matcher + +import ( + //"git.loafle.net/overflow/collector/core/scan/service/matcher/activedirectory" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/cassandra" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/dns" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/ftp" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/http" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/imap" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/ldap" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/mongodb" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/mssql" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/mysql" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/netbios" + "git.loafle.net/overflow/commons_go/matcher/oracle" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/pop" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/redis" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/rmi" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/smb" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/smtp" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/snmp" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/ssh" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/telnet" + //"git.loafle.net/overflow/collector/core/scan/service/matcher/wmi" +) + +var ( + TcpMatchers []Matcher + UdpMatchers []UDPMatcher +) + +func init() { + //TCP + //TcpMatchers = append(TcpMatchers, smtp.NewSmtpMatcher()) + //TcpMatchers = append(TcpMatchers, ldap.NewLDAPMatcher()) + //TcpMatchers = append(TcpMatchers, activedirectory.NewActiveDirectoryMatcher()) + //TcpMatchers = append(TcpMatchers, mongodb.NewMongoDBMatcher()) + //TcpMatchers = append(TcpMatchers, mysql.NewMySqlMatcher()) + //TcpMatchers = append(TcpMatchers, mssql.NewMSSqlMatcher()) + //TcpMatchers = append(TcpMatchers, redis.NewRedisMatcher()) + //TcpMatchers = append(TcpMatchers, redis.NewRedisProtectedMatcher()) + //TcpMatchers = append(TcpMatchers, netbios.NewNetBiosMatcher()) + //TcpMatchers = append(TcpMatchers, smb.NewSMBMatcher()) + //TcpMatchers = append(TcpMatchers, cassandra.NewCassandraMatcher()) + //TcpMatchers = append(TcpMatchers, imap.NewIMAPMatcher()) + TcpMatchers = append(TcpMatchers, oracle.NewOracleMatcher()) + //TcpMatchers = append(TcpMatchers, pop.NewPOPMatcher()) + //TcpMatchers = append(TcpMatchers, wmi.NewWMIMatcher()) + //TcpMatchers = append(TcpMatchers, ftp.NewFTPMatcher()) + //TcpMatchers = append(TcpMatchers, http.NewHTTPMatcher()) + //TcpMatchers = append(TcpMatchers, rmi.NewRMIMatcher()) + //TcpMatchers = append(TcpMatchers, ssh.NewSSHMatcher()) + //TcpMatchers = append(TcpMatchers, telnet.NewTelnetMatcher()) + //UdpMatchers = append(UdpMatchers, dns.NewDnsMatcher()) + //UdpMatchers = append(UdpMatchers, snmp.NewSNMPv2Matcher()) + //UdpMatchers = append(UdpMatchers, snmp.NewSNMPv3Matcher()) +} + +type Matcher interface { + Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool + + PacketCount() int + Packet(index int) *packet.Packet + ServiceName() string + + IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool + IsNoResponse(index int) bool + IsPrePacket() bool +} + +type UDPMatcher interface { + Matcher + IsSend(port int) bool +} + +func GetTcpMatchers(ispre bool) []Matcher { + + retMatchers := make([]Matcher, 0) + l := len(TcpMatchers) + for i := 0; i < l; i++ { + c := TcpMatchers[i].IsPrePacket() + if c == ispre { + retMatchers = append(retMatchers, TcpMatchers[i]) + } + } + + return retMatchers +} + +func GetUdpMatchers() []UDPMatcher { + + retMatchers := make([]UDPMatcher, 0) + l := len(UdpMatchers) + for i := 0; i < l; i++ { + retMatchers = append(retMatchers, UdpMatchers[i]) + } + + return retMatchers +} + +func GetMatcherByName(serName string) Matcher { + for _, m := range TcpMatchers { + if m.ServiceName() == serName { + return m + } + } + for _, m := range UdpMatchers { + if m.ServiceName() == serName { + return m + } + } + return nil +} diff --git a/matcher/matcher_test.go b/matcher/matcher_test.go new file mode 100644 index 0000000..83b63e9 --- /dev/null +++ b/matcher/matcher_test.go @@ -0,0 +1,87 @@ +package matcher + +import ( + "fmt" + "testing" +) + +type TestMatcher struct { +} + +func (t *TestMatcher) ServiceName() string { + return "TestMatcher" +} + +func (t *TestMatcher) Match(index int, packet []byte, info ServiceScanInfo) bool { + return true +} + +func (t *TestMatcher) PacketCount() int { + return 1 +} +func (t *TestMatcher) Packet(index int) []byte { + return nil +} + +func (t *TestMatcher) IsError(index int, packet []byte, info ServiceScanInfo) bool { + return true +} +func (t *TestMatcher) IsNoResponse(index int) bool { + return true +} +func (t *TestMatcher) IsPrePacket() bool { + return true +} + +type Animal interface { + Speak() string +} + +type Dog struct { +} + +func (d Dog) Speak() string { + return "Woof!" +} + +type Cat struct { +} + +func (c Cat) Speak() string { + return "Meow!" +} + +type Llama struct { +} + +func (l Llama) Speak() string { + return "?????" +} + +type JavaProgrammer struct { +} + +func (j JavaProgrammer) Speak() string { + return "Design patterns!" +} + +func TestMatcherTTT(t *testing.T) { + + //animals := []Animal{Dog{}, Cat{}, Llama{}, JavaProgrammer{}} + + var animals []Animal + + animals = append(animals, &Dog{}) + animals = append(animals, &Cat{}) + animals = append(animals, &Llama{}) + animals = append(animals, &JavaProgrammer{}) + + for _, a := range animals { + fmt.Println(a.Speak()) + } + +} + +func TestMatchersInit(t *testing.T) { + +} diff --git a/matcher/mongodb/mongodb.go b/matcher/mongodb/mongodb.go new file mode 100644 index 0000000..c5c0441 --- /dev/null +++ b/matcher/mongodb/mongodb.go @@ -0,0 +1,122 @@ +package mongodb + +import ( + "bytes" + "encoding/binary" + "math/rand" + + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + MONGO_OP_REQUEST uint32 = 2004 + MONGO_OP_REPLY uint32 = 1 + MONGO_FCNAME string = "admin.$cmd" + MONGO_ELEMENT string = "ismaster" +) + +var MONGO_REQUEST_ID uint32 + +type mongo struct { + MessageLength uint32 + RequestId uint32 + ResponseTo uint32 + OpCode uint32 + Flags uint32 + FullCollectionName [11]byte + NumberToSkip uint32 + NumberToReturn int32 + DocumentLength uint32 + Type_ uint8 + Element [9]byte + Value uint8 + _ uint8 +} + +type MongoDBMatcher struct { + packets []*packet.Packet +} + +func NewMongoDBMatcher() *MongoDBMatcher { + + mongoMatcher := &MongoDBMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, mongo{}) + + var fcn [11]byte + copy(fcn[:], MONGO_FCNAME) + + var elem [9]byte + copy(elem[:], MONGO_ELEMENT) + + MONGO_REQUEST_ID = rand.Uint32() + m := mongo{ + MessageLength: uint32(len(tempBuf.Bytes())), + RequestId: MONGO_REQUEST_ID, + ResponseTo: 0, + OpCode: MONGO_OP_REQUEST, + Flags: 0, + FullCollectionName: fcn, + NumberToSkip: 0, + NumberToReturn: -1, + DocumentLength: 16, + Type_: 0x08, + Element: elem, + Value: 1, + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, m) + + mongoMatcher.packets = append(mongoMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return mongoMatcher +} + +func (t *MongoDBMatcher) ServiceName() string { + return "MongoDB" +} + +func (t *MongoDBMatcher) PacketCount() int { + return len(t.packets) +} +func (t *MongoDBMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MongoDBMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *MongoDBMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MongoDBMatcher) IsPrePacket() bool { + return false +} + +func (t *MongoDBMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + m := mongo{} + if err := binary.Read(reader, binary.LittleEndian, &m); err != nil { + return false + } + + if uint32(packet.Len) != m.MessageLength || + m.ResponseTo != MONGO_REQUEST_ID || + m.OpCode != MONGO_OP_REPLY { + return false + } + + return true + +} diff --git a/matcher/mongodb/mongodb_test.go b/matcher/mongodb/mongodb_test.go new file mode 100644 index 0000000..4e09406 --- /dev/null +++ b/matcher/mongodb/mongodb_test.go @@ -0,0 +1,55 @@ +package mongodb + +// +//import ( +// "crypto/tls" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestMongoNor(t *testing.T) { +// +// conn, _ := net.Dial("tcp", "192.168.1.105:27017") +// +// defer conn.Close() +// +// MongoRun(conn, t) +// +//} +//func TestMongoTLS(t *testing.T) { +// conn, _ := tls.Dial( +// "tcp", +// "192.168.1.105:27017", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.105", +// }, +// ) +// +// defer conn.Close() +// +// MongoRun(conn, t) +//} +// +//func MongoRun(conn net.Conn, t *testing.T) { +// +// m := NewMongoDBMatcher() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("MongoDB found") +// return +// } +// +// t.Error("MongoDB not found") +// } +// +//} diff --git a/matcher/mssql/mssql.go b/matcher/mssql/mssql.go new file mode 100644 index 0000000..f5b8eab --- /dev/null +++ b/matcher/mssql/mssql.go @@ -0,0 +1,173 @@ +package mssql + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + HEADER_TYPE_PRELOGIN uint8 = 0x12 + HEADER_TYPE_RESPONSE uint8 = 0x4 + PL_OPTION_TOKEN_VERSION uint8 = 0x00 + PL_OPTION_TOKEN_ENCRYPTION uint8 = 0x01 + PL_OPTION_TOKEN_TRACEID uint8 = 0x05 + PL_OPTION_TOKEN_TERMINATOR uint8 = 0xff + + ENCRYPT_OFF string = "Encryption is available but off." + ENCRYPT_ON string = "Encryption is available and on." + ENCRYPT_NOT_SUP string = "Encryption is not available." + ENCRYPT_REQ string = "Encryption is required." +) + +type PreloginMsg struct { + VersionToken uint8 + VersionOffset uint16 + VersionLength uint16 + + EncryptionToken uint8 + EncryptionOffset uint16 + EncryptionLength uint16 + + TraceIdToken uint8 + TraceIdOffset uint16 + TraceIdLength uint16 + + Terminator uint8 + Options [7]uint8 + TraceId [36]uint8 +} + +type mssql struct { + Type_ uint8 + Status uint8 + Length uint16 + Channel uint16 + PacketNum uint8 + Window uint8 + Prelogin PreloginMsg +} + +type PreloginResponse struct { + Msg [256]uint8 +} + +type mssqlResponse struct { + Type_ uint8 + Status uint8 + Length uint16 + Channel uint16 + PacketNum uint8 + Window uint8 + PreLoginResp PreloginResponse +} + +type MSSqlMatcher struct { + packets []*packet.Packet + isSSL bool +} + +func NewMSSqlMatcher() *MSSqlMatcher { + + mssqlMatcher := &MSSqlMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, mssql{}) + + m := mssql{ + Type_: HEADER_TYPE_PRELOGIN, + Status: 0x01, + Length: uint16(len(tempBuf.Bytes())), + Channel: 0, + PacketNum: 0, + Window: 0, + Prelogin: PreloginMsg{ + VersionToken: PL_OPTION_TOKEN_VERSION, + VersionOffset: 0x0010, + VersionLength: 0x0006, + EncryptionToken: PL_OPTION_TOKEN_ENCRYPTION, + EncryptionOffset: 0x0016, + EncryptionLength: 0x0001, + TraceIdToken: PL_OPTION_TOKEN_TRACEID, + TraceIdOffset: 0x0017, + TraceIdLength: 0x0024, + Terminator: PL_OPTION_TOKEN_TERMINATOR, + }, + } + + writer := new(bytes.Buffer) + binary.Write(writer, binary.BigEndian, m) + + mssqlMatcher.packets = append(mssqlMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return mssqlMatcher +} + +func (t *MSSqlMatcher) ServiceName() string { + if t.isSSL { + return "SQL Server (SSL)" + } + return "SQL Server" +} + +func (t *MSSqlMatcher) PacketCount() int { + return len(t.packets) +} +func (t *MSSqlMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MSSqlMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *MSSqlMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MSSqlMatcher) IsPrePacket() bool { + return false +} + +func (t *MSSqlMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + m := mssqlResponse{} + + if err := binary.Read(reader, binary.BigEndian, &m); err != nil { + return false + } + + if m.Type_ != HEADER_TYPE_RESPONSE { + return false + } + + if m.Length != uint16(packet.Len) { + return false + } + + switch m.PreLoginResp.Msg[m.Length-9 : m.Length-8][0] { + case 0: + return true + case 1: + t.isSSL = true + return true + case 2: + return true + case 3: + t.isSSL = true + return true + default: + return false + } + + return false + +} diff --git a/matcher/mssql/mssql_test.go b/matcher/mssql/mssql_test.go new file mode 100644 index 0000000..199d11c --- /dev/null +++ b/matcher/mssql/mssql_test.go @@ -0,0 +1,64 @@ +package mssql + +// +//import ( +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "net" +// "testing" +//) +// +///* +//192.168.1.106:1433 - normal +//192.168.1.103:1433 - ssl +//*/ +//func TestSqlNor(t *testing.T) { +// +// conn, _ := net.Dial("tcp", "192.168.1.103:1433") +// +// defer conn.Close() +// +// sqlServerRun(conn, t) +// +//} +// +////func TestSqlTLS(t *testing.T) { +//// conn, err := tls.Dial( +//// "tcp", +//// "192.168.1.103:7680", +//// &tls.Config{ +//// InsecureSkipVerify: true, +//// ServerName: "192.168.1.103", +//// }, +//// ) +//// +//// if err != nil { +//// t.Log(err) +//// return +//// } +//// +//// defer conn.Close() +//// +//// sqlServerRun(conn, t) +////} +// +//func sqlServerRun(conn net.Conn, t *testing.T) { +// +// m := NewMSSqlMatcher() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log(m.ServiceName()) +// return +// } +// +// t.Error("MSSQL not found") +// } +// +//} diff --git a/matcher/mysql/mysql.go b/matcher/mysql/mysql.go new file mode 100644 index 0000000..e0d5944 --- /dev/null +++ b/matcher/mysql/mysql.go @@ -0,0 +1,151 @@ +package mysql + +import ( + "bytes" + "encoding/binary" + "fmt" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strconv" + "strings" +) + +type PacketSize struct { + PacketLength [3]byte + PacketNumber byte +} + +type mySql struct { + Payload PacketSize + Protocol byte + Version [256]byte + + TreadId uint32 + Salt1 [9]byte + ServerCapa uint16 + ServerLang uint8 + ServerStat uint16 + ExtServerCapa uint16 + AuthPlugLen uint8 + _ [10]uint8 + Salt2 [13]uint8 + AuthPlugName [64]uint8 +} + +type MySqlMatcher struct { + packets []*packet.Packet + version string + isErrResp bool + errCode int + errMsg string + isSSL bool +} + +func NewMySqlMatcher() *MySqlMatcher { + return &MySqlMatcher{} +} + +func (t *MySqlMatcher) ServiceName() string { + if t.isErrResp { + return "MySQL" + "(Err-" + strconv.Itoa(t.errCode) + " : " + t.errMsg + ")" + } + if t.isSSL { + return "MySQL" + "-" + t.version + "(SSL)" + } + return "MySQL" + "(" + t.version + ")" +} + +func (t *MySqlMatcher) PacketCount() int { + return len(t.packets) +} + +func (t *MySqlMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *MySqlMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *MySqlMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *MySqlMatcher) IsPrePacket() bool { + return true +} + +func (t *MySqlMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil || len(packet.Buffer) <= 0 { + return false + } + + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + m := mySql{} + if err := binary.Read(r, binary.LittleEndian, &m); err != nil { + return false + } + + buf := bytes.NewBuffer(m.Payload.PacketLength[:]) + packetLen, _ := binary.ReadUvarint(buf) + + if packetLen != uint64(packet.Len-4) { + return false + } + + if m.Protocol == 0xff { + //MySQL error response + var code [2]uint8 + copy(code[:], m.Version[:2]) + + var msg [256]uint8 + copy(msg[:], m.Version[2:]) + + errCode := binary.LittleEndian.Uint16(code[:]) + if errCode < 1000 || errCode > 1727 { + return false + } + errMsg := bytes.Trim(msg[:], "\x00") + + t.isErrResp = true + t.errCode = int(errCode) + t.errMsg = string(errMsg) + + return true + } + + if m.Protocol != 10 && m.Protocol != 9 { + return false + } + + t.checkSSL(packet) + + return true +} + +func (t *MySqlMatcher) checkSSL(packet *packet.Packet) { + + temp := make([]byte, packet.Len) + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + if err := binary.Read(r, binary.LittleEndian, &temp); err != nil { + return + } + + t.version = strings.Split(string(packet.Buffer)[5:packet.Len], "\x00")[0] + versionLen := len(t.version) + 1 + data := binary.LittleEndian.Uint16(temp[18+versionLen : 20+versionLen]) + s := fmt.Sprintf("%b", data) + + for i, b := range s { + if i == 4 { + if b == 49 { + t.isSSL = true + } + } + } +} diff --git a/matcher/mysql/mysql_test.go b/matcher/mysql/mysql_test.go new file mode 100644 index 0000000..29c294c --- /dev/null +++ b/matcher/mysql/mysql_test.go @@ -0,0 +1,71 @@ +package mysql + +// +//import ( +// "crypto/tls" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestMySql(t *testing.T) { +// +// m := NewMySqlMatcher() +// +// /* +// 192.168.1.103:3306 - normal +// 192.168.1.105:8306 - ssl +// 192.168.1.203:3306 - mysql with error code +// */ +// conn, _ := net.Dial("tcp", "192.168.1.215:3306") +// +// defer conn.Close() +// +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(0, p, nil) { +// t.Log(m.ServiceName()) +// return +// } +// +// t.Error("MySQL not found") +// +//} +// +//func TestCassandraTLS(t *testing.T) { +// +// m := NewMySqlMatcher() +// +// conn, err := tls.Dial( +// "tcp", +// "192.168.1.105:8306", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.105", +// }, +// ) +// if err != nil { +// t.Fatal(err) +// } +// +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log(m.ServiceName()) +// return +// } +// +// t.Error("MySQL not found") +// } +// +//} diff --git a/matcher/netbios/netbios.go b/matcher/netbios/netbios.go new file mode 100644 index 0000000..b314bbc --- /dev/null +++ b/matcher/netbios/netbios.go @@ -0,0 +1,109 @@ +package netbios + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + NBSS_SESSION_REQUEST uint8 = 0x81 + NBSS_POSITIVE_SESSION_RESPONSE uint8 = 0x82 + NBSS_NEGATIVE_SESSION_RESPONSE uint8 = 0x83 + ADDR string = "192.168.1.202:139" +) + +type netBios struct { + MsgType uint8 + Flags uint8 //0-6 : Reserved, must be zero. 7 : Length extension. + Length uint16 + CalledNameLen uint8 + CalledName [16]uint16 + _ uint8 + CallingNameLen uint8 + CallingName [16]uint16 + _ uint8 +} + +type NetBiosMatcher struct { + packets []*packet.Packet +} + +func NewNetBiosMatcher() *NetBiosMatcher { + + nbssMatcher := &NetBiosMatcher{} + + tempBuf := new(bytes.Buffer) + binary.Write(tempBuf, binary.BigEndian, netBios{}) + + query := netBios{ + MsgType: NBSS_SESSION_REQUEST, + Flags: 0x00, + Length: 0x4400, + CalledNameLen: 0x20, + CallingNameLen: 0x20, + } + + query.CalledName[0] = 0x4D45 // L + query.CalledName[1] = 0x4745 // F + + query.CallingName[0] = 0x4D45 + query.CallingName[1] = 0x4745 + + for i := 2; i < 16; i++ { + query.CalledName[i] = 0x4143 //Space + query.CallingName[i] = 0x4143 + } + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, query) + + nbssMatcher.packets = append(nbssMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return nbssMatcher +} + +func (t *NetBiosMatcher) ServiceName() string { + return "NBSS" +} + +func (t *NetBiosMatcher) PacketCount() int { + return len(t.packets) +} +func (t *NetBiosMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *NetBiosMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *NetBiosMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *NetBiosMatcher) IsPrePacket() bool { + return false +} + +func (t *NetBiosMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + n := netBios{} + if err := binary.Read(reader, binary.LittleEndian, &n); err != nil { + return false + } + + if NBSS_NEGATIVE_SESSION_RESPONSE != n.MsgType { + return false + } + + return true + +} diff --git a/matcher/netbios/netbios_test.go b/matcher/netbios/netbios_test.go new file mode 100644 index 0000000..701169d --- /dev/null +++ b/matcher/netbios/netbios_test.go @@ -0,0 +1,34 @@ +package netbios + +// +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestNBSS(t *testing.T) { +// +// m := NewNetBiosMatcher() +// +// conn, _ := net.Dial("tcp", "192.168.1.106:139") +// +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("NBSS found") +// return +// } +// +// t.Error("NBSS not found") +// } +// +//} diff --git a/matcher/oracle/oracle.go b/matcher/oracle/oracle.go new file mode 100644 index 0000000..6062ad1 --- /dev/null +++ b/matcher/oracle/oracle.go @@ -0,0 +1,191 @@ +package oracle + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +type OracleMatcher struct { + sendPackets []*packet.Packet +} + +func (o *OracleMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + header := header_packet{} + refuse := body_refuse{} + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + binary.Read(buf, binary.BigEndian, &header) + binary.Read(buf, binary.BigEndian, &refuse) + + //fmt.Println(header) + //fmt.Println(refuse) + + if header.Check_sum != 0 { + return false + } + if header.Types != 4 { + return false + } + if header.Reserved_byte != 0 { + return false + } + if header.Header_sum != 0 { + return false + } + if refuse.Reason_user != 34 { + return false + } + if refuse.Reason_system != 0 { + return false + } + + var dataLen int = int(refuse.Data_len) + if dataLen != packet.Len-12 { // + if dataLen != packet.Len-22 { // morformed packet error not user not service + return false + } + } + + return true +} + +func (o *OracleMatcher) PacketCount() int { + return len(o.sendPackets) +} +func (o *OracleMatcher) Packet(index int) *packet.Packet { + return o.sendPackets[index] +} +func (o *OracleMatcher) ServiceName() string { + return "OracleMatcher" +} + +func (o *OracleMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (o *OracleMatcher) IsNoResponse(index int) bool { + + return false +} + +func (o *OracleMatcher) IsPrePacket() bool { + return false +} + +func NewOracleMatcher() *OracleMatcher { + + pm := OracleMatcher{} + + hp := header_packet{ + Length: 247, + Check_sum: 0, + Types: 1, + Reserved_byte: 0, + Header_sum: 0, + } + + bc := body_connect{ + Version: 315, + Version_compatible: 300, + //Service_options: + Session_unit_size: 8192, + Maxumum_trans_data_unit_size: 65535, + //Nt_protocol_characteristics: + Line_turnaround_value: 0, + Value_of_1_in_hardware: 1, + Length_of_connect_data: 177, + Offset_to_connect_data: 70, + Maximum_receivable_connect_data: 0, + //Connect_flag0: + //Connect_flag1: + Trace_cross_facility_item_1: 0, + Trace_cross_facility_item_2: 0, + Trace_unique_connection_id: 0, + //Unknown_data: + //Connect_data: + + } + + bc.Service_options[0] = 0x0c + bc.Service_options[1] = 0x41 + + bc.Nt_protocol_characteristics[0] = 0x4f + bc.Nt_protocol_characteristics[1] = 0x98 + + bc.Connect_flag0 = 0x81 + bc.Connect_flag1 = 0x81 + + bc.Unknown_data[10] = 0x20 + bc.Unknown_data[13] = 0x20 + + conDataStr := "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=192.168.1.30)(PORT=1521))(CONNECT_DATA=(CID=(PROGRAM=JDBC Thin Client)(HOST=__jdbc__)(USER=loafle.match))(SERVICE_NAME=oracle.loafle.com1)))" + //conDataStr := "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=192.168.1.30)(PORT=1521))(CONNECT_DATA=(CID=(PROGRAM=JDBC Thin Client)(HOST=__jdbc__)(USER=Jackdaw))(SERVICE_NAME=oracle.loafle.co1m)))" + + connect_data := make([]byte, len(conDataStr)) + copy(connect_data, conDataStr) + + hpBuf := new(bytes.Buffer) + binary.Write(hpBuf, binary.BigEndian, hp) + + hpBt := hpBuf.Bytes() + + bcBuf := new(bytes.Buffer) + binary.Write(bcBuf, binary.BigEndian, bc) + bcBt := bcBuf.Bytes() + + byteSize := len(hpBt) + len(bcBt) + len(conDataStr) + sendByte := make([]byte, byteSize) + + copy(sendByte[0:], hpBt) + copy(sendByte[len(hpBt):], bcBt) + copy(sendByte[len(hpBt)+len(bcBt):], connect_data) + + pm.sendPackets = append(pm.sendPackets, packet.NewPacket(sendByte, byteSize)) + + return &pm + +} + +type header_packet struct { + Length uint16 + Check_sum uint16 + Types byte + Reserved_byte byte + Header_sum uint16 +} + +type body_connect struct { + Version uint16 + Version_compatible uint16 + Service_options [2]byte + Session_unit_size uint16 + Maxumum_trans_data_unit_size uint16 + Nt_protocol_characteristics [2]byte + Line_turnaround_value uint16 + Value_of_1_in_hardware uint16 + Length_of_connect_data uint16 + Offset_to_connect_data uint16 + Maximum_receivable_connect_data uint32 + Connect_flag0 byte + Connect_flag1 byte + Trace_cross_facility_item_1 uint32 + Trace_cross_facility_item_2 uint32 + Trace_unique_connection_id uint64 + Unknown_data [20]byte + //Connect_data []byte +} + +type body_refuse struct { + Reason_user byte + Reason_system byte + Data_len uint16 + //Data []byte +} diff --git a/matcher/oracle/oracle_test.go b/matcher/oracle/oracle_test.go new file mode 100644 index 0000000..dbef20e --- /dev/null +++ b/matcher/oracle/oracle_test.go @@ -0,0 +1,53 @@ +package oracle + +// +//import ( +// "fmt" +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "loafle.com/overflow/collector/discovery/scan/matcher/scaninfo" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestOracle(t *testing.T) { +// +// lm := NewOracleMatcher() +// +// port := types.NewPort("1521", types.NewHost("192.168.1.30"), types.TYPE_TCP) +// scanInfo := scaninfo.NewServiceScanInfo(port) +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// fmt.Println(ipport) +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// fmt.Println(pack) +// +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 1024) +// +// read, _ := client.Read(bytes) +// +// fmt.Println(bytes) +// +// b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) +// +// if b { +// t.Log("Good") +// } +// +// } +// +// t.Log(scanInfo) +// +//} diff --git a/matcher/packet/packet.go b/matcher/packet/packet.go new file mode 100644 index 0000000..9c7c1af --- /dev/null +++ b/matcher/packet/packet.go @@ -0,0 +1,13 @@ +package packet + +type Packet struct { + Buffer []byte + Len int +} + +func NewPacket(buf []byte, len int) *Packet { + return &Packet{ + Buffer: buf, + Len: len, + } +} diff --git a/matcher/pgsql/pgsql.go b/matcher/pgsql/pgsql.go new file mode 100644 index 0000000..c15875e --- /dev/null +++ b/matcher/pgsql/pgsql.go @@ -0,0 +1,182 @@ +package pgsql + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +const ( + RESPONSE_TYPE_ERR uint8 = 0x45 +) + +type pgsql struct { + Len uint32 + MessageType uint16 + _ uint16 + Name [5]byte + NameValue byte + Db [9]byte + DBValue byte + Encoding [16]byte + EncodingValue [5]byte + DateStyle [10]byte + DateStyleValue [4]byte + TimeZone [9]byte + TimeZoneValue [11]byte + ExtraDigits [19]byte + ExtraDigitsValue uint16 + End byte +} + +type pgsqlErrResponse struct { + ResponseType uint8 + Len [4]byte + Severity [6]byte + _ byte + Code [6]byte + _ byte + Message [53]byte +} + +type pgsqlErrResponse2 struct { + ResponseType uint8 + Len [4]byte + Data [128]byte + //Severity [6]byte + //_ byte + //Code [6]byte + //_ byte + //Message [53]byte +} + +type PostgreSQLMatcher struct { + packets []*packet.Packet +} + +func NewPostgreSQLMatcher() *PostgreSQLMatcher { + + pgSqlMatcher := &PostgreSQLMatcher{} + + pg := pgsql{} + + pg.Len = 0x00000065 + pg.MessageType = 0x0003 + + var name [5]byte + copy(name[:], "user") + pg.Name = name + pg.NameValue = 0x00 + + var db [9]byte + copy(db[:], "database") + pg.Db = db + pg.DBValue = 0x00 + + var encoding [16]byte + copy(encoding[:], "client_encoding") + pg.Encoding = encoding + + var encodingValue [5]byte + copy(encodingValue[:], "UTF8") + pg.EncodingValue = encodingValue + + var dateStyle [10]byte + copy(dateStyle[:], "DateStyle") + pg.DateStyle = dateStyle + + var dateStyleValue [4]byte + copy(dateStyleValue[:], "ISO") + pg.DateStyleValue = dateStyleValue + + var timeZone [9]byte + copy(timeZone[:], "TimeZone") + pg.TimeZone = timeZone + + var timeZoneValue [11]byte + copy(timeZoneValue[:], "Asia/Seoul") + pg.TimeZoneValue = timeZoneValue + + var extraDigit [19]byte + copy(extraDigit[:], "extra_float_digits") + pg.ExtraDigits = extraDigit + + pg.ExtraDigitsValue = 0x3200 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.BigEndian, pg) + + pgSqlMatcher.packets = append(pgSqlMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return pgSqlMatcher +} + +func (t *PostgreSQLMatcher) ServiceName() string { + return "PostgreSQL" +} + +func (t *PostgreSQLMatcher) PacketCount() int { + return len(t.packets) +} +func (t *PostgreSQLMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *PostgreSQLMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *PostgreSQLMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *PostgreSQLMatcher) IsPrePacket() bool { + return false +} + +func (t *PostgreSQLMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + pg := pgsqlErrResponse2{} + if err := binary.Read(reader, binary.BigEndian, &pg); err != nil { + return false + } + + if pg.ResponseType != RESPONSE_TYPE_ERR { + return false + } + + length := binary.BigEndian.Uint32(pg.Len[:]) + if length+1 != uint32(packet.Len) { + return false + } + + data := string(pg.Data[:]) + splits := strings.Split(data, "\x00") + + var findSeverity bool = false + var findErrorCode bool = false + for _, s := range splits { + if strings.Contains(s, "FATAL") { + findSeverity = true + } + if strings.Contains(s, "28000") { + findErrorCode = true + } + } + + if !findSeverity || !findErrorCode { + return false + } + + return true + +} diff --git a/matcher/pgsql/pgsql_test.go b/matcher/pgsql/pgsql_test.go new file mode 100644 index 0000000..2736687 --- /dev/null +++ b/matcher/pgsql/pgsql_test.go @@ -0,0 +1,70 @@ +package pgsql + +import ( + "crypto/tls" + "git.loafle.net/overflow/commons_go/matcher/packet" + "net" + "testing" +) + +func TestPG(t *testing.T) { + m := NewPostgreSQLMatcher() + + conn, err := net.Dial("tcp", "192.168.1.106:5432") //107 + if err != nil { + t.Error(err) + return + } + defer conn.Close() + + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("PostgreSQL found") + return + } + + t.Error("PostgreSQL not found") + } +} + +func TestSqlTLS(t *testing.T) { + conn, err := tls.Dial( + "tcp", + "192.168.1.107:5432", + &tls.Config{ + InsecureSkipVerify: true, + ServerName: "192.168.1.107", + }, + ) + + if err != nil { + t.Error(err) + return + } + + defer conn.Close() + + m := NewPostgreSQLMatcher() + for i := 0; i < m.PacketCount(); i++ { + + pack := m.Packet(i) + conn.Write(pack.Buffer) + bytes := make([]byte, 1024) + n, _ := conn.Read(bytes) + p := packet.NewPacket(bytes, n) + + if m.Match(i, p, nil) { + t.Log("PostgreSQL found") + return + } + + t.Error("PostgreSQL not found") + } +} diff --git a/matcher/pop/pop.go b/matcher/pop/pop.go new file mode 100644 index 0000000..e0e58c5 --- /dev/null +++ b/matcher/pop/pop.go @@ -0,0 +1,73 @@ +package pop + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + COMPARE_STR = "+OK" +) + +type POPMatcher struct { + sendPackets []*packet.Packet +} + +func (p *POPMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + switch index { + case 0: + fallthrough + case 1: + + recvStr := string(packet.Buffer) + + if len(recvStr) < 3 { + return false + } + + compareStr := recvStr[0:3] + + if compareStr == COMPARE_STR { + return true + } + + } + + return false +} + +func (p *POPMatcher) PacketCount() int { + return len(p.sendPackets) +} +func (p *POPMatcher) Packet(index int) *packet.Packet { + return p.sendPackets[index] +} +func (p *POPMatcher) ServiceName() string { + return "POPMatcher" +} + +func (p *POPMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (p *POPMatcher) IsNoResponse(index int) bool { + + return false +} + +func (p *POPMatcher) IsPrePacket() bool { + return true +} + +func NewPOPMatcher() *POPMatcher { + + pm := POPMatcher{} + + reqStr := "QUIT\r\n" + byte := make([]byte, len(reqStr)) + copy(byte[:], reqStr) + pm.sendPackets = append(pm.sendPackets, packet.NewPacket(byte, len(reqStr))) + + return &pm + +} diff --git a/matcher/pop/pop_test.go b/matcher/pop/pop_test.go new file mode 100644 index 0000000..721fcbc --- /dev/null +++ b/matcher/pop/pop_test.go @@ -0,0 +1,86 @@ +package pop + +// +//import ( +// "crypto/tls" +// "fmt" +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "loafle.com/overflow/collector/discovery/scan/matcher/scaninfo" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestPopTLS(t *testing.T) { +// conn, _ := tls.Dial( +// "tcp", +// "192.168.1.215:995", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.215", +// }, +// ) +// +// defer conn.Close() +// +// pop3Run(conn, t) +//} +// +//func TestPopNor(t *testing.T) { +// +// client, _ := net.Dial("tcp", "192.168.1.215:110") +// +// defer client.Close() +// +// pop3Run(client, t) +// +//} +// +//func pop3Run(client net.Conn, t *testing.T) { +// +// lm := NewPOPMatcher() +// +// port := types.NewPort("110", types.NewHost("192.168.1.215"), types.TYPE_TCP) +// scanInfo := scaninfo.NewServiceScanInfo(port) +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// fmt.Println(ipport) +// +// bytett := make([]byte, 1024) +// +// read, _ := client.Read(bytett) +// +// bb := lm.Match(0, packet.NewPacket(bytett, read), scanInfo) +// +// if bb { +// t.Log("good!") +// } +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// //fmt.Println(pack) +// +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 1024) +// +// rr, _ := client.Read(bytes) +// +// //fmt.Println(bytes) +// +// b := lm.Match(ii+1, packet.NewPacket(bytes, rr), scanInfo) +// +// if b { +// t.Log("send Good!") +// } +// +// } +// +// t.Log(scanInfo) +// +//} diff --git a/matcher/redis/redis.go b/matcher/redis/redis.go new file mode 100644 index 0000000..9720e94 --- /dev/null +++ b/matcher/redis/redis.go @@ -0,0 +1,76 @@ +package redis + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +const REDIS_PING string = "*1\r\n$4\r\nping\r\n" + +type RedisMatcher struct { + packets []*packet.Packet +} + +func NewRedisMatcher() *RedisMatcher { + + redisMatcher := &RedisMatcher{} + + redisMatcher.packets = append(redisMatcher.packets, packet.NewPacket([]byte(REDIS_PING), len(REDIS_PING))) + + return redisMatcher +} + +func (t *RedisMatcher) ServiceName() string { + return "Redis" +} + +func (t *RedisMatcher) PacketCount() int { + return len(t.packets) +} +func (t *RedisMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *RedisMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *RedisMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *RedisMatcher) IsPrePacket() bool { + return false +} + +func (t *RedisMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + resp := strings.Split(string(packet.Buffer), "\r\n")[0] + if len(resp) <= 0 { + return false + } + + sign := string([]rune(resp)[0]) + if len(sign) <= 0 { + return false + } + + if sign == "+" { + if resp == "+PONG" || resp == "+OK" { + return true + } + } + if sign == "-" { + if resp == "-NOAUTH" || resp == "-ERR" { + return true + } + } + + return false + +} diff --git a/matcher/redis/redisProtected.go b/matcher/redis/redisProtected.go new file mode 100644 index 0000000..d0e3dba --- /dev/null +++ b/matcher/redis/redisProtected.go @@ -0,0 +1,78 @@ +package redis + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + COMPARE_STR_1 = "-" + COMPARE_STR_2 = "DENIED" +) + +type RedisProtectedMatcher struct { +} + +func NewRedisProtectedMatcher() *RedisProtectedMatcher { + + redisMatcher := &RedisProtectedMatcher{} + + return redisMatcher +} + +func (r *RedisProtectedMatcher) ServiceName() string { + return "RedisProtectedMatcher" +} + +func (r *RedisProtectedMatcher) PacketCount() int { + return 0 +} +func (r *RedisProtectedMatcher) Packet(index int) *packet.Packet { + return nil +} + +func (r *RedisProtectedMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (r *RedisProtectedMatcher) IsNoResponse(index int) bool { + return false +} + +func (r *RedisProtectedMatcher) IsPrePacket() bool { + return true +} + +func (r *RedisProtectedMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + switch index { + case 0: + str := string(packet.Buffer[:packet.Len]) + + if str == "" { + return false + } + if len(str) <= 0 { + return false + } + + firstCompare := str[0:1] + seconcdCompare := str[1 : len(COMPARE_STR_2)+1] + + if firstCompare != COMPARE_STR_1 { + return false + } + if seconcdCompare != COMPARE_STR_2 { + return false + } + + return true + } + + return false + +} diff --git a/matcher/redis/redisProtected_test.go b/matcher/redis/redisProtected_test.go new file mode 100644 index 0000000..2dfc0b0 --- /dev/null +++ b/matcher/redis/redisProtected_test.go @@ -0,0 +1,33 @@ +package redis + +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestRedisProtected(t *testing.T) { +// +// m := NewRedisProtectedMatcher() +// +// conn, err := net.Dial("tcp", "192.168.1.215:8379") +// +// if err != nil { +// t.Log(err) +// return +// } +// +// defer conn.Close() +// +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// +// //fmt.Println(string(bytes[:n])) +// +// b := m.Match(0, packet.NewPacket(bytes, n), nil) +// +// if b { +// t.Log("good!") +// } +// +//} diff --git a/matcher/redis/redis_test.go b/matcher/redis/redis_test.go new file mode 100644 index 0000000..bc82304 --- /dev/null +++ b/matcher/redis/redis_test.go @@ -0,0 +1,38 @@ +package redis + +// +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//const ( +// ADDR string = "192.168.1.215:6379" +//) +// +//func TestRedisMatcher(t *testing.T) { +// +// m := NewRedisMatcher() +// +// conn, _ := net.Dial("tcp", ADDR) +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("Redis found.") +// return +// } +// +// t.Error("Redis not found") +// } +// +//} diff --git a/matcher/rmi/rmi.go b/matcher/rmi/rmi.go new file mode 100644 index 0000000..3dd0d28 --- /dev/null +++ b/matcher/rmi/rmi.go @@ -0,0 +1,111 @@ +package rmi + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + MAGIC_NUMBER = 0x4a524d49 + STREAM_PROTOCOL = 0x4b + VERSION = 0x0002 + ACK_PROTOCOL = 0x4e +) + +type RMI_SEND_MESSAGE struct { + magic uint32 + version uint16 + protocol uint8 +} + +type RMI_RECV_MESSAGE struct { + streamMessage uint8 + packetLen uint16 + host []byte + port [2]byte +} +type RMIMatcher struct { + sendPackets []*packet.Packet +} + +func (r *RMIMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + //fmt.Println("packet :", packet) + rmiRecv := RMI_RECV_MESSAGE{} + + buf := bytes.NewReader(packet.Buffer) + binary.Read(buf, binary.BigEndian, &rmiRecv.streamMessage) + binary.Read(buf, binary.BigEndian, &rmiRecv.packetLen) + + lenInt := int(rmiRecv.packetLen) + + var tempHost = make([]byte, lenInt, lenInt) + copy(rmiRecv.host, tempHost) + + rmiRecv.host = tempHost + + binary.Read(buf, binary.BigEndian, &rmiRecv.host) + binary.Read(buf, binary.BigEndian, &rmiRecv.port) + + hostIp := string(rmiRecv.host[:lenInt]) + //fmt.Println(hostIp) + + //hostPort := binary.BigEndian.Uint16(rmiRecv.port[:2]) + + if rmiRecv.streamMessage == ACK_PROTOCOL && lenInt == len(hostIp) { + result = true + } + + return result +} + +func (r *RMIMatcher) PacketCount() int { + return len(r.sendPackets) +} + +func (r *RMIMatcher) Packet(index int) *packet.Packet { + return r.sendPackets[index] +} + +func (r *RMIMatcher) ServiceName() string { + return "RMI" +} + +func (r *RMIMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (r *RMIMatcher) IsNoResponse(index int) bool { + return false +} + +func (r *RMIMatcher) IsPrePacket() bool { + return false +} + +func NewRMIMatcher() *RMIMatcher { + + r := RMIMatcher{} + rsm := RMI_SEND_MESSAGE{ + magic: MAGIC_NUMBER, + version: VERSION, + protocol: STREAM_PROTOCOL, + } + + mCache := new(bytes.Buffer) + binary.Write(mCache, binary.BigEndian, rsm) + + sendByte1 := mCache.Bytes() + pp := packet.NewPacket(sendByte1, len(sendByte1)) + + r.sendPackets = append(r.sendPackets, pp) + + return &r +} diff --git a/matcher/rmi/rmi_test.go b/matcher/rmi/rmi_test.go new file mode 100644 index 0000000..7fe8f90 --- /dev/null +++ b/matcher/rmi/rmi_test.go @@ -0,0 +1,48 @@ +package rmi + +//import ( +// "fmt" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestNew(t *testing.T) { +// r := NewRMIMatcher() +// fmt.Println("TestNew: ", r) +//} +//func TestRMIMatcher_Match(t *testing.T) { +// +// fmt.Println("Match") +// +// hm := NewRMIMatcher() +// +// port := types.NewPort("9840", types.NewHost("192.168.1.101"), types.TYPE_TCP) +// +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// //fmt.Println(ipport) +// +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// pack := hm.Packet(0) +// +// fmt.Println(pack.Buffer) +// +// //writer.WriteString(pack) +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 512) +// +// l, _ := client.Read(bytes) +// +// //fmt.Println(bytes) +// +// t1 := hm.Match(0, packet.NewPacket(bytes, l), nil) +// +// fmt.Println(t1) +//} diff --git a/matcher/smb/smb.go b/matcher/smb/smb.go new file mode 100644 index 0000000..df41796 --- /dev/null +++ b/matcher/smb/smb.go @@ -0,0 +1,161 @@ +package smb + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +const ( + SMB_COM_NEGOTIATE uint8 = 0x72 + SMB_SUCCESS uint8 = 0x00 +) + +type netBIOS struct { + MsgType byte + MsgLength [3]uint8 +} + +type smb struct { + NetBios netBIOS + Component [4]uint8 + SmbCommand uint8 + NtStatus [4]uint8 + Flags uint8 + Flags2 [2]uint8 + ProcessId uint16 + Signature uint64 + Reserved uint16 + Tid uint16 + Pid uint16 + Uid uint16 + Mid uint16 + Wct uint8 + Bcc uint16 + Bf1 uint8 + Name1 [23]uint8 + Bf2 uint8 + Name2 [10]uint8 + Bf3 uint8 + Name3 [28]uint8 + Bf4 uint8 + Name4 [10]uint8 + Bf5 uint8 + Name5 [10]uint8 + Bf6 uint8 + Name6 [11]uint8 +} + +type SMBMatcher struct { + packets []*packet.Packet +} + +func NewSMBMatcher() *SMBMatcher { + + nbssMatcher := &SMBMatcher{} + + query := smb{} + query.NetBios.MsgType = 0x00 + query.NetBios.MsgLength[2] = 0x85 + + query.Component[0] = 0xff + query.Component[1] = 'S' + query.Component[2] = 'M' + query.Component[3] = 'B' + + query.SmbCommand = SMB_COM_NEGOTIATE + query.NtStatus[3] = SMB_SUCCESS + query.Flags = 0x18 + query.Flags2[0] = 0x53 + query.Flags2[1] = 0xC8 + + query.ProcessId = 0x00 + query.Signature = 0x00 + query.Reserved = 0 + query.Tid = 0 + query.Pid = 0xfeff + query.Uid = 0 + query.Mid = 0 + query.Wct = 0 + query.Bcc = 0x0062 + + query.Bf1 = 0x02 + copy(query.Name1[:], "PC NETWORK PROGRAM 1.0") + + query.Bf2 = 0x02 + copy(query.Name2[:], "LANMAN1.0") + + query.Bf3 = 0x02 + copy(query.Name3[:], "Windows for Workgroups 3.1a") + + query.Bf4 = 0x02 + copy(query.Name4[:], "LM1.2X002") + + query.Bf5 = 0x02 + copy(query.Name5[:], "LANMAN2.1") + + query.Bf6 = 0x02 + copy(query.Name6[:], "NT LM 0.12") + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, query) + + nbssMatcher.packets = append(nbssMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return nbssMatcher +} + +func (t *SMBMatcher) ServiceName() string { + return "SMB" +} + +func (t *SMBMatcher) PacketCount() int { + return len(t.packets) +} +func (t *SMBMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SMBMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *SMBMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *SMBMatcher) IsPrePacket() bool { + return false +} + +func (t *SMBMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + s := smb{} + if err := binary.Read(reader, binary.BigEndian, &s); err != nil { + return false + } + + var des [4]byte + copy(des[1:], s.NetBios.MsgLength[:]) + packetLen := binary.BigEndian.Uint32(des[:]) + + if packetLen != uint32(packet.Len-4) { + return false + } + + if !strings.Contains(string(s.Component[:]), "SMB") { + return false + } + + return true + +} diff --git a/matcher/smb/smb_test.go b/matcher/smb/smb_test.go new file mode 100644 index 0000000..b534bc1 --- /dev/null +++ b/matcher/smb/smb_test.go @@ -0,0 +1,37 @@ +package smb + +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//const ( +// ADDR string = "192.168.1.104:139" +//) +// +//func TestSMBMatcher(t *testing.T) { +// +// m := NewSMBMatcher() +// +// conn, _ := net.Dial("tcp", ADDR) +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("SMB found.") +// return +// } +// +// t.Error("SMB not found") +// } +// +//} diff --git a/matcher/smtp/smtp.go b/matcher/smtp/smtp.go new file mode 100644 index 0000000..133ef45 --- /dev/null +++ b/matcher/smtp/smtp.go @@ -0,0 +1,71 @@ +package smtp + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +type SmtpMatcher struct { + packets []*packet.Packet +} + +func NewSmtpMatcher() *SmtpMatcher { + + m := &SmtpMatcher{} + b := []byte("helo test\r\n") + m.packets = append(m.packets, packet.NewPacket(b, len(b))) + b = []byte("quit\r\n") + m.packets = append(m.packets, packet.NewPacket(b, len(b))) + return m +} + +func (t *SmtpMatcher) ServiceName() string { + return "SMTP" +} + +func (t *SmtpMatcher) PacketCount() int { + return len(t.packets) +} +func (t *SmtpMatcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SmtpMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *SmtpMatcher) IsNoResponse(index int) bool { + return false +} + +func (t *SmtpMatcher) IsPrePacket() bool { + return true +} + +func (t *SmtpMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + if packet == nil { + return false + } + buf := string(packet.Buffer) + if len(buf) == 0 || len(buf) < 5 { + return false + } + + splits := strings.Split(buf, "\r\n") + splits = strings.Split(buf, " ") + if index == 0 { + if splits[0] == "220" { + return true + } + } else if index == 1 { + if splits[0] == "250" { + return true + } + } else if index == 2 { + if splits[0] == "221" { + return true + } + } + return false +} diff --git a/matcher/smtp/smtp_test.go b/matcher/smtp/smtp_test.go new file mode 100644 index 0000000..b2cfc08 --- /dev/null +++ b/matcher/smtp/smtp_test.go @@ -0,0 +1,68 @@ +package smtp + +// +//import ( +// "crypto/tls" +// "fmt" +// "github.com/stretchr/testify/assert" +// "net" +// "strings" +// "testing" +//) +// +//func TestSMTPTLS(t *testing.T) { +// +// conn, err := tls.Dial("tcp", +// "192.168.1.215:465", +// &tls.Config{ +// InsecureSkipVerify: true, +// ServerName: "192.168.1.215", +// }, +// ) +// +// if err != nil { +// t.Log(err) +// return +// } +// +// b := make([]byte, 1024) +// +// check(t, b, conn, "", "220") +// check(t, b, conn, "helo test\r\n", "250") +// check(t, b, conn, "quit\r\n", "221") +// +// conn.Close() +//} +// +//func TestSMTP(t *testing.T) { +// +// conn, _ := net.Dial("tcp", "192.168.1.215:25") +// +// b := make([]byte, 1024) +// +// check(t, b, conn, "", "220") +// check(t, b, conn, "helo test\r\n", "250") +// check(t, b, conn, "quit\r\n", "221") +// +// conn.Close() +// +//} +//func check(t *testing.T, b []byte, conn net.Conn, cmd string, compare string) { +// +// if cmd != "" { +// wlen, _ := conn.Write([]byte(cmd)) +// assert.Equal(t, wlen, len(cmd)) +// } +// +// rlen, _ := conn.Read(b) +// +// fmt.Println(rlen) +// fmt.Println(len(b)) +// +// data := string(b[:rlen]) +// fmt.Println(data) +// assert.Equal(t, true, rlen > 4) +// splits := strings.Split(data, " ") +// assert.Equal(t, compare, splits[0]) +// +//} diff --git a/matcher/snmp/snmpv2.go b/matcher/snmp/snmpv2.go new file mode 100644 index 0000000..2c288c3 --- /dev/null +++ b/matcher/snmp/snmpv2.go @@ -0,0 +1,200 @@ +package snmp + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + SNMP_START_SEQUENCE uint8 = 0X30 + SNMP_TYPE_INTEGER uint8 = 0X02 + SNMP_TYPE_STRING uint8 = 0X04 + SNMP_TYPE_NULL uint8 = 0X05 + SNMP_TYPE_OBJECT uint8 = 0X06 + SNMP_GET_REQUEST uint8 = 0XA0 + //SNMP_RESPONSE uint8 = 0XA2 + SNMP_NO_DESC uint16 = 0X0004 + SNMP_END_SEQUENCE uint8 = 0X30 + SNMP_PROTOCOL_VERSION_3 uint8 = 0X03 + SNMP_PROTOCOL_VERSION_2c uint8 = 0X01 + SNMP_MSG_ID_MAX_VALUE uint32 = 0xFFFFFF7F +) + +type snmpv2VarBinding struct { + VarBindindStart uint8 + VarBindLen uint8 + ObjectStart uint8 + ObjectLen uint8 + ValueType uint8 + ValueLen uint8 + ObjectValue uint64 + NullValue uint8 + EndIndicator uint8 +} + +type snmpv2Data struct { + DataType uint8 + DataLen uint8 + RequestIdType uint8 + RequestIdLen uint8 + RequestId uint8 + ErrorStatusType uint8 + ErrorStatusLen uint8 + ErrorStatus uint8 + ErrorIndexType uint8 + ErrorIndexLen uint8 + ErrorIndex uint8 + + VarBinding snmpv2VarBinding +} + +type snmpv2 struct { + StartSeq uint8 + SeqLen uint8 + SNMPVersionType uint8 + SNMPVersionLen uint8 + SNMPVersion uint8 + CommunityVersionType uint8 + CommunityVersionLen uint8 + Community [6]byte + + Data snmpv2Data +} + +type SNMPv2Matcher struct { + packets []*packet.Packet +} + +func NewSNMPv2Matcher() *SNMPv2Matcher { + + snmpMatcher := &SNMPv2Matcher{} + + snmpTempBuf := new(bytes.Buffer) + binary.Write(snmpTempBuf, binary.BigEndian, snmpv2{}) //For getting the struct size + + snmpDataTempBuf := new(bytes.Buffer) + binary.Write(snmpDataTempBuf, binary.BigEndian, snmpv2Data{}) //For getting the struct size + + snmpVarTempBuf := new(bytes.Buffer) + binary.Write(snmpVarTempBuf, binary.BigEndian, snmpv2VarBinding{}) //For getting the struct size + + q := snmpv2{} + q.StartSeq = SNMP_START_SEQUENCE + q.SeqLen = uint8(len(snmpTempBuf.Bytes())) - 2 + q.SNMPVersionType = SNMP_TYPE_INTEGER + q.SNMPVersionLen = 0x01 + q.SNMPVersion = SNMP_PROTOCOL_VERSION_2c + q.CommunityVersionType = SNMP_TYPE_STRING + q.CommunityVersionLen = 0x06 + var community [6]byte + copy(community[:], "public") + q.Community = community + q.Data.DataType = SNMP_GET_REQUEST + q.Data.DataLen = uint8(len(snmpDataTempBuf.Bytes())) - 2 + q.Data.RequestIdType = SNMP_TYPE_INTEGER + q.Data.RequestIdLen = 0x01 + q.Data.RequestId = 0x01 + q.Data.ErrorStatusType = SNMP_TYPE_INTEGER + q.Data.ErrorStatusLen = 0x01 + q.Data.ErrorStatus = 0x00 + q.Data.ErrorIndexType = SNMP_TYPE_INTEGER + q.Data.ErrorIndexLen = 0x01 + q.Data.ErrorIndex = 0x00 + q.Data.VarBinding.VarBindindStart = SNMP_START_SEQUENCE + q.Data.VarBinding.VarBindLen = uint8(len(snmpVarTempBuf.Bytes())) - 2 + q.Data.VarBinding.ObjectStart = SNMP_START_SEQUENCE + q.Data.VarBinding.ObjectLen = uint8(len(snmpVarTempBuf.Bytes())) - 4 + q.Data.VarBinding.ValueType = SNMP_TYPE_OBJECT + q.Data.VarBinding.ValueLen = 0x08 + q.Data.VarBinding.ObjectValue = 0x000001010201062b + q.Data.VarBinding.NullValue = SNMP_TYPE_NULL + q.Data.VarBinding.EndIndicator = 0x00 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, q) + + snmpMatcher.packets = append(snmpMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return snmpMatcher +} + +func (t *SNMPv2Matcher) ServiceName() string { + return "SNMP" +} + +func (t *SNMPv2Matcher) PacketCount() int { + return len(t.packets) +} +func (t *SNMPv2Matcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SNMPv2Matcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (t *SNMPv2Matcher) IsNoResponse(index int) bool { + return false +} + +func (t *SNMPv2Matcher) IsPrePacket() bool { + return false +} + +func (t *SNMPv2Matcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + s := snmpv2{} + if err := binary.Read(r, binary.BigEndian, &s); err != nil { + return false + } + + if s.StartSeq != SNMP_START_SEQUENCE { + return false + } + + var p uint8 + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + for idx := 0; idx < 5; idx++ { + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + if p == SNMP_TYPE_INTEGER { + break + } + p++ + } + + //finding protocol version type : 0x02 0x01 0x01 + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + + if p == 0x01 { + if err := binary.Read(reader, binary.BigEndian, &p); err != nil { + return false + } + if p == 0x01 { + return true + } + } + + return false + +} + +func (t *SNMPv2Matcher) IsSend(port int) bool { + if port == 161 { + return true + } + return false +} diff --git a/matcher/snmp/snmpv2_test.go b/matcher/snmp/snmpv2_test.go new file mode 100644 index 0000000..ceb0dcb --- /dev/null +++ b/matcher/snmp/snmpv2_test.go @@ -0,0 +1,37 @@ +package snmp + +// +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestSNMP2(t *testing.T) { +// +// m := NewSNMPv2Matcher() +// +// conn, err := net.Dial("udp", "192.168.1.215:161") +// if err != nil { +// t.Error(err) +// return +// } +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("SNMP found") +// return +// } +// +// t.Error("SNMP not found") +// } +// +//} diff --git a/matcher/snmp/snmpv3.go b/matcher/snmp/snmpv3.go new file mode 100644 index 0000000..68e312e --- /dev/null +++ b/matcher/snmp/snmpv3.go @@ -0,0 +1,218 @@ +package snmp + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +type snmpv3GlobalData struct { + GlobalDataStartSeq uint8 + GlobalDataLen uint8 + MsgIdType uint8 + MsgIdLen uint8 + MsgId uint32 + MsgMaxSizeType uint8 + MsgMaxSizeLen uint8 + MsgMaxSize [3]uint8 + MsgFlagsType uint8 + MsgFlagsTypeLen uint8 + MsgFlags uint8 + MsgSecurityModelType uint8 + MsgSecurityModelLen uint8 + MsgSecurityModel uint8 +} + +type snmpv3MsgData struct { + MsgDataStartSeq uint8 + MsgDataLen uint8 + ContextEngineId uint16 + ContextEngineName uint16 + SnmpType uint8 + Len uint8 + RequestIdType uint8 + RequestIdLen uint8 + RequestId uint32 + ErrorStatusType uint8 + ErrorStatusLen uint8 + ErrorStatus uint8 + ErrorIndexType uint8 + ErrorIndexLen uint8 + ErrorIndex uint8 + EndSeq uint8 + EndIndicator uint8 +} + +type snmpv3 struct { + StartSeq uint8 + SeqLen uint8 + SNMPVersionType uint8 + SNMPVersionLen uint8 + SNMPVersion uint8 + MsgGlobalData snmpv3GlobalData + Unk1 uint16 + Unk2 uint16 + MsgAuthoritativeEngineId uint16 + + MsgAuthoritativeEngineBootsType uint8 + MsgAuthoritativeEngineBootsLen uint8 + MsgAuthoritativeEngineBoots uint8 + MsgAuthoritativeEngineTimeType uint8 + MsgAuthoritativeEngineTimeLen uint8 + MsgAuthoritativeEngineTime uint8 + MsgUserName uint16 + MsgAuthenticationParam uint16 + MsgPrivacyParam uint16 + MsgData snmpv3MsgData +} + +type SNMPv3Matcher struct { + packets []*packet.Packet +} + +func NewSNMPv3Matcher() *SNMPv3Matcher { + + snmpMatcher := &SNMPv3Matcher{} + + snmpTempBuf := new(bytes.Buffer) + binary.Write(snmpTempBuf, binary.BigEndian, snmpv3{}) //For getting the struct size + + snmpMsgDataTempBuf := new(bytes.Buffer) + binary.Write(snmpMsgDataTempBuf, binary.BigEndian, snmpv3MsgData{}) //For getting the struct size + + snmpGlobalTempBuf := new(bytes.Buffer) + binary.Write(snmpGlobalTempBuf, binary.BigEndian, snmpv3GlobalData{}) //For getting the struct size + + q := snmpv3{} + q.StartSeq = SNMP_START_SEQUENCE + q.SeqLen = uint8(len(snmpTempBuf.Bytes())) - 2 + q.SNMPVersionType = SNMP_TYPE_INTEGER + q.SNMPVersionLen = 0x01 + q.SNMPVersion = SNMP_PROTOCOL_VERSION_3 + q.MsgGlobalData.GlobalDataStartSeq = SNMP_START_SEQUENCE + q.MsgGlobalData.GlobalDataLen = uint8(len(snmpGlobalTempBuf.Bytes())) - 2 + q.MsgGlobalData.MsgIdType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgIdLen = 0x04 + q.MsgGlobalData.MsgId = SNMP_MSG_ID_MAX_VALUE + q.MsgGlobalData.MsgMaxSizeType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgMaxSizeLen = 0x03 + q.MsgGlobalData.MsgMaxSize[2] = 0xe3 + q.MsgGlobalData.MsgMaxSize[1] = 0xff + q.MsgGlobalData.MsgMaxSize[0] = 0x00 + q.MsgGlobalData.MsgFlagsType = SNMP_TYPE_STRING + q.MsgGlobalData.MsgFlagsTypeLen = 0x01 + q.MsgGlobalData.MsgFlags = 0x04 + q.MsgGlobalData.MsgSecurityModelType = SNMP_TYPE_INTEGER + q.MsgGlobalData.MsgSecurityModelLen = 0x01 + q.MsgGlobalData.MsgSecurityModel = 0x03 + q.Unk1 = 0x1004 + q.Unk2 = 0x0e30 + q.MsgAuthoritativeEngineId = SNMP_NO_DESC + q.MsgAuthoritativeEngineBootsType = SNMP_TYPE_INTEGER + q.MsgAuthoritativeEngineBootsLen = 0x01 + q.MsgAuthoritativeEngineBoots = 0x00 + q.MsgAuthoritativeEngineTimeType = SNMP_TYPE_INTEGER + q.MsgAuthoritativeEngineTimeLen = 0x01 + q.MsgAuthoritativeEngineTime = 0x00 + q.MsgUserName = SNMP_NO_DESC + q.MsgAuthenticationParam = SNMP_NO_DESC + q.MsgPrivacyParam = SNMP_NO_DESC + + q.MsgData.MsgDataStartSeq = SNMP_START_SEQUENCE + q.MsgData.MsgDataLen = uint8(len(snmpMsgDataTempBuf.Bytes())) - 2 + q.MsgData.ContextEngineId = SNMP_NO_DESC + q.MsgData.ContextEngineName = SNMP_NO_DESC + q.MsgData.SnmpType = SNMP_GET_REQUEST + q.MsgData.Len = 0x0E + q.MsgData.RequestIdType = SNMP_TYPE_INTEGER + q.MsgData.RequestIdLen = 0x04 + q.MsgData.RequestId = 0x00 // + q.MsgData.ErrorStatusType = SNMP_TYPE_INTEGER + q.MsgData.ErrorStatusLen = 0x01 + q.MsgData.ErrorStatus = 0x00 + q.MsgData.ErrorIndexType = SNMP_TYPE_INTEGER + q.MsgData.ErrorIndexLen = 0x01 + q.MsgData.ErrorIndex = 0x00 + q.MsgData.EndSeq = SNMP_END_SEQUENCE + q.MsgData.EndIndicator = 0x00 + + writer := new(bytes.Buffer) + binary.Write(writer, binary.LittleEndian, q) + + snmpMatcher.packets = append(snmpMatcher.packets, packet.NewPacket(writer.Bytes(), writer.Len())) + + return snmpMatcher +} + +func (t *SNMPv3Matcher) ServiceName() string { + return "SNMP" +} + +func (t *SNMPv3Matcher) PacketCount() int { + return len(t.packets) +} + +func (t *SNMPv3Matcher) Packet(index int) *packet.Packet { + return t.packets[index] +} + +func (t *SNMPv3Matcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (t *SNMPv3Matcher) IsNoResponse(index int) bool { + return false +} + +func (t *SNMPv3Matcher) IsPrePacket() bool { + return false +} + +func (t *SNMPv3Matcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + reader := new(bytes.Buffer) + reader.Write(packet.Buffer) + + s := snmpv3{} + if err := binary.Read(reader, binary.LittleEndian, &s); err != nil { + return false + } + + if s.StartSeq != SNMP_START_SEQUENCE { + return false + } + + var p uint8 + r := new(bytes.Buffer) + r.Write(packet.Buffer) + + for { + binary.Read(r, binary.LittleEndian, &p) + + if p == SNMP_TYPE_INTEGER { + break + } + } + + binary.Read(r, binary.BigEndian, &p) + if p == 0x01 { + binary.Read(r, binary.BigEndian, &p) + if p == 0x03 { + return true + } + } + + return false + +} + +func (t *SNMPv3Matcher) IsSend(port int) bool { + if port == 161 { + return true + } + return false +} diff --git a/matcher/snmp/snmpv3_test.go b/matcher/snmp/snmpv3_test.go new file mode 100644 index 0000000..e51a31f --- /dev/null +++ b/matcher/snmp/snmpv3_test.go @@ -0,0 +1,37 @@ +package snmp + +// +//import ( +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "net" +// "testing" +//) +// +//func TestSNMP3(t *testing.T) { +// +// m := NewSNMPv3Matcher() +// +// conn, err := net.Dial("udp", "192.168.1.254:161") +// if err != nil { +// t.Error(err) +// return +// } +// defer conn.Close() +// +// for i := 0; i < m.PacketCount(); i++ { +// +// pack := m.Packet(i) +// conn.Write(pack.Buffer) +// bytes := make([]byte, 1024) +// n, _ := conn.Read(bytes) +// p := packet.NewPacket(bytes, n) +// +// if m.Match(i, p, nil) { +// t.Log("SNMP found") +// return +// } +// +// t.Error("SNMP not found") +// } +// +//} diff --git a/matcher/ssh/ssh.go b/matcher/ssh/ssh.go new file mode 100644 index 0000000..57d9c44 --- /dev/null +++ b/matcher/ssh/ssh.go @@ -0,0 +1,66 @@ +package ssh + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" + "strings" +) + +type SSHMatcher struct { + sendPackets []*packet.Packet +} + +func (ssh *SSHMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + str := string(packet.Buffer) + + //fmt.Println(str) + temps := strings.Split(str, " ") + protocol := strings.Split(temps[0], "-") + //osType := temps[1] + + if 0 == strings.Compare(protocol[0], "SSH") { + majorVersion := protocol[1] + //fmt.Println(majorVersion) + if 0 == strings.Compare(majorVersion, "2.0") || 0 == strings.Compare(majorVersion, "1.0") { + result = true + } + } + + return result +} + +func (ssh *SSHMatcher) PacketCount() int { + return len(ssh.sendPackets) +} + +func (ssh *SSHMatcher) Packet(index int) *packet.Packet { + return ssh.sendPackets[index] +} + +func (ssh *SSHMatcher) ServiceName() string { + return "SSH" +} + +func (ssh *SSHMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (ssh *SSHMatcher) IsNoResponse(index int) bool { + return false +} + +func (ssh *SSHMatcher) IsPrePacket() bool { + return true +} + +func NewSSHMatcher() *SSHMatcher { + r := SSHMatcher{} + + return &r +} diff --git a/matcher/ssh/ssh_test.go b/matcher/ssh/ssh_test.go new file mode 100644 index 0000000..17bf802 --- /dev/null +++ b/matcher/ssh/ssh_test.go @@ -0,0 +1,33 @@ +package ssh + +//import ( +// "fmt" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestSSHMatcher_Match(t *testing.T) { +// +// port := types.NewPort("22", types.NewHost("192.168.1.103"), types.TYPE_TCP) +// ssh := NewSSHMatcher() +// +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// bytes := make([]byte, 512) +// +// l, _ := client.Read(bytes) +// +// fmt.Println(bytes) +// +// b := ssh.Match(0, packet.NewPacket(bytes, l), nil) +// +// fmt.Println(b) +// +//} diff --git a/matcher/telnet/telnet.go b/matcher/telnet/telnet.go new file mode 100644 index 0000000..2299ffe --- /dev/null +++ b/matcher/telnet/telnet.go @@ -0,0 +1,79 @@ +package telnet + +import ( + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + DO = 0xfd + WONT = 0x4b + WILL = 0xfb + DONT = 0xfe + CMD = 0xff +) + +type TelnetMatcher struct { + sendPackets []*packet.Packet +} + +func (tel *TelnetMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + result := false + + if packet == nil || packet.Buffer == nil || packet.Len == 0 { + return result + } + + buf := make([]byte, 0, 0) + count := 0 + + for i := 0; i < len(packet.Buffer); i++ { + if packet.Buffer[i] > 0 { + buf = append(buf, packet.Buffer[i]) + } else if count > 2 { + break + } else { + count++ + } + } + + for idx := 0; idx < len(buf); idx += 3 { + if buf[idx] == CMD && (buf[idx+1] == DO || buf[idx+1] == WONT || buf[idx+1] == WILL || buf[idx+1] == DONT) { + result = true + } else { + result = false + } + } + + return result +} + +func (tel *TelnetMatcher) PacketCount() int { + return len(tel.sendPackets) +} + +func (tel *TelnetMatcher) Packet(index int) *packet.Packet { + return tel.sendPackets[index] +} + +func (tel *TelnetMatcher) ServiceName() string { + return "Telnet" +} + +func (tel *TelnetMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} + +func (tel *TelnetMatcher) IsNoResponse(index int) bool { + return false +} + +func (tel *TelnetMatcher) IsPrePacket() bool { + return true +} + +func NewTelnetMatcher() *TelnetMatcher { + r := TelnetMatcher{} + + return &r +} diff --git a/matcher/telnet/telnet_test.go b/matcher/telnet/telnet_test.go new file mode 100644 index 0000000..445df95 --- /dev/null +++ b/matcher/telnet/telnet_test.go @@ -0,0 +1,33 @@ +package telnet + +//import ( +// "fmt" +// "loafle.com/overflow/collector/core/scan/service/matcher/packet" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestTelnetMatcher_Match(t *testing.T) { +// +// port := types.NewPort("23", types.NewHost("192.168.1.210"), types.TYPE_TCP) +// telnet := NewTelnetMatcher() +// +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// bytes := make([]byte, 512) +// +// l, _ := client.Read(bytes) +// +// fmt.Println("length :", l) +// fmt.Println(bytes) +// +// b := telnet.Match(0, packet.NewPacket(bytes, l), nil) +// +// fmt.Println(b) +//} diff --git a/matcher/wmi/wmi.go b/matcher/wmi/wmi.go new file mode 100644 index 0000000..505003c --- /dev/null +++ b/matcher/wmi/wmi.go @@ -0,0 +1,250 @@ +package wmi + +import ( + "bytes" + "encoding/binary" + "git.loafle.net/overflow/commons_go/matcher/packet" + "git.loafle.net/overflow/commons_go/model/scaninfo" +) + +const ( + PDU_BIND = 11 + PDU_BIND_ACK = 12 + PDU_REQ = 0 + PDU_RESP = 2 + + WMI_CALL_ID_1 = 0x95 + WMI_CALL_ID_2 = 0x96 +) + +type WMIMatcher struct { + sendPackets []*packet.Packet +} + +func (w *WMIMatcher) Match(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + + if packet == nil { + return false + } + + buf := new(bytes.Buffer) + buf.Write(packet.Buffer) + + wmiRecv := DCERPC_DEFAULT{} + + binary.Read(buf, binary.LittleEndian, &wmiRecv) + + switch index { + case 0: + if wmiRecv.Call_id != WMI_CALL_ID_1 { + return false + } + + if wmiRecv.Ptype != PDU_BIND_ACK { + return false + } + + return true + case 1: + + if wmiRecv.Call_id != WMI_CALL_ID_2 { + return false + } + + if wmiRecv.Ptype != PDU_RESP { + return false + } + + return true + } + + return false +} + +func (w *WMIMatcher) PacketCount() int { + return len(w.sendPackets) +} +func (w *WMIMatcher) Packet(index int) *packet.Packet { + return w.sendPackets[index] +} +func (w *WMIMatcher) ServiceName() string { + return "WMI" +} + +func (w *WMIMatcher) IsError(index int, packet *packet.Packet, info scaninfo.ServiceScanInfo) bool { + return false +} +func (w *WMIMatcher) IsNoResponse(index int) bool { + + return false +} + +func (w *WMIMatcher) IsPrePacket() bool { + return false +} + +func NewWMIMatcher() *WMIMatcher { + + wm := WMIMatcher{} + + ds1 := DCERPC_DEFAULT{ + Rpc_ver: 5, + Rpc_ver_minor: 0, + Ptype: PDU_BIND, + Flags: 0x03, + Drep: 0x10, + Frag_len: 16 + 56, + Auth_len: 0, + Call_id: WMI_CALL_ID_1, + } + + ds2 := DCERPC_DEFAULT{ + Rpc_ver: 5, + Rpc_ver_minor: 0, + Ptype: PDU_REQ, + Flags: 0x03, + Drep: 0x10, + Frag_len: 16 + 8, + Auth_len: 0, + Call_id: WMI_CALL_ID_2, + } + + ioxidr := DCERPC_IOXIDResolver{ + MaxXmitFrag: 0x16d0, + MaxRecvFrag: 0x16d0, + + AssocGroup: 0, + NumCtxItem: 1, + + ContextId: 0, + NumTransItem: 1, + + //interfaces + + InterfaceVer: 0, + InterfaceVerMinor: 0, + + //transSyntax + + TransSyntaxVer: 2, + } + + ioxidr.Interfaces[0] = 0xc4 + ioxidr.Interfaces[1] = 0xfe + ioxidr.Interfaces[2] = 0xfc + ioxidr.Interfaces[3] = 0x99 + + ioxidr.Interfaces[4] = 0x60 + ioxidr.Interfaces[5] = 0x52 + + ioxidr.Interfaces[6] = 0x1b + ioxidr.Interfaces[7] = 0x10 + + ioxidr.Interfaces[8] = 0xbb + ioxidr.Interfaces[9] = 0xcb + + ioxidr.Interfaces[10] = 0x00 + ioxidr.Interfaces[11] = 0xaa + ioxidr.Interfaces[12] = 0x00 + ioxidr.Interfaces[13] = 0x21 + ioxidr.Interfaces[14] = 0x34 + ioxidr.Interfaces[15] = 0x7a + + ioxidr.TransSyntax[0] = 0x04 + ioxidr.TransSyntax[1] = 0x5d + ioxidr.TransSyntax[2] = 0x88 + ioxidr.TransSyntax[3] = 0x8a + + ioxidr.TransSyntax[4] = 0xeb + ioxidr.TransSyntax[5] = 0x1c + + ioxidr.TransSyntax[6] = 0xc9 + ioxidr.TransSyntax[7] = 0x11 + + ioxidr.TransSyntax[8] = 0x9f + ioxidr.TransSyntax[9] = 0xe8 + + ioxidr.TransSyntax[10] = 0x08 + ioxidr.TransSyntax[11] = 0x00 + ioxidr.TransSyntax[12] = 0x2b + ioxidr.TransSyntax[13] = 0x10 + ioxidr.TransSyntax[14] = 0x48 + ioxidr.TransSyntax[15] = 0x60 + + da := DCERPC_ALIVE{ + AllocHint: 0, + ContextId: 0, + OpNum: 3, + } + + buf1 := new(bytes.Buffer) + + binary.Write(buf1, binary.LittleEndian, ds1) + ds1Bytes := buf1.Bytes() + + buf2 := new(bytes.Buffer) + + binary.Write(buf2, binary.LittleEndian, ds2) + ds2Bytes := buf2.Bytes() + + buf3 := new(bytes.Buffer) + + binary.Write(buf3, binary.LittleEndian, ioxidr) + ioxidrBytes := buf3.Bytes() + + buf4 := new(bytes.Buffer) + + binary.Write(buf4, binary.LittleEndian, da) + daBytes := buf4.Bytes() + + firstByte := make([]byte, len(ds1Bytes)+len(ioxidrBytes)) + + copy(firstByte[0:], ds1Bytes) + copy(firstByte[len(ds1Bytes):], ioxidrBytes) + + secondByte := make([]byte, len(ds2Bytes)+len(daBytes)) + + copy(secondByte[0:], ds2Bytes) + copy(secondByte[len(ds2Bytes):], daBytes) + + wm.sendPackets = append(wm.sendPackets, packet.NewPacket(firstByte, len(ds1Bytes)+len(ioxidrBytes))) + wm.sendPackets = append(wm.sendPackets, packet.NewPacket(secondByte, len(ds2Bytes)+len(daBytes))) + + return &wm + +} + +type DCERPC_DEFAULT struct { + Rpc_ver uint8 + Rpc_ver_minor uint8 + Ptype uint8 + Flags uint8 + Drep uint32 + Frag_len uint16 + Auth_len uint16 + Call_id uint32 +} + +type DCERPC_ALIVE struct { + AllocHint uint32 + ContextId uint16 + OpNum uint16 +} + +type DCERPC_IOXIDResolver struct { + MaxXmitFrag uint16 + MaxRecvFrag uint16 + AssocGroup uint32 + NumCtxItem uint8 + UnknownCode [3]uint8 + + ContextId uint16 + NumTransItem uint8 + UnknownCode2 uint8 + + Interfaces [16]uint8 + InterfaceVer uint16 + InterfaceVerMinor uint16 + TransSyntax [16]uint8 + TransSyntaxVer uint32 +} diff --git a/matcher/wmi/wmi_test.go b/matcher/wmi/wmi_test.go new file mode 100644 index 0000000..6fd03a3 --- /dev/null +++ b/matcher/wmi/wmi_test.go @@ -0,0 +1,51 @@ +package wmi + +//import ( +// "fmt" +// "loafle.com/overflow/collector/discovery/scan/matcher/packet" +// "loafle.com/overflow/collector/discovery/scan/matcher/scaninfo" +// "loafle.com/overflow/collector/discovery/types" +// "net" +// "testing" +//) +// +//func TestWMI(t *testing.T) { +// +// lm := NewWMIMatcher() +// +// port := types.NewPort("135", types.NewHost("192.168.1.1"), types.TYPE_TCP) +// scanInfo := scaninfo.NewServiceScanInfo(port) +// var ipport string +// ipport = port.Host.Ip + ":" + string(port.Port) +// +// fmt.Println(ipport) +// client, _ := net.Dial("tcp", ipport) +// +// defer client.Close() +// +// fmt.Println(lm.PacketCount()) +// +// for ii := 0; ii < lm.PacketCount(); ii++ { +// +// pack := lm.Packet(ii) +// +// fmt.Println(pack) +// +// client.Write(pack.Buffer) +// +// bytes := make([]byte, 1024) +// +// read, _ := client.Read(bytes) +// +// //fmt.Println(bytes) +// +// b := lm.Match(ii, packet.NewPacket(bytes, read), scanInfo) +// +// if b { +// fmt.Println("Good") +// } +// +// } +// +// t.Log(scanInfo) +//} diff --git a/model/scaninfo/scaninfo.go b/model/scaninfo/scaninfo.go new file mode 100644 index 0000000..ad3ba6d --- /dev/null +++ b/model/scaninfo/scaninfo.go @@ -0,0 +1,20 @@ +package scaninfo + +type ServiceScanInfo interface { + SetHistory(history string) + GetPort() string + GetIP() string +} + +type ScanInfoImpl struct { + Ip string + Port string +} + +func (s *ScanInfoImpl) SetHistory(history string) {} +func (s *ScanInfoImpl) GetPort() string { return s.Port } +func (s *ScanInfoImpl) GetIP() string { return s.Ip } + +func NewScanInfoImpl(ip string, port string) *ScanInfoImpl { + return &ScanInfoImpl{Ip: ip, Port: port} +} diff --git a/model/timestamp/timestamp.go b/model/timestamp/timestamp.go new file mode 100644 index 0000000..6a465d0 --- /dev/null +++ b/model/timestamp/timestamp.go @@ -0,0 +1,37 @@ +package timestamp + +import ( + "fmt" + "strconv" + "time" +) + +type Timestamp time.Time + +func (t Timestamp) MarshalJSON() ([]byte, error) { + ts := time.Time(t).Unix() + stamp := fmt.Sprint(ts * 1000) + return []byte(stamp), nil +} + +func (t *Timestamp) UnmarshalJSON(b []byte) error { + ts, err := strconv.Atoi(string(b)) + if err != nil { + return err + } + *t = Timestamp(time.Unix(int64(ts)/1000, 0)) + + return nil +} + +func (t Timestamp) String() string { + return time.Time(t).String() +} + +func Now() Timestamp { + return Timestamp(time.Now()) +} + +func Date(year int, month time.Month, day int) Timestamp { + return Timestamp(time.Date(year, month, day, 0, 0, 0, 0, time.UTC)) +} diff --git a/oauth_api/main.go b/oauth_api/main.go new file mode 100644 index 0000000..42caae8 --- /dev/null +++ b/oauth_api/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "github.com/gin-gonic/gin" + "golang.org/x/oauth2" + "golang.org/x/oauth2/facebook" + "golang.org/x/oauth2/google" + "net/http" +) + +var config *oauth2.Config + +func getConfig(snsType SNSType) *oauth2.Config { + + if snsType == FB_TYPE { + return &oauth2.Config{ + ClientID: FB_CLIENT_ID, + ClientSecret: FB_CLIENT_SECRET, + RedirectURL: REDIRECT_URL, + Scopes: []string{"email", "public_profile"}, + Endpoint: facebook.Endpoint, + } + } else if snsType == GG_TYPE { + + return &oauth2.Config{ + ClientID: GG_CLIENT_ID, + ClientSecret: GG_CLIENT_SECRET, + RedirectURL: REDIRECT_URL, + Scopes: []string{"https://www.googleapis.com/auth/userinfo.email"}, + Endpoint: google.Endpoint, + } + } + + return nil +} +func Home(c *gin.Context) { + config = getConfig(GG_TYPE) + codeUrl := config.AuthCodeURL("") + http.Redirect(c.Writer, c.Request, codeUrl, http.StatusTemporaryRedirect) + +} + +func SNSCallback(c *gin.Context) { + c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") + + code := c.Request.FormValue("code") + + info := GetSNSInfo(config, code, GG_TYPE) + c.Writer.Write([]byte(fmt.Sprintf("Usern emailis %s and email is %s
", info.Name, info.Email))) +} +func main() { + r := gin.Default() + + r.GET("/", Home) + r.GET("/SNSCallback", SNSCallback) + + r.Run() +} diff --git a/oauth_api/oauth.go b/oauth_api/oauth.go new file mode 100644 index 0000000..9774f1f --- /dev/null +++ b/oauth_api/oauth.go @@ -0,0 +1,61 @@ +package main + +import ( + "encoding/json" + "fmt" + "golang.org/x/net/context" + "golang.org/x/oauth2" + "io/ioutil" + "log" + "net/http" +) + +type UserSNSInfo struct { + //Id string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + AccessToken string +} + +func GetAccessToken(config *oauth2.Config, code string) *oauth2.Token { + + token, err := config.Exchange(context.Background(), code) + + if err != nil { + log.Fatal(err) + } + + fmt.Println(token.AccessToken, "Expire :", token.Expiry, "type: ", token.TokenType) + + return token +} + +func GetSNSInfo(config *oauth2.Config, code string, snsType SNSType) UserSNSInfo { + + var response *http.Response + var err error + + accessToken := GetAccessToken(config, code) + if snsType == FB_TYPE { + response, err = http.Get("https://graph.facebook.com/me?fields=id,name,email&access_token=" + accessToken.AccessToken) + } else if snsType == GG_TYPE { + response, err = http.Get("https://www.googleapis.com/oauth2/v3/userinfo?access_token=" + accessToken.AccessToken) + } + + if err != nil { + fmt.Println(err.Error()) + } + + data, err := ioutil.ReadAll(response.Body) + + if err != nil { + fmt.Println(err.Error()) + } + userinfo := UserSNSInfo{} + json.Unmarshal([]byte(data), &userinfo) + userinfo.AccessToken = accessToken.AccessToken + + fmt.Printf("Username %s and email is %s
", userinfo.Name, userinfo.Email) + + return userinfo +} diff --git a/oauth_api/oauth_test.go b/oauth_api/oauth_test.go new file mode 100644 index 0000000..41b6925 --- /dev/null +++ b/oauth_api/oauth_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "errors" + "golang.org/x/net/context" + "golang.org/x/oauth2" + "net/http" + "testing" +) + +type appOAuthTransfer1 struct { + rt func(req *http.Request) (resp *http.Response, err error) +} + +func (t *appOAuthTransfer1) RoundTrip(req *http.Request) (resp *http.Response, err error) { + return t.rt(req) +} + +func TestExchangeRequest(t *testing.T) { + tr := &appOAuthTransfer1{ + rt: func(r *http.Request) (w *http.Response, err error) { + headerAuth := r.Header.Get("Authorization") + if headerAuth != "" { + t.Errorf("Unexpected authorization header, %v is found.", headerAuth) + } + return nil, errors.New("no response") + }, + } + + c := &http.Client{Transport: tr} + + conf := getConfig(FB_TYPE) + ctx := context.WithValue(context.Background(), oauth2.HTTPClient, c) + conf.Exchange(ctx, "code") +} diff --git a/oauth_api/snsconst.go b/oauth_api/snsconst.go new file mode 100644 index 0000000..37e4fa9 --- /dev/null +++ b/oauth_api/snsconst.go @@ -0,0 +1,16 @@ +package main + +const FB_CLIENT_ID = "808542632617259" +const FB_CLIENT_SECRET = "8982cf312e1f31ba42f4ce847564aeff" + +const GG_CLIENT_ID = "609607264245-qfq7odj24hi3minf4juhuje8445m2ro3.apps.googleusercontent.com" +const GG_CLIENT_SECRET = "LUs0jpkLLhJtpKi5Zz5bDBo4" + +const REDIRECT_URL = "http://localhost:8080/SNSCallback" + +type SNSType int + +const ( + FB_TYPE SNSType = 1 + GG_TYPE SNSType = 2 +) diff --git a/queue/queue.go b/queue/queue.go new file mode 100644 index 0000000..0fae54e --- /dev/null +++ b/queue/queue.go @@ -0,0 +1,117 @@ +package queue + +import ( + "container/heap" + "log" + "sync" + "time" +) + +type Item struct { + Value interface{} + Priority int +} + +type LoafleQueue struct { + mtx sync.Mutex + items []*Item + interval time.Duration + size int + + senderChanel chan interface{} +} + +func (lq LoafleQueue) Len() int { + return len(lq.items) +} + +func (lq LoafleQueue) Less(i, j int) bool { + + return lq.items[i].Priority < lq.items[j].Priority +} + +func (lq LoafleQueue) Swap(i, j int) { + lq.items[i], lq.items[j] = lq.items[j], lq.items[i] +} + +func (lq *LoafleQueue) getItems() { + resultItems := make([]*Item, 0) + + for { + + time.Sleep(time.Second * lq.interval) + + if lq.Len() > 0 { + lq.size = lq.Len() + + lq.mtx.Lock() + for i := 0; i < lq.size; i++ { + item := heap.Pop(lq).(*Item) + resultItems = append(resultItems, item) + } + lq.mtx.Unlock() + + lq.senderChanel <- resultItems + lq.size = lq.Len() + //log.Println("result length: ", len(resultItems)) + resultItems = nil + } + } + +} + +func (lq *LoafleQueue) Push(i interface{}) { + lq.mtx.Lock() + defer lq.mtx.Unlock() + + n := len(lq.items) + nItem := i.(*Item) + nItem.Priority = n + lq.items = append(lq.items, nItem) +} + +func (lq *LoafleQueue) Pop() interface{} { + old := lq.items + n := len(old) + nItem := old[n-1] + lq.items = old[0 : n-1] + return nItem +} +func (lq LoafleQueue) newItem(value interface{}) *Item { + return &Item{ + Value: value, + } +} + +func (lq *LoafleQueue) PushItem(v interface{}) { + + it := lq.newItem(v) + heap.Push(lq, it) + +} + +func (lq *LoafleQueue) Close() { + //if lq.eventChanel != nil { + // observer.Remove(lq.queueType, lq.eventChanel) + //} + lq.items = nil +} + +func NewQueue(interval time.Duration, senderChanel chan interface{}) *LoafleQueue { + items := make([]*Item, 0) + + log.Println("NewQueu Start") + lq := &LoafleQueue{ + items: items, + interval: interval, + senderChanel: senderChanel, + } + + heap.Init(lq) + + //observer.Add(eventType, lq.eventChanel) + //go lq.notifyEventHandler(event) + go lq.getItems() + + return lq +} diff --git a/queue/queue_test.go b/queue/queue_test.go new file mode 100644 index 0000000..0d313d2 --- /dev/null +++ b/queue/queue_test.go @@ -0,0 +1,64 @@ +package queue + +import ( + "git.loafle.net/overflow/agent_api/observer" + "git.loafle.net/overflow/agent_api/observer/messages" + "log" + "reflect" + "testing" + "time" +) + +func TestNewQueue(t *testing.T) { + senderChanel := make(chan interface{}, 0) + + lq := NewQueue(messages.QUEUE_DATA, 3, senderChanel) + + log.Println(lq) + + observer.Notify(messages.QUEUE_DATA, "data") + observer.Notify(messages.QUEUE_DATA, "data1111") + observer.Notify(messages.QUEUE_DATA, "data2222") + observer.Notify(messages.QUEUE_DATA, "data3333") + observer.Notify(messages.QUEUE_DATA, "data4444") + observer.Notify(messages.QUEUE_DATA, "data5555") + observer.Notify(messages.QUEUE_DATA, "data6666") + observer.Notify(messages.QUEUE_DATA, "data7777") + observer.Notify(messages.QUEUE_DATA, "data8888") + observer.Notify(messages.QUEUE_DATA, "data9999") + + go GetItemTest(senderChanel) + + time.Sleep(time.Second * 3) + + observer.Notify(messages.QUEUE_DATA, "data1111") + observer.Notify(messages.QUEUE_DATA, "data2222") + observer.Notify(messages.QUEUE_DATA, "data3333") + observer.Notify(messages.QUEUE_DATA, "data4444") + observer.Notify(messages.QUEUE_DATA, "data5555") + observer.Notify(messages.QUEUE_DATA, "data6666") + observer.Notify(messages.QUEUE_DATA, "data7777") + + time.Sleep(time.Second * 10) + + lq.Close() + close(senderChanel) +} + +func GetItemTest(senderChanel chan interface{}) { + + for sc := range senderChanel { + items := reflect.ValueOf(sc) + if items.Kind() != reflect.Slice { + log.Println("ddddd") + } + //log.Println(items.Len()) + for i := 0; i < items.Len(); i++ { + d := items.Index(i).Elem().Interface() + dd := d.(Item) + + log.Printf("data: %s , periorty: %d", dd.Value, dd.Priority) + } + //fmt.Println(sc) + } +} diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go new file mode 100644 index 0000000..1c46bd9 --- /dev/null +++ b/scheduler/scheduler.go @@ -0,0 +1,272 @@ +package scheduler + +import ( + "errors" + "reflect" + "runtime" + "sort" + "time" +) + +const ( + MAX_TASKS = 10000 +) + +type Task struct { + id string + intervalSec uint64 + TaskFunc string + period time.Duration + taskFunc map[string]interface{} + funcParams map[string]([]interface{}) + lastAt time.Time + nextAt time.Time + running bool +} + +func NewTask(id string, intervel uint64) *Task { + return &Task{ + id, + intervel, + "", + 0, + make(map[string]interface{}), + make(map[string]([]interface{})), + time.Unix(0, 0), + time.Unix(0, 0), + true, + } +} + +func (t *Task) runnable() bool { + return time.Now().After(t.nextAt) +} + +func (t *Task) run() { + if !t.running { + return + } + taskFunc := reflect.ValueOf(t.taskFunc[t.TaskFunc]) + taskParams := t.funcParams[t.TaskFunc] + in := make([]reflect.Value, len(taskParams)) + for i, param := range taskParams { + in[i] = reflect.ValueOf(param) + } + taskFunc.Call(in) + t.lastAt = time.Now() + t.addNextAt() +} + +func (t *Task) invoke(TaskFunc interface{}, params ...interface{}) error { + if reflect.TypeOf(TaskFunc).Kind() != reflect.Func { + return errors.New("Not a function type.") + } + + funcName := runtime.FuncForPC(reflect.ValueOf((TaskFunc)).Pointer()).Name() + t.taskFunc[funcName] = TaskFunc + t.funcParams[funcName] = params + t.TaskFunc = funcName + t.addNextAt() + + return nil +} + +func (t *Task) addNextAt() { + if t.lastAt == time.Unix(0, 0) { + t.lastAt = time.Now() + } + + if t.period == 0 { + t.period = time.Duration(t.intervalSec) + t.nextAt = t.lastAt.Add(1 * time.Second) + } else { + t.nextAt = t.lastAt.Add((t.period * time.Second) - time.Second) + } + +} + +type Scheduler struct { + Tasks [MAX_TASKS]*Task + size int + stop chan bool +} + +func (s *Scheduler) Len() int { + return s.size +} + +func (s *Scheduler) Swap(i, j int) { + s.Tasks[i], s.Tasks[j] = s.Tasks[j], s.Tasks[i] +} + +func (s *Scheduler) Less(i, j int) bool { + return s.Tasks[j].nextAt.After(s.Tasks[i].nextAt) +} + +func (s *Scheduler) RunnableTasks() ([MAX_TASKS]*Task, int) { + runnableTasks := [MAX_TASKS]*Task{} + n := 0 + sort.Sort(s) + for i := 0; i < s.size; i++ { + if s.Tasks[i].runnable() { + runnableTasks[n] = s.Tasks[i] + n++ + } else { + break + } + } + return runnableTasks, n +} + +func (s *Scheduler) nextRun() (*Task, time.Time) { + if s.size <= 0 { + return nil, time.Now() + } + sort.Sort(s) + return s.Tasks[0], s.Tasks[0].nextAt +} + +func (s *Scheduler) addTask(id string, intervalSec uint64) *Task { + s.remove(id) + + Task := NewTask(id, intervalSec) + s.Tasks[s.size] = Task + s.size++ + return Task +} + +func (s *Scheduler) runAll() { + runnableTasks, n := s.RunnableTasks() + + if n != 0 { + for i := 0; i < n; i++ { + go runnableTasks[i].run() + } + } +} + +func (s *Scheduler) remove(id string) error { + i := 0 + var ex bool + for ; i < s.size; i++ { + if s.Tasks[i].id == id { + ex = true + break + } + } + if !ex { + return nil + } + for j := (i + 1); j < s.size; j++ { + s.Tasks[i] = s.Tasks[j] + i++ + } + s.size = s.size - 1 + + return nil +} + +func (s *Scheduler) Start() { + s.stop = make(chan bool, 1) + ticker := time.NewTicker(time.Second * 1) + + go func() { + for { + select { + case <-ticker.C: + s.runAll() + case <-s.stop: + return + } + } + }() +} + +func (s *Scheduler) Stop() { + s.stop <- true +} + +func (s *Scheduler) NewSchedule(id string, interval uint64, fn interface{}) error { + return s.addTask(id, interval).invoke(fn, id) +} + +func (s *Scheduler) RemoveSchedule(id string) error { + i := 0 + var ex bool + for ; i < s.size; i++ { + if s.Tasks[i].id == id { + ex = true + break + } + } + if !ex { + return nil + } + for j := (i + 1); j < s.size; j++ { + s.Tasks[i] = s.Tasks[j] + i++ + } + s.size = s.size - 1 + + return nil +} + +func (s *Scheduler) RemoveAllSchedule() error { + for i := 0; i < s.size; i++ { + s.Tasks[i] = nil + } + s.size = 0 + + return nil +} + +func (s *Scheduler) UpdateInterval(id string, interval uint64) { + for i := 0; i < s.Len(); i++ { + if s.Tasks[i].id == id { + s.Tasks[i].intervalSec = interval + s.Tasks[i].period = 0 + } + } +} + +func (s *Scheduler) StartSchedule(id string) error { + i := 0 + var ex bool + for ; i < s.size; i++ { + if s.Tasks[i].id == id { + ex = true + break + } + } + if !ex { + return errors.New("Cannot find the task. ID : " + id) + } + if s.Tasks[i].running { + return errors.New("Already started task. ID : " + id) + } + s.Tasks[i].running = true + return nil +} + +func (s *Scheduler) StopSchedule(id string) error { + i := 0 + var ex bool + for ; i < s.size; i++ { + if s.Tasks[i].id == id { + ex = true + break + } + } + if s.Tasks[i].running { + s.Tasks[i].running = false + } + + if !ex { + return errors.New("Cannot find the task. ID : " + id) + } + if !s.Tasks[i].running { + return errors.New("Already stopped task. ID : " + id) + } + s.Tasks[i].running = false + return nil +} diff --git a/scheduler/scheduler_test.go b/scheduler/scheduler_test.go new file mode 100644 index 0000000..5659733 --- /dev/null +++ b/scheduler/scheduler_test.go @@ -0,0 +1,55 @@ +package scheduler + +import ( + "log" + "testing" + "time" +) + +func TestScheduler(t *testing.T) { + s := Scheduler{} + s.Start() + s.NewSchedule("1111", 3, callback) + time.Sleep(time.Second * 10) + s.RemoveSchedule("1111") + + time.Sleep(time.Second * 3) + s.Stop() +} + +func TestMassiveSchedule(t *testing.T) { + s := Scheduler{} + s.Start() + + for i := 0; i < 9999; i++ { + s.NewSchedule(string(i), 5, test) + } + s.NewSchedule("#######################", 5, test) + time.Sleep(time.Second * 10) + + s.NewSchedule("#######################", 1, test) + time.Sleep(time.Second * 100) +} + +func callback(id string) { + log.Println(id) +} + +func test(id string) { + if id == "#######################" { + log.Println(id) + } +} + +func TestSchedulerStopStart(t *testing.T) { + s := Scheduler{} + s.Start() + s.NewSchedule("1111", 3, callback) + time.Sleep(time.Second * 10) + s.StopSchedule("1111") + time.Sleep(time.Second * 5) + s.StartSchedule("1111") + + time.Sleep(time.Second * 10) + s.Stop() +} diff --git a/script/ubuntu/of_probe b/script/ubuntu/of_probe new file mode 100644 index 0000000..2342551 --- /dev/null +++ b/script/ubuntu/of_probe @@ -0,0 +1,71 @@ +### BEGIN INIT INFO +# Provides: overFlow +# Required-Start: $network +# Required-Stop: $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: overFlow agent +# Description: overFlow Agent +### END INIT INFO + +PROG="ofcollector" +PROG_PATH="/home/insanity/develop/overflow/overflow.collector/bin/" +PID_PATH="/var/run/" +FILE_SERVER=/var/run/of_server + +start() { + if [ -e "$PID_PATH/$PROG.pid" ]; then + ## Program is running, exit with error. + echo "$PROG is currently running!" 1>&2 + exit 1 + else + $PROG_PATH/$PROG 2>&1 > /dev/null & + echo "$PROG started" + touch "$PID_PATH/$PROG.pid" + fi +} + +stop() { + sudo echo -n 'STOP' | netcat -U $FILE_SERVER + + if [ -e "$PID_PATH/$PROG.pid" ]; then + ## Program is running, so stop it + #killall $PROG + rm "$PID_PATH/$PROG.pid" + echo "$PROG stopped" + else + ## Program is not running, exit with error. + echo "Error! $PROG not started!" 1>&2 + exit 1 + fi +} + +status() { + sudo echo -n $1 | netcat -U $FILE_SERVER +} + + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +case "$1" in + start) + start + ;; + stop) + stop + ;; + reload|restart|force-reload) + stop + start + ;; + status) + status 'STATUS' + ;; + **) + echo "Usage: $0 {start|stop|restart|status}" 1>&2 + exit 1 + ;; +esac \ No newline at end of file