first probe

.gitignore

@@ -0,0 +1,68 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+
+# Sensitive or high-churn files:
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+
+# Gradle:
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Mongo Explorer plugin:
+.idea/**/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+.idea/
+*.iml
+
+vendor/
+glide.lock
+.DS_Store
+dist/
+debug

agent_api/config_manager/config.go

@@ -0,0 +1,53 @@
+package config_manager
+
+type Connection struct {
+	Ip       string `json:"ip"`
+	Port     string `json:"port"`
+	PortType string `json:"portType"`
+	SSL      bool   `json:"ssl"`
+}
+
+type Target struct {
+	Auth       map[string]interface{} `json:"auth"`
+	Connection Connection             `json:"connection"`
+}
+
+type Schedule struct {
+	Interval string `json:"interval"`
+}
+
+type Keys struct {
+	Metric string `json:"metric"`
+	Key    string `json:"key"`
+}
+
+type QueryInfo struct {
+	Query  string                 `json:"query"`
+	Extend map[string]interface{} `json:"extend"`
+}
+
+type MappingInfo struct {
+	ParseDirection string   `json:"parseDirection"`
+	ArrayColumns   []string `json:"arrayColumns"`
+	KeyColumns     []string `json:"keyColumns"`
+	ValueColumn    string   `json:"valueColumn"`
+}
+
+type Item struct {
+	Keys        []Keys      `json:"keys"`
+	QueryInfo   QueryInfo   `json:"queryInfo"`
+	MappingInfo MappingInfo `json:"mappingInfo"`
+}
+
+type Crawler struct {
+	Name      string `json:"name"`
+	Container string `json:"container"`
+}
+
+type Config struct {
+	Id       string   `json:"id"`
+	Target   Target   `json:"target"`
+	Schedule Schedule `json:"schedule"`
+	Crawler  Crawler  `json:"crawler"`
+	Items    []Item   `json:"items"`
+}

agent_api/config_manager/config_manager.go

@@ -0,0 +1,7 @@
+package config_manager
+
+type ConfigManager interface {
+	GetGlobalConfig() *GlobalConfig
+	GetSensorById(id string) *Config
+	GetSensors() map[string]*Config
+}

agent_api/config_manager/config_test.go

@@ -0,0 +1,54 @@
+package config_manager
+
+import (
+	"encoding/json"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"log"
+	"testing"
+)
+
+func TestConfigRead(t *testing.T) {
+
+	b, err := ioutil.ReadFile("/root/gowork/src/loafle.com/overflow/agent_api/config_manager/test.json")
+	if err != nil {
+		log.Panic(err)
+	}
+	var m = Config{}
+	json.Unmarshal(b, &m)
+
+	assert.Equal(t, m.Id, "123980918237")
+	assert.Equal(t, m.Target.Connection.Ip, "192.168.1.103")
+	assert.Equal(t, m.Target.Connection.Port, "1433")
+	assert.Equal(t, m.Target.Connection.SSL, false)
+	assert.Equal(t, m.Target.Connection.PortType, "tcp")
+
+	assert.Equal(t, m.Target.Auth["url"], "jdbc:sqlserver://192.168.1.106:1433;")
+	assert.Equal(t, m.Target.Auth["id"], "sa")
+	assert.Equal(t, m.Target.Auth["pw"], "qwe123")
+	assert.Equal(t, m.Target.Auth["query"], "select * from master.dbo.sysprocesses")
+
+	assert.Equal(t, m.Schedule.Interval, "10")
+
+	assert.Equal(t, m.Crawler.Name, "health_activedirectory")
+	assert.Equal(t, m.Crawler.Container, "network_proxy")
+
+	item := m.Items[0]
+
+	assert.Equal(t, item.Keys[0].Metric, "object[$0].db[$1].datafile_size")
+	assert.Equal(t, item.Keys[0].Key, "Data File(s) Size (KB)")
+
+	assert.Equal(t, item.Keys[1].Metric, "object[$0].db[$1].logfile_size")
+	assert.Equal(t, item.Keys[1].Key, "Log File(s) Size (KB)")
+
+	assert.Equal(t, item.QueryInfo.Query, "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'")
+	assert.Equal(t, item.QueryInfo.Extend["test"], "test")
+
+	assert.Equal(t, item.MappingInfo.ParseDirection, "row")
+	assert.Equal(t, item.MappingInfo.ArrayColumns[0], "object_name")
+	assert.Equal(t, item.MappingInfo.ArrayColumns[1], "instance_name")
+
+	assert.Equal(t, item.MappingInfo.KeyColumns[0], "counter_name")
+	assert.Equal(t, item.MappingInfo.ValueColumn, "cntr_value")
+
+}

agent_api/config_manager/global_config.go

@@ -0,0 +1,31 @@
+package config_manager
+
+/* global config 예제
+central:
+  address: "http://localhost:9090"
+  port: 443
+log_Path: "./bin/log.xml"
+paths:
+  rootFolder : "/home/cm2/"
+  configFolder : "config/"
+  binaryFolder : "container/"
+  pidFolder : "pids/"
+  scriptFile : "start"
+intervalSecond: 10
+*/
+
+type GlobalConfig struct {
+	Central struct {
+		Address string `yaml:"address"`
+		Port    int    `yaml:"port"`
+	}
+	LogPath string `yaml:"logPath"`
+	Paths   struct {
+		RootFolder   string `yaml:"rootFolder"`
+		ConfigFolder string `yaml:"configFolder"`
+		BinaryFolder string `yaml:"binaryFolder"`
+		PidFolder    string `yaml:"pidFolder"`
+		ScriptFile   string `yaml:"scriptFile"`
+	}
+	IntervalSecond int `yaml:"intervalSecond"`
+}

agent_api/config_manager/test.json

@@ -0,0 +1,50 @@
+{
+  "id" : "123980918237",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "1433",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+      "url":"jdbc:sqlserver://192.168.1.106:1433;",
+      "id":"sa",
+      "pw":"qwe123",
+      "query" : "select * from master.dbo.sysprocesses"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"health_activedirectory",
+    "container":"network_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric" : "object[$0].db[$1].datafile_size",
+          "key" : "Data File(s) Size (KB)"
+        },
+        {
+          "metric" : "object[$0].db[$1].logfile_size",
+          "key" : "Log File(s) Size (KB)"
+        }
+      ],
+      "queryInfo" : {
+        "query": "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'",
+        "extend" : {
+            "test":"test"
+        }
+      },
+      "mappingInfo" : {
+        "parseDirection" : "row",
+        "arrayColumns" : [ "object_name","instance_name"],
+        "keyColumns" : ["counter_name"],
+        "valueColumn" : "cntr_value"
+      }
+    }
+  ]
+}

agent_api/crawler_manager/grpc_clientpool/grpc_clientpool.go

@@ -0,0 +1,7 @@
+package grpc_clientpool
+
+import "google.golang.org/grpc"
+
+type grpc_clientpool interface {
+	GetClient(container string) (*grpc.ClientConn, error)
+}

agent_api/messages/const.go

@@ -0,0 +1,31 @@
+package messages
+
+const (
+	INIT_AGT_STARTING = "INIT_AGT_STARTING"
+
+	EVENT_AGT_START = "EVENT_AGT_START"
+	EVENT_AGT_STOP  = "EVENT_AGT_STOP"
+	EVENT_AGT_ERROR = "EVENT_AGT_ERROR"
+
+	TASK_SENSOR_START        = "TASK_SENSOR_START"
+	TASK_SENSOR_STOP         = "TASK_SENSOR_STOP"
+	TASK_SENSOR_ADD          = "TASK_SENSOR_ADD"
+	TASK_SENSOR_REMOVE       = "TASK_SENSOR_REMOVE"
+	TASK_SENSOR_UPDATE       = "TASK_SENSOR_UPDATE"
+	TASK_CRAWLER_UPDATE      = "TASK_CRAWLER_UPDATE"
+	TASK_AGENT_UPDATE        = "TASK_AGENT_UPDATE"
+	TASK_LOG_SEND            = "TASK_LOG_SEND"
+	TASK_POL_INTERVAL_UPDATE = "TASK_POLLER_INTERVAL_UPDATE"
+
+	DISCOVERY_START         = "DISCOVERY_START"
+	DISCOVERY_HOST_START    = "DISCOVERY_HOST_START"
+	DISCOVERY_HOST_FOUND    = "DISCOVERY_HOST_FOUND"
+	DISCOVERY_HOST_DONE     = "DISCOVERY_HOST_DONE"
+	DISCOVERY_PORT_START    = "DISCOVERY_PORT_START"
+	DISCOVERY_PORT_FOUND    = "DISCOVERY_PORT_FOUND"
+	DISCOVERY_PORT_DONE     = "DISCOVERY_PORT_DONE"
+	DISCOVERY_SERVICE_START = "DISCOVERY_SERVICE_START"
+	DISCOVERY_SERVICE_FOUND = "DISCOVERY_SERVICE_FOUND"
+	DISCOVERY_SERVICE_DONE  = "DISCOVERY_SERVICE_DONE"
+	DISCOVERY_DONE          = "DISCOVERY_DONE"
+)

agent_api/messages/data.go

@@ -0,0 +1,9 @@
+package messages
+
+type Data struct {
+	AgentId    string
+	SensorId   string
+	Data       []byte
+	StartedAt  uint64
+	FinishedAt uint64
+}

agent_api/messages/event.go

@@ -0,0 +1,12 @@
+package messages
+
+type Event struct {
+	Type int
+	Id   string
+	Data []byte
+}
+
+const (
+	EVT_TYPE_TASK = 1
+	EVT_TYPE_NONE = 2
+)

agent_api/messages/task.go

@@ -0,0 +1,6 @@
+package messages
+
+type AgentTask struct {
+	Command string
+	Params  map[string]string
+}

agent_api/observer/observer.go

@@ -0,0 +1,100 @@
+package observer
+
+import (
+	"errors"
+	"sync"
+)
+
+type Observer interface {
+	Add(id string, ch chan interface{}) error
+	Remove(id string, rch chan interface{}) error
+	Notify(id string, arg interface{}) error
+}
+
+type observer struct {
+	mtx    sync.Mutex
+	events map[string][]chan interface{}
+}
+
+func (o *observer) Add(id string, ch chan interface{}) error {
+
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+
+	_, ok := o.events[id]
+	if ok {
+		o.events[id] = append(o.events[id], ch)
+	} else {
+		arr := make([]chan interface{}, 0)
+		arr = append(arr, ch)
+		o.events[id] = arr
+	}
+
+	return nil
+}
+func (o *observer) Remove(id string, rch chan interface{}) error {
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+
+	newArr := make([]chan interface{}, 0)
+	chans, ok := o.events[id]
+	if !ok {
+		return errors.New("event not found")
+	}
+
+	for _, ch := range chans {
+		if ch != rch {
+			newArr = append(newArr, ch)
+		} else {
+			close(ch)
+			ch = nil
+		}
+	}
+	if len(newArr) == 0 {
+		delete(o.events, id)
+	} else {
+		o.events[id] = newArr
+	}
+	return nil
+}
+func (o *observer) Notify(id string, arg interface{}) error {
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+
+	chans, ok := o.events[id]
+	if !ok {
+		return errors.New("event not found")
+	}
+
+	for _, ch := range chans {
+		ch <- arg
+	}
+	return nil
+}
+
+var _observer *observer
+var once sync.Once
+
+func init() {
+	once.Do(func() {
+		_observer = &observer{
+			events: make(map[string][]chan interface{}),
+		}
+	})
+}
+
+// interface
+
+func GetInstance() Observer {
+	return _observer
+}
+
+func Add(id string, ch chan interface{}) error {
+	return _observer.Add(id, ch)
+}
+func Remove(id string, rch chan interface{}) error {
+	return _observer.Remove(id, rch)
+}
+func Notify(id string, arg interface{}) error {
+	return _observer.Notify(id, arg)
+}

agent_api/observer/observer_test.go

@@ -0,0 +1,113 @@
+package observer
+
+import (
+	"fmt"
+	"git.loafle.net/overflow/agent_api/messages"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+type Test struct {
+	Id string
+}
+
+func TestAddNotifyObserver(t *testing.T) {
+
+	// others package notify call
+	go func() {
+		time.Sleep(1 * time.Second)
+		Notify("test", Test{Id: "test"})
+	}()
+
+	ch := make(chan interface{}, 0)
+	Add("test", ch)
+
+	dd := <-ch
+	bb := dd.(Test)
+	assert.Equal(t, "test", bb.Id)
+}
+
+func TestRemoveObserver(t *testing.T) {
+
+	ch := make(chan interface{}, 0)
+	Add("test", ch)
+
+	err := Remove("test", ch)
+	assert.Equal(t, nil, err)
+}
+
+// +build ignore
+func TestMultiAddObserver(t *testing.T) {
+	ch1 := make(chan interface{}, 0)
+	ch2 := make(chan interface{}, 0)
+	ch3 := make(chan interface{}, 0)
+	ch4 := make(chan interface{}, 0)
+	ch5 := make(chan interface{}, 0)
+
+	go func() {
+		d := <-ch1
+		dd := d.(Test)
+		assert.Equal(t, dd.Id, "test")
+	}()
+
+	go func() {
+		d := <-ch2
+		dd := d.(Test)
+		assert.Equal(t, dd.Id, "test")
+	}()
+
+	go func() {
+		d := <-ch3
+		dd := d.(Test)
+		assert.Equal(t, dd.Id, "test")
+	}()
+
+	go func() {
+		d := <-ch4
+		dd := d.(Test)
+		assert.Equal(t, dd.Id, "test")
+	}()
+
+	go func() {
+		d := <-ch4
+		dd := d.(Test)
+		assert.Equal(t, dd.Id, "test")
+	}()
+
+	Add("test", ch1)
+	Add("test", ch2)
+	Add("test", ch3)
+	Add("test", ch4)
+	Add("test", ch5)
+
+	// others package notify call
+	go func() {
+		time.Sleep(1 * time.Second)
+		Notify("test", Test{Id: "test"})
+
+		time.Sleep(2 * time.Second)
+		Remove(messages.CRAWLER_ADD, ch1)
+		Remove("test", ch2)
+		Remove("test", ch3)
+		Remove("test", ch4)
+		Remove("test", ch5)
+	}()
+
+	time.Sleep(1 * time.Second)
+
+}
+
+func TestStringNotify(t *testing.T) {
+	ch := make(chan interface{}, 0)
+	Add("test", ch)
+
+	go func() {
+		data := <-ch
+		str := data.(string)
+		fmt.Println(str)
+	}()
+
+	Notify("test", "testsetasetaset")
+	time.Sleep(1 * time.Second)
+}

bootstrap/shell.go

@@ -0,0 +1,77 @@
+package bootstrap
+
+import (
+	log "github.com/cihub/seelog"
+	"net"
+	"os"
+	"strings"
+)
+
+const FILE_SERVER string = "/var/run/of_server"
+
+var fd net.Conn
+
+func HandleShell(stop chan bool) {
+	go func() {
+		//os.Stdout.Close()
+		//os.Stderr.Close()
+		//os.Stdin.Close()
+
+		if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+			os.Remove(FILE_SERVER)
+		}
+
+		l, err := net.ListenUnix("unix", &net.UnixAddr{Name: FILE_SERVER, Net: "unix"})
+		if err != nil {
+			log.Error(err.Error())
+			return
+		}
+		defer func() {
+			l.Close()
+			os.Remove(FILE_SERVER)
+		}()
+
+		for {
+			fd, err = l.Accept()
+			if err != nil {
+				log.Error(err)
+				return
+			}
+
+			buf := make([]byte, 1024)
+			nr, err := fd.Read(buf)
+			if err != nil {
+				return
+			}
+
+			data := string(buf[0:nr])
+			switch strings.ToUpper(data) {
+			case "STOP":
+				if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+					os.Remove(FILE_SERVER)
+				}
+				stop <- true
+			case "STATUS":
+				fd.Write(status())
+			default:
+			}
+
+			fd.Close()
+		}
+	}()
+}
+
+func StopHandleShell() {
+	log.Info("Shell Handler stopped.")
+
+	if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+		os.Remove(FILE_SERVER)
+	}
+	if fd != nil {
+		fd.Close()
+	}
+}
+
+func status() []byte {
+	return []byte("STATUS OK\n")
+}

bootstrap/signal.go

@@ -0,0 +1,53 @@
+package bootstrap
+
+import (
+	log "github.com/cihub/seelog"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func HandleSignal(stop chan bool) {
+	sigs := make(chan os.Signal, 1)
+
+	signal.Notify(sigs,
+		os.Kill,
+		os.Interrupt,
+		syscall.SIGKILL,
+		syscall.SIGSTOP,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGQUIT,
+	)
+
+	go func() {
+		for {
+			s := <-sigs
+			switch s {
+			case os.Kill, os.Interrupt, syscall.SIGSTOP, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+				log.Infof("Signal received. [%s]", s)
+				stop <- true
+			default:
+				log.Infof("Signal received. [%s]", s)
+				stop <- true
+			}
+		}
+	}()
+}
+
+/*
+ 1) SIGHUP	 2) SIGINT	 3) SIGQUIT	 4) SIGILL	 5) SIGTRAP
+ 6) SIGABRT	 7) SIGBUS	 8) SIGFPE	 9) SIGKILL	10) SIGUSR1
+11) SIGSEGV	12) SIGUSR2	13) SIGPIPE	14) SIGALRM	15) SIGTERM
+16) SIGSTKFLT	17) SIGCHLD	18) SIGCONT	19) SIGSTOP	20) SIGTSTP
+21) SIGTTIN	22) SIGTTOU	23) SIGURG	24) SIGXCPU	25) SIGXFSZ
+26) SIGVTALRM	27) SIGPROF	28) SIGWINCH	29) SIGIO	30) SIGPWR
+31) SIGSYS	34) SIGRTMIN	35) SIGRTMIN+1	36) SIGRTMIN+2	37) SIGRTMIN+3
+38) SIGRTMIN+4	39) SIGRTMIN+5	40) SIGRTMIN+6	41) SIGRTMIN+7	42) SIGRTMIN+8
+43) SIGRTMIN+9	44) SIGRTMIN+10	45) SIGRTMIN+11	46) SIGRTMIN+12	47) SIGRTMIN+13
+48) SIGRTMIN+14	49) SIGRTMIN+15	50) SIGRTMAX-14	51) SIGRTMAX-13	52) SIGRTMAX-12
+53) SIGRTMAX-11	54) SIGRTMAX-10	55) SIGRTMAX-9	56) SIGRTMAX-8	57) SIGRTMAX-7
+58) SIGRTMAX-6	59) SIGRTMAX-5	60) SIGRTMAX-4	61) SIGRTMAX-3	62) SIGRTMAX-2
+63) SIGRTMAX-1	64) SIGRTMAX
+*/

collector/collector.go

@@ -0,0 +1,136 @@
+package collector_go
+
+import (
+	"context"
+	cm "git.loafle.net/overflow/agent_api/config_manager"
+	"git.loafle.net/overflow/crawler_go/grpc"
+	crm "git.loafle.net/overflow/crawler_manager_go"
+	s "git.loafle.net/overflow/scheduler_go"
+	"log"
+	"strconv"
+	"sync"
+	"time"
+)
+
+var (
+	instance *Collector
+	once     sync.Once
+)
+
+type Collector struct {
+	scheduler *s.Scheduler
+	cm        cm.ConfigManager
+	dataCh    chan interface{}
+}
+
+func Start(started chan bool, dataCh chan interface{}, conf cm.ConfigManager) {
+	c := GetInstance()
+	c.dataCh = dataCh
+	c.start(started, conf)
+}
+
+func Stop(stopped chan bool) {
+	c := GetInstance()
+	c.stop()
+	stopped <- true
+}
+
+func GetInstance() *Collector {
+	once.Do(func() {
+		instance = &Collector{}
+	})
+	return instance
+}
+
+func (c *Collector) start(started chan bool, conf cm.ConfigManager) {
+	go func() {
+		c.cm = conf
+		c.scheduler = &s.Scheduler{}
+		c.scheduler.Start()
+
+		for _, conf := range c.cm.GetSensors() {
+			if err := c.addSensor(conf.Id); err != nil {
+				log.Println(err)
+			}
+		}
+		started <- true
+	}()
+}
+
+func (c *Collector) stop() {
+	c.scheduler.RemoveAllSchedule()
+	c.scheduler.Stop()
+}
+
+func (c *Collector) collect(id string) {
+	go func() {
+		conf := c.cm.GetSensorById(id)
+		conn, err := crm.GetInstance().GetClient(conf.Crawler.Container)
+		if err != nil {
+			log.Println(err)
+		}
+		defer conn.Close()
+
+		dc := grpc.NewDataClient(conn)
+		in := &grpc.Input{}
+
+		in.Id = id
+		in.Name = grpc.Crawlers(grpc.Crawlers_value[conf.Crawler.Name])
+
+		startAt := time.Now().Unix()
+		out, err := dc.Get(context.Background(), in)
+		finishedAt := time.Now().Unix()
+		if err != nil {
+			log.Printf("Cannot collect [ID: %s] [Crawler : %s] - [Err : %s]\n", conf.Id, conf.Crawler.Name, err.Error())
+			return
+		}
+		out.StartDate = startAt
+		out.EndDate = finishedAt
+		log.Printf("COLLECTED. [ID: %s] [Crawler : %s] [Result : %s]\n", conf.Id, conf.Crawler.Name, out.GetData())
+		c.dataCh <- out
+	}()
+}
+
+func (c *Collector) addSensor(sensorId string) error {
+	sensor := c.cm.GetSensorById(sensorId)
+	interval, err := strconv.Atoi(sensor.Schedule.Interval)
+	if err != nil {
+		return err
+	}
+	return c.scheduler.NewSchedule(sensorId, uint64(interval), c.collect)
+}
+
+func (c *Collector) removeSensor(id string) error {
+	if err := c.scheduler.RemoveSchedule(id); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *Collector) updateSensor(id string) error {
+	err := c.removeSensor(id)
+	if err != nil {
+		return err
+	}
+	return c.addSensor(id)
+}
+
+func AddSensor(id string) error {
+	return GetInstance().addSensor(id)
+}
+
+func RemSensor(id string) error {
+	return GetInstance().removeSensor(id)
+}
+
+func UpdateSensor(id string) error {
+	return GetInstance().updateSensor(id)
+}
+
+func StartSensor(id string) error {
+	return GetInstance().scheduler.StartSchedule(id)
+}
+
+func StopSensor(id string) error {
+	return GetInstance().scheduler.StopSchedule(id)
+}

collector/collector_test.go

@@ -0,0 +1,7 @@
+package collector_go
+
+import "testing"
+
+func TestCollect(t *testing.T) {
+
+}

config_manager/config_manager.go

@@ -0,0 +1,188 @@
+package config_manager_go
+
+import (
+	"encoding/json"
+	"git.loafle.net/overflow/agent_api/config_manager"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+type configManager struct {
+	config_manager.ConfigManager // interface implements
+	configs                      map[string]*config_manager.Config
+	globalConfig                 config_manager.GlobalConfig
+}
+
+var _configManager *configManager
+
+func GetInstance() *configManager {
+	var once sync.Once
+	once.Do(func() {
+		_configManager = &configManager{
+			configs: make(map[string]*config_manager.Config, 0),
+		}
+	})
+
+	return _configManager
+
+}
+
+func (c *configManager) stop() {}
+
+func (c *configManager) getConfigPath() string {
+	return c.globalConfig.Paths.RootFolder + c.globalConfig.Paths.ConfigFolder
+}
+
+func (c *configManager) getContainerPath() string {
+	return c.getConfigPath() + "container/"
+}
+
+func (c *configManager) getConfigFilePath(config *config_manager.Config) string {
+	return c.getContainerPath() + appendSeperator(config.Crawler.Container) + appendSeperator(config.Crawler.Name) + config.Id
+}
+
+func appendSeperator(str string) string {
+	if strings.LastIndex(str, "/") != len(str)-1 {
+		return str + "/"
+	}
+	return str
+}
+
+func (c *configManager) loadGlobalConfig(path string) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+	}
+	err = yaml.Unmarshal(data, &c.globalConfig)
+	if err != nil {
+	}
+}
+
+func (c *configManager) loadCrawlerConfigAll() error {
+
+	root := c.getConfigPath()
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		log.Panic(err)
+	}
+	for _, file := range files {
+		if file.IsDir() == true {
+			c.loadCrawlerConfig(root, file.Name())
+		}
+	}
+	return nil
+}
+
+func (c *configManager) loadCrawlerConfig(root string, dir string) {
+
+	separator := ""
+	if strings.LastIndex(root, "/") != len(root)-1 {
+		separator = "/"
+	}
+	currentDir := root + separator + dir
+	files, err := ioutil.ReadDir(currentDir)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	for _, file := range files {
+
+		// 디렉터리라면 재귀 / 파일이라면 설정 로드
+		if file.IsDir() == true {
+			c.loadCrawlerConfig(currentDir, file.Name())
+		} else {
+			b, err := ioutil.ReadFile(currentDir + separator + file.Name())
+			if err != nil {
+				log.Panic(err)
+			}
+			var m = config_manager.Config{}
+			json.Unmarshal(b, &m)
+			c.configs[file.Name()] = &m
+		}
+	}
+
+}
+
+func (c *configManager) addConfig(tmp string) {
+	//for data := range c.addCh {
+
+	path := tmp
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		// error process
+		log.Panic(err)
+	}
+
+	// 임시파일을 로드 ,  Config로 변환
+	var m = config_manager.Config{}
+	err = json.Unmarshal(b, &m)
+	if err != nil {
+		// error process
+		log.Panic(err)
+	}
+
+	// agent 폴더 / config / crawler / .. / .. / .. / 에 해당하는 파일이 있는지 확인, 있다면 삭제
+	// Config 파일로 저장
+	savePath := c.getConfigFilePath(&m)
+	ioutil.WriteFile(savePath, b, 0644)
+
+	// tempfile remove
+	err = os.Remove(path)
+	if err != nil {
+		// error process
+		log.Panic(err)
+	}
+
+	// Config 맵에 저장
+	c.configs[m.Id] = &m
+
+	// notify id
+	//err = observer.Notify(messages.SCF_SENSOR_ADD_DONE, m.Id)
+	//if err != nil {
+	//	continue
+	//}
+	//}
+}
+
+func (c *configManager) removeConfig(id string) {
+	//for data := range c.removeCh {
+	removeid := id
+
+	// check exists
+	config, ok := c.configs[removeid]
+	if !ok {
+		return
+	}
+
+	// 해당 파일 삭제
+	path := c.getConfigFilePath(config)
+	err := os.Remove(path)
+	if err != nil {
+		// error check
+		log.Panic(err)
+	}
+
+	// 해당 id 삭제
+	delete(c.configs, removeid)
+
+	// notify id
+	//err = observer.Notify("TASK_DONE", removeid)
+	//if err != nil {
+	//	continue
+	//}
+	//}
+}
+
+// implements methods
+func (c *configManager) GetGlobalConfig() *config_manager.GlobalConfig {
+	return &c.globalConfig
+}
+func (c *configManager) GetSensorById(id string) *config_manager.Config {
+	return c.configs[id]
+}
+func (c *configManager) GetSensors() map[string]*config_manager.Config {
+	return c.configs
+}

config_manager/config_manager_test.go

@@ -0,0 +1,68 @@
+package config_manager_go
+
+import (
+	"testing"
+	//"github.com/stretchr/testify/assert"
+	"encoding/json"
+	"git.loafle.net/overflow/agent_api/config_manager"
+	"git.loafle.net/overflow/agent_api/messages"
+	"git.loafle.net/overflow/agent_api/observer"
+	"github.com/google/uuid"
+	"io/ioutil"
+	"log"
+	"time"
+)
+
+func TestLoadConfig(t *testing.T) {
+
+	// notify temp channel
+	//ch := make(chan interface{},0)
+	//observer.Add(messages.CFG_LOADED,ch)
+	//go func() {
+	//	data :=<- ch
+	//	c := data.(config_manager.ConfigManager)
+	//	cc := c.GetSensors()
+	//	assert.NotEqual(t, len(cc),0)
+	//}()
+
+	// make config manager after to load
+	//c := NewConfigManager()
+	//c.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	//c.loadCrawlerConfigAll()
+	//
+	//assert.NotEqual(t, len(c.configs),0)
+	//observer.Notify(messages.CFG_LOADED,c)
+	//time.Sleep(1 * time.Second)
+}
+
+func TestAddConfig(t *testing.T) {
+	_configManager.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+
+	// 원본 테스트 설정 파일 로드
+	b, err := ioutil.ReadFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/test.json")
+	if err != nil {
+		log.Panic(err)
+	}
+	var m = config_manager.Config{}
+	json.Unmarshal(b, &m)
+
+	// uuid로 원본 파일 복제
+	rid, _ := uuid.NewRandom()
+	m.Id = rid.String()
+	b, err = json.Marshal(&m)
+	ioutil.WriteFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/"+m.Id, b, 0644)
+
+	// add test
+	observer.Notify(messages.TASK_SENSOR_ADD, "/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/"+m.Id)
+	time.Sleep(1 * time.Second)
+}
+
+func TestRemoveConfig(t *testing.T) {
+	GetInstance()
+	_configManager.loadGlobalConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	_configManager.loadCrawlerConfigAll()
+
+	// remove test
+	//observer.Notify(messages.TASK_SENSOR_REMOVE,"d0fcc7b1-43a7-4acd-a7bf-c9572a9d4c9e")
+	time.Sleep(1 * time.Second)
+}

config_manager/init_method.go

@@ -0,0 +1,115 @@
+package config_manager_go
+
+import (
+	"loafle.com/overflow/agent_api/config_manager"
+)
+
+//import (
+//	"loafle.com/overflow/agent_api/observer"
+//	"loafle.com/overflow/agent_api/messages"
+//)
+//
+
+func Start(ch chan *config_manager.GlobalConfig, path string) error {
+	_configManager = GetInstance()
+	_configManager.loadGlobalConfig(path + "/" + "global.yaml")
+	ch <- _configManager.GetGlobalConfig()
+	return nil
+}
+
+func StartSensorConfig(ch chan config_manager.ConfigManager, authKey string) error {
+	if err := _configManager.loadCrawlerConfigAll(); err != nil {
+		return err
+	}
+	ch <- _configManager
+	return nil
+}
+
+func Stop(ch chan bool) {
+	GetInstance().stop()
+	ch <- true
+}
+
+func AddSensor(tmpFilePath string) {
+
+	GetInstance().addConfig(tmpFilePath)
+
+	////move file
+	//moveFile := tmpFilePath
+	////load file
+	//
+	//file, err := os.Open(moveFile)
+	//
+	//if err != nil {
+	//	log.Panic(err)
+	//}
+	//
+	//b, err := ioutil.ReadFile(moveFile)
+	//if err != nil {
+	//	log.Panic(err)
+	//}
+	//var m = config_manager.Config{}
+	//json.Unmarshal(b, &m)
+	//GetInstance().configs[file.Name()] = &m
+
+}
+
+func RemoveSensor(id string) {
+
+	//remove object
+	// remove file
+
+	GetInstance().removeConfig(id)
+
+}
+
+func UpdateSensor(tmpFile string) {
+
+	//GetInstance().
+
+	//overwrite file
+	// reload file
+
+	GetInstance().addConfig(tmpFile)
+
+}
+
+//func agentStartHandler() {
+//	agentStart := make(chan interface{}, 0)
+//	observer.Add(messages.AGT_STARTING, agentStart)
+//	go func() {
+//		data := <-agentStart
+//		path := data.(string)
+//		// load global config
+//		_configManager.loadGlobalConfig(path + "/" + "global.yaml")
+//
+//		// load all crawler configs
+//		if err := _configManager.loadCrawlerConfigAll(); err != nil {
+//			// error process
+//		}
+//
+//		observer.Remove(messages.AGT_STARTING, agentStart)
+//		observer.Notify(messages.CFG_LOADED, _configManager)
+//	}()
+//}
+//
+//func agentEndHandler() {
+//	agentEnd := make(chan interface{}, 0)
+//	observer.Add(messages.AGT_WILL_STOP, agentEnd)
+//	go func() {
+//		_ = <-agentEnd
+//		observer.Remove(messages.AGT_WILL_STOP, agentEnd)
+//		observer.Remove(messages.TASK_SENSOR_ADD,_configManager.addCh)
+//		observer.Remove(messages.TASK_SENSOR_REMOVE,_configManager.removeCh)
+//	}()
+//}
+//
+//func addSensorHandler() {
+//	observer.Add(messages.TASK_SENSOR_ADD,_configManager.addCh)
+//	go _configManager.addConfig()
+//}
+//
+//func removeSensorHandler() {
+//	observer.Add(messages.TASK_SENSOR_REMOVE,_configManager.removeCh)
+//	go _configManager.removeConfig()
+//}

config_manager/test_agent/config/container/java_proxy/jmx/747788

@@ -0,0 +1,44 @@
+{
+  "id" : "747788",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "9840",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"JMX",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {"metric" : "net.connector[$0].localPort",          "key" : "localPort"},
+        {"metric" : "net.connector[$0].port",               "key" : "port"},
+        {"metric" : "net.connector[$0].protocol",           "key" : "protocol"}
+      ],
+      "queryInfo" : {
+        "query": "*Catalina:type=Connector,*",
+        "extend" : {
+          "aliases" : [
+            {
+              "key": "port",
+              "index":0
+            }
+          ]
+        }
+      },
+      "mappingInfo" : {
+        "arrayColumns" : [ "portName" ]
+      }
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/mongodb/26021802

@@ -0,0 +1,43 @@
+{
+  "id" : "26021802",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "27017",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"MONGODB",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {"metric":"memory.usage.bits", "key":"bits"},
+        {"metric":"memory.usage.rss", "key":"resident"},
+        {"metric":"memory.usage.vmem", "key":"virtual"},
+        {"metric":"memory.usage.supported", "key":"supported"},
+        {"metric":"memory.usage.mapped", "key":"mapped"},
+        {"metric":"memory.usage.mappedWithJournal", "key":"mappedWithJournal"}
+      ],
+      "queryInfo" : {
+        "query": "mem",
+        "extend" : {
+          "dataBaseName":"admin",
+          "statusCommand": "serverStatus"
+        }
+      },
+      "mappingInfo" : {
+        "parseDirection" : "col"
+      }
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/redis/736387363

@@ -0,0 +1,67 @@
+{
+  "id" : "736387363",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"REDIS",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric":"cpu.usage.system",
+          "key":"used_cpu_sys"
+        },
+        {
+          "metric":"cpu.usage.user",
+          "key":"used_cpu_user"
+        },
+        {
+          "metric":"cpu.usage.system_children",
+          "key":"used_cpu_sys_children"
+        },
+        {
+          "metric":"cpu.usage.user_children",
+          "key":"used_cpu_user_children"
+        }
+      ],
+      "queryInfo" : {
+        "query" : "CPU"
+      },
+      "mappingInfo" : {}
+    },
+    {
+      "keys" : [
+        {
+          "metric":"memory.usage.used",
+          "key":"used_memory"
+        },
+        {
+          "metric":"memory.usage.rss",
+          "key":"used_memory_rss"
+        },
+        {
+          "metric":"memory.usage.peak",
+          "key":"used_memory_peak"
+        }
+      ],
+      "queryInfo" : {
+        "query" : "Memory"
+      },
+      "mappingInfo" : {}
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/snmp/68686868

@@ -0,0 +1,42 @@
+{
+  "id" : "68686868",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "161",
+      "ssl" : false,
+      "portType" : "udp"
+    },
+    "auth" : {
+      "version" : "v3",
+      "user" : "loafle",
+      "authType" : "MD5",
+      "authPass" : "qwer5795",
+      "privType" : "DES",
+      "privPass" : "qweqwe123"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"SNMP",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {"metric":"system.uptime", "key":"1.3.6.1.2.1.1.3.0"}
+      ],
+      "queryInfo" : {
+        "query": "mem",
+        "extend" : {
+          "method": "get"
+        }
+      },
+      "mappingInfo" : {
+        "parseDirection" : "col"
+      }
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/snmp/86868686

@@ -0,0 +1,38 @@
+{
+  "id" : "86868686",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.254",
+      "port" : "161",
+      "ssl" : false,
+      "portType" : "udp"
+    },
+    "auth" : {
+      "version":"v2c",
+      "community" : "loafle"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"SNMP",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {"metric":"system.uptime", "key":"1.3.6.1.2.1.1.3.0"}
+      ],
+      "queryInfo" : {
+        "query": "mem",
+        "extend" : {
+          "method": "get"
+        }
+      },
+      "mappingInfo" : {
+        "parseDirection" : "col"
+      }
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/sql/444441122312

@@ -0,0 +1,44 @@
+{
+  "id" : "444441122312",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+      "url":"jdbc:mysql://192.168.1.215:3306",
+      "id":"root",
+      "pw":"qwe123"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"SQL",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric":"net.mysql.connection_count",
+          "key":"Connections"
+        }
+
+      ],
+      "queryInfo" : {
+        "query":"show status where variable_name = 'Connections'"
+      },
+      "mappingInfo" : {
+        "parseDirection" : "row",
+        "valueColumn" : "Value",
+        "keyColumns" : [
+          "Variable_name"
+        ]
+      }
+    }
+  ]
+}

config_manager/test_agent/config/container/java_proxy/sql/777999444

@@ -0,0 +1,38 @@
+{
+  "id" : "777999444",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.106",
+      "port" : "1433",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+      "url":"jdbc:sqlserver://192.168.1.106:1433;",
+      "id":"sa",
+      "pw":"qwe123"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"SQL",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric" :"net.sqlserver.connection_count",
+          "key" : "connection_count"
+        }
+      ],
+      "queryInfo" : {
+        "query": "select count('session_id') as 'connection_count' from sys.dm_exec_connections where session_id = @@SPID"
+      },
+      "mappingInfo" : {}
+    }
+
+  ]
+}

config_manager/test_agent/config/container/java_proxy/sql/989238744

@@ -0,0 +1,37 @@
+{
+  "id" : "989238744",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.106",
+      "port" : "5432",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+      "url":"jdbc:postgresql://192.168.1.106:5432/postgres",
+      "id":"vertx",
+      "pw":"qwe123"
+    }
+  },
+  "schedule" : {
+    "interval" : "3"
+  },
+  "crawler" : {
+    "name":"SQL",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric":"net.pgsql.connection_count",
+          "key" : "connection_count"
+        }
+      ],
+      "queryInfo":{
+        "query" : "select count(pid) as connection_count from pg_catalog.pg_stat_activity where state <> 'idle';"
+      },
+      "mappingInfo" : {}
+    }
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_activedirectory/123980918237

@@ -0,0 +1,23 @@
+{
+  "id" : "123980918237",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.1",
+      "port" : "389",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"HEALTH_ACTIVEDIRECTORY",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_cassandra/292513

@@ -0,0 +1,23 @@
+{
+  "id" : "292513",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "165",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_CASSANDRA",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_ftp/1352231

@@ -0,0 +1,23 @@
+{
+  "id" : "1352231",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "21",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_FTP",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_ftps/135223179

@@ -0,0 +1,23 @@
+{
+  "id" : "135223179",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "2121",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_FTPS",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_http/104116116112

@@ -0,0 +1,23 @@
+{
+  "id" : "104116116112",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "80",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_HTTP",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_https/104116116112115

@@ -0,0 +1,23 @@
+{
+  "id" : "104116116112115",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "443",
+      "ssl" : true,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_HTTPS",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_imap/10510997112

@@ -0,0 +1,23 @@
+{
+  "id" : "10510997112",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "993",
+      "ssl" : true,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_IMAP",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_ldap/10810097112

@@ -0,0 +1,24 @@
+{
+  "id" : "10810097112",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "993",
+      "ssl" : true,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_LDAP",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}
+

config_manager/test_agent/config/container/network_proxy/health_mongodb/10911111010311110098

@@ -0,0 +1,23 @@
+{
+  "id" : "10911111010311110098",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "27017",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_MONGODB",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_mssql/109115115113108

@@ -0,0 +1,24 @@
+{
+  "id" : "109115115113108",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.106",
+      "port" : "1433",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_MSSQL",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}
+

config_manager/test_agent/config/container/network_proxy/health_mysql/109121115113108

@@ -0,0 +1,23 @@
+{
+  "id" : "109121115113108",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "3306",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_MYSQL",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_netbios/11010111698105111115

@@ -0,0 +1,25 @@
+{
+  "id" : "11010111698105111115",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.106",
+      "port" : "139",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_NETBIOS",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}
+
+

config_manager/test_agent/config/container/network_proxy/health_oracle/1111149799108101

@@ -0,0 +1,25 @@
+{
+  "id" : "1111149799108101",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.30",
+      "port" : "1521",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_ORACLE",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}
+
+

config_manager/test_agent/config/container/network_proxy/health_pgsql/112103115113108

@@ -0,0 +1,23 @@
+{
+  "id" : "112103115113108",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.107",
+      "port" : "5432",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_PGSQL",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_pop3/11211111251

@@ -0,0 +1,23 @@
+{
+  "id" : "11211111251",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "110",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_POP3",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_rmi/114109105

@@ -0,0 +1,25 @@
+{
+  "id" : "114109105",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "9840",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_RMI",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}
+
+

config_manager/test_agent/config/container/network_proxy/health_snmpv2c/684845

@@ -0,0 +1,23 @@
+{
+  "id" : "684845",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "161",
+      "ssl" : false,
+      "portType" : "udp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "5"
+  },
+  "crawler" : {
+    "name":"HEALTH_SNMPV2C",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_snmpv3/59797987

@@ -0,0 +1,23 @@
+{
+  "id" : "59797987",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.107",
+      "port" : "161",
+      "ssl" : false,
+      "portType" : "udp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "5"
+  },
+  "crawler" : {
+    "name":"HEALTH_SNMPV3",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_ssh/115115104

@@ -0,0 +1,23 @@
+{
+  "id" : "115115104",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "22",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_SSH",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_telnet/116101108110101116

@@ -0,0 +1,23 @@
+{
+  "id" : "116101108110101116",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.215",
+      "port" : "23",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_TELNET",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/config/container/network_proxy/health_wmi/119109105

@@ -0,0 +1,23 @@
+{
+  "id" : "119109105",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.1",
+      "port" : "135",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "7"
+  },
+  "crawler" : {
+    "name":"HEALTH_WMI",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

config_manager/test_agent/container/java_proxy/start

@@ -0,0 +1,3 @@
+#!/bin/bash
+script_dir=$(cd "$(dirname "$0")" && pwd)
+java -jar $script_dir'/grpc_java.jar' $1

config_manager/test_agent/container/network_proxy/start

@@ -0,0 +1,4 @@
+#!/bin/bash
+script_dir=$(cd "$(dirname "$0")" && pwd)
+$script_dir'/ncr' -Port=$1
+

config_manager/test_agent/global.yaml

@@ -0,0 +1,11 @@
+central:
+  address: "http://localhost:9090"
+  port: 443
+logPath: "./bin/log.xml"
+paths:
+  rootFolder : "/home/insanity/Develop/gopath/src/loafle.com/overflow/config_manager_go/test_agent/"
+  configFolder : "config/"
+  binaryFolder : "container/"
+  pidFolder : "pids/"
+  scriptFile : "start"
+intervalSecond: 10

config_manager/test_agent/test.json

@@ -0,0 +1,50 @@
+{
+  "id" : "123980918237",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.1",
+      "port" : "389",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+      "url":"jdbc:sqlserver://192.168.1.106:1433;",
+      "id":"sa",
+      "pw":"qwe123",
+      "query" : "select * from master.dbo.sysprocesses"
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"health_activedirectory",
+    "container":"network_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {
+          "metric" : "object[$0].db[$1].datafile_size",
+          "key" : "Data File(s) Size (KB)"
+        },
+        {
+          "metric" : "object[$0].db[$1].logfile_size",
+          "key" : "Log File(s) Size (KB)"
+        }
+      ],
+      "queryInfo" : {
+        "query": "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'",
+        "extend" : {
+            "test":"test"
+        }
+      },
+      "mappingInfo" : {
+        "parseDirection" : "row",
+        "arrayColumns" : [ "object_name","instance_name"],
+        "keyColumns" : ["counter_name"],
+        "valueColumn" : "cntr_value"
+      }
+    }
+  ]
+}

crawler_manager/config/example.json

@@ -0,0 +1,23 @@
+{
+  "id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"HEALTH_REDIS",
+    "container":"network_crawler"
+  },
+  "items" : [
+  ]
+}

crawler_manager/crawler_communicator.go

@@ -0,0 +1,196 @@
+package crawler_manager
+
+import (
+	"context"
+	g "git.loafle.net/overflow/crawler_go/grpc"
+	"google.golang.org/grpc"
+	"log"
+
+	"encoding/json"
+	"git.loafle.net/overflow/agent_api/config_manager"
+)
+
+func callAdd(container *string, conf *config_manager.Config) bool {
+
+	port := GetInstance().portMap[*container]
+
+	conn, err := grpc.Dial(address+port, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+		return false
+	}
+	defer conn.Close()
+
+	cc := g.NewConfigClient(conn)
+
+	in := &g.InputAdd{}
+
+	b, err := json.Marshal(conf)
+	if err != nil {
+		return false
+	}
+
+	in.Data = b
+	in.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name])
+
+	out, err := cc.Add(context.Background(), in)
+	if err != nil {
+		log.Println(err)
+	}
+	log.Println("callAdd:", out)
+
+	return true
+
+}
+
+func callInitConfigOne(container *string, conf *config_manager.Config) bool {
+
+	var cl []*config_manager.Config
+
+	cl = append(cl, conf)
+
+	return callInitConfig(container, cl)
+
+}
+
+func callInitConfig(container *string, cl []*config_manager.Config) bool {
+
+	port := GetInstance().portMap[*container]
+
+	conn, err := grpc.Dial(address+port, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+		return false
+	}
+	defer conn.Close()
+
+	cc := g.NewConfigClient(conn)
+
+	inArr := &g.InputArray{}
+
+	for _, conf := range cl {
+
+		in := &g.Init{}
+		in.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name])
+
+		b, err := json.Marshal(conf)
+		if err != nil {
+			continue
+		}
+
+		in.Data = b
+		inArr.In = append(inArr.In, in)
+	}
+
+	outInit, errInit := cc.Init(context.Background(), inArr)
+	if errInit != nil {
+		log.Println(errInit)
+		return false
+	}
+	log.Println("callInit:", outInit)
+
+	return true
+
+}
+
+//func callInit(container *string, paths *[]string) bool {
+//
+//	port := GetInstance().portMap[*container]
+//
+//	return callInitAddress(address + port, paths)
+//}
+//
+
+//
+//func callInitAddress(address string, paths *[]string) bool {
+//	conn, err := grpc.Dial(address, grpc.WithInsecure())
+//	if err != nil {
+//		log.Fatalf("did not connect: %v", err)
+//		return false
+//	}
+//	defer conn.Close()
+//
+//	cc := g.NewConfigClient(conn)
+//
+//	inArr := &g.InputArray{}
+//	base := "HEALTH_"
+//	for _, path := range *paths {
+//
+//		in := &g.Init{}
+//		//in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
+//		in.Path = path + "/"
+//		bcn := filepath.Base(path)
+//		bcn = strings.ToUpper(bcn)
+//
+//		in.Name = g.Crawlers(g.Crawlers_value[base+bcn])
+//		//in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test
+//		inArr.In = append(inArr.In, in)
+//	}
+//
+//	outInit, errInit := cc.Init(context.Background(), inArr)
+//	if errInit != nil {
+//		log.Println(errInit)
+//		return false
+//	}
+//	log.Println("callInit:",outInit)
+//
+//	return true
+//}
+
+func callRemove(container *string, conf *config_manager.Config) {
+
+	port := GetInstance().portMap[*container]
+
+	conn, err := grpc.Dial(address+port, grpc.WithInsecure())
+
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer conn.Close()
+
+	cc := g.NewConfigClient(conn)
+
+	inR := &g.Input{}
+	//inR.Id = *id  //FIXME
+	inR.Name = g.Crawlers(g.Crawlers_value[conf.Crawler.Name])
+
+	outRem, errRem := cc.Remove(context.Background(), inR)
+	if errRem != nil {
+		log.Println(errRem)
+	}
+	log.Println("callRemove:", outRem)
+
+}
+
+func callStatus(container *string) bool {
+
+	port := GetInstance().portMap[*container]
+
+	if port == "" {
+		return false
+	}
+
+	return callStatusAddress(address + port)
+
+}
+
+func callStatusAddress(addr string) bool {
+
+	conn, err := grpc.Dial(addr, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+	}
+	defer conn.Close()
+
+	c := g.NewStatusClient(conn)
+
+	e := &g.Empty{}
+	out, err := c.Status(context.Background(), e)
+
+	if err != nil {
+		log.Fatalf("could not greet: %v", err)
+	}
+
+	return out.Check
+
+}

crawler_manager/crawler_manager.go

@@ -0,0 +1,327 @@
+package crawler_manager
+
+import (
+	//"git.loafle.net/overflow/crawler_go/config"
+	//"encoding/json"
+	"google.golang.org/grpc"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strconv"
+	"time"
+
+	"errors"
+	"git.loafle.net/overflow/agent_api/config_manager"
+)
+
+const (
+	address      = "localhost:"
+	portArgsName = "-Port="
+	defaultPort  = 50000
+
+	//rootFolder = "/home/cm2/"
+	//ConfigFolder = rootFolder + "/config/container/"
+	//BinaryFolder = rootFolder + "/container/"
+	//PidFolder = rootFolder + "/pids/"
+	//runFile = "ttnc"
+)
+
+var g_CrawlerMananger *CrawlerManager = nil
+
+type CrawlerManager struct {
+	currentPort int
+	portMap     map[string]string
+	pidMap      map[string]string
+	ConfigMgr   config_manager.ConfigManager
+}
+
+func init() {
+
+	GetInstance()
+
+	listenEvent()
+
+	//g_CrawlerMananger.init()
+}
+
+func SettingPath() {
+
+	//test
+	//GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder = "/home/cm2/"
+
+	GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder
+	GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder
+	GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder
+
+}
+
+func GetInstance() *CrawlerManager {
+
+	if g_CrawlerMananger == nil {
+		g_CrawlerMananger = &CrawlerManager{portMap: make(map[string]string), pidMap: make(map[string]string), currentPort: defaultPort}
+	}
+
+	return g_CrawlerMananger
+}
+
+func (c *CrawlerManager) GetClient(container string) (*grpc.ClientConn, error) {
+
+	b := c.checkContainer(&container)
+
+	if b == false {
+		err := c.runAndInitContainerOne(&container)
+		if err != nil {
+			return nil, err
+		}
+	}
+	gport := c.portMap[container]
+
+	return grpc.Dial(address+gport, grpc.WithInsecure())
+}
+
+func (c *CrawlerManager) init() error {
+
+	c.checkPid()
+
+	cmap := c.ConfigMgr.GetSensors()
+	scm := sortContainer(cmap)
+
+	for ctn := range scm {
+		err := c.runAndInitContainer(&ctn, scm[ctn])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *CrawlerManager) checkContainer(container *string) bool {
+	return callStatus(container)
+}
+
+func (c *CrawlerManager) checkAndRunContainer(container *string) bool {
+
+	b := callStatus(container)
+
+	if b == false {
+		err := c.runAndInitContainerOne(container)
+		if err != nil {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c *CrawlerManager) checkPid() {
+
+	files, err := ioutil.ReadDir(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder)
+
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	for _, file := range files {
+
+		if file.IsDir() {
+			continue
+		}
+
+		str := file.Name()
+
+		c.stopProcess(&str)
+		c.removeProcessFile(&str)
+
+	}
+
+}
+
+func (c *CrawlerManager) runAndInitContainerOne(container *string) error {
+
+	err := c.runContainer(container)
+
+	if err != nil {
+		return err
+	}
+
+	cmap := c.ConfigMgr.GetSensors()
+	scm := sortContainer(cmap)
+
+	b := callInitConfig(container, scm[*container])
+
+	if b == false {
+		return errors.New("call init failed")
+	}
+
+	return nil
+
+}
+
+func (c *CrawlerManager) runAndInitContainer(container *string, cl []*config_manager.Config) error {
+
+	err := c.runContainer(container)
+
+	if err != nil {
+		return err
+	}
+
+	b := callInitConfig(container, cl)
+
+	if b == false {
+		return errors.New("call init failed")
+	}
+
+	return nil
+}
+
+func sortContainer(cm map[string]*config_manager.Config) map[string][]*config_manager.Config {
+
+	m := make(map[string][]*config_manager.Config)
+
+	var cn string
+	for key := range cm {
+		cn = cm[key].Crawler.Container
+		m[cn] = append(m[cn], cm[key])
+	}
+
+	return m
+}
+
+func (c *CrawlerManager) runContainer(container *string) error {
+
+	b := c.checkContainer(container)
+	if b {
+		return nil
+	}
+
+	cmdStr := getRunCommand(container)
+
+	for {
+		pArg := strconv.Itoa(c.currentPort)
+		cmd := exec.Command(cmdStr, pArg)
+
+		err := cmd.Start()
+		if err != nil {
+			//run error break;
+			log.Println(err)
+			return err
+		}
+
+		time.Sleep(time.Duration(time.Second * 2))
+
+		b := callStatusAddress(address + strconv.Itoa(c.currentPort))
+
+		if b == false {
+			log.Println("false " + strconv.Itoa(c.currentPort))
+			c.currentPort++
+			continue
+		}
+		log.Println(*container+" run success port:", c.currentPort, "pid:", cmd.Process.Pid)
+
+		writePid(cmd.Process.Pid)
+
+		c.portMap[*container] = strconv.Itoa(c.currentPort)
+		c.pidMap[*container] = strconv.Itoa(cmd.Process.Pid)
+
+		c.currentPort++
+
+		log.Println(*container + "started")
+
+		break
+	}
+
+	return nil
+}
+
+func (c *CrawlerManager) addSensor(id string) error {
+
+	conf := c.ConfigMgr.GetSensorById(id)
+
+	b := c.checkAndRunContainer(&conf.Crawler.Name)
+
+	if b == false {
+		return errors.New("run container error")
+	}
+
+	b = callAdd(&conf.Crawler.Container, conf)
+
+	if b == false {
+		return errors.New("Call Add Fail")
+	}
+
+	return nil
+}
+
+func (c *CrawlerManager) removeSensor(id string) {
+
+	conf := c.ConfigMgr.GetSensorById(id)
+
+	callRemove(&conf.Crawler.Name, conf)
+
+	//remove and stop
+
+}
+
+func (c *CrawlerManager) updateSensor(id string) error {
+
+	conf := c.ConfigMgr.GetSensorById(id)
+
+	b := callInitConfigOne(&conf.Crawler.Container, conf)
+
+	if b == false {
+		return errors.New("update sensor error")
+	}
+
+	return nil
+
+}
+
+func (c *CrawlerManager) stopContainerAll() {
+
+	for k, _ := range c.pidMap {
+		c.stopContainer(&k)
+	}
+
+}
+
+func (c *CrawlerManager) stopContainer(container *string) {
+
+	pid := c.pidMap[*container]
+
+	if len(pid) <= 0 {
+		return
+	}
+
+	c.stopProcess(&pid)
+	c.removeProcessFile(&pid)
+
+	delete(c.pidMap, *container)
+	delete(c.portMap, *container)
+
+}
+
+func (c *CrawlerManager) stopProcess(pid *string) {
+
+	pidi, err := strconv.Atoi(*pid)
+
+	if err != nil {
+		log.Println(err)
+	}
+
+	p, err := os.FindProcess(pidi)
+	if err != nil {
+		log.Println(err)
+	}
+	p.Kill()
+
+}
+
+func (c *CrawlerManager) removeProcessFile(pid *string) {
+	err := os.Remove(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder + "/" + *pid)
+	if err != nil {
+		log.Println(err)
+	}
+}

crawler_manager/crawler_manager_event.go

@@ -0,0 +1,195 @@
+package crawler_manager
+
+import (
+	//ooo "git.loafle.net/overflow/agent_api/observer"
+	//"git.loafle.net/overflow/agent_api/config_manager"
+	//"git.loafle.net/overflow/agent_api/messages"
+	//
+	//"fmt"
+	"git.loafle.net/overflow/agent_api/config_manager"
+)
+
+func Start(res chan bool, conMgr config_manager.ConfigManager) {
+
+	GetInstance().ConfigMgr = conMgr
+
+	SettingPath()
+
+	err := GetInstance().init()
+	if err != nil {
+		res <- false
+		return
+	}
+
+	res <- true
+}
+
+func Stop(res chan bool) {
+
+	GetInstance().stopContainerAll()
+
+	res <- true
+}
+
+func AddSensor(id string) error {
+
+	err := GetInstance().addSensor(id)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func RemoveSensor(id string) {
+
+	GetInstance().removeSensor(id)
+
+	//FIXME:: error return
+
+}
+
+func UpdateSensor(id string) error {
+
+	err := GetInstance().updateSensor(id)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func UpdateCRM() {
+
+	//FIXME:: update crawler
+
+}
+
+func listenEvent() {
+
+	//go listenConfigLoaded();
+	//go listenAgentStop()
+	//
+	//go listenAddSensor();
+	//go listenRemoveSensor();
+	//go listenUpdateSensor();
+	//
+	//go listenUpdateCrawler()
+}
+
+//
+//func listenConfigLoaded() {
+//
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.CFG_LOADED, ch)
+//
+//	o := <-ch
+//
+//	cm := o.(config_manager.ConfigManager)
+//
+//	GetInstance().ConfigMgr = cm;
+//
+//	SettingPath()
+//
+//	err := GetInstance().init();
+//	if err != nil {
+//		//FIXME:: noti err
+//		return;
+//	}
+//
+//
+//	observer.Notify(messages.CRM_READY, cm)
+//
+//}
+//
+//func listenAgentStop() {
+//
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.CLT_STOPPED, ch)
+//
+//	o := <-ch
+//
+//	fmt.Println(o)
+//
+//	GetInstance().stopContainerAll()
+//
+//	observer.Notify(messages.CRM_STOPPED, nil)
+//
+//
+//}
+//
+//func listenAddSensor() {
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.SCF_SENSOR_ADD_DONE, ch);
+//
+//	o := <-ch
+//
+//	str := o.(string)
+//
+//	fmt.Println(str)
+//
+//	err := GetInstance().addSensor(str)
+//
+//	if err != nil {
+//		//FIXME:: noti err
+//		return
+//	}
+//
+//	observer.Notify(messages.CRM_SENSOR_ADD_DONE, nil)
+//
+//}
+//
+//func listenRemoveSensor() {
+//
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.CLT_SENSOR_REMOVE_DONE, ch);
+//
+//	o := <-ch
+//
+//	str := o.(string)
+//
+//	GetInstance().removeSensor(str)
+//
+//
+//	observer.Notify(messages.CRM_SENSOR_REMOVE_DONE, nil)
+//
+//}
+//
+//func listenUpdateSensor() {
+//
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.SCF_SENSOR_UPDATE_DONE, ch);
+//
+//	o := <-ch
+//
+//	container := o.(string)
+//
+//	err := GetInstance().runAndInitContainerOne(&container)
+//
+//	if err != nil {
+//		//FIXME:: noti err
+//		return
+//	}
+//
+//	observer.Notify(messages.CRM_SENSOR_UPDATE_DONE, nil)
+//
+//}
+//
+//func listenUpdateCrawler() {
+//
+//	ch := make(chan interface{}, 0)
+//	observer.Add(messages.CLT_CRM_UPDATE_DONE, ch);
+//
+//	o := <-ch
+//
+//	fmt.Println(o)
+//	//FIXME::update crawler
+//
+//
+//	observer.Notify(messages.CRM_UPDATE_DONE, nil)
+//
+//}
+//

crawler_manager/crawler_manager_test.go

@@ -0,0 +1,409 @@
+package crawler_manager
+
+import (
+	"testing"
+	//grpc1 "google.golang.org/grpc"
+	"os"
+
+	"os/exec"
+
+	"path/filepath"
+
+	"io/ioutil"
+	"strings"
+
+	"encoding/json"
+	"fmt"
+	"git.loafle.net/overflow/agent_api/config_manager"
+)
+
+func TestCallInit(t *testing.T) {
+	//CallInit("")
+}
+
+func TestPid(t *testing.T) {
+
+	pp, err := os.FindProcess(12314)
+	if err != nil {
+		t.Log("err : ", err)
+	}
+
+	t.Log(pp.Pid)
+
+}
+
+func TestPr(t *testing.T) {
+	//ps -aux | awk '{print $2}' | grep 15538
+	bytes, _ := exec.Command("ps", "-aux", "awk", "{print $2}").Output()
+
+	t.Log(string(bytes))
+
+}
+
+func TestExe(t *testing.T) {
+	ps := exec.Command("ps", "-aux")
+	awk := exec.Command("awk", "{print $2}")
+	grep := exec.Command("grep", "22373")
+
+	awk.Stdin, _ = ps.StdoutPipe()
+	grep.Stdin, _ = awk.StdoutPipe()
+
+	ps.Start()
+	awk.Start()
+
+	byt, _ := grep.Output()
+
+	t.Log(len(byt))
+}
+
+func TestExeState(t *testing.T) {
+	ps := exec.Command("ps", "-aux")
+	awk := exec.Command("awk", "{print $2, $7}")
+	grep := exec.Command("grep", "22373")
+	awk2 := exec.Command("awk", "{print $2}")
+
+	awk.Stdin, _ = ps.StdoutPipe()
+	grep.Stdin, _ = awk.StdoutPipe()
+	awk2.Stdin, _ = grep.StdoutPipe()
+
+	ps.Start()
+	awk.Start()
+	grep.Start()
+
+	byt, _ := awk2.Output()
+
+	t.Log(len(byt))
+	t.Log(string(byt))
+	t.Log(len(strings.TrimSpace(string(byt))))
+}
+
+func TestPipe(t *testing.T) {
+	c1 := exec.Command("ls")
+	c2 := exec.Command("wc", "-l")
+	c2.Stdin, _ = c1.StdoutPipe()
+	//c2.Stdout = os.Stdout
+
+	_ = c1.Start()
+	aa, _ := c2.Output()
+
+	t.Log(string(aa))
+}
+
+func TestCom(t *testing.T) {
+	str := "Z"
+
+	if str == "Z" {
+		t.Log("aaa")
+	}
+}
+
+func TestReadConfig(t *testing.T) {
+
+	//c := ReadConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json")
+	//t.Log(c)
+
+}
+
+func TestDir(t *testing.T) {
+	a := filepath.Dir("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json")
+
+	t.Log(a)
+}
+
+func TestMyPid(t *testing.T) {
+	t.Log(os.Getpid())
+}
+
+func TestDir22(t *testing.T) {
+
+	//cs := IsStartContainer()
+
+	//t.Log(cs)
+
+}
+
+func TestCrateDir(t *testing.T) {
+
+	cp := "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/"
+
+	var configPath []string
+
+	rootFolder := "/home/cm2/"
+	ConfigFolder := rootFolder + "/config/container/"
+	BinaryFolder := rootFolder + "/container/"
+	PidFolder := rootFolder + "/pids/"
+	//runFile := "ttnc"
+
+	configPath = append(configPath, ConfigFolder+"/java/oracle/")
+
+	configPath = append(configPath, ConfigFolder+"/go/test/")
+
+	configPath = append(configPath, ConfigFolder+"/java/mysql/")
+	configPath = append(configPath, ConfigFolder+"/network/http/")
+	configPath = append(configPath, ConfigFolder+"/network/redis/")
+	configPath = append(configPath, ConfigFolder+"/network/ldap/")
+
+	configPath = append(configPath, PidFolder)
+
+	var containerPath []string
+
+	containerPath = append(containerPath, BinaryFolder+"/java/")
+	containerPath = append(containerPath, BinaryFolder+"/go/")
+	containerPath = append(containerPath, BinaryFolder+"/network/")
+
+	b, _ := ioutil.ReadFile(cp + "config/example.json")
+
+	for i, p := range configPath {
+		pp := filepath.Join(p)
+		os.MkdirAll(pp, os.ModePerm)
+
+		if i%2 == 0 {
+			ioutil.WriteFile(pp+"/id.json", b, os.ModePerm)
+		}
+
+	}
+
+	bbb, _ := ioutil.ReadFile(cp + "config/ttnc")
+
+	for _, p := range containerPath {
+		pp := filepath.Join(p)
+		os.MkdirAll(pp, os.ModePerm)
+
+		pp += "/"
+		pp += "/ttnc"
+		ioutil.WriteFile(pp, bbb, os.ModePerm)
+
+	}
+
+}
+
+func TestInit(t *testing.T) {
+	//InitContainer()
+}
+
+func TestDirs(t *testing.T) {
+
+	//cs := IsStartContainer()
+	//
+	//for _, c := range cs {
+	//	var ccl []string
+	//	ExistConfigFileDir(ConfigFolder, c, &ccl)
+	//	t.Log(ccl)
+	//
+	//}
+
+}
+
+func TestSplitPath(t *testing.T) {
+
+	ff := "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff"
+	a := filepath.Dir(ff)
+	t.Log(a)
+	a = filepath.Clean(a)
+	t.Log(a)
+	a = filepath.Base(a)
+
+	t.Log(a)
+
+}
+
+func TestInitCM(t *testing.T) {
+	GetInstance().init()
+
+	GetInstance().stopContainerAll()
+}
+
+func TestCallStatus(t *testing.T) {
+	//c := "java"
+	//GetInstance().runAndInitContainer(&c)
+	//
+	//callStatus(&c)
+	//
+	//GetInstance().stopContainer(&c)
+}
+
+func TestCallAdd(t *testing.T) {
+	////
+	//c := "java"
+	//cn := "HEALTH_"+"MYSQL"
+	//id := "id.json"
+	//
+	//GetInstance().runAndInitContainer(&c)
+	//
+	//callAdd(&c, &cn, &id)
+	//
+	//GetInstance().stopContainer(&c)
+
+}
+
+func TestCallRemove22(t *testing.T) {
+
+	//c := "java"
+	//cn := "HEALTH_"+"MYSQL"
+	//id := "id.json"
+	//
+	//GetInstance().runAndInitContainer(&c)
+	//
+	//callRemove(&c, &cn, &id)
+	//
+	//GetInstance().stopContainer(&c)
+
+}
+
+func TestRunRun(t *testing.T) {
+	c := "java"
+	GetInstance().runContainer(&c)
+	GetInstance().runContainer(&c)
+	GetInstance().runContainer(&c)
+
+	GetInstance().stopContainer(&c)
+
+}
+
+func TestActiveCrawler(t *testing.T) {
+
+	//c := "java"
+	//GetInstance().runContainer(&c)
+	//
+	//aa := GetInstance().activeCrawler(&c)
+	//
+	//t.Log(aa)
+	//
+	//GetInstance().stopContainer(&c)
+
+}
+
+func TestErer(t *testing.T) {
+
+	var asdf []string
+
+	asdf = nil
+
+	t.Log(len(asdf))
+
+}
+
+//
+//func TestStartContainer(t *testing.T) {
+//
+//	cs := GetStartContainer()
+//
+//	t.Log(cs)
+//
+//}
+
+//
+//func GetStartContainer() []string {
+//
+//	files, _ := ioutil.ReadDir(ConfigFolder)
+//
+//	var cs []string
+//
+//	for _,file := range files {
+//
+//		if file.IsDir() {
+//			b := existConfigFile(ConfigFolder, file.Name())
+//			if b {
+//				cs = append(cs, file.Name())
+//			}
+//		}
+//
+//	}
+//	return cs
+//}
+
+func TestJson(t *testing.T) {
+
+	c := config_manager.Config{}
+
+	c.Crawler.Name = "wmi_crawler"
+	c.Crawler.Container = "java_proxy"
+
+	c.Id = "WMI_TEST_ID_001"
+
+	b, _ := json.Marshal(c)
+
+	fmt.Println(string(b))
+
+}
+
+func TestJsonMa(t *testing.T) {
+
+	b, _ := ioutil.ReadFile("./test/example.json")
+
+	c := config_manager.Config{}
+
+	json.Unmarshal(b, &c)
+
+	t.Log(c)
+
+}
+
+func TestMock(t *testing.T) {
+
+	//GetInstance().ConfigMgr = &ConfigManagerTest{}
+	//
+	//GetInstance().init()
+
+}
+
+func TestRPC(t *testing.T) {
+
+	//c := conMgr.NewConfigManager()
+	//c.LoadGlobalConfigTTT("/home/snoop/develop/path/go/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	//c.LoadCrawlerConfigAllTTT()
+	//
+	//observer.Notify(messages.CONFIGMANAGER_LOADED,c)
+	//
+	//
+	//wg := sync.WaitGroup{}
+	//
+	//wg.Add(1)
+	//
+	//go func() {
+	//	time.Sleep(20 * time.Second)
+	//	wg.Done()
+	//}()
+	//
+	//
+	//wg.Wait()
+
+}
+
+func TestGRPC(t *testing.T) {
+	b := callStatusAddress("localhost:50052")
+
+	t.Log(b)
+}
+
+type ConfigManagerTest struct {
+}
+
+func (c *ConfigManagerTest) GetGlobalConfig() *config_manager.GlobalConfig {
+
+	g := config_manager.GlobalConfig{}
+
+	rootFolder := "/home/cm2/"
+	g.Paths.ConfigFolder = rootFolder + "/config/container/"
+	g.Paths.BinaryFolder = rootFolder + "/container/"
+	g.Paths.PidFolder = rootFolder + "/pids/"
+
+	return &g
+}
+func (c *ConfigManagerTest) GetCrawlerById(id string) *config_manager.Config {
+	return nil
+}
+func (c *ConfigManagerTest) GetCrawlers() map[string]*config_manager.Config {
+
+	var m map[string]*config_manager.Config
+
+	b, _ := ioutil.ReadFile("./test/example.json")
+
+	ccc := config_manager.Config{}
+
+	json.Unmarshal(b, &ccc)
+
+	m["wmi_crawler"] = &ccc
+
+	return m
+}

crawler_manager/crawler_util.go

@@ -0,0 +1,58 @@
+package crawler_manager
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+)
+
+func existConfigFile(prePath string, dir string) bool {
+
+	files, _ := ioutil.ReadDir(prePath + "/" + dir)
+
+	for _, file := range files {
+
+		if file.IsDir() {
+			retB := existConfigFile(prePath+"/"+dir, file.Name())
+			if retB {
+				return true
+			}
+		} else {
+			return true
+		}
+
+	}
+	return false
+}
+
+func existConfigFileDir(prePath string, dir string, configCrawler *[]string) {
+
+	files, _ := ioutil.ReadDir(prePath + "/" + dir)
+
+	for _, file := range files {
+
+		if file.IsDir() {
+			existConfigFileDir(prePath+"/"+dir, file.Name(), configCrawler)
+		} else {
+			*configCrawler = append(*configCrawler, prePath+"/"+dir)
+			return
+		}
+
+	}
+}
+
+func getRunCommand(container *string) string {
+	return GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder + "/" + *container + "/" + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ScriptFile
+}
+
+func writePid(pid int) {
+	ioutil.WriteFile(GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder+strconv.Itoa(pid), []byte(""), os.ModePerm)
+}
+
+func getConfigPaths(container *string) *[]string {
+
+	var dirs []string
+	existConfigFileDir(GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder, *container, &dirs)
+
+	return &dirs
+}

data_sender/data_sender.go

@@ -0,0 +1,187 @@
+package data_sender_go
+
+import (
+	"encoding/json"
+	cm "git.loafle.net/overflow/agent_api/config_manager"
+	"git.loafle.net/overflow/agent_api/messages"
+	pb "git.loafle.net/overflow/crawler_go/grpc"
+	q "git.loafle.net/overflow/queue_go"
+	"io/ioutil"
+	"log"
+	"os"
+	"sync"
+)
+
+const (
+	FILE_PATH        = "/overflow/tmp/data.tmp"
+	DEFAULT_INTERVAL = 10
+)
+
+var (
+	instance *DataSender
+	once     sync.Once
+)
+
+type DataSender struct {
+	once  sync.Once
+	queue *q.LoafleQueue
+	gconf *cm.GlobalConfig
+}
+
+func Start(ch chan bool, conf *cm.GlobalConfig) {
+	d := GetInstance()
+	d.start(conf)
+	ch <- true
+}
+
+func Stop(ch chan bool) {
+	GetInstance().stop()
+	ch <- true
+}
+
+func GetInstance() *DataSender {
+	once.Do(func() {
+		instance = &DataSender{}
+	})
+	return instance
+}
+
+func (ds *DataSender) start(conf *cm.GlobalConfig) {
+	qc := make(chan interface{})
+	ds.queue = q.NewQueue(DEFAULT_INTERVAL, qc)
+	ds.gconf = conf
+	go ds.handleData(qc)
+}
+
+func (ds *DataSender) stop() {}
+
+func (ds *DataSender) handleData(qc chan interface{}) {
+	for {
+		select {
+		case items := <-qc:
+			result := make([]*messages.Data, 0)
+			for _, item := range items.([]*q.Item) {
+				collectedData := item.Value.(*pb.Output)
+				d := &messages.Data{}
+				d.Data = collectedData.Data
+				d.AgentId = agentIdentifier()
+				result = append(result, d)
+			}
+			ds.send(result)
+		}
+	}
+}
+
+func AddData(data interface{}) {
+	ds := GetInstance()
+	ds.queue.PushItem(data)
+}
+
+func (ds *DataSender) send(data []*messages.Data) {
+	//for _, v := range data {
+	//	//log.Printf("SEND SENSOR RESULT : %s - %s", v.SensorId, v.Data)
+	//}
+
+	//ds.addFailedData(data)
+	//addr := ds.gconf.Central.Address + ":" + string(ds.gconf.Central.Port)
+	//conn, err := grpc.Dial(addr, grpc.WithInsecure())
+	//if err != nil {
+	//	ds.saveFailedData(data)
+	//	return
+	//}
+	//defer conn.Close()
+
+	//temporary
+	//client := pb.NewStatusClient(conn)
+	//out, err := client.Status(context.Background(), &pb.Empty{})
+	//if err != nil {
+	//	ds.saveFailedData(data)
+	//	return
+	//}
+	//ds.removeFailed()
+	//log.Print(out)
+}
+
+func (ds *DataSender) addFailedData(data []*messages.Data) {
+	bytes := ds.getFailedData()
+	if bytes != nil {
+		failed := messages.Data{}
+		err := json.Unmarshal(bytes, &failed)
+		if err != nil {
+			log.Println(err)
+		}
+		data = append([]*messages.Data{&failed}, data...) //prepend
+	}
+}
+
+func (ds *DataSender) getFailedData() []byte {
+	b, err := ioutil.ReadFile(FILE_PATH)
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+	return b
+}
+
+func (ds *DataSender) removeFailed() {
+	err := os.Remove(FILE_PATH)
+	if err != nil {
+		log.Println(err)
+	}
+}
+
+func (ds *DataSender) saveFailedData(datas []*messages.Data) {
+
+	file, err := tempFile()
+	if err != nil {
+		log.Println(err)
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Println(err)
+			return
+		}
+	}()
+
+	for _, data := range datas {
+		bytes, err := json.Marshal(&data)
+		if err != nil {
+			log.Println(err)
+		}
+		if bytes != nil {
+			log.Println("write : ", string(bytes))
+			_, err = file.Write(bytes)
+			if err != nil {
+				log.Println(err)
+			}
+		}
+	}
+}
+
+func tempFile() (*os.File, error) {
+
+	var file *os.File
+	var fileInfo os.FileInfo
+	var err error
+	if fileInfo, err = os.Stat(FILE_PATH); err != nil {
+		if os.IsNotExist(err) {
+			file, err = os.Create(FILE_PATH)
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		if fileInfo != nil {
+			file, err = os.OpenFile(FILE_PATH, os.O_RDWR|os.O_APPEND, 0660)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return file, nil
+}
+
+func agentIdentifier() string {
+	return "agentID_000000001"
+}

data_sender/data_sender_test.go

@@ -0,0 +1,64 @@
+package data_sender_go
+
+import (
+	"git.loafle.net/overflow/agent_api/observer"
+	"strconv"
+	"testing"
+	"time"
+)
+
+type Result struct {
+	Data []byte
+}
+
+func TestTotal(t *testing.T) {
+
+	time.Sleep(time.Second * 5)
+
+	observer.Notify("CONFIGMANAGER_LOADED", nil)
+
+	time.Sleep(time.Second * 5)
+
+	for i := 0; i < 20; i++ {
+		testNotify(strconv.Itoa(i))
+	}
+
+	time.Sleep(time.Second * 12)
+	for i := 20; i < 30; i++ {
+		testNotify(strconv.Itoa(i))
+	}
+
+	time.Sleep(time.Second * 100)
+}
+
+func TestSend(t *testing.T) {
+
+	ds := &DataSender{}
+	ds.start()
+
+	for i := 0; i < 20; i++ {
+		testNotify(strconv.Itoa(i))
+	}
+
+	time.Sleep(time.Second * 12)
+	for i := 20; i < 30; i++ {
+		testNotify(strconv.Itoa(i))
+	}
+
+	time.Sleep(time.Second * 100)
+}
+
+func testNotify(val string) {
+
+	result := make(map[string]string)
+	result["a"] = val
+	result["b"] = val
+	result["c"] = val
+
+	cd := &Data{
+		SensorId: "insanity",
+		Data:     result,
+	}
+
+	observer.Notify(messages.QUEUE_DATA, cd)
+}

discovery/.gitignore

@@ -0,0 +1,21 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Go template
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+.idea/
+*.iml
+
+bin/logs

discovery/bin/bridge_conf.xml

@@ -0,0 +1,6 @@
+<conf version="1.0.0">
+    <url>http://localhost:8080/discoveryHistory</url>
+    <user>sa</user>
+    <password>qwe123</password>
+    <bodyType>application/json</bodyType>
+</conf>

discovery/bin/collector.yaml

@@ -0,0 +1,4 @@
+central:
+  address: "http://localhost:9090"
+  port: 443
+log_path: "./bin/log.xml"

discovery/bin/log.xml

@@ -0,0 +1,33 @@
+<seelog type="asynctimer" asyncinterval="1000">
+    <outputs>
+
+        <!-- Exclude global log settings -->
+        <!--<exceptions>-->
+            <!--<exception funcpattern="*seelog.main.test1*" minlevel="info"/>-->
+            <!--<exception funcpattern="*seelog.main.test2*" minlevel="debug"/>-->
+            <!--<exception filepattern="*main.go" minlevel="debug"/>-->
+        <!--</exceptions>-->
+
+        <filter levels="trace, debug">
+            <console formatid="dtLevelFmt" />
+
+        </filter>
+        <filter levels="info, warn">
+            <console formatid="iwLevelFmt" />
+
+        </filter>
+        <filter levels="error, critical">
+            <console formatid="ecLevelFmt" />
+
+        </filter>
+        <!--<console formatid="basic" />-->
+        <rollingfile formatid="basic" type="date" filename="./bin/logs/log" maxrolls="5" archiveexploded="true"
+                     datepattern="02.01.2006" archivetype="zip" fullname="true" archivepath="./bin/logs/archives/"/>
+    </outputs>
+    <formats>
+        <format id="basic" format="%Date/%Time [%LEV] %File:%Line %Func()-%Msg%n"/>
+        <format id="dtLevelFmt" format="%EscM(34)%Date/%Time [%LEV] %File:%Line %Func()-%Msg%n"/>
+        <format id="iwLevelFmt" format="%EscM(35)%Date/%Time [%LEV] %File:%Line %Func()-%Msg%n"/>
+        <format id="ecLevelFmt" format="%EscM(31)%Date/%Time [%LEV] %File:%Line %Func()-%Msg%n"/>
+    </formats>
+</seelog>

discovery/bin/scripts/ubuntu/of_collector

@@ -0,0 +1,71 @@
+### BEGIN INIT INFO
+# Provides: overFlow
+# Required-Start: $network
+# Required-Stop: $network
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: of-collector agent
+# Description: overFlow Collector Agent
+### END INIT INFO
+
+PROG="ofcollector"
+PROG_PATH="/home/insanity/develop/overflow/overflow.collector/bin/"
+PID_PATH="/var/run/"
+FILE_SERVER=/var/run/of_server
+
+start() {
+    if [ -e "$PID_PATH/$PROG.pid" ]; then
+        ## Program is running, exit with error.
+        echo "$PROG is currently running!" 1>&2
+        exit 1
+    else
+        $PROG_PATH/$PROG 2>&1 > /dev/null &
+        echo "$PROG started"
+        touch "$PID_PATH/$PROG.pid"
+    fi
+}
+
+stop() {
+    sudo echo -n 'STOP' | netcat -U $FILE_SERVER
+
+    if [ -e "$PID_PATH/$PROG.pid" ]; then
+        ## Program is running, so stop it
+        #killall $PROG
+        rm "$PID_PATH/$PROG.pid"
+        echo "$PROG stopped"
+    else
+        ## Program is not running, exit with error.
+        echo "Error! $PROG not started!" 1>&2
+        exit 1
+    fi
+}
+
+status() {
+    sudo echo -n $1 | netcat -U $FILE_SERVER
+}
+
+
+if [ "$(id -u)" != "0" ]; then
+    echo "This script must be run as root" 1>&2
+    exit 1
+fi
+
+case "$1" in
+    start)
+        start
+    ;;
+    stop)
+        stop
+    ;;
+    reload|restart|force-reload)
+        stop
+        start
+    ;;
+    status)
+        status 'STATUS'
+   ;;
+    **)
+        echo "Usage: $0 {start|stop|restart|status}" 1>&2
+        exit 1
+    ;;
+esac

discovery/bootstrap/bootstrap.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"git.loafle.net/overflow/discovery"
+	"git.loafle.net/overflow/discovery/conf"
+	log "github.com/cihub/seelog"
+	"os"
+	"runtime"
+)
+
+func init() {
+	paramA := flag.String("conf", "/overflow/collector", "Config file path")
+	//paramB := flag.Int("b", 1, "int type param")
+	//paramC := flag.Bool("c", false, "bool type param")
+	flag.Parse()
+	//fmt.Println("Args: ", *paramA, *paramB, *paramC)
+	fmt.Println("Args: ", *paramA)
+	//if len(os.Args) < /* paramcount */ {
+	//  flag.Usage()
+	//  os.Exit(1)
+	//}
+
+	if err := conf.LoadConfig(*paramA); err != nil {
+		fmt.Printf("Error: %s\n", err)
+		os.Exit(1)
+	}
+}
+
+func main() {
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	defer log.Flush()
+
+	stop := make(chan bool, 1)
+	go handleShell(stop)
+	handleSignal(stop)
+
+	collector.Start()
+
+	if <-stop {
+		// Comes from shell cmd 'stop' or quit signals
+		stopHandleShell()
+		collector.Stop()
+		close(stop)
+		log.Flush()
+		os.Exit(0)
+	}
+}

discovery/bootstrap/shell.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	log "github.com/cihub/seelog"
+	"net"
+	"os"
+	"strings"
+)
+
+const FILE_SERVER string = "/var/run/of_server"
+
+var fd net.Conn
+
+func handleShell(stop chan bool) {
+
+	//os.Stdout.Close()
+	//os.Stderr.Close()
+	//os.Stdin.Close()
+
+	if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+		os.Remove(FILE_SERVER)
+	}
+
+	l, err := net.ListenUnix("unix", &net.UnixAddr{Name: FILE_SERVER, Net: "unix"})
+	if err != nil {
+		log.Error(err.Error())
+		return
+	}
+	defer func() {
+		l.Close()
+		os.Remove(FILE_SERVER)
+	}()
+
+	for {
+		fd, err = l.Accept()
+		if err != nil {
+			log.Error(err)
+			return
+		}
+
+		buf := make([]byte, 1024)
+		nr, err := fd.Read(buf)
+		if err != nil {
+			return
+		}
+
+		data := string(buf[0:nr])
+		switch strings.ToUpper(data) {
+		case "STOP":
+			if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+				os.Remove(FILE_SERVER)
+			}
+			stop <- true
+		case "STATUS":
+			fd.Write(status())
+		default:
+		}
+
+		fd.Close()
+	}
+}
+
+func stopHandleShell() {
+	log.Info("Shell Handler stopped.")
+
+	if fi, _ := os.Stat(FILE_SERVER); fi != nil {
+		os.Remove(FILE_SERVER)
+	}
+	if fd != nil {
+		fd.Close()
+	}
+}
+
+func status() []byte {
+	return []byte("STATUS OK\n")
+}

discovery/bootstrap/signal.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	log "github.com/cihub/seelog"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func handleSignal(stop chan bool) {
+	sigs := make(chan os.Signal, 1)
+
+	signal.Notify(sigs,
+		os.Kill,
+		os.Interrupt,
+		syscall.SIGKILL,
+		syscall.SIGSTOP,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGQUIT,
+	)
+
+	go func() {
+		for {
+			s := <-sigs
+			switch s {
+			case os.Kill, os.Interrupt, syscall.SIGSTOP, syscall.SIGKILL, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+				log.Infof("Signal received. [%s]", s)
+				stop <- true
+			default:
+				log.Infof("Signal received. [%s]", s)
+				stop <- true
+			}
+		}
+	}()
+}
+
+/*
+ 1) SIGHUP	 2) SIGINT	 3) SIGQUIT	 4) SIGILL	 5) SIGTRAP
+ 6) SIGABRT	 7) SIGBUS	 8) SIGFPE	 9) SIGKILL	10) SIGUSR1
+11) SIGSEGV	12) SIGUSR2	13) SIGPIPE	14) SIGALRM	15) SIGTERM
+16) SIGSTKFLT	17) SIGCHLD	18) SIGCONT	19) SIGSTOP	20) SIGTSTP
+21) SIGTTIN	22) SIGTTOU	23) SIGURG	24) SIGXCPU	25) SIGXFSZ
+26) SIGVTALRM	27) SIGPROF	28) SIGWINCH	29) SIGIO	30) SIGPWR
+31) SIGSYS	34) SIGRTMIN	35) SIGRTMIN+1	36) SIGRTMIN+2	37) SIGRTMIN+3
+38) SIGRTMIN+4	39) SIGRTMIN+5	40) SIGRTMIN+6	41) SIGRTMIN+7	42) SIGRTMIN+8
+43) SIGRTMIN+9	44) SIGRTMIN+10	45) SIGRTMIN+11	46) SIGRTMIN+12	47) SIGRTMIN+13
+48) SIGRTMIN+14	49) SIGRTMIN+15	50) SIGRTMAX-14	51) SIGRTMAX-13	52) SIGRTMAX-12
+53) SIGRTMAX-11	54) SIGRTMAX-10	55) SIGRTMAX-9	56) SIGRTMAX-8	57) SIGRTMAX-7
+58) SIGRTMAX-6	59) SIGRTMAX-5	60) SIGRTMAX-4	61) SIGRTMAX-3	62) SIGRTMAX-2
+63) SIGRTMAX-1	64) SIGRTMAX
+*/

discovery/collector.go

@@ -0,0 +1,68 @@
+package collector
+
+import (
+	"git.loafle.net/overflow/discovery/discovery"
+	"git.loafle.net/overflow/discovery/discovery/types/timestamp"
+	log "github.com/cihub/seelog"
+	"os/exec"
+	"sync"
+)
+
+type Collector struct {
+	stop chan bool `json:"-"`
+
+	ID          int                 `json:"id,omitempty"`
+	ProductId   string              `json:"productId"`
+	Version     string              `json:"version"`
+	ConfigPath  string              `json:"configPath"`
+	InstallDate timestamp.Timestamp `json:"installDate"`
+	UpdateDate  timestamp.Timestamp `json:"updateDate"`
+}
+
+var coll *Collector
+var once sync.Once
+
+func Start() *Collector {
+
+	once.Do(func() {
+		coll = &Collector{}
+		coll.info()
+	})
+	coll.start()
+
+	return coll
+}
+
+func Stop() {
+	log.Info("Collector has stopped.")
+}
+
+func (c *Collector) start() {
+	go coll.run()
+}
+
+func (c *Collector) run() {
+	log.Info("Collector is now running.")
+
+	fin := make(chan bool, 1)
+	c.stop = make(chan bool)
+
+	discovery.Discover(fin)
+
+	if <-fin {
+		log.Info("Discovery has finished.")
+	}
+}
+
+func (c *Collector) info() {
+
+	uuid, err := exec.Command("uuidgen").Output()
+	if err != nil {
+		log.Error(err)
+	}
+	c.ProductId = string(uuid)
+	c.Version = "1.0"
+	c.ConfigPath = "/root"
+	c.InstallDate = timestamp.Now()
+	c.UpdateDate = timestamp.Now()
+}

discovery/collector_test.go

@@ -0,0 +1,72 @@
+package collector
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"git.loafle.net/overflow/discovery/discovery/types/timestamp"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestTimeStampe(t *testing.T) {
+
+	aa := "1479186033399"
+
+	i, err := strconv.ParseInt(aa, 10, 64)
+	if err != nil {
+		panic(err)
+	}
+	tm := time.Unix(i/1000, 0)
+	fmt.Println(tm)
+
+}
+
+func TestJPAInsert(t *testing.T) {
+
+	c := &Collector{}
+	c.Version = "2.0"
+	c.ConfigPath = "/root"
+	c.LicenseType = LICENSE1
+	c.ProductId = "6666"
+	c.InstallDate = timestamp.Now()
+	c.UpdateDate = timestamp.Now()
+	c.LicenseDueDate = timestamp.Date(2017, 12, 25)
+
+	b, err := json.Marshal(c)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	http.Post("http://localhost:8080/collector", "application/x-spring-data-verbose+json", bytes.NewBuffer(b))
+}
+
+func TestJPASelect(t *testing.T) {
+	//res, err := http.Get("http://localhost:8080/collector/1")
+	res, err := http.Get("http://localhost:8080/collector/p/6666")
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if len(body) <= 0 {
+		t.Error("not found.")
+		return
+	}
+
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	var result Collector
+	json.Unmarshal(body, &result)
+	t.Log(result)
+
+	t.Log("ID : ", result.ID)
+	t.Log("productID : ", result.ProductId)
+	t.Log("version : ", result.Version)
+}

discovery/communicate/communicate.go

@@ -0,0 +1,26 @@
+package communicate
+
+import (
+	"fmt"
+	//"git.loafle.net/overflow/central/client/events"
+)
+
+var _c *communicator = nil
+
+func init() {
+	fmt.Println("init communicator")
+	//_c = NewCommunicator()
+	//_c.start()
+}
+
+func SetRootURL(url string) {
+	_c.RootURL = url
+}
+
+func GetRootURL() string {
+	return _c.RootURL
+}
+
+//func Send(e *events.Event) {
+//	_c.addEvent(e)
+//}

discovery/communicate/communicate_test.go

@@ -0,0 +1,83 @@
+package communicate
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/gin-gonic/gin.v1"
+	"io/ioutil"
+
+	"git.loafle.net/overflow/central/client/events"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestCommunicatorInit(t *testing.T) {
+	assert.NotNil(t, _c)
+}
+
+func makeGin() *gin.Engine {
+	r := gin.New()
+	api := r.Group("/_api")
+	{
+		collector := api.Group("/collector")
+		{
+			event := collector.Group("/event")
+			{
+				{
+					types := event.Group("/status")
+					{
+						types.POST("/:type", func(c *gin.Context) {
+							fmt.Println("called /_api/collector/event/status/:type")
+							var j events.Event
+							c.BindJSON(&j)
+							fmt.Println(j)
+							c.JSON(http.StatusOK, gin.H{"status": "ok"})
+						})
+
+					}
+				}
+			}
+		}
+	}
+	return r
+}
+
+func TestSend(t *testing.T) {
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	}))
+	defer ts.Close()
+	for i := 0; i < 10; i++ {
+		e := events.NewEvent(events.CENTRAL_EVENT, events.CollectorInstallEvent{Version: "Test"})
+		Send(e)
+		//		t.Log(e.GetResult())
+	}
+}
+
+func TestRealSendByGin(t *testing.T) {
+
+	SetRootURL("http://localhost:8080")
+
+	e := events.NewEvent(events.CENTRAL_EVENT, events.NewInstallEvent("TestInstallEvent"))
+	data, _ := json.Marshal(&e)
+
+	var u events.URLMaker
+	u = e.Data.(events.URLMaker)
+	t.Log(GetRootURL() + u.GetUrl())
+	req := httptest.NewRequest("POST", u.GetUrl(), bytes.NewReader(data))
+	w := httptest.NewRecorder()
+
+	g := makeGin()
+	g.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Errorf("Home page didn't return %v", http.StatusOK)
+	} else {
+		t.Log("OKOKOK")
+		data, _ := ioutil.ReadAll(w.Body)
+		t.Log(string(data))
+	}
+}

discovery/communicate/communicator.go

@@ -0,0 +1,53 @@
+package communicate
+
+import (
+//"bytes"
+//"compress/gzip"
+//"git.loafle.net/overflow/discovery/communicate/events"
+//"git.loafle.net/overflow/central/client/events"
+)
+
+type communicator struct {
+	//Queue   chan *events.Event
+	RootURL string
+}
+
+//
+//func NewCommunicator() *communicator {
+//	return &communicator{Queue: make(chan *events.Event, 10)}
+//}
+//
+//func (c *communicator) addEvent(e *events.Event) {
+//	c.Queue <- e
+//}
+//
+//func CompressDataGzip(data []byte) []byte {
+//	var b bytes.Buffer
+//	w := gzip.NewWriter(&b)
+//	w.Write(data)
+//	w.Close()
+//	return b.Bytes()
+//}
+//
+//func (c *communicator) start() {
+//	go func() {
+//		for e := range c.Queue {
+//			go func(event *events.Event) {
+//				//	m := event.Data.(events.URLMaker)
+//				//	data, _ := json.Marshal(event)
+//				//
+//				//	// compress , accept-encoding : gzip
+//				//	//res, err := http.Post(GetRootURL()+m.GetUrl(), "application/json", bytes.NewBuffer(CompressDataGzip(data)))
+//				//	//res, err := http.Post(GetRootURL()+m.GetUrl(), "application/json", bytes.NewBuffer(data))
+//				//
+//				//	// todo timeout,error
+//				//	if err != nil {
+//				//		return
+//				//	}
+//				//	if res.StatusCode != 200 && res.StatusCode != 201 {
+//				//
+//				//	}
+//			}(e)
+//		}
+//	}()
+//}

discovery/communicate/communicator_test.go

@@ -0,0 +1,26 @@
+package communicate
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"git.loafle.net/overflow/central/client/events"
+	"git.loafle.net/overflow/discovery/discovery/types/timestamp"
+	"net/http"
+	"testing"
+)
+
+func TestEventCollector(t *testing.T) {
+	ev := events.Event{}
+
+	ev.EventType = "install"
+	ev.Time = timestamp.Now()
+	ev.Collector_id = "1111111"
+	ev.Data = events.CollectorInstallEvent{Version: "2.2"}
+
+	bb, _ := json.Marshal(ev)
+
+	//com := CompressTest(CompressDataGzip, UnCompressDataGzip, bb)
+
+	http.Post("http://localhost:9090/_api/collector/event/status/install", "application/json", bytes.NewBuffer(bb))
+}

discovery/conf/conf.go

@@ -0,0 +1,57 @@
+package conf
+
+import (
+	"errors"
+	"fmt"
+	"github.com/cihub/seelog"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	//"git.loafle.net/overflow/discovery/communicate"
+	//"git.loafle.net/overflow/overflow.collector_backup2/src/git.loafle.net/overflow/discovery/communicate"
+)
+
+type Config struct {
+	Central struct {
+		Address string
+		Port    int
+	}
+	Log_Path string
+}
+
+var c Config
+
+func LoadConfig(f string) error {
+	if len(f) <= 0 {
+		return errors.New("conf file path is nil")
+	}
+	c = Config{}
+	data, err := ioutil.ReadFile(f)
+	if err != nil {
+		return err
+	}
+	err = yaml.Unmarshal([]byte(data), &c)
+	if err != nil {
+		return err
+	}
+
+	err = loadLogConfig(c.Log_Path)
+	if err != nil {
+		fmt.Println("Log Config Load Error : ", err)
+		return err
+	}
+	fmt.Println(c)
+
+	//communicate.SetRootURL(c.Central.Address)
+	return nil
+}
+
+func loadLogConfig(path string) error {
+
+	l, err := seelog.LoggerFromConfigAsFile(path)
+	if err != nil {
+		fmt.Println("Error : ", err)
+		return err
+	}
+	seelog.ReplaceLogger(l)
+	return err
+}

discovery/core/device/hw/cpu.go

@@ -0,0 +1 @@
+package hw

discovery/core/device/hw/hdd.go

@@ -0,0 +1 @@
+package hw

discovery/core/device/hw/memory.go

@@ -0,0 +1 @@
+package hw

discovery/core/device/hw/nic.go

@@ -0,0 +1,10 @@
+package hw
+
+type NIC struct {
+	Cidr    int32
+	Dns     [2]int64
+	Gateway int64
+	Iface   string
+	Ip      int64
+	Mac     int64
+}

discovery/core/device/sw/application.go

@@ -0,0 +1,6 @@
+package sw
+
+type Application struct {
+	Name    string
+	Version string
+}

discovery/core/device/sw/os.go

@@ -0,0 +1,7 @@
+package sw
+
+type OS struct {
+	Name        string
+	Version     string
+	DisplayName string
+}

discovery/core/device/sw/process.go

@@ -0,0 +1,7 @@
+package sw
+
+type Process struct {
+	Command string
+	Pid     int
+	User    string
+}

discovery/core/net/host.go

@@ -0,0 +1,5 @@
+package net
+
+type Host struct {
+	Name string
+}

discovery/core/net/port.go

@@ -0,0 +1,6 @@
+package net
+
+type Port struct {
+	Number   int16
+	PortType string
+}

discovery/core/net/service.go

@@ -0,0 +1,6 @@
+package net
+
+type Service struct {
+	CryptoType string
+	Name       string
+}

discovery/core/net/zone.go

@@ -0,0 +1,6 @@
+package net
+
+type Zone struct {
+	Cidr int32
+	Ip   int64
+}

discovery/core/pcapwrapper/pcap.go

@@ -0,0 +1,64 @@
+package pcap
+
+import (
+	"git.loafle.net/overflow/discovery/discovery/types"
+	"sync"
+)
+
+var pcaps map[string]*PcapWrapper = nil
+
+var m *sync.Mutex
+
+func init() {
+	pcaps = make(map[string]*PcapWrapper, 0)
+	m = new(sync.Mutex)
+}
+
+func GetInstance(zone *types.DiscoveryZone) (*PcapWrapper, error) {
+	m.Lock()
+	//
+	var p *PcapWrapper
+	var ok bool
+	p, ok = pcaps[zone.CidrStr()]
+	if !ok {
+		p, err := newPcapWrapper(zone)
+		if err != nil {
+			m.Unlock()
+			return nil, err
+		}
+		pcaps[zone.CidrStr()] = p
+		m.Unlock()
+		return p, nil
+	}
+	m.Unlock()
+	return p, nil
+}
+
+func Release(zone *types.DiscoveryZone) {
+
+	m.Lock()
+
+	p, ok := pcaps[zone.CidrStr()]
+	if ok == true {
+
+		if d := p.release(); d == true {
+
+			delete(pcaps, zone.CidrStr())
+		}
+
+	}
+	m.Unlock()
+}
+
+func RemoveAll() {
+	m.Lock()
+	if pcaps == nil {
+		m.Unlock()
+		return
+	}
+	for k, v := range pcaps {
+		v.destroy()
+		delete(pcaps, k)
+	}
+	m.Unlock()
+}

discovery/core/pcapwrapper/pcap_test.go

@@ -0,0 +1,45 @@
+package pcap
+
+import (
+	"fmt"
+	"git.loafle.net/overflow/discovery/core/scan/zone"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestMapRemove(t *testing.T) {
+
+	m := make(map[string]string, 0)
+
+	m["1"] = "1"
+	m["2"] = "2"
+	m["3"] = "3"
+	assert.Equal(t, 3, len(m))
+
+	delete(m, "1")
+	assert.Equal(t, 2, len(m))
+}
+
+func TestPcapNewRemove(t *testing.T) {
+
+	z := zone.NewZone()
+
+	GetInstance(z[0])
+	assert.Equal(t, 1, len(pcaps))
+
+	Release(z[0])
+	assert.Equal(t, 0, len(pcaps))
+}
+
+func TestRemoveAll(t *testing.T) {
+
+	z := zone.NewZone()
+
+	GetInstance(z[0])
+	fmt.Println("a9se8rypasd")
+	assert.Equal(t, 1, len(pcaps))
+
+	RemoveAll()
+	fmt.Println("a9se8rypasd")
+	assert.Equal(t, 0, len(pcaps))
+}

discovery/core/pcapwrapper/pcapwrapper.go

@@ -0,0 +1,272 @@
+package pcap
+
+import (
+	"git.loafle.net/overflow/discovery/discovery/types"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"sync"
+)
+
+type PcapWrapper struct {
+	m        *sync.RWMutex
+	HND      *pcap.Handle
+	refCount int
+	stop     chan bool
+	arps     []chan *layers.ARP
+	tcps     map[string][]chan *layers.TCP
+	udps     map[string][]chan gopacket.Packet
+}
+
+func newPcapWrapper(zone *types.DiscoveryZone) (*PcapWrapper, error) {
+
+	// new pcap handle
+	handle, err := pcap.OpenLive(zone.Iface, 65536, true, pcap.BlockForever)
+	if err != nil {
+		return nil, err
+	}
+	// set filter
+	// todo add tcp, udp filter
+	err = handle.SetBPFFilter("arp and src net " + zone.CidrStr() + " or (((tcp[tcpflags] & (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] & (tcp-rst) != 0)) and port 60000) or udp ")
+	if err != nil {
+		return nil, err
+	}
+	// make pcap wrapper
+	w := &PcapWrapper{
+		HND:      handle,
+		arps:     make([]chan *layers.ARP, 0),
+		tcps:     make(map[string][]chan *layers.TCP, 0),
+		udps:     make(map[string][]chan gopacket.Packet, 0),
+		stop:     make(chan bool),
+		refCount: 1,
+		m:        new(sync.RWMutex),
+	}
+
+	// start recv goroutine
+	go w.recvPacket()
+	return w, err
+}
+
+// arp channel funcs
+func (p *PcapWrapper) OpenARPChannel() chan *layers.ARP {
+	c := make(chan *layers.ARP, 0)
+	p.arps = append(p.arps, c)
+	return c
+}
+
+func (p *PcapWrapper) CloseARPChannel(ch chan *layers.ARP) {
+
+	var n int = -1
+	for index, value := range p.arps {
+		if ch == value {
+			close(ch)
+			n = index
+			break
+		}
+	}
+	if n != -1 {
+		p.arps = append(p.arps[:n], p.arps[n+1:]...)
+	}
+}
+
+// tcp channel funcs
+func (p *PcapWrapper) OpenTCPChannel(host *types.DiscoveryHost) chan *layers.TCP {
+	p.m.Lock()
+	defer p.m.Unlock()
+	_, ok := p.tcps[host.Ip_]
+	if !ok {
+		p.tcps[host.Ip_] = make([]chan *layers.TCP, 0)
+	}
+
+	c := make(chan *layers.TCP, 0)
+	p.tcps[host.Ip_] = append(p.tcps[host.Ip_], c)
+
+	return c
+}
+func (p *PcapWrapper) CloseTCPChannel(host *types.DiscoveryHost, ch chan *layers.TCP) {
+	p.m.Lock()
+	defer p.m.Unlock()
+
+	_, ok := p.tcps[host.Ip_]
+	if ok {
+		var n int = -1
+		for index, value := range p.tcps[host.Ip_] {
+			if ch == value {
+				close(ch)
+				n = index
+				break
+			}
+		}
+		if n != -1 {
+			p.tcps[host.Ip_] = append(p.tcps[host.Ip_][:n], p.tcps[host.Ip_][n+1:]...)
+		}
+	}
+}
+
+// udp channel funcs
+func (p *PcapWrapper) OpenUDPChannel(host *types.DiscoveryHost) chan gopacket.Packet {
+	p.m.Lock()
+	defer p.m.Unlock()
+
+	_, ok := p.udps[host.Ip_]
+	if !ok {
+		p.udps[host.Ip_] = make([]chan gopacket.Packet, 0)
+	}
+
+	c := make(chan gopacket.Packet, 0)
+	p.udps[host.Ip_] = append(p.udps[host.Ip_], c)
+
+	return c
+}
+
+func (p *PcapWrapper) CloseUDPChannel(host *types.DiscoveryHost, ch chan gopacket.Packet) {
+	p.m.Lock()
+	defer p.m.Unlock()
+
+	_, ok := p.udps[host.Ip_]
+	if ok {
+		var n int = -1
+		for index, value := range p.udps[host.Ip_] {
+			if ch == value {
+				close(ch)
+				n = index
+				break
+			}
+		}
+		if n != -1 {
+			p.udps[host.Ip_] = append(p.udps[host.Ip_][:n], p.udps[host.Ip_][n+1:]...)
+		}
+	}
+}
+
+func (p *PcapWrapper) incRefCount() {
+	p.refCount++
+}
+
+func (p *PcapWrapper) release() bool {
+	p.refCount--
+	if p.refCount == 0 {
+		p.destroy()
+		return true
+	}
+	return false
+}
+
+func (p *PcapWrapper) closeChannels() {
+
+	// close stop
+	if p.stop != nil {
+		close(p.stop)
+		p.stop = nil
+	}
+	// close tcp channels
+	for k, v := range p.tcps {
+		for _, ch := range v {
+			close(ch)
+		}
+		v = v[:0]
+		delete(p.tcps, k)
+	}
+	// close udp channels
+	for k, v := range p.udps {
+		for _, ch := range v {
+			close(ch)
+		}
+		v = v[:0]
+		delete(p.udps, k)
+	}
+	// close arp channels
+	for _, v := range p.arps {
+		close(v)
+	}
+	p.arps = p.arps[:0]
+}
+
+func (p *PcapWrapper) destroy() {
+	p.m.Lock()
+	defer p.m.Unlock()
+	p.closeChannels()
+	go p.HND.Close()
+}
+
+const arptype = 0
+const tcptype = 1
+const udptype = 2
+
+func (p *PcapWrapper) recvPacket() {
+
+	src := gopacket.NewPacketSource(p.HND, layers.LayerTypeEthernet)
+	in := src.Packets()
+
+	for {
+		var packet gopacket.Packet
+		select {
+		case <-p.stop:
+			return
+		case packet = <-in:
+			ptype := CheckProtocol(packet)
+
+			if ptype == arptype {
+				if p.arps != nil {
+					arpLayer := packet.Layer(layers.LayerTypeARP)
+					arp := arpLayer.(*layers.ARP)
+					for _, c := range p.arps {
+						c <- arp
+					}
+				}
+			} else if ptype == tcptype {
+				ipLayer := packet.Layer(layers.LayerTypeIPv4)
+				ip := ipLayer.(*layers.IPv4).SrcIP.String()
+				p.m.RLock()
+				chans, ok := p.tcps[ip]
+				if ok {
+					layer := packet.Layer(layers.LayerTypeTCP)
+					tcp, _ := layer.(*layers.TCP)
+					for _, c := range chans {
+						c <- tcp
+					}
+				}
+				p.m.RUnlock()
+			} else if ptype == udptype {
+				ipLayer := packet.Layer(layers.LayerTypeIPv4)
+				if ipLayer != nil {
+					ip := ipLayer.(*layers.IPv4).SrcIP.String()
+					p.m.RLock()
+					chans, ok := p.udps[ip]
+
+					if ok {
+						for _, c := range chans {
+							c <- packet
+						}
+					}
+
+					p.m.RUnlock()
+				}
+			}
+		}
+	}
+}
+func CheckProtocol(packet gopacket.Packet) int {
+	if packet == nil {
+		return -1
+	}
+	layer := packet.Layer(layers.LayerTypeARP)
+	if layer != nil {
+		return arptype
+	}
+
+	layer = packet.Layer(layers.LayerTypeTCP)
+	if layer != nil {
+		if _, ok := layer.(*layers.TCP); ok {
+			return tcptype
+		}
+	}
+
+	layer = packet.Layer(layers.LayerTypeUDP)
+	if layer != nil {
+		if _, ok := layer.(*layers.UDP); ok {
+			return udptype
+		}
+	}
+	return -1
+}

discovery/core/pcapwrapper/pcapwrapper_test.go

@@ -0,0 +1,76 @@
+package pcap
+
+import (
+	"fmt"
+	"github.com/google/gopacket/layers"
+	"testing"
+)
+
+func TestChanKey(t *testing.T) {
+
+	ch := make([]chan struct{}, 10)
+
+	for i := 0; i < 10; i++ {
+		ch[i] = make(chan struct{})
+	}
+
+	c := ch[5]
+
+	for n, v := range ch {
+		if c == v {
+			fmt.Println(n)
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		close(ch[i])
+	}
+}
+
+func TestDeleteSlice(t *testing.T) {
+
+	a := make([]int, 0)
+
+	for i := 0; i < 10; i++ {
+		a = append(a, i)
+	}
+
+	t.Log(a)
+
+	i := 4
+
+	a = append(a[:i], a[i+1:]...)
+	t.Log(a)
+
+	a = a[:0]
+	t.Log(len(a))
+
+}
+
+func TestMapForRange(t *testing.T) {
+	a := make(map[int]int, 0)
+
+	for i := 0; i < 10; i++ {
+		a[i] = i + 1000
+	}
+
+	for v, vv := range a {
+		t.Log(v)
+		t.Log(vv)
+	}
+}
+
+const arptest = 0
+
+func closechan(s interface{}, t int) {
+	if t == arptest {
+		close(s.(chan *layers.ARP))
+	}
+
+}
+
+func TestCloseChannels(t *testing.T) {
+	a := make(chan *layers.ARP, 0)
+	closechan(a, arptest)
+
+}

discovery/core/scan/host/host.go

@@ -0,0 +1,116 @@
+package host
+
+import (
+	p "git.loafle.net/overflow/discovery/core/pcapwrapper"
+	"git.loafle.net/overflow/discovery/discovery/types"
+	"git.loafle.net/overflow/discovery/util/converter"
+	log "github.com/cihub/seelog"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcap"
+	"net"
+	"time"
+)
+
+type ArpRecvHandler func(arp *layers.ARP)
+type HistoryHandler func(ip net.IP)
+
+// discovery
+func Scan(zone *types.DiscoveryZone, recvhandler ArpRecvHandler, historyhandler HistoryHandler) error {
+	// get channel by zone
+	w, err := p.GetInstance(zone)
+	if err != nil {
+	}
+
+	//// save singleton channel
+	ch := w.OpenARPChannel()
+	//
+	//// read channel
+	go func() {
+		for data := range ch {
+			recvhandler(data)
+		}
+	}()
+	rangeHost := MakeTargetHostRange(zone)
+	//// send arp
+	err = sendARP(w.HND, rangeHost, zone, historyhandler)
+	if err != nil {
+	}
+	//
+	//// wait
+	time.Sleep(10 * time.Second)
+	//
+	//// release arp
+	w.CloseARPChannel(ch)
+	p.Release(zone)
+
+	return nil
+}
+
+func sendARP(handle *pcap.Handle, rangeHost []net.IP, zone *types.DiscoveryZone, hh HistoryHandler) error {
+
+	hw := zone.Mac
+	localIp := converter.IntToIP4(zone.Ip)
+
+	eth := makeEthernetPacket(converter.IntToMac(hw))
+	arp := makeArpPacket(converter.IntToMac(hw), localIp.To4())
+	opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
+
+	buf := gopacket.NewSerializeBuffer()
+
+	for _, targetHost := range rangeHost {
+		if hh != nil {
+			hh(targetHost)
+		}
+		arp.DstProtAddress = []byte(targetHost)
+		gopacket.SerializeLayers(buf, opts, &eth, &arp)
+		if err := handle.WritePacketData(buf.Bytes()); err != nil {
+			log.Critical("ARP Send Error : ", err)
+			return err
+		}
+		time.Sleep(time.Microsecond * 500)
+	}
+	return nil
+}
+
+func MakeTargetHostRange(zone *types.DiscoveryZone) []net.IP {
+
+	//returnSlice := make([]net.IP, 0)
+	//
+	//for i := zone.FirstScanRange; i <= zone.LastScanRange; i++ {
+	//	cIp := converter.IntToIP4(i)
+	//	returnSlice = append(returnSlice, cIp)
+	//}
+	//
+	//return returnSlice
+
+	minIp := converter.IntToIP4(zone.FirstScanRange).To4()
+	maxIp := converter.IntToIP4(zone.LastScanRange).To4()
+
+	returnSlice := make([]net.IP, 0)
+	for i := minIp[3]; i <= maxIp[3]; i++ {
+		returnSlice = append(returnSlice, []byte{minIp[0], minIp[1], minIp[2], i})
+	}
+	return returnSlice
+}
+
+func makeArpPacket(hw net.HardwareAddr, ip net.IP) layers.ARP {
+	return layers.ARP{
+		AddrType:          layers.LinkTypeEthernet,
+		Protocol:          layers.EthernetTypeIPv4,
+		HwAddressSize:     6,
+		ProtAddressSize:   4,
+		Operation:         layers.ARPRequest,
+		SourceHwAddress:   []byte(hw),
+		SourceProtAddress: []byte(ip),
+		DstHwAddress:      []byte{0, 0, 0, 0, 0, 0},
+	}
+}
+
+func makeEthernetPacket(hw net.HardwareAddr) layers.Ethernet {
+	return layers.Ethernet{
+		SrcMAC:       hw,
+		DstMAC:       net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+		EthernetType: layers.EthernetTypeARP,
+	}
+}

discovery/core/scan/port/port.go

@@ -0,0 +1 @@
+package port

discovery/core/scan/port/port_test.go

@@ -0,0 +1 @@
+package port

discovery/core/scan/port/tcp/tcp.go

@@ -0,0 +1,135 @@
+package tcp
+
+import (
+	"git.loafle.net/overflow/discovery/discovery/types"
+	"net"
+	"time"
+
+	"git.loafle.net/overflow/discovery/discovery/types/timestamp"
+	log "github.com/cihub/seelog"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+
+	//"git.loafle.net/overflow/central/client/events"
+	//"git.loafle.net/overflow/discovery/communicate"
+	pw "git.loafle.net/overflow/discovery/core/pcapwrapper"
+	"git.loafle.net/overflow/discovery/util/converter"
+)
+
+type tcpRecvCallback func(tcp *layers.TCP)
+
+func Scan(host *types.DiscoveryHost, tcpCB tcpRecvCallback) {
+
+	//Recv(host.Zone, PcapTcpHandler{})
+
+	w, err := pw.GetInstance(host.Zone)
+	if err != nil {
+		log.Error(err)
+	}
+
+	ch := w.OpenTCPChannel(host)
+
+	//read channel
+	go func() {
+		for data := range ch {
+			tcpCB(data)
+		}
+	}()
+
+	Send(host, nil)
+
+	time.Sleep(time.Second * 3)
+
+	//communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortEndEvent(host.Zone.CidrInt64(), host.Ip, host.Histories, types.TYPE_TCP)))
+
+	w.CloseTCPChannel(host, ch)
+	pw.Release(host.Zone)
+
+}
+
+func Check(host *types.DiscoveryHost, ports []*types.DiscoveryPort) {
+
+	//Recv(host.Zone, PcapTcpHandler{})
+	Send(host, ports)
+
+	time.Sleep(time.Second * 3)
+}
+
+func Send(host *types.DiscoveryHost, ports []*types.DiscoveryPort) {
+
+	portPacket := CreatePortPacket(host.Zone, host)
+
+	defer portPacket.PacketConn.Close()
+
+	//communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortStartEvent(host.Zone.CidrInt64(), host.Ip, 10000, types.TYPE_TCP)))
+	host.PortDiscoveryTime = timestamp.Now()
+
+	if ports == nil {
+		for portNumber := host.FirstScanRange; portNumber <= host.LastScanRange; portNumber++ {
+			SendPortPacket(portNumber, portPacket, host)
+		}
+	} else {
+		for _, port := range ports {
+			SendPortPacket(port.Number, portPacket, host)
+		}
+	}
+
+}
+
+type PortPacket struct {
+	Ip         *layers.IPv4
+	Tcp        *layers.TCP
+	Opts       gopacket.SerializeOptions
+	PacketConn net.PacketConn
+}
+
+func CreatePortPacket(zone *types.DiscoveryZone, host *types.DiscoveryHost) *PortPacket {
+
+	pp := &PortPacket{}
+
+	pp.Ip = &layers.IPv4{
+		SrcIP:    converter.IntToIP4(zone.Ip),
+		DstIP:    net.ParseIP(host.Ip_),
+		Version:  4,
+		TTL:      64,
+		Protocol: layers.IPProtocolTCP,
+	}
+	pp.Tcp = &layers.TCP{
+		SrcPort: 60000,
+		DstPort: 0, // will be incremented during the scan
+		SYN:     true,
+		Seq:     0,
+	}
+	pp.Opts = gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+
+	conn, err := net.ListenPacket("ip4:tcp", "0.0.0.0")
+	if err != nil {
+		log.Error("SYN create socket error : " + err.Error())
+		return nil
+	}
+
+	pp.PacketConn = conn
+
+	return pp
+}
+
+func SendPortPacket(port uint16, pp *PortPacket, host *types.DiscoveryHost) {
+
+	pp.Tcp.DstPort = layers.TCPPort(port)
+	pp.Tcp.SetNetworkLayerForChecksum(pp.Ip)
+
+	buf := gopacket.NewSerializeBuffer()
+	if err := gopacket.SerializeLayers(buf, pp.Opts, pp.Tcp); err != nil {
+		log.Error(err)
+	}
+
+	if _, err := pp.PacketConn.WriteTo(buf.Bytes(), &net.IPAddr{IP: net.ParseIP(host.Ip_)}); err != nil {
+		log.Error(err)
+	}
+
+	types.NewPortScanHistory(host, port, types.TYPE_TCP, types.SEND, "")
+	time.Sleep(time.Microsecond * 200)
+}

discovery/core/scan/port/tcp/tcp_test.go

@@ -0,0 +1,61 @@
+package tcp
+
+import (
+	"fmt"
+	"git.loafle.net/overflow/discovery/discovery/types"
+	"git.loafle.net/overflow/discovery/discovery/types/timestamp"
+	"testing"
+
+	"github.com/google/gopacket/layers"
+)
+
+func TestPortScan(t *testing.T) {
+
+	zone := types.NewZones()[0]
+
+	hh := &types.DiscoveryHost{
+		Ip_:            "192.168.1.215",
+		Ports_:         make(map[string]*types.DiscoveryPort, 100),
+		Ip:             111,
+		Mac:            222,
+		CreateDate:     timestamp.Now(),
+		UpdateDate:     timestamp.Now(),
+		Zone:           zone,
+		FirstScanRange: 1,
+		LastScanRange:  100,
+	}
+
+	Scan(hh, func(tcp *layers.TCP) {
+		fmt.Println(tcp)
+	})
+
+}
+
+func TestPortCheck(t *testing.T) {
+
+	zone := types.NewZones()[0]
+
+	hh := &types.DiscoveryHost{
+		Ip_:            "192.168.1.215",
+		Ports_:         make(map[string]*types.DiscoveryPort, 100),
+		Ip:             111,
+		Mac:            222,
+		CreateDate:     timestamp.Now(),
+		UpdateDate:     timestamp.Now(),
+		Zone:           zone,
+		FirstScanRange: 1,
+		LastScanRange:  100,
+	}
+
+	var ports []*types.DiscoveryPort
+
+	ports = append(ports, &types.DiscoveryPort{Number: 1})
+	ports = append(ports, &types.DiscoveryPort{Number: 3})
+	ports = append(ports, &types.DiscoveryPort{Number: 5})
+	ports = append(ports, &types.DiscoveryPort{Number: 7})
+	ports = append(ports, &types.DiscoveryPort{Number: 9})
+	ports = append(ports, &types.DiscoveryPort{Number: 11})
+
+	Check(hh, ports)
+
+}

discovery/core/scan/port/udp/udp.go

@@ -0,0 +1,105 @@
+package udp
+
+import (
+	"git.loafle.net/overflow/discovery/core/scan/service/matcher"
+	"git.loafle.net/overflow/discovery/core/scan/service/matcher/packet"
+	"git.loafle.net/overflow/discovery/discovery/types"
+	log "github.com/cihub/seelog"
+	"net"
+	"time"
+
+	"github.com/google/gopacket"
+
+	//"git.loafle.net/overflow/central/client/events"
+	//"git.loafle.net/overflow/discovery/communicate"
+	pw "git.loafle.net/overflow/discovery/core/pcapwrapper"
+)
+
+type udpRecvCallback func(packet gopacket.Packet)
+
+func Scan(host *types.DiscoveryHost, udpCB udpRecvCallback) {
+
+	//Recv(host.Zone, PcapTcpHandler{})
+
+	w, err := pw.GetInstance(host.Zone)
+	if err != nil {
+		log.Error(err)
+	}
+
+	ch := w.OpenUDPChannel(host)
+
+	//read channel
+	go func() {
+		for data := range ch {
+			udpCB(data)
+		}
+	}()
+
+	Send(host, nil)
+
+	time.Sleep(time.Second * 3)
+
+	//communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortEndEvent(host.Zone.CidrInt64(), host.Ip, host.Histories, types.TYPE_UDP)))
+
+	//good game
+	w.CloseUDPChannel(host, ch)
+	pw.Release(host.Zone)
+
+}
+
+func Check(host *types.DiscoveryHost, ports []*types.DiscoveryPort) {
+
+	//Recv(host.Zone, PcapTcpHandler{})
+	Send(host, ports)
+}
+
+func Send(host *types.DiscoveryHost, ports []*types.DiscoveryPort) {
+
+	mats := matcher.GetUdpMatchers()
+
+	conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err != nil {
+		log.Error(err)
+	}
+
+	//communicate.Send(events.NewEvent(events.CENTRAL_EVENT, events.NewPortStartEvent(host.Zone.CidrInt64(), host.Ip, 10000, types.TYPE_UDP)))
+
+	for indexI := 0; indexI < len(mats); indexI++ {
+
+		mat := mats[indexI]
+
+		if ports == nil {
+			for portNumber := host.FirstScanRange; portNumber <= host.LastScanRange; portNumber++ {
+				SendUDPPortPacket(host, conn, mat, portNumber)
+			}
+		} else {
+			for _, port := range ports {
+				SendUDPPortPacket(host, conn, mat, port.Number)
+			}
+		}
+
+	}
+
+	conn.Close()
+
+}
+
+func SendUDPPortPacket(host *types.DiscoveryHost, conn *net.UDPConn, mat matcher.UDPMatcher, pn uint16) {
+
+	if mat.IsSend(int(pn)) != true {
+		return
+	}
+
+	addr := &net.UDPAddr{IP: net.ParseIP(host.Ip_), Port: int(pn)}
+	for i := 0; i < mat.PacketCount(); i++ {
+		var p *packet.Packet = mat.Packet(i)
+		if _, err := conn.WriteToUDP(p.Buffer, addr); err != nil {
+			log.Error(err)
+		}
+	}
+
+	ph := types.NewPortScanHistory(host, pn, types.TYPE_UDP, types.SEND, "")
+	host.AddHistory(ph)
+
+	time.Sleep(time.Microsecond * 200)
+}