remodeling

config.go

@@ -1,131 +0,0 @@
-package config_manager_go
-
-import (
-	"os"
-	"log"
-	"io/ioutil"
-	"loafle.com/overflow/crawler_go/config"
-	"encoding/json"
-	"strings"
-)
-
-const (
-	OVERFLOW_CONFIG_PATH = "/Overflow/Config"
-)
-type ConfigManager struct {
-	idMap map[string]string
-}
-
-func (c *ConfigManager) readByTempConfig(cn *config.Config) (error) {
-	pwd, err := os.Getwd()
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	b, err := ioutil.ReadFile(pwd+"/example.json")
-	err = json.Unmarshal(b,&cn)
-
-	return err
-}
-
-func (c *ConfigManager) writeConfig(cn config.Config) (error){
-	var fileName = cn.Id
-	fileJson, _ := json.Marshal(cn)
-	path, _ := c.getFilePath(cn)
-
-	err := ioutil.WriteFile(path + "/" + fileName + ".json", fileJson, 0644)
-
-	c.idMap[cn.Id] = path + "/"
-
-	return err
-}
-
-func (c *ConfigManager) getFilePath(cn config.Config) (string, error){
-	var parentPath string = strings.Join(configSettingFolder,"")
-
-	cctmp := cn.Crawler.Container
-	cctmps := strings.Split(cctmp,"_")
-	containerPath := cctmps[0]
-
-	cns := strings.Split(cn.Crawler.Name, "_")
-	cnf := cns[0]
-
-	rs := parentPath + OVERFLOW_CONFIG_PATH +"/"+containerPath+"/"+cnf
-
-	err := checkFolder(rs)
-
-	return rs, err
-}
-
-func (c *ConfigManager) AddConfig(b []byte) (error) {
-	var err error
-	cn := config.Config{}
-
-	if len(b) == 0 || b == nil {
-		err = c.readByTempConfig(&cn)
-	} else {
-		err = json.Unmarshal(b, &cn)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	c.writeConfig(cn)
-
-	return err
-}
-
-func (c *ConfigManager) GetConfigById(id string) (string) {
-	return c.idMap[id]
-}
-
-func checkFolder(path string) (error) {
-
-	_, err := os.Stat(path)
-
-	if err != nil {
-		log.Println(err)
-	}
-
-	if os.IsNotExist(err) {
-		err = nil
-		os.MkdirAll(path, os.ModePerm)
-	}
-	return err
-}
-
-func NewConfigManager() *ConfigManager {
-	var c ConfigManager
-	c.idMap = make(map[string]string)
-
-	return &c
-}
-
-func Start(b []byte)  {
-	// config json stream read
-	// config json byte read
-
-	var err error
-	cm := ConfigManager{}
-	c := config.Config{}
-	cm.idMap = make(map[string]string)
-
-	if len(b) == 0 || b == nil {
-		err = cm.readByTempConfig(&c)
-	} else {
-		err = json.Unmarshal(b, &c)
-	}
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	cm.writeConfig(c)
-}
-
-
-// config json save folder
-// create folder for config
-// config json parse

config_darwin.go

@@ -1,6 +0,0 @@
-package config_manager_go
-
-import "os"
-
-var hasVendorName = true
-var configSettingFolder = []string{os.Getenv("/etc/overflow/config")}

config_manager.go

@@ -0,0 +1,195 @@
+package config_manager_go
+
+import (
+	"encoding/json"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"loafle.com/overflow/agent_api/config_manager"
+	"loafle.com/overflow/agent_api/observer"
+	"loafle.com/overflow/agent_api/observer/messages"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+type configManager struct {
+	config_manager.ConfigManager // interface implements
+
+	addCh    chan interface{}
+	removeCh chan interface{}
+
+	configs      map[string]*config_manager.Config
+	globalConfig config_manager.GlobalConfig
+}
+
+var _configManager *configManager
+
+func init() {
+	var once sync.Once
+	once.Do(func() {
+		_configManager = NewConfigManager()
+	})
+
+	// regist handlers
+	agentStartHandler()
+	addSensorHandler()
+	removeSensorHandler()
+}
+
+func NewConfigManager() *configManager {
+	c := &configManager{
+		configs:  make(map[string]*config_manager.Config, 0),
+		addCh:    make(chan interface{}, 0),
+		removeCh: make(chan interface{}, 0),
+	}
+	return c
+}
+
+func (c *configManager) getConfigPath() string {
+	return c.globalConfig.Paths.RootFolder + c.globalConfig.Paths.ConfigFolder
+}
+
+func (c *configManager) getContainerPath() string {
+	return c.getConfigPath() + "container/"
+}
+
+func appendSeperator(str string) string {
+	if strings.LastIndex(str, "/") != len(str)-1 {
+		return str + "/"
+	}
+	return str
+}
+
+func (c *configManager) loadGlobalConfig(path string) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+	}
+	err = yaml.Unmarshal([]byte(data), &c.globalConfig)
+	if err != nil {
+	}
+}
+
+func (c *configManager) loadCrawlerConfigAll() error {
+
+	root := c.getConfigPath()
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		log.Panic(err)
+	}
+	for _, file := range files {
+		if file.IsDir() == true {
+			c.loadCrawlerConfig(root, file.Name())
+		}
+	}
+	return nil
+}
+
+func (c *configManager) loadCrawlerConfig(root string, dir string) {
+
+	separator := ""
+	if strings.LastIndex(root, "/") != len(root)-1 {
+		separator = "/"
+	}
+	currentDir := root + separator + dir
+	files, err := ioutil.ReadDir(currentDir)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() == true {
+			c.loadCrawlerConfig(currentDir, file.Name())
+		} else {
+			b, err := ioutil.ReadFile(currentDir + separator + file.Name())
+			if err != nil {
+				log.Panic(err)
+			}
+			var m = config_manager.Config{}
+			json.Unmarshal(b, &m)
+			c.configs[file.Name()] = &m
+		}
+	}
+
+}
+
+func (c *configManager) addConfig() {
+	for data := range c.addCh {
+
+		path := data.(string)
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			// error process
+			continue
+		}
+
+		// 임시파일을 로드 ,  Config로 변환
+		var m = config_manager.Config{}
+		err = json.Unmarshal(b, &m)
+		if err != nil {
+			// error process
+			continue
+		}
+
+		// agent 폴더 / config / crawler / .. / .. / .. / 에 해당하는 파일이 있는지 확인, 있다면 삭제
+		// Config 파일로 저장
+		savePath := c.getContainerPath() + appendSeperator(m.Crawler.Container) + appendSeperator(m.Crawler.Name) + m.Id
+		ioutil.WriteFile(savePath, b, 0644)
+
+		// tempfile remove
+		err = os.Remove(path)
+		if err != nil {
+			// error process
+			continue
+		}
+
+		// Config 맵에 저장
+		c.configs[m.Id] = &m
+
+		// notify id
+		err = observer.Notify(messages.ADD_SENSOR_1, m.Id)
+		if err != nil {
+			continue
+		}
+	}
+}
+
+func (c *configManager) removeConfig() {
+	for data := range c.removeCh {
+		removeid := data.(string)
+
+		// check exists
+		config, ok := c.configs[removeid]
+		if !ok {
+			continue
+		}
+
+		// 해당 파일 삭제
+		path := c.getContainerPath() + appendSeperator(config.Crawler.Container) + appendSeperator(config.Crawler.Name) + config.Id
+		err := os.Remove(path)
+		if err != nil {
+			// error check
+			continue
+		}
+
+		// 해당 id 삭제
+		delete(c.configs, removeid)
+
+		// notify id
+		err = observer.Notify(messages.REMOVE_SENSOR_1, removeid)
+		if err != nil {
+			continue
+		}
+	}
+}
+
+// implements methods
+func (c *configManager) GetGlobalConfig() *config_manager.GlobalConfig {
+	return &c.globalConfig
+}
+func (c *configManager) GetCrawlerById(id string) *config_manager.Config {
+	return c.configs[id]
+}
+func (c *configManager) GetCrawlers() map[string]*config_manager.Config {
+	return c.configs
+}

config_manager_test.go

@@ -0,0 +1,55 @@
+package config_manager_go
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+	"io/ioutil"
+	"log"
+	"loafle.com/overflow/agent_api/config_manager"
+	"encoding/json"
+	"github.com/google/uuid"
+	"loafle.com/overflow/agent_api/observer"
+	"loafle.com/overflow/agent_api/observer/messages"
+	"time"
+)
+
+func TestLoadConfig(t *testing.T) {
+
+	c := NewConfigManager()
+	c.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	c.loadCrawlerConfigAll()
+	
+	assert.NotEqual(t, len(c.configs),0)
+}
+
+
+func TestAddConfig(t *testing.T) {
+	
+	_configManager.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	
+	b, err := ioutil.ReadFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/test.json")
+	if err != nil {
+		log.Panic(err)
+	}
+
+	var m = config_manager.Config{}
+	json.Unmarshal(b, &m)
+
+	rid,_ := uuid.NewRandom()
+	m.Id = rid.String()
+	b,err = json.Marshal(&m)
+	ioutil.WriteFile("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/" + m.Id,b,0644)
+	
+	observer.Notify(messages.ADD_SENSOR_0,"/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/" + m.Id)
+	time.Sleep(1 * time.Second)
+}
+
+
+func TestRemoveConfig(t *testing.T) {
+	
+	_configManager.loadGlobalConfig("/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
+	_configManager.loadCrawlerConfigAll()
+	
+	observer.Notify(messages.REMOVE_SENSOR_0,"289575d2-4a34-4475-b91d-7fb0992cc3de")
+	time.Sleep(1 * time.Second)
+}

config_test.go

@@ -1,8 +0,0 @@
-package config_manager_go
-
-import "testing"
-
-func TestStart(t *testing.T) {
-
-	Start(nil)
-}

config_windows.go

@@ -1,6 +0,0 @@
-package config_manager_go
-
-import "os"
-
-var hasVendorName = true
-var configSettingFolder = []string{os.Getenv("PROGRAMDATA")}

init_method.go

@@ -0,0 +1,35 @@
+package config_manager_go
+
+import (
+	"loafle.com/overflow/agent_api/observer"
+	"loafle.com/overflow/agent_api/observer/messages"
+)
+
+func agentStartHandler() {
+	agentStart := make(chan interface{}, 0)
+	observer.Add("AGENT_STARTED", agentStart)
+	go func() {
+		data := <-agentStart
+		path := data.(string)
+		// load global config
+		_configManager.loadGlobalConfig(path + "/" + "global.yaml")
+		
+		// load all crawler configs
+		if err := _configManager.loadCrawlerConfigAll(); err != nil {
+			// error process
+		}
+		
+		observer.Remove(messages.AGENT_STARTED, agentStart)
+		observer.Notify(messages.CONFIGMANAGER_LOADED, _configManager)
+	}()
+}
+
+func removeSensorHandler() {
+	observer.Add(messages.REMOVE_SENSOR_0,_configManager.removeCh)
+	go _configManager.removeConfig()
+}
+
+func addSensorHandler() {
+	observer.Add(messages.ADD_SENSOR_0,_configManager.addCh)
+	go _configManager.addConfig()
+}

test_agent/config/container/java_proxy/redis/736387363

@@ -1,5 +1,5 @@
 {
-  "id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
+  "id" : "736387363",
   "target" : {
     "connection" : {
       "ip" : "192.168.1.104",
@@ -15,8 +15,8 @@
     "interval" : "10"
   },
   "crawler" : {
-    "name":"redis_protocol_crawler",
+    "name":"redis",
-    "container":"network_crawler"
+    "container":"java_proxy"
   },
   "items" : [
   ]

test_agent/config/container/java_proxy/sql/989238744

@@ -0,0 +1,23 @@
+{
+  "id" : "989238744",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"sql",
+    "container":"java_proxy"
+  },
+  "items" : [
+  ]
+}

test_agent/config/container/network_proxy/health_activedirectory/123980918237

@@ -0,0 +1,23 @@
+{
+  "id" : "123980918237",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"health_activedirectory",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}

test_agent/global.yaml

@@ -0,0 +1,11 @@
+central:
+  address: "http://localhost:9090"
+  port: 443
+logPath: "./bin/log.xml"
+paths:
+  rootFolder : "/root/gowork/src/loafle.com/overflow/config_manager_go/test_agent/"
+  configFolder : "config/"
+  binaryFolder : "container/"
+  pidFolder : "pids/"
+  scriptFile : "start"
+intervalSecond: 10

test_agent/test.json

@@ -0,0 +1,23 @@
+{
+  "id" : "123980918237",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.104",
+      "port" : "6379",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"health_activedirectory",
+    "container":"network_proxy"
+  },
+  "items" : [
+  ]
+}