overflow_backup_2018_0455/crawler_manager_go
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changed

Browse Source 
config logic
This commit is contained in:
snoop 2017-04-28 13:44:56 +09:00
parent 5343329213
commit 7d9a9a41de
4 changed files with 155 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
crawler_manager.go
View File

@ -20,13 +20,13 @@ import (
const (
address = "localhost:"
portArgsName = "-Port="
defaultPort = 50000
defaultPort = 50052
rootFolder = "/home/cm2/"
ConfigFolder = rootFolder + "/config/container/"
BinaryFolder = rootFolder + "/container/"
PidFolder = rootFolder + "/pids/"
runFile = "ttnc"
//rootFolder = "/home/cm2/"
//ConfigFolder = rootFolder + "/config/container/"
//BinaryFolder = rootFolder + "/container/"
//PidFolder = rootFolder + "/pids/"
//runFile = "ttnc"
)
var g_CrawlerMananger *CrawlerManager = nil
@ -38,6 +38,7 @@ type CrawlerManager struct {
ConfigMgr config_manager.ConfigManager
}
func init() {
GetInstance();
@ -47,6 +48,18 @@ func init() {
//g_CrawlerMananger.init()
}
func SettingPath() {
//test
GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder = "/home/cm2/"
GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder
GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder
GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder = GetInstance().ConfigMgr.GetGlobalConfig().Paths.RootFolder + GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder
}
func GetInstance() *CrawlerManager {
if g_CrawlerMananger == nil {
@ -111,7 +124,7 @@ func (c *CrawlerManager)checkAndRunContainer(container *string) bool {
func (c *CrawlerManager)checkPid() {
files, err := ioutil.ReadDir(PidFolder)
files, err := ioutil.ReadDir(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder)
if err != nil {
log.Println(err)
@ -197,7 +210,7 @@ func (c *CrawlerManager)runContainer(container *string) error {
cmdStr := getRunCommand(container)
for {
pArg := portArgsName + strconv.Itoa(c.currentPort)
pArg := strconv.Itoa(c.currentPort)
cmd := exec.Command(cmdStr, pArg)
err := cmd.Start()
@ -212,6 +225,7 @@ func (c *CrawlerManager)runContainer(container *string) error {
b := callStatusAddress(address + strconv.Itoa(c.currentPort))
if b == false {
log.Println("false " + strconv.Itoa(c.currentPort))
c.currentPort++
continue;
}
@ -224,6 +238,8 @@ func (c *CrawlerManager)runContainer(container *string) error {
c.currentPort++
log.Println(*container + "started")
break;
}
@ -300,7 +316,7 @@ func (c *CrawlerManager)stopProcess(pid *string) {
}
func (c *CrawlerManager)removeProcessFile(pid *string) {
err := os.Remove(PidFolder + "/" +*pid)
err := os.Remove(c.ConfigMgr.GetGlobalConfig().Paths.PidFolder + "/" +*pid)
if err != nil {
log.Println(err)
}

4
crawler_manager_event.go
View File

@ -28,8 +28,12 @@ func listenConfigLoaded() {
GetInstance().ConfigMgr = cm;
SettingPath()
GetInstance().init();
}
func listenAddSensor() {

150
crawler_manager_test.go
View File

@ -15,6 +15,12 @@ import (
"loafle.com/overflow/agent_api/config_manager"
"encoding/json"
"fmt"
"time"
"loafle.com/overflow/agent_api/observer"
"loafle.com/overflow/agent_api/observer/messages"
conMgr "loafle.com/overflow/config_manager_go"
"sync"
)
@ -135,6 +141,11 @@ func TestCrateDir(t *testing.T) {
var configPath []string;
rootFolder := "/home/cm2/"
ConfigFolder := rootFolder + "/config/container/"
BinaryFolder := rootFolder + "/container/"
PidFolder := rootFolder + "/pids/"
//runFile := "ttnc"
configPath = append(configPath, ConfigFolder + "/java/oracle/")
@ -290,34 +301,34 @@ func TestErer(t *testing.T) {
t.Log(len(asdf))
}
//
//func TestStartContainer(t *testing.T) {
//
// cs := GetStartContainer()
//
// t.Log(cs)
//
//}
func TestStartContainer(t *testing.T) {
cs := GetStartContainer()
t.Log(cs)
}
func GetStartContainer() []string {
files, _ := ioutil.ReadDir(ConfigFolder)
var cs []string
for _,file := range files {
if file.IsDir() {
b := existConfigFile(ConfigFolder, file.Name())
if b {
cs = append(cs, file.Name())
}
}
}
return cs
}
//
//func GetStartContainer() []string {
//
// files, _ := ioutil.ReadDir(ConfigFolder)
//
// var cs []string
//
// for _,file := range files {
//
// if file.IsDir() {
// b := existConfigFile(ConfigFolder, file.Name())
// if b {
// cs = append(cs, file.Name())
// }
// }
//
// }
// return cs
//}
func TestJson(t *testing.T) {
@ -335,3 +346,88 @@ func TestJson(t *testing.T) {
}
func TestJsonMa(t *testing.T) {
b, _ := ioutil.ReadFile("./test/example.json")
c := config_manager.Config{}
json.Unmarshal(b, &c)
t.Log(c)
}
func TestMock(t *testing.T) {
GetInstance().ConfigMgr = &ConfigManagerTest{}
GetInstance().init()
}
func TestRPC(t *testing.T) {
//c := conMgr.NewConfigManager()
//c.LoadGlobalConfigTTT("/home/snoop/develop/path/go/src/loafle.com/overflow/config_manager_go/test_agent/global.yaml")
//c.LoadCrawlerConfigAllTTT()
//
//observer.Notify(messages.CONFIGMANAGER_LOADED,c)
//
//
//wg := sync.WaitGroup{}
//
//wg.Add(1)
//
//go func() {
// time.Sleep(20 * time.Second)
// wg.Done()
//}()
//
//
//wg.Wait()
}
func TestGRPC(t *testing.T) {
b := callStatusAddress("localhost:50052")
t.Log(b)
}
type ConfigManagerTest struct {
}
func (c *ConfigManagerTest)GetGlobalConfig() *config_manager.GlobalConfig {
g := config_manager.GlobalConfig{}
rootFolder := "/home/cm2/"
g.Paths.ConfigFolder = rootFolder + "/config/container/"
g.Paths.BinaryFolder = rootFolder + "/container/"
g.Paths.PidFolder = rootFolder + "/pids/"
return &g
}
func (c *ConfigManagerTest)GetCrawlerById(id string) *config_manager.Config {
return nil
}
func (c *ConfigManagerTest)GetCrawlers() map[string] *config_manager.Config {
var m map[string] *config_manager.Config
b, _ := ioutil.ReadFile("./test/example.json")
ccc := config_manager.Config{}
json.Unmarshal(b, &ccc)
m["wmi_crawler"] = &ccc
return m
}

6
crawler_util.go
View File

@ -42,17 +42,17 @@ func existConfigFileDir(prePath string,dir string, configCrawler *[]string) {
}
func getRunCommand(container *string ) string {
return BinaryFolder + "/" + *container + "/" + runFile
return GetInstance().ConfigMgr.GetGlobalConfig().Paths.BinaryFolder + "/" + *container + "/" + GetInstance().ConfigMgr.GetGlobalConfig().Paths.ScriptFile
}
func writePid(pid int) {
ioutil.WriteFile(PidFolder + strconv.Itoa(pid), []byte(""), os.ModePerm)
ioutil.WriteFile(GetInstance().ConfigMgr.GetGlobalConfig().Paths.PidFolder + strconv.Itoa(pid), []byte(""), os.ModePerm)
}
func getConfigPaths(container *string) *[]string {
var dirs []string
existConfigFileDir(ConfigFolder, *container, &dirs)
existConfigFileDir(GetInstance().ConfigMgr.GetGlobalConfig().Paths.ConfigFolder, *container, &dirs)
return &dirs
}