From 3424503357a96afbbb59c13a33f561b50a24645e Mon Sep 17 00:00:00 2001 From: snoop Date: Fri, 14 Apr 2017 17:19:02 +0900 Subject: [PATCH] ing --- crawler_communicator.go | 25 ++++++---- crawler_manager.go | 45 ++---------------- crawler_manager_test.go | 33 ++++++------- crawler_runner.go | 100 ++++------------------------------------ 4 files changed, 44 insertions(+), 159 deletions(-) diff --git a/crawler_communicator.go b/crawler_communicator.go index 841da7e..09ebd9a 100644 --- a/crawler_communicator.go +++ b/crawler_communicator.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" + "strings" ) @@ -34,7 +35,7 @@ func CallAdd() { } -func CallInit2(address string, cpm *map[string][]string) bool { +func CallInit2(address string, paths *[]string) bool { conn, err := grpc.Dial(address, grpc.WithInsecure()) if err != nil { log.Fatalf("did not connect: %v", err) @@ -44,14 +45,22 @@ func CallInit2(address string, cpm *map[string][]string) bool { cc := g.NewConfigClient(conn) - in := &g.Init{} - - in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/" - //in.Name = g.Crawlers(g.Crawlers_value[""]) - in.Name = g.Crawlers_HEALTH_REDIS - inArr := &g.InputArray{} - inArr.In = append(inArr.In, in) + base := "HEALTH_" + for _, path := range *paths { + + in := &g.Init{} + in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/" + //in.Path = path + bcn := filepath.Base(path) + bcn = strings.ToUpper(bcn) + + log.Println("cn:" + base+bcn) + + //in.Name = g.Crawlers(g.Crawlers_value[base+bcn]) + in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test + inArr.In = append(inArr.In, in) + } outInit, errInit := cc.Init(context.Background(), inArr) if errInit != nil { diff --git a/crawler_manager.go b/crawler_manager.go index 279bb0f..26ecd93 100644 --- a/crawler_manager.go +++ b/crawler_manager.go @@ -3,7 +3,7 @@ package crawler_manager import ( - g "loafle.com/overflow/crawler_go/grpc" + "loafle.com/overflow/crawler_go/config" @@ -16,7 +16,7 @@ import ( const ( address = "localhost:" - defaultPort = 80 + defaultPort = 50000 rootFolder = "/home/cm/" ConfigFolder = rootFolder + "/config/" @@ -30,24 +30,18 @@ type crawler_manager struct { } - - var currentPort = defaultPort var pidMap map[string]int -var exeMap map[string]string + var portMap map[string]string func init() { pidMap = make(map[string]int) - exeMap = make(map[string]string) + portMap = make(map[string]string) - //pidMap["HEALTH_REDIS"] = 18281 - - exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc"; - } func ReadConfig(path string ) *config.Config { @@ -67,32 +61,6 @@ func ReadConfig(path string ) *config.Config { } -func AddConfig(path string) { - - c := ReadConfig(path) - - exePath := exeMap[c.Crawler.Name] - - cs := c.Crawler.Name - - pid := pidMap[cs] - - if pid > 0 { - b := IsAlive(pid) - if b == false { - ExeCrawler(c.Crawler.Name, exePath) - CallInit(c.Crawler.Name, path) - } else { - CallAdd() - } - - - } else { - ExeCrawler(c.Crawler.Name, exePath) - CallInit(c.Crawler.Name, path) - } - -} func InitContainer() { @@ -108,11 +76,6 @@ func InitContainer() { RunContainer(&c, &cpm) } - //startContainer - //isPid - - // - } func IsStartContainer() []string { diff --git a/crawler_manager_test.go b/crawler_manager_test.go index e44c5a0..c854038 100644 --- a/crawler_manager_test.go +++ b/crawler_manager_test.go @@ -13,10 +13,6 @@ import ( "io/ioutil" ) -func TestManageCrawler(t *testing.T) { - AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json") -} - func TestAdd(t *testing.T) { @@ -100,19 +96,6 @@ func TestPipe(t *testing.T) { t.Log(string(aa)) } -func TestIsAlive(t *testing.T) { - - IsAlive(15538) - -} - -func TestIsState(t *testing.T) { - - b := IsState(225931) - t.Log(b) - -} - func TestCom(t *testing.T) { str:= "Z" @@ -160,8 +143,9 @@ func TestCrateDir(t *testing.T) { configPath = append(configPath, ConfigFolder + "/java/mysql/") configPath = append(configPath, ConfigFolder + "/network/http/") - configPath = append(configPath, ConfigFolder + "/network/ldap/") configPath = append(configPath, ConfigFolder + "/network/redis/") + configPath = append(configPath, ConfigFolder + "/network/ldap/") + var containerPath []string; @@ -210,5 +194,18 @@ func TestDirs(t *testing.T) { } +} + +func TestSplitPath(t *testing.T) { + + ff :="/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff" + a := filepath.Dir(ff) + t.Log(a) + a = filepath.Clean(a) + t.Log(a) + a = filepath.Base(a) + + t.Log(a) + } \ No newline at end of file diff --git a/crawler_runner.go b/crawler_runner.go index c27dce4..adc3e99 100644 --- a/crawler_runner.go +++ b/crawler_runner.go @@ -7,7 +7,9 @@ import ( "os/exec" "time" "strconv" - "strings" + + "io/ioutil" + "os" ) @@ -25,14 +27,18 @@ func RunContainer(container *string, cpm *map[string][]string) { time.Sleep(time.Duration( time.Second * 2)) - b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm) + paths := (*cpm)[*container] + b := CallInit2("localhost:" + strconv.Itoa(currentPort), &paths) log.Println("current Port:" , currentPort) if b == false { currentPort++ continue; } log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid ) + //write pid file container_pid_port + ioutil.WriteFile(PidFolder+*container+"_"+strconv.Itoa(cmd.Process.Pid)+ "_"+strconv.Itoa(currentPort), []byte(""), os.ModePerm) currentPort++ + break; } @@ -44,93 +50,3 @@ func getRunCommand(container *string ) string { return BinaryFolder + "/" + *container + "/" + runFile } - -//FIXME:: process check!!!!!! -func ExeCrawler(crawlerName string, exePath string) { - log.Println("Run Crawler") - - for { - pArg := "-Port=" + strconv.Itoa(currentPort) - cmd := exec.Command(exePath, pArg) - - err := cmd.Start() - if err != nil { - log.Println(err) - } - - time.Sleep(time.Duration( time.Second * 2)) - - log.Println("current Pid : " , cmd.Process.Pid) - - if IsAlive(cmd.Process.Pid) == false || IsState(cmd.Process.Pid) == false { - log.Println("run fail port:",currentPort) - currentPort++ - continue - } - - portMap[crawlerName] = strconv.Itoa(currentPort) - log.Println("current port:", currentPort) - //FIXME::remove - break - } - - -} - -func IsAlive(pid int) bool { - if pid < 0 { - return false - } - - ps := exec.Command("ps", "-aux") - awk := exec.Command("awk", "{print $2}") - grep := exec.Command("grep", strconv.Itoa(pid)) - - awk.Stdin, _ = ps.StdoutPipe() - grep.Stdin,_ = awk.StdoutPipe() - - ps.Start() - awk.Start() - - byt, _ := grep.Output() - - str := strconv.Itoa(pid) - - if len(str) +1 == len(byt) { - return true - } - return false - -} - -func IsState(pid int) bool { - ps := exec.Command("ps", "-aux") - awk := exec.Command("awk", "{print $2, $8}") - grep := exec.Command("grep", strconv.Itoa(pid)) - awk2 := exec.Command("awk", "{print $2}") - - awk.Stdin, _ = ps.StdoutPipe() - grep.Stdin,_ = awk.StdoutPipe() - awk2.Stdin,_ = grep.StdoutPipe() - - ps.Start() - awk.Start() - grep.Start() - - byt, _ := awk2.Output() - - if len(byt) <= 0 { - return false - } - - str := string(byt) - str = strings.TrimSpace(str) - - log.Println(str) - idx := strings.Index(str, "Z") - if idx >= 0 { - return false - } - return true -} -