From 76ffa421e39b1fa17b4d5ea19408d68159ce0732 Mon Sep 17 00:00:00 2001 From: snoop Date: Fri, 14 Apr 2017 16:23:42 +0900 Subject: [PATCH] ing --- connection_pool.go | 9 +++ crawler_communicator.go | 37 +++++++++-- crawler_manager.go | 137 +++++++++++++++++++++++++++++++++++++--- crawler_manager_test.go | 76 +++++++++++++++++++++- crawler_runner.go | 41 +++++++++++- 5 files changed, 284 insertions(+), 16 deletions(-) create mode 100644 connection_pool.go diff --git a/connection_pool.go b/connection_pool.go new file mode 100644 index 0000000..973e899 --- /dev/null +++ b/connection_pool.go @@ -0,0 +1,9 @@ +package crawler_manager + +import "google.golang.org/grpc" + +func (c *crawler_manager)GetClient(container string) (*grpc.ClientConn, error) { + + + return nil, nil +} \ No newline at end of file diff --git a/crawler_communicator.go b/crawler_communicator.go index 11bd03a..841da7e 100644 --- a/crawler_communicator.go +++ b/crawler_communicator.go @@ -6,7 +6,7 @@ import ( "log" g "loafle.com/overflow/crawler_go/grpc" "context" - "loafle.com/overflow/crawler_go/config" + "encoding/json" ) @@ -34,9 +34,38 @@ func CallAdd() { } -func CallInit(c *config.Config, path string) { +func CallInit2(address string, cpm *map[string][]string) bool { + conn, err := grpc.Dial(address, grpc.WithInsecure()) + if err != nil { + log.Fatalf("did not connect: %v", err) + return false + } + defer conn.Close() - port := portMap[c.Crawler.Name] + cc := g.NewConfigClient(conn) + + in := &g.Init{} + + in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/" + //in.Name = g.Crawlers(g.Crawlers_value[""]) + in.Name = g.Crawlers_HEALTH_REDIS + + inArr := &g.InputArray{} + inArr.In = append(inArr.In, in) + + outInit, errInit := cc.Init(context.Background(), inArr) + if errInit != nil { + log.Println(errInit) + return false + } + log.Println(outInit) + + return true +} + +func CallInit(crawlerName string, path string) { + + port := portMap[crawlerName] conn, err := grpc.Dial(address+port, grpc.WithInsecure()) if err != nil { @@ -50,7 +79,7 @@ func CallInit(c *config.Config, path string) { in.Path = filepath.Dir(path)+ "/" - in.Name = g.Crawlers(g.Crawlers_value[c.Crawler.Name]) + in.Name = g.Crawlers(g.Crawlers_value[crawlerName]) inArr := &g.InputArray{} inArr.In = append(inArr.In, in) diff --git a/crawler_manager.go b/crawler_manager.go index 6fe112e..682af21 100644 --- a/crawler_manager.go +++ b/crawler_manager.go @@ -11,19 +11,31 @@ import ( "io/ioutil" + ) const ( address = "localhost:" - defaultPort = 50001 + defaultPort = 80 + + rootFolder = "/home/cm/" + ConfigFolder = rootFolder + "/config/" + BinaryFolder = rootFolder + "/container/" + PidFolder = rootFolder + "/pids/" + runFile = "tnc" ) + +type crawler_manager struct { + +} + + + var currentPort = defaultPort var pidMap map[string]int - var exeMap map[string]string - var portMap map[string]string func init() { @@ -55,7 +67,7 @@ func ReadConfig(path string ) *config.Config { } -func ManageCrawler(path string) { +func AddConfig(path string) { c := ReadConfig(path) @@ -68,16 +80,125 @@ func ManageCrawler(path string) { if pid > 0 { b := IsAlive(pid) if b == false { - ExeCrawler(c, exePath) - CallInit(c, path) + ExeCrawler(c.Crawler.Name, exePath) + CallInit(c.Crawler.Name, path) } else { CallAdd() } } else { - ExeCrawler(c, exePath) - CallInit(c, path) + ExeCrawler(c.Crawler.Name, exePath) + CallInit(c.Crawler.Name, path) } } + + +func InitContainer() { + + cs := IsStartContainer() + + var cpm map[string][]string = make(map[string][]string) + + var ccl []string + for _, c := range cs { + ExistConfigFileDir(ConfigFolder, c, &ccl) + cpm[c] = ccl + RunContainer(&c, &cpm) + } + + //startContainer + //isPid + + // + +} + +func IsStartContainer() []string { + + files, _ := ioutil.ReadDir(ConfigFolder) + + var cs []string + + for _,file := range files { + + if file.IsDir() { + b := ExistConfigFile(ConfigFolder, file.Name()) + if b { + cs = append(cs, file.Name()) + } + } + + } + return cs +} + +func ExistConfigFile(prePath string,dir string) bool { + + files, _ := ioutil.ReadDir(prePath + "/" +dir) + + for _,file := range files { + + if file.IsDir() { + retB := ExistConfigFile(prePath + "/" + dir, file.Name()) + if retB { + return true + } + } else { + return true + } + + } + return false +} + +func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) { + + files, _ := ioutil.ReadDir(prePath + "/" +dir) + + for _,file := range files { + + if file.IsDir() { + ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler) + } else { + *configCrawler = append(*configCrawler, prePath + "/" +dir) + return; + } + + } + + +} + + + + +//func ExistFile(path string) bool { +// +// +// +//} + + +func AddConfig22() { + +} + +func AddCrawler() { + +} + + + +//parameter crawler list +func GetCrawlerPort() { + //check crawler run + + + //crawler exe + //ExeCrawler() + + //return crawler = port + //return portMap +} \ No newline at end of file diff --git a/crawler_manager_test.go b/crawler_manager_test.go index 667b6f8..e44c5a0 100644 --- a/crawler_manager_test.go +++ b/crawler_manager_test.go @@ -10,10 +10,11 @@ import ( "path/filepath" "strings" + "io/ioutil" ) func TestManageCrawler(t *testing.T) { - ManageCrawler("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json") + AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json") } func TestAdd(t *testing.T) { @@ -138,3 +139,76 @@ func TestDir(t *testing.T) { func TestMyPid(t *testing.T) { t.Log(os.Getpid()) } + + +func TestDir22(t *testing.T) { + + cs := IsStartContainer() + + t.Log(cs) + +} + +func TestCrateDir(t *testing.T) { + + var configPath []string; + + + configPath = append(configPath, ConfigFolder + "/java/oracle/") + + configPath = append(configPath, ConfigFolder + "/go/test/") + + configPath = append(configPath, ConfigFolder + "/java/mysql/") + configPath = append(configPath, ConfigFolder + "/network/http/") + configPath = append(configPath, ConfigFolder + "/network/ldap/") + configPath = append(configPath, ConfigFolder + "/network/redis/") + + + var containerPath []string; + + containerPath = append(containerPath, BinaryFolder + "/java/") + containerPath = append(containerPath, BinaryFolder + "/go/") + containerPath = append(containerPath, BinaryFolder + "/network/") + + containerPath = append(containerPath, PidFolder) + + + + + b, _:=ioutil.ReadFile("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json") + + for i, p := range configPath { + pp := filepath.Join(p) + os.MkdirAll(pp, os.ModePerm) + + if i % 2 == 0 { + ioutil.WriteFile(pp + "/id.json", b, os.ModePerm) + } + + } + + for _, p := range containerPath { + pp := filepath.Join(p) + os.MkdirAll(pp, os.ModePerm) + + } + +} + +func TestInit(t *testing.T) { + InitContainer() +} + +func TestDirs(t *testing.T) { + + cs := IsStartContainer() + + for _, c := range cs { + var ccl []string + ExistConfigFileDir(ConfigFolder, c, &ccl) + t.Log(ccl) + + } + + +} \ No newline at end of file diff --git a/crawler_runner.go b/crawler_runner.go index a776bf9..c27dce4 100644 --- a/crawler_runner.go +++ b/crawler_runner.go @@ -3,15 +3,50 @@ package crawler_manager import ( "log" - "loafle.com/overflow/crawler_go/config" + "os/exec" "time" "strconv" "strings" ) + +func RunContainer(container *string, cpm *map[string][]string) { + cmdStr := getRunCommand(container) + + for { + pArg := "-Port=" + strconv.Itoa(currentPort) + cmd := exec.Command(cmdStr, pArg) + + err := cmd.Start() + if err != nil { + log.Println(err) + } + + time.Sleep(time.Duration( time.Second * 2)) + + b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm) + log.Println("current Port:" , currentPort) + if b == false { + currentPort++ + continue; + } + log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid ) + currentPort++ + break; + } + + + +} + +func getRunCommand(container *string ) string { + return BinaryFolder + "/" + *container + "/" + runFile +} + + //FIXME:: process check!!!!!! -func ExeCrawler(c *config.Config, exePath string) { +func ExeCrawler(crawlerName string, exePath string) { log.Println("Run Crawler") for { @@ -33,7 +68,7 @@ func ExeCrawler(c *config.Config, exePath string) { continue } - portMap[c.Crawler.Name] = strconv.Itoa(currentPort) + portMap[crawlerName] = strconv.Itoa(currentPort) log.Println("current port:", currentPort) //FIXME::remove break