ing

connection_pool.go

@@ -0,0 +1,9 @@
+package crawler_manager
+
+import "google.golang.org/grpc"
+
+func (c *crawler_manager)GetClient(container string) (*grpc.ClientConn, error) {
+
+
+	return nil, nil
+}

crawler_communicator.go

@@ -6,7 +6,7 @@ import (
 	"log"
 	g "loafle.com/overflow/crawler_go/grpc"
 	"context"
-	"loafle.com/overflow/crawler_go/config"
+
 	"encoding/json"
 )
 
@@ -34,9 +34,38 @@ func CallAdd()  {
 
 }
 
-func CallInit(c *config.Config, path string) {
+func CallInit2(address string, cpm *map[string][]string) bool {
+	conn, err := grpc.Dial(address, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("did not connect: %v", err)
+		return false
+	}
+	defer conn.Close()
 
-	port := portMap[c.Crawler.Name]
+	cc := g.NewConfigClient(conn)
+
+	in := &g.Init{}
+
+	in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
+	//in.Name = g.Crawlers(g.Crawlers_value[""])
+	in.Name = g.Crawlers_HEALTH_REDIS
+
+	inArr := &g.InputArray{}
+	inArr.In = append(inArr.In, in)
+
+	outInit, errInit := cc.Init(context.Background(), inArr)
+	if errInit != nil {
+		log.Println(errInit)
+		return false
+	}
+	log.Println(outInit)
+
+	return true
+}
+
+func CallInit(crawlerName string, path string) {
+
+	port := portMap[crawlerName]
 
 	conn, err := grpc.Dial(address+port, grpc.WithInsecure())
 	if err != nil {
@@ -50,7 +79,7 @@ func CallInit(c *config.Config, path string) {
 
 	in.Path = filepath.Dir(path)+ "/"
 
-	in.Name = g.Crawlers(g.Crawlers_value[c.Crawler.Name])
+	in.Name = g.Crawlers(g.Crawlers_value[crawlerName])
 
 	inArr := &g.InputArray{}
 	inArr.In = append(inArr.In, in)

crawler_manager.go

@@ -11,19 +11,31 @@ import (
 
 	"io/ioutil"
 
+
 )
 
 const (
 	address = "localhost:"
-	defaultPort = 50001
+	defaultPort = 80
+
+	rootFolder = "/home/cm/"
+	ConfigFolder = rootFolder + "/config/"
+	BinaryFolder = rootFolder + "/container/"
+	PidFolder = rootFolder + "/pids/"
+	runFile = "tnc"
 )
 
+
+type crawler_manager struct {
+
+}
+
+
+
 var currentPort = defaultPort
 
 var pidMap map[string]int
-
 var exeMap map[string]string
-
 var portMap map[string]string
 
 func init() {
@@ -55,7 +67,7 @@ func ReadConfig(path string ) *config.Config {
 
 }
 
-func ManageCrawler(path string) {
+func AddConfig(path string) {
 
 	c := ReadConfig(path)
 
@@ -68,16 +80,125 @@ func ManageCrawler(path string) {
 	if pid > 0 {
 		b := IsAlive(pid)
 		if b == false {
-			ExeCrawler(c, exePath)
-			CallInit(c, path)
+			ExeCrawler(c.Crawler.Name, exePath)
+			CallInit(c.Crawler.Name, path)
 		} else {
 			CallAdd()
 		}
 
 
 	} else {
-		ExeCrawler(c, exePath)
-		CallInit(c, path)
+		ExeCrawler(c.Crawler.Name, exePath)
+		CallInit(c.Crawler.Name, path)
 	}
 
 }
+
+
+func InitContainer() {
+
+	cs := IsStartContainer()
+
+	var cpm map[string][]string = make(map[string][]string)
+
+	var ccl []string
+	for _, c := range cs {
+		ExistConfigFileDir(ConfigFolder, c, &ccl)
+		cpm[c] = ccl
+		RunContainer(&c, &cpm)
+	}
+
+	//startContainer
+	//isPid
+
+	//
+
+}
+
+func IsStartContainer() []string {
+
+	files, _ := ioutil.ReadDir(ConfigFolder)
+
+	var cs []string
+
+	for _,file := range files {
+
+		if file.IsDir() {
+			b := ExistConfigFile(ConfigFolder, file.Name())
+			if b {
+				cs = append(cs, file.Name())
+			}
+		}
+
+	}
+	return cs
+}
+
+func ExistConfigFile(prePath string,dir string) bool {
+
+	files, _ := ioutil.ReadDir(prePath + "/" +dir)
+
+	for _,file := range files {
+
+		if file.IsDir() {
+			retB := ExistConfigFile(prePath + "/" + dir, file.Name())
+			if retB {
+				return true
+			}
+		} else {
+			return true
+		}
+
+	}
+	return false
+}
+
+func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) {
+
+	files, _ := ioutil.ReadDir(prePath + "/" +dir)
+
+	for _,file := range files {
+
+		if file.IsDir() {
+			ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler)
+		} else {
+			*configCrawler = append(*configCrawler, prePath + "/" +dir)
+			return;
+		}
+
+	}
+
+
+}
+
+
+
+
+//func ExistFile(path string) bool {
+//
+//
+//
+//}
+
+
+func AddConfig22() {
+
+}
+
+func AddCrawler() {
+
+}
+
+
+
+//parameter crawler list
+func GetCrawlerPort() {
+	//check crawler run
+
+
+	//crawler exe
+	//ExeCrawler()
+
+	//return crawler = port
+	//return portMap
+}

crawler_manager_test.go

@@ -10,10 +10,11 @@ import (
 	"path/filepath"
 
 	"strings"
+	"io/ioutil"
 )
 
 func TestManageCrawler(t *testing.T) {
-	ManageCrawler("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json")
+	AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json")
 }
 
 func TestAdd(t *testing.T) {
@@ -138,3 +139,76 @@ func TestDir(t *testing.T) {
 func TestMyPid(t *testing.T) {
 	t.Log(os.Getpid())
 }
+
+
+func TestDir22(t *testing.T) {
+
+	cs := IsStartContainer()
+
+	t.Log(cs)
+
+}
+
+func TestCrateDir(t *testing.T) {
+
+	var configPath []string;
+
+
+	configPath = append(configPath, ConfigFolder + "/java/oracle/")
+
+	configPath = append(configPath, ConfigFolder + "/go/test/")
+
+	configPath = append(configPath, ConfigFolder + "/java/mysql/")
+	configPath = append(configPath, ConfigFolder + "/network/http/")
+	configPath = append(configPath, ConfigFolder + "/network/ldap/")
+	configPath = append(configPath, ConfigFolder + "/network/redis/")
+
+
+	var containerPath []string;
+
+	containerPath = append(containerPath, BinaryFolder + "/java/")
+	containerPath = append(containerPath, BinaryFolder + "/go/")
+	containerPath = append(containerPath, BinaryFolder + "/network/")
+
+	containerPath = append(containerPath, PidFolder)
+
+
+
+
+	b, _:=ioutil.ReadFile("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/example.json")
+
+	for i, p := range configPath {
+		pp := filepath.Join(p)
+		os.MkdirAll(pp, os.ModePerm)
+
+		if i % 2 == 0 {
+			ioutil.WriteFile(pp + "/id.json", b, os.ModePerm)
+		}
+
+	}
+
+	for _, p := range containerPath {
+		pp := filepath.Join(p)
+		os.MkdirAll(pp, os.ModePerm)
+
+	}
+
+}
+
+func TestInit(t *testing.T) {
+	InitContainer()
+}
+
+func TestDirs(t *testing.T) {
+
+	cs := IsStartContainer()
+
+	for _, c := range cs {
+		var ccl []string
+		ExistConfigFileDir(ConfigFolder, c, &ccl)
+		t.Log(ccl)
+
+	}
+
+
+}

crawler_runner.go

@@ -3,15 +3,50 @@ package crawler_manager
 import (
 
 	"log"
-	"loafle.com/overflow/crawler_go/config"
+
 	"os/exec"
 	"time"
 	"strconv"
 	"strings"
 )
 
+
+func RunContainer(container *string, cpm *map[string][]string) {
+	cmdStr := getRunCommand(container)
+
+	for {
+		pArg := "-Port=" + strconv.Itoa(currentPort)
+		cmd := exec.Command(cmdStr, pArg)
+
+		err := cmd.Start()
+		if err != nil {
+			log.Println(err)
+		}
+
+		time.Sleep(time.Duration( time.Second * 2))
+
+		b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm)
+		log.Println("current Port:" , currentPort)
+		if b == false {
+			currentPort++
+			continue;
+		}
+		log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid )
+		currentPort++
+		break;
+	}
+
+
+
+}
+
+func getRunCommand(container *string ) string {
+	return BinaryFolder + "/" + *container + "/" + runFile
+}
+
+
 //FIXME:: process check!!!!!!
-func ExeCrawler(c *config.Config, exePath string) {
+func ExeCrawler(crawlerName string, exePath string) {
 	log.Println("Run Crawler")
 
 	for {
@@ -33,7 +68,7 @@ func ExeCrawler(c *config.Config, exePath string) {
 			continue
 		}
 
-		portMap[c.Crawler.Name] = strconv.Itoa(currentPort)
+		portMap[crawlerName] = strconv.Itoa(currentPort)
 		log.Println("current port:", currentPort)
 		//FIXME::remove
 		break