ing

crawler_communicator.go

@@ -8,6 +8,7 @@ import (
 	"context"
 
 	"encoding/json"
+	"strings"
 )
 
 
@@ -34,7 +35,7 @@ func CallAdd()  {
 
 }
 
-func CallInit2(address string, cpm *map[string][]string) bool {
+func CallInit2(address string, paths *[]string) bool {
 	conn, err := grpc.Dial(address, grpc.WithInsecure())
 	if err != nil {
 		log.Fatalf("did not connect: %v", err)
@@ -44,14 +45,22 @@ func CallInit2(address string, cpm *map[string][]string) bool {
 
 	cc := g.NewConfigClient(conn)
 
-	in := &g.Init{}
-
-	in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
-	//in.Name = g.Crawlers(g.Crawlers_value[""])
-	in.Name = g.Crawlers_HEALTH_REDIS
-
 	inArr := &g.InputArray{}
-	inArr.In = append(inArr.In, in)
+	base := "HEALTH_"
+	for _, path := range *paths {
+
+		in := &g.Init{}
+		in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
+		//in.Path = path
+		bcn := filepath.Base(path)
+		bcn = strings.ToUpper(bcn)
+
+		log.Println("cn:" + base+bcn)
+
+		//in.Name = g.Crawlers(g.Crawlers_value[base+bcn])
+		in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test
+		inArr.In = append(inArr.In, in)
+	}
 
 	outInit, errInit := cc.Init(context.Background(), inArr)
 	if errInit != nil {

crawler_manager.go

@@ -3,7 +3,7 @@ package crawler_manager
 import (
 
 
-	g "loafle.com/overflow/crawler_go/grpc"
+
 
 	"loafle.com/overflow/crawler_go/config"
 
@@ -16,7 +16,7 @@ import (
 
 const (
 	address = "localhost:"
-	defaultPort = 80
+	defaultPort = 50000
 
 	rootFolder = "/home/cm/"
 	ConfigFolder = rootFolder + "/config/"
@@ -30,24 +30,18 @@ type crawler_manager struct {
 
 }
 
-
-
 var currentPort = defaultPort
 
 var pidMap map[string]int
-var exeMap map[string]string
+
 var portMap map[string]string
 
 func init() {
 
 	pidMap = make(map[string]int)
-	exeMap = make(map[string]string)
+
 	portMap = make(map[string]string)
 
-	//pidMap["HEALTH_REDIS"] = 18281
-
-	exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";
-
 }
 
 func ReadConfig(path string ) *config.Config {
@@ -67,32 +61,6 @@ func ReadConfig(path string ) *config.Config {
 
 }
 
-func AddConfig(path string) {
-
-	c := ReadConfig(path)
-
-	exePath := exeMap[c.Crawler.Name]
-
-	cs := c.Crawler.Name
-
-	pid := pidMap[cs]
-
-	if pid > 0 {
-		b := IsAlive(pid)
-		if b == false {
-			ExeCrawler(c.Crawler.Name, exePath)
-			CallInit(c.Crawler.Name, path)
-		} else {
-			CallAdd()
-		}
-
-
-	} else {
-		ExeCrawler(c.Crawler.Name, exePath)
-		CallInit(c.Crawler.Name, path)
-	}
-
-}
 
 
 func InitContainer() {
@@ -108,11 +76,6 @@ func InitContainer() {
 		RunContainer(&c, &cpm)
 	}
 
-	//startContainer
-	//isPid
-
-	//
-
 }
 
 func IsStartContainer() []string {

crawler_manager_test.go

@@ -13,10 +13,6 @@ import (
 	"io/ioutil"
 )
 
-func TestManageCrawler(t *testing.T) {
-	AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json")
-}
-
 func TestAdd(t *testing.T) {
 
 
@@ -100,19 +96,6 @@ func TestPipe(t *testing.T) {
 	t.Log(string(aa))
 }
 
-func TestIsAlive(t *testing.T) {
-
-	IsAlive(15538)
-
-}
-
-func TestIsState(t *testing.T) {
-
-	b := IsState(225931)
-	t.Log(b)
-
-}
-
 func TestCom(t *testing.T) {
 	str:= "Z"
 
@@ -160,8 +143,9 @@ func TestCrateDir(t *testing.T) {
 
 	configPath = append(configPath, ConfigFolder + "/java/mysql/")
 	configPath = append(configPath, ConfigFolder + "/network/http/")
-	configPath = append(configPath, ConfigFolder + "/network/ldap/")
 	configPath = append(configPath, ConfigFolder + "/network/redis/")
+	configPath = append(configPath, ConfigFolder + "/network/ldap/")
+
 
 
 	var containerPath []string;
@@ -210,5 +194,18 @@ func TestDirs(t *testing.T) {
 
 	}
 
+}
+
+func TestSplitPath(t *testing.T) {
+
+	ff :="/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff"
+	a := filepath.Dir(ff)
+	t.Log(a)
+	a = filepath.Clean(a)
+	t.Log(a)
+	a = filepath.Base(a)
+
+	t.Log(a)
+
 
 }

crawler_runner.go

@@ -7,7 +7,9 @@ import (
 	"os/exec"
 	"time"
 	"strconv"
-	"strings"
+
+	"io/ioutil"
+	"os"
 )
 
 
@@ -25,14 +27,18 @@ func RunContainer(container *string, cpm *map[string][]string) {
 
 		time.Sleep(time.Duration( time.Second * 2))
 
-		b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm)
+		paths := (*cpm)[*container]
+		b := CallInit2("localhost:" + strconv.Itoa(currentPort), &paths)
 		log.Println("current Port:" , currentPort)
 		if b == false {
 			currentPort++
 			continue;
 		}
 		log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid )
+		//write pid file     container_pid_port
+		ioutil.WriteFile(PidFolder+*container+"_"+strconv.Itoa(cmd.Process.Pid)+ "_"+strconv.Itoa(currentPort), []byte(""), os.ModePerm)
 		currentPort++
+
 		break;
 	}
 
@@ -44,93 +50,3 @@ func getRunCommand(container *string ) string {
 	return BinaryFolder + "/" + *container + "/" + runFile
 }
 
-
-//FIXME:: process check!!!!!!
-func ExeCrawler(crawlerName string, exePath string) {
-	log.Println("Run Crawler")
-
-	for {
-		pArg := "-Port=" + strconv.Itoa(currentPort)
-		cmd := exec.Command(exePath, pArg)
-
-		err := cmd.Start()
-		if err != nil {
-			log.Println(err)
-		}
-
-		time.Sleep(time.Duration( time.Second * 2))
-
-		log.Println("current Pid : " , cmd.Process.Pid)
-
-		if IsAlive(cmd.Process.Pid) == false || IsState(cmd.Process.Pid) == false {
-			log.Println("run fail port:",currentPort)
-			currentPort++
-			continue
-		}
-
-		portMap[crawlerName] = strconv.Itoa(currentPort)
-		log.Println("current port:", currentPort)
-		//FIXME::remove
-		break
-	}
-
-
-}
-
-func IsAlive(pid int) bool {
-	if pid < 0 {
-		return false
-	}
-
-	ps := exec.Command("ps", "-aux")
-	awk := exec.Command("awk", "{print $2}")
-	grep := exec.Command("grep", strconv.Itoa(pid))
-
-	awk.Stdin, _ = ps.StdoutPipe()
-	grep.Stdin,_ = awk.StdoutPipe()
-
-	ps.Start()
-	awk.Start()
-
-	byt, _ := grep.Output()
-
-	str := strconv.Itoa(pid)
-
-	if len(str) +1 == len(byt) {
-		return true
-	}
-	return false
-
-}
-
-func IsState(pid int) bool {
-	ps := exec.Command("ps", "-aux")
-	awk := exec.Command("awk", "{print $2, $8}")
-	grep := exec.Command("grep", strconv.Itoa(pid))
-	awk2 := exec.Command("awk", "{print $2}")
-
-	awk.Stdin, _ = ps.StdoutPipe()
-	grep.Stdin,_ = awk.StdoutPipe()
-	awk2.Stdin,_ = grep.StdoutPipe()
-
-	ps.Start()
-	awk.Start()
-	grep.Start()
-
-	byt, _ := awk2.Output()
-
-	if len(byt) <= 0 {
-		return false
-	}
-
-	str := string(byt)
-	str = strings.TrimSpace(str)
-
-	log.Println(str)
-	idx := strings.Index(str, "Z")
-	if idx >= 0 {
-		return false
-	}
-	return true
-}
-