ing
This commit is contained in:
snoop
parent
7f0c37578e
commit
3424503357
|
|
@ -8,6 +8,7 @@ import (
|
|||
"context"
|
||||
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -34,7 +35,7 @@ func CallAdd() {
|
|||
|
||||
}
|
||||
|
||||
func CallInit2(address string, cpm *map[string][]string) bool {
|
||||
func CallInit2(address string, paths *[]string) bool {
|
||||
conn, err := grpc.Dial(address, grpc.WithInsecure())
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect: %v", err)
|
||||
|
|
@ -44,14 +45,22 @@ func CallInit2(address string, cpm *map[string][]string) bool {
|
|||
|
||||
cc := g.NewConfigClient(conn)
|
||||
|
||||
in := &g.Init{}
|
||||
|
||||
in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
|
||||
//in.Name = g.Crawlers(g.Crawlers_value[""])
|
||||
in.Name = g.Crawlers_HEALTH_REDIS
|
||||
|
||||
inArr := &g.InputArray{}
|
||||
inArr.In = append(inArr.In, in)
|
||||
base := "HEALTH_"
|
||||
for _, path := range *paths {
|
||||
|
||||
in := &g.Init{}
|
||||
in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
|
||||
//in.Path = path
|
||||
bcn := filepath.Base(path)
|
||||
bcn = strings.ToUpper(bcn)
|
||||
|
||||
log.Println("cn:" + base+bcn)
|
||||
|
||||
//in.Name = g.Crawlers(g.Crawlers_value[base+bcn])
|
||||
in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test
|
||||
inArr.In = append(inArr.In, in)
|
||||
}
|
||||
|
||||
outInit, errInit := cc.Init(context.Background(), inArr)
|
||||
if errInit != nil {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package crawler_manager
|
|||
import (
|
||||
|
||||
|
||||
g "loafle.com/overflow/crawler_go/grpc"
|
||||
|
||||
|
||||
"loafle.com/overflow/crawler_go/config"
|
||||
|
||||
|
|
@ -16,7 +16,7 @@ import (
|
|||
|
||||
const (
|
||||
address = "localhost:"
|
||||
defaultPort = 80
|
||||
defaultPort = 50000
|
||||
|
||||
rootFolder = "/home/cm/"
|
||||
ConfigFolder = rootFolder + "/config/"
|
||||
|
|
@ -30,24 +30,18 @@ type crawler_manager struct {
|
|||
|
||||
}
|
||||
|
||||
|
||||
|
||||
var currentPort = defaultPort
|
||||
|
||||
var pidMap map[string]int
|
||||
var exeMap map[string]string
|
||||
|
||||
var portMap map[string]string
|
||||
|
||||
func init() {
|
||||
|
||||
pidMap = make(map[string]int)
|
||||
exeMap = make(map[string]string)
|
||||
|
||||
portMap = make(map[string]string)
|
||||
|
||||
//pidMap["HEALTH_REDIS"] = 18281
|
||||
|
||||
exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";
|
||||
|
||||
}
|
||||
|
||||
func ReadConfig(path string ) *config.Config {
|
||||
|
|
@ -67,32 +61,6 @@ func ReadConfig(path string ) *config.Config {
|
|||
|
||||
}
|
||||
|
||||
func AddConfig(path string) {
|
||||
|
||||
c := ReadConfig(path)
|
||||
|
||||
exePath := exeMap[c.Crawler.Name]
|
||||
|
||||
cs := c.Crawler.Name
|
||||
|
||||
pid := pidMap[cs]
|
||||
|
||||
if pid > 0 {
|
||||
b := IsAlive(pid)
|
||||
if b == false {
|
||||
ExeCrawler(c.Crawler.Name, exePath)
|
||||
CallInit(c.Crawler.Name, path)
|
||||
} else {
|
||||
CallAdd()
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
ExeCrawler(c.Crawler.Name, exePath)
|
||||
CallInit(c.Crawler.Name, path)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
func InitContainer() {
|
||||
|
|
@ -108,11 +76,6 @@ func InitContainer() {
|
|||
RunContainer(&c, &cpm)
|
||||
}
|
||||
|
||||
//startContainer
|
||||
//isPid
|
||||
|
||||
//
|
||||
|
||||
}
|
||||
|
||||
func IsStartContainer() []string {
|
||||
|
|
|
|||
|
|
@ -13,10 +13,6 @@ import (
|
|||
"io/ioutil"
|
||||
)
|
||||
|
||||
func TestManageCrawler(t *testing.T) {
|
||||
AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json")
|
||||
}
|
||||
|
||||
func TestAdd(t *testing.T) {
|
||||
|
||||
|
||||
|
|
@ -100,19 +96,6 @@ func TestPipe(t *testing.T) {
|
|||
t.Log(string(aa))
|
||||
}
|
||||
|
||||
func TestIsAlive(t *testing.T) {
|
||||
|
||||
IsAlive(15538)
|
||||
|
||||
}
|
||||
|
||||
func TestIsState(t *testing.T) {
|
||||
|
||||
b := IsState(225931)
|
||||
t.Log(b)
|
||||
|
||||
}
|
||||
|
||||
func TestCom(t *testing.T) {
|
||||
str:= "Z"
|
||||
|
||||
|
|
@ -160,8 +143,9 @@ func TestCrateDir(t *testing.T) {
|
|||
|
||||
configPath = append(configPath, ConfigFolder + "/java/mysql/")
|
||||
configPath = append(configPath, ConfigFolder + "/network/http/")
|
||||
configPath = append(configPath, ConfigFolder + "/network/ldap/")
|
||||
configPath = append(configPath, ConfigFolder + "/network/redis/")
|
||||
configPath = append(configPath, ConfigFolder + "/network/ldap/")
|
||||
|
||||
|
||||
|
||||
var containerPath []string;
|
||||
|
|
@ -210,5 +194,18 @@ func TestDirs(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSplitPath(t *testing.T) {
|
||||
|
||||
ff :="/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff"
|
||||
a := filepath.Dir(ff)
|
||||
t.Log(a)
|
||||
a = filepath.Clean(a)
|
||||
t.Log(a)
|
||||
a = filepath.Base(a)
|
||||
|
||||
t.Log(a)
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -7,7 +7,9 @@ import (
|
|||
"os/exec"
|
||||
"time"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -25,14 +27,18 @@ func RunContainer(container *string, cpm *map[string][]string) {
|
|||
|
||||
time.Sleep(time.Duration( time.Second * 2))
|
||||
|
||||
b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm)
|
||||
paths := (*cpm)[*container]
|
||||
b := CallInit2("localhost:" + strconv.Itoa(currentPort), &paths)
|
||||
log.Println("current Port:" , currentPort)
|
||||
if b == false {
|
||||
currentPort++
|
||||
continue;
|
||||
}
|
||||
log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid )
|
||||
//write pid file container_pid_port
|
||||
ioutil.WriteFile(PidFolder+*container+"_"+strconv.Itoa(cmd.Process.Pid)+ "_"+strconv.Itoa(currentPort), []byte(""), os.ModePerm)
|
||||
currentPort++
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -44,93 +50,3 @@ func getRunCommand(container *string ) string {
|
|||
return BinaryFolder + "/" + *container + "/" + runFile
|
||||
}
|
||||
|
||||
|
||||
//FIXME:: process check!!!!!!
|
||||
func ExeCrawler(crawlerName string, exePath string) {
|
||||
log.Println("Run Crawler")
|
||||
|
||||
for {
|
||||
pArg := "-Port=" + strconv.Itoa(currentPort)
|
||||
cmd := exec.Command(exePath, pArg)
|
||||
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration( time.Second * 2))
|
||||
|
||||
log.Println("current Pid : " , cmd.Process.Pid)
|
||||
|
||||
if IsAlive(cmd.Process.Pid) == false || IsState(cmd.Process.Pid) == false {
|
||||
log.Println("run fail port:",currentPort)
|
||||
currentPort++
|
||||
continue
|
||||
}
|
||||
|
||||
portMap[crawlerName] = strconv.Itoa(currentPort)
|
||||
log.Println("current port:", currentPort)
|
||||
//FIXME::remove
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
func IsAlive(pid int) bool {
|
||||
if pid < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
ps := exec.Command("ps", "-aux")
|
||||
awk := exec.Command("awk", "{print $2}")
|
||||
grep := exec.Command("grep", strconv.Itoa(pid))
|
||||
|
||||
awk.Stdin, _ = ps.StdoutPipe()
|
||||
grep.Stdin,_ = awk.StdoutPipe()
|
||||
|
||||
ps.Start()
|
||||
awk.Start()
|
||||
|
||||
byt, _ := grep.Output()
|
||||
|
||||
str := strconv.Itoa(pid)
|
||||
|
||||
if len(str) +1 == len(byt) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
func IsState(pid int) bool {
|
||||
ps := exec.Command("ps", "-aux")
|
||||
awk := exec.Command("awk", "{print $2, $8}")
|
||||
grep := exec.Command("grep", strconv.Itoa(pid))
|
||||
awk2 := exec.Command("awk", "{print $2}")
|
||||
|
||||
awk.Stdin, _ = ps.StdoutPipe()
|
||||
grep.Stdin,_ = awk.StdoutPipe()
|
||||
awk2.Stdin,_ = grep.StdoutPipe()
|
||||
|
||||
ps.Start()
|
||||
awk.Start()
|
||||
grep.Start()
|
||||
|
||||
byt, _ := awk2.Output()
|
||||
|
||||
if len(byt) <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
str := string(byt)
|
||||
str = strings.TrimSpace(str)
|
||||
|
||||
log.Println(str)
|
||||
idx := strings.Index(str, "Z")
|
||||
if idx >= 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user