overflow_backup_2018_0455/crawler_manager_go
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ing

Browse Source
This commit is contained in:
snoop 2017-04-14 17:19:02 +09:00
parent 7f0c37578e
commit 3424503357
4 changed files with 44 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
crawler_communicator.go
View File

@ -8,6 +8,7 @@ import (
"context"
"encoding/json"
"strings"
)
@ -34,7 +35,7 @@ func CallAdd() {
}
func CallInit2(address string, cpm *map[string][]string) bool {
func CallInit2(address string, paths *[]string) bool {
conn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
log.Fatalf("did not connect: %v", err)
@ -44,14 +45,22 @@ func CallInit2(address string, cpm *map[string][]string) bool {
cc := g.NewConfigClient(conn)
in := &g.Init{}
in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
//in.Name = g.Crawlers(g.Crawlers_value[""])
in.Name = g.Crawlers_HEALTH_REDIS
inArr := &g.InputArray{}
base := "HEALTH_"
for _, path := range *paths {
in := &g.Init{}
in.Path = "/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/"
//in.Path = path
bcn := filepath.Base(path)
bcn = strings.ToUpper(bcn)
log.Println("cn:" + base+bcn)
//in.Name = g.Crawlers(g.Crawlers_value[base+bcn])
in.Name = g.Crawlers(g.Crawlers_value[g.Crawlers_HEALTH_REDIS.String()]) //test
inArr.In = append(inArr.In, in)
}
outInit, errInit := cc.Init(context.Background(), inArr)
if errInit != nil {

45
crawler_manager.go
View File

@ -3,7 +3,7 @@ package crawler_manager
import (
g "loafle.com/overflow/crawler_go/grpc"
"loafle.com/overflow/crawler_go/config"
@ -16,7 +16,7 @@ import (
const (
address = "localhost:"
defaultPort = 80
defaultPort = 50000
rootFolder = "/home/cm/"
ConfigFolder = rootFolder + "/config/"
@ -30,24 +30,18 @@ type crawler_manager struct {
}
var currentPort = defaultPort
var pidMap map[string]int
var exeMap map[string]string
var portMap map[string]string
func init() {
pidMap = make(map[string]int)
exeMap = make(map[string]string)
portMap = make(map[string]string)
//pidMap["HEALTH_REDIS"] = 18281
exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";
}
func ReadConfig(path string ) *config.Config {
@ -67,32 +61,6 @@ func ReadConfig(path string ) *config.Config {
}
func AddConfig(path string) {
c := ReadConfig(path)
exePath := exeMap[c.Crawler.Name]
cs := c.Crawler.Name
pid := pidMap[cs]
if pid > 0 {
b := IsAlive(pid)
if b == false {
ExeCrawler(c.Crawler.Name, exePath)
CallInit(c.Crawler.Name, path)
} else {
CallAdd()
}
} else {
ExeCrawler(c.Crawler.Name, exePath)
CallInit(c.Crawler.Name, path)
}
}
func InitContainer() {
@ -108,11 +76,6 @@ func InitContainer() {
RunContainer(&c, &cpm)
}
//startContainer
//isPid
//
}
func IsStartContainer() []string {

33
crawler_manager_test.go
View File

@ -13,10 +13,6 @@ import (
"io/ioutil"
)
func TestManageCrawler(t *testing.T) {
AddConfig("/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_manager_go/config/example.json")
}
func TestAdd(t *testing.T) {
@ -100,19 +96,6 @@ func TestPipe(t *testing.T) {
t.Log(string(aa))
}
func TestIsAlive(t *testing.T) {
IsAlive(15538)
}
func TestIsState(t *testing.T) {
b := IsState(225931)
t.Log(b)
}
func TestCom(t *testing.T) {
str:= "Z"
@ -160,8 +143,9 @@ func TestCrateDir(t *testing.T) {
configPath = append(configPath, ConfigFolder + "/java/mysql/")
configPath = append(configPath, ConfigFolder + "/network/http/")
configPath = append(configPath, ConfigFolder + "/network/ldap/")
configPath = append(configPath, ConfigFolder + "/network/redis/")
configPath = append(configPath, ConfigFolder + "/network/ldap/")
var containerPath []string;
@ -210,5 +194,18 @@ func TestDirs(t *testing.T) {
}
}
func TestSplitPath(t *testing.T) {
ff :="/home/snoop/develop/path/go/src/loafle.com/overflow/crawler_go/config/ffff"
a := filepath.Dir(ff)
t.Log(a)
a = filepath.Clean(a)
t.Log(a)
a = filepath.Base(a)
t.Log(a)
}

100
crawler_runner.go
View File

@ -7,7 +7,9 @@ import (
"os/exec"
"time"
"strconv"
"strings"
"io/ioutil"
"os"
)
@ -25,14 +27,18 @@ func RunContainer(container *string, cpm *map[string][]string) {
time.Sleep(time.Duration( time.Second * 2))
b := CallInit2("localhost:" + strconv.Itoa(currentPort), cpm)
paths := (*cpm)[*container]
b := CallInit2("localhost:" + strconv.Itoa(currentPort), &paths)
log.Println("current Port:" , currentPort)
if b == false {
currentPort++
continue;
}
log.Println(*container + " run success port:" , currentPort , "pid:", cmd.Process.Pid )
//write pid file container_pid_port
ioutil.WriteFile(PidFolder+*container+"_"+strconv.Itoa(cmd.Process.Pid)+ "_"+strconv.Itoa(currentPort), []byte(""), os.ModePerm)
currentPort++
break;
}
@ -44,93 +50,3 @@ func getRunCommand(container *string ) string {
return BinaryFolder + "/" + *container + "/" + runFile
}
//FIXME:: process check!!!!!!
func ExeCrawler(crawlerName string, exePath string) {
log.Println("Run Crawler")
for {
pArg := "-Port=" + strconv.Itoa(currentPort)
cmd := exec.Command(exePath, pArg)
err := cmd.Start()
if err != nil {
log.Println(err)
}
time.Sleep(time.Duration( time.Second * 2))
log.Println("current Pid : " , cmd.Process.Pid)
if IsAlive(cmd.Process.Pid) == false || IsState(cmd.Process.Pid) == false {
log.Println("run fail port:",currentPort)
currentPort++
continue
}
portMap[crawlerName] = strconv.Itoa(currentPort)
log.Println("current port:", currentPort)
//FIXME::remove
break
}
}
func IsAlive(pid int) bool {
if pid < 0 {
return false
}
ps := exec.Command("ps", "-aux")
awk := exec.Command("awk", "{print $2}")
grep := exec.Command("grep", strconv.Itoa(pid))
awk.Stdin, _ = ps.StdoutPipe()
grep.Stdin,_ = awk.StdoutPipe()
ps.Start()
awk.Start()
byt, _ := grep.Output()
str := strconv.Itoa(pid)
if len(str) +1 == len(byt) {
return true
}
return false
}
func IsState(pid int) bool {
ps := exec.Command("ps", "-aux")
awk := exec.Command("awk", "{print $2, $8}")
grep := exec.Command("grep", strconv.Itoa(pid))
awk2 := exec.Command("awk", "{print $2}")
awk.Stdin, _ = ps.StdoutPipe()
grep.Stdin,_ = awk.StdoutPipe()
awk2.Stdin,_ = grep.StdoutPipe()
ps.Start()
awk.Start()
grep.Start()
byt, _ := awk2.Output()
if len(byt) <= 0 {
return false
}
str := string(byt)
str = strings.TrimSpace(str)
log.Println(str)
idx := strings.Index(str, "Z")
if idx >= 0 {
return false
}
return true
}