crawler_manager_go/crawler_manager.go

174 lines
2.5 KiB
Go
Raw Normal View History

2017-04-13 02:53:00 +00:00
package crawler_manager
import (
2017-04-14 02:29:55 +00:00
2017-04-13 02:53:00 +00:00
g "loafle.com/overflow/crawler_go/grpc"
2017-04-14 02:29:55 +00:00
"loafle.com/overflow/crawler_go/config"
"encoding/json"
"io/ioutil"
2017-04-14 07:23:42 +00:00
2017-04-13 02:53:00 +00:00
)
const (
2017-04-14 02:29:55 +00:00
address = "localhost:"
2017-04-14 07:23:42 +00:00
defaultPort = 80
rootFolder = "/home/cm/"
ConfigFolder = rootFolder + "/config/"
BinaryFolder = rootFolder + "/container/"
PidFolder = rootFolder + "/pids/"
runFile = "tnc"
2017-04-13 02:53:00 +00:00
)
2017-04-14 07:23:42 +00:00
type crawler_manager struct {
}
2017-04-14 02:29:55 +00:00
var currentPort = defaultPort
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
var pidMap map[string]int
var exeMap map[string]string
var portMap map[string]string
2017-04-13 03:45:00 +00:00
2017-04-14 02:29:55 +00:00
func init() {
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
pidMap = make(map[string]int)
exeMap = make(map[string]string)
portMap = make(map[string]string)
//pidMap["HEALTH_REDIS"] = 18281
exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";
2017-04-13 02:53:00 +00:00
}
2017-04-14 02:29:55 +00:00
func ReadConfig(path string ) *config.Config {
bytes, err := ioutil.ReadFile(path)
2017-04-13 02:53:00 +00:00
if err != nil {
2017-04-14 02:29:55 +00:00
return nil
2017-04-13 02:53:00 +00:00
}
2017-04-14 02:29:55 +00:00
c := config.Config{}
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
json.Unmarshal(bytes, &c)
2017-04-13 03:45:00 +00:00
2017-04-14 02:29:55 +00:00
return &c
2017-04-13 02:53:00 +00:00
}
2017-04-14 07:23:42 +00:00
func AddConfig(path string) {
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
c := ReadConfig(path)
exePath := exeMap[c.Crawler.Name]
cs := c.Crawler.Name
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
pid := pidMap[cs]
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
if pid > 0 {
b := IsAlive(pid)
if b == false {
2017-04-14 07:23:42 +00:00
ExeCrawler(c.Crawler.Name, exePath)
CallInit(c.Crawler.Name, path)
2017-04-14 02:29:55 +00:00
} else {
CallAdd()
}
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
} else {
2017-04-14 07:23:42 +00:00
ExeCrawler(c.Crawler.Name, exePath)
CallInit(c.Crawler.Name, path)
}
}
func InitContainer() {
cs := IsStartContainer()
var cpm map[string][]string = make(map[string][]string)
var ccl []string
for _, c := range cs {
ExistConfigFileDir(ConfigFolder, c, &ccl)
cpm[c] = ccl
RunContainer(&c, &cpm)
}
//startContainer
//isPid
//
}
func IsStartContainer() []string {
files, _ := ioutil.ReadDir(ConfigFolder)
var cs []string
for _,file := range files {
if file.IsDir() {
b := ExistConfigFile(ConfigFolder, file.Name())
if b {
cs = append(cs, file.Name())
}
}
}
return cs
}
func ExistConfigFile(prePath string,dir string) bool {
files, _ := ioutil.ReadDir(prePath + "/" +dir)
for _,file := range files {
if file.IsDir() {
retB := ExistConfigFile(prePath + "/" + dir, file.Name())
if retB {
return true
}
} else {
return true
}
2017-04-13 02:53:00 +00:00
}
2017-04-14 07:23:42 +00:00
return false
}
func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) {
files, _ := ioutil.ReadDir(prePath + "/" +dir)
for _,file := range files {
if file.IsDir() {
ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler)
} else {
*configCrawler = append(*configCrawler, prePath + "/" +dir)
return;
}
}
2017-04-13 02:53:00 +00:00
2017-04-14 02:29:55 +00:00
}
2017-04-14 07:23:42 +00:00