204 lines
2.7 KiB
Go
204 lines
2.7 KiB
Go
package crawler_manager
|
|
|
|
import (
|
|
|
|
|
|
g "loafle.com/overflow/crawler_go/grpc"
|
|
|
|
"loafle.com/overflow/crawler_go/config"
|
|
|
|
"encoding/json"
|
|
|
|
"io/ioutil"
|
|
|
|
|
|
)
|
|
|
|
const (
|
|
address = "localhost:"
|
|
defaultPort = 80
|
|
|
|
rootFolder = "/home/cm/"
|
|
ConfigFolder = rootFolder + "/config/"
|
|
BinaryFolder = rootFolder + "/container/"
|
|
PidFolder = rootFolder + "/pids/"
|
|
runFile = "tnc"
|
|
)
|
|
|
|
|
|
type crawler_manager struct {
|
|
|
|
}
|
|
|
|
|
|
|
|
var currentPort = defaultPort
|
|
|
|
var pidMap map[string]int
|
|
var exeMap map[string]string
|
|
var portMap map[string]string
|
|
|
|
func init() {
|
|
|
|
pidMap = make(map[string]int)
|
|
exeMap = make(map[string]string)
|
|
portMap = make(map[string]string)
|
|
|
|
//pidMap["HEALTH_REDIS"] = 18281
|
|
|
|
exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";
|
|
|
|
}
|
|
|
|
func ReadConfig(path string ) *config.Config {
|
|
|
|
bytes, err := ioutil.ReadFile(path)
|
|
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
c := config.Config{}
|
|
|
|
json.Unmarshal(bytes, &c)
|
|
|
|
return &c
|
|
|
|
|
|
}
|
|
|
|
func AddConfig(path string) {
|
|
|
|
c := ReadConfig(path)
|
|
|
|
exePath := exeMap[c.Crawler.Name]
|
|
|
|
cs := c.Crawler.Name
|
|
|
|
pid := pidMap[cs]
|
|
|
|
if pid > 0 {
|
|
b := IsAlive(pid)
|
|
if b == false {
|
|
ExeCrawler(c.Crawler.Name, exePath)
|
|
CallInit(c.Crawler.Name, path)
|
|
} else {
|
|
CallAdd()
|
|
}
|
|
|
|
|
|
} else {
|
|
ExeCrawler(c.Crawler.Name, exePath)
|
|
CallInit(c.Crawler.Name, path)
|
|
}
|
|
|
|
}
|
|
|
|
|
|
func InitContainer() {
|
|
|
|
cs := IsStartContainer()
|
|
|
|
var cpm map[string][]string = make(map[string][]string)
|
|
|
|
var ccl []string
|
|
for _, c := range cs {
|
|
ExistConfigFileDir(ConfigFolder, c, &ccl)
|
|
cpm[c] = ccl
|
|
RunContainer(&c, &cpm)
|
|
}
|
|
|
|
//startContainer
|
|
//isPid
|
|
|
|
//
|
|
|
|
}
|
|
|
|
func IsStartContainer() []string {
|
|
|
|
files, _ := ioutil.ReadDir(ConfigFolder)
|
|
|
|
var cs []string
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
b := ExistConfigFile(ConfigFolder, file.Name())
|
|
if b {
|
|
cs = append(cs, file.Name())
|
|
}
|
|
}
|
|
|
|
}
|
|
return cs
|
|
}
|
|
|
|
func ExistConfigFile(prePath string,dir string) bool {
|
|
|
|
files, _ := ioutil.ReadDir(prePath + "/" +dir)
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
retB := ExistConfigFile(prePath + "/" + dir, file.Name())
|
|
if retB {
|
|
return true
|
|
}
|
|
} else {
|
|
return true
|
|
}
|
|
|
|
}
|
|
return false
|
|
}
|
|
|
|
func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) {
|
|
|
|
files, _ := ioutil.ReadDir(prePath + "/" +dir)
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler)
|
|
} else {
|
|
*configCrawler = append(*configCrawler, prePath + "/" +dir)
|
|
return;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//func ExistFile(path string) bool {
|
|
//
|
|
//
|
|
//
|
|
//}
|
|
|
|
|
|
func AddConfig22() {
|
|
|
|
}
|
|
|
|
func AddCrawler() {
|
|
|
|
}
|
|
|
|
|
|
|
|
//parameter crawler list
|
|
func GetCrawlerPort() {
|
|
//check crawler run
|
|
|
|
|
|
//crawler exe
|
|
//ExeCrawler()
|
|
|
|
//return crawler = port
|
|
//return portMap
|
|
} |