146 lines
2.0 KiB
Go
146 lines
2.0 KiB
Go
package crawler_manager
|
|
|
|
import (
|
|
|
|
|
|
|
|
|
|
"loafle.com/overflow/crawler_go/config"
|
|
|
|
"encoding/json"
|
|
|
|
"io/ioutil"
|
|
|
|
"google.golang.org/grpc"
|
|
)
|
|
|
|
const (
|
|
address = "localhost:"
|
|
defaultPort = 50000
|
|
|
|
rootFolder = "/home/cm/"
|
|
ConfigFolder = rootFolder + "/config/"
|
|
BinaryFolder = rootFolder + "/container/"
|
|
PidFolder = rootFolder + "/pids/"
|
|
runFile = "tnc"
|
|
)
|
|
|
|
|
|
type CrawlerManager struct {
|
|
|
|
}
|
|
|
|
var currentPort = defaultPort
|
|
|
|
var pidMap map[string]int
|
|
|
|
var portMap map[string]string
|
|
|
|
func init() {
|
|
|
|
pidMap = make(map[string]int)
|
|
|
|
portMap = make(map[string]string)
|
|
|
|
}
|
|
|
|
func (c *CrawlerManager)GetClient(container string,port string) (*grpc.ClientConn, error) {
|
|
|
|
port = portMap[container]
|
|
|
|
return grpc.Dial("localhost:"+port, grpc.WithInsecure())
|
|
}
|
|
|
|
|
|
|
|
func ReadConfig(path string ) *config.Config {
|
|
|
|
bytes, err := ioutil.ReadFile(path)
|
|
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
|
|
c := config.Config{}
|
|
|
|
json.Unmarshal(bytes, &c)
|
|
|
|
return &c
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func InitContainer() {
|
|
|
|
cs := IsStartContainer()
|
|
|
|
var cpm map[string][]string = make(map[string][]string)
|
|
|
|
var ccl []string
|
|
for _, c := range cs {
|
|
ExistConfigFileDir(ConfigFolder, c, &ccl)
|
|
cpm[c] = ccl
|
|
RunContainer(&c, &cpm)
|
|
}
|
|
|
|
}
|
|
|
|
func IsStartContainer() []string {
|
|
|
|
files, _ := ioutil.ReadDir(ConfigFolder)
|
|
|
|
var cs []string
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
b := ExistConfigFile(ConfigFolder, file.Name())
|
|
if b {
|
|
cs = append(cs, file.Name())
|
|
}
|
|
}
|
|
|
|
}
|
|
return cs
|
|
}
|
|
|
|
func ExistConfigFile(prePath string,dir string) bool {
|
|
|
|
files, _ := ioutil.ReadDir(prePath + "/" +dir)
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
retB := ExistConfigFile(prePath + "/" + dir, file.Name())
|
|
if retB {
|
|
return true
|
|
}
|
|
} else {
|
|
return true
|
|
}
|
|
|
|
}
|
|
return false
|
|
}
|
|
|
|
func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) {
|
|
|
|
files, _ := ioutil.ReadDir(prePath + "/" +dir)
|
|
|
|
for _,file := range files {
|
|
|
|
if file.IsDir() {
|
|
ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler)
|
|
} else {
|
|
*configCrawler = append(*configCrawler, prePath + "/" +dir)
|
|
return;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|