crawler_manager_go/crawler_manager.go

package crawler_manager

import (


	g "loafle.com/overflow/crawler_go/grpc"

	"loafle.com/overflow/crawler_go/config"

	"encoding/json"

	"io/ioutil"


)

const (
	address = "localhost:"
	defaultPort = 80

	rootFolder = "/home/cm/"
	ConfigFolder = rootFolder + "/config/"
	BinaryFolder = rootFolder + "/container/"
	PidFolder = rootFolder + "/pids/"
	runFile = "tnc"
)


type crawler_manager struct {

}



var currentPort = defaultPort

var pidMap map[string]int
var exeMap map[string]string
var portMap map[string]string

func init() {

	pidMap = make(map[string]int)
	exeMap = make(map[string]string)
	portMap = make(map[string]string)

	//pidMap["HEALTH_REDIS"] = 18281

	exeMap[g.Crawlers_HEALTH_REDIS.String()] = "/home/snoop/develop/path/go/src/loafle.com/overflow/tnc";

}

func ReadConfig(path string ) *config.Config {

	bytes, err := ioutil.ReadFile(path)

	if err != nil {
		return nil
	}

	c := config.Config{}

	json.Unmarshal(bytes, &c)

	return &c


}

func AddConfig(path string) {

	c := ReadConfig(path)

	exePath := exeMap[c.Crawler.Name]

	cs := c.Crawler.Name

	pid := pidMap[cs]

	if pid > 0 {
		b := IsAlive(pid)
		if b == false {
			ExeCrawler(c.Crawler.Name, exePath)
			CallInit(c.Crawler.Name, path)
		} else {
			CallAdd()
		}


	} else {
		ExeCrawler(c.Crawler.Name, exePath)
		CallInit(c.Crawler.Name, path)
	}

}


func InitContainer() {

	cs := IsStartContainer()

	var cpm map[string][]string = make(map[string][]string)

	var ccl []string
	for _, c := range cs {
		ExistConfigFileDir(ConfigFolder, c, &ccl)
		cpm[c] = ccl
		RunContainer(&c, &cpm)
	}

	//startContainer
	//isPid

	//

}

func IsStartContainer() []string {

	files, _ := ioutil.ReadDir(ConfigFolder)

	var cs []string

	for _,file := range files {

		if file.IsDir() {
			b := ExistConfigFile(ConfigFolder, file.Name())
			if b {
				cs = append(cs, file.Name())
			}
		}

	}
	return cs
}

func ExistConfigFile(prePath string,dir string) bool {

	files, _ := ioutil.ReadDir(prePath + "/" +dir)

	for _,file := range files {

		if file.IsDir() {
			retB := ExistConfigFile(prePath + "/" + dir, file.Name())
			if retB {
				return true
			}
		} else {
			return true
		}

	}
	return false
}

func ExistConfigFileDir(prePath string,dir string, configCrawler *[]string) {

	files, _ := ioutil.ReadDir(prePath + "/" +dir)

	for _,file := range files {

		if file.IsDir() {
			ExistConfigFileDir(prePath + "/" + dir, file.Name(), configCrawler)
		} else {
			*configCrawler = append(*configCrawler, prePath + "/" +dir)
			return;
		}

	}


}