collector

collector.go

@@ -1,55 +1,70 @@
 package collector_go
 
 import (
-	"context"
+	confMng "loafle.com/overflow/agent_api/config_manager"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	s "loafle.com/overflow/collector_go/scheduler"
-	conf "loafle.com/overflow/crawler_go/config"
-	g "loafle.com/overflow/crawler_go/grpc"
-	crm "loafle.com/overflow/crawler_manager_go"
 	"log"
-	"os"
+	"sync"
-	"path/filepath"
-	"strings"
 	"time"
 )
 
-const CONFIG_ROOT = "/config/container"
+var (
+	instance *Collector
+	once     sync.Once
+)
+
+func init() {
+	AddObservers()
+}
+
+func GetInstance() *Collector {
+	once.Do(func() {
+		instance = &Collector{}
+	})
+	return instance
+}
+
+func AddObservers() {
+	go handleConfigLoaded()
+}
 
 type Collector struct {
 	scheduler   s.Scheduler
-	//configs   []*conf.Config
+	cm          confMng.ConfigManager
-	configs map[string]*conf.Config
+	addSensorCh chan interface{}
+	remSensorCh chan interface{}
 }
 
-func (c *Collector) Start() {
+func (c *Collector) start(conf confMng.ConfigManager) {
 	go func() {
-		c.configs = make(map[string]*conf.Config, 0)
+		c.cm = conf
-
-		if err := c.readAllConfig(); err != nil {
-			log.Println(err)
-		}
-
 		c.scheduler = s.Scheduler{}
 		c.scheduler.Init()
+		c.addSensorCh = make(chan interface{})
+		c.remSensorCh = make(chan interface{})
+		handleSensorAdded(c.addSensorCh)
+		handleSensorRemoved(c.remSensorCh)
 
-		for _, conf := range c.configs {
+		for _, conf := range c.cm.GetSensors() {
-			if err := c.addSensor(conf); err != nil {
+			if err := c.addSensor(conf.Id); err != nil {
 				log.Println(err)
 			}
 		}
 	}()
 }
 
+func (c *Collector) Stop() {
+	cleanObserver(c.addSensorCh, c.remSensorCh)
+	c.scheduler.RemoveAllSchedule()
+	c.scheduler.Stop()
+}
+
 func (c *Collector) collect(id string) {
 
-	conf := c.configs[id]
+	conf := c.cm.GetSensorById(id)
-
 	log.Printf("COLLECT %s - [ID: %s] [Crawler : %s]", time.Now(), conf.Id, conf.Crawler.Name)
 
+	/*
 		conn, err := crm.GetInstance().GetClient(conf.Crawler.Container)
 		if err != nil {
 			log.Println(err)
@@ -68,116 +83,17 @@ func (c *Collector) collect(id string) {
 			log.Println(err)
 		}
 		log.Println(out)
+	*/
 }
 
-//func (c *Collector) AddSensor(path string) {
+func (c *Collector) addSensor(sensorId string) error {
-//	config := c.readConfig(CONFIG_ROOT + path)
+	sensor := c.cm.GetSensorById(sensorId)
-//	if config != nil {
+	return c.scheduler.NewSchedule(sensorId, sensor.Schedule.Interval, c.collect)
-//		if err := c.addSensor(config); err != nil {
-//			log.Println(err)
-//		}
-//	}
-//}
-
-func (c *Collector) AddSensor(conf *conf.Config) {
-
-	if c.checkExist(conf.Id) {
-		log.Println("The Same Id already exists.")
-		return
-	}
-	c.configs[conf.Id] = conf
-
-	if conf != nil {
-		if err := c.addSensor(conf); err != nil {
-			log.Println(err)
-		}
-	}
 }
 
-func (c *Collector) RemoveSensor(id string) {
+func (c *Collector) removeSensor(id string) {
 	if err := c.scheduler.RemoveSchedule(id); err != nil {
 		log.Println(err)
-	}
-}
-
-func (c *Collector) UpdateSensor(newConf *conf.Config) {
-	if newConf != nil {
-		if !c.checkExist(newConf.Id) {
-			log.Println("Cannot update Sensor : ID not exist [" + newConf.Id + "]")
 		return
 	}
-
-		exConf := c.configs[newConf.Id]
-
-		if exConf.Schedule.Interval != newConf.Schedule.Interval {
-			c.scheduler.UpdateSchedule(newConf.Id, newConf.Schedule.Interval)
-		}
-
-		c.configs[newConf.Id] = newConf
-	}
-}
-
-func (c *Collector) Stop() {
-	c.scheduler.RemoveAllSchedule()
-	c.scheduler.Stop()
-}
-
-func (c *Collector) addSensor(conf *conf.Config) error {
-	return c.scheduler.NewSchedule(conf.Id, conf.Schedule.Interval, c.collect)
-}
-
-func (c *Collector) readAllConfig() error {
-
-	err := filepath.Walk(CONFIG_ROOT, func(path string, f os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !f.IsDir() && strings.HasSuffix(f.Name(), ".conf") {
-			c.readConfig(path)
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	if len(c.configs) <= 0 {
-		return errors.New("No configuration file found.")
-	}
-
-	return nil
-}
-
-func (c *Collector) readConfig(path string) *conf.Config {
-	bytes, err := ioutil.ReadFile(path)
-	if err != nil {
-		fmt.Println(err)
-		return nil
-	}
-	conf := conf.Config{}
-	json.Unmarshal(bytes, &conf)
-
-	if err := c.validateConfig(&conf, path); err != nil {
-		fmt.Println(err)
-		return nil
-	}
-	c.configs[conf.Id] = &conf
-	return &conf
-}
-
-func (c *Collector) validateConfig(conf *conf.Config, configPath string) error {
-
-	//todo : some validations
-	if c.checkExist(conf.Id) {
-		return errors.New("The Same Id already exists. " + configPath)
-	}
-	return nil
-}
-
-func (c *Collector) checkExist(id string) bool {
-	if _, exists := c.configs[id]; exists {
-		return true
-	}
-	return false
 }

collector_event.go

@@ -0,0 +1,47 @@
+package collector_go
+
+import (
+	"loafle.com/overflow/agent_api/config_manager"
+	"loafle.com/overflow/agent_api/observer"
+	"loafle.com/overflow/agent_api/observer/messages"
+)
+
+func handleConfigLoaded() {
+	ch := make(chan interface{}, 0)
+	observer.Add(messages.CONFIGMANAGER_LOADED, ch)
+
+	data := <-ch
+	confMng := data.(config_manager.ConfigManager)
+
+	coll := GetInstance()
+	coll.start(confMng)
+
+	observer.Remove(messages.CONFIGMANAGER_LOADED, ch)
+}
+
+func handleSensorAdded(ch chan interface{}) {
+	coll := GetInstance()
+	observer.Add(messages.ADD_SENSOR_2_END, ch)
+
+	for {
+		configId := <-ch
+		coll.addSensor(configId.(string))
+	}
+}
+
+func handleSensorRemoved(ch chan interface{}) {
+	coll := GetInstance()
+	observer.Add(messages.REMOVE_SENSOR_1, ch)
+
+	for {
+		configId := <-ch
+		coll.removeSensor(configId.(string))
+		observer.Notify(messages.REMOVE_SENSOR_2_END, configId)
+	}
+}
+
+func cleanObserver(ach chan interface{}, rch chan interface{}) {
+	observer.Remove(messages.ADD_SENSOR_2_END, ach)
+	observer.Remove(messages.REMOVE_SENSOR_1, rch)
+	observer.Notify(messages.COLLECTOR_STOPPED, true)
+}

collector_test.go

@@ -1,58 +1,28 @@
 package collector_go
 
 import (
-	"encoding/json"
+	s "loafle.com/overflow/collector_go/scheduler"
-	"fmt"
-	"io/ioutil"
-	conf "loafle.com/overflow/crawler_go/config"
-	"loafle.com/overflow/cron_go"
 	"log"
 	"testing"
 	"time"
 )
 
-func TestCallGet(t *testing.T) {
+func TestCollector(t *testing.T) {
-	c := Collector{}
+	sc := s.Scheduler{}
-	c.Start()
+	sc.Init()
-	time.Sleep(time.Second * 10)
 
-	//log.Println("add sensor")
+	for i := 0; i < 9999; i++ {
-	//c.AddSensor("/network/smb/t2.conf")
+		sc.NewSchedule(string(i), "5", test)
-	//time.Sleep(time.Second * 3)
-
-	log.Println("update sensor")
-	c.UpdateSensor(newConf())
-
-	time.Sleep(time.Second * 30)
-}
-
-func newConf() *conf.Config {
-	bytes, err := ioutil.ReadFile("/config/container/network/smb/smb2.conf")
-	if err != nil {
-		fmt.Println(err)
-		return nil
 	}
-	conf := conf.Config{}
+	sc.NewSchedule("#######################", "5", test)
-	json.Unmarshal(bytes, &conf)
-
-	return &conf
-}
-
-func TestPooling(t *testing.T) {
-	c := &cron.Cron{}
-	c.Start()
-	c.AddTask("polling", 3).Invoke(handlePolling)
-
-	time.Sleep(time.Second * 10)
-	fmt.Println("UPDATE")
-	c.UpdateTask("polling", 1)
 	time.Sleep(time.Second * 10)
 
-	fmt.Println("UPDATE")
+	sc.NewSchedule("#######################", "1", test)
-	c.UpdateTask("polling", 3)
+	time.Sleep(time.Second * 100)
-	time.Sleep(time.Second * 10)
 }
 
-func handlePolling() {
+func test(id string) {
-	fmt.Println(time.Now(), " polling")
+	if id == "#######################" {
+		log.Println(id)
+	}
 }

scheduler/scheduler.go

@@ -3,7 +3,6 @@ package scheduler
 import (
 	c "loafle.com/overflow/crawler_go"
 	"loafle.com/overflow/cron_go"
-	"log"
 	"strconv"
 	"sync"
 )
@@ -28,6 +27,7 @@ func (s *Scheduler) Stop() {
 }
 
 func (s *Scheduler) NewSchedule(id, interval string, fn interface{}) error {
+
 	return s.newSchedule(id, interval, fn)
 }
 
@@ -54,12 +54,3 @@ func (s *Scheduler) newSchedule(id string, interval string, fn interface{}) erro
 	}
 	return cron.AddTask(id, uint64(i)).Invoke(fn, id)
 }
-
-func (s *Scheduler) requestGet(id string) {
-	data, err := s.crawler.Get(id)
-	if err != nil {
-		log.Printf("[ID: %s] An error has occurred. %s", id, err.Error())
-		return
-	}
-	log.Println(data)
-}