From 3283a7f558c27036f910dab5c00d5fa880c769ec Mon Sep 17 00:00:00 2001 From: "jackdaw@loafle.com" Date: Thu, 13 Apr 2017 16:09:16 +0900 Subject: [PATCH] example config --- config/config.go | 37 +++++++++++ crawler.go | 17 +++--- crawler_test.go | 36 ----------- .../redis_protocol.go | 3 +- socket_health_crawler.go | 20 +++--- test/crawler_test.go | 61 +++++++++++++++++++ 6 files changed, 121 insertions(+), 53 deletions(-) create mode 100644 config/config.go delete mode 100644 crawler_test.go create mode 100644 test/crawler_test.go diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..2585c4a --- /dev/null +++ b/config/config.go @@ -0,0 +1,37 @@ +package config + + +type Auth struct { +} + +type Connection struct { + Ip string `json:"ip"` + Port string `json:"port"` + PortType string `json:"portType"` + SSL bool `json:"ssl"` +} + +type Target struct { + Auth Auth + Connection Connection +} + +type Schedule struct { + Interval string `json:"interval"` +} + +type Item struct { +} + +type Crawler struct { + Name string `json:"name"` + Container string `json:"container"` +} + +type Config struct { + Id string `json:"id"` + Target Target `json:"target"` + Schedule Schedule `json:"schedule"` + Crawler Crawler `json:"crawler"` + Items []Item `json:"items"` +} diff --git a/crawler.go b/crawler.go index 8a03e90..c9fd568 100644 --- a/crawler.go +++ b/crawler.go @@ -5,10 +5,11 @@ import ( "errors" "io/ioutil" "log" + "loafle.com/overflow/crawler_go/config" ) type Internal interface { - Internal(params map[string]interface{}) ([]byte, error) + Internal(params config.Config) ([]byte, error) } type Crawler interface { @@ -20,14 +21,14 @@ type Crawler interface { type CrawlerImpl struct { Crawler - configs map[string]interface{} + configs map[string] config.Config internal Internal configPath string } func (c *CrawlerImpl) Init(path string) ([]byte, error) { if c.configs == nil { - c.configs = make(map[string]interface{}, 0) + c.configs = make(map[string]config.Config, 0) } c.configPath = path @@ -56,7 +57,7 @@ func (c *CrawlerImpl) Add(id string) ([]byte, error) { log.Panic(err) } - var m = make(map[string]interface{}, 0) + var m = config.Config{} json.Unmarshal(b, &m) c.configs[id] = m @@ -81,15 +82,15 @@ func (c *CrawlerImpl) Get(id string) ([]byte, error) { } // internal methods -func (c *CrawlerImpl) GetConfig(id string) map[string]interface{} { - return c.configs[id].(map[string]interface{}) +func (c *CrawlerImpl) GetConfig(id string) config.Config { + return c.configs[id] } func (c *CrawlerImpl) SetInternal(i Internal) { c.internal = i } -func (c *CrawlerImpl) PutConfig(name string, m map[string]interface{}) { +func (c *CrawlerImpl) PutConfig(name string, m config.Config) { if c.configs == nil { - c.configs = make(map[string]interface{}, 0) + c.configs = make(map[string]config.Config, 0) } c.configs[name] = m } diff --git a/crawler_test.go b/crawler_test.go deleted file mode 100644 index 9ccd392..0000000 --- a/crawler_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package crawler - -import ( - "testing" - "encoding/json" - "github.com/stretchr/testify/assert" -) - -func initConfig() CrawlerImpl{ - c := CrawlerImpl{} - c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/") - return c -} - - -func TestCrawlerInit(t *testing.T) { - - c := initConfig() - m := c.configs["example.json"].(map[string]interface{}) - assert.Equal(t, m["id"].(string) , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734") - -} - -func TestCrawlerAdd(t *testing.T) { - -} - -func TestCrawlerRemove(t *testing.T) { - -} - -func TestCrawlerGet(t *testing.T) { - -} - - diff --git a/health_crawler/redis_protocol_crawler_go/redis_protocol.go b/health_crawler/redis_protocol_crawler_go/redis_protocol.go index fe980ac..9cb263c 100644 --- a/health_crawler/redis_protocol_crawler_go/redis_protocol.go +++ b/health_crawler/redis_protocol_crawler_go/redis_protocol.go @@ -4,13 +4,14 @@ import ( "encoding/json" "loafle.com/overflow/commons_go/matcher/redis" "loafle.com/overflow/crawler_go" + "loafle.com/overflow/crawler_go/config" ) type RedisHeahthCrawler struct { crawler.SocketHeahthCrawler } -func (r *RedisHeahthCrawler) Internal(params map[string]interface{}) ([]byte, error) { +func (r *RedisHeahthCrawler) Internal(params config.Config) ([]byte, error) { b, err := r.CheckHeahth(params) if err != nil { diff --git a/socket_health_crawler.go b/socket_health_crawler.go index a2584c7..ab9f44e 100644 --- a/socket_health_crawler.go +++ b/socket_health_crawler.go @@ -6,6 +6,7 @@ import ( "loafle.com/overflow/commons_go/matcher/packet" "net" "loafle.com/overflow/commons_go/model/scaninfo" + "loafle.com/overflow/crawler_go/config" ) type SocketHeahthCrawler struct { @@ -17,12 +18,14 @@ func (s *SocketHeahthCrawler) SetMatcher(m matcher.Matcher) { s.m = m } -func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net.Conn, error) { - - ip := params["ip"].(string) - port := params["port"].(string) - portType := params["portType"].(string) - ssl := params["ssl"].(bool) +func (s *SocketHeahthCrawler) getConnection(params config.Config) (net.Conn, error) { + + connection := params.Target.Connection + + ip := connection.Ip + port := connection.Port + portType := connection.PortType + ssl := connection.SSL var addr string = ip addr += ":" @@ -51,7 +54,7 @@ func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net. } } -func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool, error) { +func (s *SocketHeahthCrawler) CheckHeahth(params config.Config) (bool, error) { conn, err := s.getConnection(params) if err != nil { @@ -59,7 +62,8 @@ func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool, } defer conn.Close() - info := scaninfo.NewScanInfoImpl(params["ip"].(string),params["port"].(string)) + connection := params.Target.Connection + info := scaninfo.NewScanInfoImpl(connection.Ip,connection.Port) if s.m.IsPrePacket() == true { bytes := make([]byte, 1024) diff --git a/test/crawler_test.go b/test/crawler_test.go new file mode 100644 index 0000000..d60d049 --- /dev/null +++ b/test/crawler_test.go @@ -0,0 +1,61 @@ +package test + +import ( + "testing" + "github.com/stretchr/testify/assert" + "encoding/json" + "loafle.com/overflow/crawler_go/health_crawler/redis_protocol_crawler_go" + "loafle.com/overflow/crawler_go" + +) + +func initConfig() crawler.CrawlerImpl{ + c := crawler.CrawlerImpl{} + c.SetInternal(redis_protocol_crawler_go.NewRedisHeahthCrawler()) + c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/") + return c +} + + +func TestCrawlerInit(t *testing.T) { + + c := initConfig() + m := c.GetConfig("example.json") + assert.Equal(t, m.Id , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734") + +} + +func TestCrawlerAdd(t *testing.T) { + + c := initConfig() + c.Remove("example.json") + b , _ := c.Add("example.json") + + var ck bool + json.Unmarshal(b,&ck) + + assert.Equal(t,true,ck) + +} + +func TestCrawlerRemove(t *testing.T) { + c := initConfig() + b, _ := c.Remove("example.json") + + var ck bool + json.Unmarshal(b,&ck) + + assert.Equal(t,true,ck) +} + +func TestCrawlerGet(t *testing.T) { + c := initConfig() + b, _ := c.Get("example.json") + + var ck bool + json.Unmarshal(b,&ck) + + assert.Equal(t,true,ck) +} + +