example config

config/config.go

@@ -0,0 +1,37 @@
+package config
+
+
+type Auth struct {
+}
+
+type Connection struct {
+	Ip       string `json:"ip"`
+	Port     string `json:"port"`
+	PortType string `json:"portType"`
+	SSL      bool `json:"ssl"`
+}
+
+type Target struct {
+	Auth       Auth
+	Connection Connection
+}
+
+type Schedule struct {
+	Interval string `json:"interval"`
+}
+
+type Item struct {
+}
+
+type Crawler struct {
+	Name      string `json:"name"`
+	Container string `json:"container"`
+}
+
+type Config struct {
+	Id       string   `json:"id"`
+	Target   Target   `json:"target"`
+	Schedule Schedule `json:"schedule"`
+	Crawler  Crawler  `json:"crawler"`
+	Items    []Item   `json:"items"`
+}

crawler.go

@@ -5,10 +5,11 @@ import (
 	"errors"
 	"io/ioutil"
 	"log"
+	"loafle.com/overflow/crawler_go/config"
 )
 
 type Internal interface {
-	Internal(params map[string]interface{}) ([]byte, error)
+	Internal(params config.Config) ([]byte, error)
 }
 
 type Crawler interface {
@@ -20,14 +21,14 @@ type Crawler interface {
 
 type CrawlerImpl struct {
 	Crawler
-	configs    map[string]interface{}
+	configs    map[string] config.Config
 	internal   Internal
 	configPath string
 }
 
 func (c *CrawlerImpl) Init(path string) ([]byte, error) {
 	if c.configs == nil {
-		c.configs = make(map[string]interface{}, 0)
+		c.configs = make(map[string]config.Config, 0)
 	}
 
 	c.configPath = path
@@ -56,7 +57,7 @@ func (c *CrawlerImpl) Add(id string) ([]byte, error) {
 		log.Panic(err)
 	}
 	
-	var m = make(map[string]interface{}, 0)
+	var m = config.Config{}
 	json.Unmarshal(b, &m)
 	c.configs[id] = m
 	
@@ -81,15 +82,15 @@ func (c *CrawlerImpl) Get(id string) ([]byte, error) {
 }
 
 // internal methods
-func (c *CrawlerImpl) GetConfig(id string) map[string]interface{} {
+func (c *CrawlerImpl) GetConfig(id string) config.Config {
-	return c.configs[id].(map[string]interface{})
+	return c.configs[id]
 }
 func (c *CrawlerImpl) SetInternal(i Internal) {
 	c.internal = i
 }
-func (c *CrawlerImpl) PutConfig(name string, m map[string]interface{}) {
+func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
 	if c.configs == nil {
-		c.configs = make(map[string]interface{}, 0)
+		c.configs = make(map[string]config.Config, 0)
 	}
 	c.configs[name] = m
 }

crawler_test.go

@@ -1,36 +0,0 @@
-package crawler
-
-import (
-	"testing"
-	"encoding/json"
-	"github.com/stretchr/testify/assert"
-)
-
-func initConfig() CrawlerImpl{
-	c := CrawlerImpl{}
-	c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/")
-	return c
-}
-
-
-func TestCrawlerInit(t *testing.T) {
-	
-	c := initConfig()
-	m := c.configs["example.json"].(map[string]interface{})
-	assert.Equal(t, m["id"].(string) , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734")
-	
-}
-
-func TestCrawlerAdd(t *testing.T) {
-
-}
-
-func TestCrawlerRemove(t *testing.T) {
-
-}
-
-func TestCrawlerGet(t *testing.T) {
-
-}
-
-

health_crawler/redis_protocol_crawler_go/redis_protocol.go

@@ -4,13 +4,14 @@ import (
 	"encoding/json"
 	"loafle.com/overflow/commons_go/matcher/redis"
 	"loafle.com/overflow/crawler_go"
+	"loafle.com/overflow/crawler_go/config"
 )
 
 type RedisHeahthCrawler struct {
 	crawler.SocketHeahthCrawler
 }
 
-func (r *RedisHeahthCrawler) Internal(params map[string]interface{}) ([]byte, error) {
+func (r *RedisHeahthCrawler) Internal(params config.Config) ([]byte, error) {
 
 	b, err := r.CheckHeahth(params)
 	if err != nil {

socket_health_crawler.go

@@ -6,6 +6,7 @@ import (
 	"loafle.com/overflow/commons_go/matcher/packet"
 	"net"
 	"loafle.com/overflow/commons_go/model/scaninfo"
+	"loafle.com/overflow/crawler_go/config"
 )
 
 type SocketHeahthCrawler struct {
@@ -17,12 +18,14 @@ func (s *SocketHeahthCrawler) SetMatcher(m matcher.Matcher) {
 	s.m = m
 }
 
-func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net.Conn, error) {
+func (s *SocketHeahthCrawler) getConnection(params config.Config) (net.Conn, error) {
-
+	
-	ip := params["ip"].(string)
+	connection := params.Target.Connection
-	port := params["port"].(string)
+	
-	portType := params["portType"].(string)
+	ip := connection.Ip
-	ssl := params["ssl"].(bool)
+	port := connection.Port
+	portType := connection.PortType
+	ssl := connection.SSL
 
 	var addr string = ip
 	addr += ":"
@@ -51,7 +54,7 @@ func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net.
 	}
 }
 
-func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool, error) {
+func (s *SocketHeahthCrawler) CheckHeahth(params config.Config) (bool, error) {
 
 	conn, err := s.getConnection(params)
 	if err != nil {
@@ -59,7 +62,8 @@ func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool,
 	}
 	defer conn.Close()
 	
-	info := scaninfo.NewScanInfoImpl(params["ip"].(string),params["port"].(string))
+	connection := params.Target.Connection
+	info := scaninfo.NewScanInfoImpl(connection.Ip,connection.Port)
 	
 	if s.m.IsPrePacket() == true {
 		bytes := make([]byte, 1024)

test/crawler_test.go

@@ -0,0 +1,61 @@
+package test
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+	"encoding/json"
+	"loafle.com/overflow/crawler_go/health_crawler/redis_protocol_crawler_go"
+	"loafle.com/overflow/crawler_go"
+	
+)
+
+func initConfig() crawler.CrawlerImpl{
+	c := crawler.CrawlerImpl{}
+	c.SetInternal(redis_protocol_crawler_go.NewRedisHeahthCrawler())
+	c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/")
+	return c
+}
+
+
+func TestCrawlerInit(t *testing.T) {
+	
+	c := initConfig()
+	m := c.GetConfig("example.json")
+	assert.Equal(t, m.Id , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734")
+	
+}
+
+func TestCrawlerAdd(t *testing.T) {
+	
+	c := initConfig()
+	c.Remove("example.json")
+	b , _ := c.Add("example.json")
+	
+	var ck bool
+	json.Unmarshal(b,&ck)
+	
+	assert.Equal(t,true,ck)
+	
+}
+
+func TestCrawlerRemove(t *testing.T) {
+	c := initConfig()
+	b, _ := c.Remove("example.json")
+	
+	var ck bool
+	json.Unmarshal(b,&ck)
+	
+	assert.Equal(t,true,ck)
+}
+
+func TestCrawlerGet(t *testing.T) {
+	c := initConfig()
+	b, _ := c.Get("example.json")
+	
+	var ck bool
+	json.Unmarshal(b,&ck)
+	
+	assert.Equal(t,true,ck)
+}
+
+