overflow_backup_2018_0455/crawler_go
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

example config

Browse Source
This commit is contained in:
jackdaw@loafle.com 2017-04-13 16:09:16 +09:00
parent 5e6410451c
commit 3283a7f558
6 changed files with 121 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
config/config.go Normal file
View File

@ -0,0 +1,37 @@
package config
type Auth struct {
}
type Connection struct {
Ip string `json:"ip"`
Port string `json:"port"`
PortType string `json:"portType"`
SSL bool `json:"ssl"`
}
type Target struct {
Auth Auth
Connection Connection
}
type Schedule struct {
Interval string `json:"interval"`
}
type Item struct {
}
type Crawler struct {
Name string `json:"name"`
Container string `json:"container"`
}
type Config struct {
Id string `json:"id"`
Target Target `json:"target"`
Schedule Schedule `json:"schedule"`
Crawler Crawler `json:"crawler"`
Items []Item `json:"items"`
}

17
crawler.go
View File

@ -5,10 +5,11 @@ import (
"errors"
"io/ioutil"
"log"
"loafle.com/overflow/crawler_go/config"
)
type Internal interface {
Internal(params map[string]interface{}) ([]byte, error)
Internal(params config.Config) ([]byte, error)
}
type Crawler interface {
@ -20,14 +21,14 @@ type Crawler interface {
type CrawlerImpl struct {
Crawler
configs map[string]interface{}
configs map[string] config.Config
internal Internal
configPath string
}
func (c *CrawlerImpl) Init(path string) ([]byte, error) {
if c.configs == nil {
c.configs = make(map[string]interface{}, 0)
c.configs = make(map[string]config.Config, 0)
}
c.configPath = path
@ -56,7 +57,7 @@ func (c *CrawlerImpl) Add(id string) ([]byte, error) {
log.Panic(err)
}
var m = make(map[string]interface{}, 0)
var m = config.Config{}
json.Unmarshal(b, &m)
c.configs[id] = m
@ -81,15 +82,15 @@ func (c *CrawlerImpl) Get(id string) ([]byte, error) {
}
// internal methods
func (c *CrawlerImpl) GetConfig(id string) map[string]interface{} {
return c.configs[id].(map[string]interface{})
func (c *CrawlerImpl) GetConfig(id string) config.Config {
return c.configs[id]
}
func (c *CrawlerImpl) SetInternal(i Internal) {
c.internal = i
}
func (c *CrawlerImpl) PutConfig(name string, m map[string]interface{}) {
func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
if c.configs == nil {
c.configs = make(map[string]interface{}, 0)
c.configs = make(map[string]config.Config, 0)
}
c.configs[name] = m
}

36
crawler_test.go
View File

@ -1,36 +0,0 @@
package crawler
import (
"testing"
"encoding/json"
"github.com/stretchr/testify/assert"
)
func initConfig() CrawlerImpl{
c := CrawlerImpl{}
c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/")
return c
}
func TestCrawlerInit(t *testing.T) {
c := initConfig()
m := c.configs["example.json"].(map[string]interface{})
assert.Equal(t, m["id"].(string) , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734")
}
func TestCrawlerAdd(t *testing.T) {
}
func TestCrawlerRemove(t *testing.T) {
}
func TestCrawlerGet(t *testing.T) {
}

3
health_crawler/redis_protocol_crawler_go/redis_protocol.go
View File

@ -4,13 +4,14 @@ import (
"encoding/json"
"loafle.com/overflow/commons_go/matcher/redis"
"loafle.com/overflow/crawler_go"
"loafle.com/overflow/crawler_go/config"
)
type RedisHeahthCrawler struct {
crawler.SocketHeahthCrawler
}
func (r *RedisHeahthCrawler) Internal(params map[string]interface{}) ([]byte, error) {
func (r *RedisHeahthCrawler) Internal(params config.Config) ([]byte, error) {
b, err := r.CheckHeahth(params)
if err != nil {

18
socket_health_crawler.go
View File

@ -6,6 +6,7 @@ import (
"loafle.com/overflow/commons_go/matcher/packet"
"net"
"loafle.com/overflow/commons_go/model/scaninfo"
"loafle.com/overflow/crawler_go/config"
)
type SocketHeahthCrawler struct {
@ -17,12 +18,14 @@ func (s *SocketHeahthCrawler) SetMatcher(m matcher.Matcher) {
s.m = m
}
func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net.Conn, error) {
func (s *SocketHeahthCrawler) getConnection(params config.Config) (net.Conn, error) {
ip := params["ip"].(string)
port := params["port"].(string)
portType := params["portType"].(string)
ssl := params["ssl"].(bool)
connection := params.Target.Connection
ip := connection.Ip
port := connection.Port
portType := connection.PortType
ssl := connection.SSL
var addr string = ip
addr += ":"
@ -51,7 +54,7 @@ func (s *SocketHeahthCrawler) getConnection(params map[string]interface{}) (net.
}
}
func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool, error) {
func (s *SocketHeahthCrawler) CheckHeahth(params config.Config) (bool, error) {
conn, err := s.getConnection(params)
if err != nil {
@ -59,7 +62,8 @@ func (s *SocketHeahthCrawler) CheckHeahth(params map[string]interface{}) (bool,
}
defer conn.Close()
info := scaninfo.NewScanInfoImpl(params["ip"].(string),params["port"].(string))
connection := params.Target.Connection
info := scaninfo.NewScanInfoImpl(connection.Ip,connection.Port)
if s.m.IsPrePacket() == true {
bytes := make([]byte, 1024)

61
test/crawler_test.go Normal file
View File

@ -0,0 +1,61 @@
package test
import (
"testing"
"github.com/stretchr/testify/assert"
"encoding/json"
"loafle.com/overflow/crawler_go/health_crawler/redis_protocol_crawler_go"
"loafle.com/overflow/crawler_go"
)
func initConfig() crawler.CrawlerImpl{
c := crawler.CrawlerImpl{}
c.SetInternal(redis_protocol_crawler_go.NewRedisHeahthCrawler())
c.Init("/root/gowork/src/loafle.com/overflow/crawler_go/config/")
return c
}
func TestCrawlerInit(t *testing.T) {
c := initConfig()
m := c.GetConfig("example.json")
assert.Equal(t, m.Id , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734")
}
func TestCrawlerAdd(t *testing.T) {
c := initConfig()
c.Remove("example.json")
b , _ := c.Add("example.json")
var ck bool
json.Unmarshal(b,&ck)
assert.Equal(t,true,ck)
}
func TestCrawlerRemove(t *testing.T) {
c := initConfig()
b, _ := c.Remove("example.json")
var ck bool
json.Unmarshal(b,&ck)
assert.Equal(t,true,ck)
}
func TestCrawlerGet(t *testing.T) {
c := initConfig()
b, _ := c.Get("example.json")
var ck bool
json.Unmarshal(b,&ck)
assert.Equal(t,true,ck)
}