overflow_backup_2018_0455/crawler_go
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changed

Browse Source 
parameter
This commit is contained in:
snoop 2017-04-27 19:42:46 +09:00
parent b565969109
commit 46529a530f
5 changed files with 141 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
crawler.go
View File

@ -3,7 +3,7 @@ package crawler
import ( import (
"encoding/json" "encoding/json"
"errors" "errors"
"io/ioutil"
"log" "log"
"loafle.com/overflow/crawler_go/config" "loafle.com/overflow/crawler_go/config"
) )
@ -13,8 +13,8 @@ type Internal interface {
} }
type Crawler interface { type Crawler interface {
Init(path string) ([]byte, error) Init(path []byte) ([]byte, error)
Add(id string) ([]byte, error) Add(data []byte) ([]byte, error)
Remove(id string) ([]byte, error) Remove(id string) ([]byte, error)
Get(id string) ([]byte, error) Get(id string) ([]byte, error)
} }
@ -23,43 +23,29 @@ type CrawlerImpl struct {
Crawler Crawler
configs map[string] config.Config configs map[string] config.Config
internal Internal internal Internal
configPath string
} }
func (c *CrawlerImpl) Init(path string) ([]byte, error) { func (c *CrawlerImpl) Init(data []byte) ([]byte, error) {
if c.configs == nil { if c.configs == nil {
c.configs = make(map[string]config.Config, 0) c.configs = make(map[string]config.Config, 0)
} }
c.configPath = path _,err := c.Add(data)
files, err := ioutil.ReadDir(c.configPath)
if err != nil { if err != nil {
log.Panic(err) log.Fatal(err)
} }
for _, file := range files {
if file.IsDir() == true {
continue
}
_,err := c.Add(file.Name())
if err != nil {
log.Panic(err)
}
}
// load all file in path // load all file in path
return json.Marshal(true) return json.Marshal(true)
} }
func (c *CrawlerImpl) Add(id string) ([]byte, error) { func (c *CrawlerImpl) Add(data []byte) ([]byte, error) {
b, err := ioutil.ReadFile(c.configPath + id)
if err != nil {
log.Panic(err)
}
var m = config.Config{} var m = config.Config{}
json.Unmarshal(b, &m) json.Unmarshal(data, &m)
c.configs[id] = m c.configs[m.Id] = m
return json.Marshal(true) return json.Marshal(true)
} }
@ -94,3 +80,5 @@ func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
} }
c.configs[name] = m c.configs[name] = m
} }

120
grpc/grpc.pb.go
View File

@ -14,6 +14,7 @@ It has these top-level messages:
InputArray InputArray
Init Init
Input Input
InputAdd
Output Output
*/ */
package grpc package grpc
@ -188,7 +189,7 @@ func (m *InputArray) GetIn() []*Init {
type Init struct { type Init struct {
Name Crawlers `protobuf:"varint,1,opt,name=name,enum=Crawlers" json:"name,omitempty"` Name Crawlers `protobuf:"varint,1,opt,name=name,enum=Crawlers" json:"name,omitempty"`
Path string `protobuf:"bytes,2,opt,name=path" json:"path,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
} }
func (m *Init) Reset() { *m = Init{} } func (m *Init) Reset() { *m = Init{} }
@ -203,11 +204,11 @@ func (m *Init) GetName() Crawlers {
return Crawlers_HEALTH_ACTIVEDIRECTORY return Crawlers_HEALTH_ACTIVEDIRECTORY
} }
func (m *Init) GetPath() string { func (m *Init) GetData() []byte {
if m != nil { if m != nil {
return m.Path return m.Data
} }
return "" return nil
} }
type Input struct { type Input struct {
@ -234,6 +235,30 @@ func (m *Input) GetId() string {
return "" return ""
} }
type InputAdd struct {
Name Crawlers `protobuf:"varint,1,opt,name=name,enum=Crawlers" json:"name,omitempty"`
Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
}
func (m *InputAdd) Reset() { *m = InputAdd{} }
func (m *InputAdd) String() string { return proto.CompactTextString(m) }
func (*InputAdd) ProtoMessage() {}
func (*InputAdd) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *InputAdd) GetName() Crawlers {
if m != nil {
return m.Name
}
return Crawlers_HEALTH_ACTIVEDIRECTORY
}
func (m *InputAdd) GetData() []byte {
if m != nil {
return m.Data
}
return nil
}
// response models // response models
type Output struct { type Output struct {
StartDate int64 `protobuf:"varint,1,opt,name=startDate" json:"startDate,omitempty"` StartDate int64 `protobuf:"varint,1,opt,name=startDate" json:"startDate,omitempty"`
@ -244,7 +269,7 @@ type Output struct {
func (m *Output) Reset() { *m = Output{} } func (m *Output) Reset() { *m = Output{} }
func (m *Output) String() string { return proto.CompactTextString(m) } func (m *Output) String() string { return proto.CompactTextString(m) }
func (*Output) ProtoMessage() {} func (*Output) ProtoMessage() {}
func (*Output) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } func (*Output) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *Output) GetStartDate() int64 { func (m *Output) GetStartDate() int64 {
if m != nil { if m != nil {
@ -273,6 +298,7 @@ func init() {
proto.RegisterType((*InputArray)(nil), "InputArray") proto.RegisterType((*InputArray)(nil), "InputArray")
proto.RegisterType((*Init)(nil), "Init") proto.RegisterType((*Init)(nil), "Init")
proto.RegisterType((*Input)(nil), "Input") proto.RegisterType((*Input)(nil), "Input")
proto.RegisterType((*InputAdd)(nil), "InputAdd")
proto.RegisterType((*Output)(nil), "Output") proto.RegisterType((*Output)(nil), "Output")
proto.RegisterEnum("Crawlers", Crawlers_name, Crawlers_value) proto.RegisterEnum("Crawlers", Crawlers_name, Crawlers_value)
} }
@ -288,7 +314,7 @@ const _ = grpc1.SupportPackageIsVersion4
// Client API for Config service // Client API for Config service
type ConfigClient interface { type ConfigClient interface {
Add(ctx context.Context, in *Input, opts ...grpc1.CallOption) (*Output, error) Add(ctx context.Context, in *InputAdd, opts ...grpc1.CallOption) (*Output, error)
Remove(ctx context.Context, in *Input, opts ...grpc1.CallOption) (*Output, error) Remove(ctx context.Context, in *Input, opts ...grpc1.CallOption) (*Output, error)
Init(ctx context.Context, in *InputArray, opts ...grpc1.CallOption) (*Output, error) Init(ctx context.Context, in *InputArray, opts ...grpc1.CallOption) (*Output, error)
} }
@ -301,7 +327,7 @@ func NewConfigClient(cc *grpc1.ClientConn) ConfigClient {
return &configClient{cc} return &configClient{cc}
} }
func (c *configClient) Add(ctx context.Context, in *Input, opts ...grpc1.CallOption) (*Output, error) { func (c *configClient) Add(ctx context.Context, in *InputAdd, opts ...grpc1.CallOption) (*Output, error) {
out := new(Output) out := new(Output)
err := grpc1.Invoke(ctx, "/Config/Add", in, out, c.cc, opts...) err := grpc1.Invoke(ctx, "/Config/Add", in, out, c.cc, opts...)
if err != nil { if err != nil {
@ -331,7 +357,7 @@ func (c *configClient) Init(ctx context.Context, in *InputArray, opts ...grpc1.C
// Server API for Config service // Server API for Config service
type ConfigServer interface { type ConfigServer interface {
Add(context.Context, *Input) (*Output, error) Add(context.Context, *InputAdd) (*Output, error)
Remove(context.Context, *Input) (*Output, error) Remove(context.Context, *Input) (*Output, error)
Init(context.Context, *InputArray) (*Output, error) Init(context.Context, *InputArray) (*Output, error)
} }
@ -341,7 +367,7 @@ func RegisterConfigServer(s *grpc1.Server, srv ConfigServer) {
} }
func _Config_Add_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { func _Config_Add_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
in := new(Input) in := new(InputAdd)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
@ -353,7 +379,7 @@ func _Config_Add_Handler(srv interface{}, ctx context.Context, dec func(interfac
FullMethod: "/Config/Add", FullMethod: "/Config/Add",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConfigServer).Add(ctx, req.(*Input)) return srv.(ConfigServer).Add(ctx, req.(*InputAdd))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -546,41 +572,41 @@ var _Status_serviceDesc = grpc1.ServiceDesc{
func init() { proto.RegisterFile("grpc.proto", fileDescriptor0) } func init() { proto.RegisterFile("grpc.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 569 bytes of a gzipped FileDescriptorProto // 576 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x93, 0x4f, 0x6f, 0x9b, 0x4c, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x4d, 0x6f, 0xd3, 0x40,
0x10, 0xc6, 0x63, 0x8c, 0xc1, 0x1e, 0x27, 0xce, 0x64, 0xdf, 0x24, 0x2f, 0x75, 0xfe, 0xd4, 0xa2, 0x10, 0x6d, 0x6c, 0xc7, 0x4e, 0x26, 0x6d, 0x3a, 0x1d, 0xda, 0x12, 0xd2, 0x0f, 0x22, 0x23, 0xa1,
0x52, 0x65, 0xf5, 0xc0, 0xc1, 0x91, 0x2a, 0xf5, 0xb8, 0x06, 0x1a, 0x53, 0x19, 0x43, 0x77, 0x51, 0x8a, 0x83, 0x0f, 0xa9, 0x84, 0xc4, 0x81, 0x83, 0x63, 0x9b, 0xc6, 0x28, 0x8e, 0xcd, 0xae, 0x55,
0xda, 0x9c, 0xa2, 0x6d, 0x4c, 0x13, 0xd4, 0x04, 0x2c, 0xb2, 0x69, 0x95, 0x6f, 0xd1, 0x8f, 0x5c, 0xe8, 0x09, 0xb9, 0x8d, 0x29, 0x11, 0xad, 0x13, 0xb9, 0x1b, 0x50, 0xff, 0x05, 0x3f, 0x19, 0xf9,
0x81, 0x71, 0xb3, 0x87, 0x48, 0xbd, 0xcd, 0xf3, 0x9b, 0x99, 0x67, 0x76, 0x61, 0x07, 0xe0, 0xa6, 0x23, 0xcd, 0x1e, 0x90, 0x90, 0xb8, 0xcd, 0x7b, 0x6f, 0xf6, 0xbd, 0x5d, 0x7b, 0x77, 0x00, 0x6e,
0x5c, 0x5d, 0x3b, 0xab, 0xb2, 0x90, 0x85, 0x6d, 0x42, 0xc7, 0xbf, 0x5f, 0xc9, 0x27, 0xfb, 0x35, 0xf3, 0xe5, 0x8d, 0xb5, 0xcc, 0x17, 0x62, 0x61, 0x1a, 0xd0, 0xf4, 0xee, 0x97, 0xe2, 0xd1, 0x7c,
0x98, 0xd3, 0xa2, 0xb8, 0x4b, 0x45, 0x4e, 0xf6, 0xa1, 0x73, 0x7d, 0x9b, 0x5e, 0xff, 0xb0, 0x5a, 0x09, 0xc6, 0x68, 0xb1, 0xb8, 0x4b, 0x93, 0x8c, 0xf6, 0xa1, 0x79, 0xf3, 0x3d, 0xbd, 0xf9, 0xd1,
0xa3, 0xd6, 0xb8, 0xcb, 0xd6, 0xc2, 0x7e, 0x03, 0x10, 0xe4, 0xab, 0x47, 0x49, 0xcb, 0x52, 0x3c, 0x6b, 0x0c, 0x1a, 0x67, 0x2d, 0x56, 0x01, 0xf3, 0x15, 0x80, 0x9f, 0x2d, 0x57, 0xc2, 0xce, 0xf3,
0x91, 0x03, 0xd0, 0xb2, 0xdc, 0x6a, 0x8d, 0xda, 0xe3, 0xfe, 0xa4, 0xe3, 0x04, 0x79, 0x26, 0x99, 0xe4, 0x91, 0x0e, 0x40, 0x99, 0x67, 0xbd, 0xc6, 0x40, 0x3d, 0xeb, 0x0c, 0x9b, 0x96, 0x9f, 0xcd,
0x96, 0xe5, 0xf6, 0x07, 0xd0, 0xab, 0x98, 0x9c, 0x80, 0x9e, 0x8b, 0xfb, 0xb4, 0x76, 0x18, 0x4c, 0x05, 0x53, 0xe6, 0x99, 0xf9, 0x0e, 0xb4, 0xa2, 0xa6, 0x13, 0xd0, 0xb2, 0xe4, 0x3e, 0x2d, 0x1d,
0x7a, 0x8e, 0x5b, 0x8a, 0x5f, 0x77, 0x69, 0xf9, 0xc0, 0x6a, 0x4c, 0x08, 0xe8, 0x2b, 0x21, 0x6f, 0xba, 0xc3, 0xb6, 0xe5, 0xe4, 0xc9, 0xaf, 0xbb, 0x34, 0x7f, 0x60, 0x25, 0x4d, 0x04, 0xda, 0x2c,
0x2d, 0x6d, 0xd4, 0x1a, 0xf7, 0x58, 0x1d, 0xdb, 0xef, 0xa1, 0x53, 0xfb, 0xff, 0xab, 0x77, 0x00, 0x11, 0x49, 0x4f, 0x19, 0x34, 0xce, 0xb6, 0x59, 0x59, 0x9b, 0x6f, 0xa1, 0x59, 0xfa, 0xff, 0x6b,
0x5a, 0xb6, 0x6c, 0x3a, 0xb5, 0x6c, 0x69, 0x27, 0x60, 0x44, 0x8f, 0xb2, 0x6a, 0x3c, 0x86, 0xde, 0x6d, 0x17, 0x94, 0xf9, 0xac, 0x5c, 0xd9, 0x66, 0xca, 0x7c, 0x66, 0xbe, 0x87, 0x56, 0xb5, 0xaf,
0x83, 0x14, 0xa5, 0xf4, 0x84, 0x5c, 0x77, 0xb7, 0xd9, 0x33, 0x20, 0x16, 0x98, 0x69, 0xbe, 0xac, 0xd9, 0xec, 0x7f, 0x62, 0x63, 0xd0, 0xc3, 0x95, 0x28, 0x72, 0x8f, 0xa1, 0xfd, 0x20, 0x92, 0x5c,
0x73, 0x5a, 0x9d, 0xdb, 0xc8, 0xea, 0x34, 0x4b, 0x21, 0x85, 0xd5, 0x1e, 0xb5, 0xc6, 0xdb, 0xac, 0xb8, 0x89, 0xa8, 0x1c, 0x54, 0xb6, 0x21, 0xa8, 0x07, 0x46, 0x9a, 0xcd, 0x4a, 0x4d, 0x29, 0xb5,
0x8e, 0xdf, 0xfd, 0xd6, 0xa1, 0xbb, 0x19, 0x4c, 0x86, 0x70, 0x38, 0xf3, 0xe9, 0x3c, 0x99, 0x5d, 0x35, 0x7c, 0x72, 0x55, 0x37, 0xae, 0x6f, 0x7e, 0x6b, 0xd0, 0x5a, 0x87, 0x53, 0x1f, 0x0e, 0xc7,
0x51, 0x37, 0x09, 0x2e, 0x7c, 0x2f, 0x60, 0xbe, 0x9b, 0x44, 0xec, 0x12, 0xb7, 0xc8, 0x00, 0xa0, 0x9e, 0x3d, 0x89, 0xc7, 0x5f, 0x6d, 0x27, 0xf6, 0x2f, 0x3d, 0xd7, 0x67, 0x9e, 0x13, 0x87, 0xec,
0xc9, 0x79, 0x0b, 0x8e, 0x2d, 0x45, 0x7f, 0x4c, 0x62, 0xd4, 0xc8, 0x2e, 0xf4, 0x9f, 0x35, 0xc7, 0x0a, 0xb7, 0xa8, 0x0b, 0x50, 0x6b, 0xee, 0x94, 0x63, 0x43, 0xc2, 0x1f, 0xe2, 0x08, 0x15, 0xda,
0xb6, 0x02, 0x82, 0x90, 0xc6, 0xa8, 0x2b, 0x60, 0xee, 0xd1, 0x18, 0x3b, 0x84, 0xc0, 0xa0, 0x01, 0x85, 0xce, 0x06, 0x73, 0x54, 0x25, 0xc2, 0x0f, 0xec, 0x08, 0x35, 0x89, 0x98, 0xb8, 0x76, 0x84,
0x61, 0xb4, 0x38, 0x8f, 0xbc, 0x29, 0x1a, 0x04, 0x61, 0x7b, 0xc3, 0x38, 0xff, 0x3c, 0x47, 0x53, 0x4d, 0x22, 0xe8, 0xd6, 0x44, 0x10, 0x4e, 0x2f, 0x42, 0x77, 0x84, 0x3a, 0x21, 0x6c, 0xaf, 0x39,
0x25, 0x97, 0x15, 0xe9, 0xaa, 0x7d, 0x94, 0x05, 0xd4, 0x9b, 0x62, 0x4f, 0xa9, 0x8a, 0xcf, 0xab, 0xce, 0x3f, 0x4d, 0xd0, 0x90, 0x99, 0xab, 0x82, 0x69, 0xc9, 0xeb, 0x6c, 0xe6, 0xdb, 0xee, 0x08,
0x2a, 0x50, 0xaa, 0x16, 0x7e, 0x32, 0x0d, 0x22, 0x8e, 0x7d, 0xb2, 0x07, 0x3b, 0x0d, 0x8b, 0x18, 0xdb, 0x52, 0x57, 0x74, 0x51, 0x74, 0x81, 0xd4, 0x35, 0xf5, 0xe2, 0x91, 0x1f, 0x72, 0xec, 0xd0,
0x75, 0xe7, 0x3e, 0x6e, 0x2b, 0xa7, 0x8a, 0xa3, 0xf8, 0x0c, 0x77, 0x14, 0x27, 0xe6, 0x7b, 0x01, 0x1e, 0xec, 0xd4, 0x5c, 0xc8, 0x6c, 0x67, 0xe2, 0xe1, 0xb6, 0xb4, 0xab, 0x28, 0x8c, 0xce, 0x71,
0xc7, 0x81, 0x72, 0x55, 0x16, 0x06, 0xb8, 0xab, 0x68, 0x1e, 0x4e, 0x11, 0x15, 0x0b, 0x1e, 0x26, 0x47, 0x72, 0x62, 0x9e, 0xeb, 0x73, 0xec, 0x4a, 0x47, 0x65, 0x81, 0x8f, 0xbb, 0x12, 0xe6, 0xc1,
0x31, 0xee, 0x29, 0xa3, 0xf9, 0x22, 0x8c, 0x2f, 0x26, 0x2e, 0x12, 0x65, 0x74, 0xcd, 0xce, 0xf0, 0x08, 0x51, 0xb2, 0xe0, 0x41, 0x1c, 0xe1, 0x9e, 0x14, 0xcd, 0xa7, 0x41, 0x74, 0x39, 0x74, 0x90,
0x3f, 0xd5, 0x87, 0xcf, 0x70, 0x5f, 0x29, 0x49, 0xfc, 0xf9, 0xc2, 0x4f, 0xf0, 0x40, 0x29, 0xf9, 0xa4, 0xe8, 0x92, 0x3b, 0xc7, 0x67, 0xb2, 0x0f, 0x1f, 0xe3, 0xbe, 0xd4, 0x12, 0x7b, 0x93, 0xa9,
0x12, 0x06, 0x78, 0x48, 0xf6, 0x01, 0x1b, 0xed, 0x52, 0xce, 0xe9, 0xc2, 0x63, 0x14, 0xff, 0x57, 0x17, 0xe3, 0x81, 0xd4, 0xf2, 0x39, 0xf0, 0xf1, 0x90, 0xf6, 0x01, 0x6b, 0xec, 0xd8, 0x9c, 0xdb,
0x0e, 0x30, 0x4b, 0x92, 0x18, 0x2d, 0x62, 0x42, 0xbb, 0xfa, 0x08, 0xaf, 0x48, 0x17, 0xf4, 0x6a, 0x53, 0x97, 0xd9, 0xf8, 0x5c, 0xda, 0xc0, 0x38, 0x8e, 0x23, 0xec, 0x91, 0x01, 0x6a, 0xf1, 0x11,
0x1c, 0x0e, 0x2b, 0x54, 0x59, 0x1c, 0x55, 0xc1, 0xa7, 0xf0, 0x2b, 0x1e, 0x93, 0x1e, 0x74, 0xd6, 0x5e, 0x50, 0x0b, 0xb4, 0x22, 0x0e, 0xfb, 0x05, 0x55, 0x58, 0x1c, 0x15, 0xc5, 0xc7, 0xe0, 0x0b,
0x37, 0x3c, 0x21, 0x7d, 0x30, 0x37, 0xbf, 0xe0, 0x74, 0x72, 0x05, 0x86, 0x5b, 0xe4, 0xdf, 0xb3, 0x1e, 0x53, 0x1b, 0x9a, 0xd5, 0x09, 0x4f, 0xa8, 0x03, 0xc6, 0xfa, 0x17, 0x9c, 0x0e, 0xaf, 0x41,
0x1b, 0x62, 0x41, 0x9b, 0x2e, 0x97, 0xc4, 0x70, 0xea, 0x07, 0x3b, 0x34, 0x9d, 0xf5, 0x03, 0xb4, 0x77, 0x16, 0xd9, 0xb7, 0xf9, 0x2d, 0x1d, 0x81, 0x5a, 0x5c, 0xd6, 0xb6, 0xb5, 0xbe, 0xb7, 0x7d,
0xb7, 0xc8, 0x11, 0x18, 0x2c, 0xbd, 0x2f, 0x7e, 0xa6, 0x2f, 0x25, 0x4f, 0x9b, 0xe5, 0xe8, 0x3b, 0xc3, 0xaa, 0xee, 0xa0, 0xb9, 0x45, 0x47, 0xa0, 0xb3, 0xf4, 0x7e, 0xf1, 0x33, 0x25, 0xbd, 0xd2,
0xcf, 0x8b, 0xa4, 0xe4, 0x27, 0x23, 0xd0, 0x3d, 0x21, 0x45, 0x65, 0x7f, 0x9e, 0xca, 0x17, 0x1c, 0x65, 0xf1, 0xb4, 0x7e, 0x5e, 0x1d, 0x6b, 0xf3, 0x14, 0x25, 0x7d, 0x38, 0x00, 0xcd, 0x4d, 0x44,
0x26, 0x6f, 0xc1, 0xe0, 0x52, 0xc8, 0xc7, 0x07, 0x72, 0xfc, 0x37, 0x32, 0x9c, 0x7a, 0x81, 0x87, 0x42, 0x3d, 0x50, 0x2f, 0x52, 0xf1, 0x17, 0x87, 0xe1, 0x6b, 0xd0, 0xb9, 0x48, 0xc4, 0xea, 0x81,
0x5d, 0xa7, 0xd9, 0x5f, 0x7b, 0xeb, 0x9b, 0x51, 0x2f, 0xf7, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0x8e, 0x9f, 0x2a, 0xdd, 0x2a, 0x47, 0x40, 0xbf, 0x65, 0xd5, 0x13, 0xc0, 0xdc, 0xba, 0xd6, 0xcb,
0xff, 0xc2, 0xcc, 0xf9, 0x8c, 0xea, 0x03, 0x00, 0x00, 0xf1, 0x70, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0xe6, 0x5f, 0x44, 0x44, 0x2c, 0x04, 0x00, 0x00,
} }

9
grpc/grpc.proto
View File

@ -37,7 +37,7 @@ enum Crawlers {
// services // services
service Config { service Config {
rpc Add (Input) returns (Output) {} rpc Add (InputAdd) returns (Output) {}
rpc Remove (Input) returns (Output) {} rpc Remove (Input) returns (Output) {}
rpc Init (InputArray) returns (Output) {} rpc Init (InputArray) returns (Output) {}
} }
@ -62,7 +62,7 @@ message InputArray {
message Init { message Init {
Crawlers name = 1; Crawlers name = 1;
string path = 2; bytes data = 2;
} }
message Input { message Input {
@ -70,6 +70,11 @@ message Input {
string id =2; string id =2;
} }
message InputAdd {
Crawlers name = 1;
bytes data =2;
}
// response models // response models
message Output { message Output {
int64 startDate = 1; int64 startDate = 1;

50
test/crawler_test.go
View File

@ -7,12 +7,16 @@ import (
"loafle.com/overflow/crawler_go/health_crawler/redis_protocol_crawler_go" "loafle.com/overflow/crawler_go/health_crawler/redis_protocol_crawler_go"
"loafle.com/overflow/crawler_go" "loafle.com/overflow/crawler_go"
"io/ioutil"
) )
func initConfig() crawler.CrawlerImpl{ func initConfig() crawler.CrawlerImpl{
c := crawler.CrawlerImpl{} c := crawler.CrawlerImpl{}
c.SetInternal(redis_protocol_crawler_go.NewRedisHeahthCrawler()) c.SetInternal(redis_protocol_crawler_go.NewRedisHeahthCrawler())
c.Init("./")
b, _ := ioutil.ReadFile("./example.json")
c.Init(b)
return c return c
} }
@ -20,16 +24,17 @@ func initConfig() crawler.CrawlerImpl{
func TestCrawlerInit(t *testing.T) { func TestCrawlerInit(t *testing.T) {
c := initConfig() c := initConfig()
m := c.GetConfig("example.json") m := c.GetConfig("test_id_001")
assert.Equal(t, m.Id , "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734") assert.Equal(t, m.Id , "test_id_001")
} }
func TestCrawlerAdd(t *testing.T) { func TestCrawlerAdd(t *testing.T) {
c := initConfig() c := initConfig()
c.Remove("example.json") c.Remove("test_id_001")
b , _ := c.Add("example.json") bb, _ := ioutil.ReadFile("./example.json")
b , _ := c.Add(bb)
var ck bool var ck bool
json.Unmarshal(b,&ck) json.Unmarshal(b,&ck)
@ -40,7 +45,7 @@ func TestCrawlerAdd(t *testing.T) {
func TestCrawlerRemove(t *testing.T) { func TestCrawlerRemove(t *testing.T) {
c := initConfig() c := initConfig()
b, _ := c.Remove("example.json") b, _ := c.Remove("test_id_001")
var ck bool var ck bool
json.Unmarshal(b,&ck) json.Unmarshal(b,&ck)
@ -50,7 +55,7 @@ func TestCrawlerRemove(t *testing.T) {
func TestCrawlerGet(t *testing.T) { func TestCrawlerGet(t *testing.T) {
c := initConfig() c := initConfig()
b, _ := c.Get("example.json") b, _ := c.Get("test_id_001")
var ck bool var ck bool
json.Unmarshal(b,&ck) json.Unmarshal(b,&ck)
@ -59,3 +64,34 @@ func TestCrawlerGet(t *testing.T) {
} }
func TestFolder(t *testing.T) {
configPath := "./"
files, err := ioutil.ReadDir(configPath)
if err != nil {
t.Log(err)
}
for _, file := range files {
if file.IsDir() == true {
continue
}
t.Log(file.Name())
//_,err := c.Add(file.Name())
if err != nil {
t.Log(err)
}
}
}
func TestFile(t *testing.T) {
b, _ := ioutil.ReadFile("./example.json")
t.Log(string(b))
}

2
test/example.json
View File

@ -1,5 +1,5 @@
{ {
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734", "id" : "test_id_001",
"target" : { "target" : {
"connection" : { "connection" : {
"ip" : "192.168.1.104", "ip" : "192.168.1.104",