80 lines
1.6 KiB
Go
80 lines
1.6 KiB
Go
|
package crawler
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
|
||
|
|
config "git.loafle.net/overflow/overflow_probe/agent_api/config_manager"
|
||
|
|
"log"
|
||
|
|
)
|
||
|
|
|
||
|
|
type Internal interface {
|
||
|
|
Internal(params config.Config) ([]byte, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
type Crawler interface {
|
||
|
|
Init(path []byte) ([]byte, error)
|
||
|
|
Add(data []byte) ([]byte, error)
|
||
|
|
Remove(id string) ([]byte, error)
|
||
|
|
Get(id string) ([]byte, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
type CrawlerImpl struct {
|
||
|
|
Crawler
|
||
|
|
configs map[string]config.Config
|
||
|
|
internal Internal
|
||
|
|
}
|
||
|
|
|
||
|
|
func (c *CrawlerImpl) Init(data []byte) ([]byte, error) {
|
||
|
|
if c.configs == nil {
|
||
|
|
c.configs = make(map[string]config.Config, 0)
|
||
|
|
}
|
||
|
|
|
||
|
|
_, err := c.Add(data)
|
||
|
|
|
||
|
|
if err != nil {
|
||
|
|
log.Fatal(err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// load all file in path
|
||
|
|
return json.Marshal(true)
|
||
|
|
}
|
||
|
|
func (c *CrawlerImpl) Add(data []byte) ([]byte, error) {
|
||
|
|
|
||
|
|
var m = config.Config{}
|
||
|
|
json.Unmarshal(data, &m)
|
||
|
|
c.configs[m.Id] = m
|
||
|
|
|
||
|
|
return json.Marshal(true)
|
||
|
|
}
|
||
|
|
func (c *CrawlerImpl) Remove(id string) ([]byte, error) {
|
||
|
|
//remove in config
|
||
|
|
delete(c.configs, id)
|
||
|
|
return json.Marshal(true)
|
||
|
|
}
|
||
|
|
func (c *CrawlerImpl) Get(id string) ([]byte, error) {
|
||
|
|
if c.internal != nil {
|
||
|
|
out, err := c.internal.Internal(c.GetConfig(id))
|
||
|
|
if err != nil {
|
||
|
|
//set error fail, message
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
return out, nil
|
||
|
|
}
|
||
|
|
return nil, errors.New("Not Assigned")
|
||
|
|
}
|
||
|
|
|
||
|
|
// internal methods
|
||
|
|
func (c *CrawlerImpl) GetConfig(id string) config.Config {
|
||
|
|
return c.configs[id]
|
||
|
|
}
|
||
|
|
func (c *CrawlerImpl) SetInternal(i Internal) {
|
||
|
|
c.internal = i
|
||
|
|
}
|
||
|
|
func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
|
||
|
|
if c.configs == nil {
|
||
|
|
c.configs = make(map[string]config.Config, 0)
|
||
|
|
}
|
||
|
|
c.configs[name] = m
|
||
|
|
}
|