85 lines
1.6 KiB
Go
85 lines
1.6 KiB
Go
package crawler
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
|
|
"log"
|
|
config "loafle.com/overflow/agent_api/config_manager"
|
|
)
|
|
|
|
type Internal interface {
|
|
Internal(params config.Config) ([]byte, error)
|
|
}
|
|
|
|
type Crawler interface {
|
|
Init(path []byte) ([]byte, error)
|
|
Add(data []byte) ([]byte, error)
|
|
Remove(id string) ([]byte, error)
|
|
Get(id string) ([]byte, error)
|
|
}
|
|
|
|
type CrawlerImpl struct {
|
|
Crawler
|
|
configs map[string] config.Config
|
|
internal Internal
|
|
|
|
}
|
|
|
|
func (c *CrawlerImpl) Init(data []byte) ([]byte, error) {
|
|
if c.configs == nil {
|
|
c.configs = make(map[string]config.Config, 0)
|
|
}
|
|
|
|
_,err := c.Add(data)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
|
|
// load all file in path
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Add(data []byte) ([]byte, error) {
|
|
|
|
var m = config.Config{}
|
|
json.Unmarshal(data, &m)
|
|
c.configs[m.Id] = m
|
|
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Remove(id string) ([]byte, error) {
|
|
//remove in config
|
|
delete(c.configs,id)
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Get(id string) ([]byte, error) {
|
|
|
|
if c.internal != nil {
|
|
out, err := c.internal.Internal(c.GetConfig(id))
|
|
if err != nil {
|
|
//set error fail, message
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
return nil, errors.New("Not Assign")
|
|
}
|
|
|
|
// internal methods
|
|
func (c *CrawlerImpl) GetConfig(id string) config.Config {
|
|
return c.configs[id]
|
|
}
|
|
func (c *CrawlerImpl) SetInternal(i Internal) {
|
|
c.internal = i
|
|
}
|
|
func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
|
|
if c.configs == nil {
|
|
c.configs = make(map[string]config.Config, 0)
|
|
}
|
|
c.configs[name] = m
|
|
}
|
|
|
|
|