97 lines
1.8 KiB
Go
97 lines
1.8 KiB
Go
package crawler
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"io/ioutil"
|
|
"log"
|
|
"loafle.com/overflow/crawler_go/config"
|
|
)
|
|
|
|
type Internal interface {
|
|
Internal(params config.Config) ([]byte, error)
|
|
}
|
|
|
|
type Crawler interface {
|
|
Init(path string) ([]byte, error)
|
|
Add(id string) ([]byte, error)
|
|
Remove(id string) ([]byte, error)
|
|
Get(id string) ([]byte, error)
|
|
}
|
|
|
|
type CrawlerImpl struct {
|
|
Crawler
|
|
configs map[string] config.Config
|
|
internal Internal
|
|
configPath string
|
|
}
|
|
|
|
func (c *CrawlerImpl) Init(path string) ([]byte, error) {
|
|
if c.configs == nil {
|
|
c.configs = make(map[string]config.Config, 0)
|
|
}
|
|
|
|
c.configPath = path
|
|
files, err := ioutil.ReadDir(c.configPath)
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
for _, file := range files {
|
|
if file.IsDir() == true {
|
|
continue
|
|
}
|
|
_,err := c.Add(file.Name())
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
}
|
|
|
|
// load all file in path
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Add(id string) ([]byte, error) {
|
|
|
|
b, err := ioutil.ReadFile(c.configPath + id)
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
var m = config.Config{}
|
|
json.Unmarshal(b, &m)
|
|
c.configs[id] = m
|
|
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Remove(id string) ([]byte, error) {
|
|
//remove in config
|
|
delete(c.configs,id)
|
|
return json.Marshal(true)
|
|
}
|
|
func (c *CrawlerImpl) Get(id string) ([]byte, error) {
|
|
|
|
if c.internal != nil {
|
|
out, err := c.internal.Internal(c.GetConfig(id))
|
|
if err != nil {
|
|
//set error fail, message
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
return nil, errors.New("Not Assign")
|
|
}
|
|
|
|
// internal methods
|
|
func (c *CrawlerImpl) GetConfig(id string) config.Config {
|
|
return c.configs[id]
|
|
}
|
|
func (c *CrawlerImpl) SetInternal(i Internal) {
|
|
c.internal = i
|
|
}
|
|
func (c *CrawlerImpl) PutConfig(name string, m config.Config) {
|
|
if c.configs == nil {
|
|
c.configs = make(map[string]config.Config, 0)
|
|
}
|
|
c.configs[name] = m
|
|
}
|