69 lines
1.3 KiB
Go
69 lines
1.3 KiB
Go
package crawler
|
|
|
|
import (
|
|
configM "git.loafle.net/overflow/overflow_commons_go/modules/config/model"
|
|
)
|
|
|
|
type Crawler interface {
|
|
Init(config *configM.Config) error
|
|
Add(config *configM.Config) error
|
|
Remove(id string) error
|
|
Get(id string) (map[string]string, error)
|
|
|
|
Name() string
|
|
}
|
|
|
|
type InternalCrawler interface {
|
|
InternalGet(config *configM.Config) (map[string]string, error)
|
|
}
|
|
|
|
type Crawlers struct {
|
|
configs map[string]*configM.Config
|
|
|
|
Internal InternalCrawler
|
|
}
|
|
|
|
func (c *Crawlers) GetConfig(id string) *configM.Config {
|
|
if nil == c.configs {
|
|
return nil
|
|
}
|
|
return c.configs[id]
|
|
}
|
|
|
|
func (c *Crawlers) RemoveConfig(id string) {
|
|
if nil == c.configs {
|
|
return
|
|
}
|
|
delete(c.configs, id)
|
|
}
|
|
|
|
func (c *Crawlers) PutConfig(id string, config *configM.Config) {
|
|
if nil == c.configs {
|
|
c.configs = make(map[string]*configM.Config, 0)
|
|
}
|
|
c.configs[id] = config
|
|
}
|
|
|
|
func (c *Crawlers) Init(config *configM.Config) error {
|
|
c.PutConfig(config.ID.String(), config)
|
|
return nil
|
|
}
|
|
|
|
func (c *Crawlers) Add(config *configM.Config) error {
|
|
c.PutConfig(config.ID.String(), config)
|
|
return nil
|
|
}
|
|
|
|
func (c *Crawlers) Remove(id string) error {
|
|
c.RemoveConfig(id)
|
|
return nil
|
|
}
|
|
|
|
func (c *Crawlers) Get(id string) (map[string]string, error) {
|
|
rss, err := c.Internal.InternalGet(c.GetConfig(id))
|
|
if nil != err {
|
|
return nil, err
|
|
}
|
|
return rss, nil
|
|
}
|