This commit is contained in:
crusader 2017-11-16 20:01:42 +09:00
parent 570a0241a0
commit 3579d26e52
9 changed files with 637 additions and 119 deletions

View File

@ -4,6 +4,6 @@ type Service struct {
Port *Port `json:"port,omitempty"`
ID int `json:"id,omitempty"`
TLSType string `json:"tlsType,omitempty"`
CryptoType string `json:"cryptoType,omitempty"`
ServiceName string `json:"serviceName,omitempty"`
}

109
commons/pcap/packet.go Normal file
View File

@ -0,0 +1,109 @@
package pcap
import (
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
type PacketType int
const (
PacketTypeUnknown PacketType = iota
PacketTypeARP
PacketTypeTCP
PacketTypeUDP
)
func getPacketType(packet gopacket.Packet) PacketType {
if packet == nil {
return PacketTypeUnknown
}
layer := packet.Layer(layers.LayerTypeARP)
if layer != nil {
return PacketTypeARP
}
layer = packet.Layer(layers.LayerTypeTCP)
if layer != nil {
if _, ok := layer.(*layers.TCP); ok {
return PacketTypeTCP
}
}
layer = packet.Layer(layers.LayerTypeUDP)
if layer != nil {
if _, ok := layer.(*layers.UDP); ok {
return PacketTypeUDP
}
}
return PacketTypeUnknown
}
func handlePacket(ps *pCapScan, packet gopacket.Packet) {
switch getPacketType(packet) {
case PacketTypeARP:
handlePacketARP(ps, packet)
case PacketTypeTCP:
handlePacketTCP(ps, packet)
case PacketTypeUDP:
handlePacketUDP(ps, packet)
default:
}
}
func handlePacketARP(ps *pCapScan, packet gopacket.Packet) {
ps.arpListenerChanMtx.RLock()
defer ps.arpListenerChanMtx.RUnlock()
if ps.arpListenerChans != nil {
arpLayer := packet.Layer(layers.LayerTypeARP)
arp := arpLayer.(*layers.ARP)
for _, ch := range ps.arpListenerChans {
ch <- arp
}
}
}
func handlePacketTCP(ps *pCapScan, packet gopacket.Packet) {
ipLayer := packet.Layer(layers.LayerTypeIPv4)
if nil == ipLayer {
return
}
ip := ipLayer.(*layers.IPv4).SrcIP.String()
ps.tcpListenerChanMtx.RLock()
defer ps.tcpListenerChanMtx.RUnlock()
chs, ok := ps.tcpListenerChans[ip]
if ok {
layer := packet.Layer(layers.LayerTypeTCP)
tcp, _ := layer.(*layers.TCP)
for _, ch := range chs {
ch <- tcp
}
}
}
func handlePacketUDP(ps *pCapScan, packet gopacket.Packet) {
ipLayer := packet.Layer(layers.LayerTypeIPv4)
if nil == ipLayer {
return
}
ip := ipLayer.(*layers.IPv4).SrcIP.String()
ps.udpListenerChanMtx.RLock()
defer ps.udpListenerChanMtx.RUnlock()
chs, ok := ps.udpListenerChans[ip]
if ok {
for _, ch := range chs {
ch <- packet
}
}
}

54
commons/pcap/pcap.go Normal file
View File

@ -0,0 +1,54 @@
package pcap
import (
"sync"
"git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
)
var mtx sync.Mutex
var instances map[string]PCapScanner
func init() {
instances = make(map[string]PCapScanner, 0)
}
func RetainScanner(zone *model.Zone) (PCapScanner, error) {
mtx.Lock()
defer mtx.Unlock()
var ps PCapScanner
var ok bool
if ps, ok = instances[zone.Network]; !ok {
ps = newPCapScanner(zone)
if err := ps.start(); nil != err {
return nil, err
}
instances[zone.Network] = ps
}
ps.retain()
return ps, nil
}
func ReleaseScanner(zone *model.Zone) {
mtx.Lock()
defer mtx.Unlock()
if ps, ok := instances[zone.Network]; ok {
if ps.release() {
ps.stop()
delete(instances, zone.Network)
}
}
}
func ReleaseScannerAll() {
mtx.Lock()
defer mtx.Unlock()
for k, ps := range instances {
ps.stop()
delete(instances, k)
}
}

247
commons/pcap/pcap_scan.go Normal file
View File

@ -0,0 +1,247 @@
package pcap
import (
"fmt"
"sort"
"sync"
"git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/google/gopacket/pcap"
)
func newPCapScanner(zone *model.Zone) PCapScanner {
ps := &pCapScan{
zone: zone,
}
return ps
}
type PCapScanner interface {
OpenARP() chan *layers.ARP
CloseARP(ch chan *layers.ARP)
OpenTCP(ip string) chan *layers.TCP
CloseTCP(ip string, ch chan *layers.TCP)
OpenUDP(ip string) chan gopacket.Packet
CloseUDP(ip string, ch chan gopacket.Packet)
WritePacketData(data []byte) (err error)
start() error
stop()
retain()
release() bool
}
type pCapScan struct {
zone *model.Zone
pCapHandle *pcap.Handle
arpListenerChanMtx sync.RWMutex
arpListenerChans []chan *layers.ARP
tcpListenerChanMtx sync.RWMutex
tcpListenerChans map[string][]chan *layers.TCP
udpListenerChanMtx sync.RWMutex
udpListenerChans map[string][]chan gopacket.Packet
refCount int
stopChan chan struct{}
stopWg sync.WaitGroup
}
func (ps *pCapScan) start() error {
if ps.stopChan != nil {
return fmt.Errorf("PCapScanner: already running. Stop it before starting it again")
}
// new pcap handle
var h *pcap.Handle
var err error
if h, err = pcap.OpenLive(ps.zone.Iface, 65536, true, pcap.BlockForever); nil != err {
return err
}
// set filter
// todo add tcp, udp filter
if err = h.SetBPFFilter("arp and src net " + ps.zone.Network + " or (((tcp[tcpflags] & (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] & (tcp-rst) != 0)) and port 60000) or udp "); nil != err {
h.Close()
return err
}
ps.arpListenerChans = make([]chan *layers.ARP, 0)
ps.tcpListenerChans = make(map[string][]chan *layers.TCP, 0)
ps.udpListenerChans = make(map[string][]chan gopacket.Packet, 0)
ps.stopChan = make(chan struct{})
ps.stopWg.Add(1)
go handleReceive(ps)
return nil
}
func (ps *pCapScan) stop() {
if ps.stopChan == nil {
panic("PCapScanner: must be started before stopping it")
}
close(ps.stopChan)
ps.stopWg.Wait()
ps.stopChan = nil
}
func (ps *pCapScan) OpenARP() chan *layers.ARP {
ps.arpListenerChanMtx.Lock()
defer ps.arpListenerChanMtx.Unlock()
c := make(chan *layers.ARP, 0)
ps.arpListenerChans = append(ps.arpListenerChans, c)
return c
}
func (ps *pCapScan) CloseARP(ch chan *layers.ARP) {
ps.arpListenerChanMtx.Lock()
defer ps.arpListenerChanMtx.Unlock()
i := sort.Search(len(ps.arpListenerChans), func(i int) bool {
return ch == ps.arpListenerChans[i]
})
if -1 != i {
close(ch)
ps.arpListenerChans = append(ps.arpListenerChans[:i], ps.arpListenerChans[i+1:]...)
}
}
func (ps *pCapScan) OpenTCP(ip string) chan *layers.TCP {
ps.tcpListenerChanMtx.Lock()
defer ps.tcpListenerChanMtx.Unlock()
if _, ok := ps.tcpListenerChans[ip]; !ok {
ps.tcpListenerChans[ip] = make([]chan *layers.TCP, 0)
}
ch := make(chan *layers.TCP, 0)
ps.tcpListenerChans[ip] = append(ps.tcpListenerChans[ip], ch)
return ch
}
func (ps *pCapScan) CloseTCP(ip string, ch chan *layers.TCP) {
ps.tcpListenerChanMtx.Lock()
defer ps.tcpListenerChanMtx.Unlock()
if _, ok := ps.tcpListenerChans[ip]; !ok {
return
}
chs := ps.tcpListenerChans[ip]
i := sort.Search(len(chs), func(i int) bool {
return ch == chs[i]
})
if -1 != i {
close(ch)
ps.tcpListenerChans[ip] = append(ps.tcpListenerChans[ip][:i], ps.tcpListenerChans[ip][i+1:]...)
}
}
func (ps *pCapScan) OpenUDP(ip string) chan gopacket.Packet {
ps.udpListenerChanMtx.Lock()
defer ps.udpListenerChanMtx.Unlock()
if _, ok := ps.udpListenerChans[ip]; !ok {
ps.udpListenerChans[ip] = make([]chan gopacket.Packet, 0)
}
ch := make(chan gopacket.Packet, 0)
ps.udpListenerChans[ip] = append(ps.udpListenerChans[ip], ch)
return ch
}
func (ps *pCapScan) CloseUDP(ip string, ch chan gopacket.Packet) {
ps.udpListenerChanMtx.Lock()
defer ps.udpListenerChanMtx.Unlock()
if _, ok := ps.udpListenerChans[ip]; !ok {
return
}
chs := ps.udpListenerChans[ip]
i := sort.Search(len(chs), func(i int) bool {
return ch == chs[i]
})
if -1 != i {
close(ch)
ps.udpListenerChans[ip] = append(ps.udpListenerChans[ip][:i], ps.udpListenerChans[ip][i+1:]...)
}
}
func (ps *pCapScan) WritePacketData(data []byte) (err error) {
return ps.pCapHandle.WritePacketData(data)
}
func (ps *pCapScan) retain() {
ps.refCount++
}
func (ps *pCapScan) release() bool {
ps.refCount--
if 0 > ps.refCount {
ps.refCount = 0
}
return 0 == ps.refCount
}
func (ps *pCapScan) destroy() {
ps.tcpListenerChanMtx.Lock()
for k, v := range ps.tcpListenerChans {
for _, ch := range v {
close(ch)
}
v = v[:0]
delete(ps.tcpListenerChans, k)
}
ps.tcpListenerChanMtx.Unlock()
ps.udpListenerChanMtx.Lock()
for k, v := range ps.udpListenerChans {
for _, ch := range v {
close(ch)
}
v = v[:0]
delete(ps.udpListenerChans, k)
}
ps.udpListenerChanMtx.Unlock()
ps.arpListenerChanMtx.Lock()
for _, v := range ps.arpListenerChans {
close(v)
}
ps.arpListenerChans = ps.arpListenerChans[:0]
ps.arpListenerChanMtx.Unlock()
ps.pCapHandle.Close()
}
func handleReceive(ps *pCapScan) {
defer ps.stopWg.Done()
pSrc := gopacket.NewPacketSource(ps.pCapHandle, layers.LayerTypeEthernet)
inPacket := pSrc.Packets()
for {
select {
case packet := <-inPacket:
handlePacket(ps, packet)
case <-ps.stopChan:
ps.destroy()
}
}
}

View File

@ -71,140 +71,60 @@ func (d *discovery) stop() {
}
func (d *discovery) discoverZone(dz *model.DiscoveryZone) {
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverZone is start"))
d.stopWg.Add(1)
resultChan := make(chan *model.Zone)
errChan := make(chan error)
doneChan := make(chan struct{})
defer func() {
close(resultChan)
close(errChan)
close(doneChan)
d.stopWg.Done()
}()
go scanZone(dz, resultChan, errChan, doneChan)
for {
select {
case z := <-resultChan:
taskScan(d,
func(resultChan chan interface{}, errChan chan error, doneChan chan struct{}) {
scanZone(dz, resultChan, errChan, doneChan)
},
func(result interface{}) {
z := result.(*model.Zone)
logging.Logger().Info(fmt.Sprintf("zone: %v", z))
if nil != dz.DiscoveryHost {
d.discoverHost(z, dz.DiscoveryHost)
}
case err := <-errChan:
logging.Logger().Info(fmt.Sprintf("zone err: %v", err))
case <-d.stopChan:
return
case <-doneChan:
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverZone is complete"))
return
}
}
},
)
}
func (d *discovery) discoverHost(zone *model.Zone, dh *model.DiscoveryHost) {
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverHost is start"))
d.stopWg.Add(1)
resultChan := make(chan *model.Host)
errChan := make(chan error)
doneChan := make(chan struct{})
defer func() {
close(resultChan)
close(errChan)
close(doneChan)
d.stopWg.Done()
}()
go scanHost(zone, dh, resultChan, errChan, doneChan)
for {
select {
case h := <-resultChan:
taskScan(d,
func(resultChan chan interface{}, errChan chan error, doneChan chan struct{}) {
scanHost(zone, dh, resultChan, errChan, doneChan)
},
func(result interface{}) {
h := result.(*model.Host)
logging.Logger().Info(fmt.Sprintf("host: %v", h))
if nil != dh.DiscoveryPort {
d.discoverPort(h, dh.DiscoveryPort)
}
case err := <-errChan:
logging.Logger().Info(fmt.Sprintf("host err: %v", err))
case <-d.stopChan:
return
case <-doneChan:
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverHost is complete"))
return
}
}
},
)
}
func (d *discovery) discoverPort(host *model.Host, dp *model.DiscoveryPort) {
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverPort is start"))
d.stopWg.Add(1)
resultChan := make(chan *model.Port)
errChan := make(chan error)
doneChan := make(chan struct{})
defer func() {
close(resultChan)
close(errChan)
close(doneChan)
d.stopWg.Done()
}()
go scanPort(host, dp, resultChan, errChan, doneChan)
for {
select {
case p := <-resultChan:
taskScan(d,
func(resultChan chan interface{}, errChan chan error, doneChan chan struct{}) {
scanPort(host, dp, resultChan, errChan, doneChan)
},
func(result interface{}) {
p := result.(*model.Port)
logging.Logger().Info(fmt.Sprintf("port: %v", p))
if nil != dp.DiscoveryService {
d.discoverService(p, dp.DiscoveryService)
}
case err := <-errChan:
logging.Logger().Info(fmt.Sprintf("port err: %v", err))
case <-d.stopChan:
return
case <-doneChan:
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverPort is complete"))
return
}
}
},
)
}
func (d *discovery) discoverService(port *model.Port, ds *model.DiscoveryService) {
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverService is start"))
d.stopWg.Add(1)
resultChan := make(chan *model.Service)
errChan := make(chan error)
doneChan := make(chan struct{})
defer func() {
close(resultChan)
close(errChan)
close(doneChan)
d.stopWg.Done()
}()
go scanService(port, ds, resultChan, errChan, doneChan)
for {
select {
case s := <-resultChan:
taskScan(d,
func(resultChan chan interface{}, errChan chan error, doneChan chan struct{}) {
scanService(port, ds, resultChan, errChan, doneChan)
},
func(result interface{}) {
s := result.(*model.Service)
logging.Logger().Info(fmt.Sprintf("service: %v", s))
case err := <-errChan:
logging.Logger().Info(fmt.Sprintf("service err: %v", err))
case <-d.stopChan:
return
case <-doneChan:
logging.Logger().Debug(fmt.Sprintf("Discovery: DiscoverService is complete"))
return
}
}
},
)
}
func (d *discovery) sendResult() {
@ -214,3 +134,33 @@ func (d *discovery) sendResult() {
func (d *discovery) sendError() {
}
func taskScan(d *discovery, task func(resultChan chan interface{}, errChan chan error, doneChan chan struct{}), onResult func(result interface{})) {
d.stopWg.Add(1)
resultChan := make(chan interface{})
errChan := make(chan error)
doneChan := make(chan struct{})
defer func() {
close(resultChan)
close(errChan)
close(doneChan)
d.stopWg.Done()
}()
go task(resultChan, errChan, doneChan)
for {
select {
case r := <-resultChan:
onResult(r)
case err := <-errChan:
logging.Logger().Info(fmt.Sprintf("task err: %v", err))
case <-doneChan:
logging.Logger().Debug(fmt.Sprintf("Discovery: task is complete"))
return
case <-d.stopChan:
return
}
}
}

View File

@ -1,8 +1,166 @@
package discovery
import "git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
import (
"bytes"
"fmt"
"net"
"time"
func scanHost(zone *model.Zone, dh *model.DiscoveryHost, resultChan chan *model.Host, errChan chan error, doneChan chan<- struct{}) {
"git.loafle.net/commons_go/logging"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
doneChan <- struct{}{}
"git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
"git.loafle.net/overflow/overflow_discovery/commons/pcap"
)
func scanHost(zone *model.Zone, dh *model.DiscoveryHost, resultChan chan interface{}, errChan chan error, doneChan chan<- struct{}) {
ps, err := pcap.RetainScanner(zone)
if nil != err {
errChan <- fmt.Errorf("Discovery: Cannot retain pcap instance %v", err)
// doneChan <- struct{}{}
return
}
arpChan := ps.OpenARP()
defer func() {
if nil != ps {
ps.CloseARP(arpChan)
pcap.ReleaseScanner(zone)
}
doneChan <- struct{}{}
}()
go func() {
for {
select {
case packet, ok := <-arpChan:
if !ok {
logging.Logger().Debug(fmt.Sprintf("Discovery: arp channel is closed"))
return
}
logging.Logger().Debug(fmt.Sprintf("Discovery: arp packet %v", packet))
}
}
}()
time.Sleep(10 * time.Second)
}
func sendARP(ps pcap.PCapScanner, zone *model.Zone, dh *model.DiscoveryHost) error {
hwAddr, err := net.ParseMAC(zone.Mac)
if nil != err {
return err
}
ip := net.ParseIP(zone.IP)
if nil != ip {
return fmt.Errorf("Discovery: IP(%s) of zone is not valid", zone.IP)
}
ranges, err := getTargetHostRange(zone, dh)
if nil != err {
return err
}
ethPacket := makePacketEthernet(hwAddr)
arpPacket := makePacketARP(hwAddr, ip)
opts := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
buf := gopacket.NewSerializeBuffer()
for _, targetHost := range ranges {
arpPacket.DstProtAddress = []byte(targetHost)
gopacket.SerializeLayers(buf, opts, ethPacket, arpPacket)
// if err := ps.WritePacketData(buf.Bytes()); err != nil {
// return err
// }
time.Sleep(time.Microsecond * 500)
}
return nil
}
func getTargetHostRange(zone *model.Zone, dh *model.DiscoveryHost) ([]net.IP, error) {
nIP, nIPNet, err := net.ParseCIDR(zone.Network)
if nil != err {
return nil, fmt.Errorf("Discovery: Network(%s) of zone is not valid", zone.Network)
}
firstIP := net.ParseIP(dh.FirstScanRange)
lastIP := net.ParseIP(dh.LastScanRange)
if nil == firstIP || nil == lastIP {
return nil, fmt.Errorf("Discovery: IP Range(First:%s, Last:%s) is not valid", dh.FirstScanRange, dh.LastScanRange)
}
firstIP16 := firstIP.To16()
lastIP16 := lastIP.To16()
if nil == firstIP16 || nil == lastIP16 {
return nil, fmt.Errorf("Discovery: IP Range(First:%s, Last:%s) is not valid", dh.FirstScanRange, dh.LastScanRange)
}
excludeIP16s := make([]net.IP, 0)
for _, eHost := range dh.ExcludeHosts {
eIP := net.ParseIP(eHost)
if nil == eIP {
return nil, fmt.Errorf("Discovery: IP(%v) of exclude host is not valid", eHost)
}
eIP16 := eIP.To16()
if nil == eIP16 {
return nil, fmt.Errorf("Discovery: IP(%v) of exclude host is not valid", eHost)
}
excludeIP16s = append(excludeIP16s, eIP16)
}
ranges := make([]net.IP, 0)
ip := nIP
Loop:
for ip := ip.Mask(nIPNet.Mask); nIPNet.Contains(ip); inc(ip) {
ip16 := ip.To16()
if nil == ip16 {
return nil, fmt.Errorf("Discovery: IP(%v) converting is failed", ip)
}
if 0 > bytes.Compare(ip16, firstIP16) || 0 < bytes.Compare(ip16, lastIP16) {
continue
}
for _, eIP16 := range excludeIP16s {
if 0 == bytes.Compare(eIP16, ip16) {
continue Loop
}
}
ranges = append(ranges, ip)
}
return ranges, nil
}
func inc(ip net.IP) {
for j := len(ip) - 1; j >= 0; j-- {
ip[j]++
if ip[j] > 0 {
break
}
}
}
func makePacketEthernet(hw net.HardwareAddr) *layers.Ethernet {
return &layers.Ethernet{
SrcMAC: hw,
DstMAC: net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
EthernetType: layers.EthernetTypeARP,
}
}
func makePacketARP(hw net.HardwareAddr, ip net.IP) *layers.ARP {
return &layers.ARP{
AddrType: layers.LinkTypeEthernet,
Protocol: layers.EthernetTypeIPv4,
HwAddressSize: 6,
ProtAddressSize: 4,
Operation: layers.ARPRequest,
SourceHwAddress: []byte(hw),
SourceProtAddress: []byte(ip),
DstHwAddress: []byte{0, 0, 0, 0, 0, 0},
}
}

View File

@ -2,6 +2,6 @@ package discovery
import "git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
func scanPort(host *model.Host, dp *model.DiscoveryPort, resultChan chan *model.Port, errChan chan error, doneChan chan<- struct{}) {
func scanPort(host *model.Host, dp *model.DiscoveryPort, resultChan chan interface{}, errChan chan error, doneChan chan<- struct{}) {
doneChan <- struct{}{}
}

View File

@ -2,6 +2,6 @@ package discovery
import "git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
func scanService(port *model.Port, ds *model.DiscoveryService, resultChan chan *model.Service, errChan chan error, doneChan chan<- struct{}) {
func scanService(port *model.Port, ds *model.DiscoveryService, resultChan chan interface{}, errChan chan error, doneChan chan<- struct{}) {
doneChan <- struct{}{}
}

View File

@ -8,7 +8,7 @@ import (
"git.loafle.net/overflow/overflow_discovery/api/module/discovery/model"
)
func scanZone(dz *model.DiscoveryZone, resultChan chan *model.Zone, errChan chan error, doneChan chan<- struct{}) {
func scanZone(dz *model.DiscoveryZone, resultChan chan interface{}, errChan chan error, doneChan chan<- struct{}) {
var err error
var ifaces []net.Interface
var addrs []net.Addr