overflow_cloud/container_discovery
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ing

Browse Source
This commit is contained in:
crusader 2018-04-28 00:33:22 +09:00
parent 7d2e8170d7
commit bbb569b9da
6 changed files with 311 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

193
internal/discoverer/discoverer_test.go Normal file
View File

@ -0,0 +1,193 @@
package discoverer_test
import (
"encoding/json"
"log"
"strconv"
"testing"
logging "git.loafle.net/commons/logging-go"
occc "git.loafle.net/overflow/commons-go/core/constants"
ocmd "git.loafle.net/overflow/commons-go/model/discovery"
"git.loafle.net/overflow/container_discovery/internal/discoverer"
)
var (
dz = &ocmd.DiscoveryZone{
ExcludePatterns: []string{},
}
z = &ocmd.Zone{
Network: "192.168.1.0/24",
IP: "192.168.1.101",
Iface: "enp3s0",
Mac: "44:8a:5b:f1:f1:f3",
}
dh = &ocmd.DiscoveryHost{
FirstScanRange: "192.168.1.1",
LastScanRange: "192.168.1.254",
}
h = &ocmd.Host{
Zone: z,
IP: "192.168.1.10",
Mac: "d0:50:99:97:5d:99",
}
dp = &ocmd.DiscoveryPort{
FirstScanRange: 1,
LastScanRange: 50000,
ExcludePorts: []int{
631,
},
IncludeTCP: true,
IncludeUDP: true,
}
p = &ocmd.Port{
Host: h,
PortType: occc.PortTypeTCP,
PortNumber: json.Number(strconv.Itoa(80)),
}
ds = &ocmd.DiscoveryService{
IncludeServices: []string{
"ACTIVEDIRECTORY",
"CASSANDRA",
"DHCP",
"DNS",
"FTP",
"HTTP",
"IMAP",
"LDAP",
"MONGODB",
"MSSQL",
"MYSQL",
"NETBIOS",
"ORACLE",
"POP",
"POSTGRESQL",
"REDIS",
"RMI",
"SMB",
"SMTP",
"SNMP",
"SSH",
"TELNET",
"WMI",
"UNKNOWN",
"SSH",
"WMI",
"SNMP",
},
}
dha = &ocmd.DiscoveryHost{
FirstScanRange: "192.168.1.1",
LastScanRange: "192.168.1.254",
DiscoveryPort: &ocmd.DiscoveryPort{
FirstScanRange: 1,
LastScanRange: 50000,
ExcludePorts: []int{
631,
},
IncludeTCP: true,
IncludeUDP: false,
DiscoveryService: ds,
},
}
)
func init() {
// runtime.LockOSThread()
logging.InitializeLogger("")
}
// func TestMain(m *testing.M) {
// go func() {
// os.Exit(m.Run())
// }()
// }
func TestDiscoverZone(t *testing.T) {
_discoverer := discoverer.GetDiscoverer()
handleDiscovery(_discoverer, func(dataChan chan *discoverer.DiscoveryData) {
_discoverer.DiscoverZone(dataChan, dz)
})
}
func TestDiscoverHost(t *testing.T) {
_discoverer := discoverer.GetDiscoverer()
handleDiscovery(_discoverer, func(dataChan chan *discoverer.DiscoveryData) {
_discoverer.DiscoverHost(dataChan, z, dh)
})
}
func TestDiscoverPort(t *testing.T) {
_discoverer := discoverer.GetDiscoverer()
handleDiscovery(_discoverer, func(dataChan chan *discoverer.DiscoveryData) {
_discoverer.DiscoverPort(dataChan, h, dp)
})
}
func TestDiscoverService(t *testing.T) {
_discoverer := discoverer.GetDiscoverer()
handleDiscovery(_discoverer, func(dataChan chan *discoverer.DiscoveryData) {
_discoverer.DiscoverSerice(dataChan, p, ds)
})
}
func TestDiscoverHostAll(t *testing.T) {
_discoverer := discoverer.GetDiscoverer()
handleDiscovery(_discoverer, func(dataChan chan *discoverer.DiscoveryData) {
_discoverer.DiscoverHost(dataChan, z, dha)
})
}
func handleDiscovery(_discoverer discoverer.Discoverer, discoveryFunc func(dataChan chan *discoverer.DiscoveryData)) error {
var dataChan chan *discoverer.DiscoveryData
retainChan := make(chan struct{})
go func() {
dataChan = _discoverer.Retain()
close(retainChan)
}()
select {
case <-retainChan:
}
defer func() {
_discoverer.Release(dataChan)
}()
go discoveryFunc(dataChan)
for {
select {
case data, ok := <-dataChan:
if !ok {
return nil
}
switch data.Type {
case discoverer.DiscoveryDataTypeStart:
log.Printf("DiscoveryService.DiscoveryStart")
case discoverer.DiscoveryDataTypeStop:
log.Printf("DiscoveryService.DiscoveryStop")
data.Release()
return nil
case discoverer.DiscoveryDataTypeError:
log.Printf("DiscoveryService.DiscoveryDataTypeError %v", data.Error)
case discoverer.DiscoveryDataTypeZone:
log.Printf("DiscoveryService.DiscoveryDataTypeZone %v", data.Result)
case discoverer.DiscoveryDataTypeHost:
log.Printf("DiscoveryService.DiscoveryDataTypeHost %v", data.Result)
case discoverer.DiscoveryDataTypePort:
log.Printf("DiscoveryService.DiscoveryDataTypePort %v", data.Result)
case discoverer.DiscoveryDataTypeService:
log.Printf("DiscoveryService.DiscoveryDataTypeService %v", data.Result)
}
data.Release()
}
}
}

19
internal/discoverer/ipv4/host.go
View File

@ -3,6 +3,7 @@ package ipv4
import (
"fmt"
"net"
"sync/atomic"
"time"
"git.loafle.net/commons/logging-go"
@ -42,8 +43,14 @@ func ScanHost(zone *ocmd.Zone, dh *ocmd.DiscoveryHost, resultChan chan interface
ps.CloseARP(arpChan)
}()
timerStopped := make(chan struct{})
go func() {
hosts := make(map[string]*ocmd.Host)
var delay atomic.Value
delay.Store(false)
ticker := time.NewTicker(time.Millisecond * 500)
for {
select {
case packet, ok := <-arpChan:
@ -51,9 +58,17 @@ func ScanHost(zone *ocmd.Zone, dh *ocmd.DiscoveryHost, resultChan chan interface
logging.Logger().Debugf("arp channel is closed")
return
}
delay.Store(true)
if h := handlePacketARP(zone, hostRanges, hosts, packet); nil != h {
resultChan <- h
}
case <-ticker.C:
if false == delay.Load().(bool) {
ticker.Stop()
timerStopped <- struct{}{}
return
}
delay.Store(false)
case <-stopChan:
return
}
@ -65,12 +80,10 @@ func ScanHost(zone *ocmd.Zone, dh *ocmd.DiscoveryHost, resultChan chan interface
return
}
timer := time.NewTimer(10 * time.Second)
select {
case <-stopChan:
return
case <-timer.C:
case <-timerStopped:
return
}

21
internal/discoverer/ipv4/port_tcp.go
View File

@ -6,6 +6,7 @@ import (
"net"
"strconv"
"sync"
"sync/atomic"
"time"
"git.loafle.net/commons/logging-go"
@ -28,17 +29,21 @@ func scanPortTCP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
return
}
defer func() {
pcap.ReleaseScanner(host.Zone)
go pcap.ReleaseScanner(host.Zone)
}()
tcpChan := ps.OpenTCP(host.IP)
defer func() {
ps.CloseTCP(host.IP, tcpChan)
go ps.CloseTCP(host.IP, tcpChan)
}()
timerStopped := make(chan struct{})
go func() {
ports := make(map[int]*ocmd.Port)
var delay atomic.Value
delay.Store(false)
ticker := time.NewTicker(time.Millisecond * 500)
for {
select {
case packet, ok := <-tcpChan:
@ -46,9 +51,17 @@ func scanPortTCP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
logging.Logger().Debugf("Discovery: tcp channel is closed")
return
}
delay.Store(true)
if p := handlePacketTCP(host, dp, ports, packet); nil != p {
resultChan <- p
}
case <-ticker.C:
if false == delay.Load().(bool) {
ticker.Stop()
timerStopped <- struct{}{}
return
}
delay.Store(false)
case <-stopChan:
return
}
@ -60,12 +73,10 @@ func scanPortTCP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
return
}
timer := time.NewTimer(20 * time.Second)
select {
case <-stopChan:
return
case <-timer.C:
case <-timerStopped:
return
}
}

18
internal/discoverer/ipv4/port_udp.go
View File

@ -6,6 +6,7 @@ import (
"net"
"strconv"
"sync"
"sync/atomic"
"time"
"git.loafle.net/commons/logging-go"
@ -37,9 +38,14 @@ func scanPortUDP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
ps.CloseUDP(host.IP, udpChan)
}()
timerStopped := make(chan struct{})
go func() {
ports := make(map[int]*ocmd.Port)
var delay atomic.Value
delay.Store(false)
ticker := time.NewTicker(time.Second * 3)
for {
select {
case packet, ok := <-udpChan:
@ -47,9 +53,17 @@ func scanPortUDP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
logging.Logger().Debugf("Discovery: udp channel is closed")
return
}
delay.Store(true)
if p := handlePacketUDP(host, dp, ports, packet); nil != p {
resultChan <- p
}
case <-ticker.C:
if false == delay.Load().(bool) {
ticker.Stop()
timerStopped <- struct{}{}
return
}
delay.Store(false)
case <-stopChan:
return
}
@ -61,12 +75,10 @@ func scanPortUDP(host *ocmd.Host, dp *ocmd.DiscoveryPort, resultChan chan interf
return
}
timer := time.NewTimer(3 * time.Second)
select {
case <-stopChan:
return
case <-timer.C:
case <-timerStopped:
return
}
}

22
internal/discoverer/ipv4/service_conn.go
View File

@ -8,16 +8,16 @@ import (
)
type serviceConnector interface {
Type() string
CryptoType() string
Dial(ip string, port int) (net.Conn, error)
}
type normalServiceConn struct {
t string
cryptoType string
}
func (nsc *normalServiceConn) Type() string {
return nsc.t
func (nsc *normalServiceConn) CryptoType() string {
return nsc.cryptoType
}
func (nsc *normalServiceConn) Dial(ip string, port int) (net.Conn, error) {
@ -27,19 +27,15 @@ func (nsc *normalServiceConn) Dial(ip string, port int) (net.Conn, error) {
return nil, err
}
err = conn.SetDeadline(time.Now().Add(3 * time.Second))
if err != nil {
return nil, err
}
return conn, err
}
type tlsServiceConn struct {
t string
cryptoType string
}
func (tsc *tlsServiceConn) Type() string {
return tsc.t
func (tsc *tlsServiceConn) CryptoType() string {
return tsc.cryptoType
}
func (tsc *tlsServiceConn) Dial(ip string, port int) (net.Conn, error) {
@ -60,9 +56,5 @@ func (tsc *tlsServiceConn) Dial(ip string, port int) (net.Conn, error) {
return nil, err
}
err = conn.SetDeadline(time.Now().Add(3 * time.Second))
if err != nil {
return nil, err
}
return conn, err
}

100
internal/discoverer/ipv4/service_tcp.go
View File

@ -3,6 +3,7 @@ package ipv4
import (
"fmt"
"net"
"time"
"git.loafle.net/commons/logging-go"
csm "git.loafle.net/commons/service_matcher-go"
@ -13,7 +14,6 @@ import (
)
func scanServiceTCP(port *ocmd.Port, ds *ocmd.DiscoveryService, resultChan chan interface{}, errChan chan error, stopChan chan struct{}) bool {
hostIP := port.Host.IP
portNumber, err := cuej.NumberToInt(port.PortNumber)
if err != nil {
@ -26,24 +26,30 @@ func scanServiceTCP(port *ocmd.Port, ds *ocmd.DiscoveryService, resultChan chan
scs := []serviceConnector{
&normalServiceConn{
t: occc.PortTypeTCP.String(),
cryptoType: occc.CryptoTypeNONE.String(),
},
&tlsServiceConn{
t: occc.CryptoTypeTLS.String(),
cryptoType: occc.CryptoTypeTLS.String(),
},
}
LOOP:
for i := 0; i < len(scs); i++ {
sc := scs[i]
conn, err := sc.Dial(hostIP, portNumber)
if err != nil {
errChan <- fmt.Errorf("Discovery: Service scan[%s] on %s:%d error has occurred %v ", sc.Type(), hostIP, portNumber, err)
break
errChan <- fmt.Errorf("Discovery: Service scan[%s] on %s:%d error has occurred %v ", sc.CryptoType(), hostIP, portNumber, err)
return false
}
logging.Logger().Debugf("Discovery: Service scan connected[%s:%d] %s", hostIP, portNumber, sc.Type())
logging.Logger().Debugf("Discovery: Service scan connected[%s:%d] %s", hostIP, portNumber, sc.CryptoType())
buf := make([]byte, 1024)
if err := conn.SetReadDeadline(time.Now().Add(1 * time.Second)); nil != err {
logging.Logger().Debugf("Discovery: cannot set readdeadline connected[%s:%d] %s", hostIP, portNumber, sc.CryptoType())
return false
}
rn, err := conn.Read(buf)
if err != nil {
rn = 0
@ -56,12 +62,13 @@ func scanServiceTCP(port *ocmd.Port, ds *ocmd.DiscoveryService, resultChan chan
}
if nil != s {
break
break LOOP
}
select {
case <-stopChan:
return true
return false
default:
}
}
@ -80,13 +87,13 @@ func hadlePrePacket(info csm.MatchInfo, sc serviceConnector, conn net.Conn, pack
}()
// logging.Logger().Debugf("Discovery: Service scan pre packet length[%d], buf[%v]", packet.Len, packet.Buffer)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d pre packet length[%d]", sc.Type(), info.IP(), info.Port(), packet.Len)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d pre packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), packet.Len)
ms := matcher.GetTCPMatchers(true)
buf := make([]byte, 1024)
var s *ocmd.Service
Loop:
LOOP:
for i := 0; i < len(ms); i++ {
m := ms[i]
@ -96,9 +103,9 @@ Loop:
if 0 == packetCount {
s = &ocmd.Service{
ServiceName: m.Name(),
CryptoType: occc.ToCryptoType(sc.Type()),
CryptoType: occc.ToCryptoType(sc.CryptoType()),
}
break Loop
break LOOP
}
found := false
@ -106,41 +113,50 @@ Loop:
for j := 0; j < packetCount; j++ {
tPacket := m.Packet(j)
// logging.Logger().Debugf("Discovery: Service scan send packet length[%d], buf[%v]", tPacket.Len, tPacket.Buffer)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d]", sc.Type(), info.IP(), info.Port(), tPacket.Len)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
if err := conn.SetWriteDeadline(time.Now().Add(1 * time.Second)); nil != err {
logging.Logger().Debugf("Discovery: cannot set writeDeadLine Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
break LOOP
}
wn, err := conn.Write(tPacket.Buffer)
if nil != err {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet error %v", sc.Type(), info.IP(), info.Port(), err)
break
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet error %v", sc.CryptoType(), info.IP(), info.Port(), err)
continue LOOP
}
if wn != tPacket.Len {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d] not same with %d", sc.Type(), info.IP(), info.Port(), wn, tPacket.Len)
break
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d] not same with %d", sc.CryptoType(), info.IP(), info.Port(), wn, tPacket.Len)
continue LOOP
}
if err := conn.SetReadDeadline(time.Now().Add(1 * time.Second)); nil != err {
logging.Logger().Debugf("Discovery: cannot set readDeadLine Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
break LOOP
}
rn, err := conn.Read(buf)
if nil != err {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive packet error %v", sc.Type(), info.IP(), info.Port(), err)
break
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive packet error %v", sc.CryptoType(), info.IP(), info.Port(), err)
break LOOP
}
if m.Match(info, j+1, csm.NewPacket(buf, rn)) {
// logging.Logger().Debugf("Discovery: Service scan receive match length[%d], buf[%v]", rn, buf)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive match length[%d]", sc.Type(), info.IP(), info.Port(), rn)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive match length[%d]", sc.CryptoType(), info.IP(), info.Port(), rn)
found = true
} else {
// logging.Logger().Debugf("Discovery: Service scan receive not match length[%d], buf[%v]", rn, buf)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive not match length[%d]", sc.Type(), info.IP(), info.Port(), rn)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive not match length[%d]", sc.CryptoType(), info.IP(), info.Port(), rn)
found = false
break
continue LOOP
}
}
if found {
s = &ocmd.Service{
ServiceName: m.Name(),
CryptoType: occc.ToCryptoType(sc.Type()),
CryptoType: occc.ToCryptoType(sc.CryptoType()),
}
break Loop
break LOOP
}
}
}
@ -149,46 +165,58 @@ Loop:
}
func hadlePostPacket(info csm.MatchInfo, sc serviceConnector) *ocmd.Service {
defer func() {
}()
ms := matcher.GetTCPMatchers(false)
buf := make([]byte, 1024)
var s *ocmd.Service
Loop:
LOOP:
for i := 0; i < len(ms); i++ {
m := ms[i]
conn, err := sc.Dial(info.IP(), info.Port())
if err != nil {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d socket dial error %v", sc.Type(), info.IP(), info.Port(), err)
break Loop
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d socket dial error %v", sc.CryptoType(), info.IP(), info.Port(), err)
break LOOP
}
packetCount := m.PacketCount()
for j := 0; j < packetCount; j++ {
tPacket := m.Packet(j)
// logging.Logger().Debugf("Discovery: Service scan send packet length[%d], buf[%v]", tPacket.Len, tPacket.Buffer)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d]", sc.Type(), info.IP(), info.Port(), tPacket.Len)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
if err := conn.SetWriteDeadline(time.Now().Add(1 * time.Second)); nil != err {
logging.Logger().Debugf("Discovery: cannot set writeDeadLine Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
break
}
wn, err := conn.Write(tPacket.Buffer)
if nil != err {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet error %v", sc.Type(), info.IP(), info.Port(), err)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet error %v", sc.CryptoType(), info.IP(), info.Port(), err)
break
}
if wn != tPacket.Len {
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d] not same with %d", sc.Type(), info.IP(), info.Port(), wn, tPacket.Len)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d send packet length[%d] not same with %d", sc.CryptoType(), info.IP(), info.Port(), wn, tPacket.Len)
break
}
if err := conn.SetReadDeadline(time.Now().Add(1 * time.Second)); nil != err {
logging.Logger().Debugf("Discovery: cannot set readDeadLine Service scan[%s] on %s:%d send packet length[%d]", sc.CryptoType(), info.IP(), info.Port(), tPacket.Len)
break
}
rn, err := conn.Read(buf)
if nil != err {
if !m.HasResponse(j) {
s = &ocmd.Service{
ServiceName: m.Name(),
CryptoType: occc.ToCryptoType(sc.Type()),
CryptoType: occc.ToCryptoType(sc.CryptoType()),
}
break
}
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive packet error %v", sc.Type(), info.IP(), info.Port(), err)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive packet error %v", sc.CryptoType(), info.IP(), info.Port(), err)
break
}
@ -196,24 +224,24 @@ Loop:
if packetCount-1 == j {
s = &ocmd.Service{
ServiceName: m.Name(),
CryptoType: occc.ToCryptoType(sc.Type()),
CryptoType: occc.ToCryptoType(sc.CryptoType()),
}
break
}
// logging.Logger().Debugf("Discovery: Service scan receive match length[%d], buf[%v]", rn, buf)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive match length[%d]", sc.Type(), info.IP(), info.Port(), rn)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive match length[%d]", sc.CryptoType(), info.IP(), info.Port(), rn)
continue
} else {
// logging.Logger().Debugf("Discovery: Service scan receive not match length[%d], buf[%v]", rn, buf)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive not match length[%d]", sc.Type(), info.IP(), info.Port(), rn)
logging.Logger().Debugf("Discovery: Service scan[%s] on %s:%d receive not match length[%d]", sc.CryptoType(), info.IP(), info.Port(), rn)
break
}
}
conn.Close()
if nil != s {
break Loop
break LOOP
}
}