From 670993e9244a371bc6a4ff927afb2cd90705f7e2 Mon Sep 17 00:00:00 2001 From: snoop Date: Wed, 12 Apr 2017 15:57:11 +0900 Subject: [PATCH 1/5] removed comment --- .../com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java | 6 ------ .../loafle/overflow/crawler/wmi/WMICrawlerWindows.java | 8 +------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java index 70b0893..fda8bd7 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java @@ -18,12 +18,6 @@ public class WMICrawlerLinux extends WMICrawlerOS { public Object processCommand(Map params) throws Exception { -// String id = "administrator"; -// String pw = "dbseogns18"; -// String nameSpace = "root/cimv2"; -// String query = "select * from Win32_OperatingSystem"; -// String ip = "192.168.1.106"; - String id = (String)params.get("id"); String pw = (String)params.get("pw"); String nameSpace = (String)params.get("nameSpace"); diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java index 4ee0d82..9674ba7 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java @@ -15,13 +15,7 @@ import java.util.Map; */ public class WMICrawlerWindows extends WMICrawlerOS { - public Object processWMI(Map params) { - -// String id = "administrator"; -// String pw = "!@#$qwer1234"; -// String nameSpace = "root/cimv2"; -// String query = "select * from Win32_OperatingSystem"; -// String ip = "192.168.1.1"; + public Object processWMI(Map params) throws Exception { String id = (String)params.get("id"); String pw = (String)params.get("pw"); From 400eaf8457899d50eed8719886788a822f192764 Mon Sep 17 00:00:00 2001 From: "jackdaw@loafle.com" Date: Thu, 13 Apr 2017 17:02:59 +0900 Subject: [PATCH 2/5] . --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a348c43..a8f3e29 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ com.loafle.overflow - crawler_java + crawler 1.0.0-SNAPSHOT From ec0e8d4419eae1051846be38673814302aefcb7c Mon Sep 17 00:00:00 2001 From: "jackdaw@loafle.com" Date: Fri, 14 Apr 2017 10:49:23 +0900 Subject: [PATCH 3/5] . --- .../loafle/overflow/crawler/wmi/WMICrawler.java | 6 ++++-- .../overflow/crawler/wmi/WMICrawlerTest.java | 16 ++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java index 84bb232..9ede7a4 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java @@ -1,6 +1,7 @@ package com.loafle.overflow.crawler.wmi; import com.loafle.overflow.crawler.Crawler; +import com.loafle.overflow.crawler.config.Config; import java.util.Map; @@ -10,8 +11,9 @@ import java.util.Map; */ public class WMICrawler extends Crawler { - public Object getInternal(Map map) throws Exception { - return WMICrawlerOS.getInstance().processWMI(map); + public Object getInternal(Config config) throws Exception { + //return WMICrawlerOS.getInstance().processWMI(map); + return null; } diff --git a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java index b094f74..14c7ea7 100644 --- a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java +++ b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java @@ -65,14 +65,14 @@ public class WMICrawlerTest { map.put("ip", "192.168.1.1"); - List> resultList = (List>)wmiCrawler.getInternal(map); - - - for( Map rMap : resultList) { - for( String key : rMap.keySet() ){ - System.out.println( String.format("key : %s ||| value : %s", key, rMap.get(key)) ); - } - } +// List> resultList = (List>)wmiCrawler.getInternal(map); +// +// +// for( Map rMap : resultList) { +// for( String key : rMap.keySet() ){ +// System.out.println( String.format("key : %s ||| value : %s", key, rMap.get(key)) ); +// } +// } } From 8f3da472c4d98eab51b0491512bff458e11e89fb Mon Sep 17 00:00:00 2001 From: snoop Date: Fri, 21 Apr 2017 18:26:32 +0900 Subject: [PATCH 4/5] config ing --- pom.xml | 2 +- .../overflow/crawler/wmi/WMICrawler.java | 9 +- .../overflow/crawler/wmi/WMICrawlerLinux.java | 15 +- .../overflow/crawler/wmi/WMICrawlerOS.java | 4 +- .../crawler/wmi/WMICrawlerWindows.java | 193 ++++++++++++++---- .../overflow/crawler/wmi/WMICrawlerTest.java | 121 +++++++++-- src/test/resources/config/example.json | 65 ++++++ 7 files changed, 347 insertions(+), 62 deletions(-) create mode 100644 src/test/resources/config/example.json diff --git a/pom.xml b/pom.xml index a348c43..a8f3e29 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ com.loafle.overflow - crawler_java + crawler 1.0.0-SNAPSHOT diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java index 84bb232..eb4d826 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java @@ -1,6 +1,7 @@ package com.loafle.overflow.crawler.wmi; import com.loafle.overflow.crawler.Crawler; +import com.loafle.overflow.crawler.config.Config; import java.util.Map; @@ -10,9 +11,9 @@ import java.util.Map; */ public class WMICrawler extends Crawler { - public Object getInternal(Map map) throws Exception { - return WMICrawlerOS.getInstance().processWMI(map); + @Override + public Object getInternal(Config config) throws Exception { + + return WMICrawlerOS.getInstance().processWMI(config); } - - } diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java index fda8bd7..9af667c 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerLinux.java @@ -1,5 +1,7 @@ package com.loafle.overflow.crawler.wmi; +import com.loafle.overflow.crawler.config.Config; + import java.util.*; @@ -12,11 +14,18 @@ public class WMICrawlerLinux extends WMICrawlerOS { private final String DELIMITER = "||"; private final String DELIMITER_SPLIT = "\\|\\|"; - public Object processWMI(Map params) throws Exception { - return processCommand(params); + public Object processWMI(Config config) throws Exception { + return processCommand(config); } - public Object processCommand(Map params) throws Exception { + public Object processCommand(Config config) throws Exception { + + Map params = new HashMap<>(); + params.put("id", "administrator"); + params.put("pw", "!@#$qwer1234"); + params.put("nameSpace", "root/cimv2"); + params.put("query", "select * from Win32_OperatingSystem"); + params.put("ip", "192.168.1.1"); String id = (String)params.get("id"); String pw = (String)params.get("pw"); diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerOS.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerOS.java index be2ce64..25689b5 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerOS.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerOS.java @@ -1,5 +1,7 @@ package com.loafle.overflow.crawler.wmi; +import com.loafle.overflow.crawler.config.Config; + import java.util.Map; /** @@ -26,5 +28,5 @@ public abstract class WMICrawlerOS { return WMICrawlerOS.wmiCrawlerOS; } - public abstract Object processWMI(Map params) throws Exception; + public abstract Object processWMI(Config config) throws Exception; } diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java index 9674ba7..e155c04 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawlerWindows.java @@ -4,6 +4,9 @@ import com.jacob.activeX.ActiveXComponent; import com.jacob.com.Dispatch; import com.jacob.com.EnumVariant; import com.jacob.com.Variant; +import com.loafle.overflow.crawler.config.Config; +import com.loafle.overflow.crawler.config.Item; +import com.loafle.overflow.crawler.config.Query; import java.util.ArrayList; import java.util.HashMap; @@ -15,82 +18,145 @@ import java.util.Map; */ public class WMICrawlerWindows extends WMICrawlerOS { - public Object processWMI(Map params) throws Exception { + public Object processWMI(Config config) throws Exception { - String id = (String)params.get("id"); - String pw = (String)params.get("pw"); - String nameSpace = (String)params.get("nameSpace"); - String query = (String)params.get("query"); - String ip = (String)params.get("ip"); + String query = ""; - ActiveXComponent wmi = null; - wmi = new ActiveXComponent("WbemScripting.SWbemLocator"); + Item cItem = null; + Query cQuery = null; - Variant vIp = new Variant(ip); - Variant vNs = new Variant(nameSpace); - Variant vId = new Variant(id); - Variant vPw = new Variant(pw); + int idxMetric = 0; - Variant conRet = wmi.invoke("ConnectServer", vIp, vNs, vId, vPw); + Map resultMap = new HashMap<>(); + ActiveXComponent wmiconnect = null; + for(int cIndexI = 0 ; cIndexI < config.getItems().size() ; ++cIndexI) { - vIp.safeRelease(); - vNs.safeRelease(); - vId.safeRelease(); - vPw.safeRelease(); + cItem = config.getItems().get(cIndexI); - Dispatch disConret = conRet.toDispatch(); + for( int indexJ = 0; indexJ < cItem.getQueries().size(); ++indexJ) { - ActiveXComponent wmiconnect = new ActiveXComponent(disConret); - disConret.safeRelease(); + cQuery = cItem.getQueries().get(indexJ); + query =cQuery.getQuery(); - Variant vQuery = new Variant(query); + wmiconnect = connectServer(config); - Variant vCollection = wmiconnect.invoke("ExecQuery", vQuery); + Variant vQuery = new Variant(query); - wmiconnect.safeRelease(); - vQuery.safeRelease(); + Variant vCollection = wmiconnect.invoke("ExecQuery", vQuery); - EnumVariant enumVariant = new EnumVariant(vCollection.toDispatch()); + wmiconnect.safeRelease(); + vQuery.safeRelease(); - wmi.safeRelease(); - conRet.safeRelease(); - vCollection.safeRelease(); + Dispatch dConnection = vCollection.toDispatch(); + vCollection.safeRelease(); + int count = getCount(dConnection); + + EnumVariant enumVariant = new EnumVariant(dConnection); + dConnection.safeRelease(); + + if( count <= 1 ) { + getSingleValue(enumVariant, cQuery, cItem, idxMetric, resultMap); + } + else { + getMultiValue(enumVariant, cQuery, cItem, idxMetric, resultMap); + } + + + enumVariant.safeRelease(); + idxMetric += cQuery.getKeys().size(); + + } + + + + } + return resultMap; + } + + protected void getMultiValue(EnumVariant enumVariant, Query cQuery, Item cItem, int idxMetric, Map resultMap) { + Variant vItem = null; Dispatch item = null; - List columns = null; - Map resultMap = null; - List> resultMapList = new ArrayList>(); + + int keyIdx = 0; while (enumVariant.hasMoreElements()) { vItem = enumVariant.nextElement(); item = vItem.toDispatch(); - if (columns == null) { - columns = getColumns(item); - } - resultMap = new HashMap(); String name = null; String value = null; Variant vValue = null; - for(int indexI = 0 ; indexI < columns.size(); ++indexI) { - name = columns.get(indexI); + for(int indexI = 0 ; indexI < cQuery.getKeys().size(); ++indexI) { + name = cQuery.getKeys().get(indexI); vValue = Dispatch.call(item, name); value = vValue.toString(); - resultMap.put(name, value); + String keyStr = convertArrayKey(cItem.getMetrics().get(indexI),keyIdx ); + resultMap.put(keyStr, value); vValue.safeRelease(); } - resultMapList.add(resultMap); + vItem.safeRelease(); + item.safeRelease(); + ++idxMetric; + ++keyIdx; + } + + } + + protected String convertArrayKey(String key, int index) { + int idx = key.indexOf("["); + String ret = ""; + + if(idx <= 0) { + idx = key.indexOf("."); + ret += key.substring(0, idx); + ret += "["; + ret += String.valueOf(index); + ret += "]"; + ret += key.substring(idx +1); + }else { + + ret += key.substring(0, idx +1); + ret += String.valueOf(index); + ret += key.substring(idx +1); + } + + + return ret; + } + + protected void getSingleValue(EnumVariant enumVariant, Query cQuery, Item cItem, int idxMetric, Map resultMap) { + + Variant vItem = null; + Dispatch item = null; + + while (enumVariant.hasMoreElements()) { + + vItem = enumVariant.nextElement(); + item = vItem.toDispatch(); + + String name = null; + String value = null; + Variant vValue = null; + + for(int indexI = 0 ; indexI < cQuery.getKeys().size(); ++indexI) { + name = cQuery.getKeys().get(indexI); + vValue = Dispatch.call(item, name); + value = vValue.toString(); + resultMap.put(cItem.getMetrics().get(idxMetric++), value); + vValue.safeRelease(); + } vItem.safeRelease(); item.safeRelease(); } - return resultMapList; + } protected List getColumns(Dispatch item) { @@ -134,4 +200,51 @@ public class WMICrawlerWindows extends WMICrawlerOS { } + protected int getCount(Dispatch d) { + + Variant c = Dispatch.call(d, "Count"); + + int retInt = Integer.valueOf(c.toString()); + + c.safeRelease(); + + return retInt; + } + + protected ActiveXComponent connectServer(Config config) { + + String id = (String)config.getTarget().getAuth().get("id"); + String pw = (String)config.getTarget().getAuth().get("pw"); + String nameSpace = ""; +// String query = (String)config.get.get("query"); + + String ip = config.getTarget().getConnection().getIp(); + + ActiveXComponent wmi = null; + wmi = new ActiveXComponent("WbemScripting.SWbemLocator"); + + Variant vIp = new Variant(ip); + Variant vNs = new Variant(nameSpace); + Variant vId = new Variant(id); + Variant vPw = new Variant(pw); + + Variant conRet = wmi.invoke("ConnectServer", vIp, vNs, vId, vPw); + + vIp.safeRelease(); + vNs.safeRelease(); + vId.safeRelease(); + vPw.safeRelease(); + + Dispatch disConret = conRet.toDispatch(); + + ActiveXComponent wmiconnect = new ActiveXComponent(disConret); + disConret.safeRelease(); + + wmi.safeRelease(); + conRet.safeRelease(); + + return wmiconnect; + + } + } diff --git a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java index b094f74..0b6ece3 100644 --- a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java +++ b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java @@ -1,10 +1,16 @@ package com.loafle.overflow.crawler.wmi; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.loafle.overflow.crawler.config.Config; +import com.sun.xml.internal.ws.api.ResourceLoader; import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import java.io.File; +import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -58,21 +64,31 @@ public class WMICrawlerTest { // String query = "select * from Win32_OperatingSystem"; // String ip = "192.168.1.1"; - map.put("id", "administrator"); - map.put("pw", "!@#$qwer1234"); - map.put("nameSpace", "root/cimv2"); - map.put("query", "select * from Win32_OperatingSystem"); - map.put("ip", "192.168.1.1"); +// map.put("id", "administrator"); +// map.put("pw", "!@#$qwer1234"); +// map.put("nameSpace", "root/cimv2"); +// map.put("query", "select * from Win32_OperatingSystem"); +// map.put("ip", "192.168.1.1"); + + Config config = new Config(); + + ObjectMapper mapper = new ObjectMapper(); + ClassLoader classLoader = getClass().getClassLoader(); + + String s = classLoader.getResource("config").getFile(); + System.out.println(s); + +// Config c = mapper.readValue(new File(classLoader.getResource("config").getFile()),Config.class); - List> resultList = (List>)wmiCrawler.getInternal(map); - - - for( Map rMap : resultList) { - for( String key : rMap.keySet() ){ - System.out.println( String.format("key : %s ||| value : %s", key, rMap.get(key)) ); - } - } +// List> resultList = (List>)wmiCrawler.getInternal(map); +// +// +// for( Map rMap : resultList) { +// for( String key : rMap.keySet() ){ +// System.out.println( String.format("key : %s ||| value : %s", key, rMap.get(key)) ); +// } +// } } @@ -83,4 +99,83 @@ public class WMICrawlerTest { } + @Test + public void testConfig() throws IOException { + + + ObjectMapper mapper = new ObjectMapper(); + + String p = ResourceLoader.class.getClass().getResource("/config/example.json").getPath(); + + System.out.println(p); + + + Config c = mapper.readValue(new File(p),Config.class); + + System.out.println(c); + } + + @Test + public void testWMIJson() throws Exception { + + ObjectMapper mapper = new ObjectMapper(); + String p = ResourceLoader.class.getClass().getResource("/config/example.json").getPath(); + Config c = mapper.readValue(new File(p),Config.class); + +// Map map = new HashMap<>(); +// map.put("id", "administrator"); +// map.put("pw", "!@#$qwer1234"); + +// c.getTarget().setAuth(map); + + c.getTarget().getAuth().put("id", "administrator"); + c.getTarget().getAuth().put("pw", "!@#$qwer1234"); + + + WMICrawler wmiCrawler = new WMICrawler(); + + + Map resultMap = (Map)wmiCrawler.getInternal(c); + + + for( String key : resultMap.keySet() ){ + System.out.println( String.format("key : %s ||| value : %s", key, resultMap.get(key)) ); + } + + + } + + @Test + public void testInt() { + + Integer a = 1; + + aaa(a); + + System.out.println(a); + + } + + public void aaa(Integer a) { + ++a; + + } + + @Test + public void testString() { + + String str = "cpu[].usage.system"; + + int idx = str.indexOf("["); + int arrIdx = 0; + + String ret = ""; + ret += str.substring(0, idx +1); + ret += String.valueOf(arrIdx); + ret += str.substring(idx +1); + + System.out.println(ret); + + } + } \ No newline at end of file diff --git a/src/test/resources/config/example.json b/src/test/resources/config/example.json new file mode 100644 index 0000000..e989218 --- /dev/null +++ b/src/test/resources/config/example.json @@ -0,0 +1,65 @@ +{ + "id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734", + "target" : { + "connection" : { + "ip" : "192.168.1.1", + "port" : "135", + "ssl" : false, + "portType" : "tcp" + }, + "auth" : { + + } + }, + "schedule" : { + "interval" : "10" + }, + "crawler" : { + "name":"redis_protocol_crawler", + "container":"network_crawler" + }, + "items" : [ + { + "metrics": [ + "cpu.usage.system", + "cpu.usage.idle", + "cpu.usage.user" + ], + "queries": [ + { + "query":"select PercentProcessorTime, PercentIdleTime, PercentUserTime from Win32_PerfFormattedData_PerfOS_Processor where Name='_Total'", + "keys":["PercentProcessorTime","PercentIdleTime", "PercentUserTime"] + } + ] + }, + { + "metrics": [ + "cpu[].usage.system", + "cpu[].usage.idle", + "cpu[].usage.user" + ], + "queries": [ + { + "query":"select PercentProcessorTime, PercentIdleTime, PercentUserTime from Win32_PerfFormattedData_PerfOS_Processor where Name!='_Total'", + "keys":["PercentProcessorTime","PercentIdleTime", "PercentUserTime"] + } + ] + } + + ,{ + "metrics": [ + "cpu.usage.system", + "cpu.usage.idle", + "cpu.usage.user" + ], + "queries": [ + { + "query":"select PercentProcessorTime, PercentIdleTime, PercentUserTime from Win32_PerfFormattedData_PerfOS_Processor", + "keys":["PercentProcessorTime","PercentIdleTime", "PercentUserTime"] + } + ] + } + + ] + +} \ No newline at end of file From 8f024d8c8b56d0de2f631a1b29f333071f477591 Mon Sep 17 00:00:00 2001 From: snoop Date: Mon, 24 Apr 2017 11:15:08 +0900 Subject: [PATCH 5/5] config --- src/test/resources/config/example.json | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/test/resources/config/example.json b/src/test/resources/config/example.json index e989218..0b804ac 100644 --- a/src/test/resources/config/example.json +++ b/src/test/resources/config/example.json @@ -46,20 +46,6 @@ ] } - ,{ - "metrics": [ - "cpu.usage.system", - "cpu.usage.idle", - "cpu.usage.user" - ], - "queries": [ - { - "query":"select PercentProcessorTime, PercentIdleTime, PercentUserTime from Win32_PerfFormattedData_PerfOS_Processor", - "keys":["PercentProcessorTime","PercentIdleTime", "PercentUserTime"] - } - ] - } - ] } \ No newline at end of file