From eba6757f1ddb75bf3aa298fb29d7e7ea3ff333f9 Mon Sep 17 00:00:00 2001 From: snoop Date: Fri, 24 Nov 2017 15:40:10 +0900 Subject: [PATCH] added meta crawler --- .../overflow/crawler/wmi/WMICrawler.java | 57 ++++++++++ src/main/resources/config/meta/meta_cpu.json | 49 +++++++++ .../config/meta/meta_filesystem.json | 61 +++++++++++ .../resources/config/meta/meta_memory.json | 53 +++++++++ .../resources/config/meta/meta_network.json | 53 +++++++++ .../overflow/crawler/wmi/WMICrawlerTest.java | 101 ++++++++++++++++-- 6 files changed, 367 insertions(+), 7 deletions(-) create mode 100644 src/main/resources/config/meta/meta_cpu.json create mode 100644 src/main/resources/config/meta/meta_filesystem.json create mode 100644 src/main/resources/config/meta/meta_memory.json create mode 100644 src/main/resources/config/meta/meta_network.json diff --git a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java index add281d..8bbbe52 100644 --- a/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java +++ b/src/main/java/com/loafle/overflow/crawler/wmi/WMICrawler.java @@ -1,8 +1,15 @@ package com.loafle.overflow.crawler.wmi; +import com.fasterxml.jackson.databind.ObjectMapper; import com.loafle.overflow.crawler.Crawler; import com.loafle.overflow.crawler.config.Config; +import com.loafle.overflow.crawler.result.OFResultSet; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @@ -16,4 +23,54 @@ public class WMICrawler extends Crawler { return WMICrawlerOS.getInstance().process(config); } + + public Object getInternalMeta(Config config) throws Exception { + + List metaConfigList = loadMetaConfig(); + + + List retlist = new ArrayList<>(); + WMICrawlerOS os = WMICrawlerOS.getInstance(); + OFResultSet ofResultSet = null; + for(Config metaConfig : metaConfigList) { + + for(int indexI = 0; indexI < metaConfig.getItems().size(); ++indexI) { + ofResultSet = os.processWMI(config, metaConfig.getItems().get(indexI)); + if(ofResultSet != null) { + retlist.add(ofResultSet); + } + } + } + + return null; + } + + private List loadMetaConfig() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + ObjectMapper mapper = new ObjectMapper(); + + URL url = classLoader.getResource("config/meta"); + if(url == null) { + return null; + } + + String path = url.getFile(); + + File pathFile = new File(path); + + System.out.println(path); + String[] metaFiles = pathFile.list(); + List retList = new ArrayList<>(); + Config metaConfig = null; + for(String fi : metaFiles) { + if(fi.indexOf("meta_") > -1) { + metaConfig = mapper.readValue(new File(path+"/"+fi), Config.class); + if(metaConfig != null) { + retList.add(metaConfig); + } + } + } + + return retList; + } } diff --git a/src/main/resources/config/meta/meta_cpu.json b/src/main/resources/config/meta/meta_cpu.json new file mode 100644 index 0000000..8402774 --- /dev/null +++ b/src/main/resources/config/meta/meta_cpu.json @@ -0,0 +1,49 @@ +{ + "id": "4", + "schedule": { + "interval": "5" + }, + "items": [{ + "keys": [{ + "metric": "cpu.caption", + "key": "Caption" + }, + { + "metric": "cpu.Manufacturer", + "key": "Manufacturer" + }, + { + "metric": "cpu.MaxClockSpeed", + "key": "MaxClockSpeed" + }, + { + "metric": "cpu.CurrentClockSpeed", + "key": "CurrentClockSpeed" + }, + { + "metric": "cpu.AddressWidth", + "key": "AddressWidth" + }, + { + "metric": "cpu.DataWidth", + "key": "DataWidth" + }, + { + "metric": "cpu.Architecture", + "key": "Architecture" + }], + "queryInfo": { + "query": "SELECT Caption, Manufacturer, MaxClockSpeed, CurrentClockSpeed, AddressWidth, DataWidth, Architecture from Win32_Processor ", + "extend": { + "nameSpace": "root/cimv2", + "wmicPath": "/home/snoop/temp/wmic" + } + }, + "mappingInfo": { + "parseDirection": "col", + "arrayColumns": null, + "keyColumns": null, + "valueColumn": null + } + }] +} \ No newline at end of file diff --git a/src/main/resources/config/meta/meta_filesystem.json b/src/main/resources/config/meta/meta_filesystem.json new file mode 100644 index 0000000..d77ec1c --- /dev/null +++ b/src/main/resources/config/meta/meta_filesystem.json @@ -0,0 +1,61 @@ +{ + "id": "4", + "target": { + "auth": { + "pw": "dbseogns18", + "id": "administrator" + }, + "connection": { + "ip": "192.168.1.106", + "port": "135", + "portType": "tcp", + "ssl": false + } + }, + "schedule": { + "interval": "5" + }, + "crawler": { + "name": "WMI_CRAWLER", + "container": "java_proxy" + }, + "items": [{ + "keys": [{ + "metric": "filesystem[$0].InterfaceType", + "key": "InterfaceType" + }, + { + "metric": "filesystem[$0].Status", + "key": "Status" + }, + { + "metric": "filesystem[$0].Manufacturer", + "key": "Manufacturer" + }, + { + "metric": "filesystem[$0].Model", + "key": "Model" + }, + { + "metric": "filesystem[$0].MediaType", + "key": "MediaType" + }, + { + "metric": "filesystem[$0].Size", + "key": "Size" + }], + "queryInfo": { + "query": "SELECT InterfaceType, Status, Manufacturer, Model, MediaType, Size from Win32_DiskDrive ", + "extend": { + "nameSpace": "root/cimv2", + "wmicPath": "/home/snoop/temp/wmic" + } + }, + "mappingInfo": { + "parseDirection": "col", + "arrayColumns": ["Model"], + "keyColumns": null, + "valueColumn": null + } + }] +} \ No newline at end of file diff --git a/src/main/resources/config/meta/meta_memory.json b/src/main/resources/config/meta/meta_memory.json new file mode 100644 index 0000000..6af69d1 --- /dev/null +++ b/src/main/resources/config/meta/meta_memory.json @@ -0,0 +1,53 @@ +{ + "id": "4", + "target": { + "auth": { + "pw": "dbseogns18", + "id": "administrator" + }, + "connection": { + "ip": "192.168.1.106", + "port": "135", + "portType": "tcp", + "ssl": false + } + }, + "schedule": { + "interval": "5" + }, + "crawler": { + "name": "WMI_CRAWLER", + "container": "java_proxy" + }, + "items": [{ + "keys": [{ + "metric": "memory.BankLabel", + "key": "BankLabel" + }, + { + "metric": "memory.Capacity", + "key": "Capacity" + }, + { + "metric": "memory.MemoryType", + "key": "MemoryType" + }, + { + "metric": "memory.DataWidth", + "key": "DataWidth" + }], + "queryInfo": { + "query": "SELECT BankLabel, Capacity, MemoryType, DataWidth from Win32_PhysicalMemory ", + "extend": { + "nameSpace": "root/cimv2", + "wmicPath": "/home/snoop/temp/wmic" + } + }, + "mappingInfo": { + "parseDirection": "col", + "arrayColumns": null, + "keyColumns": null, + "valueColumn": null + } + }] +} \ No newline at end of file diff --git a/src/main/resources/config/meta/meta_network.json b/src/main/resources/config/meta/meta_network.json new file mode 100644 index 0000000..7216486 --- /dev/null +++ b/src/main/resources/config/meta/meta_network.json @@ -0,0 +1,53 @@ +{ + "id": "4", + "target": { + "auth": { + "pw": "dbseogns18", + "id": "administrator" + }, + "connection": { + "ip": "192.168.1.106", + "port": "135", + "portType": "tcp", + "ssl": false + } + }, + "schedule": { + "interval": "5" + }, + "crawler": { + "name": "WMI_CRAWLER", + "container": "java_proxy" + }, + "items": [{ + "keys": [{ + "metric": "network[$0].MACAddress", + "key": "MACAddress" + }, + { + "metric": "network[$0].AdapterType", + "key": "AdapterType" + }, + { + "metric": "network[$0].Manufacturer", + "key": "Manufacturer" + }, + { + "metric": "network[$0].Description", + "key": "Description" + }], + "queryInfo": { + "query": "SELECT MACAddress, AdapterType, Manufacturer, Description, NetConnectionID from Win32_NetworkAdapter WHERE PhysicalAdapter='TRUE'", + "extend": { + "nameSpace": "root/cimv2", + "wmicPath": "/home/snoop/temp/wmic" + } + }, + "mappingInfo": { + "parseDirection": "col", + "arrayColumns": ["NetConnectionID"], + "keyColumns": null, + "valueColumn": null + } + }] +} \ No newline at end of file diff --git a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java index 255fd1a..ad8d76d 100644 --- a/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java +++ b/src/test/java/com/loafle/overflow/crawler/wmi/WMICrawlerTest.java @@ -3,19 +3,14 @@ package com.loafle.overflow.crawler.wmi; import com.fasterxml.jackson.databind.ObjectMapper; import com.loafle.overflow.crawler.config.Config; import com.loafle.overflow.crawler.result.OFResultSet; -import com.sun.xml.internal.ws.api.ResourceLoader; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Created by root on 17. 4. 11. @@ -123,7 +118,7 @@ public class WMICrawlerTest { ObjectMapper mapper = new ObjectMapper(); ClassLoader classLoader = getClass().getClassLoader(); - String p = classLoader.getResource("config/test.json").getFile(); + String p = classLoader.getResource("config/meta/meta_network.json").getFile(); Config c = mapper.readValue(new File(p),Config.class); // Map map = new HashMap<>(); @@ -141,6 +136,23 @@ public class WMICrawlerTest { List resultList = (List)wmiCrawler.getInternal(c); + PrintResultList(resultList); +// for(int indexI = 0 ; indexI < resultList.size(); ++indexI) { +// Map resultMap = resultList.get(indexI).getData(); +// System.out.println("============================================"); +// for( String key : resultMap.keySet() ){ +// System.out.println( String.format("key : %s ||| value : %s", key, resultMap.get(key)) ); +// } +// System.out.println("============================================"); +// } + + + + + + } + + public void PrintResultList(List resultList) { for(int indexI = 0 ; indexI < resultList.size(); ++indexI) { Map resultMap = resultList.get(indexI).getData(); System.out.println("============================================"); @@ -149,11 +161,68 @@ public class WMICrawlerTest { } System.out.println("============================================"); } + } + + @Test + public void testMetaProcess() throws Exception { + + // get targetinfo + ObjectMapper mapper = new ObjectMapper(); + ClassLoader classLoader = getClass().getClassLoader(); + String p = classLoader.getResource("config/test.json").getFile(); + Config c = mapper.readValue(new File(p),Config.class); + + List metaConfigList = loadMetaConfig(); + + + List retlist = new ArrayList<>(); + WMICrawlerOS os = WMICrawlerOS.getInstance(); + OFResultSet ofResultSet = null; + for(Config metaConfig : metaConfigList) { + + for(int indexI = 0; indexI < metaConfig.getItems().size(); ++indexI) { + ofResultSet = os.processWMI(c, metaConfig.getItems().get(indexI)); + if(ofResultSet != null) { + retlist.add(ofResultSet); + } + } + } + PrintResultList(retlist); + + } + + public List loadMetaConfig() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + ObjectMapper mapper = new ObjectMapper(); + + URL url = classLoader.getResource("config/meta"); + if(url == null) { + return null; + } + + String path = url.getFile(); + + File pathFile = new File(path); + + System.out.println(path); + String[] metaFiles = pathFile.list(); + List retList = new ArrayList<>(); + Config metaConfig = null; + for(String fi : metaFiles) { + if(fi.indexOf("meta_") > -1) { + metaConfig = mapper.readValue(new File(path+"/"+fi), Config.class); + if(metaConfig != null) { + retList.add(metaConfig); + } + } + } + + return retList; } @Test @@ -232,4 +301,22 @@ public class WMICrawlerTest { } + @Test + public void testRes() throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + + URL url = classLoader.getResource("config/meta"); + + + File file = new File(url.getFile()); + + String[] fl = file.list(); + + for(String f : fl) { + System.out.println(f); + } + + + } + } \ No newline at end of file