.

src/main/java/com/loafle/overflow/crawler/jmx/JmxCrawler.java

@@ -3,6 +3,7 @@ package com.loafle.overflow.crawler.jmx;
 import com.loafle.overflow.crawler.Crawler;
 import com.loafle.overflow.crawler.config.*;
 import com.loafle.overflow.crawler.config.Query;
+import com.loafle.overflow.crawler.result.OFResultSet;
 
 import javax.management.*;
 import javax.management.openmbean.CompositeData;
@@ -33,235 +34,281 @@ public class JmxCrawler extends Crawler{
     private String username;
     private String password;
     private boolean ssl;
-    private MBeanReceiver receiver;
+//    private MBeanReceiver receiver;
     private List<String> whiteObjectNameList = new ArrayList<>();
-    private List<List<String>> keys = new ArrayList<>();
+    private List<Keys> keys = new ArrayList<>();
-    private List<String> metrics = new ArrayList<>();
+    private List<Keys> metrics = new ArrayList<>();
-
     private Map<String, String> re = new HashMap<String, String>();
 
+    public JmxCrawler() {}
+
+    private List<String> parse(Map<String, String> dataMap, Map<String, Integer> meta) {
+
+        List<String> row =  new ArrayList<>(Arrays.asList(new String[meta.size()]));
+        for (Map.Entry<String, Integer> info : meta.entrySet()) {
+            String data = dataMap.get(info.getKey());
+            data = data.trim();
+            row.set(info.getValue().intValue(),data);
+        }
+        return row;
+    }
+
+    private Map<String, String> parseAttribute(MBeanServerConnection beanCon, ObjectName mbeanName) throws IntrospectionException, ReflectionException, InstanceNotFoundException, IOException, AttributeNotFoundException, MBeanException {
+
+        Map<String, String> returnMap = new HashMap<>();
+        MBeanInfo info = beanCon.getMBeanInfo(mbeanName);
+
+        for (MBeanAttributeInfo attr : info.getAttributes()) {
+
+            if (!attr.isReadable()) {
+                logCrawler(mbeanName, attr, "not readable");
+                continue;
+            }
+            Object att = beanCon.getAttribute(mbeanName, attr.getName());
+            if (att != null)
+                returnMap.put(attr.getName(), att.toString());
+        }
+        return returnMap;
+    }
+
     @Override
     public Object getInternal(Config config) throws Exception {
 
-        MBeanReceiver m = (domain, attrKeys, attrName, attrType, attrDescription, value) -> re.put(attrName, value.toString());
+//        MBeanReceiver m = (domain, attrKeys, attrName, attrType, attrDescription, value)
+//                -> re.put(attrName, value.toString());
+//        this.receiver = m;
+
 
-//        MBeanReceiver m = new MBeanReceiver() {
-//            @Override
-//            public void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
-//                re.put(attrName, value.toString());
-//            }
-//        };
-//        "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi";
         String hostIp = config.getTarget().getConnection().getIp();
         String port = config.getTarget().getConnection().getPort();
 
         this.jmxUrl = "service:jmx:rmi:///jndi/rmi://" + hostIp + ":" +port + "/jmxrmi";
         this.username = null;
         this.password = null;
-        this.receiver = m;
 
-        for (Item item : config.getItems()) {
+        // connection
-            List<com.loafle.overflow.crawler.config.Query> querys = item.getQueries();
-            this.metrics = item.getMetrics();
-            for (Query q : querys) {
-                this.whiteObjectNameList.add(q.getQuery());
-                this.keys.add(q.getKeys());
-            }
-        }
-
-
-        this.doCrawler();
-
-        Map<String, String> result = new LinkedHashMap<>();
-        int idx = 0;
-
-        for (List<String> ks : this.keys) {
-            for (String k : ks) {
-                for (String key : re.keySet()) {
-                    if (k.equals(key)) {
-                        String metricKey = this.metrics.get(idx);
-                        result.put(metricKey, re.get(key));
-                        idx++;
-                    }
-                }
-            }
-        }
-
-
-        return result;
-    }
-
-    public static interface MBeanReceiver {
-        void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value);
-    }
-
-    public JmxCrawler() {
-
-    }
-//    public JmxCrawler(String jmxUrl, String username, String password, boolean ssl, MBeanReceiver receiver) {
-//        this.jmxUrl = jmxUrl;
-//        this.username = username;
-//        this.password = password;
-//        this.ssl = ssl;
-//        this.receiver = receiver;
-//    }
-
-    // 커넥터 생성
-    public void doCrawler() throws Exception {
         MBeanServerConnection beanCon;
         JMXConnector jmxconn = null;
-
-        if (this.jmxUrl.isEmpty()) {
-            beanCon = ManagementFactory.getPlatformMBeanServer();
-        } else {
-            Map<String, Object> environment = getAuthority();
-            jmxconn = JMXConnectorFactory.connect(new JMXServiceURL(this.jmxUrl), environment);
-            beanCon = jmxconn.getMBeanServerConnection();
-        }
-
         try {
-            Set<ObjectInstance> mBeanNames = new HashSet();
+
+            if (this.jmxUrl.isEmpty()) {
+                beanCon = ManagementFactory.getPlatformMBeanServer();
+            } else {
+                Map<String, Object> environment = getAuthority();
+                jmxconn = JMXConnectorFactory.connect(new JMXServiceURL(this.jmxUrl), environment);
+                beanCon = jmxconn.getMBeanServerConnection();
+            }
+
+            List<OFResultSet> resultSetList = new ArrayList<>();
             List<ObjectName> objectNames = new LinkedList<ObjectName>();
             objectNames.add(null);
-
-//            for (ObjectName name : objectNames) {
-//                Set<ObjectInstance> objectInstances = beanCon.queryMBeans(name, null);
-//
-//                for (ObjectInstance oi : objectInstances ) {
-//                    System.out.println("oi = " + oi.getClassName());
-//                }
-//                mBeanNames.addAll(beanCon.queryMBeans(name, null));
-////                beanCon.queryMBeans(new ObjectName("Catalina:type=Engine"), null);
-//            }
-
-//            for (ObjectInstance name : mBeanNames) {
-//                crawlerBean(beanCon, name.getObjectName());
-//            }
-
-//            ObjectName on = new ObjectName("Catalina:type=*,*");
-//            ObjectName on = new ObjectName("org.apache.cassandra.db:type=*,*");
-
             ObjectName on = null;
 
-            for (String q : this.whiteObjectNameList) {
+            for (Item item : config.getItems()) {
-                on = new ObjectName(q);
+
-                mBeanNames.addAll(beanCon.queryMBeans(on,null));
+                OFResultSet resultSet = OFResultSet.newInstance(item);
+
+                // get object
+                String query = item.getQueryInfo().getQuery();
+                on = new ObjectName(query);
+                Set<ObjectInstance> names = beanCon.queryMBeans(on, null);
+
+                List<Map<String,Object>> keys = (List<Map<String,Object>>) item.getQueryInfo().getExtend().get("aliases");
+
+                for (ObjectInstance name : names) {
+
+                    Map<String, String> attrs = parseAttribute(beanCon,name.getObjectName());
+                    if (keys != null) {
+                        for ( Map<String,Object> keyMap : keys ) {
+                            String key = (String) keyMap.get("key");
+                            int index =  (int) keyMap.get("index");
+                            String objectName = parseObjectName(name.toString(),key);
+                            String alias = item.getMappingInfo().getArrayColumns().get(index);
+                            attrs.put(alias,objectName);
+                        }
+                    }
+
+                    resultSet.addRow(parse(attrs, resultSet.getMeta()));
+                }
+
+                resultSetList.add(resultSet);
             }
-            for (ObjectInstance name : mBeanNames) {
+
-                crawlerBean(beanCon, name.getObjectName());
+            return resultSetList;
-            }
+        } catch(Exception e) {
-        }finally {
+            e.printStackTrace();
+            throw e;
+        } finally {
             if (jmxconn != null) {
                 jmxconn.close();
             }
         }
     }
 
+    private String parseObjectName(String s, String key) {
+
+        int start = s.indexOf("[");
+        int end = s.lastIndexOf("]");
+        s = s.substring(start,end);
+
+        String [] slices = s.split(",");
+        String returnData = "";
+        for (String item : slices) {
+            String [] itemSplit = item.split("=");
+            if (itemSplit == null || itemSplit.length != 2) {continue;}
+            if (itemSplit[0].equals(key)) {
+                returnData = itemSplit[1];
+            }
+        }
+
+        return returnData;
+    }
+
+//    public static interface MBeanReceiver {
+//        void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value);
+//    }
+//
+//
+//
+//    // 커넥터 생성
+//    public void doCrawler() throws Exception {
+//        MBeanServerConnection beanCon;
+//        JMXConnector jmxconn = null;
+//
+//        if (this.jmxUrl.isEmpty()) {
+//            beanCon = ManagementFactory.getPlatformMBeanServer();
+//        } else {
+//            Map<String, Object> environment = getAuthority();
+//            jmxconn = JMXConnectorFactory.connect(new JMXServiceURL(this.jmxUrl), environment);
+//            beanCon = jmxconn.getMBeanServerConnection();
+//        }
+//
+//        try {
+//            Set<ObjectInstance> mBeanNames = new HashSet();
+//            List<ObjectName> objectNames = new LinkedList<ObjectName>();
+//            objectNames.add(null);
+//
+//            ObjectName on = null;
+//
+//            for (String q : this.whiteObjectNameList) {
+//                on = new ObjectName(q);
+//                mBeanNames.addAll(beanCon.queryMBeans(on, null));
+//            }
+//            for (ObjectInstance name : mBeanNames) {
+//                crawlerBean(beanCon, name.getObjectName());
+//            }
+//        }finally {
+//            if (jmxconn != null) {
+//                jmxconn.close();
+//            }
+//        }
+//    }
+
     // 수집
-    private void crawlerBean(MBeanServerConnection beanCon, ObjectName mbeanName) {
+//    private void crawlerBean(MBeanServerConnection beanCon, ObjectName mbeanName) {
-        MBeanInfo info;
+//        MBeanInfo info;
+//
+//        try {
+//            info = beanCon.getMBeanInfo(mbeanName);
+//        } catch (IOException e) {
+//            logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
+//            return;
+//        } catch (JMException e) {
+//            logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
+//            return;
+//        }
+//
+//        MBeanAttributeInfo[] attrInfos = info.getAttributes();
+//
+//        for (int idx = 0; idx < attrInfos.length; ++idx) {
+//            MBeanAttributeInfo attr = attrInfos[idx];
+//
+//            if (!attr.isReadable()) {
+//                logCrawler(mbeanName, attr, "not readable");
+//                continue;
+//            }
+//
+//            Object value;
+//
+//            try {
+//                value = beanCon.getAttribute(mbeanName, attr.getName());
+//            } catch (Exception e) {
+//                logCrawler(mbeanName, attr, "Fail: " + e);
+//                continue;
+//            }
+//
+//            logCrawler(mbeanName, attr, "process");
+//            processBeanValue(
+//                    mbeanName.getDomain(),
+//                    new LinkedList<String>(),
+//                    attr.getName(),
+//                    attr.getType(),
+//                    attr.getDescription(),
+//                    value
+//            );
+//        }
+//    }
 
-        try {
+//    private void processBeanValue(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
-            info = beanCon.getMBeanInfo(mbeanName);
+//        if (value == null) {
-        } catch (IOException e) {
+//            logCrawler("Value Null : "+domain + attrName, "null");
-            logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
+//        } else if (value instanceof Number || value instanceof String || value instanceof Boolean) {
-            return;
+//            logCrawler(domain + attrName, value.toString());
-        } catch (JMException e) {
+//            this.receiver.recordBean(domain, attrKeys, attrName, attrType, attrDescription,value);
-            logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
+//            // JSON 결과 생성
-            return;
+//        } else if (value instanceof CompositeData) {
-        }
+//            CompositeData composite = (CompositeData)value;
-
+//            CompositeType type = composite.getCompositeType();
-        MBeanAttributeInfo[] attrInfos = info.getAttributes();
+//            attrKeys = new LinkedList<String>();
-
+//            attrKeys.add(attrName);
-        for (int idx = 0; idx < attrInfos.length; ++idx) {
+//
-            MBeanAttributeInfo attr = attrInfos[idx];
+//            for (String key : type.keySet()) {
-
+//                String typ = type.getType(key).getTypeName();
-            if (!attr.isReadable()) {
+//                Object val = composite.get(key);
-                logCrawler(mbeanName, attr, "not readable");
+//
-                continue;
+//                processBeanValue(domain, attrKeys, key, typ, type.getDescription(), val);
-            }
+//            }
-
+//        } else if (value instanceof TabularData) {
-            Object value;
+//            TabularData tds = (TabularData)value;
-
+//            TabularType tt = tds.getTabularType();
-            try {
+//
-                value = beanCon.getAttribute(mbeanName, attr.getName());
+//            CompositeType type = tt.getRowType();
-            } catch (Exception e) {
+//            Set<String> valueKeys = new TreeSet<String>(type.keySet());
-                logCrawler(mbeanName, attr, "Fail: " + e);
+//
-                continue;
+//            LinkedList<String> extendedAttrKeys = new LinkedList<String>(attrKeys);
-            }
+//            extendedAttrKeys.add(attrName);
-
+//
-            logCrawler(mbeanName, attr, "process");
+//            for (Object valu : tds.values()) {
-            processBeanValue(
+//                if (valu instanceof CompositeData) {
-                    mbeanName.getDomain(),
+//                    CompositeData composite = (CompositeData) valu;
-                    new LinkedList<String>(),
+//
-                    attr.getName(),
+//                    for(String valueIdx : valueKeys) {
-                    attr.getType(),
+//                        LinkedList<String> attrNames = extendedAttrKeys;
-                    attr.getDescription(),
+//                        String typ = type.getType(valueIdx).getTypeName();
-                    value
+//                        String name = valueIdx;
-            );
+//                        if (valueIdx.toLowerCase().equals("value")) {
-        }
+//                            // Skip appending 'value' to the name
-    }
+//                            attrNames = attrKeys;
-
+//                            name = attrName;
-    private void processBeanValue(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
+//                        }
-        if (value == null) {
+//                        processBeanValue(
-            logCrawler("Value Null : "+domain + attrName, "null");
+//                                domain,
-        } else if (value instanceof Number || value instanceof String || value instanceof Boolean) {
+//                                attrNames,
-            logCrawler(domain + attrName, value.toString());
+//                                name,
-            this.receiver.recordBean(domain, attrKeys, attrName, attrType, attrDescription,value);
+//                                typ,
-            // JSON 결과 생성
+//                                type.getDescription(),
-        } else if (value instanceof CompositeData) {
+//                                composite.get(valueIdx));
-            CompositeData composite = (CompositeData)value;
+//                    }
-            CompositeType type = composite.getCompositeType();
+//                } else {
-            attrKeys = new LinkedList<String>();
+//                    logCrawler(domain, "not a correct tabulardata format");
-            attrKeys.add(attrName);
+//                }
-
+//            }
-            for (String key : type.keySet()) {
+//        } else if (value.getClass().isArray()) {
-                String typ = type.getType(key).getTypeName();
+//            logCrawler(domain, "arrays are unsupported");
-                Object val = composite.get(key);
+//        } else {
-
+//            logCrawler(domain, attrType + "is not exported");
-                processBeanValue(domain, attrKeys, key, typ, type.getDescription(), val);
+//        }
-            }
+//    }
-        } else if (value instanceof TabularData) {
-            TabularData tds = (TabularData)value;
-            TabularType tt = tds.getTabularType();
-
-            CompositeType type = tt.getRowType();
-            Set<String> valueKeys = new TreeSet<String>(type.keySet());
-
-            LinkedList<String> extendedAttrKeys = new LinkedList<String>(attrKeys);
-            extendedAttrKeys.add(attrName);
-
-            for (Object valu : tds.values()) {
-                if (valu instanceof CompositeData) {
-                    CompositeData composite = (CompositeData) valu;
-
-                    for(String valueIdx : valueKeys) {
-                        LinkedList<String> attrNames = extendedAttrKeys;
-                        String typ = type.getType(valueIdx).getTypeName();
-                        String name = valueIdx;
-                        if (valueIdx.toLowerCase().equals("value")) {
-                            // Skip appending 'value' to the name
-                            attrNames = attrKeys;
-                            name = attrName;
-                        }
-                        processBeanValue(
-                                domain,
-                                attrNames,
-                                name,
-                                typ,
-                                type.getDescription(),
-                                composite.get(valueIdx));
-                    }
-                } else {
-                    logCrawler(domain, "not a correct tabulardata format");
-                }
-            }
-        } else if (value.getClass().isArray()) {
-            logCrawler(domain, "arrays are unsupported");
-        } else {
-            logCrawler(domain, attrType + "is not exported");
-        }
-    }
 
     private Map<String, Object> getAuthority() {
 
@@ -290,42 +337,4 @@ public class JmxCrawler extends Crawler{
         logCrawler(mbeanName + "'_'" + attr.getName(), msg);
     }
 
-    private static class StdoutWriter implements MBeanReceiver {
-        public void recordBean(
-                String domain,
-                LinkedList<String> attrKeys,
-                String attrName,
-                String attrType,
-                String attrDescription,
-                Object value) {
-
-                    System.out.println(domain +
-                    " " + attrKeys + " "  +
-                    " " + attrName + " " +
-                    " " + attrDescription + " " +
-                    " " + value);
-        }
-    }
-
-//    public static void main(String[] args) throws Exception {
-
-//        JmxCrawler jmxCrawler = new JmxCrawler(
-////                "service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi",
-//                "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi",
-////                "",
-//                "",
-//                "",
-//                false,
-//                new StdoutWriter()
-//        );
-//
-//        jmxCrawler.doCrawler();
-
-//        JmxCrawler d = new JmxCrawler();
-//        Map<String, Object> m = new HashMap<>();
-//        m.put("ip","service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi");
-//
-//        Object r = d.getInternal(m);
-//        System.out.println("r = " + r);
-//    }
 }

src/test/java/com/loafle/AppTest.java

@@ -2,13 +2,18 @@
 package com.loafle;
 import static org.junit.Assert.*;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.loafle.overflow.crawler.Crawler;
 import com.loafle.overflow.crawler.config.*;
 import com.loafle.overflow.crawler.jmx.JmxCrawler;
+import com.loafle.overflow.crawler.result.OFResultSet;
 import org.junit.Test;
 
+import java.io.File;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class AppTest {
     @Test
@@ -18,71 +23,43 @@ public class AppTest {
 
     @Test
     public void TestDoCrawler() throws Exception {
-//        JmxCrawler jmxCrawler = new JmxCrawler(
-////                "service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi",
-//                "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi",
-////                "",
-//                "",
-//                "",
-//                false,
-//                new JmxCrawler.StdoutWriter()
-//        );
-//
-//        jmxCrawler.doCrawler();
 
-        Config config = new Config();
+        ClassLoader classLoader = getClass().getClassLoader();
+        String path = classLoader.getResource("config/example.json").getFile();
 
-        config.setId("SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734");
+        ObjectMapper mapper = new ObjectMapper();
-        Connection cn = new Connection();
+        Config c = mapper.readValue(new File(path),Config.class);
-        cn.setIp("192.168.1.103");
-        cn.setPort("9840");
-        cn.setSsl(false);
-        cn.setPortType("tcp");
-
-        Target t = new Target();
-        t.setConnection(cn);
-
-        Schedule s = new Schedule();
-        s.setInterval("10");
-
-        com.loafle.overflow.crawler.config.Crawler cc = new com.loafle.overflow.crawler.config.Crawler();
-        cc.setName("jmx_crawler");
-        cc.setContainer("java_crawler");
-
-        List<Item> ls = new ArrayList<>();
-        List<String> metrics = new ArrayList<>();
-        List<Query> queries = new ArrayList<>();
-        List<String> keys = new ArrayList<>();
-        keys.add("ProcessCpuLoad");
-        keys.add("ProcessCpuTime");
-        keys.add("SystemCpuLoad");
-
-        Query query = new Query();
-        query.setKeys(keys);
-        query.setQuery("java.lang:type=OperatingSystem");
-
-        queries.add(query);
-
-        metrics.add("cpu.system");
-        metrics.add("cpu.user");
-        metrics.add("cpu.system_children");
-        metrics.add("cpu.user_children");
-
-        Item it = new Item();
-        it.setMetrics(metrics);
-        it.setQueries(queries);
-
-        ls.add(it);
-
-        config.setTarget(t);
-        config.setSchedule(s);
-        config.setCrawler(cc);
-        config.setItems(ls);
 
         JmxCrawler cr = new JmxCrawler();
+        List<OFResultSet> result = (List<OFResultSet>) cr.getInternal(c);
 
-       Object o =  cr.getInternal(config);
+        print("",result);
 
-        System.out.println("o = " + o);
     }
+
+    private void print(String s, List<OFResultSet> mm) {
+
+        System.out.println(s);
+
+        Map<String,String> m = new HashMap<>();
+        for(OFResultSet r : mm) {
+            m.putAll(r.getData());
+        }
+
+        for (Map.Entry<String, String> item : m.entrySet()) {
+            System.out.println("key=" + item.getKey() +  " |||||||||||  value=" +  item.getValue());
+        }
+    }
+
+    @Test
+    public void testParseObject () {
+
+        String s = "org.apache.catalina.mbeans.ConnectorMBean[Catalina:type=Connector,port=8080]";
+
+        int start = s.indexOf("[");
+        int end = s.lastIndexOf("]");
+        s = s.substring(start + 1,end);
+        assertEquals("Catalina:type=Connector,port=8080",s);
+    }
+
 }

src/test/resources/config/example.json

@@ -0,0 +1,44 @@
+{
+  "id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
+  "target" : {
+    "connection" : {
+      "ip" : "192.168.1.103",
+      "port" : "9840",
+      "ssl" : false,
+      "portType" : "tcp"
+    },
+    "auth" : {
+
+    }
+  },
+  "schedule" : {
+    "interval" : "10"
+  },
+  "crawler" : {
+    "name":"jmx",
+    "container":"java_proxy"
+  },
+  "items" : [
+    {
+      "keys" : [
+        {"metric" : "net.connector[$0].localPort",          "key" : "localPort"},
+        {"metric" : "net.connector[$0].port",               "key" : "port"},
+        {"metric" : "net.connector[$0].protocol",           "key" : "protocol"}
+      ],
+      "queryInfo" : {
+        "query": "*Catalina:type=Connector,*",
+        "extend" : {
+          "aliases" : [
+            {
+              "key": "port",
+              "index":0
+            }
+          ]
+        }
+      },
+      "mappingInfo" : {
+        "arrayColumns" : [ "portName" ]
+      }
+    }
+  ]
+}