overflow_backup_2018_0455/crawler_jmx_java
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jackdaw@loafle.com 2017-04-26 18:06:00 +09:00
parent ac3873a6ca
commit d305a43850
3 changed files with 327 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

463
src/main/java/com/loafle/overflow/crawler/jmx/JmxCrawler.java
View File

@ -3,6 +3,7 @@ package com.loafle.overflow.crawler.jmx;
import com.loafle.overflow.crawler.Crawler;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.crawler.config.Query;
import com.loafle.overflow.crawler.result.OFResultSet;
import javax.management.*;
import javax.management.openmbean.CompositeData;
@ -33,83 +34,62 @@ public class JmxCrawler extends Crawler{
private String username;
private String password;
private boolean ssl;
private MBeanReceiver receiver;
// private MBeanReceiver receiver;
private List<String> whiteObjectNameList = new ArrayList<>();
private List<List<String>> keys = new ArrayList<>();
private List<String> metrics = new ArrayList<>();
private List<Keys> keys = new ArrayList<>();
private List<Keys> metrics = new ArrayList<>();
private Map<String, String> re = new HashMap<String, String>();
public JmxCrawler() {}
private List<String> parse(Map<String, String> dataMap, Map<String, Integer> meta) {
List<String> row = new ArrayList<>(Arrays.asList(new String[meta.size()]));
for (Map.Entry<String, Integer> info : meta.entrySet()) {
String data = dataMap.get(info.getKey());
data = data.trim();
row.set(info.getValue().intValue(),data);
}
return row;
}
private Map<String, String> parseAttribute(MBeanServerConnection beanCon, ObjectName mbeanName) throws IntrospectionException, ReflectionException, InstanceNotFoundException, IOException, AttributeNotFoundException, MBeanException {
Map<String, String> returnMap = new HashMap<>();
MBeanInfo info = beanCon.getMBeanInfo(mbeanName);
for (MBeanAttributeInfo attr : info.getAttributes()) {
if (!attr.isReadable()) {
logCrawler(mbeanName, attr, "not readable");
continue;
}
Object att = beanCon.getAttribute(mbeanName, attr.getName());
if (att != null)
returnMap.put(attr.getName(), att.toString());
}
return returnMap;
}
@Override
public Object getInternal(Config config) throws Exception {
MBeanReceiver m = (domain, attrKeys, attrName, attrType, attrDescription, value) -> re.put(attrName, value.toString());
// MBeanReceiver m = (domain, attrKeys, attrName, attrType, attrDescription, value)
// -> re.put(attrName, value.toString());
// this.receiver = m;
// MBeanReceiver m = new MBeanReceiver() {
// @Override
// public void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
// re.put(attrName, value.toString());
// }
// };
// "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi";
String hostIp = config.getTarget().getConnection().getIp();
String port = config.getTarget().getConnection().getPort();
this.jmxUrl = "service:jmx:rmi:///jndi/rmi://" + hostIp + ":" +port + "/jmxrmi";
this.username = null;
this.password = null;
this.receiver = m;
for (Item item : config.getItems()) {
List<com.loafle.overflow.crawler.config.Query> querys = item.getQueries();
this.metrics = item.getMetrics();
for (Query q : querys) {
this.whiteObjectNameList.add(q.getQuery());
this.keys.add(q.getKeys());
}
}
this.doCrawler();
Map<String, String> result = new LinkedHashMap<>();
int idx = 0;
for (List<String> ks : this.keys) {
for (String k : ks) {
for (String key : re.keySet()) {
if (k.equals(key)) {
String metricKey = this.metrics.get(idx);
result.put(metricKey, re.get(key));
idx++;
}
}
}
}
return result;
}
public static interface MBeanReceiver {
void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value);
}
public JmxCrawler() {
}
// public JmxCrawler(String jmxUrl, String username, String password, boolean ssl, MBeanReceiver receiver) {
// this.jmxUrl = jmxUrl;
// this.username = username;
// this.password = password;
// this.ssl = ssl;
// this.receiver = receiver;
// }
// 커넥터 생성
public void doCrawler() throws Exception {
// connection
MBeanServerConnection beanCon;
JMXConnector jmxconn = null;
try {
if (this.jmxUrl.isEmpty()) {
beanCon = ManagementFactory.getPlatformMBeanServer();
@ -119,37 +99,45 @@ public class JmxCrawler extends Crawler{
beanCon = jmxconn.getMBeanServerConnection();
}
try {
Set<ObjectInstance> mBeanNames = new HashSet();
List<OFResultSet> resultSetList = new ArrayList<>();
List<ObjectName> objectNames = new LinkedList<ObjectName>();
objectNames.add(null);
// for (ObjectName name : objectNames) {
// Set<ObjectInstance> objectInstances = beanCon.queryMBeans(name, null);
//
// for (ObjectInstance oi : objectInstances ) {
// System.out.println("oi = " + oi.getClassName());
// }
// mBeanNames.addAll(beanCon.queryMBeans(name, null));
//// beanCon.queryMBeans(new ObjectName("Catalina:type=Engine"), null);
// }
// for (ObjectInstance name : mBeanNames) {
// crawlerBean(beanCon, name.getObjectName());
// }
// ObjectName on = new ObjectName("Catalina:type=*,*");
// ObjectName on = new ObjectName("org.apache.cassandra.db:type=*,*");
ObjectName on = null;
for (String q : this.whiteObjectNameList) {
on = new ObjectName(q);
mBeanNames.addAll(beanCon.queryMBeans(on,null));
for (Item item : config.getItems()) {
OFResultSet resultSet = OFResultSet.newInstance(item);
// get object
String query = item.getQueryInfo().getQuery();
on = new ObjectName(query);
Set<ObjectInstance> names = beanCon.queryMBeans(on, null);
List<Map<String,Object>> keys = (List<Map<String,Object>>) item.getQueryInfo().getExtend().get("aliases");
for (ObjectInstance name : names) {
Map<String, String> attrs = parseAttribute(beanCon,name.getObjectName());
if (keys != null) {
for ( Map<String,Object> keyMap : keys ) {
String key = (String) keyMap.get("key");
int index = (int) keyMap.get("index");
String objectName = parseObjectName(name.toString(),key);
String alias = item.getMappingInfo().getArrayColumns().get(index);
attrs.put(alias,objectName);
}
for (ObjectInstance name : mBeanNames) {
crawlerBean(beanCon, name.getObjectName());
}
resultSet.addRow(parse(attrs, resultSet.getMeta()));
}
resultSetList.add(resultSet);
}
return resultSetList;
} catch(Exception e) {
e.printStackTrace();
throw e;
} finally {
if (jmxconn != null) {
jmxconn.close();
@ -157,111 +145,170 @@ public class JmxCrawler extends Crawler{
}
}
private String parseObjectName(String s, String key) {
int start = s.indexOf("[");
int end = s.lastIndexOf("]");
s = s.substring(start,end);
String [] slices = s.split(",");
String returnData = "";
for (String item : slices) {
String [] itemSplit = item.split("=");
if (itemSplit == null || itemSplit.length != 2) {continue;}
if (itemSplit[0].equals(key)) {
returnData = itemSplit[1];
}
}
return returnData;
}
// public static interface MBeanReceiver {
// void recordBean(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value);
// }
//
//
//
// // 커넥터 생성
// public void doCrawler() throws Exception {
// MBeanServerConnection beanCon;
// JMXConnector jmxconn = null;
//
// if (this.jmxUrl.isEmpty()) {
// beanCon = ManagementFactory.getPlatformMBeanServer();
// } else {
// Map<String, Object> environment = getAuthority();
// jmxconn = JMXConnectorFactory.connect(new JMXServiceURL(this.jmxUrl), environment);
// beanCon = jmxconn.getMBeanServerConnection();
// }
//
// try {
// Set<ObjectInstance> mBeanNames = new HashSet();
// List<ObjectName> objectNames = new LinkedList<ObjectName>();
// objectNames.add(null);
//
// ObjectName on = null;
//
// for (String q : this.whiteObjectNameList) {
// on = new ObjectName(q);
// mBeanNames.addAll(beanCon.queryMBeans(on, null));
// }
// for (ObjectInstance name : mBeanNames) {
// crawlerBean(beanCon, name.getObjectName());
// }
// }finally {
// if (jmxconn != null) {
// jmxconn.close();
// }
// }
// }
// 수집
private void crawlerBean(MBeanServerConnection beanCon, ObjectName mbeanName) {
MBeanInfo info;
// private void crawlerBean(MBeanServerConnection beanCon, ObjectName mbeanName) {
// MBeanInfo info;
//
// try {
// info = beanCon.getMBeanInfo(mbeanName);
// } catch (IOException e) {
// logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
// return;
// } catch (JMException e) {
// logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
// return;
// }
//
// MBeanAttributeInfo[] attrInfos = info.getAttributes();
//
// for (int idx = 0; idx < attrInfos.length; ++idx) {
// MBeanAttributeInfo attr = attrInfos[idx];
//
// if (!attr.isReadable()) {
// logCrawler(mbeanName, attr, "not readable");
// continue;
// }
//
// Object value;
//
// try {
// value = beanCon.getAttribute(mbeanName, attr.getName());
// } catch (Exception e) {
// logCrawler(mbeanName, attr, "Fail: " + e);
// continue;
// }
//
// logCrawler(mbeanName, attr, "process");
// processBeanValue(
// mbeanName.getDomain(),
// new LinkedList<String>(),
// attr.getName(),
// attr.getType(),
// attr.getDescription(),
// value
// );
// }
// }
try {
info = beanCon.getMBeanInfo(mbeanName);
} catch (IOException e) {
logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
return;
} catch (JMException e) {
logCrawler(mbeanName.toString(), "getMbeanInfo Fail: " + e);
return;
}
MBeanAttributeInfo[] attrInfos = info.getAttributes();
for (int idx = 0; idx < attrInfos.length; ++idx) {
MBeanAttributeInfo attr = attrInfos[idx];
if (!attr.isReadable()) {
logCrawler(mbeanName, attr, "not readable");
continue;
}
Object value;
try {
value = beanCon.getAttribute(mbeanName, attr.getName());
} catch (Exception e) {
logCrawler(mbeanName, attr, "Fail: " + e);
continue;
}
logCrawler(mbeanName, attr, "process");
processBeanValue(
mbeanName.getDomain(),
new LinkedList<String>(),
attr.getName(),
attr.getType(),
attr.getDescription(),
value
);
}
}
private void processBeanValue(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
if (value == null) {
logCrawler("Value Null : "+domain + attrName, "null");
} else if (value instanceof Number || value instanceof String || value instanceof Boolean) {
logCrawler(domain + attrName, value.toString());
this.receiver.recordBean(domain, attrKeys, attrName, attrType, attrDescription,value);
// JSON 결과 생성
} else if (value instanceof CompositeData) {
CompositeData composite = (CompositeData)value;
CompositeType type = composite.getCompositeType();
attrKeys = new LinkedList<String>();
attrKeys.add(attrName);
for (String key : type.keySet()) {
String typ = type.getType(key).getTypeName();
Object val = composite.get(key);
processBeanValue(domain, attrKeys, key, typ, type.getDescription(), val);
}
} else if (value instanceof TabularData) {
TabularData tds = (TabularData)value;
TabularType tt = tds.getTabularType();
CompositeType type = tt.getRowType();
Set<String> valueKeys = new TreeSet<String>(type.keySet());
LinkedList<String> extendedAttrKeys = new LinkedList<String>(attrKeys);
extendedAttrKeys.add(attrName);
for (Object valu : tds.values()) {
if (valu instanceof CompositeData) {
CompositeData composite = (CompositeData) valu;
for(String valueIdx : valueKeys) {
LinkedList<String> attrNames = extendedAttrKeys;
String typ = type.getType(valueIdx).getTypeName();
String name = valueIdx;
if (valueIdx.toLowerCase().equals("value")) {
// Skip appending 'value' to the name
attrNames = attrKeys;
name = attrName;
}
processBeanValue(
domain,
attrNames,
name,
typ,
type.getDescription(),
composite.get(valueIdx));
}
} else {
logCrawler(domain, "not a correct tabulardata format");
}
}
} else if (value.getClass().isArray()) {
logCrawler(domain, "arrays are unsupported");
} else {
logCrawler(domain, attrType + "is not exported");
}
}
// private void processBeanValue(String domain, LinkedList<String> attrKeys, String attrName, String attrType, String attrDescription, Object value) {
// if (value == null) {
// logCrawler("Value Null : "+domain + attrName, "null");
// } else if (value instanceof Number || value instanceof String || value instanceof Boolean) {
// logCrawler(domain + attrName, value.toString());
// this.receiver.recordBean(domain, attrKeys, attrName, attrType, attrDescription,value);
// // JSON 결과 생성
// } else if (value instanceof CompositeData) {
// CompositeData composite = (CompositeData)value;
// CompositeType type = composite.getCompositeType();
// attrKeys = new LinkedList<String>();
// attrKeys.add(attrName);
//
// for (String key : type.keySet()) {
// String typ = type.getType(key).getTypeName();
// Object val = composite.get(key);
//
// processBeanValue(domain, attrKeys, key, typ, type.getDescription(), val);
// }
// } else if (value instanceof TabularData) {
// TabularData tds = (TabularData)value;
// TabularType tt = tds.getTabularType();
//
// CompositeType type = tt.getRowType();
// Set<String> valueKeys = new TreeSet<String>(type.keySet());
//
// LinkedList<String> extendedAttrKeys = new LinkedList<String>(attrKeys);
// extendedAttrKeys.add(attrName);
//
// for (Object valu : tds.values()) {
// if (valu instanceof CompositeData) {
// CompositeData composite = (CompositeData) valu;
//
// for(String valueIdx : valueKeys) {
// LinkedList<String> attrNames = extendedAttrKeys;
// String typ = type.getType(valueIdx).getTypeName();
// String name = valueIdx;
// if (valueIdx.toLowerCase().equals("value")) {
// // Skip appending 'value' to the name
// attrNames = attrKeys;
// name = attrName;
// }
// processBeanValue(
// domain,
// attrNames,
// name,
// typ,
// type.getDescription(),
// composite.get(valueIdx));
// }
// } else {
// logCrawler(domain, "not a correct tabulardata format");
// }
// }
// } else if (value.getClass().isArray()) {
// logCrawler(domain, "arrays are unsupported");
// } else {
// logCrawler(domain, attrType + "is not exported");
// }
// }
private Map<String, Object> getAuthority() {
@ -290,42 +337,4 @@ public class JmxCrawler extends Crawler{
logCrawler(mbeanName + "'_'" + attr.getName(), msg);
}
private static class StdoutWriter implements MBeanReceiver {
public void recordBean(
String domain,
LinkedList<String> attrKeys,
String attrName,
String attrType,
String attrDescription,
Object value) {
System.out.println(domain +
" " + attrKeys + " " +
" " + attrName + " " +
" " + attrDescription + " " +
" " + value);
}
}
// public static void main(String[] args) throws Exception {
// JmxCrawler jmxCrawler = new JmxCrawler(
//// "service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi",
// "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi",
//// "",
// "",
// "",
// false,
// new StdoutWriter()
// );
//
// jmxCrawler.doCrawler();
// JmxCrawler d = new JmxCrawler();
// Map<String, Object> m = new HashMap<>();
// m.put("ip","service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi");
//
// Object r = d.getInternal(m);
// System.out.println("r = " + r);
// }
}

97
src/test/java/com/loafle/AppTest.java
View File

@ -2,13 +2,18 @@
package com.loafle;
import static org.junit.Assert.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.loafle.overflow.crawler.Crawler;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.crawler.jmx.JmxCrawler;
import com.loafle.overflow.crawler.result.OFResultSet;
import org.junit.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AppTest {
@Test
@ -18,71 +23,43 @@ public class AppTest {
@Test
public void TestDoCrawler() throws Exception {
// JmxCrawler jmxCrawler = new JmxCrawler(
//// "service:jmx:rmi:///jndi/rmi://192.168.1.207:7199/jmxrmi",
// "service:jmx:rmi:///jndi/rmi://192.168.1.103:9840/jmxrmi",
//// "",
// "",
// "",
// false,
// new JmxCrawler.StdoutWriter()
// );
//
// jmxCrawler.doCrawler();
Config config = new Config();
ClassLoader classLoader = getClass().getClassLoader();
String path = classLoader.getResource("config/example.json").getFile();
config.setId("SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734");
Connection cn = new Connection();
cn.setIp("192.168.1.103");
cn.setPort("9840");
cn.setSsl(false);
cn.setPortType("tcp");
Target t = new Target();
t.setConnection(cn);
Schedule s = new Schedule();
s.setInterval("10");
com.loafle.overflow.crawler.config.Crawler cc = new com.loafle.overflow.crawler.config.Crawler();
cc.setName("jmx_crawler");
cc.setContainer("java_crawler");
List<Item> ls = new ArrayList<>();
List<String> metrics = new ArrayList<>();
List<Query> queries = new ArrayList<>();
List<String> keys = new ArrayList<>();
keys.add("ProcessCpuLoad");
keys.add("ProcessCpuTime");
keys.add("SystemCpuLoad");
Query query = new Query();
query.setKeys(keys);
query.setQuery("java.lang:type=OperatingSystem");
queries.add(query);
metrics.add("cpu.system");
metrics.add("cpu.user");
metrics.add("cpu.system_children");
metrics.add("cpu.user_children");
Item it = new Item();
it.setMetrics(metrics);
it.setQueries(queries);
ls.add(it);
config.setTarget(t);
config.setSchedule(s);
config.setCrawler(cc);
config.setItems(ls);
ObjectMapper mapper = new ObjectMapper();
Config c = mapper.readValue(new File(path),Config.class);
JmxCrawler cr = new JmxCrawler();
List<OFResultSet> result = (List<OFResultSet>) cr.getInternal(c);
Object o = cr.getInternal(config);
print("",result);
System.out.println("o = " + o);
}
private void print(String s, List<OFResultSet> mm) {
System.out.println(s);
Map<String,String> m = new HashMap<>();
for(OFResultSet r : mm) {
m.putAll(r.getData());
}
for (Map.Entry<String, String> item : m.entrySet()) {
System.out.println("key=" + item.getKey() + " ||||||||||| value=" + item.getValue());
}
}
@Test
public void testParseObject () {
String s = "org.apache.catalina.mbeans.ConnectorMBean[Catalina:type=Connector,port=8080]";
int start = s.indexOf("[");
int end = s.lastIndexOf("]");
s = s.substring(start + 1,end);
assertEquals("Catalina:type=Connector,port=8080",s);
}
}

44
src/test/resources/config/example.json Normal file
View File

@ -0,0 +1,44 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.103",
"port" : "9840",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"jmx",
"container":"java_proxy"
},
"items" : [
{
"keys" : [
{"metric" : "net.connector[$0].localPort", "key" : "localPort"},
{"metric" : "net.connector[$0].port", "key" : "port"},
{"metric" : "net.connector[$0].protocol", "key" : "protocol"}
],
"queryInfo" : {
"query": "*Catalina:type=Connector,*",
"extend" : {
"aliases" : [
{
"key": "port",
"index":0
}
]
}
},
"mappingInfo" : {
"arrayColumns" : [ "portName" ]
}
}
]
}