overflow_cloud/central
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ing

Browse Source
This commit is contained in:
crusader 2018-04-24 20:51:41 +09:00
parent 0eda0683de
commit 2b4aeea3e6
7 changed files with 317 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@ -179,12 +179,6 @@
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.loafle.overflow</groupId>
<artifactId>crawler</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.loafle.overflow</groupId>
<artifactId>commons-java</artifactId>

4
src/main/java/com/loafle/overflow/central/module/generator/service/GenerateUtil.java
View File

@ -1,12 +1,12 @@
package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.core.exception.OverflowException;
import com.loafle.overflow.crawler.config.Crawler;
import com.loafle.overflow.crawler.config.Keys;
import com.loafle.overflow.model.meta.MetaCrawler;
import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.meta.type.MetaCrawlerEnum;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Crawler;
import com.loafle.overflow.model.sensorconfig.Keys;
import com.loafle.overflow.service.central.meta.MetaSensorItemKeyService;
import org.springframework.beans.factory.annotation.Autowired;

34
src/main/java/com/loafle/overflow/central/module/generator/service/InfraHostGenerator.java
View File

@ -2,10 +2,7 @@ package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.central.commons.utils.StringConvertor;
import com.loafle.overflow.core.exception.OverflowException;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.crawler.config.Connection;
import com.loafle.overflow.crawler.config.Crawler;
import com.loafle.overflow.crawler.config.Schedule;
import com.loafle.overflow.core.type.PortType;
import com.loafle.overflow.model.auth.AuthCrawler;
import com.loafle.overflow.model.infra.Infra;
import com.loafle.overflow.model.infra.InfraHost;
@ -13,6 +10,11 @@ import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.meta.type.MetaCrawlerEnum;
import com.loafle.overflow.model.sensor.Sensor;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Connection;
import com.loafle.overflow.model.sensorconfig.Crawler;
import com.loafle.overflow.model.sensorconfig.Schedule;
import com.loafle.overflow.model.sensorconfig.SensorConfig;
import com.loafle.overflow.model.sensorconfig.Target;
import com.loafle.overflow.service.central.auth.AuthCrawlerService;
import org.codehaus.jackson.map.ObjectMapper;
@ -43,36 +45,36 @@ public class InfraHostGenerator {
InfraHost infraHost = (InfraHost) infra;
Config config = new Config();
config.setId(String.valueOf(dbSensor.getId()));
SensorConfig sensorConfig = new SensorConfig();
sensorConfig.setConfigID(String.valueOf(dbSensor.getId()));
com.loafle.overflow.crawler.config.Target target = this.createTarget(infraHost, dbSensor);
Target target = this.createTarget(infraHost, dbSensor);
if (target == null) {
return null;
}
config.setTarget(target);
sensorConfig.setTarget(target);
// FIXME: Interval
Schedule schedule = new Schedule();
schedule.setInterval("5");
config.setSchedule(schedule);
sensorConfig.setSchedule(schedule);
Crawler crawler = this.generateUtil.getCrawler(dbSensor.getCrawler());
config.setCrawler(crawler);
sensorConfig.setCrawler(crawler);
Map<Integer, MetaSensorItemKey> keyMap = this.generateUtil.initMappingMap(dbSensor.getCrawler());
if (dbSensor.getCrawler().getId() == MetaCrawlerEnum.WMI_CRAWLER.getValue()) {
this.infraHostWMIGenerator.process(sensorItems, keyMap, dbSensor, config);
this.infraHostWMIGenerator.process(sensorItems, keyMap, dbSensor, sensorConfig);
}
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.writeValueAsString(config);
return objectMapper.writeValueAsString(sensorConfig);
}
private com.loafle.overflow.crawler.config.Target createTarget(InfraHost infraHost, Sensor dbSensor)
private Target createTarget(InfraHost infraHost, Sensor dbSensor)
throws Exception {
AuthCrawler authCrawler = this.authCrawlerService.readAuth(dbSensor.getCrawler(), dbSensor.getTarget());
@ -81,7 +83,7 @@ public class InfraHostGenerator {
return null;
}
com.loafle.overflow.crawler.config.Target target = new com.loafle.overflow.crawler.config.Target();
Target target = new Target();
Connection connection = new Connection();
connection.setIp(StringConvertor.intToIp(infraHost.getIp()));
@ -91,8 +93,8 @@ public class InfraHostGenerator {
});
if (dbSensor.getCrawler().getId() == MetaCrawlerEnum.WMI_CRAWLER.getValue()) {
connection.setPort("135");
connection.setPortType("tcp");
connection.setPort(135);
connection.setPortType(PortType.TCP);
connection.setSsl(false);
target.setConnection(connection);

67
src/main/java/com/loafle/overflow/central/module/generator/service/InfraHostWMIGenerator.java
View File

@ -1,9 +1,13 @@
package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.sensor.Sensor;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Item;
import com.loafle.overflow.model.sensorconfig.Keys;
import com.loafle.overflow.model.sensorconfig.MappingInfo;
import com.loafle.overflow.model.sensorconfig.QueryInfo;
import com.loafle.overflow.model.sensorconfig.SensorConfig;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
@ -25,20 +29,21 @@ public class InfraHostWMIGenerator {
@Autowired
private GenerateUtil generateUtil;
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor, Config config) throws Exception {
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor,
SensorConfig sensorConfig) throws Exception {
Map<String, List<MetaSensorItemKey>> metricMap = null;
metricMap = this.generateUtil.sortItems(sensorItems, keyMap);
// MetaSensorItemKey itemKey = null;
//
// for(SensorItem sItem : sensorItems) {
// itemKey = keyMap.get(sItem.getItem().getId());
// if(metricMap.containsKey(itemKey.getFroms()) == false) {
// metricMap.put(itemKey.getFroms(), new ArrayList<>());
// }
// metricMap.get(itemKey.getFroms()).add(itemKey);
// }
// MetaSensorItemKey itemKey = null;
//
// for(SensorItem sItem : sensorItems) {
// itemKey = keyMap.get(sItem.getItem().getId());
// if(metricMap.containsKey(itemKey.getFroms()) == false) {
// metricMap.put(itemKey.getFroms(), new ArrayList<>());
// }
// metricMap.get(itemKey.getFroms()).add(itemKey);
// }
List<Item> itemList = new ArrayList<>();
Item item = null;
@ -51,44 +56,43 @@ public class InfraHostWMIGenerator {
extendMap.put("nameSpace", "root/cimv2");
extendMap.put("wmicPath", "/home/snoop/temp/wmic");
List<Keys> keysList = null;
MetaSensorItemKey tempItemKey = null;
Keys keys = null;
for(Map.Entry<String, List<MetaSensorItemKey>> elems : metricMap.entrySet()) {
for (Map.Entry<String, List<MetaSensorItemKey>> elems : metricMap.entrySet()) {
keysList = new ArrayList<>();
item = new Item();
stringBuffer.setLength(0);
stringBuffer.append("SELECT ");
for(int indexI = 0; indexI < elems.getValue().size(); ++indexI) {
for (int indexI = 0; indexI < elems.getValue().size(); ++indexI) {
tempItemKey = elems.getValue().get(indexI);
stringBuffer.append(tempItemKey.getKey());
if(indexI + 1 < elems.getValue().size()) {
if (indexI + 1 < elems.getValue().size()) {
stringBuffer.append(", ");
}
// keys = new Keys();
// keys.setKey(tempItemKey.getKey());
// keys.setMetric(tempItemKey.getItem().getKey());
// keys = new Keys();
// keys.setKey(tempItemKey.getKey());
// keys.setMetric(tempItemKey.getItem().getKey());
keysList.add(this.generateUtil.createKeys(tempItemKey));
}
List<String> arrayColumns = null;
String json = tempItemKey.getOption();
if(json != null && json.length() > 0) {
HashMap<String,Object> optionMap;
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String,Object>>() {});
if (json != null && json.length() > 0) {
HashMap<String, Object> optionMap;
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String, Object>>() {
});
Object obj = null;
obj = optionMap.get("appends");
if(obj != null) {
List<String> appendsList = (List<String>)obj;
for(String append : appendsList) {
if (obj != null) {
List<String> appendsList = (List<String>) obj;
for (String append : appendsList) {
stringBuffer.append(", ");
stringBuffer.append(append);
}
@ -98,19 +102,18 @@ public class InfraHostWMIGenerator {
stringBuffer.append(elems.getKey());
obj = optionMap.get("where");
if(obj != null) {
String where = (String)obj;
if (obj != null) {
String where = (String) obj;
stringBuffer.append(" WHERE ");
stringBuffer.append(where);
}
obj = optionMap.get("arrayColumns");
if(obj != null) {
arrayColumns = (List<String>)obj;
if (obj != null) {
arrayColumns = (List<String>) obj;
}
}
else {
} else {
stringBuffer.append(" FROM ");
stringBuffer.append(elems.getKey());
}
@ -131,7 +134,7 @@ public class InfraHostWMIGenerator {
}
config.setItems(itemList);
sensorConfig.setItems(itemList);
}
}

31
src/main/java/com/loafle/overflow/central/module/generator/service/InfraServiceGenerator.java
View File

@ -1,7 +1,7 @@
package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.central.commons.utils.StringConvertor;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.core.type.PortType;
import com.loafle.overflow.model.auth.AuthCrawler;
import com.loafle.overflow.model.infra.Infra;
import com.loafle.overflow.model.infra.InfraService;
@ -9,6 +9,11 @@ import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.meta.type.MetaCrawlerEnum;
import com.loafle.overflow.model.sensor.Sensor;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Connection;
import com.loafle.overflow.model.sensorconfig.Crawler;
import com.loafle.overflow.model.sensorconfig.Schedule;
import com.loafle.overflow.model.sensorconfig.SensorConfig;
import com.loafle.overflow.model.sensorconfig.Target;
import com.loafle.overflow.service.central.auth.AuthCrawlerService;
import org.codehaus.jackson.map.ObjectMapper;
@ -42,8 +47,8 @@ public class InfraServiceGenerator {
public String process(Sensor dbSensor, List<SensorItem> sensorItems, Infra infra) throws Exception {
com.loafle.overflow.model.infra.InfraService infraService = (com.loafle.overflow.model.infra.InfraService)infra;
Config config = new Config();
config.setId(String.valueOf(dbSensor.getId()));
SensorConfig sensorConfig = new SensorConfig();
sensorConfig.setConfigID(String.valueOf(dbSensor.getId()));
Target target = this.createTarget(infraService, dbSensor);
@ -52,26 +57,26 @@ public class InfraServiceGenerator {
}
config.setTarget(target);
sensorConfig.setTarget(target);
// FIXME: Interval
Schedule schedule = new Schedule();
schedule.setInterval("5");
config.setSchedule(schedule);
sensorConfig.setSchedule(schedule);
Crawler crawler = this.generateUtil.getCrawler(dbSensor.getCrawler());
config.setCrawler(crawler);
sensorConfig.setCrawler(crawler);
Map<Integer, MetaSensorItemKey> keyMap = this.generateUtil.initMappingMap(dbSensor.getCrawler());
if(dbSensor.getCrawler().getId() == MetaCrawlerEnum.MYSQL_CRAWLER.getValue()) {
this.infraServiceMysqlGenerator.process(sensorItems, keyMap, dbSensor, config);
this.infraServiceMysqlGenerator.process(sensorItems, keyMap, dbSensor, sensorConfig);
} else if (dbSensor.getCrawler().getId() == MetaCrawlerEnum.JMX_CRAWLER.getValue()) {
this.infraServiceJMXGenerator.process(sensorItems, keyMap, dbSensor, config);
this.infraServiceJMXGenerator.process(sensorItems, keyMap, dbSensor, sensorConfig);
}
ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.writeValueAsString(config);
return objectMapper.writeValueAsString(sensorConfig);
}
private Target createTarget(InfraService infraService, Sensor sensor) throws Exception {
@ -82,11 +87,11 @@ public class InfraServiceGenerator {
return null;
}
Target target = new com.loafle.overflow.crawler.config.Target();
Target target = new Target();
Connection connection = new Connection();
connection.setIp(StringConvertor.intToIp(infraService.getHost().getIp()));
connection.setPort(String.valueOf(infraService.getPort()));
connection.setPortType(infraService.getPortType());
connection.setPort(infraService.getPort());
connection.setPortType(PortType.valueOf(infraService.getPortType()));
connection.setSsl(infraService.isTlsType());
target.setConnection(connection);
@ -103,7 +108,7 @@ public class InfraServiceGenerator {
} else if (sensor.getCrawler().getId() == MetaCrawlerEnum.JMX_CRAWLER.getValue()) {
auth.put("id", optionMap.get("ID")); // FIXME: Auth Info
auth.put("pw", optionMap.get("PassWord")); // FIXME: Auth Info
connection.setPort("9840");
connection.setPort(9840);
}
target.setAuth(auth);

19
src/main/java/com/loafle/overflow/central/module/generator/service/InfraServiceJMXGenerator.java
View File

@ -1,9 +1,13 @@
package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.sensor.Sensor;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Item;
import com.loafle.overflow.model.sensorconfig.Keys;
import com.loafle.overflow.model.sensorconfig.MappingInfo;
import com.loafle.overflow.model.sensorconfig.QueryInfo;
import com.loafle.overflow.model.sensorconfig.SensorConfig;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
@ -24,7 +28,8 @@ public class InfraServiceJMXGenerator {
@Autowired
private GenerateUtil generateUtil;
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor, Config config) throws Exception {
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor,
SensorConfig sensorConfig) throws Exception {
Map<String, List<MetaSensorItemKey>> metricMap = null;
metricMap = this.generateUtil.sortItems(sensorItems, keyMap);
@ -66,18 +71,19 @@ public class InfraServiceJMXGenerator {
if (json != null && json.length() > 0) {
HashMap<String, Object> optionMap;
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String,Object>>() {});
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String, Object>>() {
});
Object obj = null;
obj = optionMap.get("aliases");
if (obj != null) {
aliases = (List<String>)obj;
aliases = (List<String>) obj;
}
obj = optionMap.get("arrayColumns");
if (obj != null) {
arrayCol = (List<String>)obj;
arrayCol = (List<String>) obj;
}
}
@ -94,8 +100,7 @@ public class InfraServiceJMXGenerator {
item.setKeys(keysList);
itemList.add(item);
}
config.setItems(itemList);
sensorConfig.setItems(itemList);
}
}

124
src/main/java/com/loafle/overflow/central/module/generator/service/InfraServiceMysqlGenerator.java
View File

@ -1,10 +1,14 @@
package com.loafle.overflow.central.module.generator.service;
import com.loafle.overflow.core.exception.OverflowException;
import com.loafle.overflow.crawler.config.*;
import com.loafle.overflow.model.meta.MetaSensorItemKey;
import com.loafle.overflow.model.sensor.Sensor;
import com.loafle.overflow.model.sensor.SensorItem;
import com.loafle.overflow.model.sensorconfig.Item;
import com.loafle.overflow.model.sensorconfig.Keys;
import com.loafle.overflow.model.sensorconfig.MappingInfo;
import com.loafle.overflow.model.sensorconfig.QueryInfo;
import com.loafle.overflow.model.sensorconfig.SensorConfig;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
@ -28,17 +32,17 @@ public class InfraServiceMysqlGenerator {
@Autowired
private GenerateUtil generateUtil;
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor, Config config) throws Exception {
public void process(List<SensorItem> sensorItems, Map<Integer, MetaSensorItemKey> keyMap, Sensor dbSensor,
SensorConfig sensorConfig) throws Exception {
// List<Keys> keysList = new ArrayList<>();
// for(SensorItem sItem : sensorItems) {
// keys = new Keys();
// keys.setMetric(sItem.getItem().getKey());
// keys.setKey(KeyMap.get(sItem.getItem().getId()).getKey());
// keysList.add(keys);
// }
// item.setKeys(keysList);
// List<Keys> keysList = new ArrayList<>();
// for(SensorItem sItem : sensorItems) {
// keys = new Keys();
// keys.setMetric(sItem.getItem().getKey());
// keys.setKey(KeyMap.get(sItem.getItem().getId()).getKey());
// keysList.add(keys);
// }
// item.setKeys(keysList);
Map<String, List<MetaSensorItemKey>> metricMap = null;
metricMap = this.generateUtil.sortItems(sensorItems, keyMap);
@ -52,7 +56,7 @@ public class InfraServiceMysqlGenerator {
Keys keys = null;
Item item = null;
StringBuffer stringBuffer = new StringBuffer();
for(Map.Entry<String, List<MetaSensorItemKey>> elems : metricMap.entrySet()) {
for (Map.Entry<String, List<MetaSensorItemKey>> elems : metricMap.entrySet()) {
keysList = new ArrayList<>();
item = new Item();
@ -61,38 +65,37 @@ public class InfraServiceMysqlGenerator {
stringBuffer.append(elems.getKey());
stringBuffer.append(" where ");
for(int indexI = 0; indexI < elems.getValue().size(); ++indexI) {
for (int indexI = 0; indexI < elems.getValue().size(); ++indexI) {
tempItemKey = elems.getValue().get(indexI);
stringBuffer.append("variable_name = '");
stringBuffer.append(tempItemKey.getKey());
stringBuffer.append("'");
if(indexI + 1 < elems.getValue().size()) {
if (indexI + 1 < elems.getValue().size()) {
stringBuffer.append(" or ");
}
keysList.add(this.generateUtil.createKeys(tempItemKey));
}
List<String> keyColumns = null;
String valueColumn = null;
String json = tempItemKey.getOption();
if(json != null && json.length() > 0) {
HashMap<String,Object> optionMap;
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String,Object>>() {});
if (json != null && json.length() > 0) {
HashMap<String, Object> optionMap;
optionMap = new ObjectMapper().readValue(json, new TypeReference<HashMap<String, Object>>() {
});
Object obj = null;
obj = optionMap.get("valueColumn");
if(obj != null) {
valueColumn = (String)obj;
if (obj != null) {
valueColumn = (String) obj;
}
obj = optionMap.get("keyColumns");
if(obj != null) {
keyColumns = (List<String>)obj;
if (obj != null) {
keyColumns = (List<String>) obj;
}
}
@ -112,45 +115,44 @@ public class InfraServiceMysqlGenerator {
}
config.setItems(itemList);
sensorConfig.setItems(itemList);
// ObjectMapper objectMapper = new ObjectMapper();
//
// return objectMapper.writeValueAsString(config);
// ObjectMapper objectMapper = new ObjectMapper();
//
// return objectMapper.writeValueAsString(config);
}
// public void setQueryAndMapping(MetaCrawler metaCrawler, List<Keys> keysList, QueryInfo queryInfo, MappingInfo mappingInfo) {
//
// switch (metaCrawler.getId()) {
// case 11: // mysql
// {
// String query = "show status where ";
//// queryInfo.setQuery("show status where ");
//
// Keys keys = null;
// for(int indexI = 0 ; indexI < keysList.size(); ++indexI) {
// keys = keysList.get(indexI);
// query += "variable_name = '";
// query += keys.getKey();
// query += "'";
// if(indexI + 1 < keysList.size()) {
// query += " or ";
// }
// }
//
// queryInfo.setQuery(query);
//
// mappingInfo.setParseDirection("row");
// mappingInfo.setValueColumn("Value");
//
// List<String> keyColumns = new ArrayList<>();
// keyColumns.add("Variable_name");
//
// mappingInfo.setKeyColumns(keyColumns);
// }
// break;
// }
//
// }
// public void setQueryAndMapping(MetaCrawler metaCrawler, List<Keys> keysList, QueryInfo queryInfo, MappingInfo mappingInfo) {
//
// switch (metaCrawler.getId()) {
// case 11: // mysql
// {
// String query = "show status where ";
//// queryInfo.setQuery("show status where ");
//
// Keys keys = null;
// for(int indexI = 0 ; indexI < keysList.size(); ++indexI) {
// keys = keysList.get(indexI);
// query += "variable_name = '";
// query += keys.getKey();
// query += "'";
// if(indexI + 1 < keysList.size()) {
// query += " or ";
// }
// }
//
// queryInfo.setQuery(query);
//
// mappingInfo.setParseDirection("row");
// mappingInfo.setValueColumn("Value");
//
// List<String> keyColumns = new ArrayList<>();
// keyColumns.add("Variable_name");
//
// mappingInfo.setKeyColumns(keyColumns);
// }
// break;
// }
//
// }
}