overflow_backup_2018_0455/mongodb_crawler_java
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new config

Browse Source 
resultset
This commit is contained in:
snoop 2017-04-25 19:51:39 +09:00
parent 9f2b8adec9
commit 142d897ab0
3 changed files with 66 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/main/java/com/loafle/overflow/crawler/mongo/MongoCrawler.java
View File

@ -4,12 +4,15 @@ import com.loafle.overflow.crawler.Crawler;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.crawler.config.Item;
import com.loafle.overflow.crawler.config.Query;
import com.loafle.overflow.crawler.result.OFResultSet;
import com.loafle.overflow.crawler.result.OFResultSetCol;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoDatabase;
import org.bson.Document;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -18,17 +21,17 @@ import java.util.Map;
public class MongoCrawler extends Crawler {
public Map<String, Object> collectMetric(Config c) throws Exception {
public Object collectMetric(Config c) throws Exception {
MongoClient mongoClient = null;
Map<String, Object> returnMap = new HashMap<String, Object>();
// Map<String, Object> returnMap = new HashMap<String, Object>();
String dataBaseName = "admin";
String statusCommand = "serverStatus";
String dataBaseName = null;
String statusCommand = null;
String targetIP = c.getTarget().getConnection().getIp();
short targetPort = Short.parseShort(c.getTarget().getConnection().getPort());
List<OFResultSet> ofResultSets = null;
try {
mongoClient = new MongoClient(new ServerAddress(targetIP, targetPort),
@ -38,23 +41,43 @@ public class MongoCrawler extends Crawler {
.socketTimeout(1000).build()
);
MongoDatabase database = mongoClient.getDatabase(dataBaseName);
Document serverStatus = database.runCommand(new Document(statusCommand, 1));
Map<String,Object> resultMap = new HashMap<String, Object>();
for (Map.Entry<String, Object> set : serverStatus.entrySet()){
resultMap.put(set.getKey(), set.getValue());
}
ofResultSets = new ArrayList<>();
OFResultSet ofResultSet = null;
List<Item> items = c.getItems();
List<String> row = null;
for (Item item : items) {
List<String> metrics = item.getMetrics();
for (Query query : item.getQueries()) {
Map<String, String> re = (Map<String, String>) resultMap.get(query.getQuery());
for (int index =0 ; index < query.getKeys().size() ; ++index) {
returnMap.put(metrics.get(index),re.get(query.getKeys().get(index)));
}
row = new ArrayList<>();
ofResultSet = new OFResultSetCol(item);
Query query = item.getQuery();
dataBaseName = (String)item.getQuery().getQueryInfo().get("dataBaseName");
statusCommand = (String)item.getQuery().getQueryInfo().get("statusCommand");
MongoDatabase database = mongoClient.getDatabase(dataBaseName);
Document serverStatus = database.runCommand(new Document(statusCommand, 1));
Map<String, String> re = (Map<String, String>)serverStatus.get(query.getQueryInfo().get("query"));
// Map<String,Object> resultMap = new HashMap<String, Object>();
// for (Map.Entry<String, Object> set : serverStatus.entrySet()){
// resultMap.put(set.getKey(), set.getValue());
// }
// Map<String, String> re = (Map<String, String>) resultMap.get(query.getQueryInfo().get("query"));
for (int index =0 ; index < query.getKeys().size() ; ++index) {
row.add(String.valueOf(re.get(query.getKeys().get(index))));
}
ofResultSet.addRow(row);
ofResultSets.add(ofResultSet);
}
}catch (Exception e) {
throw e;
@ -63,7 +86,7 @@ public class MongoCrawler extends Crawler {
mongoClient.close();
}
}
return returnMap;
return ofResultSets;
}
@Override

17
src/test/java/com/loafle/overflow/crawler/MongoCrawlerTest.java
View File

@ -5,6 +5,7 @@ import static org.junit.Assert.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.crawler.mongo.MongoCrawler;
import com.loafle.overflow.crawler.result.OFResultSet;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
@ -15,6 +16,7 @@ import org.junit.Test;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class MongoCrawlerTest {
@ -32,7 +34,18 @@ public class MongoCrawlerTest {
Config c = mapper.readValue(new File(path),Config.class);
MongoCrawler cr = new MongoCrawler();
Map<String,Object> m = cr.collectMetric(c);
assertEquals(6, m.size());
List<OFResultSet> rl = (List<OFResultSet>)cr.collectMetric(c);
for(int indexI = 0 ; indexI < rl.size(); ++indexI) {
Map<String, String> m = rl.get(indexI).getData();
for( String key : m.keySet()) {
System.out.print("key : " + key);
System.out.print("|| value : ");
System.out.println(m.get(key));
}
}
// assertEquals(6, m.size());
}
}

11
src/test/resources/config/example.json
View File

@ -28,9 +28,14 @@
"memory.usage.mapped",
"memory.usage.mappedWithJournal"
],
"queries":[
"query":
{
"query" : "mem" ,
"queryInfo": {
"query":"mem",
"dataBaseName":"admin",
"statusCommand":"serverStatus",
"parseDirection" : "col"
},
"keys" : [
"bits",
"resident",
@ -40,7 +45,7 @@
"mappedWithJournal"
]
}
]
}
]
}