overflow_backup_2018_0455/sql_cralwer_java
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ResultSet 처리

Browse Source
This commit is contained in:
jackdaw@loafle.com 2017-04-25 19:47:42 +09:00
parent 7fb48cfc81
commit ed726f368b
7 changed files with 257 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -42,13 +42,11 @@
<scope>system</scope>
<systemPath>${basedir}/src/lib/ojdbc7.jar</systemPath>
</dependency>
<dependency>
<groupId>com.loafle.overflow</groupId>
<artifactId>crawler</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

87
src/main/java/com/loafle/overflow/crawler/sql/SQLCrawler.java
View File

@ -1,13 +1,15 @@
package com.loafle.overflow.crawler.sql;
import com.loafle.overflow.crawler.Crawler;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.crawler.config.Item;
import com.loafle.overflow.crawler.config.Query;
import com.loafle.overflow.crawler.result.*;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.sql.ResultSet;
import java.util.*;
/**
* Created by insanity on 17. 4. 11.
@ -27,66 +29,47 @@ public class SQLCrawler extends Crawler {
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
// conn = DriverManager.getConnection(
// (String)config.get("url"),
// (String)config.get("user"),
// (String)config.get("pw"));
String url = (String)config.getTarget().getAuth().get("url");
String id = (String)config.getTarget().getAuth().get("id");
String pw = (String)config.getTarget().getAuth().get("pw");
List<OFResultSet> resultSets = new ArrayList<>();
try {
conn = DriverManager.getConnection(url, id, pw);
stmt = conn.createStatement();
// rs = stmt.executeQuery((String)config.get("query"));
for (Item item : config.getItems()) {
ResultSetMetaData md = rs.getMetaData();
int cnt = md.getColumnCount();
Query query = item.getQuery();
OFResultSet resultSet = OFResultSetRow.newInstance(item);
Map<String,Integer> meta = resultSet.getMeta();
List<DBMetaInfo> metaList = new ArrayList<DBMetaInfo>();
rs = stmt.executeQuery((String)query.getQueryInfo().get("query"));
for (int i =0 ; i < cnt ; ++i) {
DBMetaInfo o = new DBMetaInfo();
o.setName(md.getColumnName(i+1));
metaList.add(o);
}
List<Map<String, String>> datas = new ArrayList<Map<String, String>>();
while(rs.next()){
Map<String, String> row = new HashMap<String, String>();
for (DBMetaInfo o : metaList) {
row.put(o.getName(), rs.getString(o.getName()));
while(rs.next()) {
List<String> row = new ArrayList<>(Arrays.asList(new String[meta.size()]));
for (Map.Entry<String, Integer> info : meta.entrySet()) {
String data = rs.getString(info.getKey());
data = data.trim();
row.set(info.getValue().intValue(),data);
}
resultSet.addRow(row);
}
datas.add(row);
}
return datas;
resultSets.add(resultSet);
rs.close();
}
return resultSets;
} catch (Exception e) {
e.printStackTrace();
new Exception(e.getMessage()).printStackTrace();
} finally {
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (conn != null) {try {conn.close();} catch (SQLException e) {e.printStackTrace();}}
if (rs != null) {try {rs.close();} catch (SQLException e) {e.printStackTrace();}}
if (stmt != null) {try {stmt.close();} catch (SQLException e) {e.printStackTrace();}}
}
return null;
}

71
src/test/java/com/loafle/overflow/AppTest.java
View File

@ -1,16 +1,21 @@
package com.loafle.overflow;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.crawler.result.OFResultSet;
import com.loafle.overflow.crawler.sql.SQLCrawler;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class AppTest {
// //MySQL
// //MySQL
// info.setUrl("jdbc:mysql://192.168.1.215:3306"); // ?useSSL=true&verifyServerCertificate=false
// info.setId("root");
// info.setPw("qwe123");
@ -34,23 +39,57 @@ public class AppTest {
// info.setPw("qwer5795QWER");
// info.setSSL(false);
// info.setQuery("select * from v$sysstat");
private Config getConfig(String path) throws IOException {
ClassLoader classLoader = getClass().getClassLoader();
String filePath = classLoader.getResource(path).getFile();
ObjectMapper mapper = new ObjectMapper();
Config c = mapper.readValue(new File(filePath),Config.class);
return c;
}
private void print(String s, List<OFResultSet> mm) {
System.out.println(s);
Map<String,String> m = new HashMap<>();
for(OFResultSet r : mm) {
m.putAll(r.getData());
}
for (Map.Entry<String, String> item : m.entrySet()) {
System.out.println("key=" + item.getKey() + " ||||||||||| value=" + item.getValue());
}
}
@Test
@Ignore
public void testSQL() {
// SQLCrawler sc = new SQLCrawler();
// Map config = new HashMap();
// config.put("url", "jdbc:mysql://192.168.1.215:3306");
// config.put("user", "root");
// config.put("pw", "qwe123");
// config.put("ssl", false);
// config.put("query", "show session status");
//
// Object result = sc.getInternal(config);
// for(Map<String, String> m : (List<Map<String,String>>)result) {
// for (Map.Entry<String, String> entry : m.entrySet()) {
// System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue());
// }
// }
public void testMYSQL() throws IOException {
Object result = new SQLCrawler().getInternal( getConfig("config/mysql.json"));
print("Mysql row", (List<OFResultSet>) result);
}
@Test
@Ignore
public void testPGSQL() throws IOException {
Object result = new SQLCrawler().getInternal(getConfig("config/pgsql.json"));
print("PGSql", (List<OFResultSet>) result);
}
@Test
@Ignore
public void testSQLServerConnection_Count() throws IOException {
Object result = new SQLCrawler().getInternal(getConfig("config/sqlserver_connection_count.json"));
print("MSSql Server", (List<OFResultSet>) result);
}
@Test
@Ignore
public void testSQLServerMultiKeyArray() throws IOException {
Object result = new SQLCrawler().getInternal(getConfig("config/sqlserver_multiple_key_array.json"));
print("MSSql Server MuoltiArray", (List< OFResultSet>) result);
}
}

44
src/test/resources/config/mysql.json Normal file
View File

@ -0,0 +1,44 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.104",
"port" : "6379",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
"url":"jdbc:mysql://192.168.1.215:3306",
"id":"root",
"pw":"qwe123"
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"mysql",
"container":"java_proxy"
},
"items" : [
{
"metrics" : [
"net.connection_count"
],
"query":
{
"queryInfo" : {
"query":"show status where `variable_name` = 'Connections'",
"parseDirection" : "row",
"valueColumn" : "Value",
"keyColumns" : [
"Variable_name"
]
},
"keys" : [
"Connections"
]
}
}
]
}

39
src/test/resources/config/pgsql.json Normal file
View File

@ -0,0 +1,39 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.107",
"port" : "5432",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
"url":"jdbc:postgresql://192.168.1.107:5432/postgres",
"id":"postgres",
"pw":"!@#$qwer1234"
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"pgsql",
"container":"java_proxy"
},
"items" : [
{
"metrics" : [
"net.connection_count"
],
"query": {
"queryInfo":{
"query" : "select count(pid) as connection_count from pg_catalog.pg_stat_activity where state <> 'idle'" ,
"parseDirection":"col"
},
"keys" : [
"connection_count"
]
}
}
]
}

39
src/test/resources/config/sqlserver_connection_count.json Normal file
View File

@ -0,0 +1,39 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.103",
"port" : "1433",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
"url":"jdbc:sqlserver://192.168.1.106:1433;",
"id":"sa",
"pw":"qwe123"
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"mssql",
"container":"java_proxy"
},
"items" : [
{
"metrics" : [
"net.connection_count"
],
"query": {
"queryInfo" : {
"query": "select count(session_id) as connection_count from sys.dm_exec_connections where session_id = @@SPID",
"parseDirection" : "col"
},
"keys" : [
"connection_count"
]
}
}
]
}

45
src/test/resources/config/sqlserver_multiple_key_array.json Normal file
View File

@ -0,0 +1,45 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.103",
"port" : "1433",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
"url":"jdbc:sqlserver://192.168.1.106:1433;",
"id":"sa",
"pw":"qwe123",
"query" : "select * from master.dbo.sysprocesses"
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"mssql",
"container":"java_proxy"
},
"items" : [
{
"metrics" : [
"object[$0].db[$1].datafile_size",
"object[$0].db[$1].logfile_size"
],
"query": {
"queryInfo" : {
"query": "select object_name,instance_name, counter_name, cntr_value from sys.dm_os_performance_counters where ( counter_name = 'Data File(s) Size (KB)' or counter_name = 'Log File(s) Size (KB)' ) AND object_name = 'SQLServer:Databases'",
"parseDirection" : "row",
"arrayColumns" : [ "object_name","instance_name"],
"keyColumns" : ["counter_name"],
"valueColumn" : "cntr_value"
},
"keys" : [
"Data File(s) Size (KB)",
"Log File(s) Size (KB)"
]
}
}
]
}