overflow_backup_2018_0455/grpc_java
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test init add remove get

Browse Source
This commit is contained in:
jackdaw@loafle.com 2017-04-13 19:20:17 +09:00
parent d28aa42640
commit cb97bc2da9
5 changed files with 234 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -69,7 +69,7 @@
<dependency>
<groupId>com.loafle.overflow</groupId>
<artifactId>crawler_mongo</artifactId>
<artifactId>crawler_mongodb</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>

53
src/main/java/com/loafle/overflow/Server.java
View File

@ -17,6 +17,7 @@ import java.io.IOException;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Logger;
@ -42,21 +43,20 @@ public class Server {
addDelegate(Crawlers.JMX.name(), new JmxCrawler());
}
private int addDelegate(String name, Crawler crawler) {
if(this.crawlerMap == null) {
this.crawlerMap = new HashMap<String, Crawler>();
}
if(this.crawlerMap.containsKey(name)) {
return 1;
}
public int addDelegate(String name, Crawler crawler) {
if(this.crawlerMap == null) {this.crawlerMap = new HashMap<String, Crawler>();}
this.crawlerMap.put(name, crawler);
return 0;
}
private void start() throws IOException {
static public byte[] convertObjectToBytes(Object o) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutput out = new ObjectOutputStream(bos);
out.writeObject(o);
return bos.toByteArray();
}
public void start() throws IOException {
/* The port on which the server should run */
int port = 50052;
server = ServerBuilder.forPort(port)
@ -76,7 +76,7 @@ public class Server {
});
}
private void stop() {
public void stop() {
if (server != null) {
server.shutdown();
}
@ -98,8 +98,9 @@ public class Server {
if(crawler != null) {
try {
Object obj = crawler.get(req.getId());
Output reply = Output.newBuilder()
.setData(null)
.setData(ByteString.copyFrom(convertObjectToBytes(obj)))
.setStartDate(333)
.setEndDate(333)
.build();
@ -116,13 +117,29 @@ public class Server {
@Override
public void init(InputArray reqs, StreamObserver<Output> responseObserver) {
List<Init> list = reqs.getInList();
Object obj = null;
for (Init in : list) {
Crawler crawler = crawlerMap.get(in.getName().name());
try {
obj = crawler.init(in.getPath());
} catch (Exception e) {
e.printStackTrace();
}
}
try {
Output reply = Output.newBuilder()
.setData(ByteString.copyFromUtf8("Init"))
.setData(ByteString.copyFrom(convertObjectToBytes(obj)))
.setStartDate(0)
.setEndDate(0)
.build();
responseObserver.onNext(reply);
responseObserver.onCompleted();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
@ -131,12 +148,8 @@ public class Server {
if(crawler != null) {
try {
Object obj = crawler.add(req.getId());
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutput out = new ObjectOutputStream(bos);
out.writeObject(obj);
Output reply = Output.newBuilder()
//.setData(ByteString.copyFrom(bos.toByteArray())) ///////////////////
.setData(ByteString.copyFrom(bos.toByteArray())) ///////////////////
.setData(ByteString.copyFrom(convertObjectToBytes(obj)))
.setStartDate(0)
.setEndDate(0)
.build();
@ -155,7 +168,7 @@ public class Server {
try {
Object obj = crawler.remove(req.getId());
Output reply = Output.newBuilder()
.setData(null) ///////////////////
.setData(ByteString.copyFrom(convertObjectToBytes(obj)))
.setStartDate(0)
.setEndDate(0)
.build();

12
src/main/proto/remote.proto
View File

@ -37,6 +37,7 @@ enum Crawlers {
MONGODB = 30;
}
// services
service Config {
rpc Add (Input) returns (Output) {}
rpc Remove (Input) returns (Output) {}
@ -47,16 +48,23 @@ service Data {
rpc Get (Input) returns (Output) {}
}
// request models
message InputArray {
repeated Input in = 1;
repeated Init in = 1;
}
message Init {
Crawlers name = 1;
string path = 2;
}
message Input {
Crawlers name = 1;
string id =2;
string path = 3;
}
// response models
message Output {
int64 startDate = 1;
int64 endDate = 2;

160
src/test/java/com/loafle/overflow/ServerTest.java Normal file
View File

@ -0,0 +1,160 @@
package com.loafle.overflow;
import com.loafle.overflow.crawler.Crawler;
import com.loafle.overflow.crawler.config.Config;
import com.loafle.overflow.rpc.*;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import junit.framework.Assert;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.util.logging.Logger;
import static org.junit.Assert.assertEquals;
/**
* Created by root on 17. 4. 13.
*/
public class ServerTest {
private static final Logger logger = Logger.getLogger(ServerTest.class.getName());
public class TestCrawlerImpl extends Crawler {
@Override
public Object getInternal(Config config) throws Exception {
return config.getId();
}
}
Server server;
@Before
public void setUp() throws Exception {
server = new Server();
server.addDelegate(Crawlers.REDIS.name(),new TestCrawlerImpl());
server.start();
}
@After
public void tearDown() throws Exception {
server.stop();
}
public DataGrpc.DataBlockingStub newDataClient(String ip, int port) {
return DataGrpc.newBlockingStub(newChannel(ip,port));
}
public ConfigGrpc.ConfigBlockingStub newConfigClient(String ip, int port) {
return ConfigGrpc.newBlockingStub(newChannel(ip,port));
}
public ManagedChannel newChannel(String ip, int port) {
return ManagedChannelBuilder.forAddress(ip,port).usePlaintext(true).build();
}
public Output init() {
ConfigGrpc.ConfigBlockingStub client = newConfigClient("localhost",50052);
ClassLoader classLoader = getClass().getClassLoader();
// path is test resources/config/example.json
Init input = Init.newBuilder().
setPath(classLoader.getResource("config").getFile()).
setName(Crawlers.REDIS).build();
InputArray arr = InputArray.newBuilder().addIn(input).build();
return client.init(arr);
}
@Test
public void TestInit() throws IOException, ClassNotFoundException {
Output out = init();
ByteArrayInputStream bis = new ByteArrayInputStream(out.getData().toByteArray());
ObjectInput in = new ObjectInputStream(bis);
Object o = in.readObject();
logger.info("!!!!!!!!!!!!1 " + o);
assertEquals(true,o);
}
@Test
public void TestAdd() throws Exception {
init();
ConfigGrpc.ConfigBlockingStub client = newConfigClient("localhost",50052);
ClassLoader classLoader = getClass().getClassLoader();
// path is test resources/config/example.json
Input input = Input.newBuilder().
setId("example.json").
setName(Crawlers.REDIS).build();
Output out = client.add(input);
ByteArrayInputStream bis = new ByteArrayInputStream(out.getData().toByteArray());
ObjectInput in = new ObjectInputStream(bis);
Object o = in.readObject();
logger.info("!!!!!!!!!!!!1 " + o);
assertEquals(true,o);
}
@Test
public void TestRemove() throws Exception {
init();
ConfigGrpc.ConfigBlockingStub client = newConfigClient("localhost",50052);
ClassLoader classLoader = getClass().getClassLoader();
// path is test resources/config/example.json
Input input = Input.newBuilder().
setId("example.json").
setName(Crawlers.REDIS).build();
Output out = client.remove(input);
ByteArrayInputStream bis = new ByteArrayInputStream(out.getData().toByteArray());
ObjectInput in = new ObjectInputStream(bis);
Object o = in.readObject();
logger.info("!!!!!!!!!!!!1 " + o);
assertEquals(true,o);
}
@Test
public void TestGet() throws Exception {
init();
DataGrpc.DataBlockingStub client = newDataClient("localhost",50052);
Input input = Input.newBuilder().
setId("example.json").
setName(Crawlers.REDIS).build();
Output out = client.get(input);
ByteArrayInputStream bis = new ByteArrayInputStream(out.getData().toByteArray());
ObjectInput in = new ObjectInputStream(bis);
Object o = in.readObject();
logger.info("!!!!!!!!!!!!!!!! " + o);
assertEquals("SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",o);
}
}

23
src/test/resources/config/example.json Normal file
View File

@ -0,0 +1,23 @@
{
"id" : "SOEJWEOJWOEJOSDJFOASDJFOSDFO2903870928734",
"target" : {
"connection" : {
"ip" : "192.168.1.104",
"port" : "6379",
"ssl" : false,
"portType" : "tcp"
},
"auth" : {
}
},
"schedule" : {
"interval" : "10"
},
"crawler" : {
"name":"redis_protocol_crawler",
"container":"network_crawler"
},
"items" : [
]
}