ChronosXYZ/Influence-P2P
1
0
Fork 0
mirror of https://github.com/ChronosX88/Influence-P2P.git synced 2024-11-24 08:12:19 +00:00
Code Issues Projects Releases Wiki Activity

[WIP] Changed storage backend to MapDB

Browse Source
This commit is contained in:
ChronosX88 2019-04-04 18:35:36 +04:00
parent 046abcb465
commit ad357c779e
7 changed files with 223 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/build.gradle
View File

@ -40,9 +40,9 @@ dependencies {
implementation 'androidx.constraintlayout:constraintlayout:2.0.0-alpha3'
implementation "androidx.room:room-runtime:2.1.0-alpha04"
annotationProcessor "androidx.room:room-compiler:2.1.0-alpha04"
implementation('net.tomp2p:tomp2p-all:5.0-Beta8') //{
//exclude group: 'org.mapdb', module: 'mapdb'
//}
implementation('net.tomp2p:tomp2p-all:5.0-Beta8') {
exclude group: 'net.tomp2p', module: 'tomp2p-storage'
}
implementation 'org.slf4j:slf4j-log4j12:1.7.26'
implementation group: 'com.h2database', name: 'h2-mvstore', version: '1.4.197'
implementation 'com.google.android.material:material:1.1.0-alpha04'
@ -50,4 +50,5 @@ dependencies {
implementation 'com.google.code.gson:gson:2.8.5'
implementation group: 'org.springframework.security', name: 'spring-security-crypto', version: '3.1.0.RELEASE'
implementation 'de.hdodenhof:circleimageview:3.0.0'
implementation 'org.mapdb:mapdb:2.0-beta13'
}

2
app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java
View File

@ -17,7 +17,7 @@ public class Converter {
}
@TypeConverter
public static String fromArrayLisr(ArrayList<String> list) {
public static String fromArrayList(ArrayList<String> list) {
Gson gson = new Gson();
return gson.toJson(list);
}

194
app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerEx.java Normal file
View File

@ -0,0 +1,194 @@
package io.github.chronosx88.influence.helpers;
import android.util.Log;
import net.tomp2p.connection.SignatureFactory;
import net.tomp2p.peers.Number160;
import net.tomp2p.storage.AlternativeCompositeByteBuf;
import net.tomp2p.storage.Data;
import org.mapdb.DataIO;
import org.mapdb.Serializer;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.InvalidKeyException;
import java.security.KeyPair;
import java.security.SignatureException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
public class DataSerializerEx extends Serializer<Data> implements Serializable {
private static final long serialVersionUID = 1428836065493792295L;
//TODO: test the performance impact
private static final int MAX_SIZE = 10 * 1024;
private File path;
private SignatureFactory signatureFactory;
final private static KeyPairManager keyPairManager = new KeyPairManager();
public DataSerializerEx() { }
public DataSerializerEx(File path, SignatureFactory signatureFactory) {
this.path = path;
this.signatureFactory = signatureFactory;
}
@Override
public void serialize(DataOutput out, Data value) throws IOException {
if (value.length() > MAX_SIZE) {
// header, 1 means stored on disk in a file
out.writeByte(1);
serializeFile(out, value);
} else {
// header, 0 means stored on disk with MapDB
out.writeByte(0);
serializeMapDB(out, value);
}
}
private void serializeMapDB(DataOutput out, Data value) throws IOException {
KeyPair keyPair = keyPairManager.getKeyPair("mainKeyForSign");
value.sign(keyPair);
AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP);
// store data to disk
// header first
value.encodeHeader(acb, signatureFactory);
write(out, acb.nioBuffers());
acb.skipBytes(acb.writerIndex());
// next data - no need to copy to another buffer, just take the data
// from memory
write(out, value.toByteBuffers());
// rest
try {
value.encodeDone(acb, signatureFactory);
write(out, acb.nioBuffers());
} catch (InvalidKeyException e) {
throw new IOException(e);
} catch (SignatureException e) {
throw new IOException(e);
}
}
private void serializeFile(DataOutput out, Data value) throws IOException, FileNotFoundException {
Number160 hash = value.hash();
// store file name
out.write(hash.toByteArray());
// store as external file, create path
RandomAccessFile file = null;
FileChannel rwChannel = null;
try {
file = new RandomAccessFile(new File(path, hash.toString()), "rw");
rwChannel = file.getChannel();
AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP);
// store data to disk
// header first
value.encodeHeader(acb, signatureFactory);
rwChannel.write(acb.nioBuffers());
// next data - no need to copy to another buffer, just take the
// data from memory
rwChannel.write(value.toByteBuffers());
// rest
try {
value.encodeDone(acb, signatureFactory);
rwChannel.write(acb.nioBuffers());
} catch (InvalidKeyException e) {
throw new IOException(e);
} catch (SignatureException e) {
throw new IOException(e);
}
} finally {
if (rwChannel != null) {
rwChannel.close();
}
if (file != null) {
file.close();
}
}
}
private void write(DataOutput out, ByteBuffer[] nioBuffers) throws IOException {
final int length = nioBuffers.length;
for(int i=0;i < length; i++) {
int remaining = nioBuffers[i].remaining();
if(nioBuffers[i].hasArray()) {
out.write(nioBuffers[i].array(), nioBuffers[i].arrayOffset(), remaining);
} else {
byte[] me = new byte[remaining];
nioBuffers[i].get(me);
out.write(me);
}
}
}
@Override
public Data deserialize(DataInput in, int available) throws IOException {
int header = in.readByte();
if(header == 1) {
return deserializeFile(in);
} else if(header == 0) {
return deserializeMapDB(in);
} else {
throw new IOException("unexpected header: " + header);
}
}
private Data deserializeMapDB(DataInput in) throws IOException {
ByteBuf buf = Unpooled.buffer();
Data data = null;
while(data == null) {
buf.writeByte(in.readByte());
data = Data.decodeHeader(buf, signatureFactory);
}
int len = data.length();
byte me[] = new byte[len];
in.readFully(me);
buf = Unpooled.wrappedBuffer(me);
boolean retVal = data.decodeBuffer(buf);
if(!retVal) {
throw new IOException("data could not be read");
}
retVal = data.decodeDone(buf, signatureFactory);
if(!retVal) {
//throw new IOException("signature could not be read");
Log.e("DataSerializerEx", "# Signature could not be read!");
}
DataIO.DataInputByteArray di = (DataIO.DataInputByteArray) in;
di.setPos(di.internalByteArray().length);
return data;
}
private Data deserializeFile(DataInput in) throws IOException, FileNotFoundException {
byte[] me = new byte[Number160.BYTE_ARRAY_SIZE];
in.readFully(me);
Number160 hash = new Number160(me);
RandomAccessFile file = new RandomAccessFile(new File(path, hash.toString()), "r");
FileChannel inChannel = file.getChannel();
MappedByteBuffer buffer = inChannel.map(FileChannel.MapMode.READ_ONLY, 0, inChannel.size());
buffer.load();
ByteBuf buf = Unpooled.wrappedBuffer(buffer);
Data data = Data.decodeHeader(buf, signatureFactory);
data.decodeBuffer(buf);
data.decodeDone(buf, signatureFactory);
file.close();
return data;
}
@Override
public int fixedSize() {
return -1;
}
}

6
app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java
View File

@ -1,8 +1,11 @@
package io.github.chronosx88.influence.helpers;
import android.util.Log;
import net.tomp2p.dht.Storage;
public class JVMShutdownHook extends Thread {
Storage storage;
public JVMShutdownHook(Storage storage) {
@ -12,6 +15,9 @@ public class JVMShutdownHook extends Thread {
@Override
public void run() {
super.run();
Log.d("JVMShutdownHook", "# Closing storage...");
storage.close();
Log.d("JVMShutdownHook", "# Storage is closed");
}
}

27
app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java
View File

@ -1,5 +1,16 @@
package io.github.chronosx88.influence.helpers;
import net.tomp2p.connection.SignatureFactory;
import net.tomp2p.dht.Storage;
import net.tomp2p.peers.Number160;
import net.tomp2p.peers.Number320;
import net.tomp2p.peers.Number480;
import net.tomp2p.peers.Number640;
import net.tomp2p.storage.Data;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import java.io.File;
import java.security.PublicKey;
import java.util.ArrayList;
@ -15,18 +26,6 @@ import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentNavigableMap;
import net.tomp2p.connection.SignatureFactory;
import net.tomp2p.dht.Storage;
import net.tomp2p.peers.Number160;
import net.tomp2p.peers.Number320;
import net.tomp2p.peers.Number480;
import net.tomp2p.peers.Number640;
import net.tomp2p.storage.Data;
import net.tomp2p.storage.DataSerializer;
import org.mapdb.DB;
import org.mapdb.DBMaker;
public class StorageMapDB implements Storage {
// Core
final private NavigableMap<Number640, Data> dataMap;
@ -47,7 +46,7 @@ public class StorageMapDB implements Storage {
//for full control
public StorageMapDB(DB db, Number160 peerId, File path, SignatureFactory signatureFactory, int storageCheckIntervalMillis) {
this.db = db;
DataSerializer dataSerializer = new DataSerializer(path, signatureFactory);
DataSerializerEx dataSerializer = new DataSerializerEx(path, signatureFactory);
this.dataMap = db.createTreeMap("dataMap_" + peerId.toString()).valueSerializer(dataSerializer).makeOrGet();
this.timeoutMap = db.createTreeMap("timeoutMap_" + peerId.toString()).makeOrGet();
this.timeoutMapRev = db.createTreeMap("timeoutMapRev_" + peerId.toString()).makeOrGet();
@ -60,7 +59,7 @@ public class StorageMapDB implements Storage {
//set parameter to a reasonable default
public StorageMapDB(Number160 peerId, File path, SignatureFactory signatureFactory) {
this(DBMaker.newFileDB(new File(path, "tomp2p")).closeOnJvmShutdown().make(),
this(DBMaker.newFileDB(new File(path, "coreDB")).closeOnJvmShutdown().make(),
peerId, path, signatureFactory, 60 * 1000);
}

10
app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java
View File

@ -7,9 +7,10 @@ import android.util.Log;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import net.tomp2p.connection.RSASignatureFactory;
import net.tomp2p.connection.DSASignatureFactory;
import net.tomp2p.dht.PeerBuilderDHT;
import net.tomp2p.dht.PeerDHT;
import net.tomp2p.dht.StorageMemory;
import net.tomp2p.futures.FutureBootstrap;
import net.tomp2p.futures.FutureDiscover;
import net.tomp2p.nat.FutureRelayNAT;
@ -21,7 +22,6 @@ import net.tomp2p.peers.PeerAddress;
import net.tomp2p.relay.tcp.TCPRelayClientConfig;
import net.tomp2p.replication.AutoReplication;
import net.tomp2p.storage.Data;
import net.tomp2p.storage.StorageDisk;
import java.io.IOException;
import java.net.Inet4Address;
@ -43,9 +43,7 @@ import io.github.chronosx88.influence.helpers.JVMShutdownHook;
import io.github.chronosx88.influence.helpers.KeyPairManager;
import io.github.chronosx88.influence.helpers.NetworkHandler;
import io.github.chronosx88.influence.helpers.P2PUtils;
import io.github.chronosx88.influence.helpers.StorageMVStore;
import io.github.chronosx88.influence.helpers.StorageMapDB;
import io.github.chronosx88.influence.helpers.actions.NetworkActions;
import io.github.chronosx88.influence.helpers.actions.UIActions;
import io.github.chronosx88.influence.models.PublicUserProfile;
@ -85,13 +83,15 @@ public class MainLogic implements IMainLogicContract {
new Thread(() -> {
try {
StorageMapDB storageMapDB = new StorageMapDB(peerID, context.getFilesDir(), new DSASignatureFactory());
peerDHT = new PeerBuilderDHT(
new PeerBuilder(peerID)
.ports(7243)
.start()
)
.storage(new StorageMVStore(peerID, context.getFilesDir()))
.storage(storageMapDB)
.start();
Runtime.getRuntime().addShutdownHook(new JVMShutdownHook(storageMapDB));
try {
String bootstrapIP = this.preferences.getString("bootstrapAddress", null);
if(bootstrapIP == null) {

2
app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java
View File

@ -66,6 +66,4 @@ public class StartChatFragment extends Fragment implements IStartChatViewContrac
}
});
}
// TODO: clear text input
}