diff --git a/app/build.gradle b/app/build.gradle index 9433b60..4b1669c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -40,9 +40,9 @@ dependencies { implementation 'androidx.constraintlayout:constraintlayout:2.0.0-alpha3' implementation "androidx.room:room-runtime:2.1.0-alpha04" annotationProcessor "androidx.room:room-compiler:2.1.0-alpha04" - implementation('net.tomp2p:tomp2p-all:5.0-Beta8') //{ - //exclude group: 'org.mapdb', module: 'mapdb' - //} + implementation('net.tomp2p:tomp2p-all:5.0-Beta8') { + exclude group: 'net.tomp2p', module: 'tomp2p-storage' + } implementation 'org.slf4j:slf4j-log4j12:1.7.26' implementation group: 'com.h2database', name: 'h2-mvstore', version: '1.4.197' implementation 'com.google.android.material:material:1.1.0-alpha04' @@ -50,4 +50,5 @@ dependencies { implementation 'com.google.code.gson:gson:2.8.5' implementation group: 'org.springframework.security', name: 'spring-security-crypto', version: '3.1.0.RELEASE' implementation 'de.hdodenhof:circleimageview:3.0.0' + implementation 'org.mapdb:mapdb:2.0-beta13' } diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java b/app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java index 2fba7a2..e14f50a 100644 --- a/app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java @@ -17,7 +17,7 @@ public class Converter { } @TypeConverter - public static String fromArrayLisr(ArrayList list) { + public static String fromArrayList(ArrayList list) { Gson gson = new Gson(); return gson.toJson(list); } diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerEx.java b/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerEx.java new file mode 100644 index 0000000..cebe499 --- /dev/null +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerEx.java @@ -0,0 +1,194 @@ +package io.github.chronosx88.influence.helpers; + +import android.util.Log; + +import net.tomp2p.connection.SignatureFactory; +import net.tomp2p.peers.Number160; +import net.tomp2p.storage.AlternativeCompositeByteBuf; +import net.tomp2p.storage.Data; + +import org.mapdb.DataIO; +import org.mapdb.Serializer; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.security.InvalidKeyException; +import java.security.KeyPair; +import java.security.SignatureException; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +public class DataSerializerEx extends Serializer implements Serializable { + + private static final long serialVersionUID = 1428836065493792295L; + //TODO: test the performance impact + private static final int MAX_SIZE = 10 * 1024; + + private File path; + private SignatureFactory signatureFactory; + final private static KeyPairManager keyPairManager = new KeyPairManager(); + + public DataSerializerEx() { } + + public DataSerializerEx(File path, SignatureFactory signatureFactory) { + this.path = path; + this.signatureFactory = signatureFactory; + } + + @Override + public void serialize(DataOutput out, Data value) throws IOException { + if (value.length() > MAX_SIZE) { + // header, 1 means stored on disk in a file + out.writeByte(1); + serializeFile(out, value); + } else { + // header, 0 means stored on disk with MapDB + out.writeByte(0); + serializeMapDB(out, value); + } + } + + private void serializeMapDB(DataOutput out, Data value) throws IOException { + KeyPair keyPair = keyPairManager.getKeyPair("mainKeyForSign"); + value.sign(keyPair); + AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP); + // store data to disk + // header first + value.encodeHeader(acb, signatureFactory); + write(out, acb.nioBuffers()); + acb.skipBytes(acb.writerIndex()); + // next data - no need to copy to another buffer, just take the data + // from memory + write(out, value.toByteBuffers()); + // rest + try { + value.encodeDone(acb, signatureFactory); + write(out, acb.nioBuffers()); + } catch (InvalidKeyException e) { + throw new IOException(e); + } catch (SignatureException e) { + throw new IOException(e); + } + } + + private void serializeFile(DataOutput out, Data value) throws IOException, FileNotFoundException { + Number160 hash = value.hash(); + // store file name + out.write(hash.toByteArray()); + // store as external file, create path + RandomAccessFile file = null; + FileChannel rwChannel = null; + try { + file = new RandomAccessFile(new File(path, hash.toString()), "rw"); + rwChannel = file.getChannel(); + AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP); + // store data to disk + // header first + value.encodeHeader(acb, signatureFactory); + rwChannel.write(acb.nioBuffers()); + // next data - no need to copy to another buffer, just take the + // data from memory + rwChannel.write(value.toByteBuffers()); + // rest + try { + value.encodeDone(acb, signatureFactory); + rwChannel.write(acb.nioBuffers()); + } catch (InvalidKeyException e) { + throw new IOException(e); + } catch (SignatureException e) { + throw new IOException(e); + } + } finally { + + if (rwChannel != null) { + rwChannel.close(); + } + if (file != null) { + file.close(); + } + } + } + + private void write(DataOutput out, ByteBuffer[] nioBuffers) throws IOException { + final int length = nioBuffers.length; + for(int i=0;i < length; i++) { + int remaining = nioBuffers[i].remaining(); + if(nioBuffers[i].hasArray()) { + out.write(nioBuffers[i].array(), nioBuffers[i].arrayOffset(), remaining); + } else { + byte[] me = new byte[remaining]; + nioBuffers[i].get(me); + out.write(me); + } + } + } + + @Override + public Data deserialize(DataInput in, int available) throws IOException { + int header = in.readByte(); + if(header == 1) { + return deserializeFile(in); + } else if(header == 0) { + return deserializeMapDB(in); + } else { + throw new IOException("unexpected header: " + header); + } + } + + private Data deserializeMapDB(DataInput in) throws IOException { + ByteBuf buf = Unpooled.buffer(); + Data data = null; + while(data == null) { + buf.writeByte(in.readByte()); + data = Data.decodeHeader(buf, signatureFactory); + } + int len = data.length(); + byte me[] = new byte[len]; + in.readFully(me); + buf = Unpooled.wrappedBuffer(me); + boolean retVal = data.decodeBuffer(buf); + if(!retVal) { + throw new IOException("data could not be read"); + } + retVal = data.decodeDone(buf, signatureFactory); + if(!retVal) { + //throw new IOException("signature could not be read"); + Log.e("DataSerializerEx", "# Signature could not be read!"); + } + DataIO.DataInputByteArray di = (DataIO.DataInputByteArray) in; + di.setPos(di.internalByteArray().length); + + return data; + } + + private Data deserializeFile(DataInput in) throws IOException, FileNotFoundException { + byte[] me = new byte[Number160.BYTE_ARRAY_SIZE]; + in.readFully(me); + Number160 hash = new Number160(me); + RandomAccessFile file = new RandomAccessFile(new File(path, hash.toString()), "r"); + FileChannel inChannel = file.getChannel(); + MappedByteBuffer buffer = inChannel.map(FileChannel.MapMode.READ_ONLY, 0, inChannel.size()); + buffer.load(); + ByteBuf buf = Unpooled.wrappedBuffer(buffer); + Data data = Data.decodeHeader(buf, signatureFactory); + data.decodeBuffer(buf); + data.decodeDone(buf, signatureFactory); + file.close(); + return data; + } + + @Override + public int fixedSize() { + return -1; + } +} + diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java b/app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java index 008a28e..4e5c7bd 100644 --- a/app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java @@ -1,8 +1,11 @@ package io.github.chronosx88.influence.helpers; +import android.util.Log; + import net.tomp2p.dht.Storage; public class JVMShutdownHook extends Thread { + Storage storage; public JVMShutdownHook(Storage storage) { @@ -12,6 +15,9 @@ public class JVMShutdownHook extends Thread { @Override public void run() { super.run(); + Log.d("JVMShutdownHook", "# Closing storage..."); storage.close(); + Log.d("JVMShutdownHook", "# Storage is closed"); } + } diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java b/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java index 4cb4087..d2c7489 100644 --- a/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java @@ -1,5 +1,16 @@ package io.github.chronosx88.influence.helpers; +import net.tomp2p.connection.SignatureFactory; +import net.tomp2p.dht.Storage; +import net.tomp2p.peers.Number160; +import net.tomp2p.peers.Number320; +import net.tomp2p.peers.Number480; +import net.tomp2p.peers.Number640; +import net.tomp2p.storage.Data; + +import org.mapdb.DB; +import org.mapdb.DBMaker; + import java.io.File; import java.security.PublicKey; import java.util.ArrayList; @@ -15,18 +26,6 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentNavigableMap; -import net.tomp2p.connection.SignatureFactory; -import net.tomp2p.dht.Storage; -import net.tomp2p.peers.Number160; -import net.tomp2p.peers.Number320; -import net.tomp2p.peers.Number480; -import net.tomp2p.peers.Number640; -import net.tomp2p.storage.Data; -import net.tomp2p.storage.DataSerializer; - -import org.mapdb.DB; -import org.mapdb.DBMaker; - public class StorageMapDB implements Storage { // Core final private NavigableMap dataMap; @@ -47,7 +46,7 @@ public class StorageMapDB implements Storage { //for full control public StorageMapDB(DB db, Number160 peerId, File path, SignatureFactory signatureFactory, int storageCheckIntervalMillis) { this.db = db; - DataSerializer dataSerializer = new DataSerializer(path, signatureFactory); + DataSerializerEx dataSerializer = new DataSerializerEx(path, signatureFactory); this.dataMap = db.createTreeMap("dataMap_" + peerId.toString()).valueSerializer(dataSerializer).makeOrGet(); this.timeoutMap = db.createTreeMap("timeoutMap_" + peerId.toString()).makeOrGet(); this.timeoutMapRev = db.createTreeMap("timeoutMapRev_" + peerId.toString()).makeOrGet(); @@ -60,7 +59,7 @@ public class StorageMapDB implements Storage { //set parameter to a reasonable default public StorageMapDB(Number160 peerId, File path, SignatureFactory signatureFactory) { - this(DBMaker.newFileDB(new File(path, "tomp2p")).closeOnJvmShutdown().make(), + this(DBMaker.newFileDB(new File(path, "coreDB")).closeOnJvmShutdown().make(), peerId, path, signatureFactory, 60 * 1000); } diff --git a/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java b/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java index 76010dd..beb4ced 100644 --- a/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java +++ b/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java @@ -7,9 +7,10 @@ import android.util.Log; import com.google.gson.Gson; import com.google.gson.JsonObject; -import net.tomp2p.connection.RSASignatureFactory; +import net.tomp2p.connection.DSASignatureFactory; import net.tomp2p.dht.PeerBuilderDHT; import net.tomp2p.dht.PeerDHT; +import net.tomp2p.dht.StorageMemory; import net.tomp2p.futures.FutureBootstrap; import net.tomp2p.futures.FutureDiscover; import net.tomp2p.nat.FutureRelayNAT; @@ -21,7 +22,6 @@ import net.tomp2p.peers.PeerAddress; import net.tomp2p.relay.tcp.TCPRelayClientConfig; import net.tomp2p.replication.AutoReplication; import net.tomp2p.storage.Data; -import net.tomp2p.storage.StorageDisk; import java.io.IOException; import java.net.Inet4Address; @@ -43,9 +43,7 @@ import io.github.chronosx88.influence.helpers.JVMShutdownHook; import io.github.chronosx88.influence.helpers.KeyPairManager; import io.github.chronosx88.influence.helpers.NetworkHandler; import io.github.chronosx88.influence.helpers.P2PUtils; -import io.github.chronosx88.influence.helpers.StorageMVStore; import io.github.chronosx88.influence.helpers.StorageMapDB; -import io.github.chronosx88.influence.helpers.actions.NetworkActions; import io.github.chronosx88.influence.helpers.actions.UIActions; import io.github.chronosx88.influence.models.PublicUserProfile; @@ -85,13 +83,15 @@ public class MainLogic implements IMainLogicContract { new Thread(() -> { try { + StorageMapDB storageMapDB = new StorageMapDB(peerID, context.getFilesDir(), new DSASignatureFactory()); peerDHT = new PeerBuilderDHT( new PeerBuilder(peerID) .ports(7243) .start() ) - .storage(new StorageMVStore(peerID, context.getFilesDir())) + .storage(storageMapDB) .start(); + Runtime.getRuntime().addShutdownHook(new JVMShutdownHook(storageMapDB)); try { String bootstrapIP = this.preferences.getString("bootstrapAddress", null); if(bootstrapIP == null) { diff --git a/app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java b/app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java index ff93cd1..b307167 100644 --- a/app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java +++ b/app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java @@ -66,6 +66,4 @@ public class StartChatFragment extends Fragment implements IStartChatViewContrac } }); } - - // TODO: clear text input }