[WIP] Changed storage backend to MapDB

app/build.gradle

@@ -40,9 +40,9 @@ dependencies {
     implementation 'androidx.constraintlayout:constraintlayout:2.0.0-alpha3'
     implementation "androidx.room:room-runtime:2.1.0-alpha04"
     annotationProcessor "androidx.room:room-compiler:2.1.0-alpha04"
-    implementation('net.tomp2p:tomp2p-all:5.0-Beta8') //{
+    implementation('net.tomp2p:tomp2p-all:5.0-Beta8') {
-        //exclude group: 'org.mapdb', module: 'mapdb'
+        exclude group: 'net.tomp2p', module: 'tomp2p-storage'
-    //}
+    }
     implementation 'org.slf4j:slf4j-log4j12:1.7.26'
     implementation group: 'com.h2database', name: 'h2-mvstore', version: '1.4.197'
     implementation 'com.google.android.material:material:1.1.0-alpha04'
@@ -50,4 +50,5 @@ dependencies {
     implementation 'com.google.code.gson:gson:2.8.5'
     implementation group: 'org.springframework.security', name: 'spring-security-crypto', version: '3.1.0.RELEASE'
     implementation 'de.hdodenhof:circleimageview:3.0.0'
+    implementation 'org.mapdb:mapdb:2.0-beta13'
 }

app/src/main/java/io/github/chronosx88/influence/helpers/Converter.java

@@ -17,7 +17,7 @@ public class Converter {
     }
 
     @TypeConverter
-    public static String fromArrayLisr(ArrayList<String> list) {
+    public static String fromArrayList(ArrayList<String> list) {
         Gson gson = new Gson();
         return gson.toJson(list);
     }

app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerEx.java

@@ -0,0 +1,194 @@
+package io.github.chronosx88.influence.helpers;
+
+import android.util.Log;
+
+import net.tomp2p.connection.SignatureFactory;
+import net.tomp2p.peers.Number160;
+import net.tomp2p.storage.AlternativeCompositeByteBuf;
+import net.tomp2p.storage.Data;
+
+import org.mapdb.DataIO;
+import org.mapdb.Serializer;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.SignatureException;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+
+public class DataSerializerEx extends Serializer<Data> implements Serializable {
+
+    private static final long serialVersionUID = 1428836065493792295L;
+    //TODO: test the performance impact
+    private static final int MAX_SIZE = 10 * 1024;
+
+    private File path;
+    private SignatureFactory signatureFactory;
+    final private static KeyPairManager keyPairManager = new KeyPairManager();
+
+    public DataSerializerEx() { }
+
+    public DataSerializerEx(File path, SignatureFactory signatureFactory) {
+        this.path = path;
+        this.signatureFactory = signatureFactory;
+    }
+
+    @Override
+    public void serialize(DataOutput out, Data value) throws IOException {
+        if (value.length() > MAX_SIZE) {
+            // header, 1 means stored on disk in a file
+            out.writeByte(1);
+            serializeFile(out, value);
+        } else {
+            // header, 0 means stored on disk with MapDB
+            out.writeByte(0);
+            serializeMapDB(out, value);
+        }
+    }
+
+    private void serializeMapDB(DataOutput out, Data value) throws IOException {
+        KeyPair keyPair =  keyPairManager.getKeyPair("mainKeyForSign");
+        value.sign(keyPair);
+        AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP);
+        // store data to disk
+        // header first
+        value.encodeHeader(acb, signatureFactory);
+        write(out, acb.nioBuffers());
+        acb.skipBytes(acb.writerIndex());
+        // next data - no need to copy to another buffer, just take the data
+        // from memory
+        write(out, value.toByteBuffers());
+        // rest
+        try {
+            value.encodeDone(acb, signatureFactory);
+            write(out, acb.nioBuffers());
+        } catch (InvalidKeyException e) {
+            throw new IOException(e);
+        } catch (SignatureException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private void serializeFile(DataOutput out, Data value) throws IOException, FileNotFoundException {
+        Number160 hash = value.hash();
+        // store file name
+        out.write(hash.toByteArray());
+        // store as external file, create path
+        RandomAccessFile file = null;
+        FileChannel rwChannel = null;
+        try {
+            file = new RandomAccessFile(new File(path, hash.toString()), "rw");
+            rwChannel = file.getChannel();
+            AlternativeCompositeByteBuf acb = AlternativeCompositeByteBuf.compBuffer(AlternativeCompositeByteBuf.UNPOOLED_HEAP);
+            // store data to disk
+            // header first
+            value.encodeHeader(acb, signatureFactory);
+            rwChannel.write(acb.nioBuffers());
+            // next data - no need to copy to another buffer, just take the
+            // data from memory
+            rwChannel.write(value.toByteBuffers());
+            // rest
+            try {
+                value.encodeDone(acb, signatureFactory);
+                rwChannel.write(acb.nioBuffers());
+            } catch (InvalidKeyException e) {
+                throw new IOException(e);
+            } catch (SignatureException e) {
+                throw new IOException(e);
+            }
+        } finally {
+
+            if (rwChannel != null) {
+                rwChannel.close();
+            }
+            if (file != null) {
+                file.close();
+            }
+        }
+    }
+
+    private void write(DataOutput out, ByteBuffer[] nioBuffers) throws IOException {
+        final int length = nioBuffers.length;
+        for(int i=0;i < length; i++) {
+            int remaining = nioBuffers[i].remaining();
+            if(nioBuffers[i].hasArray()) {
+                out.write(nioBuffers[i].array(), nioBuffers[i].arrayOffset(), remaining);
+            } else {
+                byte[] me = new byte[remaining];
+                nioBuffers[i].get(me);
+                out.write(me);
+            }
+        }
+    }
+
+    @Override
+    public Data deserialize(DataInput in, int available) throws IOException {
+        int header = in.readByte();
+        if(header == 1) {
+            return deserializeFile(in);
+        } else if(header == 0) {
+            return deserializeMapDB(in);
+        } else {
+            throw new IOException("unexpected header: " + header);
+        }
+    }
+
+    private Data deserializeMapDB(DataInput in) throws IOException {
+        ByteBuf buf = Unpooled.buffer();
+        Data data = null;
+        while(data == null) {
+            buf.writeByte(in.readByte());
+            data = Data.decodeHeader(buf, signatureFactory);
+        }
+        int len = data.length();
+        byte me[] = new byte[len];
+        in.readFully(me);
+        buf = Unpooled.wrappedBuffer(me);
+        boolean retVal = data.decodeBuffer(buf);
+        if(!retVal) {
+            throw new IOException("data could not be read");
+        }
+        retVal = data.decodeDone(buf, signatureFactory);
+        if(!retVal) {
+            //throw new IOException("signature could not be read");
+            Log.e("DataSerializerEx", "# Signature could not be read!");
+        }
+        DataIO.DataInputByteArray di = (DataIO.DataInputByteArray) in;
+        di.setPos(di.internalByteArray().length);
+
+        return data;
+    }
+
+    private Data deserializeFile(DataInput in) throws IOException, FileNotFoundException {
+        byte[] me = new byte[Number160.BYTE_ARRAY_SIZE];
+        in.readFully(me);
+        Number160 hash = new Number160(me);
+        RandomAccessFile file = new RandomAccessFile(new File(path, hash.toString()), "r");
+        FileChannel inChannel = file.getChannel();
+        MappedByteBuffer buffer = inChannel.map(FileChannel.MapMode.READ_ONLY, 0, inChannel.size());
+        buffer.load();
+        ByteBuf buf = Unpooled.wrappedBuffer(buffer);
+        Data data = Data.decodeHeader(buf, signatureFactory);
+        data.decodeBuffer(buf);
+        data.decodeDone(buf, signatureFactory);
+        file.close();
+        return data;
+    }
+
+    @Override
+    public int fixedSize() {
+        return -1;
+    }
+}
+

app/src/main/java/io/github/chronosx88/influence/helpers/JVMShutdownHook.java

@@ -1,8 +1,11 @@
 package io.github.chronosx88.influence.helpers;
 
+import android.util.Log;
+
 import net.tomp2p.dht.Storage;
 
 public class JVMShutdownHook extends Thread {
+
     Storage storage;
 
     public JVMShutdownHook(Storage storage) {
@@ -12,6 +15,9 @@ public class JVMShutdownHook extends Thread {
     @Override
     public void run() {
         super.run();
+        Log.d("JVMShutdownHook", "# Closing storage...");
         storage.close();
+        Log.d("JVMShutdownHook", "# Storage is closed");
     }
+
 }

app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.java

@@ -1,5 +1,16 @@
 package io.github.chronosx88.influence.helpers;
 
+import net.tomp2p.connection.SignatureFactory;
+import net.tomp2p.dht.Storage;
+import net.tomp2p.peers.Number160;
+import net.tomp2p.peers.Number320;
+import net.tomp2p.peers.Number480;
+import net.tomp2p.peers.Number640;
+import net.tomp2p.storage.Data;
+
+import org.mapdb.DB;
+import org.mapdb.DBMaker;
+
 import java.io.File;
 import java.security.PublicKey;
 import java.util.ArrayList;
@@ -15,18 +26,6 @@ import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentNavigableMap;
 
-import net.tomp2p.connection.SignatureFactory;
-import net.tomp2p.dht.Storage;
-import net.tomp2p.peers.Number160;
-import net.tomp2p.peers.Number320;
-import net.tomp2p.peers.Number480;
-import net.tomp2p.peers.Number640;
-import net.tomp2p.storage.Data;
-import net.tomp2p.storage.DataSerializer;
-
-import org.mapdb.DB;
-import org.mapdb.DBMaker;
-
 public class StorageMapDB implements Storage {
     // Core
     final private NavigableMap<Number640, Data> dataMap;
@@ -47,7 +46,7 @@ public class StorageMapDB implements Storage {
     //for full control
     public StorageMapDB(DB db, Number160 peerId, File path, SignatureFactory signatureFactory, int storageCheckIntervalMillis) {
         this.db = db;
-        DataSerializer dataSerializer = new DataSerializer(path, signatureFactory);
+        DataSerializerEx dataSerializer = new DataSerializerEx(path, signatureFactory);
         this.dataMap = db.createTreeMap("dataMap_" + peerId.toString()).valueSerializer(dataSerializer).makeOrGet();
         this.timeoutMap = db.createTreeMap("timeoutMap_" + peerId.toString()).makeOrGet();
         this.timeoutMapRev = db.createTreeMap("timeoutMapRev_" + peerId.toString()).makeOrGet();
@@ -60,7 +59,7 @@ public class StorageMapDB implements Storage {
 
     //set parameter to a reasonable default
     public StorageMapDB(Number160 peerId, File path, SignatureFactory signatureFactory) {
-        this(DBMaker.newFileDB(new File(path, "tomp2p")).closeOnJvmShutdown().make(),
+        this(DBMaker.newFileDB(new File(path, "coreDB")).closeOnJvmShutdown().make(),
                 peerId, path, signatureFactory, 60 * 1000);
     }
 

app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java

@@ -7,9 +7,10 @@ import android.util.Log;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 
-import net.tomp2p.connection.RSASignatureFactory;
+import net.tomp2p.connection.DSASignatureFactory;
 import net.tomp2p.dht.PeerBuilderDHT;
 import net.tomp2p.dht.PeerDHT;
+import net.tomp2p.dht.StorageMemory;
 import net.tomp2p.futures.FutureBootstrap;
 import net.tomp2p.futures.FutureDiscover;
 import net.tomp2p.nat.FutureRelayNAT;
@@ -21,7 +22,6 @@ import net.tomp2p.peers.PeerAddress;
 import net.tomp2p.relay.tcp.TCPRelayClientConfig;
 import net.tomp2p.replication.AutoReplication;
 import net.tomp2p.storage.Data;
-import net.tomp2p.storage.StorageDisk;
 
 import java.io.IOException;
 import java.net.Inet4Address;
@@ -43,9 +43,7 @@ import io.github.chronosx88.influence.helpers.JVMShutdownHook;
 import io.github.chronosx88.influence.helpers.KeyPairManager;
 import io.github.chronosx88.influence.helpers.NetworkHandler;
 import io.github.chronosx88.influence.helpers.P2PUtils;
-import io.github.chronosx88.influence.helpers.StorageMVStore;
 import io.github.chronosx88.influence.helpers.StorageMapDB;
-import io.github.chronosx88.influence.helpers.actions.NetworkActions;
 import io.github.chronosx88.influence.helpers.actions.UIActions;
 import io.github.chronosx88.influence.models.PublicUserProfile;
 
@@ -85,13 +83,15 @@ public class MainLogic implements IMainLogicContract {
 
         new Thread(() -> {
             try {
+                StorageMapDB storageMapDB = new StorageMapDB(peerID, context.getFilesDir(), new DSASignatureFactory());
                 peerDHT = new PeerBuilderDHT(
                         new PeerBuilder(peerID)
                                 .ports(7243)
                                 .start()
                 )
-                        .storage(new StorageMVStore(peerID, context.getFilesDir()))
+                        .storage(storageMapDB)
                         .start();
+                Runtime.getRuntime().addShutdownHook(new JVMShutdownHook(storageMapDB));
                 try {
                     String bootstrapIP = this.preferences.getString("bootstrapAddress", null);
                     if(bootstrapIP == null) {

app/src/main/java/io/github/chronosx88/influence/views/fragments/StartChatFragment.java

@@ -66,6 +66,4 @@ public class StartChatFragment extends Fragment implements IStartChatViewContrac
             }
         });
     }
-
-    // TODO: clear text input
 }