diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml deleted file mode 100644 index 30aa626..0000000 --- a/.idea/codeStyles/Project.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index c63c3e6..015c555 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -44,7 +44,9 @@ dependencies { implementation "androidx.room:room-runtime:2.1.0-alpha04" annotationProcessor "androidx.room:room-compiler:2.1.0-alpha04" implementation 'org.slf4j:slf4j-log4j12:1.7.26' - implementation 'net.tomp2p:tomp2p-all:5.0-Beta8' + implementation('net.tomp2p:tomp2p-all:5.0-Beta8') { + exclude group: 'net.tomp2p', module: 'tomp2p-storage' + } implementation 'com.google.android.material:material:1.1.0-alpha04' implementation 'androidx.preference:preference:1.1.0-alpha03' implementation 'com.google.code.gson:gson:2.8.5' @@ -54,7 +56,19 @@ dependencies { implementation "org.jetbrains.anko:anko:0.10.8" implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.2.0' implementation 'com.esotericsoftware:kryo:5.0.0-RC1' + + implementation 'org.eclipse.collections:eclipse-collections-api:7.0.0' + implementation 'org.eclipse.collections:eclipse-collections:7.0.0' + implementation 'org.eclipse.collections:eclipse-collections-forkjoin:7.0.0' + implementation('com.google.guava:guava:15.0') { + exclude group: 'com.google.guava', module: 'listenablefuture' + } + implementation 'net.jpountz.lz4:lz4:1.3.0' + implementation 'org.mapdb:elsa:3.0.0-M5' } repositories { mavenCentral() } +configurations { + all*.exclude group: 'com.google.guava', module:'listenablefuture' +} \ No newline at end of file diff --git a/app/libs/mapdb-3.0.7-sources.jar b/app/libs/mapdb-3.0.7-sources.jar new file mode 100644 index 0000000..6a59d68 Binary files /dev/null and b/app/libs/mapdb-3.0.7-sources.jar differ diff --git a/app/libs/mapdb-3.0.7.jar b/app/libs/mapdb-3.0.7.jar new file mode 100644 index 0000000..a756239 Binary files /dev/null and b/app/libs/mapdb-3.0.7.jar differ diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerMapDB.kt b/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerMapDB.kt new file mode 100644 index 0000000..abe3037 --- /dev/null +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/DataSerializerMapDB.kt @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2019 ChronosX88 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.github.chronosx88.influence.helpers + +import io.netty.buffer.Unpooled +import net.tomp2p.connection.SignatureFactory +import net.tomp2p.p2p.PeerBuilder +import net.tomp2p.storage.Data +import org.mapdb.DataInput2 +import org.mapdb.DataOutput2 +import org.mapdb.serializer.GroupSerializerObjectArray +import java.io.* +import java.nio.ByteBuffer +import java.security.InvalidKeyException +import java.security.SignatureException + + +class DataSerializerMapDB(private val signatureFactory: SignatureFactory) : GroupSerializerObjectArray() { + private val LOG_TAG = "DataSerializerMapDB" + + override fun serialize(out: DataOutput2, value: Data) { + val dataOut = DataOutputStream(out) + + val acb = Unpooled.buffer() + // store data to disk + // header first + if(value.publicKey().equals(PeerBuilder.EMPTY_PUBLIC_KEY)) { + value.publicKey(null) + } + value.encodeHeader(acb, signatureFactory) + writeData(dataOut, acb.nioBuffers()) + acb.skipBytes(acb.writerIndex()) + // next data - no need to copy to another buffer, just take the data + // from memory + writeData(dataOut, value.toByteBuffers()) + // rest + try { + value.encodeDone(acb, signatureFactory) + writeData(dataOut, acb.nioBuffers()) + } catch (e: InvalidKeyException) { + throw IOException(e) + } catch (e: SignatureException) { + throw IOException(e) + } + } + + override fun deserialize(input: DataInput2, available: Int): Data { + val dataInput = DataInputStream(DataInput2.DataInputToStream(input)) + + var buf = Unpooled.buffer() + var data: Data? = null + while (data == null) { + buf.writeByte(dataInput.readByte().toInt()) + data = Data.decodeHeader(buf, signatureFactory) + } + val len = data.length() + var me = ByteArray(len) + dataInput.readFully(me) + buf = Unpooled.wrappedBuffer(me) + var retVal = data.decodeBuffer(buf) + if (!retVal) { + throw IOException("data could not be read") + } + if (data.isSigned) { + me = ByteArray(signatureFactory.signatureSize()) + dataInput.readFully(me) + buf = Unpooled.wrappedBuffer(me) + } + retVal = data.decodeDone(buf, signatureFactory) + if (!retVal) { + throw IOException("signature could not be read") + } + return data + } + + @Throws(IOException::class) + private fun writeData(out: OutputStream, nioBuffers: Array) { + val length = nioBuffers.size + for (i in 0 until length) { + val remaining = nioBuffers[i].remaining() + if (nioBuffers[i].hasArray()) { + out.write(nioBuffers[i].array(), nioBuffers[i].arrayOffset(), remaining) + } else { + val me = ByteArray(remaining) + nioBuffers[i].get(me) + out.write(me) + } + } + } + + companion object { + private const val serialVersionUID = 1428836065493792295L + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.kt b/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.kt new file mode 100644 index 0000000..85a9b52 --- /dev/null +++ b/app/src/main/java/io/github/chronosx88/influence/helpers/StorageMapDB.kt @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2019 ChronosX88 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.github.chronosx88.influence.helpers + +import com.esotericsoftware.kryo.serializers.JavaSerializer +import net.tomp2p.connection.SignatureFactory +import net.tomp2p.dht.Storage +import net.tomp2p.peers.Number160 +import net.tomp2p.peers.Number320 +import net.tomp2p.peers.Number480 +import net.tomp2p.peers.Number640 +import net.tomp2p.storage.Data +import org.mapdb.BTreeMap +import org.mapdb.DB +import org.mapdb.DBMaker +import org.mapdb.serializer.SerializerJava +import java.io.File +import java.security.PublicKey +import java.util.* +import java.util.concurrent.ConcurrentHashMap +import kotlin.collections.ArrayList + + +class StorageMapDB(peerId: Number160, path : File, signatureFactory: SignatureFactory) : Storage { + // Core + private val dataMap: BTreeMap + // Maintenance + private val timeoutMap: BTreeMap + private val timeoutMapRev: BTreeMap> + // Protection + private val protectedDomainMap: BTreeMap + private val protectedEntryMap: BTreeMap + // Responsibility + private val responsibilityMap: BTreeMap + private val responsibilityMapRev: BTreeMap> + + private val db: DB = DBMaker.fileDB(path).closeOnJvmShutdown().make() + private val storageCheckIntervalMillis: Int = 60 * 1000 + + init { + val dataSerializer = DataSerializerMapDB(signatureFactory) + //val kryoSerializer = KryoSerializer() // We use this because Elsa serializer works very poor + val javaSerializer = SerializerJava() + dataMap = db + .treeMap("dataMap_$peerId") + .valueSerializer(dataSerializer) + .createOrOpen() as BTreeMap + timeoutMap = db + .treeMap("timeoutMap_$peerId") + .createOrOpen() as BTreeMap + timeoutMapRev = db + .treeMap("timeoutMapRev_$peerId") + .createOrOpen() as BTreeMap> + protectedDomainMap = db + .treeMap("protectedDomainMap_$peerId") + .createOrOpen() as BTreeMap + protectedEntryMap = db + .treeMap("protectedEntryMap_$peerId") + .createOrOpen() as BTreeMap + responsibilityMap = db + .treeMap("responsibilityMap_$peerId") + .createOrOpen() as BTreeMap + responsibilityMapRev = db + .treeMap("responsibilityMapRev_$peerId") + .createOrOpen() as BTreeMap> + } + + + override fun contains(key: Number640?): Boolean { + return dataMap.containsKey(key) + } + + override fun contains(from: Number640?, to: Number640?): Int { + return dataMap.subMap(from, true, to, true).size + } + + override fun findContentForResponsiblePeerID(peerID: Number160?): MutableSet? { + return responsibilityMapRev[peerID] as MutableSet? + } + + override fun findPeerIDsForResponsibleContent(locationKey: Number160?): Number160? { + return responsibilityMap[locationKey] + } + + override fun put(key: Number640?, value: Data?): Data? { + val oldData = dataMap.put(key, value) + db.commit() + return oldData + } + + override fun get(key: Number640?): Data? { + return dataMap[key] + } + + override fun remove(key: Number640?, returnData: Boolean): Data? { + val retVal = dataMap.remove(key) + db.commit() + return retVal + } + + override fun remove(from: Number640?, to: Number640?): NavigableMap { + val tmp = dataMap.subMap(from, true, to, true) + val retVal = TreeMap() + for(entry : Map.Entry in tmp.entries) { + retVal[entry.key] = entry.value + } + tmp.clear() + db.commit() + return retVal + } + + override fun addTimeout(key: Number640, expiration: Long) { + val oldExpiration = timeoutMap.put(key, expiration) + putIfAbsent2(expiration, key) + if (oldExpiration == null) { + return + } + removeRevTimeout(key, oldExpiration) + db.commit() + } + + private fun putIfAbsent2(expiration: Long, key: Number640) { + var timeouts = timeoutMapRev[expiration] + //var timeouts : MutableSet = timeoutMapRev[expiration] as MutableSet + if (timeouts == null) { + timeouts = Collections.newSetFromMap(ConcurrentHashMap()) + } + (timeouts as MutableSet).add(key) + timeoutMapRev[expiration] = timeouts + } + + private fun removeRevTimeout(key: Number640, expiration: Long?) { + val tmp = timeoutMapRev[expiration] as MutableSet? + if (tmp != null) { + tmp.remove(key) + if (tmp.isEmpty()) { + timeoutMapRev.remove(expiration) + } else { + timeoutMapRev[expiration!!] = tmp + } + } + } + + override fun updateResponsibilities(locationKey: Number160, peerId: Number160?): Boolean { + val oldPeerID = responsibilityMap.put(locationKey, peerId) + val hasChanged: Boolean + if (oldPeerID != null) { + if (oldPeerID == peerId) { + hasChanged = false + } else { + removeRevResponsibility(oldPeerID, locationKey) + hasChanged = true + } + } else { + hasChanged = true + } + var contentIDs: MutableSet? = responsibilityMapRev[peerId] as MutableSet + if (contentIDs == null) { + contentIDs = HashSet() + } + contentIDs.add(locationKey) + responsibilityMapRev[peerId] = contentIDs + db.commit() + return hasChanged + } + + private fun removeRevResponsibility(peerId: Number160, locationKey: Number160) { + val contentIDs = responsibilityMapRev[peerId] as MutableSet? + if (contentIDs != null) { + contentIDs.remove(locationKey) + if (contentIDs.isEmpty()) { + responsibilityMapRev.remove(peerId) + } else { + responsibilityMapRev[peerId] = contentIDs + } + } + } + + override fun protectDomain(key: Number320?, publicKey: PublicKey?): Boolean { + protectedDomainMap[key] = publicKey + db.commit() + return true + } + + override fun storageCheckIntervalMillis(): Int { + return storageCheckIntervalMillis + } + + override fun isDomainProtectedByOthers(key: Number320?, publicKey: PublicKey?): Boolean { + val other = protectedDomainMap[key] ?: return false + return other != publicKey + } + + override fun removeTimeout(key: Number640) { + val expiration = timeoutMap.remove(key) ?: return + removeRevTimeout(key, expiration) + db.commit() + } + + override fun removeResponsibility(locationKey: Number160) { + val peerId = responsibilityMap.remove(locationKey) + if (peerId != null) { + removeRevResponsibility(peerId, locationKey) + } + db.commit() + } + + override fun protectEntry(key: Number480?, publicKey: PublicKey?): Boolean { + publicKey ?: return true + protectedEntryMap[key] = publicKey + return true + } + + override fun map(): NavigableMap { + val retVal = TreeMap() + for ((key, value) in dataMap) { + retVal[key] = value + } + + return retVal + } + + override fun isEntryProtectedByOthers(key: Number480?, publicKey: PublicKey?): Boolean { + val other = protectedEntryMap[key] ?: return false + return other != publicKey + } + + override fun subMap(from: Number640?, to: Number640?, limit: Int, ascending: Boolean): NavigableMap { + val tmp = dataMap.subMap(from, true, to, true) + val retVal = TreeMap() + if (limit < 0) { + for ((key, value) in if (ascending) tmp else tmp.descendingMap()) { + retVal[key] = value + } + } else { + val limit1 = Math.min(limit, tmp.size) + val iterator = if (ascending) + tmp.entries.iterator() + else + tmp.descendingMap().entries.iterator() + var i = 0 + while (iterator.hasNext() && i < limit1) { + val entry = iterator.next() + retVal[entry.key] = entry.value + i++ + } + } + return retVal + } + + override fun subMapTimeout(to: Long): MutableCollection { + val tmp = timeoutMapRev.subMap(0L, to) + val toRemove = ArrayList() + for (set in tmp.values) { + toRemove.addAll(set) + } + return toRemove + } + + override fun close() { + db.close() + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java b/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java index 91a4532..ee7b7f8 100644 --- a/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java +++ b/app/src/main/java/io/github/chronosx88/influence/logic/MainLogic.java @@ -14,6 +14,7 @@ import net.tomp2p.connection.Ports; import net.tomp2p.connection.RSASignatureFactory; import net.tomp2p.dht.PeerBuilderDHT; import net.tomp2p.dht.PeerDHT; +import net.tomp2p.dht.Storage; import net.tomp2p.futures.FutureBootstrap; import net.tomp2p.futures.FutureDiscover; import net.tomp2p.nat.FutureRelayNAT; @@ -26,6 +27,7 @@ import net.tomp2p.relay.tcp.TCPRelayClientConfig; import net.tomp2p.replication.IndirectReplication; import net.tomp2p.storage.Data; +import java.io.File; import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; @@ -42,7 +44,7 @@ import io.github.chronosx88.influence.helpers.JVMShutdownHook; import io.github.chronosx88.influence.helpers.KeyPairManager; import io.github.chronosx88.influence.helpers.NetworkHandler; import io.github.chronosx88.influence.helpers.P2PUtils; -import io.github.chronosx88.influence.helpers.StorageBerkeleyDB; +import io.github.chronosx88.influence.helpers.StorageMapDB; import io.github.chronosx88.influence.helpers.actions.UIActions; import io.github.chronosx88.influence.models.PublicUserProfile; @@ -59,7 +61,7 @@ public class MainLogic implements CoreContracts.IMainLogicContract { private IndirectReplication replication; private KeyPairManager keyPairManager; private Thread checkNewChatsThread = null; - private StorageBerkeleyDB storage; + private Storage storage; public MainLogic() { this.context = AppHelper.getContext(); @@ -83,8 +85,9 @@ public class MainLogic implements CoreContracts.IMainLogicContract { new Thread(() -> { try { - StorageBerkeleyDB storageBerkeleyDB = new StorageBerkeleyDB(peerID, context.getFilesDir(), new RSASignatureFactory()); - this.storage = storageBerkeleyDB; + //StorageBerkeleyDB storageBerkeleyDB = new StorageBerkeleyDB(peerID, context.getFilesDir(), new RSASignatureFactory()); + Storage storageMapDB = new StorageMapDB(peerID, new File(context.getFilesDir(), "dhtDB.db"), new RSASignatureFactory()); + this.storage = storageMapDB; peerDHT = new PeerBuilderDHT( new PeerBuilder(peerID) .ports(7243) @@ -92,9 +95,9 @@ public class MainLogic implements CoreContracts.IMainLogicContract { .channelServerConfiguration(createChannelServerConfig()) .start() ) - .storage(storageBerkeleyDB) + .storage(storageMapDB) .start(); - Runtime.getRuntime().addShutdownHook(new JVMShutdownHook(storageBerkeleyDB)); + Runtime.getRuntime().addShutdownHook(new JVMShutdownHook(storageMapDB)); try { String bootstrapIP = this.preferences.getString("bootstrapAddress", null); if(bootstrapIP == null) { diff --git a/app/src/main/java/io/github/chronosx88/influence/logic/SettingsLogic.kt b/app/src/main/java/io/github/chronosx88/influence/logic/SettingsLogic.kt index da08ee9..e324a34 100644 --- a/app/src/main/java/io/github/chronosx88/influence/logic/SettingsLogic.kt +++ b/app/src/main/java/io/github/chronosx88/influence/logic/SettingsLogic.kt @@ -40,7 +40,7 @@ class SettingsLogic : CoreContracts.ISettingsLogic { data!!.protectEntry(mainKeyPair) val isSuccess = P2PUtils.put(username, null, data, mainKeyPair) - Log.i(LOG_TAG, if (isSuccess) "Username $username is published!" else "Username $username isn't published!") + Log.i(LOG_TAG, if (isSuccess) "# Username $username is published!" else "# Username $username isn't published!") } ?: run { return } diff --git a/gradle.properties b/gradle.properties index abc7787..9e35377 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,4 +16,3 @@ android.useAndroidX=true android.enableJetifier=true android.enableD8=true -