commit 5c067e723a291a78eb374199f3051c67be76216b Author: Joshua Kissoon Date: Sun May 11 23:54:20 2014 +0530 commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..838458f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/dist/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d5bc2fb --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Joshua Kissoon + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..42a3088 --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +Kademlia +======== + +This is an implementation of the Kademlia (http://en.wikipedia.org/wiki/Kademlia) routing protocol and DHT. +Kademlia original Publication: http://link.springer.com/chapter/10.1007/3-540-45748-8_5 + +Note: This repository is a Netbeans project which you can simply download and import. + +Usage +----- +The Implementation is meant to be self contained and very easy to setup and use. There are several tests (https://github.com/JoshuaKissoon/Kademlia/tree/master/src/kademlia/tests) which demonstrates the usage of the protocol and DHT. + + +**Configuration** + +There is a configuration file available in the kademlia.core package which have all settings used throughout the protocol, all of these settings are described in depth in the Configuration file. + + +**Creating a Kad Instance** + +All of Kademlia's sub-components (DHT, Node, Routing Table, Server, etc) are wrapped within the Kademlia object to simplify the usage of the protocol. To create an instance, simply call: + +```Java +Kademlia kad1 = new Kademlia("OwnerName1", new NodeId("ASF45678947584567463"), 12049); +Kademlia kad2 = new Kademlia("OwnerName2", new NodeId(), 12057); // Random NodeId will be generated +``` +Param 1: The Name of the owner of this instance, can be any name. +Param 2: A NodeId for this node +Param 3: The port on which this Kademlia instance will run on. + +After this initialization phase, the 2 Kad instances will basically be 2 separate networks. Lets connect them so they'll be in the same network. + + +**Connecting Nodes** + +Test: https://github.com/JoshuaKissoon/Kademlia/blob/master/src/kademlia/tests/NodeConnectionTest.java +```Java +kad2.bootstrap(kad1.getNode()); // Bootstrap kad2 by using kad1 as the main network node +``` + + +**Storing Content** + +Test: https://github.com/JoshuaKissoon/Kademlia/blob/master/src/kademlia/tests/ContentSendingTest.java +```Java +/* Working example at: https://github.com/JoshuaKissoon/Kademlia/blob/master/src/kademlia/tests/ContentSendingTest.java */ +DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), "Some Data"); // Create a content +kad2.put(c); // Put the content on the network + +``` + + +**Retrieving Content** + +Test: https://github.com/JoshuaKissoon/Kademlia/blob/master/src/kademlia/tests/ContentSendingTest.java +```Java +/* Create a GetParameter object with the parameters of the content to retrieve */ +GetParameter gp = new GetParameter(c.getKey()); // Lets look for content by key +gp.setType(DHTContentImpl.TYPE); // We also only want content of this type +gp.setOwnerId(c.getOwnerId()); // And content from this owner + +/* Now we call get specifying the GetParameters and the Number of results we want */ +List conte = kad2.get(gp, 1); +``` + + +**Saving and Retrieving a Node State** + +Test: https://github.com/JoshuaKissoon/Kademlia/blob/master/src/kademlia/tests/SaveStateTest.java + +You may want to save the Node state when your application is shut down and Retrieve the Node state on startup to remove the need of rebuilding the Node State (Routing Table, DHT Content Entries, etc). Lets look at how we do this. + +```Java +/** + * Shutting down the Kad instance. + * Calling .shutdown() ill automatically store the node state in the location specified in the Configuration file + */ +kad1.shutdown(); + +/** + * Retrieving the Node state + * This is done by simply building the Kademlia instance by calling .loadFromFile() + * and passing in the instance Owner name as a parameter + */ + Kademlia kad1Reloaded = Kademlia.loadFromFile("OwnerName1"); +``` + +For more information on using Kademlia, check the tests at: https://github.com/JoshuaKissoon/Kademlia/tree/master/src/kademlia/tests + + +Usage in a Real Project +----------------------- +I am currently using this implementation of Kademlia in developing a Distributed Online Social Network Architecture, you can look at that project at https://github.com/JoshuaKissoon/DOSNA for more ideas on using Kademlia. diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..4167536 --- /dev/null +++ b/build.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + Builds, tests, and runs the project Kademlia. + + + diff --git a/manifest.mf b/manifest.mf new file mode 100644 index 0000000..1574df4 --- /dev/null +++ b/manifest.mf @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +X-COMMENT: Main-Class will be added automatically by build + diff --git a/nbproject/build-impl.xml b/nbproject/build-impl.xml new file mode 100644 index 0000000..9092fc0 --- /dev/null +++ b/nbproject/build-impl.xml @@ -0,0 +1,1413 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set src.dir + Must set test.src.dir + Must set build.dir + Must set dist.dir + Must set build.classes.dir + Must set dist.javadoc.dir + Must set build.test.classes.dir + Must set build.test.results.dir + Must set build.classes.excludes + Must set dist.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No tests executed. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set JVM to use for profiling in profiler.info.jvm + Must set profiler agent JVM arguments in profiler.info.jvmargs.agent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + To run this application from the command line without Ant, try: + + java -jar "${dist.jar.resolved}" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + Must select one file in the IDE or set run.class + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set debug.class + + + + + Must select one file in the IDE or set debug.class + + + + + Must set fix.includes + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + Must select one file in the IDE or set profile.class + This target only works when run from inside the NetBeans IDE. + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + + + Must select some files in the IDE or set test.includes + + + + + Must select one file in the IDE or set run.class + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + Some tests failed; see details above. + + + + + + + + + Must select some files in the IDE or set test.includes + + + + Some tests failed; see details above. + + + + Must select some files in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + Some tests failed; see details above. + + + + + Must select one file in the IDE or set test.class + + + + Must select one file in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + + + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/nbproject/genfiles.properties b/nbproject/genfiles.properties new file mode 100644 index 0000000..cc9dd93 --- /dev/null +++ b/nbproject/genfiles.properties @@ -0,0 +1,8 @@ +build.xml.data.CRC32=7e563d6e +build.xml.script.CRC32=c3cd04bd +build.xml.stylesheet.CRC32=8064a381@1.68.1.46 +# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. +# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. +nbproject/build-impl.xml.data.CRC32=7e563d6e +nbproject/build-impl.xml.script.CRC32=934ae712 +nbproject/build-impl.xml.stylesheet.CRC32=876e7a8f@1.74.1.48 diff --git a/nbproject/project.properties b/nbproject/project.properties new file mode 100644 index 0000000..35adc9e --- /dev/null +++ b/nbproject/project.properties @@ -0,0 +1,77 @@ +annotation.processing.enabled=true +annotation.processing.enabled.in.editor=false +annotation.processing.processors.list= +annotation.processing.run.all.processors=true +annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output +application.title=Kademlia +application.vendor=Joshua +build.classes.dir=${build.dir}/classes +build.classes.excludes=**/*.java,**/*.form +# This directory is removed when the project is cleaned: +build.dir=build +build.generated.dir=${build.dir}/generated +build.generated.sources.dir=${build.dir}/generated-sources +# Only compile against the classpath explicitly listed here: +build.sysclasspath=ignore +build.test.classes.dir=${build.dir}/test/classes +build.test.results.dir=${build.dir}/test/results +# Uncomment to specify the preferred debugger connection transport: +#debug.transport=dt_socket +debug.classpath=\ + ${run.classpath} +debug.test.classpath=\ + ${run.test.classpath} +# Files in build.classes.dir which should be excluded from distribution jar +dist.archive.excludes= +# This directory is removed when the project is cleaned: +dist.dir=dist +dist.jar=${dist.dir}/Kademlia.jar +dist.javadoc.dir=${dist.dir}/javadoc +endorsed.classpath= +excludes= +file.reference.gson-2.2.4.jar=C:\\Users\\Joshua\\Documents\\NetBeansProjects\\Libraries\\gson-2.2.4.jar +includes=** +jar.compress=false +javac.classpath=\ + ${file.reference.gson-2.2.4.jar} +# Space-separated list of extra javac options +javac.compilerargs= +javac.deprecation=false +javac.processorpath=\ + ${javac.classpath} +javac.source=1.8 +javac.target=1.8 +javac.test.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +javac.test.processorpath=\ + ${javac.test.classpath} +javadoc.additionalparam= +javadoc.author=false +javadoc.encoding=${source.encoding} +javadoc.noindex=false +javadoc.nonavbar=false +javadoc.notree=false +javadoc.private=false +javadoc.splitindex=true +javadoc.use=true +javadoc.version=false +javadoc.windowtitle= +main.class=kademlia.KademliaBasic +manifest.file=manifest.mf +meta.inf.dir=${src.dir}/META-INF +mkdist.disabled=false +platform.active=default_platform +run.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +# Space-separated list of JVM arguments used when running the project. +# You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. +# To set system properties for unit tests define test-sys-prop.name=value: +run.jvmargs= +run.test.classpath=\ + ${javac.test.classpath}:\ + ${build.test.classes.dir} +source.encoding=UTF-8 +src.dir=src +test.src.dir=test diff --git a/nbproject/project.xml b/nbproject/project.xml new file mode 100644 index 0000000..29e8f9a --- /dev/null +++ b/nbproject/project.xml @@ -0,0 +1,15 @@ + + + org.netbeans.modules.java.j2seproject + + + Kademlia + + + + + + + + + diff --git a/src/kademlia/KadStatistician.java b/src/kademlia/KadStatistician.java new file mode 100644 index 0000000..2f409e1 --- /dev/null +++ b/src/kademlia/KadStatistician.java @@ -0,0 +1,81 @@ +package kademlia; + +/** + * Specification for class that keeps statistics for a Kademlia instance. + * + * These statistics are temporary and will be lost when Kad is shut down. + * + * @author Joshua Kissoon + * @since 20140507 + */ +public interface KadStatistician +{ + + /** + * Used to indicate some data is sent + * + * @param size The size of the data sent + */ + public void sentData(long size); + + /** + * @return The total data sent in KiloBytes + */ + public long getTotalDataSent(); + + /** + * Used to indicate some data was received + * + * @param size The size of the data received + */ + public void receivedData(long size); + + /** + * @return The total data received in KiloBytes + */ + public long getTotalDataReceived(); + + /** + * Sets the bootstrap time for this Kademlia Node + * + * @param time The bootstrap time in nanoseconds + */ + public void setBootstrapTime(long time); + + /** + * @return How long the system took to bootstrap in milliseconds + */ + public long getBootstrapTime(); + + /** + * Add the timing for a new content lookup operation that took place + * + * @param time The time the content lookup took in nanoseconds + * @param routeLength The length of the route it took to get the content + */ + public void addContentLookup(long time, int routeLength); + + /** + * @return The total number of content lookups performed. + */ + public int numContentLookups(); + + /** + * @return The total time spent on content lookups. + */ + public long totalContentLookupTime(); + + /** + * Compute the average time a content lookup took + * + * @return The average time in milliseconds + */ + public double averageContentLookupTime(); + + /** + * Compute the average route length of content lookup operations. + * + * @return The average route length + */ + public double averageContentLookupRouteLength(); +} diff --git a/src/kademlia/KademliaNode.java b/src/kademlia/KademliaNode.java new file mode 100644 index 0000000..586e94c --- /dev/null +++ b/src/kademlia/KademliaNode.java @@ -0,0 +1,503 @@ +package kademlia; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.util.NoSuchElementException; +import java.util.Timer; +import java.util.TimerTask; +import kademlia.core.DefaultConfiguration; +import kademlia.dht.GetParameter; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; +import kademlia.dht.KadContent; +import kademlia.dht.StorageEntry; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.exceptions.RoutingException; +import kademlia.message.MessageFactory; +import kademlia.node.Node; +import kademlia.node.KademliaId; +import kademlia.operation.ConnectOperation; +import kademlia.operation.ContentLookupOperation; +import kademlia.operation.Operation; +import kademlia.operation.KadRefreshOperation; +import kademlia.operation.StoreOperation; +import kademlia.routing.RoutingTable; +import kademlia.util.serializer.JsonDHTSerializer; +import kademlia.util.serializer.JsonRoutingTableSerializer; +import kademlia.util.serializer.JsonSerializer; + +/** + * The main Kademlia Node on the network, this node manages everything for this local system. + * + * @author Joshua Kissoon + * @since 20140215 + * + * @todo When we receive a store message - if we have a newer version of the content, re-send this newer version to that node so as to update their version + * @todo Handle IPv6 Addresses + * + */ +public class KademliaNode +{ + + /* Kademlia Attributes */ + private final String ownerId; + + /* Objects to be used */ + private final transient Node localNode; + private final transient KadServer server; + private final transient DHT dht; + private transient RoutingTable routingTable; + private final int udpPort; + private transient KadConfiguration config; + + /* Timer used to execute refresh operations */ + private transient Timer refreshOperationTimer; + private transient TimerTask refreshOperationTTask; + + /* Factories */ + private final transient MessageFactory messageFactory; + + /* Statistics */ + private final transient KadStatistician statistician; + + + { + statistician = new Statistician(); + } + + /** + * Creates a Kademlia DistributedMap using the specified name as filename base. + * If the id cannot be read from disk the specified defaultId is used. + * The instance is bootstraped to an existing network by specifying the + * address of a bootstrap node in the network. + * + * @param ownerId The Name of this node used for storage + * @param localNode The Local Node for this Kad instance + * @param udpPort The UDP port to use for routing messages + * @param dht The DHT for this instance + * @param config + * @param routingTable + * + * @throws IOException If an error occurred while reading id or local map + * from disk or a network error occurred while + * attempting to bootstrap to the network + * */ + public KademliaNode(String ownerId, Node localNode, int udpPort, DHT dht, RoutingTable routingTable, KadConfiguration config) throws IOException + { + this.ownerId = ownerId; + this.udpPort = udpPort; + this.localNode = localNode; + this.dht = dht; + this.config = config; + this.routingTable = routingTable; + this.messageFactory = new MessageFactory(this, this.dht, this.config); + this.server = new KadServer(udpPort, this.messageFactory, this.localNode, this.config, this.statistician); + this.startRefreshOperation(); + } + + /** + * Schedule the recurring refresh operation + */ + public final void startRefreshOperation() + { + this.refreshOperationTimer = new Timer(true); + refreshOperationTTask = new TimerTask() + { + @Override + public void run() + { + try + { + /* Runs a DHT RefreshOperation */ + KademliaNode.this.refresh(); + } + catch (IOException e) + { + System.err.println("KademliaNode: Refresh Operation Failed; Message: " + e.getMessage()); + } + } + }; + refreshOperationTimer.schedule(refreshOperationTTask, this.config.restoreInterval(), this.config.restoreInterval()); + } + + public final void stopRefreshOperation() + { + /* Close off the timer tasks */ + this.refreshOperationTTask.cancel(); + this.refreshOperationTimer.cancel(); + this.refreshOperationTimer.purge(); + } + + public KademliaNode(String ownerId, Node node, int udpPort, RoutingTable routingTable, KadConfiguration config) throws IOException + { + this( + ownerId, + node, + udpPort, + new DHT(ownerId, config), + routingTable, + config + ); + } + + public KademliaNode(String ownerId, Node node, int udpPort, KadConfiguration config) throws IOException + { + this( + ownerId, + node, + udpPort, + new RoutingTable(node, config), + config + ); + } + + public KademliaNode(String ownerId, KademliaId defaultId, int udpPort) throws IOException + { + this( + ownerId, + new Node(defaultId, InetAddress.getLocalHost(), udpPort), + udpPort, + new DefaultConfiguration() + ); + } + + /** + * Load Stored state using default configuration + * + * @param ownerId The ID of the owner for the stored state + * + * @return A Kademlia instance loaded from a stored state in a file + * + * @throws java.io.FileNotFoundException + * @throws java.lang.ClassNotFoundException + */ + public static KademliaNode loadFromFile(String ownerId) throws FileNotFoundException, IOException, ClassNotFoundException + { + return KademliaNode.loadFromFile(ownerId, new DefaultConfiguration()); + } + + /** + * Load Stored state + * + * @param ownerId The ID of the owner for the stored state + * @param iconfig Configuration information to work with + * + * @return A Kademlia instance loaded from a stored state in a file + * + * @throws java.io.FileNotFoundException + * @throws java.lang.ClassNotFoundException + */ + public static KademliaNode loadFromFile(String ownerId, KadConfiguration iconfig) throws FileNotFoundException, IOException, ClassNotFoundException + { + DataInputStream din; + + /** + * @section Read Basic Kad data + */ + din = new DataInputStream(new FileInputStream(getStateStorageFolderName(ownerId, iconfig) + File.separator + "kad.kns")); + KademliaNode ikad = new JsonSerializer().read(din); + + /** + * @section Read the routing table + */ + din = new DataInputStream(new FileInputStream(getStateStorageFolderName(ownerId, iconfig) + File.separator + "routingtable.kns")); + RoutingTable irtbl = new JsonRoutingTableSerializer(iconfig).read(din); + + /** + * @section Read the node state + */ + din = new DataInputStream(new FileInputStream(getStateStorageFolderName(ownerId, iconfig) + File.separator + "node.kns")); + Node inode = new JsonSerializer().read(din); + + /** + * @section Read the DHT + */ + din = new DataInputStream(new FileInputStream(getStateStorageFolderName(ownerId, iconfig) + File.separator + "dht.kns")); + DHT idht = new JsonDHTSerializer().read(din); + idht.setConfiguration(iconfig); + + return new KademliaNode(ownerId, inode, ikad.getPort(), idht, irtbl, iconfig); + } + + /** + * @return Node The local node for this system + */ + public Node getNode() + { + return this.localNode; + } + + /** + * @return The KadServer used to send/receive messages + */ + public KadServer getServer() + { + return this.server; + } + + /** + * @return The DHT for this kad instance + */ + public DHT getDHT() + { + return this.dht; + } + + /** + * @return The current KadConfiguration object being used + */ + public KadConfiguration getCurrentConfiguration() + { + return this.config; + } + + /** + * Connect to an existing peer-to-peer network. + * + * @param n The known node in the peer-to-peer network + * + * @throws RoutingException If the bootstrap node could not be contacted + * @throws IOException If a network error occurred + * @throws IllegalStateException If this object is closed + * */ + public synchronized final void bootstrap(Node n) throws IOException, RoutingException + { + long startTime = System.nanoTime(); + Operation op = new ConnectOperation(this.server, this, n, this.config); + op.execute(); + long endTime = System.nanoTime(); + this.statistician.setBootstrapTime(endTime - startTime); + } + + /** + * Stores the specified value under the given key + * This value is stored on K nodes on the network, or all nodes if there are > K total nodes in the network + * + * @param content The content to put onto the DHT + * + * @return Integer How many nodes the content was stored on + * + * @throws java.io.IOException + * + */ + public int put(KadContent content) throws IOException + { + return this.put(new StorageEntry(content)); + } + + /** + * Stores the specified value under the given key + * This value is stored on K nodes on the network, or all nodes if there are > K total nodes in the network + * + * @param entry The StorageEntry with the content to put onto the DHT + * + * @return Integer How many nodes the content was stored on + * + * @throws java.io.IOException + * + */ + private int put(StorageEntry entry) throws IOException + { + StoreOperation sop = new StoreOperation(this.server, this, entry, this.dht, this.config); + sop.execute(); + + /* Return how many nodes the content was stored on */ + return sop.numNodesStoredAt(); + } + + /** + * Store a content on the local node's DHT + * + * @param content The content to put on the DHT + * + * @throws java.io.IOException + */ + public void putLocally(KadContent content) throws IOException + { + this.dht.store(new StorageEntry(content)); + } + + /** + * Get some content stored on the DHT + * + * @param param The parameters used to search for the content + * + * @return DHTContent The content + * + * @throws java.io.IOException + * @throws kademlia.exceptions.ContentNotFoundException + */ + public StorageEntry get(GetParameter param) throws NoSuchElementException, IOException, ContentNotFoundException + { + if (this.dht.contains(param)) + { + /* If the content exist in our own DHT, then return it. */ + return this.dht.get(param); + } + + /* Seems like it doesn't exist in our DHT, get it from other Nodes */ + long startTime = System.nanoTime(); + ContentLookupOperation clo = new ContentLookupOperation(server, this, param, this.config); + clo.execute(); + long endTime = System.nanoTime(); + this.statistician.addContentLookup(endTime - startTime, clo.routeLength()); + return clo.getContentFound(); + } + + /** + * Allow the user of the System to call refresh even out of the normal Kad refresh timing + * + * @throws java.io.IOException + */ + public void refresh() throws IOException + { + new KadRefreshOperation(this.server, this, this.dht, this.config).execute(); + } + + /** + * @return String The ID of the owner of this local network + */ + public String getOwnerId() + { + return this.ownerId; + } + + /** + * @return Integer The port on which this kad instance is running + */ + public int getPort() + { + return this.udpPort; + } + + /** + * Here we handle properly shutting down the Kademlia instance + * + * @param saveState Whether to save the application state or not + * + * @throws java.io.FileNotFoundException + */ + public void shutdown(final boolean saveState) throws IOException + { + /* Shut down the server */ + this.server.shutdown(); + + this.stopRefreshOperation(); + + /* Save this Kademlia instance's state if required */ + if (saveState) + { + /* Save the system state */ + this.saveKadState(); + } + } + + /** + * Saves the node state to a text file + * + * @throws java.io.FileNotFoundException + */ + private void saveKadState() throws IOException + { + DataOutputStream dout; + + /** + * @section Store Basic Kad data + */ + dout = new DataOutputStream(new FileOutputStream(getStateStorageFolderName(this.ownerId, this.config) + File.separator + "kad.kns")); + new JsonSerializer().write(this, dout); + + /** + * @section Save the node state + */ + dout = new DataOutputStream(new FileOutputStream(getStateStorageFolderName(this.ownerId, this.config) + File.separator + "node.kns")); + new JsonSerializer().write(this.localNode, dout); + + /** + * @section Save the routing table + * We need to save the routing table separate from the node since the routing table will contain the node and the node will contain the routing table + * This will cause a serialization recursion, and in turn a Stack Overflow + */ + dout = new DataOutputStream(new FileOutputStream(getStateStorageFolderName(this.ownerId, this.config) + File.separator + "routingtable.kns")); + new JsonRoutingTableSerializer(this.config).write(this.getRoutingTable(), dout); + + /** + * @section Save the DHT + */ + dout = new DataOutputStream(new FileOutputStream(getStateStorageFolderName(this.ownerId, this.config) + File.separator + "dht.kns")); + new JsonDHTSerializer().write(this.dht, dout); + + } + + /** + * Get the name of the folder for which a content should be stored + * + * @return String The name of the folder to store node states + */ + private static String getStateStorageFolderName(String ownerId, KadConfiguration iconfig) + { + /* Setup the nodes storage folder if it doesn't exist */ + String path = iconfig.getNodeDataFolder(ownerId) + File.separator + "nodeState"; + File nodeStateFolder = new File(path); + if (!nodeStateFolder.isDirectory()) + { + nodeStateFolder.mkdir(); + } + return nodeStateFolder.toString(); + } + + /** + * @return The routing table for this node. + */ + public RoutingTable getRoutingTable() + { + return this.routingTable; + } + + /** + * @return The statistician that manages all statistics + */ + public KadStatistician getStatistician() + { + return this.statistician; + } + + /** + * Creates a string containing all data about this Kademlia instance + * + * @return The string representation of this Kad instance + */ + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("\n\nPrinting Kad State for instance with owner: "); + sb.append(this.ownerId); + sb.append("\n\n"); + + sb.append("\n"); + sb.append("Local Node"); + sb.append(this.localNode); + sb.append("\n"); + + sb.append("\n"); + sb.append("Routing Table: "); + sb.append(this.getRoutingTable()); + sb.append("\n"); + + sb.append("\n"); + sb.append("DHT: "); + sb.append(this.dht); + sb.append("\n"); + + sb.append("\n\n\n"); + + return sb.toString(); + } +} diff --git a/src/kademlia/Statistician.java b/src/kademlia/Statistician.java new file mode 100644 index 0000000..679dd29 --- /dev/null +++ b/src/kademlia/Statistician.java @@ -0,0 +1,151 @@ +package kademlia; + +import java.text.DecimalFormat; + +/** + * Class that keeps statistics for this Kademlia instance. + * + * These statistics are temporary and will be lost when Kad is shut down. + * + * @author Joshua Kissoon + * @since 20140505 + */ +public class Statistician implements KadStatistician +{ + + /* How much data was sent and received by the server over the network */ + private long totalDataSent, totalDataReceived; + private long numDataSent, numDataReceived; + + /* Bootstrap timings */ + private long bootstrapTime; + + /* Content lookup operation timing & route length */ + private int numContentLookups; + private long totalContentLookupTime; + private long totalRouteLength; + + + { + this.totalDataSent = 0; + this.totalDataReceived = 0; + this.bootstrapTime = 0; + this.numContentLookups = 0; + this.totalContentLookupTime = 0; + this.totalRouteLength = 0; + } + + @Override + public void sentData(long size) + { + this.totalDataSent += size; + this.numDataSent++; + } + + @Override + public long getTotalDataSent() + { + return this.totalDataSent / 1000L; + } + + @Override + public void receivedData(long size) + { + this.totalDataReceived += size; + this.numDataReceived++; + } + + @Override + public long getTotalDataReceived() + { + return this.totalDataReceived / 1000L; + } + + @Override + public void setBootstrapTime(long time) + { + this.bootstrapTime = time; + } + + @Override + public long getBootstrapTime() + { + return this.bootstrapTime / 1000000L; + } + + @Override + public void addContentLookup(long time, int routeLength) + { + this.numContentLookups++; + this.totalContentLookupTime += time; + this.totalRouteLength += routeLength; + } + + @Override + public int numContentLookups() + { + return this.numContentLookups; + } + + @Override + public long totalContentLookupTime() + { + return this.totalContentLookupTime; + } + + @Override + public double averageContentLookupTime() + { + double avg = (double) ((double) this.totalContentLookupTime / (double) this.numContentLookups) / 1000000D; + DecimalFormat df = new DecimalFormat("#.00"); + return new Double(df.format(avg)); + } + + @Override + public double averageContentLookupRouteLength() + { + double avg = (double) ((double) this.totalRouteLength / (double) this.numContentLookups); + DecimalFormat df = new DecimalFormat("#.00"); + return new Double(df.format(avg)); + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("Statistician: ["); + + sb.append("Bootstrap Time: "); + sb.append(this.getBootstrapTime()); + sb.append("; "); + + sb.append("Data Sent: "); + sb.append("("); + sb.append(this.numDataSent); + sb.append(") "); + sb.append(this.getTotalDataSent()); + sb.append(" bytes; "); + + sb.append("Data Received: "); + sb.append("("); + sb.append(this.numDataReceived); + sb.append(") "); + sb.append(this.getTotalDataReceived()); + sb.append(" bytes; "); + + sb.append("Num Content Lookups: "); + sb.append(this.numContentLookups()); + sb.append("; "); + + sb.append("Avg Content Lookup Time: "); + sb.append(this.averageContentLookupTime()); + sb.append("; "); + + sb.append("Avg Content Lookup Route Lth: "); + sb.append(this.averageContentLookupRouteLength()); + sb.append("; "); + + sb.append("]"); + + return sb.toString(); + } +} diff --git a/src/kademlia/Todo b/src/kademlia/Todo new file mode 100644 index 0000000..798da63 --- /dev/null +++ b/src/kademlia/Todo @@ -0,0 +1,4 @@ +# What's left to add to the implementation + +1. Implement the "Optimized Contact Accounting" features as mentioned in the paper +2. Implement the "Accelerated Lookups" featured as described in the paper \ No newline at end of file diff --git a/src/kademlia/core/DefaultConfiguration.java b/src/kademlia/core/DefaultConfiguration.java new file mode 100644 index 0000000..8f56aca --- /dev/null +++ b/src/kademlia/core/DefaultConfiguration.java @@ -0,0 +1,101 @@ +package kademlia.core; + +import java.io.File; + +/** + * A set of Kademlia configuration parameters. Default values are + * supplied and can be changed by the application as necessary. + * + */ +public class DefaultConfiguration implements KadConfiguration +{ + + private final static long RESTORE_INTERVAL = 60 * 1000; // in milliseconds + private final static long RESPONSE_TIMEOUT = 2000; + private final static long OPERATION_TIMEOUT = 2000; + private final static int CONCURRENCY = 10; + private final static int K = 5; + private final static int RCSIZE = 3; + private final static int STALE = 1; + private final static String LOCAL_FOLDER = "kademlia"; + + private final static boolean IS_TESTING = true; + + /** + * Default constructor to support Gson Serialization + */ + public DefaultConfiguration() + { + + } + + @Override + public long restoreInterval() + { + return RESTORE_INTERVAL; + } + + @Override + public long responseTimeout() + { + return RESPONSE_TIMEOUT; + } + + @Override + public long operationTimeout() + { + return OPERATION_TIMEOUT; + } + + @Override + public int maxConcurrentMessagesTransiting() + { + return CONCURRENCY; + } + + @Override + public int k() + { + return K; + } + + @Override + public int replacementCacheSize() + { + return RCSIZE; + } + + @Override + public int stale() + { + return STALE; + } + + @Override + public String getNodeDataFolder(String ownerId) + { + /* Setup the main storage folder if it doesn't exist */ + String path = System.getProperty("user.home") + File.separator + DefaultConfiguration.LOCAL_FOLDER; + File folder = new File(path); + if (!folder.isDirectory()) + { + folder.mkdir(); + } + + /* Setup subfolder for this owner if it doesn't exist */ + File ownerFolder = new File(folder + File.separator + ownerId); + if (!ownerFolder.isDirectory()) + { + ownerFolder.mkdir(); + } + + /* Return the path */ + return ownerFolder.toString(); + } + + @Override + public boolean isTesting() + { + return IS_TESTING; + } +} diff --git a/src/kademlia/core/KadConfiguration.java b/src/kademlia/core/KadConfiguration.java new file mode 100644 index 0000000..e77b65e --- /dev/null +++ b/src/kademlia/core/KadConfiguration.java @@ -0,0 +1,63 @@ +package kademlia.core; + +/** + * Interface that defines a KadConfiguration object + * + * @author Joshua Kissoon + * @since 20140329 + */ +public interface KadConfiguration +{ + + /** + * @return Interval in milliseconds between execution of RestoreOperations. + */ + public long restoreInterval(); + + /** + * If no reply received from a node in this period (in milliseconds) + * consider the node unresponsive. + * + * @return The time it takes to consider a node unresponsive + */ + public long responseTimeout(); + + /** + * @return Maximum number of milliseconds for performing an operation. + */ + public long operationTimeout(); + + /** + * @return Maximum number of concurrent messages in transit. + */ + public int maxConcurrentMessagesTransiting(); + + /** + * @return K-Value used throughout Kademlia + */ + public int k(); + + /** + * @return Size of replacement cache. + */ + public int replacementCacheSize(); + + /** + * @return # of times a node can be marked as stale before it is actually removed. + */ + public int stale(); + + /** + * Creates the folder in which this node data is to be stored. + * + * @param ownerId + * + * @return The folder path + */ + public String getNodeDataFolder(String ownerId); + + /** + * @return Whether we're in a testing or production system. + */ + public boolean isTesting(); +} diff --git a/src/kademlia/core/KadServer.java b/src/kademlia/core/KadServer.java new file mode 100644 index 0000000..fe1a846 --- /dev/null +++ b/src/kademlia/core/KadServer.java @@ -0,0 +1,352 @@ +package kademlia.core; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.SocketException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; +import kademlia.KadStatistician; +import kademlia.exceptions.KadServerDownException; +import kademlia.message.Message; +import kademlia.message.MessageFactory; +import kademlia.node.Node; +import kademlia.message.Receiver; + +/** + * The server that handles sending and receiving messages between nodes on the Kad Network + * + * @author Joshua Kissoon + * @created 20140215 + */ +public class KadServer +{ + + /* Maximum size of a Datagram Packet */ + private static final int DATAGRAM_BUFFER_SIZE = 64 * 1024; // 64KB + + /* Basic Kad Objects */ + private final transient KadConfiguration config; + + /* Server Objects */ + private final DatagramSocket socket; + private transient boolean isRunning; + private final Map receivers; + private final Timer timer; // Schedule future tasks + private final Map tasks; // Keep track of scheduled tasks + + private final Node localNode; + + /* Factories */ + private final MessageFactory messageFactory; + + private final KadStatistician statistician; + + + { + isRunning = true; + this.tasks = new HashMap<>(); + this.receivers = new HashMap<>(); + this.timer = new Timer(true); + } + + /** + * Initialize our KadServer + * + * @param udpPort The port to listen on + * @param mFactory Factory used to create messages + * @param localNode Local node on which this server runs on + * @param config + * @param statistician A statistician to manage the server statistics + * + * @throws java.net.SocketException + */ + public KadServer(int udpPort, MessageFactory mFactory, Node localNode, KadConfiguration config, KadStatistician statistician) throws SocketException + { + this.config = config; + this.socket = new DatagramSocket(udpPort); + this.localNode = localNode; + this.messageFactory = mFactory; + this.statistician = statistician; + + /* Start listening for incoming requests in a new thread */ + this.startListener(); + } + + /** + * Starts the listener to listen for incoming messages + */ + private void startListener() + { + new Thread() + { + @Override + public void run() + { + listen(); + } + }.start(); + } + + /** + * Sends a message + * + * @param msg The message to send + * @param to The node to send the message to + * @param recv The receiver to handle the response message + * + * @return Integer The communication ID of this message + * + * @throws IOException + * @throws kademlia.exceptions.KadServerDownException + */ + public synchronized int sendMessage(Node to, Message msg, Receiver recv) throws IOException, KadServerDownException + { + if (!isRunning) + { + throw new KadServerDownException(this.localNode + " - Kad Server is not running."); + } + + /* Generate a random communication ID */ + int comm = new Random().nextInt(); + + /* If we have a receiver */ + if (recv != null) + { + try + { + /* Setup the receiver to handle message response */ + receivers.put(comm, recv); + TimerTask task = new TimeoutTask(comm, recv); + timer.schedule(task, this.config.responseTimeout()); + tasks.put(comm, task); + } + catch (IllegalStateException ex) + { + /* The timer is already cancelled so we cannot do anything here really */ + } + } + + /* Send the message */ + sendMessage(to, msg, comm); + + return comm; + } + + /** + * Method called to reply to a message received + * + * @param to The Node to send the reply to + * @param msg The reply message + * @param comm The communication ID - the one received + * + * @throws java.io.IOException + */ + public synchronized void reply(Node to, Message msg, int comm) throws IOException + { + if (!isRunning) + { + throw new IllegalStateException("Kad Server is not running."); + } + sendMessage(to, msg, comm); + } + + /** + * Internal sendMessage method called by the public sendMessage method after a communicationId is generated + */ + private void sendMessage(Node to, Message msg, int comm) throws IOException + { + /* Use a try-with resource to auto-close streams after usage */ + try (ByteArrayOutputStream bout = new ByteArrayOutputStream(); DataOutputStream dout = new DataOutputStream(bout);) + { + /* Setup the message for transmission */ + dout.writeInt(comm); + dout.writeByte(msg.code()); + msg.toStream(dout); + dout.close(); + + byte[] data = bout.toByteArray(); + + if (data.length > DATAGRAM_BUFFER_SIZE) + { + throw new IOException("Message is too big"); + } + + /* Everything is good, now create the packet and send it */ + DatagramPacket pkt = new DatagramPacket(data, 0, data.length); + pkt.setSocketAddress(to.getSocketAddress()); + socket.send(pkt); + + /* Lets inform the statistician that we've sent some data */ + this.statistician.sentData(data.length); + } + } + + /** + * Listen for incoming messages in a separate thread + */ + private void listen() + { + try + { + while (isRunning) + { + try + { + /* Wait for a packet */ + byte[] buffer = new byte[DATAGRAM_BUFFER_SIZE]; + DatagramPacket packet = new DatagramPacket(buffer, buffer.length); + socket.receive(packet); + + /* Lets inform the statistician that we've received some data */ + this.statistician.receivedData(packet.getLength()); + + if (this.config.isTesting()) + { + /** + * Simulating network latency + * We pause for 1 millisecond/100 bytes + */ + int pause = packet.getLength() / 100; + try + { + Thread.sleep(pause); + } + catch (InterruptedException ex) + { + + } + } + + /* We've received a packet, now handle it */ + try (ByteArrayInputStream bin = new ByteArrayInputStream(packet.getData(), packet.getOffset(), packet.getLength()); + DataInputStream din = new DataInputStream(bin);) + { + + /* Read in the conversation Id to know which handler to handle this response */ + int comm = din.readInt(); + byte messCode = din.readByte(); + + Message msg = messageFactory.createMessage(messCode, din); + din.close(); + + /* Get a receiver for this message */ + Receiver receiver; + if (this.receivers.containsKey(comm)) + { + /* If there is a reciever in the receivers to handle this */ + synchronized (this) + { + receiver = this.receivers.remove(comm); + TimerTask task = (TimerTask) tasks.remove(comm); + if (task != null) + { + task.cancel(); + } + } + } + else + { + /* There is currently no receivers, try to get one */ + receiver = messageFactory.createReceiver(messCode, this); + } + + /* Invoke the receiver */ + if (receiver != null) + { + receiver.receive(msg, comm); + } + } + } + catch (IOException e) + { + //this.isRunning = false; + System.err.println("Server ran into a problem in listener method. Message: " + e.getMessage()); + } + } + } + finally + { + if (!socket.isClosed()) + { + socket.close(); + } + this.isRunning = false; + } + } + + /** + * Remove a conversation receiver + * + * @param comm The id of this conversation + */ + private synchronized void unregister(int comm) + { + receivers.remove(comm); + this.tasks.remove(comm); + } + + /** + * Stops listening and shuts down the server + */ + public synchronized void shutdown() + { + this.isRunning = false; + this.socket.close(); + timer.cancel(); + } + + /** + * Task that gets called by a separate thread if a timeout for a receiver occurs. + * When a reply arrives this task must be canceled using the cancel() + * method inherited from TimerTask. In this case the caller is + * responsible for removing the task from the tasks map. + * */ + class TimeoutTask extends TimerTask + { + + private final int comm; + private final Receiver recv; + + public TimeoutTask(int comm, Receiver recv) + { + this.comm = comm; + this.recv = recv; + } + + @Override + public void run() + { + if (!KadServer.this.isRunning) + { + return; + } + + try + { + unregister(comm); + recv.timeout(comm); + } + catch (IOException e) + { + System.err.println("Cannot unregister a receiver. Message: " + e.getMessage()); + } + } + } + + public void printReceivers() + { + for (Integer r : this.receivers.keySet()) + { + System.out.println("Receiver for comm: " + r + "; Receiver: " + this.receivers.get(r)); + } + } + +} diff --git a/src/kademlia/dht/DHT.java b/src/kademlia/dht/DHT.java new file mode 100644 index 0000000..051387d --- /dev/null +++ b/src/kademlia/dht/DHT.java @@ -0,0 +1,325 @@ +package kademlia.dht; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.NoSuchElementException; +import kademlia.core.KadConfiguration; +import kademlia.exceptions.ContentExistException; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.node.KademliaId; +import kademlia.util.serializer.JsonSerializer; +import kademlia.util.serializer.KadSerializer; + +/** + * The main Distributed Hash Table class that manages the entire DHT + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class DHT +{ + + private transient StoredContentManager contentManager; + private transient KadSerializer serializer = null; + private transient KadConfiguration config; + + private final String ownerId; + + public DHT(String ownerId, KadConfiguration config) + { + this.ownerId = ownerId; + this.config = config; + this.initialize(); + } + + /** + * Initialize this DHT to it's default state + */ + public final void initialize() + { + contentManager = new StoredContentManager(); + } + + /** + * Set a new configuration. Mainly used when we restore the DHT state from a file + * + * @param con The new configuration file + */ + public void setConfiguration(KadConfiguration con) + { + this.config = con; + } + + /** + * Creates a new Serializer or returns an existing serializer + * + * @return The new ContentSerializer + */ + public KadSerializer getSerializer() + { + if (null == serializer) + { + serializer = new JsonSerializer<>(); + } + + return serializer; + } + + /** + * Handle storing content locally + * + * @param content The DHT content to store + * + * @return boolean true if we stored the content, false if the content already exists and is up to date + * + * @throws java.io.IOException + */ + public boolean store(StorageEntry content) throws IOException + { + /* Lets check if we have this content and it's the updated version */ + if (this.contentManager.contains(content.getContentMetadata())) + { + StorageEntryMetadata current = this.contentManager.get(content.getContentMetadata()); + + /* update the last republished time */ + current.updateLastRepublished(); + + if (current.getLastUpdatedTimestamp() >= content.getContentMetadata().getLastUpdatedTimestamp()) + { + /* We have the current content, no need to update it! just leave this method now */ + return false; + } + else + { + /* We have this content, but not the latest version, lets delete it so the new version will be added below */ + try + { + //System.out.println("Removing older content to update it"); + this.remove(content.getContentMetadata()); + } + catch (ContentNotFoundException ex) + { + /* This won't ever happen at this point since we only get here if the content is found, lets ignore it */ + } + } + } + + /** + * If we got here means we don't have this content, or we need to update the content + * If we need to update the content, the code above would've already deleted it, so we just need to re-add it + */ + try + { + //System.out.println("Adding new content."); + /* Keep track of this content in the entries manager */ + StorageEntryMetadata sEntry = this.contentManager.put(content.getContentMetadata()); + + /* Now we store the content locally in a file */ + String contentStorageFolder = this.getContentStorageFolderName(content.getContentMetadata().getKey()); + + try (FileOutputStream fout = new FileOutputStream(contentStorageFolder + File.separator + sEntry.hashCode() + ".kct"); + DataOutputStream dout = new DataOutputStream(fout)) + { + this.getSerializer().write(content, dout); + } + return true; + } + catch (ContentExistException e) + { + /** + * Content already exist on the DHT + * This won't happen because above takes care of removing the content if it's older and needs to be updated, + * or returning if we already have the current content version. + */ + return false; + } + } + + public boolean store(KadContent content) throws IOException + { + return this.store(new StorageEntry(content)); + } + + /** + * Retrieves a Content from local storage + * + * @param key The Key of the content to retrieve + * @param hashCode The hash code of the content to retrieve + * + * @return A KadContent object + */ + private StorageEntry retrieve(KademliaId key, int hashCode) throws FileNotFoundException, IOException, ClassNotFoundException + { + String folder = this.getContentStorageFolderName(key); + DataInputStream din = new DataInputStream(new FileInputStream(folder + File.separator + hashCode + ".kct")); + return this.getSerializer().read(din); + } + + /** + * Check if any content for the given criteria exists in this DHT + * + * @param param The content search criteria + * + * @return boolean Whether any content exist that satisfy the criteria + */ + public boolean contains(GetParameter param) + { + return this.contentManager.contains(param); + } + + /** + * Retrieve and create a KadContent object given the StorageEntry object + * + * @param entry The StorageEntry used to retrieve this content + * + * @return KadContent The content object + * + * @throws java.io.IOException + */ + public StorageEntry get(StorageEntryMetadata entry) throws IOException, NoSuchElementException + { + try + { + return this.retrieve(entry.getKey(), entry.hashCode()); + } + catch (FileNotFoundException e) + { + System.err.println("Error while loading file for content. Message: " + e.getMessage()); + } + catch (ClassNotFoundException e) + { + System.err.println("The class for some content was not found. Message: " + e.getMessage()); + } + + /* If we got here, means we got no entries */ + throw new NoSuchElementException(); + } + + /** + * Get the StorageEntry for the content if any exist, + * retrieve the KadContent from the storage system and return it + * + * @param param The parameters used to filter the content needed + * + * @return KadContent A KadContent found on the DHT satisfying the given criteria + * + * @throws java.io.IOException + */ + public StorageEntry get(GetParameter param) throws NoSuchElementException, IOException + { + /* Load a KadContent if any exist for the given criteria */ + try + { + StorageEntryMetadata e = this.contentManager.get(param); + return this.retrieve(e.getKey(), e.hashCode()); + } + catch (FileNotFoundException e) + { + System.err.println("Error while loading file for content. Message: " + e.getMessage()); + } + catch (ClassNotFoundException e) + { + System.err.println("The class for some content was not found. Message: " + e.getMessage()); + } + + /* If we got here, means we got no entries */ + throw new NoSuchElementException(); + } + + /** + * Delete a content from local storage + * + * @param content The Content to Remove + * + * + * @throws kademlia.exceptions.ContentNotFoundException + */ + public void remove(KadContent content) throws ContentNotFoundException + { + this.remove(new StorageEntryMetadata(content)); + } + + public void remove(StorageEntryMetadata entry) throws ContentNotFoundException + { + String folder = this.getContentStorageFolderName(entry.getKey()); + File file = new File(folder + File.separator + entry.hashCode() + ".kct"); + + contentManager.remove(entry); + + if (file.exists()) + { + file.delete(); + } + else + { + throw new ContentNotFoundException(); + } + } + + /** + * Get the name of the folder for which a content should be stored + * + * @param key The key of the content + * + * @return String The name of the folder + */ + private String getContentStorageFolderName(KademliaId key) + { + /** + * Each content is stored in a folder named after the first 2 characters of the NodeId + * + * The name of the file containing the content is the hash of this content + */ + String folderName = key.hexRepresentation().substring(0, 2); + File contentStorageFolder = new File(this.config.getNodeDataFolder(ownerId) + File.separator + folderName); + + /* Create the content folder if it doesn't exist */ + if (!contentStorageFolder.isDirectory()) + { + contentStorageFolder.mkdir(); + } + + return contentStorageFolder.toString(); + } + + /** + * @return A List of all StorageEntries for this node + */ + public List getStorageEntries() + { + return contentManager.getAllEntries(); + } + + /** + * Used to add a list of storage entries for existing content to the DHT. + * Mainly used when retrieving StorageEntries from a saved state file. + * + * @param ientries The entries to add + */ + public void putStorageEntries(List ientries) + { + for (StorageEntryMetadata e : ientries) + { + try + { + this.contentManager.put(e); + } + catch (ContentExistException ex) + { + /* Entry already exist, no need to store it again */ + } + } + } + + @Override + public synchronized String toString() + { + return this.contentManager.toString(); + } +} diff --git a/src/kademlia/dht/GetParameter.java b/src/kademlia/dht/GetParameter.java new file mode 100644 index 0000000..c1c12cf --- /dev/null +++ b/src/kademlia/dht/GetParameter.java @@ -0,0 +1,117 @@ +package kademlia.dht; + +import kademlia.node.KademliaId; + +/** + * A GET request can get content based on Key, Owner, Type, etc + * + * This is a class containing the parameters to be passed in a GET request + * + * We use a class since the number of filtering parameters can change later + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class GetParameter +{ + + private KademliaId key; + private String ownerId = null; + private String type = null; + + /** + * Construct a GetParameter to search for data by NodeId and owner + * + * @param key + * @param type + */ + public GetParameter(KademliaId key, String type) + { + this.key = key; + this.type = type; + } + + /** + * Construct a GetParameter to search for data by NodeId, owner, type + * + * @param key + * @param type + * @param owner + */ + public GetParameter(KademliaId key, String type, String owner) + { + this(key, owner); + this.type = type; + } + + /** + * Construct our get parameter from a Content + * + * @param c + */ + public GetParameter(KadContent c) + { + this.key = c.getKey(); + + if (c.getType() != null) + { + this.type = c.getType(); + } + + if (c.getOwnerId() != null) + { + this.ownerId = c.getOwnerId(); + } + } + + /** + * Construct our get parameter from a StorageEntryMeta data + * + * @param md + */ + public GetParameter(StorageEntryMetadata md) + { + this.key = md.getKey(); + + if (md.getType() != null) + { + this.type = md.getType(); + } + + if (md.getOwnerId() != null) + { + this.ownerId = md.getOwnerId(); + } + } + + public KademliaId getKey() + { + return this.key; + } + + public void setOwnerId(String ownerId) + { + this.ownerId = ownerId; + } + + public String getOwnerId() + { + return this.ownerId; + } + + public void setType(String type) + { + this.type = type; + } + + public String getType() + { + return this.type; + } + + @Override + public String toString() + { + return "GetParameter - [Key: " + key + "][Owner: " + this.ownerId + "][Type: " + this.type + "]"; + } +} diff --git a/src/kademlia/dht/KadContent.java b/src/kademlia/dht/KadContent.java new file mode 100644 index 0000000..9cae1dc --- /dev/null +++ b/src/kademlia/dht/KadContent.java @@ -0,0 +1,65 @@ +package kademlia.dht; + +import kademlia.node.KademliaId; + +/** + * Any piece of content that needs to be stored on the DHT + * + * @author Joshua Kissoon + * + * @since 20140224 + */ +public interface KadContent +{ + + /** + * @return NodeId The DHT key for this content + */ + public KademliaId getKey(); + + /** + * @return String The type of content + */ + public String getType(); + + /** + * Each content will have an created date + * This allows systems to know when to delete a content form his/her machine + * + * @return long The create date of this content + */ + public long getCreatedTimestamp(); + + /** + * Each content will have an update timestamp + * This allows the DHT to keep only the latest version of a content + * + * @return long The timestamp of when this content was last updated + */ + public long getLastUpdatedTimestamp(); + + /** + * @return The ID of the owner of this content + */ + public String getOwnerId(); + + /** + * Each content needs to be in byte format for transporting and storage, + * this method takes care of that. + * + * Each object is responsible for transforming itself to byte format since the + * structure of methods may differ. + * + * @return byte[] The content in byte format + */ + public byte[] toBytes(); + + /** + * Given the Content in byte format, read it + * + * @param data The object in byte format + * + * @return A new object from the given byte[] + */ + public KadContent fromBytes(byte[] data); +} diff --git a/src/kademlia/dht/StorageEntry.java b/src/kademlia/dht/StorageEntry.java new file mode 100644 index 0000000..14bdc97 --- /dev/null +++ b/src/kademlia/dht/StorageEntry.java @@ -0,0 +1,35 @@ +package kademlia.dht; + +/** + * A StorageEntry class that is used to store a content on the DHT + * + * @author Joshua Kissoon + * @since 20140402 + */ +public class StorageEntry +{ + + private final String content; + private final StorageEntryMetadata metadata; + + public StorageEntry(KadContent content) + { + this(content, new StorageEntryMetadata(content)); + } + + public StorageEntry(KadContent content, StorageEntryMetadata metadata) + { + this.content = new String(content.toBytes()); + this.metadata = metadata; + } + + public String getContent() + { + return this.content; + } + + public StorageEntryMetadata getContentMetadata() + { + return this.metadata; + } +} diff --git a/src/kademlia/dht/StorageEntryMetadata.java b/src/kademlia/dht/StorageEntryMetadata.java new file mode 100644 index 0000000..6c2d91a --- /dev/null +++ b/src/kademlia/dht/StorageEntryMetadata.java @@ -0,0 +1,144 @@ +package kademlia.dht; + +import java.util.Objects; +import kademlia.node.KademliaId; + +/** + * Keeps track of data for a Content stored in the DHT + * Used by the StorageEntryManager class + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class StorageEntryMetadata +{ + + private final KademliaId key; + private final String ownerId; + private final String type; + private final int contentHash; + private final long updatedTs; + + /* This value is the last time this content was last updated from the network */ + private long lastRepublished; + + public StorageEntryMetadata(KadContent content) + { + this.key = content.getKey(); + this.ownerId = content.getOwnerId(); + this.type = content.getType(); + this.contentHash = content.hashCode(); + this.updatedTs = content.getLastUpdatedTimestamp(); + + this.lastRepublished = System.currentTimeMillis() / 1000L; + } + + public KademliaId getKey() + { + return this.key; + } + + public String getOwnerId() + { + return this.ownerId; + } + + public String getType() + { + return this.type; + } + + public int getContentHash() + { + return this.contentHash; + } + + public long getLastUpdatedTimestamp() + { + return this.updatedTs; + } + + /** + * When a node is looking for content, he sends the search criteria in a GetParameter object + * Here we take this GetParameter object and check if this StorageEntry satisfies the given parameters + * + * @param params + * + * @return boolean Whether this content satisfies the parameters + */ + public boolean satisfiesParameters(GetParameter params) + { + /* Check that owner id matches */ + if ((params.getOwnerId() != null) && (!params.getOwnerId().equals(this.ownerId))) + { + return false; + } + + /* Check that type matches */ + if ((params.getType() != null) && (!params.getType().equals(this.type))) + { + return false; + } + + /* Check that key matches */ + if ((params.getKey() != null) && (!params.getKey().equals(this.key))) + { + return false; + } + + return true; + } + + public long lastRepublished() + { + return this.lastRepublished; + } + + /** + * Whenever we republish a content or get this content from the network, we update the last republished time + */ + public void updateLastRepublished() + { + this.lastRepublished = System.currentTimeMillis() / 1000L; + } + + @Override + public boolean equals(Object o) + { + if (o instanceof StorageEntryMetadata) + { + return this.hashCode() == o.hashCode(); + } + + return false; + } + + @Override + public int hashCode() + { + int hash = 3; + hash = 23 * hash + Objects.hashCode(this.key); + hash = 23 * hash + Objects.hashCode(this.ownerId); + hash = 23 * hash + Objects.hashCode(this.type); + return hash; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("[StorageEntry: "); + + sb.append("{Key: "); + sb.append(this.key); + sb.append("} "); + sb.append("{Owner: "); + sb.append(this.ownerId); + sb.append("} "); + sb.append("{Type: "); + sb.append(this.type); + sb.append("} "); + sb.append("]"); + + return sb.toString(); + } +} diff --git a/src/kademlia/dht/StoredContentManager.java b/src/kademlia/dht/StoredContentManager.java new file mode 100644 index 0000000..c414d52 --- /dev/null +++ b/src/kademlia/dht/StoredContentManager.java @@ -0,0 +1,202 @@ +package kademlia.dht; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import kademlia.exceptions.ContentExistException; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.node.KademliaId; + +/** + * It would be infeasible to keep all content in memory to be send when requested + * Instead we store content into files + * We use this Class to keep track of all content stored + * + * @author Joshua Kissoon + * @since 20140226 + */ +class StoredContentManager +{ + + private final Map> entries; + + + { + entries = new HashMap<>(); + } + + /** + * Add a new entry to our storage + * + * @param content The content to store a reference to + */ + public StorageEntryMetadata put(KadContent content) throws ContentExistException + { + return this.put(new StorageEntryMetadata(content)); + } + + /** + * Add a new entry to our storage + * + * @param entry The StorageEntry to store + */ + public StorageEntryMetadata put(StorageEntryMetadata entry) throws ContentExistException + { + if (!this.entries.containsKey(entry.getKey())) + { + this.entries.put(entry.getKey(), new ArrayList<>()); + } + + /* If this entry doesn't already exist, then we add it */ + if (!this.contains(entry)) + { + this.entries.get(entry.getKey()).add(entry); + + return entry; + } + else + { + throw new ContentExistException("Content already exists on this DHT"); + } + } + + /** + * Checks if our DHT has a Content for the given criteria + * + * @param param The parameters used to search for a content + * + * @return boolean + */ + public synchronized boolean contains(GetParameter param) + { + if (this.entries.containsKey(param.getKey())) + { + /* Content with this key exist, check if any match the rest of the search criteria */ + for (StorageEntryMetadata e : this.entries.get(param.getKey())) + { + /* If any entry satisfies the given parameters, return true */ + if (e.satisfiesParameters(param)) + { + return true; + } + } + } + else + { + } + return false; + } + + /** + * Check if a content exist in the DHT + */ + public synchronized boolean contains(KadContent content) + { + return this.contains(new GetParameter(content)); + } + + /** + * Check if a StorageEntry exist on this DHT + */ + public synchronized boolean contains(StorageEntryMetadata entry) + { + return this.contains(new GetParameter(entry)); + } + + /** + * Checks if our DHT has a Content for the given criteria + * + * @param param The parameters used to search for a content + * + * @return List of content for the specific search parameters + */ + public StorageEntryMetadata get(GetParameter param) throws NoSuchElementException + { + if (this.entries.containsKey(param.getKey())) + { + /* Content with this key exist, check if any match the rest of the search criteria */ + for (StorageEntryMetadata e : this.entries.get(param.getKey())) + { + /* If any entry satisfies the given parameters, return true */ + if (e.satisfiesParameters(param)) + { + return e; + } + } + + /* If we got here, means we didn't find any entry */ + throw new NoSuchElementException(); + } + else + { + throw new NoSuchElementException("No content exist for the given parameters"); + } + } + + public StorageEntryMetadata get(StorageEntryMetadata md) + { + return this.get(new GetParameter(md)); + } + + /** + * @return A list of all storage entries + */ + public synchronized List getAllEntries() + { + List entriesRet = new ArrayList<>(); + + for (List entrySet : this.entries.values()) + { + if (entrySet.size() > 0) + { + entriesRet.addAll(entrySet); + } + } + + return entriesRet; + } + + public void remove(KadContent content) throws ContentNotFoundException + { + this.remove(new StorageEntryMetadata(content)); + } + + public void remove(StorageEntryMetadata entry) throws ContentNotFoundException + { + if (contains(entry)) + { + this.entries.get(entry.getKey()).remove(entry); + } + else + { + throw new ContentNotFoundException("This content does not exist in the Storage Entries"); + } + } + + @Override + public synchronized String toString() + { + StringBuilder sb = new StringBuilder("Stored Content: \n"); + int count = 0; + for (List es : this.entries.values()) + { + if (entries.size() < 1) + { + continue; + } + + for (StorageEntryMetadata e : es) + { + sb.append(++count); + sb.append(". "); + sb.append(e); + sb.append("\n"); + } + } + + sb.append("\n"); + return sb.toString(); + } +} diff --git a/src/kademlia/exceptions/ContentExistException.java b/src/kademlia/exceptions/ContentExistException.java new file mode 100644 index 0000000..96e7c2a --- /dev/null +++ b/src/kademlia/exceptions/ContentExistException.java @@ -0,0 +1,21 @@ +package kademlia.exceptions; + +/** + * An exception used to indicate that a content already exist on the DHT + * + * @author Joshua Kissoon + * @created 20140322 + */ +public class ContentExistException extends Exception +{ + + public ContentExistException() + { + super(); + } + + public ContentExistException(String message) + { + super(message); + } +} diff --git a/src/kademlia/exceptions/ContentNotFoundException.java b/src/kademlia/exceptions/ContentNotFoundException.java new file mode 100644 index 0000000..4b7bf84 --- /dev/null +++ b/src/kademlia/exceptions/ContentNotFoundException.java @@ -0,0 +1,21 @@ +package kademlia.exceptions; + +/** + * An exception used to indicate that a content does not exist on the DHT + * + * @author Joshua Kissoon + * @created 20140322 + */ +public class ContentNotFoundException extends Exception +{ + + public ContentNotFoundException() + { + super(); + } + + public ContentNotFoundException(String message) + { + super(message); + } +} diff --git a/src/kademlia/exceptions/KadServerDownException.java b/src/kademlia/exceptions/KadServerDownException.java new file mode 100644 index 0000000..92cf5d1 --- /dev/null +++ b/src/kademlia/exceptions/KadServerDownException.java @@ -0,0 +1,21 @@ +package kademlia.exceptions; + +/** + * An exception to be thrown whenever the Kad Server is down + * + * @author Joshua Kissoon + * @created 20140428 + */ +public class KadServerDownException extends RoutingException +{ + + public KadServerDownException() + { + super(); + } + + public KadServerDownException(String message) + { + super(message); + } +} diff --git a/src/kademlia/exceptions/RoutingException.java b/src/kademlia/exceptions/RoutingException.java new file mode 100644 index 0000000..aa0697c --- /dev/null +++ b/src/kademlia/exceptions/RoutingException.java @@ -0,0 +1,23 @@ +package kademlia.exceptions; + +import java.io.IOException; + +/** + * An exception to be thrown whenever there is a routing problem + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class RoutingException extends IOException +{ + + public RoutingException() + { + super(); + } + + public RoutingException(String message) + { + super(message); + } +} diff --git a/src/kademlia/exceptions/UnknownMessageException.java b/src/kademlia/exceptions/UnknownMessageException.java new file mode 100644 index 0000000..a7116ec --- /dev/null +++ b/src/kademlia/exceptions/UnknownMessageException.java @@ -0,0 +1,21 @@ +package kademlia.exceptions; + +/** + * An exception used to indicate an unknown message type or communication identifier + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class UnknownMessageException extends RuntimeException +{ + + public UnknownMessageException() + { + super(); + } + + public UnknownMessageException(String message) + { + super(message); + } +} diff --git a/src/kademlia/message/AcknowledgeMessage.java b/src/kademlia/message/AcknowledgeMessage.java new file mode 100644 index 0000000..0f4fef5 --- /dev/null +++ b/src/kademlia/message/AcknowledgeMessage.java @@ -0,0 +1,59 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.node.Node; + +/** + * A message used to acknowledge a request from a node; can be used in many situations. + * - Mainly used to acknowledge a connect message + * + * @author Joshua Kissoon + * @created 20140218 + */ +public class AcknowledgeMessage implements Message +{ + + private Node origin; + public static final byte CODE = 0x01; + + public AcknowledgeMessage(Node origin) + { + this.origin = origin; + } + + public AcknowledgeMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + origin.toStream(out); + } + + public Node getOrigin() + { + return this.origin; + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public String toString() + { + return "AcknowledgeMessage[origin=" + origin.getNodeId() + "]"; + } +} diff --git a/src/kademlia/message/ConnectMessage.java b/src/kademlia/message/ConnectMessage.java new file mode 100644 index 0000000..25629be --- /dev/null +++ b/src/kademlia/message/ConnectMessage.java @@ -0,0 +1,58 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.node.Node; + +/** + * A message sent to another node requesting to connect to them. + * + * @author Joshua Kissoon + * @created 20140218 + */ +public class ConnectMessage implements Message +{ + + private Node origin; + public static final byte CODE = 0x02; + + public ConnectMessage(Node origin) + { + this.origin = origin; + } + + public ConnectMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + origin.toStream(out); + } + + public Node getOrigin() + { + return this.origin; + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public String toString() + { + return "ConnectMessage[origin NodeId=" + origin.getNodeId() + "]"; + } +} diff --git a/src/kademlia/message/ConnectReceiver.java b/src/kademlia/message/ConnectReceiver.java new file mode 100644 index 0000000..236a2d1 --- /dev/null +++ b/src/kademlia/message/ConnectReceiver.java @@ -0,0 +1,58 @@ +package kademlia.message; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadServer; + +/** + * Receives a ConnectMessage and sends an AcknowledgeMessage as reply. + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class ConnectReceiver implements Receiver +{ + + private final KadServer server; + private final KademliaNode localNode; + + public ConnectReceiver(KadServer server, KademliaNode local) + { + this.server = server; + this.localNode = local; + } + + /** + * Handle receiving a ConnectMessage + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public void receive(Message incoming, int comm) throws IOException + { + ConnectMessage mess = (ConnectMessage) incoming; + + /* Update the local space by inserting the origin node. */ + this.localNode.getRoutingTable().insert(mess.getOrigin()); + + /* Respond to the connect request */ + AcknowledgeMessage msg = new AcknowledgeMessage(this.localNode.getNode()); + + /* Reply to the connect message with an Acknowledgement */ + this.server.reply(mess.getOrigin(), msg, comm); + } + + /** + * We don't need to do anything here + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public void timeout(int comm) throws IOException + { + } +} diff --git a/src/kademlia/message/ContentLookupMessage.java b/src/kademlia/message/ContentLookupMessage.java new file mode 100644 index 0000000..9d8a85f --- /dev/null +++ b/src/kademlia/message/ContentLookupMessage.java @@ -0,0 +1,80 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.dht.GetParameter; +import kademlia.node.Node; +import kademlia.util.serializer.JsonSerializer; + +/** + * Messages used to send to another node requesting content. + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class ContentLookupMessage implements Message +{ + + public static final byte CODE = 0x03; + + private Node origin; + private GetParameter params; + + /** + * @param origin The node where this lookup came from + * @param params The parameters used to find the content + */ + public ContentLookupMessage(Node origin, GetParameter params) + { + this.origin = origin; + this.params = params; + } + + public ContentLookupMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + public GetParameter getParameters() + { + return this.params; + } + + public Node getOrigin() + { + return this.origin; + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + this.origin.toStream(out); + + /* Write the params to the stream */ + new JsonSerializer().write(this.params, out); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + + /* Read the params from the stream */ + try + { + this.params = new JsonSerializer().read(in); + } + catch (ClassNotFoundException e) + { + e.printStackTrace(); + } + } + + @Override + public byte code() + { + return CODE; + } + +} diff --git a/src/kademlia/message/ContentLookupReceiver.java b/src/kademlia/message/ContentLookupReceiver.java new file mode 100644 index 0000000..26aedf9 --- /dev/null +++ b/src/kademlia/message/ContentLookupReceiver.java @@ -0,0 +1,69 @@ +package kademlia.message; + +import java.io.IOException; +import java.util.NoSuchElementException; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; + +/** + * Responds to a ContentLookupMessage by sending a ContentMessage containing the requested content; + * if the requested content is not found, a NodeReplyMessage containing the K closest nodes to the request key is sent. + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class ContentLookupReceiver implements Receiver +{ + + private final KadServer server; + private final KademliaNode localNode; + private final DHT dht; + private final KadConfiguration config; + + public ContentLookupReceiver(KadServer server, KademliaNode localNode, DHT dht, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.dht = dht; + this.config = config; + } + + @Override + public void receive(Message incoming, int comm) throws IOException + { + ContentLookupMessage msg = (ContentLookupMessage) incoming; + this.localNode.getRoutingTable().insert(msg.getOrigin()); + + /* Check if we can have this data */ + if (this.dht.contains(msg.getParameters())) + { + try + { + /* Return a ContentMessage with the required data */ + ContentMessage cMsg = new ContentMessage(localNode.getNode(), this.dht.get(msg.getParameters())); + server.reply(msg.getOrigin(), cMsg, comm); + } + catch (NoSuchElementException ex) + { + /* @todo Not sure why this exception is thrown here, checkup the system when tests are writtem*/ + } + } + else + { + /** + * Return a the K closest nodes to this content identifier + * We create a NodeLookupReceiver and let this receiver handle this operation + */ + NodeLookupMessage lkpMsg = new NodeLookupMessage(msg.getOrigin(), msg.getParameters().getKey()); + new NodeLookupReceiver(server, localNode, this.config).receive(lkpMsg, comm); + } + } + + @Override + public void timeout(int comm) + { + + } +} diff --git a/src/kademlia/message/ContentMessage.java b/src/kademlia/message/ContentMessage.java new file mode 100644 index 0000000..1a35304 --- /dev/null +++ b/src/kademlia/message/ContentMessage.java @@ -0,0 +1,85 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.dht.StorageEntry; +import kademlia.node.Node; +import kademlia.util.serializer.JsonSerializer; + +/** + * A Message used to send content between nodes + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class ContentMessage implements Message +{ + + public static final byte CODE = 0x04; + + private StorageEntry content; + private Node origin; + + /** + * @param origin Where the message came from + * @param content The content to be stored + * + */ + public ContentMessage(Node origin, StorageEntry content) + { + this.content = content; + this.origin = origin; + } + + public ContentMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + this.origin.toStream(out); + + /* Serialize the KadContent, then send it to the stream */ + new JsonSerializer().write(content, out); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + + try + { + this.content = new JsonSerializer().read(in); + } + catch (ClassNotFoundException e) + { + System.err.println("ClassNotFoundException when reading StorageEntry; Message: " + e.getMessage()); + } + } + + public Node getOrigin() + { + return this.origin; + } + + public StorageEntry getContent() + { + return this.content; + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public String toString() + { + return "ContentMessage[origin=" + origin + ",content=" + content + "]"; + } +} diff --git a/src/kademlia/message/Message.java b/src/kademlia/message/Message.java new file mode 100644 index 0000000..c1fd105 --- /dev/null +++ b/src/kademlia/message/Message.java @@ -0,0 +1,14 @@ +package kademlia.message; + +public interface Message extends Streamable +{ + + /** + * The unique code for the message type, used to differentiate all messages + * from each other. Since this is of byte type there can + * be at most 256 different message types. + * + * @return byte A unique code representing the message type + * */ + public byte code(); +} diff --git a/src/kademlia/message/MessageFactory.java b/src/kademlia/message/MessageFactory.java new file mode 100644 index 0000000..c06621e --- /dev/null +++ b/src/kademlia/message/MessageFactory.java @@ -0,0 +1,74 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; + +/** + * Handles creating messages and receivers + * + * @author Joshua Kissoon + * @since 20140202 + */ +public class MessageFactory +{ + + private final KademliaNode localNode; + private final DHT dht; + private final KadConfiguration config; + + public MessageFactory(KademliaNode local, DHT dht, KadConfiguration config) + { + this.localNode = local; + this.dht = dht; + this.config = config; + } + + public Message createMessage(byte code, DataInputStream in) throws IOException + { + switch (code) + { + case AcknowledgeMessage.CODE: + return new AcknowledgeMessage(in); + case ConnectMessage.CODE: + return new ConnectMessage(in); + case ContentMessage.CODE: + return new ContentMessage(in); + case ContentLookupMessage.CODE: + return new ContentLookupMessage(in); + case NodeLookupMessage.CODE: + return new NodeLookupMessage(in); + case NodeReplyMessage.CODE: + return new NodeReplyMessage(in); + case SimpleMessage.CODE: + return new SimpleMessage(in); + case StoreContentMessage.CODE: + return new StoreContentMessage(in); + default: + //System.out.println(this.localNode + " - No Message handler found for message. Code: " + code); + return new SimpleMessage(in); + + } + } + + public Receiver createReceiver(byte code, KadServer server) + { + switch (code) + { + case ConnectMessage.CODE: + return new ConnectReceiver(server, this.localNode); + case ContentLookupMessage.CODE: + return new ContentLookupReceiver(server, this.localNode, this.dht, this.config); + case NodeLookupMessage.CODE: + return new NodeLookupReceiver(server, this.localNode, this.config); + case StoreContentMessage.CODE: + return new StoreContentReceiver(server, this.localNode, this.dht); + default: + //System.out.println("No receiver found for message. Code: " + code); + return new SimpleReceiver(); + } + } +} diff --git a/src/kademlia/message/NodeLookupMessage.java b/src/kademlia/message/NodeLookupMessage.java new file mode 100644 index 0000000..d1a9b5d --- /dev/null +++ b/src/kademlia/message/NodeLookupMessage.java @@ -0,0 +1,75 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.node.Node; +import kademlia.node.KademliaId; + +/** + * A message sent to other nodes requesting the K-Closest nodes to a key sent in this message. + * + * @author Joshua Kissoon + * @created 20140218 + */ +public class NodeLookupMessage implements Message +{ + + private Node origin; + private KademliaId lookupId; + + public static final byte CODE = 0x05; + + /** + * A new NodeLookupMessage to find nodes + * + * @param origin The Node from which the message is coming from + * @param lookup The key for which to lookup nodes for + */ + public NodeLookupMessage(Node origin, KademliaId lookup) + { + this.origin = origin; + this.lookupId = lookup; + } + + public NodeLookupMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + this.lookupId = new KademliaId(in); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + this.origin.toStream(out); + this.lookupId.toStream(out); + } + + public Node getOrigin() + { + return this.origin; + } + + public KademliaId getLookupId() + { + return this.lookupId; + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public String toString() + { + return "NodeLookupMessage[origin=" + origin + ",lookup=" + lookupId + "]"; + } +} diff --git a/src/kademlia/message/NodeLookupReceiver.java b/src/kademlia/message/NodeLookupReceiver.java new file mode 100644 index 0000000..7f4ac0b --- /dev/null +++ b/src/kademlia/message/NodeLookupReceiver.java @@ -0,0 +1,69 @@ +package kademlia.message; + +import java.io.IOException; +import java.util.List; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.node.Node; + +/** + * Receives a NodeLookupMessage and sends a NodeReplyMessage as reply with the K-Closest nodes to the ID sent. + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class NodeLookupReceiver implements Receiver +{ + + private final KadServer server; + private final KademliaNode localNode; + private final KadConfiguration config; + + public NodeLookupReceiver(KadServer server, KademliaNode local, KadConfiguration config) + { + this.server = server; + this.localNode = local; + this.config = config; + } + + /** + * Handle receiving a NodeLookupMessage + * Find the set of K nodes closest to the lookup ID and return them + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public void receive(Message incoming, int comm) throws IOException + { + NodeLookupMessage msg = (NodeLookupMessage) incoming; + + Node origin = msg.getOrigin(); + + /* Update the local space by inserting the origin node. */ + this.localNode.getRoutingTable().insert(origin); + + /* Find nodes closest to the LookupId */ + List nodes = this.localNode.getRoutingTable().findClosest(msg.getLookupId(), this.config.k()); + + /* Respond to the NodeLookupMessage */ + Message reply = new NodeReplyMessage(this.localNode.getNode(), nodes); + + /* Let the Server send the reply */ + this.server.reply(origin, reply, comm); + } + + /** + * We don't need to do anything here + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public void timeout(int comm) throws IOException + { + } +} diff --git a/src/kademlia/message/NodeReplyMessage.java b/src/kademlia/message/NodeReplyMessage.java new file mode 100644 index 0000000..07a1f72 --- /dev/null +++ b/src/kademlia/message/NodeReplyMessage.java @@ -0,0 +1,94 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import kademlia.node.Node; + +/** + * A message used to connect nodes. + * When a NodeLookup Request comes in, we respond with a NodeReplyMessage. + * + * @author Joshua Kissoon + * @created 20140218 + */ +public class NodeReplyMessage implements Message +{ + + private Node origin; + public static final byte CODE = 0x06; + private List nodes; + + public NodeReplyMessage(Node origin, List nodes) + { + this.origin = origin; + this.nodes = nodes; + } + + public NodeReplyMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + /* Read in the origin */ + this.origin = new Node(in); + + /* Get the number of incoming nodes */ + int len = in.readInt(); + this.nodes = new ArrayList<>(len); + + /* Read in all nodes */ + for (int i = 0; i < len; i++) + { + this.nodes.add(new Node(in)); + } + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + /* Add the origin node to the stream */ + origin.toStream(out); + + /* Add all other nodes to the stream */ + int len = this.nodes.size(); + if (len > 255) + { + throw new IndexOutOfBoundsException("Too many nodes in list to send in NodeReplyMessage. Size: " + len); + } + + /* Writing the nodes to the stream */ + out.writeInt(len); + for (Node n : this.nodes) + { + n.toStream(out); + } + } + + public Node getOrigin() + { + return this.origin; + } + + @Override + public byte code() + { + return CODE; + } + + public List getNodes() + { + return this.nodes; + } + + @Override + public String toString() + { + return "NodeReplyMessage[origin NodeId=" + origin.getNodeId() + "]"; + } +} diff --git a/src/kademlia/message/Receiver.java b/src/kademlia/message/Receiver.java new file mode 100644 index 0000000..eda37c0 --- /dev/null +++ b/src/kademlia/message/Receiver.java @@ -0,0 +1,33 @@ +package kademlia.message; + +import java.io.IOException; + +/** + * A receiver waits for incoming messages and perform some action when the message is received + * + * @author Joshua Kissoon + * @created 20140218 + */ +public interface Receiver +{ + + /** + * Message is received, now handle it + * + * @param conversationId The ID of this conversation, used for further conversations + * @param incoming The incoming + * + * @throws java.io.IOException + */ + public void receive(Message incoming, int conversationId) throws IOException; + + /** + * If no reply is received in MessageServer.TIMEOUT seconds for the + * message with communication id comm, the MessageServer calls this method + * + * @param conversationId The conversation ID of this communication + * + * @throws IOException if an I/O error occurs + * */ + public void timeout(int conversationId) throws IOException; +} diff --git a/src/kademlia/message/SimpleMessage.java b/src/kademlia/message/SimpleMessage.java new file mode 100644 index 0000000..2b2b157 --- /dev/null +++ b/src/kademlia/message/SimpleMessage.java @@ -0,0 +1,72 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** + * A simple message used for testing the system; Default message constructed if the message type sent is not available + * + * @author Joshua Kissoon + * @created 20140217 + */ +public class SimpleMessage implements Message +{ + + /* Message constants */ + public static final byte CODE = 0x07; + + private String content; + + public SimpleMessage(String message) + { + this.content = message; + } + + public SimpleMessage(DataInputStream in) + { + this.fromStream(in); + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public void toStream(DataOutputStream out) + { + try + { + out.writeInt(this.content.length()); + out.writeBytes(this.content); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + + @Override + public final void fromStream(DataInputStream in) + { + try + { + byte[] buff = new byte[in.readInt()]; + in.readFully(buff); + + this.content = new String(buff); + } + catch (IOException e) + { + e.printStackTrace(); + } + } + + @Override + public String toString() + { + return this.content; + } +} diff --git a/src/kademlia/message/SimpleReceiver.java b/src/kademlia/message/SimpleReceiver.java new file mode 100644 index 0000000..fd178cf --- /dev/null +++ b/src/kademlia/message/SimpleReceiver.java @@ -0,0 +1,25 @@ +package kademlia.message; + +import java.io.IOException; + +/** + * Default receiver if none other is called + * + * @author Joshua Kissoon + * @created 20140202 + */ +public class SimpleReceiver implements Receiver +{ + + @Override + public void receive(Message incoming, int conversationId) + { + + } + + @Override + public void timeout(int conversationId) throws IOException + { + + } +} diff --git a/src/kademlia/message/StoreContentMessage.java b/src/kademlia/message/StoreContentMessage.java new file mode 100644 index 0000000..edd8075 --- /dev/null +++ b/src/kademlia/message/StoreContentMessage.java @@ -0,0 +1,84 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import kademlia.dht.StorageEntry; +import kademlia.node.Node; +import kademlia.util.serializer.JsonSerializer; + +/** + * A StoreContentMessage used to send a store message to a node + * + * @author Joshua Kissoon + * @since 20140225 + */ +public class StoreContentMessage implements Message +{ + + public static final byte CODE = 0x08; + + private StorageEntry content; + private Node origin; + + /** + * @param origin Where the message came from + * @param content The content to be stored + * + */ + public StoreContentMessage(Node origin, StorageEntry content) + { + this.content = content; + this.origin = origin; + } + + public StoreContentMessage(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + this.origin.toStream(out); + + /* Serialize the KadContent, then send it to the stream */ + new JsonSerializer().write(content, out); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + this.origin = new Node(in); + try + { + this.content = new JsonSerializer().read(in); + } + catch (ClassNotFoundException e) + { + e.printStackTrace(); + } + } + + public Node getOrigin() + { + return this.origin; + } + + public StorageEntry getContent() + { + return this.content; + } + + @Override + public byte code() + { + return CODE; + } + + @Override + public String toString() + { + return "StoreContentMessage[origin=" + origin + ",content=" + content + "]"; + } +} diff --git a/src/kademlia/message/StoreContentReceiver.java b/src/kademlia/message/StoreContentReceiver.java new file mode 100644 index 0000000..e170444 --- /dev/null +++ b/src/kademlia/message/StoreContentReceiver.java @@ -0,0 +1,57 @@ +package kademlia.message; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadServer; +import kademlia.dht.DHT; + +/** + * Receiver for incoming StoreContentMessage + * + * @author Joshua Kissoon + * @since 20140225 + */ +public class StoreContentReceiver implements Receiver +{ + + private final KadServer server; + private final KademliaNode localNode; + private final DHT dht; + + public StoreContentReceiver(KadServer server, KademliaNode localNode, DHT dht) + { + this.server = server; + this.localNode = localNode; + this.dht = dht; + } + + @Override + public void receive(Message incoming, int comm) + { + /* It's a StoreContentMessage we're receiving */ + StoreContentMessage msg = (StoreContentMessage) incoming; + + /* Insert the message sender into this node's routing table */ + this.localNode.getRoutingTable().insert(msg.getOrigin()); + + try + { + /* Store this Content into the DHT */ + this.dht.store(msg.getContent()); + } + catch (IOException e) + { + System.err.println("Unable to store received content; Message: " + e.getMessage()); + } + + } + + @Override + public void timeout(int comm) + { + /** + * This receiver only handles Receiving content when we've received the message, + * so no timeout will happen with this receiver. + */ + } +} diff --git a/src/kademlia/message/Streamable.java b/src/kademlia/message/Streamable.java new file mode 100644 index 0000000..fd038f3 --- /dev/null +++ b/src/kademlia/message/Streamable.java @@ -0,0 +1,42 @@ +package kademlia.message; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** + * A Streamable object is able to write it's state to an output stream and + * a class implementing Streamable must be able to recreate an instance of + * the class from an input stream. No information about class name is written + * to the output stream so it must be known what class type is expected when + * reading objects back in from an input stream. This gives a space + * advantage over Serializable. + *

+ * Since the exact class must be known anyway prior to reading, it is incouraged + * that classes implementing Streamble also provide a constructor of the form: + *

+ * Streamable(DataInput in) throws IOException; + * */ +public interface Streamable +{ + + /** + * Writes the internal state of the Streamable object to the output stream + * in a format that can later be read by the same Streamble class using + * the {@link #fromStream} method. + * + * @param out + * + * @throws java.io.IOException + */ + public void toStream(DataOutputStream out) throws IOException; + + /** + * Reads the internal state of the Streamable object from the input stream. + * + * @param out + * + * @throws java.io.IOException + */ + public void fromStream(DataInputStream out) throws IOException; +} diff --git a/src/kademlia/node/KademliaId.java b/src/kademlia/node/KademliaId.java new file mode 100644 index 0000000..1664cf0 --- /dev/null +++ b/src/kademlia/node/KademliaId.java @@ -0,0 +1,262 @@ +/** + * @author Joshua Kissoon + * @created 20140215 + * @desc Represents a Kademlia Node ID + */ +package kademlia.node; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Random; +import kademlia.message.Streamable; + +public class KademliaId implements Streamable +{ + + public final transient static int ID_LENGTH = 160; + private byte[] keyBytes; + + /** + * Construct the NodeId from some string + * + * @param data The user generated key string + */ + public KademliaId(String data) + { + keyBytes = data.getBytes(); + if (keyBytes.length != ID_LENGTH / 8) + { + throw new IllegalArgumentException("Specified Data need to be " + (ID_LENGTH / 8) + " characters long."); + } + } + + /** + * Generate a random key + */ + public KademliaId() + { + keyBytes = new byte[ID_LENGTH / 8]; + new Random().nextBytes(keyBytes); + } + + /** + * Generate the NodeId from a given byte[] + * + * @param bytes + */ + public KademliaId(byte[] bytes) + { + if (bytes.length != ID_LENGTH / 8) + { + throw new IllegalArgumentException("Specified Data need to be " + (ID_LENGTH / 8) + " characters long. Data Given: '" + new String(bytes) + "'"); + } + this.keyBytes = bytes; + } + + /** + * Load the NodeId from a DataInput stream + * + * @param in The stream from which to load the NodeId + * + * @throws IOException + */ + public KademliaId(DataInputStream in) throws IOException + { + this.fromStream(in); + } + + public byte[] getBytes() + { + return this.keyBytes; + } + + /** + * @return The BigInteger representation of the key + */ + public BigInteger getInt() + { + return new BigInteger(1, this.getBytes()); + } + + /** + * Compares a NodeId to this NodeId + * + * @param o The NodeId to compare to this NodeId + * + * @return boolean Whether the 2 NodeIds are equal + */ + @Override + public boolean equals(Object o) + { + if (o instanceof KademliaId) + { + KademliaId nid = (KademliaId) o; + return this.hashCode() == nid.hashCode(); + } + return false; + } + + @Override + public int hashCode() + { + int hash = 7; + hash = 83 * hash + Arrays.hashCode(this.keyBytes); + return hash; + } + + /** + * Checks the distance between this and another NodeId + * + * @param nid + * + * @return The distance of this NodeId from the given NodeId + */ + public KademliaId xor(KademliaId nid) + { + byte[] result = new byte[ID_LENGTH / 8]; + byte[] nidBytes = nid.getBytes(); + + for (int i = 0; i < ID_LENGTH / 8; i++) + { + result[i] = (byte) (this.keyBytes[i] ^ nidBytes[i]); + } + + KademliaId resNid = new KademliaId(result); + + return resNid; + } + + /** + * Generates a NodeId that is some distance away from this NodeId + * + * @param distance in number of bits + * + * @return NodeId The newly generated NodeId + */ + public KademliaId generateNodeIdByDistance(int distance) + { + byte[] result = new byte[ID_LENGTH / 8]; + + /* Since distance = ID_LENGTH - prefixLength, we need to fill that amount with 0's */ + int numByteZeroes = (ID_LENGTH - distance) / 8; + int numBitZeroes = 8 - (distance % 8); + + /* Filling byte zeroes */ + for (int i = 0; i < numByteZeroes; i++) + { + result[i] = 0; + } + + /* Filling bit zeroes */ + BitSet bits = new BitSet(8); + bits.set(0, 8); + + for (int i = 0; i < numBitZeroes; i++) + { + /* Shift 1 zero into the start of the value */ + bits.clear(i); + } + bits.flip(0, 8); // Flip the bits since they're in reverse order + result[numByteZeroes] = (byte) bits.toByteArray()[0]; + + /* Set the remaining bytes to Maximum value */ + for (int i = numByteZeroes + 1; i < result.length; i++) + { + result[i] = Byte.MAX_VALUE; + } + + return this.xor(new KademliaId(result)); + } + + /** + * Counts the number of leading 0's in this NodeId + * + * @return Integer The number of leading 0's + */ + public int getFirstSetBitIndex() + { + int prefixLength = 0; + + for (byte b : this.keyBytes) + { + if (b == 0) + { + prefixLength += 8; + } + else + { + /* If the byte is not 0, we need to count how many MSBs are 0 */ + int count = 0; + for (int i = 7; i >= 0; i--) + { + boolean a = (b & (1 << i)) == 0; + if (a) + { + count++; + } + else + { + break; // Reset the count if we encounter a non-zero number + } + } + + /* Add the count of MSB 0s to the prefix length */ + prefixLength += count; + + /* Break here since we've now covered the MSB 0s */ + break; + } + } + return prefixLength; + } + + /** + * Gets the distance from this NodeId to another NodeId + * + * @param to + * + * @return Integer The distance + */ + public int getDistance(KademliaId to) + { + /** + * Compute the xor of this and to + * Get the index i of the first set bit of the xor returned NodeId + * The distance between them is ID_LENGTH - i + */ + return ID_LENGTH - this.xor(to).getFirstSetBitIndex(); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + /* Add the NodeId to the stream */ + out.write(this.getBytes()); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + byte[] input = new byte[ID_LENGTH / 8]; + in.readFully(input); + this.keyBytes = input; + } + + public String hexRepresentation() + { + /* Returns the hex format of this NodeId */ + BigInteger bi = new BigInteger(1, this.keyBytes); + return String.format("%0" + (this.keyBytes.length << 1) + "X", bi); + } + + @Override + public String toString() + { + return this.hexRepresentation(); + } + +} diff --git a/src/kademlia/node/KeyComparator.java b/src/kademlia/node/KeyComparator.java new file mode 100644 index 0000000..01bbbe4 --- /dev/null +++ b/src/kademlia/node/KeyComparator.java @@ -0,0 +1,44 @@ +package kademlia.node; + +import java.math.BigInteger; +import java.util.Comparator; + +/** + * A Comparator to compare 2 keys to a given key + * + * @author Joshua Kissoon + * @since 20140322 + */ +public class KeyComparator implements Comparator +{ + + private final BigInteger key; + + /** + * @param key The NodeId relative to which the distance should be measured. + */ + public KeyComparator(KademliaId key) + { + this.key = key.getInt(); + } + + /** + * Compare two objects which must both be of type Node + * and determine which is closest to the identifier specified in the + * constructor. + * + * @param n1 Node 1 to compare distance from the key + * @param n2 Node 2 to compare distance from the key + */ + @Override + public int compare(Node n1, Node n2) + { + BigInteger b1 = n1.getNodeId().getInt(); + BigInteger b2 = n2.getNodeId().getInt(); + + b1 = b1.xor(key); + b2 = b2.xor(key); + + return b1.abs().compareTo(b2.abs()); + } +} diff --git a/src/kademlia/node/Node.java b/src/kademlia/node/Node.java new file mode 100644 index 0000000..fdfeb2a --- /dev/null +++ b/src/kademlia/node/Node.java @@ -0,0 +1,133 @@ +package kademlia.node; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import kademlia.message.Streamable; + +/** + * A Node in the Kademlia network - Contains basic node network information. + * + * @author Joshua Kissoon + * @since 20140202 + * @version 0.1 + */ +public class Node implements Streamable +{ + + private KademliaId nodeId; + private InetAddress inetAddress; + private int port; + private final String strRep; + + public Node(KademliaId nid, InetAddress ip, int port) + { + this.nodeId = nid; + this.inetAddress = ip; + this.port = port; + this.strRep = this.nodeId.toString(); + } + + /** + * Load the Node's data from a DataInput stream + * + * @param in + * + * @throws IOException + */ + public Node(DataInputStream in) throws IOException + { + this.fromStream(in); + this.strRep = this.nodeId.toString(); + } + + /** + * Set the InetAddress of this node + * + * @param addr The new InetAddress of this node + */ + public void setInetAddress(InetAddress addr) + { + this.inetAddress = addr; + } + + /** + * @return The NodeId object of this node + */ + public KademliaId getNodeId() + { + return this.nodeId; + } + + /** + * Creates a SocketAddress for this node + * + * @return + */ + public InetSocketAddress getSocketAddress() + { + return new InetSocketAddress(this.inetAddress, this.port); + } + + @Override + public void toStream(DataOutputStream out) throws IOException + { + /* Add the NodeId to the stream */ + this.nodeId.toStream(out); + + /* Add the Node's IP address to the stream */ + byte[] a = inetAddress.getAddress(); + if (a.length != 4) + { + throw new RuntimeException("Expected InetAddress of 4 bytes, got " + a.length); + } + out.write(a); + + /* Add the port to the stream */ + out.writeInt(port); + } + + @Override + public final void fromStream(DataInputStream in) throws IOException + { + /* Load the NodeId */ + this.nodeId = new KademliaId(in); + + /* Load the IP Address */ + byte[] ip = new byte[4]; + in.readFully(ip); + this.inetAddress = InetAddress.getByAddress(ip); + + /* Read in the port */ + this.port = in.readInt(); + } + + @Override + public boolean equals(Object o) + { + if (o instanceof Node) + { + Node n = (Node) o; + if (n == this) + { + return true; + } + return this.getNodeId().equals(n.getNodeId()); + } + return false; + } + + @Override + public int hashCode() + { + return this.getNodeId().hashCode(); + } + + @Override + public String toString() + { + return this.getNodeId().toString(); + } +} diff --git a/src/kademlia/operation/BucketRefreshOperation.java b/src/kademlia/operation/BucketRefreshOperation.java new file mode 100644 index 0000000..226ee77 --- /dev/null +++ b/src/kademlia/operation/BucketRefreshOperation.java @@ -0,0 +1,66 @@ +package kademlia.operation; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.node.KademliaId; + +/** + * At each time interval t, nodes need to refresh their K-Buckets + * This operation takes care of refreshing this node's K-Buckets + * + * @author Joshua Kissoon + * @created 20140224 + */ +public class BucketRefreshOperation implements Operation +{ + + private final KadServer server; + private final KademliaNode localNode; + private final KadConfiguration config; + + public BucketRefreshOperation(KadServer server, KademliaNode localNode, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.config = config; + } + + /** + * Each bucket need to be refreshed at every time interval t. + * Find an identifier in each bucket's range, use it to look for nodes closest to this identifier + * allowing the bucket to be refreshed. + * + * Then Do a NodeLookupOperation for each of the generated NodeIds, + * This will find the K-Closest nodes to that ID, and update the necessary K-Bucket + * + * @throws java.io.IOException + */ + @Override + public synchronized void execute() throws IOException + { + for (int i = 1; i < KademliaId.ID_LENGTH; i++) + { + /* Construct a NodeId that is i bits away from the current node Id */ + final KademliaId current = this.localNode.getNode().getNodeId().generateNodeIdByDistance(i); + + /* Run the Node Lookup Operation, each in a different thread to speed up things */ + new Thread() + { + @Override + public void run() + { + try + { + new NodeLookupOperation(server, localNode, current, BucketRefreshOperation.this.config).execute(); + } + catch (IOException e) + { + //System.err.println("Bucket Refresh Operation Failed. Msg: " + e.getMessage()); + } + } + }.start(); + } + } +} diff --git a/src/kademlia/operation/ConnectOperation.java b/src/kademlia/operation/ConnectOperation.java new file mode 100644 index 0000000..06687e7 --- /dev/null +++ b/src/kademlia/operation/ConnectOperation.java @@ -0,0 +1,139 @@ +/** + * @author Joshua Kissoon + * @created 20140218 + * @desc Operation that handles connecting to an existing Kademlia network using a bootstrap node + */ +package kademlia.operation; + +import kademlia.message.Receiver; +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.exceptions.RoutingException; +import kademlia.message.AcknowledgeMessage; +import kademlia.message.ConnectMessage; +import kademlia.message.Message; +import kademlia.node.Node; + +public class ConnectOperation implements Operation, Receiver +{ + + public static final int MAX_CONNECT_ATTEMPTS = 5; // Try 5 times to connect to a node + + private final KadServer server; + private final KademliaNode localNode; + private final Node bootstrapNode; + private final KadConfiguration config; + + private boolean error; + private int attempts; + + /** + * @param server The message server used to send/receive messages + * @param local The local node + * @param bootstrap Node to use to bootstrap the local node onto the network + * @param config + */ + public ConnectOperation(KadServer server, KademliaNode local, Node bootstrap, KadConfiguration config) + { + this.server = server; + this.localNode = local; + this.bootstrapNode = bootstrap; + this.config = config; + } + + @Override + public synchronized void execute() throws IOException + { + try + { + /* Contact the bootstrap node */ + this.error = true; + this.attempts = 0; + Message m = new ConnectMessage(this.localNode.getNode()); + + /* Send a connect message to the bootstrap node */ + server.sendMessage(this.bootstrapNode, m, this); + + /* If we haven't finished as yet, wait for a maximum of config.operationTimeout() time */ + int totalTimeWaited = 0; + int timeInterval = 50; // We re-check every 300 milliseconds + while (totalTimeWaited < this.config.operationTimeout()) + { + if (error) + { + wait(timeInterval); + totalTimeWaited += timeInterval; + } + else + { + break; + } + } + if (error) + { + /* If we still haven't received any responses by then, do a routing timeout */ + throw new RoutingException("ConnectOperation: Bootstrap node did not respond: " + bootstrapNode); + } + + /* Perform lookup for our own ID to get nodes close to us */ + Operation lookup = new NodeLookupOperation(this.server, this.localNode, this.localNode.getNode().getNodeId(), this.config); + lookup.execute(); + + /** + * Refresh buckets to get a good routing table + * After the above lookup operation, K nodes will be in our routing table, + * Now we try to populate all of our buckets. + */ + new BucketRefreshOperation(this.server, this.localNode, this.config).execute(); + } + catch (InterruptedException e) + { + System.err.println("Connect operation was interrupted. "); + } + } + + /** + * Receives an AcknowledgeMessage from the bootstrap node. + * + * @param comm + */ + @Override + public synchronized void receive(Message incoming, int comm) + { + /* The incoming message will be an acknowledgement message */ + AcknowledgeMessage msg = (AcknowledgeMessage) incoming; + + /* The bootstrap node has responded, insert it into our space */ + this.localNode.getRoutingTable().insert(this.bootstrapNode); + + /* We got a response, so the error is false */ + error = false; + + /* Wake up any waiting thread */ + notify(); + } + + /** + * Resends a ConnectMessage to the boot strap node a maximum of MAX_ATTEMPTS + * times. + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public synchronized void timeout(int comm) throws IOException + { + if (++this.attempts < MAX_CONNECT_ATTEMPTS) + { + this.server.sendMessage(this.bootstrapNode, new ConnectMessage(this.localNode.getNode()), this); + } + else + { + /* We just exit, so notify all other threads that are possibly waiting */ + notify(); + } + } +} diff --git a/src/kademlia/operation/ContentLookupOperation.java b/src/kademlia/operation/ContentLookupOperation.java new file mode 100644 index 0000000..a9189be --- /dev/null +++ b/src/kademlia/operation/ContentLookupOperation.java @@ -0,0 +1,334 @@ +package kademlia.operation; + +import kademlia.message.Receiver; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import kademlia.KademliaNode; +import kademlia.dht.GetParameter; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.StorageEntry; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.exceptions.RoutingException; +import kademlia.exceptions.UnknownMessageException; +import kademlia.message.ContentLookupMessage; +import kademlia.message.ContentMessage; +import kademlia.message.Message; +import kademlia.message.NodeReplyMessage; +import kademlia.node.KeyComparator; +import kademlia.node.Node; +import kademlia.util.RouteLengthChecker; + +/** + * Looks up a specified identifier and returns the value associated with it + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class ContentLookupOperation implements Operation, Receiver +{ + + /* Constants */ + private static final Byte UNASKED = (byte) 0x00; + private static final Byte AWAITING = (byte) 0x01; + private static final Byte ASKED = (byte) 0x02; + private static final Byte FAILED = (byte) 0x03; + + private final KadServer server; + private final KademliaNode localNode; + private StorageEntry contentFound = null; + private final KadConfiguration config; + + private final ContentLookupMessage lookupMessage; + + private boolean isContentFound; + private final SortedMap nodes; + + /* Tracks messages in transit and awaiting reply */ + private final Map messagesTransiting; + + /* Used to sort nodes */ + private final Comparator comparator; + + /* Statistical information */ + private final RouteLengthChecker routeLengthChecker; + + + { + messagesTransiting = new HashMap<>(); + isContentFound = false; + routeLengthChecker = new RouteLengthChecker(); + } + + /** + * @param server + * @param localNode + * @param params The parameters to search for the content which we need to find + * @param config + */ + public ContentLookupOperation(KadServer server, KademliaNode localNode, GetParameter params, KadConfiguration config) + { + /* Construct our lookup message */ + this.lookupMessage = new ContentLookupMessage(localNode.getNode(), params); + + this.server = server; + this.localNode = localNode; + this.config = config; + + /** + * We initialize a TreeMap to store nodes. + * This map will be sorted by which nodes are closest to the lookupId + */ + this.comparator = new KeyComparator(params.getKey()); + this.nodes = new TreeMap(this.comparator); + } + + /** + * @throws java.io.IOException + * @throws kademlia.exceptions.RoutingException + */ + @Override + public synchronized void execute() throws IOException, RoutingException + { + try + { + /* Set the local node as already asked */ + nodes.put(this.localNode.getNode(), ASKED); + + /** + * We add all nodes here instead of the K-Closest because there may be the case that the K-Closest are offline + * - The operation takes care of looking at the K-Closest. + */ + List allNodes = this.localNode.getRoutingTable().getAllNodes(); + this.addNodes(allNodes); + + /* Also add the initial set of nodes to the routeLengthChecker */ + this.routeLengthChecker.addInitialNodes(allNodes); + + /** + * If we haven't found the requested amount of content as yet, + * keey trying until config.operationTimeout() time has expired + */ + int totalTimeWaited = 0; + int timeInterval = 10; // We re-check every n milliseconds + while (totalTimeWaited < this.config.operationTimeout()) + { + if (!this.askNodesorFinish() && !isContentFound) + { + wait(timeInterval); + totalTimeWaited += timeInterval; + } + else + { + break; + } + } + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + /** + * Add nodes from this list to the set of nodes to lookup + * + * @param list The list from which to add nodes + */ + public void addNodes(List list) + { + for (Node o : list) + { + /* If this node is not in the list, add the node */ + if (!nodes.containsKey(o)) + { + nodes.put(o, UNASKED); + } + } + } + + /** + * Asks some of the K closest nodes seen but not yet queried. + * Assures that no more than DefaultConfiguration.CONCURRENCY messages are in transit at a time + * + * This method should be called every time a reply is received or a timeout occurs. + * + * If all K closest nodes have been asked and there are no messages in transit, + * the algorithm is finished. + * + * @return true if finished OR false otherwise + */ + private boolean askNodesorFinish() throws IOException + { + /* If >= CONCURRENCY nodes are in transit, don't do anything */ + if (this.config.maxConcurrentMessagesTransiting() <= this.messagesTransiting.size()) + { + return false; + } + + /* Get unqueried nodes among the K closest seen that have not FAILED */ + List unasked = this.closestNodesNotFailed(UNASKED); + + if (unasked.isEmpty() && this.messagesTransiting.isEmpty()) + { + /* We have no unasked nodes nor any messages in transit, we're finished! */ + return true; + } + + /* Sort nodes according to criteria */ + Collections.sort(unasked, this.comparator); + + /** + * Send messages to nodes in the list; + * making sure than no more than CONCURRENCY messsages are in transit + */ + for (int i = 0; (this.messagesTransiting.size() < this.config.maxConcurrentMessagesTransiting()) && (i < unasked.size()); i++) + { + Node n = (Node) unasked.get(i); + + int comm = server.sendMessage(n, lookupMessage, this); + + this.nodes.put(n, AWAITING); + this.messagesTransiting.put(comm, n); + } + + /* We're not finished as yet, return false */ + return false; + } + + /** + * Find The K closest nodes to the target lookupId given that have not FAILED. + * From those K, get those that have the specified status + * + * @param status The status of the nodes to return + * + * @return A List of the closest nodes + */ + private List closestNodesNotFailed(Byte status) + { + List closestNodes = new ArrayList<>(this.config.k()); + int remainingSpaces = this.config.k(); + + for (Map.Entry e : this.nodes.entrySet()) + { + if (!FAILED.equals(e.getValue())) + { + if (status.equals(e.getValue())) + { + /* We got one with the required status, now add it */ + closestNodes.add((Node) e.getKey()); + } + + if (--remainingSpaces == 0) + { + break; + } + } + } + + return closestNodes; + } + + @Override + public synchronized void receive(Message incoming, int comm) throws IOException, RoutingException + { + if (this.isContentFound) + { + return; + } + + if (incoming instanceof ContentMessage) + { + /* The reply received is a content message with the required content, take it in */ + ContentMessage msg = (ContentMessage) incoming; + + /* Add the origin node to our routing table */ + this.localNode.getRoutingTable().insert(msg.getOrigin()); + + /* Get the Content and check if it satisfies the required parameters */ + StorageEntry content = msg.getContent(); + this.contentFound = content; + this.isContentFound = true; + } + else + { + /* The reply received is a NodeReplyMessage with nodes closest to the content needed */ + NodeReplyMessage msg = (NodeReplyMessage) incoming; + + /* Add the origin node to our routing table */ + Node origin = msg.getOrigin(); + this.localNode.getRoutingTable().insert(origin); + + /* Set that we've completed ASKing the origin node */ + this.nodes.put(origin, ASKED); + + /* Remove this msg from messagesTransiting since it's completed now */ + this.messagesTransiting.remove(comm); + + /* Add the received nodes to the routeLengthChecker */ + this.routeLengthChecker.addNodes(msg.getNodes(), origin); + + /* Add the received nodes to our nodes list to query */ + this.addNodes(msg.getNodes()); + this.askNodesorFinish(); + } + } + + /** + * A node does not respond or a packet was lost, we set this node as failed + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public synchronized void timeout(int comm) throws IOException + { + /* Get the node associated with this communication */ + Node n = this.messagesTransiting.get(new Integer(comm)); + + if (n == null) + { + throw new UnknownMessageException("Unknown comm: " + comm); + } + + /* Mark this node as failed and inform the routing table that it's unresponsive */ + this.nodes.put(n, FAILED); + this.localNode.getRoutingTable().setUnresponsiveContact(n); + this.messagesTransiting.remove(comm); + + this.askNodesorFinish(); + } + + /** + * @return The list of all content found during the lookup operation + * + * @throws kademlia.exceptions.ContentNotFoundException + */ + public StorageEntry getContentFound() throws ContentNotFoundException + { + if (this.isContentFound) + { + return this.contentFound; + } + else + { + throw new ContentNotFoundException("No Value was found for the given key."); + } + } + + /** + * @return How many hops it took in order to get to the content. + */ + public int routeLength() + { + return this.routeLengthChecker.getRouteLength(); + } +} diff --git a/src/kademlia/operation/ContentRefreshOperation.java b/src/kademlia/operation/ContentRefreshOperation.java new file mode 100644 index 0000000..2ad54c9 --- /dev/null +++ b/src/kademlia/operation/ContentRefreshOperation.java @@ -0,0 +1,99 @@ +package kademlia.operation; + +import java.io.IOException; +import java.util.List; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; +import kademlia.dht.StorageEntryMetadata; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.message.Message; +import kademlia.message.StoreContentMessage; +import kademlia.node.Node; + +/** + * Refresh/Restore the data on this node by sending the data to the K-Closest nodes to the data + * + * @author Joshua Kissoon + * @since 20140306 + */ +public class ContentRefreshOperation implements Operation +{ + + private final KadServer server; + private final KademliaNode localNode; + private final DHT dht; + private final KadConfiguration config; + + public ContentRefreshOperation(KadServer server, KademliaNode localNode, DHT dht, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.dht = dht; + this.config = config; + } + + /** + * For each content stored on this DHT, distribute it to the K closest nodes + * Also delete the content if this node is no longer one of the K closest nodes + * + * We assume that our RoutingTable is updated, and we can get the K closest nodes from that table + * + * @throws java.io.IOException + */ + @Override + public void execute() throws IOException + { + /* Get a list of all storage entries for content */ + List entries = this.dht.getStorageEntries(); + + /* If a content was last republished before this time, then we need to republish it */ + final long minRepublishTime = (System.currentTimeMillis() / 1000L) - this.config.restoreInterval(); + + /* For each storage entry, distribute it */ + for (StorageEntryMetadata e : entries) + { + /* Check last update time of this entry and only distribute it if it has been last updated > 1 hour ago */ + if (e.lastRepublished() > minRepublishTime) + { + continue; + } + + /* Set that this content is now republished */ + e.updateLastRepublished(); + + /* Get the K closest nodes to this entries */ + List closestNodes = this.localNode.getRoutingTable().findClosest(e.getKey(), this.config.k()); + + /* Create the message */ + Message msg = new StoreContentMessage(this.localNode.getNode(), dht.get(e)); + + /*Store the message on all of the K-Nodes*/ + for (Node n : closestNodes) + { + /*We don't need to again store the content locally, it's already here*/ + if (!n.equals(this.localNode.getNode())) + { + /* Send a contentstore operation to the K-Closest nodes */ + this.server.sendMessage(n, msg, null); + } + } + + /* Delete any content on this node that this node is not one of the K-Closest nodes to */ + try + { + if (!closestNodes.contains(this.localNode.getNode())) + { + this.dht.remove(e); + } + } + catch (ContentNotFoundException cnfe) + { + /* It would be weird if the content is not found here */ + System.err.println("ContentRefreshOperation: Removing content from local node, content not found... Message: " + cnfe.getMessage()); + } + } + + } +} diff --git a/src/kademlia/operation/KadRefreshOperation.java b/src/kademlia/operation/KadRefreshOperation.java new file mode 100644 index 0000000..7904d45 --- /dev/null +++ b/src/kademlia/operation/KadRefreshOperation.java @@ -0,0 +1,40 @@ +package kademlia.operation; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; + +/** + * An operation that handles refreshing the entire Kademlia Systems including buckets and content + * + * @author Joshua Kissoon + * @since 20140306 + */ +public class KadRefreshOperation implements Operation +{ + + private final KadServer server; + private final KademliaNode localNode; + private final DHT dht; + private final KadConfiguration config; + + public KadRefreshOperation(KadServer server, KademliaNode localNode, DHT dht, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.dht = dht; + this.config = config; + } + + @Override + public void execute() throws IOException + { + /* Run our BucketRefreshOperation to refresh buckets */ + new BucketRefreshOperation(this.server, this.localNode, this.config).execute(); + + /* After buckets have been refreshed, we refresh content */ + new ContentRefreshOperation(this.server, this.localNode, this.dht, this.config).execute(); + } +} diff --git a/src/kademlia/operation/NodeLookupOperation.java b/src/kademlia/operation/NodeLookupOperation.java new file mode 100644 index 0000000..2f3a702 --- /dev/null +++ b/src/kademlia/operation/NodeLookupOperation.java @@ -0,0 +1,339 @@ +package kademlia.operation; + +import kademlia.message.Receiver; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.exceptions.RoutingException; +import kademlia.message.Message; +import kademlia.message.NodeLookupMessage; +import kademlia.message.NodeReplyMessage; +import kademlia.node.KeyComparator; +import kademlia.node.Node; +import kademlia.node.KademliaId; + +/** + * Finds the K closest nodes to a specified identifier + * The algorithm terminates when it has gotten responses from the K closest nodes it has seen. + * Nodes that fail to respond are removed from consideration + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class NodeLookupOperation implements Operation, Receiver +{ + + /* Constants */ + private static final String UNASKED = "UnAsked"; + private static final String AWAITING = "Awaiting"; + private static final String ASKED = "Asked"; + private static final String FAILED = "Failed"; + + private final KadServer server; + private final KademliaNode localNode; + private final KadConfiguration config; + + private boolean error; + + private final Message lookupMessage; // Message sent to each peer + private final Map nodes; + + /* Tracks messages in transit and awaiting reply */ + private final Map messagesTransiting; + + /* Used to sort nodes */ + private final Comparator comparator; + + + { + messagesTransiting = new HashMap<>(); + } + + /** + * @param server KadServer used for communication + * @param localNode The local node making the communication + * @param lookupId The ID for which to find nodes close to + * @param config + */ + public NodeLookupOperation(KadServer server, KademliaNode localNode, KademliaId lookupId, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.config = config; + + this.lookupMessage = new NodeLookupMessage(localNode.getNode(), lookupId); + + /** + * We initialize a TreeMap to store nodes. + * This map will be sorted by which nodes are closest to the lookupId + */ + this.comparator = new KeyComparator(lookupId); + this.nodes = new TreeMap(this.comparator); + } + + /** + * @throws java.io.IOException + * @throws kademlia.exceptions.RoutingException + */ + @Override + public synchronized void execute() throws IOException, RoutingException + { + try + { + error = true; + + /* Set the local node as already asked */ + nodes.put(this.localNode.getNode(), ASKED); + + /** + * We add all nodes here instead of the K-Closest because there may be the case that the K-Closest are offline + * - The operation takes care of looking at the K-Closest. + */ + this.addNodes(this.localNode.getRoutingTable().getAllNodes()); + + /* If we haven't finished as yet, wait for a maximum of config.operationTimeout() time */ + int totalTimeWaited = 0; + int timeInterval = 10; // We re-check every 300 milliseconds + while (totalTimeWaited < this.config.operationTimeout()) + { + if (!this.askNodesorFinish()) + { + wait(timeInterval); + totalTimeWaited += timeInterval; + } + else + { + break; + } + } + + /** + * There is no need to throw an exception here! + * If the operation times out means we didn't get replies from all nodes, + * so lets just simply return the K-Closest nodes we knoe + */ +// if (error) +// { +// /* If we still haven't received any responses by then, do a routing timeout */ +// throw new RoutingException("Node Lookup Timeout."); +// } + + /* Now after we've finished, we would have an idea of offline nodes, lets update our routing table */ + this.localNode.getRoutingTable().setUnresponsiveContacts(this.getFailedNodes()); + + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + public List getClosestNodes() + { + return this.closestNodes(ASKED); + } + + /** + * Add nodes from this list to the set of nodes to lookup + * + * @param list The list from which to add nodes + */ + public void addNodes(List list) + { + for (Node o : list) + { + /* If this node is not in the list, add the node */ + if (!nodes.containsKey(o)) + { + nodes.put(o, UNASKED); + } + } + } + + /** + * Asks some of the K closest nodes seen but not yet queried. + * Assures that no more than DefaultConfiguration.CONCURRENCY messages are in transit at a time + * + * This method should be called every time a reply is received or a timeout occurs. + * + * If all K closest nodes have been asked and there are no messages in transit, + * the algorithm is finished. + * + * @return true if finished OR false otherwise + */ + private boolean askNodesorFinish() throws IOException + { + /* If >= CONCURRENCY nodes are in transit, don't do anything */ + if (this.config.maxConcurrentMessagesTransiting() <= this.messagesTransiting.size()) + { + return false; + } + + /* Get unqueried nodes among the K closest seen that have not FAILED */ + List unasked = this.closestNodesNotFailed(UNASKED); + + if (unasked.isEmpty() && this.messagesTransiting.isEmpty()) + { + /* We have no unasked nodes nor any messages in transit, we're finished! */ + error = false; + return true; + } + + /** + * Send messages to nodes in the list; + * making sure than no more than CONCURRENCY messsages are in transit + */ + for (int i = 0; (this.messagesTransiting.size() < this.config.maxConcurrentMessagesTransiting()) && (i < unasked.size()); i++) + { + Node n = (Node) unasked.get(i); + + int comm = server.sendMessage(n, lookupMessage, this); + + this.nodes.put(n, AWAITING); + this.messagesTransiting.put(comm, n); + } + + /* We're not finished as yet, return false */ + return false; + } + + /** + * @param status The status of the nodes to return + * + * @return The K closest nodes to the target lookupId given that have the specified status + */ + private List closestNodes(String status) + { + List closestNodes = new ArrayList<>(this.config.k()); + int remainingSpaces = this.config.k(); + + for (Map.Entry e : this.nodes.entrySet()) + { + if (status.equals(e.getValue())) + { + /* We got one with the required status, now add it */ + closestNodes.add((Node) e.getKey()); + if (--remainingSpaces == 0) + { + break; + } + } + } + + return closestNodes; + } + + /** + * Find The K closest nodes to the target lookupId given that have not FAILED. + * From those K, get those that have the specified status + * + * @param status The status of the nodes to return + * + * @return A List of the closest nodes + */ + private List closestNodesNotFailed(String status) + { + List closestNodes = new ArrayList<>(this.config.k()); + int remainingSpaces = this.config.k(); + + for (Map.Entry e : this.nodes.entrySet()) + { + if (!FAILED.equals(e.getValue())) + { + if (status.equals(e.getValue())) + { + /* We got one with the required status, now add it */ + closestNodes.add(e.getKey()); + } + + if (--remainingSpaces == 0) + { + break; + } + } + } + + return closestNodes; + } + + /** + * Receive and handle the incoming NodeReplyMessage + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public synchronized void receive(Message incoming, int comm) throws IOException + { + if (!(incoming instanceof NodeReplyMessage)) + { + /* Not sure why we get a message of a different type here... @todo Figure it out. */ + return; + } + /* We receive a NodeReplyMessage with a set of nodes, read this message */ + NodeReplyMessage msg = (NodeReplyMessage) incoming; + + /* Add the origin node to our routing table */ + Node origin = msg.getOrigin(); + this.localNode.getRoutingTable().insert(origin); + + /* Set that we've completed ASKing the origin node */ + this.nodes.put(origin, ASKED); + + /* Remove this msg from messagesTransiting since it's completed now */ + this.messagesTransiting.remove(comm); + + /* Add the received nodes to our nodes list to query */ + this.addNodes(msg.getNodes()); + this.askNodesorFinish(); + } + + /** + * A node does not respond or a packet was lost, we set this node as failed + * + * @param comm + * + * @throws java.io.IOException + */ + @Override + public synchronized void timeout(int comm) throws IOException + { + /* Get the node associated with this communication */ + Node n = this.messagesTransiting.get(comm); + + if (n == null) + { + return; + } + + /* Mark this node as failed and inform the routing table that it is unresponsive */ + this.nodes.put(n, FAILED); + this.localNode.getRoutingTable().setUnresponsiveContact(n); + this.messagesTransiting.remove(comm); + + this.askNodesorFinish(); + } + + public List getFailedNodes() + { + List failedNodes = new ArrayList<>(); + + for (Map.Entry e : this.nodes.entrySet()) + { + if (e.getValue().equals(FAILED)) + { + failedNodes.add(e.getKey()); + } + } + + return failedNodes; + } +} diff --git a/src/kademlia/operation/Operation.java b/src/kademlia/operation/Operation.java new file mode 100644 index 0000000..cb01922 --- /dev/null +++ b/src/kademlia/operation/Operation.java @@ -0,0 +1,21 @@ +package kademlia.operation; + +import java.io.IOException; +import kademlia.exceptions.RoutingException; + +/** + * An operation in the Kademlia routing protocol + * + * @author Joshua Kissoon + * @created 20140218 + */ +public interface Operation +{ + + /** + * Starts an operation and returns when the operation is finished + * + * @throws kademlia.exceptions.RoutingException + */ + public void execute() throws IOException, RoutingException; +} diff --git a/src/kademlia/operation/PingOperation.java b/src/kademlia/operation/PingOperation.java new file mode 100644 index 0000000..6cfe352 --- /dev/null +++ b/src/kademlia/operation/PingOperation.java @@ -0,0 +1,39 @@ +/** + * Implementation of the Kademlia Ping operation, + * This is on hold at the moment since I'm not sure if we'll use ping given the improvements mentioned in the paper. + * + * @author Joshua Kissoon + * @since 20140218 + */ +package kademlia.operation; + +import java.io.IOException; +import kademlia.core.KadServer; +import kademlia.exceptions.RoutingException; +import kademlia.node.Node; + +public class PingOperation implements Operation +{ + + private final KadServer server; + private final Node localNode; + private final Node toPing; + + /** + * @param server The Kademlia server used to send & receive messages + * @param local The local node + * @param toPing The node to send the ping message to + */ + public PingOperation(KadServer server, Node local, Node toPing) + { + this.server = server; + this.localNode = local; + this.toPing = toPing; + } + + @Override + public void execute() throws IOException, RoutingException + { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } +} diff --git a/src/kademlia/operation/StoreOperation.java b/src/kademlia/operation/StoreOperation.java new file mode 100644 index 0000000..09f4dc3 --- /dev/null +++ b/src/kademlia/operation/StoreOperation.java @@ -0,0 +1,84 @@ +package kademlia.operation; + +import java.io.IOException; +import java.util.List; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.core.KadServer; +import kademlia.dht.DHT; +import kademlia.dht.KadContent; +import kademlia.dht.StorageEntry; +import kademlia.message.Message; +import kademlia.message.StoreContentMessage; +import kademlia.node.Node; + +/** + * Operation that stores a DHT Content onto the K closest nodes to the content Key + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class StoreOperation implements Operation +{ + + private final KadServer server; + private final KademliaNode localNode; + private final StorageEntry storageEntry; + private final DHT localDht; + private final KadConfiguration config; + + /** + * @param server + * @param localNode + * @param storageEntry The content to be stored on the DHT + * @param localDht The local DHT + * @param config + */ + public StoreOperation(KadServer server, KademliaNode localNode, StorageEntry storageEntry, DHT localDht, KadConfiguration config) + { + this.server = server; + this.localNode = localNode; + this.storageEntry = storageEntry; + this.localDht = localDht; + this.config = config; + } + + @Override + public synchronized void execute() throws IOException + { + /* Get the nodes on which we need to store the content */ + NodeLookupOperation ndlo = new NodeLookupOperation(this.server, this.localNode, this.storageEntry.getContentMetadata().getKey(), this.config); + ndlo.execute(); + List nodes = ndlo.getClosestNodes(); + + /* Create the message */ + Message msg = new StoreContentMessage(this.localNode.getNode(), this.storageEntry); + + /*Store the message on all of the K-Nodes*/ + for (Node n : nodes) + { + if (n.equals(this.localNode.getNode())) + { + /* Store the content locally */ + this.localDht.store(this.storageEntry); + } + else + { + /** + * @todo Create a receiver that receives a store acknowledgement message to count how many nodes a content have been stored at + */ + this.server.sendMessage(n, msg, null); + } + } + } + + /** + * @return The number of nodes that have stored this content + * + * @todo Implement this method + */ + public int numNodesStoredAt() + { + return 1; + } +} diff --git a/src/kademlia/routing/Contact.java b/src/kademlia/routing/Contact.java new file mode 100644 index 0000000..896d1df --- /dev/null +++ b/src/kademlia/routing/Contact.java @@ -0,0 +1,118 @@ +package kademlia.routing; + +import kademlia.node.Node; + +/** + * Keeps information about contacts of the Node; Contacts are stored in the Buckets in the Routing Table. + * + * Contacts are used instead of nodes because more information is needed than just the node information. + * - Information such as + * -- Last seen time + * + * @author Joshua Kissoon + * @since 20140425 + * @updated 20140426 + */ +public class Contact implements Comparable +{ + + private final Node n; + private long lastSeen; + + /** + * Stale as described by Kademlia paper page 64 + * When a contact fails to respond, if the replacement cache is empty and there is no replacement for the contact, + * just mark it as stale. + * + * Now when a new contact is added, if the contact is stale, it is removed. + */ + private int staleCount; + + /** + * Create a contact object + * + * @param n The node associated with this contact + */ + public Contact(Node n) + { + this.n = n; + this.lastSeen = System.currentTimeMillis() / 1000L; + } + + public Node getNode() + { + return this.n; + } + + /** + * When a Node sees a contact a gain, the Node will want to update that it's seen recently, + * this method updates the last seen timestamp for this contact. + */ + public void setSeenNow() + { + this.lastSeen = System.currentTimeMillis() / 1000L; + } + + /** + * When last was this contact seen? + * + * @return long The last time this contact was seen. + */ + public long lastSeen() + { + return this.lastSeen; + } + + @Override + public boolean equals(Object c) + { + if (c instanceof Contact) + { + return ((Contact) c).getNode().equals(this.getNode()); + } + + return false; + } + + /** + * Increments the amount of times this count has failed to respond to a request. + */ + public void incrementStaleCount() + { + staleCount++; + } + + /** + * @return Integer Stale count + */ + public int staleCount() + { + return this.staleCount; + } + + /** + * Reset the stale count of the contact if it's recently seen + */ + public void resetStaleCount() + { + this.staleCount = 0; + } + + @Override + public int compareTo(Contact o) + { + if (this.getNode().equals(o.getNode())) + { + return 0; + } + + return (this.lastSeen() > o.lastSeen()) ? 1 : -1; + } + + @Override + public int hashCode() + { + return this.getNode().hashCode(); + } + +} diff --git a/src/kademlia/routing/ContactLastSeenComparator.java b/src/kademlia/routing/ContactLastSeenComparator.java new file mode 100644 index 0000000..6c1d097 --- /dev/null +++ b/src/kademlia/routing/ContactLastSeenComparator.java @@ -0,0 +1,34 @@ +package kademlia.routing; + +import java.util.Comparator; + +/** + * A Comparator to compare 2 contacts by their last seen time + * + * @author Joshua Kissoon + * @since 20140426 + */ +public class ContactLastSeenComparator implements Comparator +{ + + /** + * Compare two contacts to determine their order in the Bucket, + * Contacts are ordered by their last seen timestamp. + * + * @param c1 Contact 1 + * @param c2 Contact 2 + */ + @Override + public int compare(Contact c1, Contact c2) + { + if (c1.getNode().equals(c2.getNode())) + { + return 0; + } + else + { + /* We may have 2 different contacts with same last seen values so we can't return 0 here */ + return c1.lastSeen() > c2.lastSeen() ? 1 : -1; + } + } +} diff --git a/src/kademlia/routing/KadBucket.java b/src/kademlia/routing/KadBucket.java new file mode 100644 index 0000000..b0de8d5 --- /dev/null +++ b/src/kademlia/routing/KadBucket.java @@ -0,0 +1,87 @@ +package kademlia.routing; + +import java.util.List; +import kademlia.node.Node; + +/** + * A bucket used to store Contacts in the routing table. + * + * @author Joshua Kissoon + * @created 20140215 + */ +public interface KadBucket +{ + + /** + * Adds a contact to the bucket + * + * @param c the new contact + */ + public void insert(Contact c); + + /** + * Create a new contact and insert it into the bucket. + * + * @param n The node to create the contact from + */ + public void insert(Node n); + + /** + * Checks if this bucket contain a contact + * + * @param c The contact to check for + * + * @return boolean + */ + public boolean containsContact(Contact c); + + /** + * Checks if this bucket contain a node + * + * @param n The node to check for + * + * @return boolean + */ + public boolean containsNode(Node n); + + /** + * Remove a contact from this bucket. + * + * If there are replacement contacts in the replacement cache, + * select the last seen one and put it into the bucket while removing the required contact. + * + * If there are no contacts in the replacement cache, then we just mark the contact requested to be removed as stale. + * Marking as stale would actually be incrementing the stale count of the contact. + * + * @param c The contact to remove + * + * @return Boolean whether the removal was successful. + */ + public boolean removeContact(Contact c); + + /** + * Remove the contact object related to a node from this bucket + * + * @param n The node of the contact to remove + * + * @return Boolean whether the removal was successful. + */ + public boolean removeNode(Node n); + + /** + * Counts the number of contacts in this bucket. + * + * @return Integer The number of contacts in this bucket + */ + public int numContacts(); + + /** + * @return Integer The depth of this bucket in the RoutingTable + */ + public int getDepth(); + + /** + * @return An Iterable structure with all contacts in this bucket + */ + public List getContacts(); +} diff --git a/src/kademlia/routing/KadBucketImpl.java b/src/kademlia/routing/KadBucketImpl.java new file mode 100644 index 0000000..8f83e5d --- /dev/null +++ b/src/kademlia/routing/KadBucketImpl.java @@ -0,0 +1,261 @@ +package kademlia.routing; + +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.TreeSet; +import kademlia.core.KadConfiguration; +import kademlia.node.Node; + +/** + * A bucket in the Kademlia routing table + * + * @author Joshua Kissoon + * @created 20140215 + */ +public class KadBucketImpl implements KadBucket +{ + + /* How deep is this bucket in the Routing Table */ + private final int depth; + + /* Contacts stored in this routing table */ + private final TreeSet contacts; + + /* A set of last seen contacts that can replace any current contact that is unresponsive */ + private final TreeSet replacementCache; + + private final KadConfiguration config; + + + { + contacts = new TreeSet<>(); + replacementCache = new TreeSet<>(); + } + + /** + * @param depth How deep in the routing tree is this bucket + * @param config + */ + public KadBucketImpl(int depth, KadConfiguration config) + { + this.depth = depth; + this.config = config; + } + + @Override + public synchronized void insert(Contact c) + { + if (this.contacts.contains(c)) + { + /** + * If the contact is already in the bucket, lets update that we've seen it + * We need to remove and re-add the contact to get the Sorted Set to update sort order + */ + Contact tmp = this.removeFromContacts(c.getNode()); + tmp.setSeenNow(); + tmp.resetStaleCount(); + this.contacts.add(tmp); + } + else + { + /* If the bucket is filled, so put the contacts in the replacement cache */ + if (contacts.size() >= this.config.k()) + { + /* If the cache is empty, we check if any contacts are stale and replace the stalest one */ + Contact stalest = null; + for (Contact tmp : this.contacts) + { + if (tmp.staleCount() >= this.config.stale()) + { + /* Contact is stale */ + if (stalest == null) + { + stalest = tmp; + } + else if (tmp.staleCount() > stalest.staleCount()) + { + stalest = tmp; + } + } + } + + /* If we have a stale contact, remove it and add the new contact to the bucket */ + if (stalest != null) + { + this.contacts.remove(stalest); + this.contacts.add(c); + } + else + { + /* No stale contact, lets insert this into replacement cache */ + this.insertIntoReplacementCache(c); + } + } + else + { + this.contacts.add(c); + } + } + } + + @Override + public synchronized void insert(Node n) + { + this.insert(new Contact(n)); + } + + @Override + public synchronized boolean containsContact(Contact c) + { + return this.contacts.contains(c); + } + + @Override + public synchronized boolean containsNode(Node n) + { + return this.containsContact(new Contact(n)); + } + + @Override + public synchronized boolean removeContact(Contact c) + { + /* If the contact does not exist, then we failed to remove it */ + if (!this.contacts.contains(c)) + { + return false; + } + + /* Contact exist, lets remove it only if our replacement cache has a replacement */ + if (!this.replacementCache.isEmpty()) + { + /* Replace the contact with one from the replacement cache */ + this.contacts.remove(c); + Contact replacement = this.replacementCache.first(); + this.contacts.add(replacement); + this.replacementCache.remove(replacement); + } + else + { + /* There is no replacement, just increment the contact's stale count */ + this.getFromContacts(c.getNode()).incrementStaleCount(); + } + + return true; + } + + private synchronized Contact getFromContacts(Node n) + { + for (Contact c : this.contacts) + { + if (c.getNode().equals(n)) + { + return c; + } + } + + /* This contact does not exist */ + throw new NoSuchElementException("The contact does not exist in the contacts list."); + } + + private synchronized Contact removeFromContacts(Node n) + { + for (Contact c : this.contacts) + { + if (c.getNode().equals(n)) + { + this.contacts.remove(c); + return c; + } + } + + /* We got here means this element does not exist */ + throw new NoSuchElementException("Node does not exist in the replacement cache. "); + } + + @Override + public synchronized boolean removeNode(Node n) + { + return this.removeContact(new Contact(n)); + } + + @Override + public synchronized int numContacts() + { + return this.contacts.size(); + } + + @Override + public synchronized int getDepth() + { + return this.depth; + } + + @Override + public synchronized List getContacts() + { + return (this.contacts.isEmpty()) ? new ArrayList<>() : new ArrayList<>(this.contacts); + } + + /** + * When the bucket is filled, we keep extra contacts in the replacement cache. + */ + private synchronized void insertIntoReplacementCache(Contact c) + { + /* Just return if this contact is already in our replacement cache */ + if (this.replacementCache.contains(c)) + { + /** + * If the contact is already in the bucket, lets update that we've seen it + * We need to remove and re-add the contact to get the Sorted Set to update sort order + */ + Contact tmp = this.removeFromReplacementCache(c.getNode()); + tmp.setSeenNow(); + this.replacementCache.add(tmp); + } + else if (this.replacementCache.size() > this.config.k()) + { + /* if our cache is filled, we remove the least recently seen contact */ + this.replacementCache.remove(this.replacementCache.last()); + this.replacementCache.add(c); + } + else + { + this.replacementCache.add(c); + } + } + + private synchronized Contact removeFromReplacementCache(Node n) + { + for (Contact c : this.replacementCache) + { + if (c.getNode().equals(n)) + { + this.replacementCache.remove(c); + return c; + } + } + + /* We got here means this element does not exist */ + throw new NoSuchElementException("Node does not exist in the replacement cache. "); + } + + @Override + public synchronized String toString() + { + StringBuilder sb = new StringBuilder("Bucket at depth: "); + sb.append(this.depth); + sb.append("\n Nodes: \n"); + for (Contact n : this.contacts) + { + sb.append("Node: "); + sb.append(n.getNode().getNodeId().toString()); + sb.append(" (stale: "); + sb.append(n.staleCount()); + sb.append(")"); + sb.append("\n"); + } + + return sb.toString(); + } +} diff --git a/src/kademlia/routing/KadRoutingTable.java b/src/kademlia/routing/KadRoutingTable.java new file mode 100644 index 0000000..6abdcc1 --- /dev/null +++ b/src/kademlia/routing/KadRoutingTable.java @@ -0,0 +1,91 @@ +package kademlia.routing; + +import java.util.List; +import kademlia.core.KadConfiguration; +import kademlia.node.Node; +import kademlia.node.KademliaId; + +/** + * Specification for Kademlia's Routing Table + * + * @author Joshua Kissoon + * @since 20140501 + */ +public interface KadRoutingTable +{ + + /** + * Initialize the RoutingTable to it's default state + */ + public void initialize(); + + /** + * Sets the configuration file for this routing table + * + * @param config + */ + public void setConfiguration(KadConfiguration config); + + /** + * Adds a contact to the routing table based on how far it is from the LocalNode. + * + * @param c The contact to add + */ + public void insert(Contact c); + + /** + * Adds a node to the routing table based on how far it is from the LocalNode. + * + * @param n The node to add + */ + public void insert(Node n); + + /** + * Compute the bucket ID in which a given node should be placed; the bucketId is computed based on how far the node is away from the Local Node. + * + * @param nid The NodeId for which we want to find which bucket it belong to + * + * @return Integer The bucket ID in which the given node should be placed. + */ + public int getBucketId(KademliaId nid); + + /** + * Find the closest set of contacts to a given NodeId + * + * @param target The NodeId to find contacts close to + * @param numNodesRequired The number of contacts to find + * + * @return List A List of contacts closest to target + */ + public List findClosest(KademliaId target, int numNodesRequired); + + /** + * @return List A List of all Nodes in this RoutingTable + */ + public List getAllNodes(); + + /** + * @return List A List of all Nodes in this RoutingTable + */ + public List getAllContacts(); + + /** + * @return Bucket[] The buckets in this Kad Instance + */ + public KadBucket[] getBuckets(); + + /** + * Method used by operations to notify the routing table of any contacts that have been unresponsive. + * + * @param contacts The set of unresponsive contacts + */ + public void setUnresponsiveContacts(List contacts); + + /** + * Method used by operations to notify the routing table of any contacts that have been unresponsive. + * + * @param n + */ + public void setUnresponsiveContact(Node n); + +} diff --git a/src/kademlia/routing/RoutingTable.java b/src/kademlia/routing/RoutingTable.java new file mode 100644 index 0000000..465d1e9 --- /dev/null +++ b/src/kademlia/routing/RoutingTable.java @@ -0,0 +1,238 @@ +package kademlia.routing; + +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import kademlia.core.KadConfiguration; +import kademlia.node.KeyComparator; +import kademlia.node.Node; +import kademlia.node.KademliaId; + +/** + * Implementation of a Kademlia routing table + * + * @author Joshua Kissoon + * @created 20140215 + */ +public class RoutingTable implements KadRoutingTable +{ + + private final Node localNode; // The current node + private transient KadBucket[] buckets; + + private transient KadConfiguration config; + + public RoutingTable(Node localNode, KadConfiguration config) + { + this.localNode = localNode; + this.config = config; + + /* Initialize all of the buckets to a specific depth */ + this.initialize(); + + /* Insert the local node */ + this.insert(localNode); + } + + /** + * Initialize the RoutingTable to it's default state + */ + @Override + public final void initialize() + { + this.buckets = new KadBucket[KademliaId.ID_LENGTH]; + for (int i = 0; i < KademliaId.ID_LENGTH; i++) + { + buckets[i] = new KadBucketImpl(i, this.config); + } + } + + @Override + public void setConfiguration(KadConfiguration config) + { + this.config = config; + } + + /** + * Adds a contact to the routing table based on how far it is from the LocalNode. + * + * @param c The contact to add + */ + @Override + public synchronized final void insert(Contact c) + { + this.buckets[this.getBucketId(c.getNode().getNodeId())].insert(c); + } + + /** + * Adds a node to the routing table based on how far it is from the LocalNode. + * + * @param n The node to add + */ + @Override + public synchronized final void insert(Node n) + { + this.buckets[this.getBucketId(n.getNodeId())].insert(n); + } + + /** + * Compute the bucket ID in which a given node should be placed; the bucketId is computed based on how far the node is away from the Local Node. + * + * @param nid The NodeId for which we want to find which bucket it belong to + * + * @return Integer The bucket ID in which the given node should be placed. + */ + @Override + public final int getBucketId(KademliaId nid) + { + int bId = this.localNode.getNodeId().getDistance(nid) - 1; + + /* If we are trying to insert a node into it's own routing table, then the bucket ID will be -1, so let's just keep it in bucket 0 */ + return bId < 0 ? 0 : bId; + } + + /** + * Find the closest set of contacts to a given NodeId + * + * @param target The NodeId to find contacts close to + * @param numNodesRequired The number of contacts to find + * + * @return List A List of contacts closest to target + */ + @Override + public synchronized final List findClosest(KademliaId target, int numNodesRequired) + { + TreeSet sortedSet = new TreeSet<>(new KeyComparator(target)); + sortedSet.addAll(this.getAllNodes()); + + List closest = new ArrayList<>(numNodesRequired); + + /* Now we have the sorted set, lets get the top numRequired */ + int count = 0; + for (Node n : sortedSet) + { + closest.add(n); + if (++count == numNodesRequired) + { + break; + } + } + return closest; + } + + /** + * @return List A List of all Nodes in this RoutingTable + */ + @Override + public synchronized final List getAllNodes() + { + List nodes = new ArrayList<>(); + + for (KadBucket b : this.buckets) + { + for (Contact c : b.getContacts()) + { + nodes.add(c.getNode()); + } + } + + return nodes; + } + + /** + * @return List A List of all Nodes in this RoutingTable + */ + @Override + public final List getAllContacts() + { + List contacts = new ArrayList<>(); + + for (KadBucket b : this.buckets) + { + contacts.addAll(b.getContacts()); + } + + return contacts; + } + + /** + * @return Bucket[] The buckets in this Kad Instance + */ + @Override + public final KadBucket[] getBuckets() + { + return this.buckets; + } + + /** + * Set the KadBuckets of this routing table, mainly used when retrieving saved state + * + * @param buckets + */ + public final void setBuckets(KadBucket[] buckets) + { + this.buckets = buckets; + } + + /** + * Method used by operations to notify the routing table of any contacts that have been unresponsive. + * + * @param contacts The set of unresponsive contacts + */ + @Override + public void setUnresponsiveContacts(List contacts) + { + if (contacts.isEmpty()) + { + return; + } + for (Node n : contacts) + { + this.setUnresponsiveContact(n); + } + } + + /** + * Method used by operations to notify the routing table of any contacts that have been unresponsive. + * + * @param n + */ + @Override + public synchronized void setUnresponsiveContact(Node n) + { + int bucketId = this.getBucketId(n.getNodeId()); + + /* Remove the contact from the bucket */ + this.buckets[bucketId].removeNode(n); + } + + @Override + public synchronized final String toString() + { + StringBuilder sb = new StringBuilder("\nPrinting Routing Table Started ***************** \n"); + int totalContacts = 0; + for (KadBucket b : this.buckets) + { + if (b.numContacts() > 0) + { + totalContacts += b.numContacts(); + sb.append("# nodes in Bucket with depth "); + sb.append(b.getDepth()); + sb.append(": "); + sb.append(b.numContacts()); + sb.append("\n"); + sb.append(b.toString()); + sb.append("\n"); + } + } + + sb.append("\nTotal Contacts: "); + sb.append(totalContacts); + sb.append("\n\n"); + + sb.append("Printing Routing Table Ended ******************** "); + + return sb.toString(); + } + +} diff --git a/src/kademlia/tests/AutoRefreshOperationTest.java b/src/kademlia/tests/AutoRefreshOperationTest.java new file mode 100644 index 0000000..6e2fc3e --- /dev/null +++ b/src/kademlia/tests/AutoRefreshOperationTest.java @@ -0,0 +1,87 @@ +package kademlia.tests; + +import java.util.Timer; +import java.util.TimerTask; +import kademlia.core.DefaultConfiguration; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.node.KademliaId; + +/** + * Testing the Kademlia Auto Content and Node table refresh operations + * + * @author Joshua Kissoon + * @since 20140309 + */ +public class AutoRefreshOperationTest +{ + + public AutoRefreshOperationTest() + { + try + { + /* Setting up 2 Kad networks */ + final KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF456789djem45674DH"), 12049); + final KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("AJDHR678947584567464"), 4585); + final KademliaNode kad3 = new KademliaNode("Shameer", new KademliaId("AS84k6789KRNS45KFJ8W"), 8104); + final KademliaNode kad4 = new KademliaNode("Lokesh.", new KademliaId("ASF45678947A845674GG"), 8335); + final KademliaNode kad5 = new KademliaNode("Chandu.", new KademliaId("AS84kUD894758456dyrj"), 13345); + + /* Connecting nodes */ + System.out.println("Connecting Nodes"); + kad2.bootstrap(kad1.getNode()); + kad3.bootstrap(kad2.getNode()); + kad4.bootstrap(kad2.getNode()); + kad5.bootstrap(kad4.getNode()); + + DHTContentImpl c = new DHTContentImpl(new KademliaId("AS84k678947584567465"), kad1.getOwnerId()); + c.setData("Setting the data"); + + System.out.println("\n Content ID: " + c.getKey()); + System.out.println(kad1.getNode() + " Distance from content: " + kad1.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad2.getNode() + " Distance from content: " + kad2.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad3.getNode() + " Distance from content: " + kad3.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad4.getNode() + " Distance from content: " + kad4.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad5.getNode() + " Distance from content: " + kad5.getNode().getNodeId().getDistance(c.getKey())); + System.out.println("\nSTORING CONTENT 1 locally on " + kad1.getOwnerId() + "\n\n\n\n"); + + kad1.putLocally(c); + + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + System.out.println(kad4); + System.out.println(kad5); + + /* Print the node states every few minutes */ + KadConfiguration config = new DefaultConfiguration(); + Timer timer = new Timer(true); + timer.schedule( + new TimerTask() + { + @Override + public void run() + { + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + System.out.println(kad4); + System.out.println(kad5); + } + }, + // Delay // Interval + config.restoreInterval(), config.restoreInterval() + ); + } + + catch (Exception e) + { + e.printStackTrace(); + } + } + + public static void main(String[] args) + { + new AutoRefreshOperationTest(); + } +} diff --git a/src/kademlia/tests/AutoRefreshOperationTest2.java b/src/kademlia/tests/AutoRefreshOperationTest2.java new file mode 100644 index 0000000..8536567 --- /dev/null +++ b/src/kademlia/tests/AutoRefreshOperationTest2.java @@ -0,0 +1,76 @@ +package kademlia.tests; + +import java.util.Timer; +import java.util.TimerTask; +import kademlia.core.DefaultConfiguration; +import kademlia.KademliaNode; +import kademlia.core.KadConfiguration; +import kademlia.node.KademliaId; + +/** + * Testing the Kademlia Auto Content and Node table refresh operations + * + * @author Joshua Kissoon + * @since 20140309 + */ +public class AutoRefreshOperationTest2 +{ + + public AutoRefreshOperationTest2() + { + try + { + /* Setting up 2 Kad networks */ + final KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF456789djem4567463"), 12049); + final KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("AS84k678DJRW84567465"), 4585); + final KademliaNode kad3 = new KademliaNode("Shameer", new KademliaId("AS84k67894758456746A"), 8104); + + /* Connecting nodes */ + System.out.println("Connecting Nodes"); + kad2.bootstrap(kad1.getNode()); + kad3.bootstrap(kad2.getNode()); + + DHTContentImpl c = new DHTContentImpl(new KademliaId("AS84k678947584567465"), kad1.getOwnerId()); + c.setData("Setting the data"); + kad1.putLocally(c); + + System.out.println("\n Content ID: " + c.getKey()); + System.out.println(kad1.getNode() + " Distance from content: " + kad1.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad2.getNode() + " Distance from content: " + kad2.getNode().getNodeId().getDistance(c.getKey())); + System.out.println(kad3.getNode() + " Distance from content: " + kad3.getNode().getNodeId().getDistance(c.getKey())); + System.out.println("\nSTORING CONTENT 1 locally on " + kad1.getOwnerId() + "\n\n\n\n"); + + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + + /* Print the node states every few minutes */ + KadConfiguration config = new DefaultConfiguration(); + Timer timer = new Timer(true); + timer.schedule( + new TimerTask() + { + @Override + public void run() + { + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + } + }, + // Delay // Interval + config.restoreInterval(), config.restoreInterval() + ); + } + + catch (Exception e) + { + e.printStackTrace(); + } + } + + public static void main(String[] args) + { + new AutoRefreshOperationTest2(); + } +} diff --git a/src/kademlia/tests/ContentSendingTest.java b/src/kademlia/tests/ContentSendingTest.java new file mode 100644 index 0000000..c1193e2 --- /dev/null +++ b/src/kademlia/tests/ContentSendingTest.java @@ -0,0 +1,60 @@ +package kademlia.tests; + +import java.io.IOException; +import java.util.UUID; +import kademlia.dht.GetParameter; +import kademlia.KademliaNode; +import kademlia.dht.StorageEntry; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.node.KademliaId; + +/** + * Testing sending and receiving content between 2 Nodes on a network + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class ContentSendingTest +{ + + public static void main(String[] args) + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567467"), 7574); + System.out.println("Created Node Kad 1: " + kad1.getNode().getNodeId()); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASERTKJDHGVHERJHGFLK"), 7572); + System.out.println("Created Node Kad 2: " + kad2.getNode().getNodeId()); + kad2.bootstrap(kad1.getNode()); + + /** + * Lets create the content and share it + */ + String data = ""; + for (int i = 0; i < 500; i++) + { + data += UUID.randomUUID(); + } + System.out.println(data); + DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), data); + kad2.put(c); + + /** + * Lets retrieve the content + */ + System.out.println("Retrieving Content"); + GetParameter gp = new GetParameter(c.getKey(), DHTContentImpl.TYPE); + gp.setOwnerId(c.getOwnerId()); + System.out.println("Get Parameter: " + gp); + StorageEntry conte = kad2.get(gp); + System.out.println("Content Found: " + new DHTContentImpl().fromBytes(conte.getContent().getBytes())); + System.out.println("Content Metadata: " + conte.getContentMetadata()); + + } + catch (IOException | ContentNotFoundException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/kademlia/tests/ContentUpdatingTest.java b/src/kademlia/tests/ContentUpdatingTest.java new file mode 100644 index 0000000..9b35883 --- /dev/null +++ b/src/kademlia/tests/ContentUpdatingTest.java @@ -0,0 +1,59 @@ +package kademlia.tests; + +import java.io.IOException; +import kademlia.dht.GetParameter; +import kademlia.KademliaNode; +import kademlia.dht.StorageEntry; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.node.KademliaId; + +/** + * Testing sending and receiving content between 2 Nodes on a network + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class ContentUpdatingTest +{ + + public static void main(String[] args) + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567467"), 7574); + System.out.println("Created Node Kad 1: " + kad1.getNode().getNodeId()); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASERTKJDHGVHERJHGFLK"), 7572); + System.out.println("Created Node Kad 2: " + kad2.getNode().getNodeId()); + kad2.bootstrap(kad1.getNode()); + + /* Lets create the content and share it */ + DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), "Some Data"); + kad2.put(c); + + /* Lets retrieve the content */ + System.out.println("Retrieving Content"); + GetParameter gp = new GetParameter(c.getKey(), DHTContentImpl.TYPE, c.getOwnerId()); + + System.out.println("Get Parameter: " + gp); + StorageEntry conte = kad2.get(gp); + System.out.println("Content Found: " + new DHTContentImpl().fromBytes(conte.getContent().getBytes())); + System.out.println("Content Metadata: " + conte.getContentMetadata()); + + /* Lets update the content and put it again */ + c.setData("Some New Data"); + kad2.put(c); + + /* Lets retrieve the content */ + System.out.println("Retrieving Content Again"); + conte = kad2.get(gp); + System.out.println("Content Found: " + new DHTContentImpl().fromBytes(conte.getContent().getBytes())); + System.out.println("Content Metadata: " + conte.getContentMetadata()); + + } + catch (IOException | ContentNotFoundException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/kademlia/tests/DHTContentImpl.java b/src/kademlia/tests/DHTContentImpl.java new file mode 100644 index 0000000..71d0085 --- /dev/null +++ b/src/kademlia/tests/DHTContentImpl.java @@ -0,0 +1,111 @@ +package kademlia.tests; + +import com.google.gson.Gson; +import kademlia.dht.KadContent; +import kademlia.node.KademliaId; + +/** + * A simple DHT Content object to test DHT storage + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class DHTContentImpl implements KadContent +{ + + public static final transient String TYPE = "DHTContentImpl"; + + private KademliaId key; + private String data; + private String ownerId; + private final long createTs; + private long updateTs; + + + { + this.createTs = this.updateTs = System.currentTimeMillis() / 1000L; + } + + public DHTContentImpl() + { + + } + + public DHTContentImpl(String ownerId, String data) + { + this.ownerId = ownerId; + this.data = data; + this.key = new KademliaId(); + } + + public DHTContentImpl(KademliaId key, String ownerId) + { + this.key = key; + this.ownerId = ownerId; + } + + public void setData(String newData) + { + this.data = newData; + this.setUpdated(); + } + + @Override + public KademliaId getKey() + { + return this.key; + } + + @Override + public String getType() + { + return TYPE; + } + + @Override + public String getOwnerId() + { + return this.ownerId; + } + + /** + * Set the content as updated + */ + public void setUpdated() + { + this.updateTs = System.currentTimeMillis() / 1000L; + } + + @Override + public long getCreatedTimestamp() + { + return this.createTs; + } + + @Override + public long getLastUpdatedTimestamp() + { + return this.updateTs; + } + + @Override + public byte[] toBytes() + { + Gson gson = new Gson(); + return gson.toJson(this).getBytes(); + } + + @Override + public DHTContentImpl fromBytes(byte[] data) + { + Gson gson = new Gson(); + DHTContentImpl val = gson.fromJson(new String(data), DHTContentImpl.class); + return val; + } + + @Override + public String toString() + { + return "DHTContentImpl[{data=" + this.data + "{ {key:" + this.key + "}]"; + } +} diff --git a/src/kademlia/tests/NodeConnectionTest.java b/src/kademlia/tests/NodeConnectionTest.java new file mode 100644 index 0000000..fc7a25a --- /dev/null +++ b/src/kademlia/tests/NodeConnectionTest.java @@ -0,0 +1,70 @@ +package kademlia.tests; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.node.KademliaId; + +/** + * Testing connecting 2 nodes to each other + * + * @author Joshua Kissoon + * @created 20140219 + */ +public class NodeConnectionTest +{ + + public static void main(String[] args) + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567467"), 7574); + System.out.println("Created Node Kad 1: " + kad1.getNode().getNodeId()); + + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASERTKJDHGVHERJHGFLK"), 7572); + //NodeId diff12 = kad1.getNode().getNodeId().xor(kad2.getNode().getNodeId()); + System.out.println("Created Node Kad 2: " + kad2.getNode().getNodeId()); +// System.out.println(kad1.getNode().getNodeId() + " ^ " + kad2.getNode().getNodeId() + " = " + diff12); +// System.out.println("Kad 1 - Kad 2 distance: " + diff12.getFirstSetBitIndex()); + + /* Connecting 2 to 1 */ + System.out.println("Connecting Kad 1 and Kad 2"); + kad1.bootstrap(kad2.getNode()); + +// System.out.println("Kad 1: "); +// System.out.println(kad1.getNode().getRoutingTable()); +// System.out.println("Kad 2: "); +// System.out.println(kad2.getNode().getRoutingTable()); + + /* Creating a new node 3 and connecting it to 1, hoping it'll get onto 2 also */ + KademliaNode kad3 = new KademliaNode("Jessica", new KademliaId("ASERTKJDOLKMNBVFR45G"), 7783); + System.out.println("\n\n\n\n\n\nCreated Node Kad 3: " + kad3.getNode().getNodeId()); + + System.out.println("Connecting Kad 3 and Kad 2"); + kad3.bootstrap(kad2.getNode()); + +// NodeId diff32 = kad3.getNode().getNodeId().xor(kad2.getNode().getNodeId()); +// NodeId diff31 = kad1.getNode().getNodeId().xor(kad3.getNode().getNodeId()); +// System.out.println("Kad 3 - Kad 1 distance: " + diff31.getFirstSetBitIndex()); +// System.out.println("Kad 3 - Kad 2 distance: " + diff32.getFirstSetBitIndex()); + KademliaNode kad4 = new KademliaNode("Sandy", new KademliaId("ASERTK85OLKMN85FR4SS"), 7789); + System.out.println("\n\n\n\n\n\nCreated Node Kad 4: " + kad4.getNode().getNodeId()); + + System.out.println("Connecting Kad 4 and Kad 2"); + kad4.bootstrap(kad2.getNode()); + + System.out.println("\n\nKad 1: " + kad1.getNode().getNodeId() + " Routing Table: "); + System.out.println(kad1.getRoutingTable()); + System.out.println("\n\nKad 2: " + kad2.getNode().getNodeId() + " Routing Table: "); + System.out.println(kad2.getRoutingTable()); + System.out.println("\n\nKad 3: " + kad3.getNode().getNodeId() + " Routing Table: "); + System.out.println(kad3.getRoutingTable()); + System.out.println("\n\nKad 4: " + kad4.getNode().getNodeId() + " Routing Table: "); + System.out.println(kad4.getRoutingTable()); + } + catch (IOException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/kademlia/tests/RefreshOperationTest.java b/src/kademlia/tests/RefreshOperationTest.java new file mode 100644 index 0000000..5044403 --- /dev/null +++ b/src/kademlia/tests/RefreshOperationTest.java @@ -0,0 +1,45 @@ +package kademlia.tests; + +import java.io.IOException; +import kademlia.dht.GetParameter; +import kademlia.KademliaNode; +import kademlia.dht.StorageEntry; +import kademlia.exceptions.ContentNotFoundException; +import kademlia.node.KademliaId; + +/** + * Testing sending and receiving content between 2 Nodes on a network + * + * @author Joshua Kissoon + * @since 20140224 + */ +public class RefreshOperationTest +{ + + public static void main(String[] args) + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567467"), 7574); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASERTKJDHGVHERJHGFLK"), 7572); + kad2.bootstrap(kad1.getNode()); + + /* Lets create the content and share it */ + DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), "Some Data"); + kad2.put(c); + + /* Lets retrieve the content */ + GetParameter gp = new GetParameter(c.getKey(), DHTContentImpl.TYPE); + gp.setType(DHTContentImpl.TYPE); + gp.setOwnerId(c.getOwnerId()); + StorageEntry conte = kad2.get(gp); + + kad2.refresh(); + } + catch (IOException | ContentNotFoundException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/kademlia/tests/RoutingTableSimulation.java b/src/kademlia/tests/RoutingTableSimulation.java new file mode 100644 index 0000000..90858bc --- /dev/null +++ b/src/kademlia/tests/RoutingTableSimulation.java @@ -0,0 +1,57 @@ +package kademlia.tests; + +import kademlia.KademliaNode; +import kademlia.node.KademliaId; +import kademlia.routing.RoutingTable; + +/** + * Testing how the routing table works and checking if everything works properly + * + * @author Joshua Kissoon + * @since 20140426 + */ +public class RoutingTableSimulation +{ + + public RoutingTableSimulation() + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567463"), 12049); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASF45678947584567464"), 4585); + KademliaNode kad3 = new KademliaNode("Shameer", new KademliaId("ASF45678947584567465"), 8104); + KademliaNode kad4 = new KademliaNode("Lokesh", new KademliaId("ASF45678947584567466"), 8335); + KademliaNode kad5 = new KademliaNode("Chandu", new KademliaId("ASF45678947584567467"), 13345); + + RoutingTable rt = kad1.getRoutingTable(); + + rt.insert(kad2.getNode()); + rt.insert(kad3.getNode()); + rt.insert(kad4.getNode()); + System.out.println(rt); + + rt.insert(kad5.getNode()); + System.out.println(rt); + + rt.insert(kad3.getNode()); + System.out.println(rt); + + + /* Lets shut down a node and then try putting a content on the network. We'll then see how the un-responsive contacts work */ + } + catch (IllegalStateException e) + { + + } + catch (Exception e) + { + e.printStackTrace(); + } + } + + public static void main(String[] args) + { + new RoutingTableSimulation(); + } +} diff --git a/src/kademlia/tests/RoutingTableStateTesting.java b/src/kademlia/tests/RoutingTableStateTesting.java new file mode 100644 index 0000000..6dd0ee2 --- /dev/null +++ b/src/kademlia/tests/RoutingTableStateTesting.java @@ -0,0 +1,149 @@ +package kademlia.tests; + +import java.io.IOException; +import java.util.Scanner; +import kademlia.KademliaNode; +import kademlia.dht.KadContent; +import kademlia.node.KademliaId; + +/** + * Testing how the routing table works and it's state after different operations + * + * @author Joshua Kissoon + * @since 20140426 + */ +public class RoutingTableStateTesting +{ + + KademliaNode[] kads; + + public int numKads = 10; + + public RoutingTableStateTesting() + { + try + { + /* Setting up Kad networks */ + kads = new KademliaNode[numKads]; + + kads[0] = new KademliaNode("user0", new KademliaId("HRF456789SD584567460"), 1334); + kads[1] = new KademliaNode("user1", new KademliaId("ASF456789475DS567461"), 1209); + kads[2] = new KademliaNode("user2", new KademliaId("AFG45678947584567462"), 4585); + kads[3] = new KademliaNode("user3", new KademliaId("FSF45J38947584567463"), 8104); + kads[4] = new KademliaNode("user4", new KademliaId("ASF45678947584567464"), 8335); + kads[5] = new KademliaNode("user5", new KademliaId("GHF4567894DR84567465"), 13345); + kads[6] = new KademliaNode("user6", new KademliaId("ASF45678947584567466"), 12049); + kads[7] = new KademliaNode("user7", new KademliaId("AE345678947584567467"), 14585); + kads[8] = new KademliaNode("user8", new KademliaId("ASAA5678947584567468"), 18104); + kads[9] = new KademliaNode("user9", new KademliaId("ASF456789475845674U9"), 18335); + + for (int i = 1; i < numKads; i++) + { + kads[i].bootstrap(kads[0].getNode()); + } + + /* Lets shut down a node and then try putting a content on the network. We'll then see how the un-responsive contacts work */ + } + catch (Exception e) + { + e.printStackTrace(); + } + } + + public KadContent putContent(String content, KademliaNode owner) + { + DHTContentImpl c = null; + try + { + c = new DHTContentImpl(owner.getOwnerId(), "Some Data"); + owner.put(c); + return c; + } + catch (IOException e) + { + System.err.println("Error whiles putting content " + content + " from owner: " + owner.getOwnerId()); + } + + return c; + } + + public void shutdownKad(KademliaNode kad) + { + try + { + kad.shutdown(false); + } + catch (IOException ex) + { + System.err.println("Error whiles shutting down node with owner: " + kad.getOwnerId()); + } + } + + public void printRoutingTable(int kadId) + { + System.out.println(kads[kadId].getRoutingTable()); + } + + public void printRoutingTables() + { + for (int i = 0; i < numKads; i++) + { + this.printRoutingTable(i); + } + } + + public void printStorage(int kadId) + { + System.out.println(kads[kadId].getDHT()); + } + + public void printStorage() + { + for (int i = 0; i < numKads; i++) + { + this.printStorage(i); + } + } + + public static void main(String[] args) + { + + RoutingTableStateTesting rtss = new RoutingTableStateTesting(); + + try + { + rtss.printRoutingTables(); + + /* Lets shut down a node to test the node removal operation */ + rtss.shutdownKad(rtss.kads[3]); + + rtss.putContent("Content owned by kad0", rtss.kads[0]); + rtss.printStorage(); + + Thread.sleep(1000); + + /* kad3 should be removed from their routing tables by now. */ + rtss.printRoutingTables(); + } + catch (InterruptedException ex) + { + + } + + Scanner sc = new Scanner(System.in); + while (true) + { + System.out.println("\n\n ************************* Options **************************** \n"); + System.out.println("1 i - Print routing table of node i"); + int val1 = sc.nextInt(); + int val2 = sc.nextInt(); + + switch (val1) + { + case 1: + rtss.printRoutingTable(val2); + break; + } + } + } +} diff --git a/src/kademlia/tests/SaveStateTest.java b/src/kademlia/tests/SaveStateTest.java new file mode 100644 index 0000000..1b46593 --- /dev/null +++ b/src/kademlia/tests/SaveStateTest.java @@ -0,0 +1,96 @@ +package kademlia.tests; + +import kademlia.KademliaNode; +import kademlia.node.KademliaId; + +/** + * Testing the save and retrieve state operations + * + * @author Joshua Kissoon + * @since 20140309 + */ +public class SaveStateTest +{ + + public SaveStateTest() + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567463"), 12049); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASF45678947584567464"), 4585); + KademliaNode kad3 = new KademliaNode("Shameer", new KademliaId("ASF45678947584567465"), 8104); + KademliaNode kad4 = new KademliaNode("Lokesh", new KademliaId("ASF45678947584567466"), 8335); + KademliaNode kad5 = new KademliaNode("Chandu", new KademliaId("ASF45678947584567467"), 13345); + + /* Connecting 2 to 1 */ + System.out.println("Connecting Nodes 1 & 2"); + kad2.bootstrap(kad1.getNode()); + System.out.println(kad1); + System.out.println(kad2); + + kad3.bootstrap(kad2.getNode()); + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + + kad4.bootstrap(kad2.getNode()); + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + System.out.println(kad4); + + kad5.bootstrap(kad4.getNode()); + + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + System.out.println(kad4); + System.out.println(kad5); + + synchronized (this) + { + System.out.println("\n\n\n\nSTORING CONTENT 1\n\n\n\n"); + DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), "Some Data"); + System.out.println(c); + kad2.put(c); + } + + synchronized (this) + { + System.out.println("\n\n\n\nSTORING CONTENT 2\n\n\n\n"); + DHTContentImpl c2 = new DHTContentImpl(kad2.getOwnerId(), "Some other Data"); + System.out.println(c2); + kad4.put(c2); + } + + System.out.println(kad1); + System.out.println(kad2); + System.out.println(kad3); + System.out.println(kad4); + System.out.println(kad5); + + /* Shutting down kad1 and restarting it */ + System.out.println("\n\n\nShutting down Kad instance"); + System.out.println(kad2); + kad1.shutdown(true); + + System.out.println("\n\n\nReloading Kad instance from file"); + KademliaNode kadR2 = KademliaNode.loadFromFile("JoshuaK"); + System.out.println(kadR2); + } + catch (IllegalStateException e) + { + + } + catch (Exception e) + { + e.printStackTrace(); + } + } + + public static void main(String[] args) + { + new SaveStateTest(); + } +} diff --git a/src/kademlia/tests/SaveStateTest2.java b/src/kademlia/tests/SaveStateTest2.java new file mode 100644 index 0000000..a013070 --- /dev/null +++ b/src/kademlia/tests/SaveStateTest2.java @@ -0,0 +1,70 @@ +package kademlia.tests; + +import kademlia.KademliaNode; +import kademlia.dht.GetParameter; +import kademlia.dht.StorageEntry; +import kademlia.node.KademliaId; + +/** + * Testing the save and retrieve state operations. + * Here we also try to look for content on a restored node + * + * @author Joshua Kissoon + * @since 20140309 + */ +public class SaveStateTest2 +{ + + public SaveStateTest2() + { + try + { + /* Setting up 2 Kad networks */ + KademliaNode kad1 = new KademliaNode("JoshuaK", new KademliaId("ASF45678947584567463"), 12049); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("ASF45678947584567464"), 4585); + + /* Connecting 2 to 1 */ + System.out.println("Connecting Nodes 1 & 2"); + kad2.bootstrap(kad1.getNode()); + System.out.println(kad1); + System.out.println(kad2); + + DHTContentImpl c; + synchronized (this) + { + System.out.println("\n\n\n\nSTORING CONTENT 1\n\n\n\n"); + c = new DHTContentImpl(kad2.getOwnerId(), "Some Data"); + System.out.println(c); + kad1.putLocally(c); + } + + System.out.println(kad1); + System.out.println(kad2); + + /* Shutting down kad1 and restarting it */ + System.out.println("\n\n\nShutting down Kad 1 instance"); + kad1.shutdown(true); + + System.out.println("\n\n\nReloading Kad instance from file"); + kad1 = KademliaNode.loadFromFile("JoshuaK"); + kad1.bootstrap(kad2.getNode()); + System.out.println(kad2); + + /* Trying to get a content stored on the restored node */ + GetParameter gp = new GetParameter(c.getKey(), kad2.getOwnerId(), c.getType()); + StorageEntry content = kad2.get(gp); + DHTContentImpl cc = new DHTContentImpl().fromBytes(content.getContent().getBytes()); + System.out.println("Content received: " + cc); + } + + catch (Exception e) + { + e.printStackTrace(); + } + } + + public static void main(String[] args) + { + new SaveStateTest2(); + } +} diff --git a/src/kademlia/tests/SimpleMessageTest.java b/src/kademlia/tests/SimpleMessageTest.java new file mode 100644 index 0000000..7ba24b9 --- /dev/null +++ b/src/kademlia/tests/SimpleMessageTest.java @@ -0,0 +1,32 @@ +package kademlia.tests; + +import java.io.IOException; +import kademlia.KademliaNode; +import kademlia.message.SimpleMessage; +import kademlia.node.KademliaId; +import kademlia.message.SimpleReceiver; + +/** + * Test 1: Try sending a simple message between nodes + * + * @author Joshua Kissoon + * @created 20140218 + */ +public class SimpleMessageTest +{ + + public static void main(String[] args) + { + try + { + KademliaNode kad1 = new KademliaNode("Joshua", new KademliaId("12345678901234567890"), 7574); + KademliaNode kad2 = new KademliaNode("Crystal", new KademliaId("12345678901234567891"), 7572); + + kad1.getServer().sendMessage(kad2.getNode(), new SimpleMessage("Some Message"), new SimpleReceiver()); + } + catch (IOException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/kademlia/util/HashCalculator.java b/src/kademlia/util/HashCalculator.java new file mode 100644 index 0000000..2bd2bab --- /dev/null +++ b/src/kademlia/util/HashCalculator.java @@ -0,0 +1,100 @@ +package kademlia.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * A class that is used to calculate the hash of strings. + * + * @author Joshua Kissoon + * @since 20140405 + */ +public class HashCalculator +{ + + /** + * Computes the SHA-1 Hash. + * + * @param toHash The string to hash + * + * @return byte[20] The hashed string + * + * @throws java.security.NoSuchAlgorithmException + */ + public static byte[] sha1Hash(String toHash) throws NoSuchAlgorithmException + { + /* Create a MessageDigest */ + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + /* Add password bytes to digest */ + md.update(toHash.getBytes()); + + /* Get the hashed bytes */ + return md.digest(); + } + + /** + * Computes the SHA-1 Hash using a Salt. + * + * @param toHash The string to hash + * @param salt A salt used to blind the hash + * + * @return byte[20] The hashed string + * + * @throws java.security.NoSuchAlgorithmException + */ + public static byte[] sha1Hash(String toHash, String salt) throws NoSuchAlgorithmException + { + /* Create a MessageDigest */ + MessageDigest md = MessageDigest.getInstance("SHA-1"); + + /* Add password bytes to digest */ + md.update(toHash.getBytes()); + + /* Get the hashed bytes */ + return md.digest(salt.getBytes()); + } + + /** + * Computes the MD5 Hash. + * + * @param toHash The string to hash + * + * @return byte[16] The hashed string + * + * @throws java.security.NoSuchAlgorithmException + */ + public static byte[] md5Hash(String toHash) throws NoSuchAlgorithmException + { + /* Create a MessageDigest */ + MessageDigest md = MessageDigest.getInstance("MD5"); + + /* Add password bytes to digest */ + md.update(toHash.getBytes()); + + /* Get the hashed bytes */ + return md.digest(); + } + + /** + * Computes the MD5 Hash using a salt. + * + * @param toHash The string to hash + * @param salt A salt used to blind the hash + * + * @return byte[16] The hashed string + * + * @throws java.security.NoSuchAlgorithmException + */ + public static byte[] md5Hash(String toHash, String salt) throws NoSuchAlgorithmException + { + /* Create a MessageDigest */ + MessageDigest md = MessageDigest.getInstance("MD5"); + + /* Add password bytes to digest */ + md.update(toHash.getBytes()); + + /* Get the hashed bytes */ + return md.digest(salt.getBytes()); + } +} diff --git a/src/kademlia/util/RouteLengthChecker.java b/src/kademlia/util/RouteLengthChecker.java new file mode 100644 index 0000000..6fc7d27 --- /dev/null +++ b/src/kademlia/util/RouteLengthChecker.java @@ -0,0 +1,92 @@ +package kademlia.util; + +import java.util.Collection; +import java.util.HashMap; +import kademlia.node.Node; + +/** + * Class that helps compute the route length taken to complete an operation. + * + * Only used for routing operations - mainly the NodeLookup and ContentLookup Operations. + * + * Idea: + * - Add the original set of nodes with route length 0; + * - When we get a node reply with a set of nodes, we add those nodes and set the route length to their sender route length + 1 + * + * @author Joshua Kissoon + * @since 20140510 + */ +public class RouteLengthChecker +{ + + /* Store the nodes and their route length (RL) */ + private final HashMap nodes; + + /* Lets cache the max route length instead of having to go and search for it later */ + private int maxRouteLength; + + + { + this.nodes = new HashMap<>(); + this.maxRouteLength = 1; + } + + /** + * Add the initial nodes in the routing operation + * + * @param initialNodes The set of initial nodes + */ + public void addInitialNodes(Collection initialNodes) + { + for (Node n : initialNodes) + { + this.nodes.put(n, 1); + } + } + + /** + * Add any nodes that we get from a node reply. + * + * The route length of these nodes will be their sender + 1; + * + * @param inputSet The set of nodes we receive + * @param sender The node who send the set + */ + public void addNodes(Collection inputSet, Node sender) + { + if (!this.nodes.containsKey(sender)) + { + return; + } + + /* Get the route length of the input set - sender RL + 1 */ + int inputSetRL = this.nodes.get(sender) + 1; + + if (inputSetRL > this.maxRouteLength) + { + this.maxRouteLength = inputSetRL; + } + + /* Add the nodes to our set */ + for (Node n : inputSet) + { + /* We only add if the node is not already there... */ + if (!this.nodes.containsKey(n)) + { + this.nodes.put(n, inputSetRL); + } + } + } + + /** + * Get the route length of the operation! + * + * It will be the max route length of all the nodes here. + * + * @return The route length + */ + public int getRouteLength() + { + return this.maxRouteLength; + } +} diff --git a/src/kademlia/util/serializer/JsonDHTSerializer.java b/src/kademlia/util/serializer/JsonDHTSerializer.java new file mode 100644 index 0000000..a9b2307 --- /dev/null +++ b/src/kademlia/util/serializer/JsonDHTSerializer.java @@ -0,0 +1,94 @@ +package kademlia.util.serializer; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.lang.reflect.Type; +import java.util.List; +import kademlia.dht.DHT; +import kademlia.dht.StorageEntryMetadata; + +/** + * A KadSerializer that serializes DHT to JSON format + * The generic serializer is not working for DHT + * + * Why a DHT specific serializer? + * The DHT structure: + * - DHT + * -- StorageEntriesManager + * --- Map> + * ---- NodeId:KeyBytes + * ---- List + * ----- StorageEntry: Key, OwnerId, Type, Hash + * + * The above structure seems to be causing some problem for Gson, especially at the Map part. + * + * Solution + * - Make the StorageEntriesManager transient + * - Simply store all StorageEntry in the serialized object + * - When reloading, re-add all StorageEntry to the DHT + * + * @author Joshua Kissoon + * + * @since 20140310 + */ +public class JsonDHTSerializer implements KadSerializer +{ + + private final Gson gson; + private final Type storageEntriesCollectionType; + + + { + gson = new Gson(); + + storageEntriesCollectionType = new TypeToken>() + { + }.getType(); + } + + @Override + public void write(DHT data, DataOutputStream out) throws IOException + { + try (JsonWriter writer = new JsonWriter(new OutputStreamWriter(out))) + { + writer.beginArray(); + + /* Write the basic DHT */ + gson.toJson(data, DHT.class, writer); + + /* Now Store the Entries */ + gson.toJson(data.getStorageEntries(), this.storageEntriesCollectionType, writer); + + writer.endArray(); + } + + } + + @Override + public DHT read(DataInputStream in) throws IOException, ClassNotFoundException + { + try (DataInputStream din = new DataInputStream(in); + JsonReader reader = new JsonReader(new InputStreamReader(in))) + { + reader.beginArray(); + + /* Read the basic DHT */ + DHT dht = gson.fromJson(reader, DHT.class); + dht.initialize(); + + /* Now get the entries and add them back to the DHT */ + List entries = gson.fromJson(reader, this.storageEntriesCollectionType); + dht.putStorageEntries(entries); + + reader.endArray(); + return dht; + } + } +} diff --git a/src/kademlia/util/serializer/JsonRoutingTableSerializer.java b/src/kademlia/util/serializer/JsonRoutingTableSerializer.java new file mode 100644 index 0000000..0ec55ed --- /dev/null +++ b/src/kademlia/util/serializer/JsonRoutingTableSerializer.java @@ -0,0 +1,111 @@ +package kademlia.util.serializer; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import kademlia.routing.RoutingTable; +import java.lang.reflect.Type; +import java.util.List; +import kademlia.core.KadConfiguration; +import kademlia.routing.Contact; + +/** + * A KadSerializer that serializes routing tables to JSON format + * The generic serializer is not working for routing tables + * + * Why a RoutingTable specific serializer? + * The routing table structure: + * - RoutingTable + * -- Buckets[] + * --- Map + * ---- NodeId:KeyBytes + * ---- Node: NodeId, InetAddress, Port + * + * The above structure seems to be causing some problem for Gson, + * especially at the Map part. + * + * Solution + * - Make the Buckets[] transient + * - Simply store all Nodes in the serialized object + * - When reloading, re-add all nodes to the RoutingTable + * + * @author Joshua Kissoon + * + * @since 20140310 + */ +public class JsonRoutingTableSerializer implements KadSerializer +{ + + private final Gson gson; + + Type contactCollectionType = new TypeToken>() + { + }.getType(); + + private final KadConfiguration config; + + + { + gson = new Gson(); + } + + /** + * Initialize the class + * + * @param config + */ + public JsonRoutingTableSerializer(KadConfiguration config) + { + this.config = config; + } + + @Override + public void write(RoutingTable data, DataOutputStream out) throws IOException + { + try (JsonWriter writer = new JsonWriter(new OutputStreamWriter(out))) + { + writer.beginArray(); + + /* Write the basic RoutingTable */ + gson.toJson(data, RoutingTable.class, writer); + + /* Now Store the Contacts */ + gson.toJson(data.getAllContacts(), contactCollectionType, writer); + + writer.endArray(); + } + } + + @Override + public RoutingTable read(DataInputStream in) throws IOException, ClassNotFoundException + { + try (DataInputStream din = new DataInputStream(in); + JsonReader reader = new JsonReader(new InputStreamReader(in))) + { + reader.beginArray(); + + /* Read the basic RoutingTable */ + RoutingTable tbl = gson.fromJson(reader, RoutingTable.class); + tbl.setConfiguration(config); + + /* Now get the Contacts and add them back to the RoutingTable */ + List contacts = gson.fromJson(reader, contactCollectionType); + tbl.initialize(); + + for (Contact c : contacts) + { + tbl.insert(c); + } + + reader.endArray(); + /* Read and return the Content*/ + return tbl; + } + } +} diff --git a/src/kademlia/util/serializer/JsonSerializer.java b/src/kademlia/util/serializer/JsonSerializer.java new file mode 100644 index 0000000..b59e605 --- /dev/null +++ b/src/kademlia/util/serializer/JsonSerializer.java @@ -0,0 +1,67 @@ +package kademlia.util.serializer; + +import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; + +/** + * A KadSerializer that serializes content to JSON format + * + * @param The type of content to serialize + * + * @author Joshua Kissoon + * + * @since 20140225 + */ +public class JsonSerializer implements KadSerializer +{ + + private final Gson gson; + + + { + gson = new Gson(); + } + + @Override + public void write(T data, DataOutputStream out) throws IOException + { + try (JsonWriter writer = new JsonWriter(new OutputStreamWriter(out))) + { + writer.beginArray(); + + /* Store the content type */ + gson.toJson(data.getClass().getName(), String.class, writer); + + /* Now Store the content */ + gson.toJson(data, data.getClass(), writer); + + writer.endArray(); + } + } + + @Override + public T read(DataInputStream in) throws IOException, ClassNotFoundException + { + try (DataInputStream din = new DataInputStream(in); + JsonReader reader = new JsonReader(new InputStreamReader(in))) + { + reader.beginArray(); + + /* Read the class name */ + String className = gson.fromJson(reader, String.class); + + /* Read and return the Content*/ + T ret = gson.fromJson(reader, Class.forName(className)); + + reader.endArray(); + + return ret; + } + } +} diff --git a/src/kademlia/util/serializer/KadSerializer.java b/src/kademlia/util/serializer/KadSerializer.java new file mode 100644 index 0000000..824a1d1 --- /dev/null +++ b/src/kademlia/util/serializer/KadSerializer.java @@ -0,0 +1,41 @@ +package kademlia.util.serializer; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** + * A Serializer is used to transform data to and from a specified form. + * + * Here we define the structure of any Serializer used in Kademlia + * + * @author Joshua Kissoon + * @param The type of content being serialized + * + * @since 20140225 + */ +public interface KadSerializer +{ + + /** + * Write a KadContent to a DataOutput stream + * + * @param data The data to write + * @param out The output Stream to write to + * + * @throws java.io.IOException + */ + public void write(T data, DataOutputStream out) throws IOException; + + /** + * Read data of type T from a DataInput Stream + * + * @param in The InputStream to read the data from + * + * @return T Data of type T + * + * @throws java.io.IOException + * @throws java.lang.ClassNotFoundException + */ + public T read(DataInputStream in) throws IOException, ClassNotFoundException; +}