ChronosXYZ/KademliaDHT
1
0
Fork 0
mirror of https://github.com/ChronosX88/KademliaDHT.git synced 2024-11-24 19:22:18 +00:00
Code Issues Projects Releases Wiki Activity

Content Store, Retrieve and DHT File and Table management working well

Browse Source 
Got a few Todos now to finish off
This commit is contained in:
Joshua Kissoon 2014-02-26 21:35:37 +05:30
parent a12e075cec
commit e4726b93f8
22 changed files with 158 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/kademlia/core/Configuration.java
View File

@ -21,7 +21,7 @@ public class Configuration
/**
* Maximum number of milliseconds for performing an operation.
* */
public static long OPERATION_TIMEOUT = 10000;
public static long OPERATION_TIMEOUT = 3000;
/**
* Maximum number of concurrent messages in transit.

16
src/kademlia/core/GetParameter.java
View File

@ -41,13 +41,29 @@ public class GetParameter
return this.key;
}
public void setOwnerId(String ownerId)
{
this.ownerId = ownerId;
}
public String getOwnerId()
{
return this.ownerId;
}
public void setType(String type)
{
this.type = type;
}
public String getType()
{
return this.type;
}
@Override
public String toString()
{
return "GetParameter - [Key: " + key + "][Owner: " + this.ownerId + "][Type: " + this.type + "]";
}
}

3
src/kademlia/core/KadServer.java
View File

@ -170,12 +170,13 @@ public class KadServer
Message msg = messageFactory.createMessage(messCode, din);
din.close();
System.out.println(this.localNode.getNodeId() + " Message Received: " + msg);
System.out.println(this.localNode.getNodeId() + " Message Received: [Comm: " + comm + "] " + msg);
/* Get a receiver for this message */
Receiver receiver;
if (this.receivers.containsKey(comm))
{
System.out.println("Receiver found");
/* If there is a reciever in the receivers to handle this */
synchronized (this)
{

11
src/kademlia/core/Kademlia.java
View File

@ -2,6 +2,7 @@ package kademlia.core;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Timer;
@ -153,18 +154,24 @@ public class Kademlia
*/
public List<KadContent> get(GetParameter param, int numResultsReq) throws NoSuchElementException, IOException
{
List contentFound;
if (this.dht.contains(param))
{
/* If the content exist in our own DHT, then return it. */
return this.dht.get(param);
System.out.println("Found content locally");
contentFound = new ArrayList<>();
contentFound.add(this.dht.get(param));
}
else
{
/* Seems like it doesn't exist in our DHT, get it from other Nodes */
System.out.println("Looking for content on foreign nodes");
ContentLookupOperation clo = new ContentLookupOperation(server, localNode, param, numResultsReq);
clo.execute();
return clo.getContentFound();
contentFound = clo.getContentFound();
}
return contentFound;
}
/**

27
src/kademlia/dht/DHT.java
View File

@ -7,8 +7,6 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.NoSuchElementException;
import kademlia.core.Configuration;
import kademlia.core.GetParameter;
@ -25,10 +23,12 @@ public class DHT
{
private final StorageEntryManager entriesManager;
private final JsonSerializer<KadContent> contentSerializer;
{
entriesManager = new StorageEntryManager();
contentSerializer = new JsonSerializer<>();
}
/**
@ -46,7 +46,7 @@ public class DHT
/* Now we store the content locally in a file */
String contentStorageFolder = this.getContentStorageFolderName(content.getKey());
DataOutputStream dout = new DataOutputStream(new FileOutputStream(contentStorageFolder + File.separator + content.hashCode() + ".kct"));
new JsonSerializer().write(content, dout);
contentSerializer.write(content, dout);
}
/**
@ -61,7 +61,7 @@ public class DHT
{
String folder = this.getContentStorageFolderName(key);
DataInputStream in = new DataInputStream(new FileInputStream(folder + File.separator + hashCode + ".kct"));
return new JsonSerializer().read(in);
return contentSerializer.read(in);
}
/**
@ -86,17 +86,14 @@ public class DHT
*
* @throws java.io.IOException
*/
public List<KadContent> get(GetParameter param) throws NoSuchElementException, IOException
public KadContent get(GetParameter param) throws NoSuchElementException, IOException
{
/* Load all KadContent for the entries if any exist */
List<KadContent> values = new ArrayList<>();
/* Load a KadContent if any exist for the given criteria */
try
{
for (StorageEntry e : this.entriesManager.get(param))
{
values.add(this.retrieve(e.getKey(), e.getContentHash()));
}
StorageEntry e = this.entriesManager.get(param);
return this.retrieve(e.getKey(), e.getContentHash());
}
catch (FileNotFoundException e)
{
@ -107,14 +104,10 @@ public class DHT
System.err.println("The class for some content was not found.");
}
if (values.isEmpty())
{
/* If we got here, means we got no entries */
throw new NoSuchElementException();
}
return values;
}
/**
* Get the name of the folder for which a content should be stored
*

15
src/kademlia/dht/StorageEntryManager.java
View File

@ -38,7 +38,6 @@ public class StorageEntryManager
{
this.entries.put(entry.getKey(), new ArrayList<StorageEntry>());
}
this.entries.get(entry.getKey()).add(entry);
}
@ -77,31 +76,23 @@ public class StorageEntryManager
*
* @return List of content for the specific search parameters
*/
public List<StorageEntry> get(GetParameter param) throws NoSuchElementException
public StorageEntry get(GetParameter param) throws NoSuchElementException
{
if (this.entries.containsKey(param.getKey()))
{
/* Content with this key exist, check if any match the rest of the search criteria */
List<StorageEntry> results = new ArrayList<>();
for (StorageEntry e : this.entries.get(param.getKey()))
{
/* If any entry satisfies the given parameters, return true */
if (e.satisfiesParameters(param))
{
results.add(e);
return e;
}
}
if (results.size() > 0)
{
return results;
}
else
{
/* If we got here, means we didn't find any entry */
throw new NoSuchElementException();
}
}
else
{
throw new NoSuchElementException("No content exist for the given parameters");

2
src/kademlia/message/AcknowledgeMessage.java
View File

@ -5,9 +5,7 @@
*/
package kademlia.message;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import kademlia.node.Node;

2
src/kademlia/message/ConnectMessage.java
View File

@ -5,9 +5,7 @@
*/
package kademlia.message;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import kademlia.node.Node;

18
src/kademlia/message/ContentLookupMessage.java
View File

@ -1,10 +1,13 @@
package kademlia.message;
import com.google.gson.JsonSerializationContext;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import kademlia.core.GetParameter;
import kademlia.dht.KadContent;
import kademlia.node.Node;
import kademlia.serializer.JsonSerializer;
/**
* Messages used to send to another node requesting content
@ -15,7 +18,7 @@ import kademlia.node.Node;
public class ContentLookupMessage implements Message
{
public static final byte CODE = 0x47;
public static final byte CODE = 0x03;
private Node origin;
private GetParameter params;
@ -49,12 +52,25 @@ public class ContentLookupMessage implements Message
public void toStream(DataOutputStream out) throws IOException
{
this.origin.toStream(out);
/* Write the params to the stream */
new JsonSerializer<GetParameter>().write(this.params, out);
}
@Override
public final void fromStream(DataInputStream in) throws IOException
{
this.origin = new Node(in);
/* Read the params from the stream */
try
{
this.params = new JsonSerializer<GetParameter>().read(in);
}
catch (ClassNotFoundException e)
{
e.printStackTrace();
}
}
@Override

2
src/kademlia/message/ContentLookupReceiver.java
View File

@ -31,12 +31,14 @@ public class ContentLookupReceiver implements Receiver
public void receive(Message incoming, int comm) throws IOException
{
ContentLookupMessage msg = (ContentLookupMessage) incoming;
this.localNode.getRoutingTable().insert(msg.getOrigin());
/* Check if we can have this data */
if (this.dht.contains(msg.getParameters()))
{
/* Return a ContentMessage with the required data */
ContentMessage cMsg = new ContentMessage(localNode, this.dht.get(msg.getParameters()));
server.reply(msg.getOrigin(), cMsg, comm);
}
else
{

10
src/kademlia/message/ContentMessage.java
View File

@ -16,7 +16,7 @@ import kademlia.serializer.JsonSerializer;
public class ContentMessage implements Message
{
public static final byte CODE = 0x56;
public static final byte CODE = 0x04;
private KadContent content;
private Node origin;
@ -43,17 +43,17 @@ public class ContentMessage implements Message
this.origin.toStream(out);
/* Serialize the KadContent, then send it to the stream */
JsonSerializer serializer = new JsonSerializer();
serializer.write(content, out);
new JsonSerializer<KadContent>().write(content, out);
}
@Override
public final void fromStream(DataInputStream in) throws IOException
{
this.origin = new Node(in);
try
{
this.content = new JsonSerializer().read(in);
this.content = new JsonSerializer<KadContent>().read(in);
}
catch (ClassNotFoundException e)
{
@ -80,6 +80,6 @@ public class ContentMessage implements Message
@Override
public String toString()
{
return "StoreMessage[origin=" + origin + ",content=" + content + "]";
return "ContentMessage[origin=" + origin + ",content=" + content + "]";
}
}

34
src/kademlia/message/MessageFactory.java
View File

@ -29,20 +29,24 @@ public class MessageFactory
{
switch (code)
{
case SimpleMessage.CODE:
return new SimpleMessage(in);
case ConnectMessage.CODE:
return new ConnectMessage(in);
case AcknowledgeMessage.CODE:
return new AcknowledgeMessage(in);
case NodeReplyMessage.CODE:
return new NodeReplyMessage(in);
case ConnectMessage.CODE:
return new ConnectMessage(in);
case ContentMessage.CODE:
return new ContentMessage(in);
case ContentLookupMessage.CODE:
return new ContentLookupMessage(in);
case NodeLookupMessage.CODE:
return new NodeLookupMessage(in);
case NodeReplyMessage.CODE:
return new NodeReplyMessage(in);
case SimpleMessage.CODE:
return new SimpleMessage(in);
case StoreContentMessage.CODE:
return new StoreContentMessage(in);
default:
System.out.println("No Message handler found for message. Code: " + code);
System.out.println(this.localNode + " - No Message handler found for message. Code: " + code);
return new SimpleMessage(in);
}
@ -52,17 +56,19 @@ public class MessageFactory
{
switch (code)
{
default:
case SimpleMessage.CODE:
return new SimpleReceiver();
case ConnectMessage.CODE:
return new ConnectReceiver(server, this.localNode);
case NodeLookupMessage.CODE:
return new NodeLookupReceiver(server, this.localNode);
case StoreContentMessage.CODE:
return new StoreContentReceiver(server, this.localNode, this.dht);
case ContentLookupMessage.CODE:
return new ContentLookupReceiver(server, localNode, dht);
case NodeLookupMessage.CODE:
return new NodeLookupReceiver(server, this.localNode);
case SimpleMessage.CODE:
return new SimpleReceiver();
case StoreContentMessage.CODE:
return new StoreContentReceiver(server, this.localNode, this.dht);
default:
System.out.println("No reveiver found for message. Code: " + code);
return new SimpleReceiver();
}
}
}

4
src/kademlia/message/NodeLookupMessage.java
View File

@ -5,9 +5,7 @@
*/
package kademlia.message;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import kademlia.node.Node;
@ -19,7 +17,7 @@ public class NodeLookupMessage implements Message
private Node origin;
private NodeId lookupId;
public static final byte CODE = 0x03;
public static final byte CODE = 0x05;
/**
* A new NodeLookupMessage to find nodes

2
src/kademlia/message/NodeReplyMessage.java
View File

@ -16,7 +16,7 @@ public class NodeReplyMessage implements Message
{
private Node origin;
public static final byte CODE = 0x04;
public static final byte CODE = 0x06;
private List<Node> nodes;
public NodeReplyMessage(Node origin, List<Node> nodes)

4
src/kademlia/message/SimpleMessage.java
View File

@ -5,9 +5,7 @@
*/
package kademlia.message;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
@ -15,7 +13,7 @@ public class SimpleMessage implements Message
{
/* Message constants */
public static final byte CODE = 0x05;
public static final byte CODE = 0x07;
private String content;

9
src/kademlia/message/StoreContentMessage.java
View File

@ -16,7 +16,7 @@ import kademlia.serializer.JsonSerializer;
public class StoreContentMessage implements Message
{
public static final byte CODE = 0x55;
public static final byte CODE = 0x08;
private KadContent content;
private Node origin;
@ -43,8 +43,7 @@ public class StoreContentMessage implements Message
this.origin.toStream(out);
/* Serialize the KadContent, then send it to the stream */
JsonSerializer serializer = new JsonSerializer();
serializer.write(content, out);
new JsonSerializer<KadContent>().write(content, out);
}
@Override
@ -53,7 +52,7 @@ public class StoreContentMessage implements Message
this.origin = new Node(in);
try
{
this.content = new JsonSerializer().read(in);
this.content = new JsonSerializer<KadContent>().read(in);
}
catch (ClassNotFoundException e)
{
@ -80,6 +79,6 @@ public class StoreContentMessage implements Message
@Override
public String toString()
{
return "StoreMessage[origin=" + origin + ",content=" + content + "]";
return "StoreContentMessage[origin=" + origin + ",content=" + content + "]";
}
}

26
src/kademlia/operation/ContentLookupOperation.java
View File

@ -105,12 +105,21 @@ public class ContentLookupOperation implements Operation, Receiver
if (!this.askNodesorFinish())
{
/* If we haven't finished as yet, wait a while */
/**
* @todo Get rid of this wait here!
* We should run this until there are no nodes left to ask from the K closest nodes
* and only pause for short intervals in between
*
* @todo Do the same for the NodeLookupOperation
*/
wait(Configuration.OPERATION_TIMEOUT);
/* If we still haven't received any responses by then, do a routing timeout */
if (error)
{
throw new RoutingException("Lookup Timeout.");
/* Lets not throw any exception */
//throw new RoutingException("Content Lookup Operation Timeout.");
}
}
}
@ -132,16 +141,9 @@ public class ContentLookupOperation implements Operation, Receiver
/* If this node is not in the list, add the node */
if (!nodes.containsKey(o))
{
System.out.println("Adding node " + o.getNodeId());
nodes.put(o, UNASKED);
}
}
System.out.println(this.localNode.getNodeId() + " Nodes List: ");
for (Node o : this.nodes.keySet())
{
System.out.println(o.getNodeId() + " hash: " + o.hashCode());
}
}
/**
@ -230,6 +232,11 @@ public class ContentLookupOperation implements Operation, Receiver
@Override
public synchronized void receive(Message incoming, int comm) throws IOException, RoutingException
{
if (!this.isRunning)
{
return;
}
if (incoming instanceof ContentMessage)
{
/* The reply received is a content message with the required content, take it in */
@ -240,6 +247,7 @@ public class ContentLookupOperation implements Operation, Receiver
/* Get the Content and check if it satisfies the required parameters */
KadContent content = msg.getContent();
System.out.println("Content Received: " + content);
/*@todo Check if the content matches the given criteria */
this.contentFound.add(content);
@ -247,6 +255,8 @@ public class ContentLookupOperation implements Operation, Receiver
if (this.contentFound.size() == this.numResultsReq)
{
/* We've got all the content required, let's stop the loopup operation */
System.out.println("We good");
this.isRunning = false;
}
}
else

20
src/kademlia/operation/NodeLookupOperation.java
View File

@ -128,16 +128,16 @@ public class NodeLookupOperation implements Operation, Receiver
/* If this node is not in the list, add the node */
if (!nodes.containsKey(o))
{
System.out.println("Adding node " + o.getNodeId());
//System.out.println("Adding node " + o.getNodeId());
nodes.put(o, UNASKED);
}
}
System.out.println(this.localNode.getNodeId() + " Nodes List: ");
for (Node o : this.nodes.keySet())
{
System.out.println(o.getNodeId() + " hash: " + o.hashCode());
}
// System.out.println(this.localNode.getNodeId() + " Nodes List: ");
// for (Node o : this.nodes.keySet())
// {
// System.out.println(o.getNodeId() + " hash: " + o.hashCode());
// }
}
/**
@ -161,10 +161,10 @@ public class NodeLookupOperation implements Operation, Receiver
/* Get unqueried nodes among the K closest seen that have not FAILED */
List<Node> unasked = this.closestNodesNotFailed(UNASKED);
for (Node nn : unasked)
{
System.out.println(nn.getNodeId());
}
// for (Node nn : unasked)
// {
// System.out.println(nn.getNodeId());
// }
if (unasked.isEmpty() && this.messagesTransiting.isEmpty())
{

17
src/kademlia/serializer/JsonSerializer.java
View File

@ -4,22 +4,21 @@ import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import kademlia.dht.KadContent;
/**
* A KadContentSerializer that serializes content to JSON format
*
* @param <T> The type of content to serialize
*
* @author Joshua Kissoon
*
* @since 20140225
*/
public class JsonSerializer implements KadContentSerializer
public class JsonSerializer<T> implements KadContentSerializer<T>
{
private final Gson gson;
@ -30,17 +29,17 @@ public class JsonSerializer implements KadContentSerializer
}
@Override
public void write(KadContent content, DataOutputStream out) throws IOException
public void write(T data, DataOutputStream out) throws IOException
{
try (JsonWriter writer = new JsonWriter(new OutputStreamWriter(out)))
{
writer.beginArray();
/* Store the content type */
gson.toJson(content.getClass().getName(), String.class, writer);
gson.toJson(data.getClass().getName(), String.class, writer);
/* Now Store the content */
gson.toJson(content, content.getClass(), writer);
gson.toJson(data, data.getClass(), writer);
writer.endArray();
}
@ -48,7 +47,7 @@ public class JsonSerializer implements KadContentSerializer
}
@Override
public KadContent read(DataInputStream in) throws IOException, ClassNotFoundException
public T read(DataInputStream in) throws IOException, ClassNotFoundException
{
try (DataInputStream din = new DataInputStream(in);
JsonReader reader = new JsonReader(new InputStreamReader(in)))

17
src/kademlia/serializer/KadContentSerializer.java
View File

@ -3,9 +3,6 @@ package kademlia.serializer;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import kademlia.dht.KadContent;
/**
* A Serializer is used to transform data to and from a specified form.
@ -13,30 +10,32 @@ import kademlia.dht.KadContent;
* Here we define the structure of any Serializer used in Kademlia
*
* @author Joshua Kissoon
* @param <T> The type of content being serialized
*
* @since 20140225
*/
public interface KadContentSerializer
public interface KadContentSerializer<T>
{
/**
* Write a KadContent to a DataOutput stream
*
* @param content The content to write
* @param data The data to write
* @param out The output Stream to write to
*
* @throws java.io.IOException
*/
public void write(KadContent content, DataOutputStream out) throws IOException;
public void write(T data, DataOutputStream out) throws IOException;
/**
* Read a KadContent from a DataInput Stream
* Read data of type T from a DataInput Stream
*
* @param in The InputStream to read the data from
*
* @return KadContent
* @return T Data of type T
*
* @throws java.io.IOException
* @throws java.lang.ClassNotFoundException
*/
public KadContent read(DataInputStream in) throws IOException, ClassNotFoundException;
public T read(DataInputStream in) throws IOException, ClassNotFoundException;
}

17
src/kademlia/tests/ContentSendingTest.java
View File

@ -1,7 +1,10 @@
package kademlia.tests;
import java.io.IOException;
import java.util.List;
import kademlia.core.GetParameter;
import kademlia.core.Kademlia;
import kademlia.dht.KadContent;
import kademlia.node.NodeId;
/**
@ -30,6 +33,20 @@ public class ContentSendingTest
DHTContentImpl c = new DHTContentImpl(kad2.getOwnerId(), "Some Data");
kad2.put(c);
/**
* Lets retrieve the content
*/
System.out.println("Retrieving Content");
GetParameter gp = new GetParameter(c.getKey());
gp.setType(DHTContentImpl.TYPE);
gp.setOwnerId(c.getOwnerId());
System.out.println("Get Parameter: " + gp);
List<KadContent> conte = kad2.get(gp, 1);
for (KadContent cc : conte)
{
System.out.println("Content Found: " + cc);
}
}
catch (IOException e)
{

4
src/kademlia/tests/DHTContentImpl.java
View File

@ -17,7 +17,7 @@ public class DHTContentImpl implements KadContent
private final String ownerId;
private final long createTs;
private static final String type = "DHTContentImpl";
public static final String TYPE = "DHTContentImpl";
{
@ -56,7 +56,7 @@ public class DHTContentImpl implements KadContent
@Override
public String getType()
{
return type;
return TYPE;
}
@Override