From a2d0be612487fe72011147c66b5a0b7d704f0caa Mon Sep 17 00:00:00 2001 From: Joshua Kissoon Date: Wed, 26 Feb 2014 17:07:18 +0530 Subject: [PATCH] Finished DHT content storage and retrieval Started working on Content lookup and sending content --- src/kademlia/core/GetParameter.java | 18 ++- src/kademlia/core/Kademlia.java | 19 +++- src/kademlia/dht/DHT.java | 104 ++++++++++++++++-- src/kademlia/dht/StorageEntry.java | 49 +++++++++ src/kademlia/dht/StorageEntryManager.java | 61 ++++++++-- .../operation/ContentLookupOperation.java | 31 ++++++ 6 files changed, 258 insertions(+), 24 deletions(-) create mode 100644 src/kademlia/operation/ContentLookupOperation.java diff --git a/src/kademlia/core/GetParameter.java b/src/kademlia/core/GetParameter.java index 17f5edc..ac265e4 100644 --- a/src/kademlia/core/GetParameter.java +++ b/src/kademlia/core/GetParameter.java @@ -16,7 +16,7 @@ public class GetParameter { private NodeId key; - private String owner = null; + private String ownerId = null; private String type = null; public GetParameter(NodeId key) @@ -27,7 +27,7 @@ public class GetParameter public GetParameter(NodeId key, String owner) { this(key); - this.owner = owner; + this.ownerId = owner; } public GetParameter(NodeId key, String owner, String type) @@ -36,4 +36,18 @@ public class GetParameter this.type = type; } + public NodeId getKey() + { + return this.key; + } + + public String getOwnerId() + { + return this.ownerId; + } + + public String getType() + { + return this.type; + } } diff --git a/src/kademlia/core/Kademlia.java b/src/kademlia/core/Kademlia.java index 616cfbd..3081372 100644 --- a/src/kademlia/core/Kademlia.java +++ b/src/kademlia/core/Kademlia.java @@ -2,6 +2,8 @@ package kademlia.core; import java.io.IOException; import java.net.InetAddress; +import java.util.List; +import java.util.NoSuchElementException; import java.util.Timer; import java.util.TimerTask; import kademlia.dht.DHT; @@ -24,6 +26,7 @@ import kademlia.operation.StoreOperation; * @todo When we receive a store message - if we have a newer version of the content, re-send this newer version to that node so as to update their version * @todo Handle IPv6 Addresses * @todo Handle compressing data + * @todo Allow optional storing of content locally using the put method */ public class Kademlia { @@ -138,13 +141,23 @@ public class Kademlia * The content returned is a JSON String in byte format; this string is parsed into a class * * @param param The parameters used to search for the content - * @param c The class to cast the returned object to * * @return DHTContent The content + * + * @throws java.io.IOException */ - public KadContent get(GetParameter param, Class c) + public List get(GetParameter param) throws NoSuchElementException, IOException { - return null; + if (this.dht.contains(param)) + { + /* If the content exist in our own DHT, then return it. */ + return this.dht.get(param); + } + else + { + /* Seems like it doesn't exist in our DHT, get it from other Nodes */ + return new DataLookupOperation().execute().getContent(); + } } /** diff --git a/src/kademlia/dht/DHT.java b/src/kademlia/dht/DHT.java index 87c373d..060ff77 100644 --- a/src/kademlia/dht/DHT.java +++ b/src/kademlia/dht/DHT.java @@ -1,10 +1,18 @@ package kademlia.dht; +import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; import kademlia.core.Configuration; +import kademlia.core.GetParameter; +import kademlia.node.NodeId; import kademlia.serializer.JsonSerializer; /** @@ -33,10 +41,90 @@ public class DHT public void store(KadContent content) throws IOException { /* Keep track of this content in the entries manager */ - this.entriesManager.put(new StorageEntry(content)); + this.entriesManager.put(content); + /* Now we store the content locally in a file */ + String contentStorageFolder = this.getContentStorageFolderName(content.getKey()); + DataOutputStream dout = new DataOutputStream(new FileOutputStream(contentStorageFolder + File.separator + content.hashCode() + ".kct")); + new JsonSerializer().write(content, dout); + } + + /** + * Retrieves a Content from local storage + * + * @param key The Key of the content to retrieve + * @param hashCode The hash code of the content to retrieve + * + * @return A KadContent object + */ + private KadContent retrieve(NodeId key, int hashCode) throws FileNotFoundException, IOException, ClassNotFoundException + { + String folder = this.getContentStorageFolderName(key); + DataInputStream in = new DataInputStream(new FileInputStream(folder + File.separator + hashCode + ".kct")); + return new JsonSerializer().read(in); + } + + /** + * Check if any content for the given criteria exists in this DHT + * + * @param param The content search criteria + * + * @return boolean Whether any content exist that satisfy the criteria + */ + public boolean contains(GetParameter param) + { + return this.entriesManager.contains(param); + } + + /** + * Get the List for the content if any exist, + * retrieve the content from the storage system and return it + * + * @param param The parameters used to filter the content needed + * + * @return List A list with the content found on the DHT satisfying the given criteria + * + * @throws java.io.IOException + */ + public List get(GetParameter param) throws NoSuchElementException, IOException + { + /* Load all KadContent for the entries if any exist */ + List values = new ArrayList<>(); + + try + { + for (StorageEntry e : this.entriesManager.get(param)) + { + values.add(this.retrieve(e.getKey(), e.getContentHash())); + } + } + catch (FileNotFoundException e) + { + System.err.println("Error while loading file for content."); + } + catch (ClassNotFoundException e) + { + System.err.println("The class for some content was not found."); + } + + if (values.isEmpty()) + { + throw new NoSuchElementException(); + } + + return values; + } + + /** + * Get the name of the folder for which a content should be stored + * + * @param key The key of the content + * + * @return String The name of the folder + */ + private String getContentStorageFolderName(NodeId key) + { /** - * Now we store the content locally in a file * Each content is stored in a folder named after the first 10 characters of the NodeId * * The name of the file containing the content is the hash of this content @@ -44,23 +132,21 @@ public class DHT String storagePath = System.getProperty("user.home") + File.separator + Configuration.localFolder; File mainStorageFolder = new File(storagePath); - /* Create the folder if it doesn't exist */ + /* Create the main storage folder if it doesn't exist */ if (!mainStorageFolder.isDirectory()) { mainStorageFolder.mkdir(); } - /* Check if a folder after the first 10 characters of Hex(nodeId) exist, if not, create it */ - String folderName = content.getKey().hexRepresentation().substring(0, 20); + String folderName = key.hexRepresentation().substring(0, 10); File contentStorageFolder = new File(mainStorageFolder + File.separator + folderName); + + /* Create the content folder if it doesn't exist */ if (!contentStorageFolder.isDirectory()) { contentStorageFolder.mkdir(); } - /* Write the content to a file and store it in the folder */ - File contentFile = new File(String.valueOf(content.hashCode()) + ".kct"); - DataOutputStream dout = new DataOutputStream(new FileOutputStream(contentStorageFolder + File.separator + contentFile)); - new JsonSerializer().write(content, dout); + return mainStorageFolder + File.separator + folderName; } } diff --git a/src/kademlia/dht/StorageEntry.java b/src/kademlia/dht/StorageEntry.java index 673a53b..b087bc0 100644 --- a/src/kademlia/dht/StorageEntry.java +++ b/src/kademlia/dht/StorageEntry.java @@ -1,6 +1,7 @@ package kademlia.dht; import java.util.Objects; +import kademlia.core.GetParameter; import kademlia.node.NodeId; /** @@ -16,12 +17,14 @@ public class StorageEntry private final NodeId key; private final String ownerId; private final String type; + private final int contentHash; public StorageEntry(KadContent content) { this.key = content.getKey(); this.ownerId = content.getOwnerId(); this.type = content.getType(); + this.contentHash = content.hashCode(); } public NodeId getKey() @@ -29,6 +32,52 @@ public class StorageEntry return this.key; } + public String getOwnerId() + { + return this.ownerId; + } + + public String getType() + { + return this.type; + } + + public int getContentHash() + { + return this.contentHash; + } + + /** + * When a node is looking for content, he sends the search criteria in a GetParameter object + * Here we take this GetParameter object and check if this StorageEntry satisfies the given parameters + * + * @param params + * + * @return boolean Whether this content satisfies the parameters + */ + public boolean satisfiesParameters(GetParameter params) + { + /* Check that owner id matches */ + if ((params.getOwnerId() != null) && (!params.getOwnerId().equals(this.ownerId))) + { + return false; + } + + /* Check that type matches */ + if ((params.getType() != null) && (!params.getType().equals(this.type))) + { + return false; + } + + /* Check that key matches */ + if ((params.getKey() != null) && (!params.getKey().equals(this.key))) + { + return false; + } + + return true; + } + @Override public boolean equals(Object o) { diff --git a/src/kademlia/dht/StorageEntryManager.java b/src/kademlia/dht/StorageEntryManager.java index 674f74b..0fe79f6 100644 --- a/src/kademlia/dht/StorageEntryManager.java +++ b/src/kademlia/dht/StorageEntryManager.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; +import kademlia.core.GetParameter; import kademlia.node.NodeId; /** @@ -27,15 +29,16 @@ public class StorageEntryManager /** * Add a new entry to our storage * - * @param entry + * @param content The content to store a reference to */ - public void put(StorageEntry entry) + public void put(KadContent content) { + StorageEntry entry = new StorageEntry(content); if (!this.entries.containsKey(entry.getKey())) { this.entries.put(entry.getKey(), new ArrayList()); } - + this.entries.get(entry.getKey()).add(entry); } @@ -44,27 +47,65 @@ public class StorageEntryManager * * @todo Add searching for content by type and ownerID * - * @param key + * @param param The parameters used to search for a content * * @return boolean */ - public boolean contains(NodeId key) + public boolean contains(GetParameter param) { - return this.entries.containsKey(key); + if (this.entries.containsKey(param.getKey())) + { + /* Content with this key exist, check if any match the rest of the search criteria */ + for (StorageEntry e : this.entries.get(param.getKey())) + { + /* If any entry satisfies the given parameters, return true */ + if (e.satisfiesParameters(param)) + { + return true; + } + } + } + return false; } /** * Checks if our DHT has a Content for the given criteria * - * @todo Add finding for content by type and ownerID + * @param param The parameters used to search for a content * - * @param key + * @todo Add finding for content by type and ownerID * * @return List of content for the specific search parameters */ - public List get(NodeId key) + public List get(GetParameter param) throws NoSuchElementException { - return this.entries.get(key); + if (this.entries.containsKey(param.getKey())) + { + /* Content with this key exist, check if any match the rest of the search criteria */ + List results = new ArrayList<>(); + + for (StorageEntry e : this.entries.get(param.getKey())) + { + /* If any entry satisfies the given parameters, return true */ + if (e.satisfiesParameters(param)) + { + results.add(e); + } + } + + if (results.size() > 0) + { + return results; + } + else + { + throw new NoSuchElementException(); + } + } + else + { + throw new NoSuchElementException("No content exist for the given parameters"); + } } } diff --git a/src/kademlia/operation/ContentLookupOperation.java b/src/kademlia/operation/ContentLookupOperation.java new file mode 100644 index 0000000..b55dcf9 --- /dev/null +++ b/src/kademlia/operation/ContentLookupOperation.java @@ -0,0 +1,31 @@ +package kademlia.operation; + +import kademlia.core.KadServer; +import kademlia.node.Node; +import kademlia.node.NodeId; + +/** + * Looks up a specified identifier and returns the value associated with it + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class ContentLookupOperation implements Operation +{ + + private final KadServer server; + private final Node localNode; + private final NodeId key; + + /** + * @param server + * @param localNode + * @param key The key for the content which we need to find + */ + public ContentLookupOperation(KadServer server, Node localNode, NodeId key) + { + this.server = server; + this.localNode = localNode; + this.key = key; + } +}