From 9190b122c7f917e79d4c7bd990f4ee0e860abc63 Mon Sep 17 00:00:00 2001 From: Joshua Kissoon Date: Wed, 2 Apr 2014 16:35:08 +0530 Subject: [PATCH] The current mechanism for serializing content will not work for complex content types. Now we need to add methods to KadContent objects to provide the byte[] format of their content and to re-read the byte format of their content. --- src/kademlia/dht/DHT.java | 38 +++--- src/kademlia/dht/KadContent.java | 10 ++ src/kademlia/dht/StorageEntry.java | 110 ++-------------- src/kademlia/dht/StorageEntryManager.java | 34 ++--- src/kademlia/dht/StorageEntryMetadata.java | 117 ++++++++++++++++++ src/kademlia/message/ContentMessage.java | 20 ++- .../operation/ContentRefreshOperation.java | 6 +- .../serializer/JsonDHTSerializer.java | 6 +- src/kademlia/serializer/JsonSerializer.java | 1 - src/kademlia/tests/DHTContentImpl.java | 2 +- 10 files changed, 191 insertions(+), 153 deletions(-) create mode 100644 src/kademlia/dht/StorageEntryMetadata.java diff --git a/src/kademlia/dht/DHT.java b/src/kademlia/dht/DHT.java index 0036cd1..404ad76 100644 --- a/src/kademlia/dht/DHT.java +++ b/src/kademlia/dht/DHT.java @@ -84,12 +84,18 @@ public class DHT /* Keep track of this content in the entries manager */ try { - StorageEntry sEntry = this.entriesManager.put(content); + StorageEntryMetadata sEntry = this.entriesManager.put(content); /* Now we store the content locally in a file */ String contentStorageFolder = this.getContentStorageFolderName(content.getKey()); - DataOutputStream dout = new DataOutputStream(new FileOutputStream(contentStorageFolder + File.separator + sEntry.hashCode() + ".kct")); - getContentSerializer().write(content, dout); + + try (FileOutputStream fout = new FileOutputStream(contentStorageFolder + File.separator + sEntry.hashCode() + ".kct"); + DataOutputStream dout = new DataOutputStream(fout)) + { + byte[] data = content.toBytes(); + dout.writeInt(data.length); + dout.write(data); + } } catch (ContentExistException e) { @@ -105,11 +111,15 @@ public class DHT * * @return A KadContent object */ - private KadContent retrieve(NodeId key, int hashCode) throws FileNotFoundException, IOException, ClassNotFoundException + private byte[] retrieve(NodeId key, int hashCode) throws FileNotFoundException, IOException, ClassNotFoundException { String folder = this.getContentStorageFolderName(key); - DataInputStream in = new DataInputStream(new FileInputStream(folder + File.separator + hashCode + ".kct")); - return getContentSerializer().read(in); + DataInputStream din = new DataInputStream(new FileInputStream(folder + File.separator + hashCode + ".kct")); + int length = din.readInt(); + + byte[] data = new byte[length]; + din.read(data); + return data; } /** @@ -133,7 +143,7 @@ public class DHT * * @throws java.io.IOException */ - public KadContent get(StorageEntry entry) throws IOException, NoSuchElementException + public byte[] get(StorageEntryMetadata entry) throws IOException, NoSuchElementException { try { @@ -162,12 +172,12 @@ public class DHT * * @throws java.io.IOException */ - public KadContent get(GetParameter param) throws NoSuchElementException, IOException + public byte[] get(GetParameter param) throws NoSuchElementException, IOException { /* Load a KadContent if any exist for the given criteria */ try { - StorageEntry e = this.entriesManager.get(param); + StorageEntryMetadata e = this.entriesManager.get(param); return this.retrieve(e.getKey(), e.hashCode()); } catch (FileNotFoundException e) @@ -193,10 +203,10 @@ public class DHT */ public void remove(KadContent content) throws ContentNotFoundException { - this.remove(new StorageEntry(content)); + this.remove(new StorageEntryMetadata(content)); } - public void remove(StorageEntry entry) throws ContentNotFoundException + public void remove(StorageEntryMetadata entry) throws ContentNotFoundException { String folder = this.getContentStorageFolderName(entry.getKey()); File file = new File(folder + File.separator + entry.hashCode() + ".kct"); @@ -242,7 +252,7 @@ public class DHT /** * @return A List of all StorageEntries for this node */ - public List getStorageEntries() + public List getStorageEntries() { return entriesManager.getAllEntries(); } @@ -253,9 +263,9 @@ public class DHT * * @param ientries The entries to add */ - public void putStorageEntries(List ientries) + public void putStorageEntries(List ientries) { - for (StorageEntry e : ientries) + for (StorageEntryMetadata e : ientries) { try { diff --git a/src/kademlia/dht/KadContent.java b/src/kademlia/dht/KadContent.java index bb03c3a..2fd3715 100644 --- a/src/kademlia/dht/KadContent.java +++ b/src/kademlia/dht/KadContent.java @@ -41,4 +41,14 @@ public interface KadContent * @return The ID of the owner of this content */ public String getOwnerId(); + + public default byte[] toBytes() + { + return new byte[2]; + } + + public default void fromBytes(byte[] data) + { + + } } diff --git a/src/kademlia/dht/StorageEntry.java b/src/kademlia/dht/StorageEntry.java index 2bdb94d..fb748d0 100644 --- a/src/kademlia/dht/StorageEntry.java +++ b/src/kademlia/dht/StorageEntry.java @@ -1,117 +1,23 @@ package kademlia.dht; -import java.util.Objects; -import kademlia.core.GetParameter; -import kademlia.node.NodeId; - /** - * Keeps track of data for a Content stored in the DHT - * Used by the StorageEntryManager class + * A StorageEntry class that is used to store a content on the DHT * * @author Joshua Kissoon - * @since 20140226 + * @since 20140402 */ public class StorageEntry { - - private final NodeId key; - private final String ownerId; - private final String type; - private final int contentHash; - private final long lastUpdated; - + private byte[] content; + private StorageEntryMetadata metadata; + public StorageEntry(KadContent content) { - this.key = content.getKey(); - this.ownerId = content.getOwnerId(); - this.type = content.getType(); - this.contentHash = content.hashCode(); - this.lastUpdated = content.getLastUpdatedTimestamp(); - } - - public NodeId getKey() - { - return this.key; - } - - public String getOwnerId() - { - return this.ownerId; - } - - public String getType() - { - return this.type; - } - - public int getContentHash() - { - return this.contentHash; - } - - /** - * When a node is looking for content, he sends the search criteria in a GetParameter object - * Here we take this GetParameter object and check if this StorageEntry satisfies the given parameters - * - * @param params - * - * @return boolean Whether this content satisfies the parameters - */ - public boolean satisfiesParameters(GetParameter params) - { - /* Check that owner id matches */ - if ((params.getOwnerId() != null) && (!params.getOwnerId().equals(this.ownerId))) - { - return false; - } - /* Check that type matches */ - if ((params.getType() != null) && (!params.getType().equals(this.type))) - { - return false; - } - - /* Check that key matches */ - return (params.getKey() != null) && (params.getKey().equals(this.key)); } - - @Override - public boolean equals(Object o) + + public StorageEntry(KadContent content, StorageEntryMetadata metadata) { - if (o instanceof StorageEntry) - { - return this.hashCode() == o.hashCode(); - } - - return false; - } - - @Override - public int hashCode() - { - int hash = 3; - hash = 23 * hash + Objects.hashCode(this.key); - hash = 23 * hash + Objects.hashCode(this.ownerId); - hash = 23 * hash + Objects.hashCode(this.type); - return hash; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder("[StorageEntry: "); - - sb.append("{Key: "); - sb.append(this.key); - sb.append("} "); - sb.append("{Owner: "); - sb.append(this.ownerId); - sb.append("} "); - sb.append("{Type: "); - sb.append(this.type); - sb.append("} "); - sb.append("]"); - - return sb.toString(); + this.content = content.g } } diff --git a/src/kademlia/dht/StorageEntryManager.java b/src/kademlia/dht/StorageEntryManager.java index 27fa91e..c9b5255 100644 --- a/src/kademlia/dht/StorageEntryManager.java +++ b/src/kademlia/dht/StorageEntryManager.java @@ -21,7 +21,7 @@ import kademlia.node.NodeId; class StorageEntryManager { - private final Map> entries; + private final Map> entries; { @@ -33,9 +33,9 @@ class StorageEntryManager * * @param content The content to store a reference to */ - public StorageEntry put(KadContent content) throws ContentExistException + public StorageEntryMetadata put(KadContent content) throws ContentExistException { - return this.put(new StorageEntry(content)); + return this.put(new StorageEntryMetadata(content)); } /** @@ -43,11 +43,11 @@ class StorageEntryManager * * @param entry The StorageEntry to store */ - public StorageEntry put(StorageEntry entry) throws ContentExistException + public StorageEntryMetadata put(StorageEntryMetadata entry) throws ContentExistException { if (!this.entries.containsKey(entry.getKey())) { - this.entries.put(entry.getKey(), new ArrayList()); + this.entries.put(entry.getKey(), new ArrayList<>()); } /* If this entry doesn't already exist, then we add it */ @@ -78,7 +78,7 @@ class StorageEntryManager { System.out.println("Does contain the key"); /* Content with this key exist, check if any match the rest of the search criteria */ - for (StorageEntry e : this.entries.get(param.getKey())) + for (StorageEntryMetadata e : this.entries.get(param.getKey())) { /* If any entry satisfies the given parameters, return true */ if (e.satisfiesParameters(param)) @@ -100,13 +100,13 @@ class StorageEntryManager */ public boolean contains(KadContent content) { - return this.contains(new StorageEntry(content)); + return this.contains(new StorageEntryMetadata(content)); } /** * Check if a StorageEntry exist on this DHT */ - private boolean contains(StorageEntry entry) + private boolean contains(StorageEntryMetadata entry) { if (this.entries.containsKey(entry.getKey())) { @@ -125,12 +125,12 @@ class StorageEntryManager * * @return List of content for the specific search parameters */ - public StorageEntry get(GetParameter param) throws NoSuchElementException + public StorageEntryMetadata get(GetParameter param) throws NoSuchElementException { if (this.entries.containsKey(param.getKey())) { /* Content with this key exist, check if any match the rest of the search criteria */ - for (StorageEntry e : this.entries.get(param.getKey())) + for (StorageEntryMetadata e : this.entries.get(param.getKey())) { /* If any entry satisfies the given parameters, return true */ if (e.satisfiesParameters(param)) @@ -151,11 +151,11 @@ class StorageEntryManager /** * @return A list of all storage entries */ - public List getAllEntries() + public List getAllEntries() { - List entriesRet = new ArrayList<>(); + List entriesRet = new ArrayList<>(); - for (List entrySet : this.entries.values()) + for (List entrySet : this.entries.values()) { if (entrySet.size() > 0) { @@ -168,10 +168,10 @@ class StorageEntryManager public void remove(KadContent content) throws ContentNotFoundException { - this.remove(new StorageEntry(content)); + this.remove(new StorageEntryMetadata(content)); } - public void remove(StorageEntry entry) throws ContentNotFoundException + public void remove(StorageEntryMetadata entry) throws ContentNotFoundException { if (contains(entry)) { @@ -187,14 +187,14 @@ class StorageEntryManager public String toString() { StringBuilder sb = new StringBuilder("Stored Content: \n"); - for (List es : this.entries.values()) + for (List es : this.entries.values()) { if (entries.size() < 1) { continue; } - for (StorageEntry e : es) + for (StorageEntryMetadata e : es) { sb.append(e); sb.append("\n"); diff --git a/src/kademlia/dht/StorageEntryMetadata.java b/src/kademlia/dht/StorageEntryMetadata.java new file mode 100644 index 0000000..aa923cd --- /dev/null +++ b/src/kademlia/dht/StorageEntryMetadata.java @@ -0,0 +1,117 @@ +package kademlia.dht; + +import java.util.Objects; +import kademlia.core.GetParameter; +import kademlia.node.NodeId; + +/** + * Keeps track of data for a Content stored in the DHT + * Used by the StorageEntryManager class + * + * @author Joshua Kissoon + * @since 20140226 + */ +public class StorageEntryMetadata +{ + + private final NodeId key; + private final String ownerId; + private final String type; + private final int contentHash; + private final long lastUpdated; + + public StorageEntryMetadata(KadContent content) + { + this.key = content.getKey(); + this.ownerId = content.getOwnerId(); + this.type = content.getType(); + this.contentHash = content.hashCode(); + this.lastUpdated = content.getLastUpdatedTimestamp(); + } + + public NodeId getKey() + { + return this.key; + } + + public String getOwnerId() + { + return this.ownerId; + } + + public String getType() + { + return this.type; + } + + public int getContentHash() + { + return this.contentHash; + } + + /** + * When a node is looking for content, he sends the search criteria in a GetParameter object + * Here we take this GetParameter object and check if this StorageEntry satisfies the given parameters + * + * @param params + * + * @return boolean Whether this content satisfies the parameters + */ + public boolean satisfiesParameters(GetParameter params) + { + /* Check that owner id matches */ + if ((params.getOwnerId() != null) && (!params.getOwnerId().equals(this.ownerId))) + { + return false; + } + + /* Check that type matches */ + if ((params.getType() != null) && (!params.getType().equals(this.type))) + { + return false; + } + + /* Check that key matches */ + return (params.getKey() != null) && (params.getKey().equals(this.key)); + } + + @Override + public boolean equals(Object o) + { + if (o instanceof StorageEntryMetadata) + { + return this.hashCode() == o.hashCode(); + } + + return false; + } + + @Override + public int hashCode() + { + int hash = 3; + hash = 23 * hash + Objects.hashCode(this.key); + hash = 23 * hash + Objects.hashCode(this.ownerId); + hash = 23 * hash + Objects.hashCode(this.type); + return hash; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder("[StorageEntry: "); + + sb.append("{Key: "); + sb.append(this.key); + sb.append("} "); + sb.append("{Owner: "); + sb.append(this.ownerId); + sb.append("} "); + sb.append("{Type: "); + sb.append(this.type); + sb.append("} "); + sb.append("]"); + + return sb.toString(); + } +} diff --git a/src/kademlia/message/ContentMessage.java b/src/kademlia/message/ContentMessage.java index 8015751..7e117e2 100644 --- a/src/kademlia/message/ContentMessage.java +++ b/src/kademlia/message/ContentMessage.java @@ -18,7 +18,7 @@ public class ContentMessage implements Message public static final byte CODE = 0x04; - private KadContent content; + private byte[] content; private Node origin; /** @@ -26,7 +26,7 @@ public class ContentMessage implements Message * @param content The content to be stored * */ - public ContentMessage(Node origin, KadContent content) + public ContentMessage(Node origin, byte[] content) { this.content = content; this.origin = origin; @@ -43,7 +43,8 @@ public class ContentMessage implements Message this.origin.toStream(out); /* Serialize the KadContent, then send it to the stream */ - new JsonSerializer().write(content, out); + out.writeInt(content.length); + out.write(content); } @Override @@ -51,14 +52,9 @@ public class ContentMessage implements Message { this.origin = new Node(in); - try - { - this.content = new JsonSerializer().read(in); - } - catch (ClassNotFoundException e) - { - e.printStackTrace(); - } + final int length = in.readInt(); + this.content = new byte[length]; + in.read(content); } public Node getOrigin() @@ -66,7 +62,7 @@ public class ContentMessage implements Message return this.origin; } - public KadContent getContent() + public byte[] getContent() { return this.content; } diff --git a/src/kademlia/operation/ContentRefreshOperation.java b/src/kademlia/operation/ContentRefreshOperation.java index c42ed84..1818a93 100644 --- a/src/kademlia/operation/ContentRefreshOperation.java +++ b/src/kademlia/operation/ContentRefreshOperation.java @@ -6,7 +6,7 @@ import kademlia.core.DefaultConfiguration; import kademlia.core.KadConfiguration; import kademlia.core.KadServer; import kademlia.dht.DHT; -import kademlia.dht.StorageEntry; +import kademlia.dht.StorageEntryMetadata; import kademlia.exceptions.ContentNotFoundException; import kademlia.message.Message; import kademlia.message.StoreContentMessage; @@ -46,10 +46,10 @@ public class ContentRefreshOperation implements Operation public void execute() throws IOException { /* Get a list of all storage entries for content */ - List entries = this.dht.getStorageEntries(); + List entries = this.dht.getStorageEntries(); /* For each storage entry, distribute it */ - for (StorageEntry e : entries) + for (StorageEntryMetadata e : entries) { /** * @todo - Paper improvement 1 - diff --git a/src/kademlia/serializer/JsonDHTSerializer.java b/src/kademlia/serializer/JsonDHTSerializer.java index 050ed76..6f88215 100644 --- a/src/kademlia/serializer/JsonDHTSerializer.java +++ b/src/kademlia/serializer/JsonDHTSerializer.java @@ -12,7 +12,7 @@ import java.io.OutputStreamWriter; import java.lang.reflect.Type; import java.util.List; import kademlia.dht.DHT; -import kademlia.dht.StorageEntry; +import kademlia.dht.StorageEntryMetadata; /** * A KadSerializer that serializes DHT to JSON format @@ -48,7 +48,7 @@ public class JsonDHTSerializer implements KadSerializer { gson = new Gson(); - storageEntriesCollectionType = new TypeToken>() + storageEntriesCollectionType = new TypeToken>() { }.getType(); } @@ -84,7 +84,7 @@ public class JsonDHTSerializer implements KadSerializer dht.initialize(); /* Now get the entries and add them back to the DHT */ - List entries = gson.fromJson(reader, this.storageEntriesCollectionType); + List entries = gson.fromJson(reader, this.storageEntriesCollectionType); dht.putStorageEntries(entries); reader.endArray(); diff --git a/src/kademlia/serializer/JsonSerializer.java b/src/kademlia/serializer/JsonSerializer.java index 8ad88f9..a43170e 100644 --- a/src/kademlia/serializer/JsonSerializer.java +++ b/src/kademlia/serializer/JsonSerializer.java @@ -43,7 +43,6 @@ public class JsonSerializer implements KadSerializer writer.endArray(); } - } @Override diff --git a/src/kademlia/tests/DHTContentImpl.java b/src/kademlia/tests/DHTContentImpl.java index 93b2a5e..6b1864b 100644 --- a/src/kademlia/tests/DHTContentImpl.java +++ b/src/kademlia/tests/DHTContentImpl.java @@ -42,7 +42,7 @@ public class DHTContentImpl implements KadContent this.data = newData; } - public String getData() + public String toBytes() { return this.data; }