[Archipelago-submits] [185] trunk/archipelago: working but incomplete dump with test
nobody at rubyforge.org
nobody at rubyforge.org
Tue Jan 23 09:57:30 EST 2007
Revision: 185
Author: zond
Date: 2007-01-23 09:57:30 -0500 (Tue, 23 Jan 2007)
Log Message:
-----------
working but incomplete dump with test
Modified Paths:
--------------
trunk/archipelago/lib/archipelago/dump.rb
trunk/archipelago/lib/archipelago/hashish.rb
trunk/archipelago/lib/archipelago/sanitation.rb
trunk/archipelago/tests/test_helper.rb
Added Paths:
-----------
trunk/archipelago/tests/dump_test.rb
Modified: trunk/archipelago/lib/archipelago/dump.rb
===================================================================
--- trunk/archipelago/lib/archipelago/dump.rb 2007-01-22 16:10:14 UTC (rev 184)
+++ trunk/archipelago/lib/archipelago/dump.rb 2007-01-23 14:57:30 UTC (rev 185)
@@ -15,10 +15,11 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+require 'rubygems'
+require 'oneliner'
require 'archipelago/disco'
require 'archipelago/hashish'
-require 'archipelago/exxon'
-require 'archipelago/raider'
+require 'archipelago/sanitation'
module Archipelago
@@ -37,9 +38,9 @@
#
@persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("cove_tanker.db"))
- @db = @persistence_provider.get_hashish("db")
+ @officer = options[:officer] || Archipelago::Sanitation::CLEANER
- @valdez = options[:dumper] || Archipelago::Exxon::HAZELWOOD
+ @dbs = {}
#
# Use the given options to initialize the publishable
@@ -49,18 +50,37 @@
end
- def store_part(key, split_part)
- @db[key] = Marshal.dump(split_part)
- end
-
- def get_part(key)
- if @db.include?(key)
- return Marshal.load(@db[key])
+ def insert!(key, value, owner_id)
+ db = @dbs[owner_id] ||= @persistence_provider.get_dup_hashish(owner_id)
+ db[key] = value
+ if owner_id == service_id
+ subscribe_as_master(key)
else
- return nil
+ subscribe_as_slave(owner_id)
end
end
+ def fetch(key, owner_id)
+ db = @dbs[owner_id] ||= @persistence_provider.get_dup_hashish(owner_id)
+ return db.duplicates(key)
+ end
+
+ def delete(key, owner_id)
+ db = @dbs[owner_id] ||= @persistence_provider.get_dup_hashish(owner_id)
+ db.delete(key)
+ return nil
+ end
+
+ private
+
+ def subscribe_as_master(key)
+ # IMPLEMENT ME DAMNIT
+ end
+
+ def subscribe_as_slave(owner_id)
+ # IMPLEMENT ME AS WELL!
+ end
+
end
end
Modified: trunk/archipelago/lib/archipelago/hashish.rb
===================================================================
--- trunk/archipelago/lib/archipelago/hashish.rb 2007-01-22 16:10:14 UTC (rev 184)
+++ trunk/archipelago/lib/archipelago/hashish.rb 2007-01-23 14:57:30 UTC (rev 185)
@@ -275,7 +275,7 @@
# using +name+.
#
def get_dup_hashish(name)
- db = BDB::Hash.open(File.join(@env.configuration["home"], name),
+ db = BDB::Hash.open(Pathname.new(File.join(@env.home, name)).expand_path,
nil,
BDB::CREATE | BDB::NOMMAP,
0,
@@ -308,9 +308,9 @@
#
def unlink!
close!
- home = Pathname.new(@env.home)
+ home = Pathname.new(@env.home).expand_path
@env.close
- home.rmtree if home.exist?
+ home.rmtree if home.exist?
end
end
Modified: trunk/archipelago/lib/archipelago/sanitation.rb
===================================================================
--- trunk/archipelago/lib/archipelago/sanitation.rb 2007-01-22 16:10:14 UTC (rev 184)
+++ trunk/archipelago/lib/archipelago/sanitation.rb 2007-01-23 14:57:30 UTC (rev 185)
@@ -25,7 +25,10 @@
:class => 'Archipelago::Dump::Site'
}
- PARTS = 3
+ MINIMUM_CHUNK_SIZE = 40
+ MINIMUM_NR_OF_CHUNKS = 10
+ MINIMUM_REDUNDANCY_RATIO = 2
+ METADATA_OVERHEAD = 8
#
# Raised when you try to do stuff without any remote database
@@ -37,6 +40,16 @@
end
end
+ #
+ # Raised when you try to fetch data that we find traces of
+ # but are unable to fully restore.
+ #
+ class NotEnoughDataException < RuntimeError
+ def initialize(officer, key)
+ super("#{officer} can not find enough data to restore the value for #{key}")
+ end
+ end
+
class Officer < Archipelago::Client::Base
attr_reader :sites
def initialize(options = {})
@@ -47,30 +60,92 @@
def setup(options = {})
super(options)
-
- @parts = options[:parts] || PARTS
+ @minimum_chunk_size = options[:minimum_chunk_size] || MINIMUM_CHUNK_SIZE
+ @minimum_nr_of_chunks = options[:minimum_nr_of_chunks] || MINIMUM_NR_OF_CHUNKS
+ @minimum_redundancy_ratio = options[:minimum_redundancy_ratio] || MINIMUM_REDUNDANCY_RATIO
+ @metadata_overhead = options[:metadata_overhead] || METADATA_OVERHEAD
@site_description = SITE_DESCRIPTION.merge(options[:site_description] || {})
end
def []=(key, value)
+ super_string = Oneliner::SuperString.new(value)
+ chunk_size = (super_string.size / (@minimum_nr_of_chunks / @minimum_redundancy_ratio)) + @metadata_overhead
+ chunk_size = @minimum_chunk_size if chunk_size < @minimum_chunk_size
+
+ dumps = responsible_dumps(key, @minimum_nr_of_chunks)
+ owner_id = dumps.first[:service_id]
+ dumps.each do |dump|
+ dump[:service].insert!(key,
+ super_string.encode(chunk_size),
+ owner_id)
+ end
end
def [](key)
+ dumps = responsible_dumps(key, @minimum_nr_of_chunks)
+ owner_id = dumps.first[:service_id]
+
+ rval = Oneliner::SuperString.new
+ decoded = false
+ found_chunks = false
+ while !decoded && dumps.size > 0
+ dumps.shift.fetch(key, owner_id).each do |chunk|
+ found_chunks = true
+ decoded = decoded || rval.decode!(chunk)
+ end
+ end
+
+ if decoded
+ return rval.to_s
+ else
+ raise NotEnoughDataException.new(self, key) if found_chunks
+ return nil
+ end
+
end
def update_services!
@sites = @jockey.lookup(Archipelago::Disco::Query.new(@site_description), 0)
end
- def responsible_tankers(key, n)
- end
+ #
+ # Returns the +n+ services in our @dumps Hash that have keys
+ # greater than +key+.
+ #
+ # Will loop to the beginning if the number of elements run out.
+ #
+ def responsible_dumps(key, n)
+ raise NoRemoteDatabaseAvailableException.new(self) if @dumps.empty?
- def responsible_tanker(key)
- return responsible_tankers(key, 1).first
+ get_least_greater_than(@dumps, key, n).collect do |id|
+ @dumps[id]
+ end
end
private
+ #
+ # Gets the +n+ smallest keys from +hash+ that
+ # are greater than +o+.
+ #
+ # Will loop to the beginning if the number of elements run out.
+ #
+ def get_least_greater_than(hash, o, n)
+ sorted_key_array = hash.keys.sort
+ 0.upto(sorted_key_array.size - 1) do |index|
+ key = sorted_key_array[index]
+ if key > o
+ return get_some(sorted_key_array, index, n)
+ end
+ end
+ return get_some(sorted_key_array, 0, n)
+ end
+
+ #
+ # Gets +n+ values of +array+ starting at +index+.
+ #
+ # Will loop to the beginning if the elements run out.
+ #
def get_some(array, index, n)
rval = []
while rval.size < n
Added: trunk/archipelago/tests/dump_test.rb
===================================================================
--- trunk/archipelago/tests/dump_test.rb (rev 0)
+++ trunk/archipelago/tests/dump_test.rb 2007-01-23 14:57:30 UTC (rev 185)
@@ -0,0 +1,35 @@
+
+require File.join(File.dirname(__FILE__), 'test_helper')
+
+class DumpTest < Test::Unit::TestCase
+
+ def setup
+ DRb.start_service
+ @d = Archipelago::Dump::Site.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("site.db")))
+ end
+
+ def teardown
+ @d.instance_eval do @persistence_provider.unlink! end
+ DRb.stop_service
+ end
+
+ def test_insert_fetch
+ @d.insert!("key", "value", "owner")
+ @d.insert!("key", "value1", "owner")
+ @d.insert!("key2", "value2", "owner")
+ @d.insert!("key2", "value3", "owner2")
+ assert_equal(["value", "value1"].sort,
+ @d.fetch("key", "owner").sort)
+ assert_equal(["value2"],
+ @d.fetch("key2", "owner"))
+ assert_equal(["value3"],
+ @d.fetch("key2", "owner2"))
+ @d.delete("key", "owner")
+ @d.delete("key2", "owner2")
+ assert_equal([],
+ @d.fetch("key", "owner"))
+ assert_equal([],
+ @d.fetch("key2", "owner2"))
+ end
+
+end
Modified: trunk/archipelago/tests/test_helper.rb
===================================================================
--- trunk/archipelago/tests/test_helper.rb 2007-01-22 16:10:14 UTC (rev 184)
+++ trunk/archipelago/tests/test_helper.rb 2007-01-23 14:57:30 UTC (rev 185)
@@ -11,6 +11,7 @@
require 'archipelago'
require 'archipelago/treasure'
require 'archipelago/pirate'
+require 'archipelago/dump'
require 'benchmark'
require 'socket'
require 'ipaddr'
More information about the Archipelago-submits
mailing list