From nobody at rubyforge.org Sun Mar 4 13:49:54 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Sun, 4 Mar 2007 13:49:54 -0500 (EST) Subject: [Archipelago-submits] [220] trunk/archipelago: made debug output from dump better. Message-ID: <20070304184954.63E6352420C3@rubyforge.org> Revision: 220 Author: zond Date: 2007-03-04 13:49:53 -0500 (Sun, 04 Mar 2007) Log Message: ----------- made debug output from dump better. made console services able to select drburi as well. lowered validation interval slightly. added lots of debug stuff to dump. made dump not run redistribute within the each-block when redistributing orphan keys. added options to hashish db getters. made sanitation synchronize on a unique lock when fetching instead of the return value. Modified Paths: -------------- trunk/archipelago/lib/archipelago/disco.rb trunk/archipelago/lib/archipelago/dump.rb trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/sanitation.rb trunk/archipelago/script/officer.rb trunk/archipelago/script/pirate.rb trunk/archipelago/script/services.rb Modified: trunk/archipelago/lib/archipelago/disco.rb =================================================================== --- trunk/archipelago/lib/archipelago/disco.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/lib/archipelago/disco.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -54,7 +54,7 @@ # Default pause between trying to validate all services we # know about. # - VALIDATION_INTERVAL = 60 + VALIDATION_INTERVAL = 30 # # Only save stuff that we KNOW we want. # Modified: trunk/archipelago/lib/archipelago/dump.rb =================================================================== --- trunk/archipelago/lib/archipelago/dump.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/lib/archipelago/dump.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -125,6 +125,7 @@ values = @db.duplicates(key).collect do |value| [value[0...4], value[4..-1]] end + return values end # @@ -178,6 +179,7 @@ @officer.redistribute(key) @db.delete(key) rescue Archipelago::Sanitation::NotEnoughDataException => e + @debug_callable.call("#{service_id}.check_key(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable # What shall we do in this case? # * Nothing, and wait for the admins to make a manual check? # * Send an email someplace? @@ -211,7 +213,7 @@ break if check_key(key) end rescue Exception => e - # /moo + @debug_callable.call("#{service_id}.start_edge_check() got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable end sleep(@check_interval) end @@ -242,6 +244,7 @@ @debug_callable.call("#{service_id}.redistribute_key(#{key}) called") if @debug_callable @officer.redistribute(key) rescue Archipelago::Sanitation::NotEnoughDataException => e + @debug_callable.call("#{service_id}.redistribute_key(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable # What shall we do in this case? # * Nothing, and wait for the admins to make a manual check? # * Send an email someplace? @@ -263,20 +266,24 @@ # def lost_peer(record) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) called") if @debug_callable + keys_to_redistribute = [] if right_before?(record[:service_id]) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) is right_before, redistributing from #{@officer.second_master_to(service_id)}") if @debug_callable @db.reverse_each_key do |key| - redistribute_key(key) + keys_to_redistribute << key break if key < @officer.second_master_to(service_id) end end if right_after?(record[:service_id]) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) is right_after, redistributing up to #{record[:service_id]}") if @debug_callable @db.each_key do |key| - redistribute_key(key) + keys_to_redistribute << key break if key > record[:service_id] end end + keys_to_redistribute.each do |key| + redistribute_key(key) + end end end Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -271,13 +271,13 @@ return hashish end # - # Returns something acting like an uncached Berkeley Hash DB instance allowing duplicate entries + # Returns something acting like a Berkeley Hash DB instance allowing duplicate entries # and transactions using +name+. # - def get_dup_tree(name) + def get_dup_tree(name, flags = BDB::CREATE | BDB::NOMMAP) db = BDB::Hash.open(Pathname.new(File.join(@env.home, name)).expand_path, nil, - BDB::CREATE | BDB::NOMMAP, + flags, 0, "env" => @env, "set_flags" => BDB::DUP) @@ -285,10 +285,10 @@ return db end # - # Returns something acting like an uncached Berkeley Hash DB instance using +name+. + # Returns something acting like a Berkeley Hash DB instance using +name+. # - def get_hashish(name) - db = @env.open_db(BDB::HASH, name, nil, BDB::CREATE | BDB::NOMMAP) + def get_hashish(name, flags = BDB::CREATE | BDB::NOMMAP) + db = @env.open_db(BDB::HASH, name, nil, flags) @bdb_dbs << db return db end Modified: trunk/archipelago/lib/archipelago/sanitation.rb =================================================================== --- trunk/archipelago/lib/archipelago/sanitation.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -214,18 +214,17 @@ newest_timestamp = "\000\000\000\000" threads = [] rval = Oneliner::SuperString.new - rval.extend(MonitorMixin) + lock = Archipelago::Current::Lock.new dump_hash.t_each do |dump_id, nr_of_chunks_available| site = @sites[dump_id][:service] begin chunks = site.fetch(key) - rval.mon_synchronize do + lock.mon_synchronize do while chunks.size > 0 t, data = chunks.shift if t > newest_timestamp rval = Oneliner::SuperString.new - rval.extend(MonitorMixin) newest_timestamp = t end Modified: trunk/archipelago/script/officer.rb =================================================================== --- trunk/archipelago/script/officer.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/script/officer.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -5,6 +5,6 @@ begin DRb.uri rescue DRb::DRbServerNotFound - DRb.start_service + DRb.start_service(ENV["DRBURI"]) end @o = Archipelago::Sanitation::Officer.new Modified: trunk/archipelago/script/pirate.rb =================================================================== --- trunk/archipelago/script/pirate.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/script/pirate.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -5,7 +5,7 @@ begin DRb.uri rescue DRb::DRbServerNotFound - DRb.start_service + DRb.start_service(ENV["DRBURI"]) end @p = Archipelago::Pirate::Captain.new @p.evaluate!(File.join(File.dirname(__FILE__), 'overloads.rb')) Modified: trunk/archipelago/script/services.rb =================================================================== --- trunk/archipelago/script/services.rb 2007-02-22 22:33:52 UTC (rev 219) +++ trunk/archipelago/script/services.rb 2007-03-04 18:49:53 UTC (rev 220) @@ -26,7 +26,7 @@ c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "chest")))) c.publish! puts "published #{c.class} with id #{c.service_id}" -d = Archipelago::Dump::Site.new(:debug_callable => Proc.new do |msg| puts msg end, +d = Archipelago::Dump::Site.new(:debug_callable => Proc.new do |msg| puts "#{$$}:#{Thread.current}:#{Time.new.to_f}: #{msg}" end, :persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "dump")))) d.publish! puts "published #{d.class} with id #{d.service_id}" From nobody at rubyforge.org Sun Mar 4 17:49:23 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Sun, 4 Mar 2007 17:49:23 -0500 (EST) Subject: [Archipelago-submits] [221] trunk/oneliner/lib/oneliner/superstring.rb: better error message Message-ID: <20070304224923.3F37252420F6@rubyforge.org> Revision: 221 Author: zond Date: 2007-03-04 17:49:22 -0500 (Sun, 04 Mar 2007) Log Message: ----------- better error message Modified Paths: -------------- trunk/oneliner/lib/oneliner/superstring.rb Modified: trunk/oneliner/lib/oneliner/superstring.rb =================================================================== --- trunk/oneliner/lib/oneliner/superstring.rb 2007-03-04 18:49:53 UTC (rev 220) +++ trunk/oneliner/lib/oneliner/superstring.rb 2007-03-04 22:49:22 UTC (rev 221) @@ -65,7 +65,7 @@ # Returns whether decoding is done. # def decode!(chunk) - raise "chunk is too small for metadata (8 bytes)" unless chunk.size > 7 + raise "#{chunk.inspect} is too small for metadata (8 bytes)" unless chunk.size > 7 @decode_done = nil From nobody at rubyforge.org Sun Mar 4 18:32:37 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Sun, 4 Mar 2007 18:32:37 -0500 (EST) Subject: [Archipelago-submits] [222] trunk/archipelago: commented the big sanitation test. Message-ID: <20070304233237.A08DC52420CB@rubyforge.org> Revision: 222 Author: zond Date: 2007-03-04 18:32:37 -0500 (Sun, 04 Mar 2007) Log Message: ----------- commented the big sanitation test. added recovery and maintenance tests to the big sanitation test. fixed a small irregularity in publishable. made dump not do the redistribute within the each loop. renamed a few methods in dump. made sanitation#next_to? work even if one id doesnt exist anymore. Modified Paths: -------------- trunk/archipelago/lib/archipelago/disco.rb trunk/archipelago/lib/archipelago/dump.rb trunk/archipelago/lib/archipelago/sanitation.rb trunk/archipelago/tests/sanitation_test.rb Modified: trunk/archipelago/lib/archipelago/disco.rb =================================================================== --- trunk/archipelago/lib/archipelago/disco.rb 2007-03-04 22:49:22 UTC (rev 221) +++ trunk/archipelago/lib/archipelago/disco.rb 2007-03-04 23:32:37 UTC (rev 222) @@ -352,7 +352,7 @@ attr_reader :hash include Archipelago::Current::Synchronized include Archipelago::Current::ThreadedCollection - def_delegators :@hash, :[], :size, :each, :empty?, :delete, :values, :keys, :include? + def_delegators :@hash, :[], :size, :each, :empty?, :values, :keys, :include? def initialize(options = {}) super @hash = options[:hash] || {} @@ -371,8 +371,9 @@ # Delete +key+. # def delete(key) - value = @hash.delete(key) + value = @hash[key] @jockey.instance_eval do notify_subscribers(:lost, value) end if @jockey && value + @hash.delete(key) end # # Merge this locker with another. Modified: trunk/archipelago/lib/archipelago/dump.rb =================================================================== --- trunk/archipelago/lib/archipelago/dump.rb 2007-03-04 22:49:22 UTC (rev 221) +++ trunk/archipelago/lib/archipelago/dump.rb 2007-03-04 23:32:37 UTC (rev 222) @@ -160,33 +160,21 @@ end # - # Checks whether +key+ should reside here. - # - # If not, will ask our Archipelago::Sanitation::Officer to + # Will ask our Archipelago::Sanitation::Officer to # redistribute it. # # If that succeeded will delete it from our database. # - # Will return whether it belongs here. - # - def check_key(key) - if belongs_here?(key) - @debug_callable.call("#{service_id}.check_key(#{key}) returns true") if @debug_callable - return true - else - begin - @debug_callable.call("#{service_id}.check_key(#{key}) redistributes, deletes and returns false") if @debug_callable - @officer.redistribute(key) - @db.delete(key) - rescue Archipelago::Sanitation::NotEnoughDataException => e - @debug_callable.call("#{service_id}.check_key(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable - # What shall we do in this case? - # * Nothing, and wait for the admins to make a manual check? - # * Send an email someplace? - # * Store it somewhere so that the manual check goes faster? - ensure - return false - end + def redistribute_and_delete(key) + begin + @officer.redistribute(key) + @db.delete(key) + rescue Archipelago::Sanitation::NotEnoughDataException => e + @debug_callable.call("#{service_id}.redistribute_and_delete(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable + # What shall we do in this case? + # * Nothing, and wait for the admins to make a manual check? + # * Send an email someplace? + # * Store it somewhere so that the manual check goes faster? end end @@ -206,16 +194,22 @@ loop do begin @debug_callable.call("#{service_id}.start_edge_check doing its thang") if @debug_callable + keys_to_check = Set.new @db.reverse_each_key do |key| - break if check_key(key) + break if belongs_here?(key) + keys_to_check << key end @db.each_key do |key| - break if check_key(key) + break if belongs_here?(key) + keys_to_check << key end + keys_to_check.each do |key| + redistribute_and_delete(key) + end + sleep(@check_interval) rescue Exception => e @debug_callable.call("#{service_id}.start_edge_check() got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable end - sleep(@check_interval) end end end @@ -239,12 +233,11 @@ # # Tell the officer to redistribute +key+ and ignore any errors. # - def redistribute_key(key) + def redistribute(key) begin - @debug_callable.call("#{service_id}.redistribute_key(#{key}) called") if @debug_callable @officer.redistribute(key) rescue Archipelago::Sanitation::NotEnoughDataException => e - @debug_callable.call("#{service_id}.redistribute_key(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable + @debug_callable.call("#{service_id}.redistribute(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable # What shall we do in this case? # * Nothing, and wait for the admins to make a manual check? # * Send an email someplace? @@ -266,7 +259,7 @@ # def lost_peer(record) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) called") if @debug_callable - keys_to_redistribute = [] + keys_to_redistribute = Set.new if right_before?(record[:service_id]) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) is right_before, redistributing from #{@officer.second_master_to(service_id)}") if @debug_callable @db.reverse_each_key do |key| @@ -282,7 +275,7 @@ end end keys_to_redistribute.each do |key| - redistribute_key(key) + redistribute(key) end end Modified: trunk/archipelago/lib/archipelago/sanitation.rb =================================================================== --- trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-04 22:49:22 UTC (rev 221) +++ trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-04 23:32:37 UTC (rev 222) @@ -187,11 +187,17 @@ end # - # Returns whether +service_id1+ and +service_id2+ - # are in that order in the array. + # Returns whether +service_id1+ and +service_id2+ would come in that + # order in the site array if both existed. # def next_to?(service_id1, service_id2) - return @sites.include?(service_id1) && get_least_greater_than(@sites, service_id1, 1).first == service_id2 + if @sites.include?(service_id1) + return get_least_greater_than(@sites, service_id1, 1).first <= service_id2 + elsif @sites.include?(service_id2) + return get_greatest_less_than(@sites, service_id2, 1).first >= service_id1 + else + return false + end end # Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-04 22:49:22 UTC (rev 221) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-04 23:32:37 UTC (rev 222) @@ -34,6 +34,9 @@ end def test_navigation + # + # Setup the network + # @d.stop! dumps = [] cleaner2 = Archipelago::Sanitation::Officer.new(:minimum_nr_of_chunks => 3) @@ -44,10 +47,16 @@ :persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("master_test_#{n}.db"))) dumps[n].publish! end + # + # Wait until it is alive + # assert_within(30) do cleaner2.update_services! dumps.collect do |d| d.service_id end.sort == cleaner2.sites.keys.sort end + # + # Perform checks on all dumps + # 10.times do |n| if n < 9 assert(dumps[n].instance_eval do right_before?(dumps[n+1].service_id) end, @@ -61,6 +70,9 @@ "#{dumps[0].service_id} is supposed to be right_after? #{dumps[n].service_id}, but isnt. services are #{cleaner2.sites.keys.inspect}") end end + # + # Check that only fresh data is returned + # s1 = Oneliner::SuperString.new("brappa") s2 = Oneliner::SuperString.new("brappa2") assert_equal(0, dumps[0].db.size) @@ -69,7 +81,10 @@ dumps[1].insert!("a", [s2.encode(200)], "aaab") assert(cleaner2.responsible_sites("a").include?(dumps[0].service_id)) assert(cleaner2.responsible_sites("a").include?(dumps[1].service_id)) - assert_equal("brappa2", cleaner2["a"]) + assert_equal(s2.to_s, cleaner2["a"]) + # + # Check the utility methods of sanitation + # assert_equal([dumps[9].service_id, dumps[0].service_id, dumps[1].service_id].sort, cleaner2.instance_eval do get_greatest_less_than(@sites, dumps[2].service_id, 3) @@ -82,6 +97,55 @@ cleaner2.second_master_to(dumps[4].service_id)) assert_equal(dumps[9].service_id, cleaner2.second_master_to(dumps[0].service_id)) + # + # Check the recovery methods + # + assert_equal({ + "6"=>0, + "7"=>0, + "8"=>0, + "9"=>0, + "0"=>1, + "1"=>1, + "2"=>0, + "3"=>0, + "4"=>0, + "5"=>0 + }, + chunks_by_id(cleaner2, "a")) + cleaner2.redistribute("a") + healthy_chunks = { + "6"=>0, + "7"=>0, + "8"=>0, + "9"=>0, + "0"=>1, + "1"=>1, + "2"=>1, + "3"=>0, + "4"=>0, + "5"=>0 + } + assert_equal(healthy_chunks, + chunks_by_id(cleaner2, "a")) + dumps[1].instance_eval do @valid = false end + cleaner2.update_services! + dumps[0].instance_eval do lost_peer({:service_id => dumps[1].service_id}) end + dumps[2].instance_eval do lost_peer({:service_id => dumps[1].service_id}) end + assert_equal({"6"=>0, "7"=>0, "8"=>0, "9"=>0, "0"=>1, "2"=>1, "3"=>1, "4"=>0, "5"=>0}, + chunks_by_id(cleaner2, "a")) + + dumps[1].instance_eval do @valid = true end + assert_within(5) do + cleaner2.update_services! + ["0","1","2","3","4","5","6","7","8","9"].sort == cleaner2.sites.keys.sort + end + assert_equal({"0"=>1, "1"=>1, "2"=>1}, + cleaner2.responsible_sites("a")) + dumps[3].instance_eval do @edge_check_thread.wakeup end + assert_within(5) do + chunks_by_id(cleaner2, "a") == healthy_chunks + end ensure dumps.extend(Archipelago::Current::ThreadedCollection) dumps.t_each do |dump| @@ -98,4 +162,12 @@ assert_equal(@c["hej"], nil) end + private + + def chunks_by_id(officer, key) + officer.sites.values.inject({}) do |result, service_desc| + result.merge({service_desc[:service_id] => service_desc[:service].fetch(key).size}) + end + end + end From nobody at rubyforge.org Mon Mar 5 04:08:53 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Mon, 5 Mar 2007 04:08:53 -0500 (EST) Subject: [Archipelago-submits] [223] trunk/archipelago: gave sanitation_test more generous limits. Message-ID: <20070305090854.CAAC05240C45@rubyforge.org> Revision: 223 Author: zond Date: 2007-03-05 04:08:51 -0500 (Mon, 05 Mar 2007) Log Message: ----------- gave sanitation_test more generous limits. readded a few debug messages to dump. made dump more thrifty when recovering lost peers. made hashish actually return a TREE not a HASH when asked for one :O. Modified Paths: -------------- trunk/archipelago/lib/archipelago/dump.rb trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/sanitation.rb trunk/archipelago/tests/sanitation_test.rb Modified: trunk/archipelago/lib/archipelago/dump.rb =================================================================== --- trunk/archipelago/lib/archipelago/dump.rb 2007-03-04 23:32:37 UTC (rev 222) +++ trunk/archipelago/lib/archipelago/dump.rb 2007-03-05 09:08:51 UTC (rev 223) @@ -167,8 +167,10 @@ # def redistribute_and_delete(key) begin + @debug_callable.call("#{service_id}.redistribute_and_delete(#{key}) gonna redist and delete") if @debug_callable @officer.redistribute(key) @db.delete(key) + @debug_callable.call("#{service_id}.redistribute_and_delete(#{key}) done redistributing") if @debug_callable rescue Archipelago::Sanitation::NotEnoughDataException => e @debug_callable.call("#{service_id}.redistribute_and_delete(#{key}) got #{e}: #{PP.pp(e.backtrace, "")}") if @debug_callable # What shall we do in this case? @@ -262,14 +264,13 @@ keys_to_redistribute = Set.new if right_before?(record[:service_id]) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) is right_before, redistributing from #{@officer.second_master_to(service_id)}") if @debug_callable - @db.reverse_each_key do |key| + @db.each_key(@officer.second_master_to(service_id)) do |key| keys_to_redistribute << key - break if key < @officer.second_master_to(service_id) end end if right_after?(record[:service_id]) @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) is right_after, redistributing up to #{record[:service_id]}") if @debug_callable - @db.each_key do |key| + @db.each_key(@officer.predecessor(record[:service_id])) do |key| keys_to_redistribute << key break if key > record[:service_id] end Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-03-04 23:32:37 UTC (rev 222) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-05 09:08:51 UTC (rev 223) @@ -271,11 +271,11 @@ return hashish end # - # Returns something acting like a Berkeley Hash DB instance allowing duplicate entries + # Returns something acting like a Berkeley Btree DB instance allowing duplicate entries # and transactions using +name+. # def get_dup_tree(name, flags = BDB::CREATE | BDB::NOMMAP) - db = BDB::Hash.open(Pathname.new(File.join(@env.home, name)).expand_path, + db = BDB::Btree.open(Pathname.new(File.join(@env.home, name)).expand_path, nil, flags, 0, Modified: trunk/archipelago/lib/archipelago/sanitation.rb =================================================================== --- trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-04 23:32:37 UTC (rev 222) +++ trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-05 09:08:51 UTC (rev 223) @@ -208,6 +208,14 @@ return get_greatest_less_than(@sites, service_id, @minimum_nr_of_chunks - 1).first end + # + # Gets the predecessor of +service_id+ in the array of services. + # + def predecessor(service_id) + update_services! + return get_greatest_less_than(@sites, service_id, 1).first + end + private # Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-04 23:32:37 UTC (rev 222) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-05 09:08:51 UTC (rev 223) @@ -136,14 +136,16 @@ chunks_by_id(cleaner2, "a")) dumps[1].instance_eval do @valid = true end - assert_within(5) do + assert_within(10) do cleaner2.update_services! ["0","1","2","3","4","5","6","7","8","9"].sort == cleaner2.sites.keys.sort end assert_equal({"0"=>1, "1"=>1, "2"=>1}, cleaner2.responsible_sites("a")) +# dumps[3].instance_eval do @debug_callable = Proc.new do |m| puts m end end dumps[3].instance_eval do @edge_check_thread.wakeup end - assert_within(5) do + assert_within(10) do +# pp chunks_by_id(cleaner2, "a") chunks_by_id(cleaner2, "a") == healthy_chunks end ensure From nobody at rubyforge.org Tue Mar 6 08:28:17 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Tue, 6 Mar 2007 08:28:17 -0500 (EST) Subject: [Archipelago-submits] [224] trunk/archipelago: moved from configurable persistence provider to configurable persistence directory. Message-ID: <20070306132818.1F5A952409A3@rubyforge.org> Revision: 224 Author: zond Date: 2007-03-06 08:28:17 -0500 (Tue, 06 Mar 2007) Log Message: ----------- moved from configurable persistence provider to configurable persistence directory. renamed console to console.rb Modified Paths: -------------- trunk/archipelago/lib/archipelago/disco.rb trunk/archipelago/lib/archipelago/dump.rb trunk/archipelago/script/services.rb trunk/archipelago/tests/dump_test.rb trunk/archipelago/tests/pirate_test.rb trunk/archipelago/tests/sanitation_test.rb trunk/archipelago/tests/treasure_benchmark.rb trunk/archipelago/tests/treasure_test.rb Added Paths: ----------- trunk/archipelago/script/console.rb Removed Paths: ------------- trunk/archipelago/script/console Modified: trunk/archipelago/lib/archipelago/disco.rb =================================================================== --- trunk/archipelago/lib/archipelago/disco.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/lib/archipelago/disco.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -161,7 +161,7 @@ # # The provider of happy magic persistent hashes of different kinds. # - @persistence_provider ||= options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join(self.class.name + ".db")) + @persistence_provider ||= Archipelago::Hashish::BerkeleyHashishProvider.new(options[:persistence_directory] || Pathname.new(File.expand_path(__FILE__)).parent.join(self.class.name + ".db")) # # Stuff that didnt fit in any of the other databases. # Modified: trunk/archipelago/lib/archipelago/dump.rb =================================================================== --- trunk/archipelago/lib/archipelago/dump.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/lib/archipelago/dump.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -49,12 +49,8 @@ attr_accessor :db, :debug_callable, :officer, :persistence_provider def initialize(options = {}) + initialize_publishable(options) # - # The provider of happy magic persistent hashes of different kinds. - # - @persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("cove_tanker.db")) - - # # The callable object that will get sent our debug messages if it exists. # @debug_callable = options[:debug_callable] @@ -73,12 +69,6 @@ # The minimum pause between checking if keys belong with us. # @check_interval = options[:check_interval] || CHECK_INTERVAL - - # - # Use the given options to initialize the publishable - # instance variables. - # - initialize_publishable(options) end # Deleted: trunk/archipelago/script/console =================================================================== --- trunk/archipelago/script/console 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/script/console 2007-03-06 13:28:17 UTC (rev 224) @@ -1,3 +0,0 @@ -#!/usr/bin/env ruby - -exec 'irb', '-r', File.join(File.dirname(__FILE__), 'pirate.rb'), '-r', File.join(File.dirname(__FILE__), 'officer.rb'), "-I", 'lib' Copied: trunk/archipelago/script/console.rb (from rev 219, trunk/archipelago/script/console) =================================================================== --- trunk/archipelago/script/console.rb (rev 0) +++ trunk/archipelago/script/console.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby + +exec 'irb', '-r', File.join(File.dirname(__FILE__), 'pirate.rb'), '-r', File.join(File.dirname(__FILE__), 'officer.rb'), "-I", 'lib' Modified: trunk/archipelago/script/services.rb =================================================================== --- trunk/archipelago/script/services.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/script/services.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -20,14 +20,14 @@ DRb.start_service end -t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tranny")))) +t = Archipelago::Tranny::Manager.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "tranny"))) t.publish! puts "published #{t.class} with id #{t.service_id}" -c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "chest")))) +c = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "chest"))) c.publish! puts "published #{c.class} with id #{c.service_id}" d = Archipelago::Dump::Site.new(:debug_callable => Proc.new do |msg| puts "#{$$}:#{Thread.current}:#{Time.new.to_f}: #{msg}" end, - :persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "dump")))) + :persistence_directory => Pathname.new(File.join(ARGV[0], "dump"))) d.publish! puts "published #{d.class} with id #{d.service_id}" Modified: trunk/archipelago/tests/dump_test.rb =================================================================== --- trunk/archipelago/tests/dump_test.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/tests/dump_test.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -5,7 +5,7 @@ def setup DRb.start_service - @d = Archipelago::Dump::Site.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("site.db"))) + @d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site.db")) end def teardown Modified: trunk/archipelago/tests/pirate_test.rb =================================================================== --- trunk/archipelago/tests/pirate_test.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/tests/pirate_test.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -7,11 +7,11 @@ DRb.start_service("druby://localhost:#{rand(1000) + 5000}") @p = Archipelago::Pirate::Captain.new(:chest_description => {:class => "TestChest"}, :tranny_description => {:class => "TestManager"}) - @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db"))) + @c = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest.db")) @c.publish! - @c2 = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest2.db"))) + @c2 = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest2.db")) @c2.publish! - @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny.db"))) + @tm = TestManager.new(:persistence_directory => Pathname.new(__FILE__).parent.join("tranny.db")) @tm.publish! assert_within(10) do @p.update_services! Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -9,7 +9,7 @@ def setup DRb.start_service - @d = Archipelago::Dump::Site.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("site.db"))) + @d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site.db")) @d.publish! @c = Archipelago::Sanitation::Officer.new assert_within(10) do @@ -44,7 +44,7 @@ 10.times do |n| dumps[n] = Archipelago::Dump::Site.new(:officer => cleaner2, :service_description => {:service_id => n.to_s}, - :persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("master_test_#{n}.db"))) + :persistence_directory => Pathname.new(__FILE__).parent.join("master_test_#{n}.db")) dumps[n].publish! end # Modified: trunk/archipelago/tests/treasure_benchmark.rb =================================================================== --- trunk/archipelago/tests/treasure_benchmark.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/tests/treasure_benchmark.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -5,7 +5,7 @@ def setup DRb.start_service - @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db"))) + @c = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest.db")) end def teardown Modified: trunk/archipelago/tests/treasure_test.rb =================================================================== --- trunk/archipelago/tests/treasure_test.rb 2007-03-05 09:08:51 UTC (rev 223) +++ trunk/archipelago/tests/treasure_test.rb 2007-03-06 13:28:17 UTC (rev 224) @@ -24,9 +24,9 @@ def setup DRb.start_service - @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db"))) - @c2 = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest2.db"))) - @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny1.db"))) + @c = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest.db")) + @c2 = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest2.db")) + @tm = TestManager.new(:persistence_directory => Pathname.new(__FILE__).parent.join("tranny1.db")) $BURKMAT = 0 $BURKMAT2 = 0 $BURKMAT3 = 0 @@ -87,7 +87,7 @@ def test_around_load @c["brunis"] = A.new("hirr") @c.persistence_provider.close! - @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db"))) + @c = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest.db")) a = @c["brunis"] assert_equal(1, $BURKMAT3) assert_equal(1, $BURKMAT4) From nobody at rubyforge.org Tue Mar 6 09:17:40 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Tue, 6 Mar 2007 09:17:40 -0500 (EST) Subject: [Archipelago-submits] [225] trunk/archipelago: created a sanitation_benchmark. Message-ID: <20070306141740.7D3A352409A0@rubyforge.org> Revision: 225 Author: zond Date: 2007-03-06 09:17:40 -0500 (Tue, 06 Mar 2007) Log Message: ----------- created a sanitation_benchmark. improved the verbosity of the sanitation_test. prettified disco.rb. improved comments in treasure.rb and tranny.rb Modified Paths: -------------- trunk/archipelago/lib/archipelago/disco.rb trunk/archipelago/lib/archipelago/tranny.rb trunk/archipelago/lib/archipelago/treasure.rb trunk/archipelago/tests/sanitation_test.rb Added Paths: ----------- trunk/archipelago/tests/sanitation_benchmark.rb Modified: trunk/archipelago/lib/archipelago/disco.rb =================================================================== --- trunk/archipelago/lib/archipelago/disco.rb 2007-03-06 13:28:17 UTC (rev 224) +++ trunk/archipelago/lib/archipelago/disco.rb 2007-03-06 14:17:40 UTC (rev 225) @@ -90,7 +90,7 @@ @jockey.stop! if defined?(@jockey) && (!defined?(Archipelago::Disco::MC) || @jockey != Archipelago::Disco::MC) @jockey_options ||= {} - jockey_options = (@jockey_options || {}).merge(options[:jockey_options] || {}) + jockey_options = @jockey_options.merge(options[:jockey_options] || {}) if options[:jockey] @jockey = options[:jockey] @@ -98,14 +98,14 @@ @jockey.setup(jockey_options) end else - unless jockey_options.empty? - @jockey = Archipelago::Disco::Jockey.new(jockey_options) - else + if jockey_options.empty? if defined?(Archipelago::Disco::MC) @jockey = Archipelago::Disco::MC else @jockey = Archipelago::Disco::Jockey.new end + else + @jockey = Archipelago::Disco::Jockey.new(jockey_options) end end end Modified: trunk/archipelago/lib/archipelago/tranny.rb =================================================================== --- trunk/archipelago/lib/archipelago/tranny.rb 2007-03-06 13:28:17 UTC (rev 224) +++ trunk/archipelago/lib/archipelago/tranny.rb 2007-03-06 14:17:40 UTC (rev 225) @@ -99,7 +99,7 @@ # # Will use a BerkeleyHashishProvider using tranny_manager.db in the same dir to get its hashes - # if not :persistence_provider is given. + # if not :persistence_directory is given. # # Will create Transactions timing out after :transaction_timeout seconds or TRANSACTION_TIMEOUT # if none is given. Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 13:28:17 UTC (rev 224) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 14:17:40 UTC (rev 225) @@ -245,7 +245,7 @@ # Initialize a Chest # # Will use a BerkeleyHashishProvider using treasure_chest.db in the same dir to get its hashes - # if not :persistence_provider is given. + # if not :persistence_directory is given. # # Will try to recover crashed transaction every :transaction_recovery_interval seconds # or TRANSACTION_RECOVERY_INTERVAL if none is given. Added: trunk/archipelago/tests/sanitation_benchmark.rb =================================================================== --- trunk/archipelago/tests/sanitation_benchmark.rb (rev 0) +++ trunk/archipelago/tests/sanitation_benchmark.rb 2007-03-06 14:17:40 UTC (rev 225) @@ -0,0 +1,43 @@ + +require File.join(File.dirname(__FILE__), 'test_helper') + +class SanitationBenchmark < Test::Unit::TestCase + + def setup + DRb.start_service + @d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site.db")) + @d.publish! + @c = Archipelago::Sanitation::Officer.new + assert_within(10) do + @c.update_services! + s = @c.instance_eval do @sites end + s.keys == [@d.service_id] + end + end + + def teardown + @d.stop! + @d.instance_eval do @persistence_provider.unlink! end + DRb.stop_service + end + + def test_set_get + k = "apa" + v = "gnu" + bm("Sanitation#[]=", :n => 100) do + @c[k] = v + k = k.next + end + k = "apa" + bm("Sanitation#[]", :n => 100) do + t = @c[k] + k = k.next + end + k = "apa" + bm("Sanitation#delete", :n => 100) do + t = @c.delete!(k) + k = k.next + end + end + +end Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-06 13:28:17 UTC (rev 224) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-06 14:17:40 UTC (rev 225) @@ -3,10 +3,6 @@ class SanitationTest < Test::Unit::TestCase - def test_truth - assert(true) - end - def setup DRb.start_service @d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site.db")) @@ -144,9 +140,13 @@ cleaner2.responsible_sites("a")) # dumps[3].instance_eval do @debug_callable = Proc.new do |m| puts m end end dumps[3].instance_eval do @edge_check_thread.wakeup end - assert_within(10) do -# pp chunks_by_id(cleaner2, "a") - chunks_by_id(cleaner2, "a") == healthy_chunks + begin + assert_within(20) do + chunks_by_id(cleaner2, "a") == healthy_chunks + end + rescue Test::Unit::AssertionFailedError => e + pp chunks_by_id(cleaner2, "a") + raise e end ensure dumps.extend(Archipelago::Current::ThreadedCollection) From nobody at rubyforge.org Tue Mar 6 10:49:26 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Tue, 6 Mar 2007 10:49:26 -0500 (EST) Subject: [Archipelago-submits] [226] trunk/archipelago: fixed a borkination in sanitation_test. Message-ID: <20070306154926.F01A05241D74@rubyforge.org> Revision: 226 Author: zond Date: 2007-03-06 10:49:26 -0500 (Tue, 06 Mar 2007) Log Message: ----------- fixed a borkination in sanitation_test. broke BerkeleyHashish into CachedHashish and BerkeleyHashish. added some goddamn lying comments (so far) into treasure.rb Modified Paths: -------------- trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/tranny.rb trunk/archipelago/lib/archipelago/treasure.rb trunk/archipelago/tests/sanitation_test.rb Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-03-06 14:17:40 UTC (rev 225) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-06 15:49:26 UTC (rev 226) @@ -25,49 +25,62 @@ # module Hashish - # - # In essence a Berkeley Database backed Hash. - # - # Will cache all values having been written or read - # in a normal Hash cache for fast access. - # - # Will save the last update timestamp for all keys - # in a separate Hash cache AND a separate Berkeley Database. - # - class BerkeleyHashish - include Archipelago::Current::Synchronized + module CachedHashish # - # Initialize an instance with the +name+ and BDB::Env +env+. + # Returns true if what +key+ points to is no longer +value+. # - def initialize(name, env) - super() - @content_db = env.open_db(BDB::HASH, name, "content", BDB::CREATE) - @content = {} - @timestamps_db = env.open_db(BDB::HASH, name, "timestamps", BDB::CREATE | BDB::NOMMAP) - @timestamps = {} - @lock = Archipelago::Current::Lock.new + def changed?(key, value) + raise "You have to implement me!" end # - # Close the @content_db and @timestamps_db behind this BerkeleyHashish + # Get the serialized timestamp for +serialized_key+ from the persistent backer. # + def do_get_timestamp_from_db(serialized_key) + raise "You have to implement me!" + end + # + # Actually writes +key+ serialized as +serialized_key+ an + # +serialized_value+ to the db. Used by write_to_db. + # + def do_write_to_db(key, serialized_key, serialized_value) + raise "You have to implement me!" + end + # + # Close the persistent backers behind this CachedHashish. + # def close! - @content_db.close - @timestamps_db.close + raise "You have to implement me!" end # - # Returns a deep ( Marshal.load(Marshal.dump(o)) ) clone - # of the object represented by +key+. + # Returns whether the persistent backer contains +key+. # - def get_deep_clone(key) - return Marshal.load(@content_db[Marshal.dump(key)]) + def db_include? + raise "You have to implement me!" end # - # Returns true if this BerkeleyHashish include +key+. + # Will perform an actual fetching of +serialized_key+ from the persistent backer. # - def include?(key) - @content.include?(key) || !@content_db[Marshal.dump(key)].nil? + def do_get_from_db(serialized_key) + raise "You have to implement me!" end # + # Actually deletes +serialized_key+ from the persistent backer. + # + def do_delete_from_persistence(serialized_key) + raise "You have to implement me!" + end + + # + # Will do +callable+.call(key, value) for each + # key-and-value pair in this Hashish. + # + # NB: This is totaly thread-unsafe, only do this + # for management or rescue! + # + def each(callable) + raise "You have to implement me!" + end + # # Simply get the value for the +key+. # # Will call value.load_hook and send it @@ -129,17 +142,70 @@ else - serialized_value = Marshal.dump(value) - serialized_key = Marshal.dump(key) - old_serialized_value = @content_db[serialized_key] + write_to_db(key, serialized_key, serialized_value, value) if changed?(key, value) - write_to_db(key, serialized_key, serialized_value, value) if old_serialized_value && old_serialized_value != serialized_value - end end end # + # Returns a deep ( Marshal.load(Marshal.dump(o)) ) clone + # of the object represented by +key+. + # + def get_deep_clone(key) + return Marshal.load(Marshal.dump(@content[key])) + end + # + # Initializes the cache hashes for this CachedHashish. + # + def initialize_cached_hashish + @content = {} + @timestamps = {} + @lock = Archipelago::Current::Lock.new + end + # + # Delete +key+ and its value and timestamp. + # + def delete(key) + @lock.synchronize_on(key) do + + serialized_key = Marshal.dump(key) + + @content.delete(key) + @timestamps.delete(key) + do_delete_from_persistence(serialized_key) + end + end + # + # Read +key+ from db and if it is found + # put it in the cache Hash. + # + # Will call value.load_hook and send it + # a block that does the actuall insertion of the value + # into the live hash if value.respond_to?(:load_hook). + # + def get_from_db(key) + serialized_key = Marshal.dump(key) + serialized_value = do_get_from_db(serialized_key) + return nil unless serialized_value + + value = Marshal.load(serialized_value) + if value.respond_to?(:load_hook) + value.load_hook do + @content[key] = value + end + else + @content[key] = value + end + return value + end + # + # Returns true if this BerkeleyHashish include +key+. + # + def include?(key) + @content.include?(key) || db_include?(key) + end + # # Returns the last time the value under +key+ was changed. # def timestamp(key) @@ -149,7 +215,7 @@ return timestamp if timestamp serialized_key = Marshal.dump(key) - serialized_timestamp = @timestamps_db[serialized_key] + serialized_timestamp = do_get_timestamp_from_db(serialized_key) return nil unless serialized_timestamp timestamp = Marshal.load(serialized_timestamp) @@ -159,6 +225,55 @@ end end # + # Write +key+, serialized as +serialized_key+ and + # +serialized_value+ to the db. + # + # Will call value.save_hook(old_value) and send + # it a block that does the actual saving if + # value.respond_to?(:save_hook). + # + def write_to_db(key, serialized_key, serialized_value, value) + if value.respond_to?(:save_hook) + old_serialized_value = do_get_from_db(serialized_key) + old_value = old_serialized_value ? Marshal.load(old_serialized_value) : nil + value.save_hook(old_value) do + do_write_to_db(key, serialized_key, serialized_value) + end + else + do_write_to_db(key, serialized_key, serialized_value) + end + end + end + + # + # In essence a Berkeley Database backed Hash. + # + # Will cache all values having been written or read + # in a normal Hash cache for fast access. + # + # Will save the last update timestamp for all keys + # in a separate Hash cache AND a separate Berkeley Database. + # + class BerkeleyHashish + include Archipelago::Current::Synchronized + include Archipelago::Hashish::CachedHashish + # + # Initialize an instance with the +name+ and BDB::Env +env+. + # + def initialize(name, env) + super() + initialize_cached_hashish + @content_db = env.open_db(BDB::HASH, name, "content", BDB::CREATE) + @timestamps_db = env.open_db(BDB::HASH, name, "timestamps", BDB::CREATE | BDB::NOMMAP) + end + # + # Close the @content_db and @timestamps_db behind this BerkeleyHashish + # + def close! + @content_db.close + @timestamps_db.close + end + # # Will do +callable+.call(key, value) for each # key-and-value pair in this Hashish. # @@ -171,42 +286,32 @@ callable.call(key, self.[](key)) end end + + private + # - # Delete +key+ and its value and timestamp. + # Get the serialized timestamp for +serialized_key+ from the persistent backer. # - def delete(key) - @lock.synchronize_on(key) do + def do_get_timestamp_from_db(serialized_key) + return @timestamps_db[serialized_key] + end - serialized_key = Marshal.dump(key) - - @content.delete(key) + # + # Actually deletes +serialized_key+ from the persistent backer. + # + def do_delete_from_persistence(serialized_key) + @timestamps_db[serialized_key] = nil @content_db[serialized_key] = nil - @timestamps.delete(key) - @timestamps_db[serialized_key] = nil - - end end - - private - + # - # Write +key+, serialized as +serialized_key+ and - # +serialized_value+ to the db. + # Returns true if what +key+ points to is no longer +value+. # - # Will call value.save_hook(old_value) and send - # it a block that does the actual saving if - # value.respond_to?(:save_hook). - # - def write_to_db(key, serialized_key, serialized_value, value) - if value.respond_to?(:save_hook) - old_serialized_value = @content_db[serialized_key] - old_value = old_serialized_value ? Marshal.load(old_serialized_value) : nil - value.save_hook(old_value) do - do_write_to_db(key, serialized_key, serialized_value) - end - else - do_write_to_db(key, serialized_key, serialized_value) - end + def changed?(key, value) + serialized_value = Marshal.dump(value) + serialized_key = Marshal.dump(key) + old_serialized_value = @content_db[serialized_key] + return old_serialized_value && old_serialized_value != serialized_value end # @@ -220,30 +325,21 @@ @timestamps_db[serialized_key] = Marshal.dump(now) @timestamps[key] = now end + + # + # Returns whether the persistent backer contains +key+. + # + def db_include?(key) + return !@content_db[Marshal.dump(key)].nil? + end # - # Read +key+ from db and if it is found - # put it in the cache Hash. + # Will perform an actual fetching of +serialized_key+ from the persistent backer. # - # Will call value.load_hook and send it - # a block that does the actuall insertion of the value - # into the live hash if value.respond_to?(:load_hook). - # - def get_from_db(key) - serialized_key = Marshal.dump(key) - serialized_value = @content_db[serialized_key] - return nil unless serialized_value - - value = Marshal.load(serialized_value) - if value.respond_to?(:load_hook) - value.load_hook do - @content[key] = value - end - else - @content[key] = value - end - return value + def do_get_from_db(serialized_key) + return @content_db[serialized_key] end + end # @@ -265,10 +361,13 @@ # hash-like instance (see Archipelago::Hashish::BerkeleyHashish) # using +name+. # - def get_cached_hashish(name) - hashish = BerkeleyHashish.new(name, @env) - @berkeley_hashishes << hashish - return hashish + def get_cached_hashish(options) + if options[:officer] + else + hashish = BerkeleyHashish.new(options[:name], @env) + @berkeley_hashishes << hashish + return hashish + end end # # Returns something acting like a Berkeley Btree DB instance allowing duplicate entries Modified: trunk/archipelago/lib/archipelago/tranny.rb =================================================================== --- trunk/archipelago/lib/archipelago/tranny.rb 2007-03-06 14:17:40 UTC (rev 225) +++ trunk/archipelago/lib/archipelago/tranny.rb 2007-03-06 15:49:26 UTC (rev 226) @@ -111,7 +111,7 @@ @transaction_timeout = options[:transaction_timeout] || TRANSACTION_TIMEOUT - @db = @persistence_provider.get_cached_hashish("db") + @db = @persistence_provider.get_cached_hashish(:name => "db") end # Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 14:17:40 UTC (rev 225) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 15:49:26 UTC (rev 226) @@ -250,6 +250,9 @@ # Will try to recover crashed transaction every :transaction_recovery_interval seconds # or TRANSACTION_RECOVERY_INTERVAL if none is given. # + # Will store the actual data in a remote Archipelago::Dump network if :officer is given + # or in a local hash if not. + # # Will use Archipelago::Disco::Publishable by calling initialize_publishable with +options+. # def initialize(options = {}) @@ -285,7 +288,7 @@ # The magical persistent map that defines how we actually # store our data. # - @db = @persistence_provider.get_cached_hashish("db") + @db = @persistence_provider.get_cached_hashish(:name => "db", :officer => options[:officer] || nil) initialize_prepared(options[:transaction_recovery_interval] || TRANSACTION_RECOVERY_INTERVAL) Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-06 14:17:40 UTC (rev 225) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-06 15:49:26 UTC (rev 226) @@ -23,7 +23,7 @@ def test_missing_bits s1 = Oneliner::SuperString.new("brappa") - @d.insert!("a", s1.encode(9), "abab") + @d.insert!("a", [s1.encode(9)], "abab") assert_raise(Archipelago::Sanitation::NotEnoughDataException) do @c["a"] end From nobody at rubyforge.org Tue Mar 6 11:44:42 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Tue, 6 Mar 2007 11:44:42 -0500 (EST) Subject: [Archipelago-submits] [227] trunk/archipelago/lib/archipelago: created a trial Dump backed hashish. Message-ID: <20070306164442.B59605242072@rubyforge.org> Revision: 227 Author: zond Date: 2007-03-06 11:44:42 -0500 (Tue, 06 Mar 2007) Log Message: ----------- created a trial Dump backed hashish. made treasure smarter when selecting hashish Modified Paths: -------------- trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/treasure.rb Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-03-06 15:49:26 UTC (rev 226) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-06 16:44:42 UTC (rev 227) @@ -25,7 +25,15 @@ # module Hashish + # + # In essence a persistence backed Hash. + # + # Will cache everything including timestamps for last + # modification in normal Hashes, but keep everything stored + # in a persistency backer defined by the subclass. + # module CachedHashish + include Archipelago::Current::Synchronized # # Returns true if what +key+ points to is no longer +value+. # @@ -40,8 +48,11 @@ end # # Actually writes +key+ serialized as +serialized_key+ an - # +serialized_value+ to the db. Used by write_to_db. + # +serialized_value+ to the db, and a new timestamp to the + # timestamp db, and the same timestamp to the @timestamp attribute. # + # Used by write_to_db. + # def do_write_to_db(key, serialized_key, serialized_value) raise "You have to implement me!" end @@ -69,7 +80,6 @@ def do_delete_from_persistence(serialized_key) raise "You have to implement me!" end - # # Will do +callable+.call(key, value) for each # key-and-value pair in this Hashish. @@ -237,25 +247,68 @@ old_serialized_value = do_get_from_db(serialized_key) old_value = old_serialized_value ? Marshal.load(old_serialized_value) : nil value.save_hook(old_value) do - do_write_to_db(key, serialized_key, serialized_value) + now = Time.now + do_write_to_db(key, serialized_key, serialized_value, now) + @timestamps[key] = now end else - do_write_to_db(key, serialized_key, serialized_value) + now = Time.now + do_write_to_db(key, serialized_key, serialized_value, now) + @timestamps[key] = now end end end # - # In essence a Berkeley Database backed Hash. + # An Archipelago::Dump network backed CachedHashish. # - # Will cache all values having been written or read - # in a normal Hash cache for fast access. + class DumpHashish + include Archipelago::Hashish::CachedHashish + # + # Initialize a new DumpHashish with the given +officer+ to find Dumps. + # + def initialize(officer) + super() + initialize_cached_hashish + @officer = officer + @hash_by_key = {} + end + def changed?(key, value) + return (old_hash = @hash_by_key[Marshal.dump(key)]) && old_hash != Digest::SHA1.hexdigest(Marshal.dump(value)) + end + def do_get_timestamp_from_db(serialized_key) + return @officer[officer_key(serialized_key, "timestamp")] + end + def do_write_to_db(key, serialized_key, serialized_value, now) + @officer[officer_key(serialized_key, "content")] = serialized_value + @officer[officer_key(serialized_key, "timestamp")] = Marshal.dump(now) + @hash_by_key[serialized_key] = Digest::SHA1.hexdigest(serialized_value) + end + def close! + end + def officer_key(serialized_key, space) + Digest::SHA1.hexdigest("DumpHashish:#{space}:#{serialized_key}") + end + def db_include?(key) + return !@officer[officer_key(Marshal.dump(key), "content")].nil? + end + def do_get_from_db(serialized_key) + return @officer[officer_key(Marshal.dump(key), "content")] + end + def do_delete_from_persistence(serialized_key) + @officer.delete(officer_key(serialized_key, "content")) + @officer.delete(officer_key(serialized_key, "timestamp")) + @hash_by_key[serialized_key] = Digest::SHA1.hexdigest(serialized_value) + end + def each(callable) + raise "You have to implement me!" + end + end + # - # Will save the last update timestamp for all keys - # in a separate Hash cache AND a separate Berkeley Database. + # A CachedHashish backed by a few BDB::Hashes. # class BerkeleyHashish - include Archipelago::Current::Synchronized include Archipelago::Hashish::CachedHashish # # Initialize an instance with the +name+ and BDB::Env +env+. @@ -266,80 +319,39 @@ @content_db = env.open_db(BDB::HASH, name, "content", BDB::CREATE) @timestamps_db = env.open_db(BDB::HASH, name, "timestamps", BDB::CREATE | BDB::NOMMAP) end - # - # Close the @content_db and @timestamps_db behind this BerkeleyHashish - # def close! @content_db.close @timestamps_db.close end - # - # Will do +callable+.call(key, value) for each - # key-and-value pair in this Hashish. - # - # NB: This is totaly thread-unsafe, only do this - # for management or rescue! - # def each(callable) @content_db.each do |serialized_key, serialized_value| key = Marshal.load(serialized_key) callable.call(key, self.[](key)) end end - - private - - # - # Get the serialized timestamp for +serialized_key+ from the persistent backer. - # def do_get_timestamp_from_db(serialized_key) return @timestamps_db[serialized_key] end - - # - # Actually deletes +serialized_key+ from the persistent backer. - # def do_delete_from_persistence(serialized_key) - @timestamps_db[serialized_key] = nil - @content_db[serialized_key] = nil + @timestamps_db[serialized_key] = nil + @content_db[serialized_key] = nil end - - # - # Returns true if what +key+ points to is no longer +value+. - # def changed?(key, value) serialized_value = Marshal.dump(value) serialized_key = Marshal.dump(key) old_serialized_value = @content_db[serialized_key] return old_serialized_value && old_serialized_value != serialized_value end - - # - # Actually writes +key+ serialized as +serialized_key+ an - # +serialized_value+ to the db. Used by write_to_db. - # - def do_write_to_db(key, serialized_key, serialized_value) - now = Time.now - + def do_write_to_db(key, serialized_key, serialized_value, now) @content_db[serialized_key] = serialized_value @timestamps_db[serialized_key] = Marshal.dump(now) - @timestamps[key] = now end - - # - # Returns whether the persistent backer contains +key+. - # def db_include?(key) return !@content_db[Marshal.dump(key)].nil? end - - # - # Will perform an actual fetching of +serialized_key+ from the persistent backer. - # def do_get_from_db(serialized_key) return @content_db[serialized_key] end - end # @@ -356,23 +368,17 @@ @berkeley_hashishes = [] @bdb_dbs = [] end - # - # Returns a cleverly cached (but slightly inefficient) - # hash-like instance (see Archipelago::Hashish::BerkeleyHashish) - # using +name+. - # def get_cached_hashish(options) if options[:officer] - else + hashish = DumpHashish.new(options[:officer]) + @berkeley_hashishes << hashish + return hashish + elsif options[:name] hashish = BerkeleyHashish.new(options[:name], @env) @berkeley_hashishes << hashish return hashish end end - # - # Returns something acting like a Berkeley Btree DB instance allowing duplicate entries - # and transactions using +name+. - # def get_dup_tree(name, flags = BDB::CREATE | BDB::NOMMAP) db = BDB::Btree.open(Pathname.new(File.join(@env.home, name)).expand_path, nil, @@ -383,17 +389,11 @@ @bdb_dbs << db return db end - # - # Returns something acting like a Berkeley Hash DB instance using +name+. - # def get_hashish(name, flags = BDB::CREATE | BDB::NOMMAP) db = @env.open_db(BDB::HASH, name, nil, flags) @bdb_dbs << db return db end - # - # Closes databases opened by this instance. - # def close! @berkeley_hashishes.each do |h| h.close! @@ -402,9 +402,6 @@ d.close end end - # - # Closes databases opened by this instance and removes the persistent files. - # def unlink! close! home = Pathname.new(@env.home).expand_path Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 15:49:26 UTC (rev 226) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 16:44:42 UTC (rev 227) @@ -288,7 +288,11 @@ # The magical persistent map that defines how we actually # store our data. # - @db = @persistence_provider.get_cached_hashish(:name => "db", :officer => options[:officer] || nil) + if options[:officer] + @db = @persistence_provider.get_cached_hashish(:officer => options[:officer]) + else + @db = @persistence_provider.get_cached_hashish(:name => "db") + end initialize_prepared(options[:transaction_recovery_interval] || TRANSACTION_RECOVERY_INTERVAL) From nobody at rubyforge.org Wed Mar 7 05:18:09 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 05:18:09 -0500 (EST) Subject: [Archipelago-submits] [228] trunk/archipelago/lib/archipelago: made Captain#responsible_chest public. Message-ID: <20070307101809.2DDAB5241CCC@rubyforge.org> Revision: 228 Author: zond Date: 2007-03-07 05:18:06 -0500 (Wed, 07 Mar 2007) Log Message: ----------- made Captain#responsible_chest public. improved documentation in hashish.rb. made Dubloons more error resilient. Modified Paths: -------------- trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/pirate.rb trunk/archipelago/lib/archipelago/treasure.rb Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-03-06 16:44:42 UTC (rev 227) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-07 10:18:06 UTC (rev 228) @@ -48,12 +48,12 @@ end # # Actually writes +key+ serialized as +serialized_key+ an - # +serialized_value+ to the db, and a new timestamp to the - # timestamp db, and the same timestamp to the @timestamp attribute. + # +serialized_value+ to the db, and +timestamp+ to the + # timestamp db. # # Used by write_to_db. # - def do_write_to_db(key, serialized_key, serialized_value) + def do_write_to_db(key, serialized_key, serialized_value, timestamp) raise "You have to implement me!" end # @@ -293,7 +293,9 @@ return !@officer[officer_key(Marshal.dump(key), "content")].nil? end def do_get_from_db(serialized_key) - return @officer[officer_key(Marshal.dump(key), "content")] + serialized_value = @officer[officer_key(serialized_key, "content")] + @hash_by_key[serialized_key] = Digest::SHA1.hexdigest(serialized_value) + return serialized_value end def do_delete_from_persistence(serialized_key) @officer.delete(officer_key(serialized_key, "content")) Modified: trunk/archipelago/lib/archipelago/pirate.rb =================================================================== --- trunk/archipelago/lib/archipelago/pirate.rb 2007-03-06 16:44:42 UTC (rev 227) +++ trunk/archipelago/lib/archipelago/pirate.rb 2007-03-07 10:18:06 UTC (rev 228) @@ -256,8 +256,6 @@ @trannies = @jockey.lookup(Archipelago::Disco::Query.new(@tranny_description), 0) end - private - # # Get the chest responsible for +key+. # @@ -272,6 +270,8 @@ return @chests[sorted_chest_ids.first] end + private + # # Make sure all our known chests have evaluated # all files we need them to. Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-06 16:44:42 UTC (rev 227) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:18:06 UTC (rev 228) @@ -162,7 +162,7 @@ end # # Load some instance variables and replace @chest if we know that - # it actually is not correct. + # it is incorrect. # def self._load(s) key, chest, transaction, chest_id = Marshal.load(s) @@ -212,17 +212,31 @@ begin return @chest.call_instance_method(@key, meth, @transaction, *args, &block) rescue DRb::DRbConnError => e - if defined?(Archipelago::Disco::MC) - possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id})) - raise e if possible_replacements.empty? - - @chest = possible_replacements[@chest_id][:service] - CHEST_BY_SERVICE_ID[@chest_id] = @chest - - retry - else - raise e + # + # Here is some fancy rescuing being done. + # + # First: If we have an MC we will try to reconnect to our actual chest. + # + # Second: If that failed, we will try to reconnect to the chest that (according to the BLACKBEARD) + # is responsible for our key - if it actually HAS our key. + # + # If this fails, lets raise hell. + # + possible_replacements = [] + possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id})) if defined?(Archipelago::Disco::MC) + if possible_replacements.empty? && defined?(Archipelago::Pirate::BLACKBEARD) + responsible_chest = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key) + possible_replacements << responsible_chest if responsible_chest[:service].include?(@key, @transaction) end + raise e if possible_replacements.empty? + + # + # And for all the future dubloons that we unmarshal, lets remember this. + # + @chest = possible_replacements[@chest_id][:service] + CHEST_BY_SERVICE_ID[@chest_id] = @chest + + retry end end From nobody at rubyforge.org Wed Mar 7 05:18:47 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 05:18:47 -0500 (EST) Subject: [Archipelago-submits] [229] trunk/archipelago/lib/archipelago/treasure.rb: beautified code in treasure.rb Message-ID: <20070307101847.6DB745241DFB@rubyforge.org> Revision: 229 Author: zond Date: 2007-03-07 05:18:46 -0500 (Wed, 07 Mar 2007) Log Message: ----------- beautified code in treasure.rb Modified Paths: -------------- trunk/archipelago/lib/archipelago/treasure.rb Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:18:06 UTC (rev 228) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:18:46 UTC (rev 229) @@ -223,7 +223,9 @@ # If this fails, lets raise hell. # possible_replacements = [] - possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id})) if defined?(Archipelago::Disco::MC) + if defined?(Archipelago::Disco::MC) + possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id})) + end if possible_replacements.empty? && defined?(Archipelago::Pirate::BLACKBEARD) responsible_chest = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key) possible_replacements << responsible_chest if responsible_chest[:service].include?(@key, @transaction) From nobody at rubyforge.org Wed Mar 7 05:25:44 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 05:25:44 -0500 (EST) Subject: [Archipelago-submits] [230] trunk/archipelago/lib/archipelago/treasure.rb: fixed bugs and beautified the dubloon rescuing Message-ID: <20070307102545.128DF5242320@rubyforge.org> Revision: 230 Author: zond Date: 2007-03-07 05:25:44 -0500 (Wed, 07 Mar 2007) Log Message: ----------- fixed bugs and beautified the dubloon rescuing Modified Paths: -------------- trunk/archipelago/lib/archipelago/treasure.rb Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:18:46 UTC (rev 229) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:25:44 UTC (rev 230) @@ -222,21 +222,23 @@ # # If this fails, lets raise hell. # - possible_replacements = [] + new_chest_record = nil if defined?(Archipelago::Disco::MC) - possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id})) + new_chest_record = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({ + :service_id => @chest_id + }))[@chest_id] end - if possible_replacements.empty? && defined?(Archipelago::Pirate::BLACKBEARD) - responsible_chest = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key) - possible_replacements << responsible_chest if responsible_chest[:service].include?(@key, @transaction) + if new_chest_record.nil? && defined?(Archipelago::Pirate::BLACKBEARD) + new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key) + new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction) end - raise e if possible_replacements.empty? + raise e unless new_chest_record # # And for all the future dubloons that we unmarshal, lets remember this. # - @chest = possible_replacements[@chest_id][:service] - CHEST_BY_SERVICE_ID[@chest_id] = @chest + @chest = new_chest_record[:service] + CHEST_BY_SERVICE_ID[new_chest_record[:service_id]] = @chest retry end From nobody at rubyforge.org Wed Mar 7 06:46:27 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 06:46:27 -0500 (EST) Subject: [Archipelago-submits] [231] trunk/archipelago: added a test for a sanitation backed chest. Message-ID: <20070307114627.8261D524234E@rubyforge.org> Revision: 231 Author: zond Date: 2007-03-07 06:46:27 -0500 (Wed, 07 Mar 2007) Log Message: ----------- added a test for a sanitation backed chest. renamed test_navigation to test_multi. made services.rb back the chest with sanitation instead of with bdb. made publishable.stop! also close the @persistence_provider. removed the loops from disco again. removed a lot of over zealous update_services! and added them where they were actually needed. made hashish work properly by moving some serialization from out of the changed? method and not digesting nil in do_get_from_db. Modified Paths: -------------- trunk/archipelago/lib/archipelago/disco.rb trunk/archipelago/lib/archipelago/dump.rb trunk/archipelago/lib/archipelago/hashish.rb trunk/archipelago/lib/archipelago/sanitation.rb trunk/archipelago/lib/archipelago/treasure.rb trunk/archipelago/script/services.rb trunk/archipelago/tests/sanitation_test.rb Modified: trunk/archipelago/lib/archipelago/disco.rb =================================================================== --- trunk/archipelago/lib/archipelago/disco.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/lib/archipelago/disco.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -242,6 +242,7 @@ end end end + @persistence_provider.close! end end @@ -362,18 +363,20 @@ # Set +key+ to +value+. # def []=(key, value) - if @jockey && !@hash.include?(key) + existed_before = @hash.include?(key) + @hash[key] = value + # Notifying AFTER the fact to avoid loops. + if @jockey && !existed_before @jockey.instance_eval do notify_subscribers(:found, value) end end - @hash[key] = value end # # Delete +key+. # def delete(key) - value = @hash[key] + value = @hash.delete(key) + # Notifying AFTER the fact to avoid loops. @jockey.instance_eval do notify_subscribers(:lost, value) end if @jockey && value - @hash.delete(key) end # # Merge this locker with another. Modified: trunk/archipelago/lib/archipelago/dump.rb =================================================================== --- trunk/archipelago/lib/archipelago/dump.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/lib/archipelago/dump.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -195,6 +195,7 @@ break if belongs_here?(key) keys_to_check << key end + @officer.update_services! unless keys_to_check.empty? keys_to_check.each do |key| redistribute_and_delete(key) end @@ -250,6 +251,7 @@ # it was master to. # def lost_peer(record) + @officer.update_services! @debug_callable.call("#{service_id}.lost_peer(#{record[:service_id]}) called") if @debug_callable keys_to_redistribute = Set.new if right_before?(record[:service_id]) Modified: trunk/archipelago/lib/archipelago/hashish.rb =================================================================== --- trunk/archipelago/lib/archipelago/hashish.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/lib/archipelago/hashish.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -152,7 +152,9 @@ else - write_to_db(key, serialized_key, serialized_value, value) if changed?(key, value) + serialized_value = Marshal.dump(value) + serialized_key = Marshal.dump(key) + write_to_db(key, serialized_key, serialized_value, value) if changed?(serialized_key, serialized_value) end @@ -273,8 +275,8 @@ @officer = officer @hash_by_key = {} end - def changed?(key, value) - return (old_hash = @hash_by_key[Marshal.dump(key)]) && old_hash != Digest::SHA1.hexdigest(Marshal.dump(value)) + def changed?(serialized_key, serialized_value) + return (old_hash = @hash_by_key[serialized_key]) && old_hash != Digest::SHA1.hexdigest(serialized_value) end def do_get_timestamp_from_db(serialized_key) return @officer[officer_key(serialized_key, "timestamp")] @@ -294,7 +296,7 @@ end def do_get_from_db(serialized_key) serialized_value = @officer[officer_key(serialized_key, "content")] - @hash_by_key[serialized_key] = Digest::SHA1.hexdigest(serialized_value) + @hash_by_key[serialized_key] = Digest::SHA1.hexdigest(serialized_value) if serialized_value return serialized_value end def do_delete_from_persistence(serialized_key) @@ -338,9 +340,7 @@ @timestamps_db[serialized_key] = nil @content_db[serialized_key] = nil end - def changed?(key, value) - serialized_value = Marshal.dump(value) - serialized_key = Marshal.dump(key) + def changed?(serialized_key, serialized_value) old_serialized_value = @content_db[serialized_key] return old_serialized_value && old_serialized_value != serialized_value end Modified: trunk/archipelago/lib/archipelago/sanitation.rb =================================================================== --- trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/lib/archipelago/sanitation.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -110,7 +110,6 @@ chunk_size = (super_string.size / nr_of_needed_chunks) + @metadata_overhead chunk_size = @minimum_recoverable_size / nr_of_needed_chunks if chunk_size < @minimum_recoverable_size / nr_of_needed_chunks - update_services! dump_hash = responsible_sites(key) super_string.encode(8) dump_hash.t_each do |dump_id, nr_of_chunks_needed| @@ -123,7 +122,6 @@ end def delete!(key) - update_services! dump_hash = responsible_sites(key) dump_hash.t_each do |dump_id, nr_of_chunks_available| @sites[dump_id][:service].delete!(key) @@ -212,7 +210,6 @@ # Gets the predecessor of +service_id+ in the array of services. # def predecessor(service_id) - update_services! return get_greatest_less_than(@sites, service_id, 1).first end @@ -222,7 +219,6 @@ # Returns [the value for +key+, the timestamp for the value]. # def fetch(key) - update_services! dump_hash = responsible_sites(key) dump_ids = dump_hash.keys newest_timestamp = "\000\000\000\000" Modified: trunk/archipelago/lib/archipelago/treasure.rb =================================================================== --- trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/lib/archipelago/treasure.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -229,6 +229,7 @@ }))[@chest_id] end if new_chest_record.nil? && defined?(Archipelago::Pirate::BLACKBEARD) + Archipelago::Pirate::BLACKBEARD.update_services! new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key) new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction) end Modified: trunk/archipelago/script/services.rb =================================================================== --- trunk/archipelago/script/services.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/script/services.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -19,16 +19,19 @@ else DRb.start_service end +puts "using URI #{DRb.uri}" t = Archipelago::Tranny::Manager.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "tranny"))) t.publish! puts "published #{t.class} with id #{t.service_id}" -c = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "chest"))) -c.publish! -puts "published #{c.class} with id #{c.service_id}" -d = Archipelago::Dump::Site.new(:debug_callable => Proc.new do |msg| puts "#{$$}:#{Thread.current}:#{Time.new.to_f}: #{msg}" end, - :persistence_directory => Pathname.new(File.join(ARGV[0], "dump"))) + +d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "dump"))) d.publish! puts "published #{d.class} with id #{d.service_id}" +c = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "chest")), + :officer => Archipelago::Sanitation::CLEANER) +c.publish! +puts "published #{c.class} with id #{c.service_id}" + DRb.thread.join Modified: trunk/archipelago/tests/sanitation_test.rb =================================================================== --- trunk/archipelago/tests/sanitation_test.rb 2007-03-07 10:25:44 UTC (rev 230) +++ trunk/archipelago/tests/sanitation_test.rb 2007-03-07 11:46:27 UTC (rev 231) @@ -21,6 +21,30 @@ DRb.stop_service end + def test_sanitation_backed_chest + chest = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("site_chest.db"), + :officer => @c) + begin + assert_equal(nil, chest["klibb"]) + assert_equal("kladd", chest["klibb"] = "kladd") + assert_equal("kladd", chest["klibb"]) + assert_equal("KLADD", chest["klibb"].upcase!) + klibb = chest["klibb"] + assert_equal("kladd", klibb.downcase) + assert_equal("kladd", klibb.downcase!) + assert_equal("KLADD", klibb.upcase) + assert_equal("kladd", klibb) + @d.stop! + @c.update_services! + assert_raise(Archipelago::Sanitation::NoRemoteDatabaseAvailableException) do + klibb.upcase! + end + ensure + chest.stop! + chest.instance_eval do @persistence_provider.unlink! end + end + end + def test_missing_bits s1 = Oneliner::SuperString.new("brappa") @d.insert!("a", [s1.encode(9)], "abab") @@ -29,7 +53,15 @@ end end - def test_navigation + + def test_get_set + @c["hej"] = "hoho" + assert_equal("hoho", @c["hej"]) + @c.delete!("hej") + assert_equal(@c["hej"], nil) + end + + def test_multi # # Setup the network # @@ -157,13 +189,6 @@ end end - def test_get_set - @c["hej"] = "hoho" - assert_equal("hoho", @c["hej"]) - @c.delete!("hej") - assert_equal(@c["hej"], nil) - end - private def chunks_by_id(officer, key) From nobody at rubyforge.org Wed Mar 7 07:18:32 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 07:18:32 -0500 (EST) Subject: [Archipelago-submits] [232] trunk/archipelago/TODO: updated TODO Message-ID: <20070307121832.A1FC85242322@rubyforge.org> Revision: 232 Author: zond Date: 2007-03-07 07:18:32 -0500 (Wed, 07 Mar 2007) Log Message: ----------- updated TODO Modified Paths: -------------- trunk/archipelago/TODO Modified: trunk/archipelago/TODO =================================================================== --- trunk/archipelago/TODO 2007-03-07 11:46:27 UTC (rev 231) +++ trunk/archipelago/TODO 2007-03-07 12:18:32 UTC (rev 232) @@ -1,19 +1,4 @@ - * Create a failover/redundancy framework - * For example: Create a new HashishProvider with built in redundancy, - for example using the Chord project: http://pdos.csail.mit.edu/chord/ - * Or: Create migration methods that move objects between Chests opon - startup and shutdown, and make them keep backups at each others - persistence backends. - * Or: Create something that stores data the way Chord does (with erasure - codes) but doesnt use the same look up mechanism. - * Problem: We still have to implement the entire maintenance protocol - of Chord (continously checking if our data is safely replicated across - the network, continously checking that our data belong with us) - - * Replace Raider with some well known near-optimal erasure code, for example - Online Codes: http://en.wikipedia.org/wiki/Online_codes - * Make Chest aware about whether transactions have affected it 'for real' ie check whether the instance before the call differs from the instance after the call. Preferably without incurring performance lossage. @@ -30,3 +15,5 @@ * Create a memcached-client that uses Disco instance to find all memcached instances in the network and distribute requests among them. + * Create a file server services that handles really big files within transactions + and allows the POSTing and GETing of them via HTTP. From nobody at rubyforge.org Wed Mar 7 07:19:12 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Wed, 7 Mar 2007 07:19:12 -0500 (EST) Subject: [Archipelago-submits] [233] trunk/hyperactive/TODO: updated a link Message-ID: <20070307121912.3D0FE524234F@rubyforge.org> Revision: 233 Author: zond Date: 2007-03-07 07:19:12 -0500 (Wed, 07 Mar 2007) Log Message: ----------- updated a link Modified Paths: -------------- trunk/hyperactive/TODO Modified: trunk/hyperactive/TODO =================================================================== --- trunk/hyperactive/TODO 2007-03-07 12:18:32 UTC (rev 232) +++ trunk/hyperactive/TODO 2007-03-07 12:19:12 UTC (rev 233) @@ -1,4 +1,4 @@ * Create a validation framework for Hyperactive::Record::Bass - * Create a sorted data structure, for example AA trees: http://www.eternallyconfuzzled.com/tuts/andersson.html \ No newline at end of file + * Create a sorted data structure, for example AA trees: http://www.eternallyconfuzzled.com/tuts/datastructures/jsw_tut_andersson.aspx \ No newline at end of file From nobody at rubyforge.org Thu Mar 8 03:45:22 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Thu, 8 Mar 2007 03:45:22 -0500 (EST) Subject: [Archipelago-submits] [234] trunk/hyperactive/lib/hyperactive: made sure Persistent includes Pimp. Message-ID: <20070308084522.4A8C252423E4@rubyforge.org> Revision: 234 Author: zond Date: 2007-03-08 03:45:21 -0500 (Thu, 08 Mar 2007) Log Message: ----------- made sure Persistent includes Pimp. made sure with_transaction retains the old transaction. Modified Paths: -------------- trunk/hyperactive/lib/hyperactive/index.rb trunk/hyperactive/lib/hyperactive/record.rb Modified: trunk/hyperactive/lib/hyperactive/index.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/index.rb 2007-03-07 12:19:12 UTC (rev 233) +++ trunk/hyperactive/lib/hyperactive/index.rb 2007-03-08 08:45:21 UTC (rev 234) @@ -142,7 +142,7 @@ super base.class_eval do # - # We depend on lots of hooks. + # We depend on hooks. # include(Hyperactive::Hooker::Pimp) end Modified: trunk/hyperactive/lib/hyperactive/record.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/record.rb 2007-03-07 12:19:12 UTC (rev 233) +++ trunk/hyperactive/lib/hyperactive/record.rb 2007-03-08 08:45:21 UTC (rev 234) @@ -65,6 +65,10 @@ # The transaction we are currently in. # attr_reader :transaction + # + # We depend on hooks. + # + include(Hyperactive::Hooker::Pimp) end base.extend(ClassMethods) @@ -86,11 +90,12 @@ # See Hyperactive::List::Head and Hyperactive::Hash::Head for examples of this behaviour. # def with_transaction(transaction, &block) + old_transaction = @transaction if defined?(@transaction) @transaction = transaction begin return yield ensure - @transaction = nil + @transaction = old_transaction end end From nobody at rubyforge.org Thu Mar 8 11:30:28 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Thu, 8 Mar 2007 11:30:28 -0500 (EST) Subject: [Archipelago-submits] [235] trunk/hyperactive: added prototype for anderson trees Message-ID: <20070308163028.D148B5241E8E@rubyforge.org> Revision: 235 Author: zond Date: 2007-03-08 11:30:27 -0500 (Thu, 08 Mar 2007) Log Message: ----------- added prototype for anderson trees Modified Paths: -------------- trunk/hyperactive/lib/hyperactive/hash.rb trunk/hyperactive/lib/hyperactive/index.rb trunk/hyperactive/lib/hyperactive/record.rb Added Paths: ----------- trunk/hyperactive/lib/hyperactive/tree.rb trunk/hyperactive/tests/tree_test.rb Modified: trunk/hyperactive/lib/hyperactive/hash.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/hash.rb 2007-03-08 08:45:21 UTC (rev 234) +++ trunk/hyperactive/lib/hyperactive/hash.rb 2007-03-08 16:30:27 UTC (rev 235) @@ -25,8 +25,8 @@ module Hyperactive # - # The package containing the Hash class that provides any - # kind of index for your Hyperactive classes. + # The package containing the Hash class that provides unsorted + # indices for your Hyperactive classes. # # Is supposed to be constantly scaling, but preliminary benchmarks show some # problems with that assumption? @@ -110,7 +110,7 @@ # Insert +value+ under +key+ in this Hash. # def []=(key, value) - self.list = Hyperactive::List::Head.get_instance_with_transaction(@transaction) unless self.list + self.list ||= Hyperactive::List::Head.get_instance_with_transaction(@transaction) if (element = Archipelago::Pirate::BLACKBEARD[my_key_for(key), @transaction]) element.value = value @@ -126,7 +126,7 @@ # Delete +key+ from this Hash. # def delete(key) - self.list = Hyperactive::List::Head.get_instance_with_transaction(@transaction) unless self.list + self.list ||= Hyperactive::List::Head.get_instance_with_transaction(@transaction) return_value = nil @@ -177,7 +177,7 @@ # Get my private key for a given +key+. # def my_key_for(key) - Digest::SHA1.hexdigest("#{Marshal.dump(key)}#{self.record_id}") + Digest::SHA1.hexdigest("Hyperactive::Hash::Head:#{self.record_id}:#{Marshal.dump(key)}") end end Modified: trunk/hyperactive/lib/hyperactive/index.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/index.rb 2007-03-08 08:45:21 UTC (rev 234) +++ trunk/hyperactive/lib/hyperactive/index.rb 2007-03-08 16:30:27 UTC (rev 235) @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. require 'hyperactive/hash' +require 'hyperactive/tree' require 'hyperactive/hooker' module Hyperactive Modified: trunk/hyperactive/lib/hyperactive/record.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/record.rb 2007-03-08 08:45:21 UTC (rev 234) +++ trunk/hyperactive/lib/hyperactive/record.rb 2007-03-08 16:30:27 UTC (rev 235) @@ -180,9 +180,19 @@ # which is not what you usually want. Every other time you fetch it using a select or other # method you will instead receive a proxy object to the database. This means that nothing you # do to it at that point will be persistent or even necessarily have a defined result. - # Therefore: do not use the instantiated object, instead call my_instance.save + # Therefore: do not use the instantiated object, instead call my_instance.create or MyClass.get_instance # to get a proxy to the object stored into the database. # + # NB: When a subclass is created with get_instance, initialize is called + # and then create is called and its return value returned. + # If you instead use get_instance_with_transaction, initialize will be called (like before) + # and then create with the given transaction. This means that you can not use transactions in + # initialize and at the same time make get_instance_with_transaction work, since + # the initialize method wont know about the transaction sent to get_instance_with_transaction. + # Of course, this can be avoided using varying degrees of ugly hack, but a simpler way is to try and + # make initialize not do transaction-dependant stuff, like create new Hyperactive::Record::Bass + # instances, but instead do that in (for example) create_hooks (See Hyperactive::Hooker::Pimp). + # class Bass include Hyperactive::Record::Persistent Added: trunk/hyperactive/lib/hyperactive/tree.rb =================================================================== --- trunk/hyperactive/lib/hyperactive/tree.rb (rev 0) +++ trunk/hyperactive/lib/hyperactive/tree.rb 2007-03-08 16:30:27 UTC (rev 235) @@ -0,0 +1,115 @@ +# Archipelago - a distributed computing toolkit for ruby +# Copyright (C) 2006 Martin Kihlgren +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +require 'rubygems' +require 'archipelago/pirate' +require 'hyperactive/record' +require 'digest/sha1' + +module Hyperactive + + # + # The package containing the Tree class that provides sorted + # indices for your Hyperactive classes. + # + # Is supposed to be logarithmically scaling. + # + module Tree + + class Node < Hyperactive::Record::Bass + + @@nil_node = nil + @@nil_node_record_id = nil + + include Hyperactive::Cleaner::Accessors + + attr_accessor :right, :left, :level, :value, :key + + def initialize(options = {}) + super() + self.right = options[:right] || self.class.nil_node + self.right = self if self.right == :self + self.left = options[:left] || self.class.nil_node + self.left = self if self.left == :self + self.level = options[:level] || 1 + self.value = options[:value] + self.key = options[:key] + end + + def insert(key, value) + if self.nil_node? + return Node.get_instance_with_transaction(transaction, :key => key, :value => value) + else + side = :left + side = :right if self.key < key + self.send("#{side}=", self.send(side).insert(key, value)) + rval = self.skew + rval = rval.split + return rval + end + end + + def skew + end + + def split + end + + def nil_node? + self.record_id == self.class.nil_node_record_id + end + + def self.nil_node_record_id + @@nil_node_record_id ||= self.nil_node.record_id + end + + def self.nil_node + nil_node_key = Digest::SHA1.hexdigest("Hyperactive::Tree::Node::nil_node") + @@nil_node ||= if (existing_element = Archipelago::Pirate::BLACKBEARD[nil_node_key]) + existing_element + else + rval = Node.get_instance(:left => :self, + :right => :self, + :level => 0) + Archipelago::Pirate::BLACKBEARD[nil_node_key] = rval + rval + end + end + + end + + class Root < Hyperactive::Record::Bass + + include Hyperactive::Cleaner::Accessors + + attr_accessor :root + + def initialize + super + self.root = Node.nil_node + end + + def []=(key, value) + self.root = root.insert(key, value) + end + + end + + end + +end Added: trunk/hyperactive/tests/tree_test.rb =================================================================== --- trunk/hyperactive/tests/tree_test.rb (rev 0) +++ trunk/hyperactive/tests/tree_test.rb 2007-03-08 16:30:27 UTC (rev 235) @@ -0,0 +1,42 @@ + +require File.join(File.dirname(__FILE__), 'test_helper') + +class TreeTest < Test::Unit::TestCase + + def setup + @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db"))) + @c.publish! + @c2 = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest2.db"))) + @c2.publish! + @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny1.db"))) + @tm.publish! + Archipelago::Pirate::BLACKBEARD.setup(:chest_description => {:class => 'TestChest'}, + :tranny_description => {:class => 'TestManager'}) + Archipelago::Pirate::BLACKBEARD.update_services! + assert_within(10) do + Archipelago::Pirate::BLACKBEARD.chests.keys.sort == [@c.service_id, @c2.service_id].sort + end + assert_within(10) do + Archipelago::Pirate::BLACKBEARD.trannies.keys == [@tm.service_id] + end + end + + def teardown + @c.stop! + @c.persistence_provider.unlink! + @c2.stop! + @c2.persistence_provider.unlink! + @tm.stop! + @tm.persistence_provider.unlink! + end + + def test_insert + t = Hyperactive::Tree::Root.get_instance + assert(t.root.nil_node?) + t["epa"] = "apa" + assert_equal("apa", t.root.value) + assert(t.root.left.nil_node?) + assert(t.root.right.nil_node?) + end + +end From nobody at rubyforge.org Fri Mar 9 05:28:46 2007 From: nobody at rubyforge.org (nobody at rubyforge.org) Date: Fri, 9 Mar 2007 05:28:46 -0500 (EST) Subject: [Archipelago-submits] [236] trunk: added a patched version of rbtree Message-ID: <20070309102847.301385240952@rubyforge.org> Revision: 236 Author: zond Date: 2007-03-09 05:28:46 -0500 (Fri, 09 Mar 2007) Log Message: ----------- added a patched version of rbtree Added Paths: ----------- trunk/rbtree-0.2.0/ trunk/rbtree-0.2.0/ChangeLog trunk/rbtree-0.2.0/LICENSE trunk/rbtree-0.2.0/MANIFEST trunk/rbtree-0.2.0/README trunk/rbtree-0.2.0/depend trunk/rbtree-0.2.0/dict.c trunk/rbtree-0.2.0/dict.h trunk/rbtree-0.2.0/extconf.rb trunk/rbtree-0.2.0/rbtree.c trunk/rbtree-0.2.0/test.rb Property changes on: trunk/rbtree-0.2.0 ___________________________________________________________________ Name: svn:ignore + rbtree.bundle mkmf.log Makefile Added: trunk/rbtree-0.2.0/ChangeLog =================================================================== --- trunk/rbtree-0.2.0/ChangeLog (rev 0) +++ trunk/rbtree-0.2.0/ChangeLog 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,421 @@ +2007-02-01 OZAWA Takuma + + * version 0.2.0 released. + +2007-01-25 OZAWA Takuma + + * rbtree.c (rbtree_readjust): remove a warning. + + * rbtree.c (rbtree_default): should not call default procedure if + no key is given. + + * rbtree.c (rbtree_equal): returns true if two rbtrees have same + set of key-value set. + +2004-10-27 OZAWA Takuma + + * version 0.1.3 released. + +2004-07-29 OZAWA Takuma + + * rbtree.c (rbtree_bound): RBTree#bound(lower, upper = lower). + +2004-07-11 OZAWA Takuma + + * rbtree.c (MultiRBTree): new class. MultiRBTree allows duplicates + of keys. + + * rbtree.c (rbtree_dump, rbtree_s_load): use Array as a storage + for performance improvement. + + * rbtree.c (rbtree_equal, rbtree_initialize_copy, rbtree_update): + should use rb_obj_is_kind_of. + + * rbtree.c (iter_lev): added to count iterator level for nesting + and thread-safety. + +2004-06-29 OZAWA Takuma + + * dict.c: remove codes Ruby/RBTree doesn't need. now not supposed + to use dict.c with other program. + + * dict.h: ditto. + + * dict.c (dict_equal): remove key_eql argument. use + dict->dict_compare to compare keys. + +2004-06-20 OZAWA Takuma + + * rbtree.c (rbtree_cmp): use rb_str_cmp if the type of keys is + string. + + * rbtree.c (rbtree_cmp): use rb_cmpint. + + * rbtree.c (rbtree_user_cmp): ditto. + +2004-06-12 OZAWA Takuma + + * version 0.1.2 release. + +2004-05-31 OZAWA Takuma + + * rbtree.c (rbtree_initialize_copy): if an exception is raised in + the method rbtree is not modified. + + * rbtree.c (rbtree_delete_if): if an exception is raised in the + block rbtree is not modified(no keys are deleted). + + * rbtree.c (rbtree_readjust): use rb_gc_force_recycle. + +2004-05-24 OZAWA Takuma + + * rbtree.c (rbtree_initialize_copy): use aset_i. duplicating a key + of string is fast because of copy-on-write. + + * rbtree.c (rbtree_readjust): ditto. + + * rbtree.c (rbtree_update): ditto. + + * rbtree.c (rbtree_aset): no dict_lookup for better performance. + + * rbtree.c (rbtree_update): was same whether a block is given or not. + +2004-05-23 OZAWA Takuma + + * rbtree.c (inspect_rbtree): add OBJ_INFECTs. + + * rbtree.c (inspect_rbtree): change ``compare'' to ``cmp_proc''. + + * rbtree.c (pp_block): ditto. + +2004-04-26 OZAWA Takuma + + * rbtree.c (rbtree_s_create): accept Hash argument. + + * rbtree.c (rbtree_s_create): should just copy keys and values. + +2004-02-19 OZAWA Takuma + + * rbtree.c (pp_object_group): use id_object_group. + +2004-02-16 OZAWA Takuma + + * version 0.1.1 release. + +2004-02-13 OZAWA Takuma + + * README: rewritten. + + * rbtree.c (document): incomplete document for rdoc. + +2004-02-08 OZAWA Takuma + + * test.rb (test_pp): add pretty printing test. + +2004-02-07 OZAWA Takuma + + * rbtree.c (rbtree_mark): should check rbtree and dict is + initialized. Thanks to Neil Spring. + +2004-02-05 OZAWA Takuma + + * rbtree.c (prettyprint): polish code. + + * test.rb (assert_raise): alias of assert_raises for Ruby 1.6.x. + +2004-02-02 OZAWA Takuma + + * test.rb: clean code. no more RUnit support. use Test::Unit. + +2004-01-29 OZAWA Takuma + + * rbtree.c (rbtree_dump): optimization. + + * rbtree.c (rbtree_s_load): ditto. + +2004-01-27 OZAWA Takuma + + * rbtree.c (readjust): RBTree#readjust() just readjusts elements + using current comparison block. use RBTree#readjust(nil) to set + default comparison block. + + * extconf.rb (assertion): removed. + +2004-01-14 OZAWA Takuma + + * ctest/test.c: removed. + + * ctest/testlib.c: ditto. + + * ctest/testlib.h: ditto. + + * ctest/Makefile: ditto. + +2004-01-04 OZAWA Takuma + + * version 0.1.0 release. + +2004-01-03 OZAWA Takuma + + * rbtree.c (rbtree_to_rbtree): new method. + + * rbtree.c (each_pair_i, rbtree_bound_body, rbtree_delete_if_body, + select_i, update_block_i): use rb_yield_values. + + * rbtree.c (rbtree_each): RBTree#each should yield single value. + + * rbtree.c (rbtree_select): select(key..) is removed. + + * rbtree.c (rbtree_fetch): always warn if default argument and a + block are supplied at the same time. + +2003-08-12 takuma ozawa + + * version 0.0.7 release. + +2003-08-11 takuma ozawa + + * rbtree.c (rbtree_to_hash): copy default value. + + * rbtree.c (rbtree_readjust): takes a Proc argument. + + * rbtree.c (to_hash_i): optimization. + +2003-08-09 takuma ozawa + + * rbtree.c (rbtree_merge): new method. + + * rbtree.c (rbtree_select): select(key..) is deprecated. + + * rbtree.c (rbtree_values_at): new method. + + * rbtree.c (rbtree_initialized_copy): rbtree_copy_object changed to. + +2003-07-27 takuma ozawa + + * rbtree.c (rbtree_dump): new method based on Ara Howard's code. + Thanks. + + * rbtree.c (rbtree_s_load): ditto. + +2003-03-25 takuma ozawa + + * version 0.0.6 release. + +2003-02-26 takuma ozawa + + * rbtree.c (rbtree_readjust): rbtree_modify. + +2003-02-23 takuma ozawa + + * rbtree.c (rbtree_copy_object): use copy_i. + + * rbtree.c (rbtree_readjust): ditto. + + * rbtree.c (rbtree_aset): not freeze a key. + +2003-01-24 takuma ozawa + + * rbtree.c (rbtree_aset): optimization. + +2003-01-18 takuma ozawa + + * rbtree.c (rbtree_aset): not raise an exception if a dict is + full and the key has been contained. + +2003-01-16 takuma ozawa + + * version 0.0.5 release. + +2002-12-26 takuma ozawa + + * rbtree.c (rbtree_alloc): new allocation framework. + + * rbtree.c (rbtree_copy_object): changed become to copy_object. + + * rbtree.c (rbtree_cmp): use NUM2INT in case nil returned. + + * extconf.rb (assertion): assertion is off by default. + + * rbtree.c (rbtree_aset): fixed a memory leak occured if the + comparison block raises an exception. + +2002-11-24 takuma ozawa + + * rbtree.c (rbtree_pretty_print): new method. + + * rbtree.c (rbtree_pretty_print_cycle): new method. + +2002-11-22 takuma ozawa + + * rbtree.c (rbtree_inspect): format changed. + + * rbtree.c (rbtree_user_cmp): use NUM2INT. + +2002-10-29 takuma ozawa + + * version 0.0.4 release. + +2002-10-18 takuma ozawa + + * extconf.rb (assertion): change name to assertion. + +2002-10-12 takuma ozawa + + * rbtree.c (rbtree_readjust): assign Qnil to other's dict_context + after swap. + + * rbtree.c (rbtree_bound): use dict_compare for range check. + +2002-10-10 takuma ozawa + + * rbtree.c (rbtree_readjust): empty dict check. + + * rbtree.c (rbtree_readjust): not use dict_readjust for better + readability. + + * rbtree.c (rbtree_become): must copy dict_compare and + dict_context before copying nodes. + + * rbtree.c (rbtree_delete_if): rbtree_modify. + + * rbtree.c (rbtree_shift_pop): ditto. + + * rbtree.c (rbtree_update): ditto. + + * test.rb: Test::Unit. + + * rbtree.c (rbtree_readjust): clear other's nodes after swap. + + * rbtree.c (rbtree_readjust): assign Qnil to other's ifnone after + swap. + +2002-10-07 takuma ozawa + + * rbtree.c (rbtree_update): self assignment check. + + * rbtree.c (rbtree_update): replace duplicate value by return + value of block if given. + +2002-10-03 takuma ozawa + + * rbtree.c (rbtree_become): self assignment check. + +2002-09-24 takuma ozawa + + * rbtree.c (rbtree_cmp_proc): new method. + + * rbtree.c (rbtree_readjust): if no block given, must assign Qnil + to context. + + * rbtree.c (rbtree_inspect): format changed. + +2002-09-23 takuma ozawa + + * dict.c (dict_equal): empty test must be after similar test. + +2002-09-22 takuma ozawa + + * rbtree.c (rbtree_update): convert argument to RBTree. + + * rbtree.c (rbtree_become): ditto. + + * rbtree.c (rbtree_eq): use rb_equal. + + * rbtree.c (rbtree_eql): change name to rbtree_eq. + +2002-09-20 takuma ozawa + + * rbtree.c (RBTREE_FL_COPY): removed. + + * ctest/test.c: rewritten in C. CppUnit is not required. + +2002-09-18 takuma ozawa + + * depend: new file. + + * rbtree.c (rbtree_equal): no need to check RBTREE_PROC_DEFAULT. + + * extconf.rb (inline) check for inline keyword. + +2002-09-17 takuma ozawa + + * version 0.0.3 release. + +2002-09-16 takuma ozawa + + * rbtree.c (rbtree_reverse_each): new method. + +2002-09-12 takuma ozawa + + * rbtree.c (rbtree_s_create): unset RBTREE_PROC_DEFAULT if the + argument is RBTree. + + * rbtree.c (rbtree_clone): use rbtree_become. + + * rbtree.c (version.h): not included. + +2002-09-11 takuma ozawa + + * rbtree.c (rbtree_to_a): use OBJ_INFECT. + + * rbtree.c (rbtree_to_hash): ditto. + + * rbtree.c (rbtree_become): replaced by rbtree_replace. + + * rbtree.c (rbtree_replace): removed. replace is implemented + useing rbtree_become. + + * rbtree.c (rbtree_first_last): first or pop from empty tree + should not return its default proc. + +2002-09-09 takuma ozawa + + * rbtree.c (rbtree_become): new method. + + * rbtree.c (rbtree_clone): unset RBTREE_IN_ITERATION. + + * rbtree.c (rbtree_replace): should copy ifnone. + + * rbtree.c (rbtree_shift_pop): shift or pop from empty tree should + not return its default proc. + + * rbtree.c (rbtree_default_proc): new method. + + * rbtree.c (rbtree_equal): should check default values and + RBTREE_PROC_DEFAULT. + +2002-07-19 takuma ozawa + + * dict.c (alloc_node): must return NULL if a memory allocation + fails. + +2002-07-13 takuma ozawa + + * version 0.0.2 release. + +2002-07-11 takuma ozawa + + * rbtree.c (rbtree_readjust): new method. + +2002-07-10 takuma ozawa + + * dict.c (dict_clone): clone object must copy must_unfreeze + variable. + +2002-06-25 takuma ozawa + + * dict.h (dnode_t): add must_freeze flag not to unfreeze already + freezed object. + +2002-06-22 takuma ozawa + + * rbtree.c (rbtree_first): new method. + + * rbtree.c (rbtree_last): new method. + + * rbtree.c (rbtree_pop): new method. + +2002-06-14 takuma ozawa + + * initial release. Added: trunk/rbtree-0.2.0/LICENSE =================================================================== --- trunk/rbtree-0.2.0/LICENSE (rev 0) +++ trunk/rbtree-0.2.0/LICENSE 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,22 @@ +Copyright (c) 2002-2004, 2007 OZAWA Takuma + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. Added: trunk/rbtree-0.2.0/MANIFEST =================================================================== --- trunk/rbtree-0.2.0/MANIFEST (rev 0) +++ trunk/rbtree-0.2.0/MANIFEST 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,10 @@ +ChangeLog +LICENSE +MANIFEST +README +depend +dict.c +dict.h +extconf.rb +rbtree.c +test.rb Added: trunk/rbtree-0.2.0/README =================================================================== --- trunk/rbtree-0.2.0/README (rev 0) +++ trunk/rbtree-0.2.0/README 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,74 @@ +=begin + += Ruby/RBTree + +RBTree is a sorted associative collection using Red-Black Tree as +the internal data structure. The elements of RBTree are ordered and +the interface is the almost same as Hash, so simply you can +consider RBTree sorted Hash. + +Red-Black Tree is a kind of binary tree that automatically balances +by itself when a node is inserted or deleted. Thus the complexity +for insert, search and delete is O(log N) in expected and worst +case. On the other hand the complexity of Hash is O(1). Because +Hash is unordered the data structure is more effective than +Red-Black Tree as an associative collection. + +The interface of RBTree is the almost same as Hash although there +are some limitations. + + * While iterating (e.g. in RBTree#each block), RBTree is + unmodifiable. + + * Comparison is done using <=> method of key objects. So all types + of keys in RBTree should be comparable each other or an arbitrary + Proc might be set by RBTree#readjust. + +RBTree has a few searching methods that Hash doesn't have. They are +RBTree#lower_bound, RBTree#upper_bound and RBTree#bound. See document +of each method for details. + +Pretty printing is available for RBTree by using pp.rb. The output +of pp is easier than p to read. Just call Kernel#pp for the object. + +MultiRBTree that allows duplicates of keys is also available. + +== Download + + * ((<"Ruby/RBTree 0.2.0"|URL:rbtree-0.2.0.tar.gz>)) + +== Requirement + + * Ruby 1.6.7 or higher + +== Install + + $ tar xzf rbtree-x.x.x.tar.gz + $ cd rbtree-x.x.x.tar.gz + $ ruby extconf.rb + $ make + $ make site-install + +== Test + + $ ruby test.rb + +== Incomplete Document + + $ rdoc rbtree.c + +== License + +MIT License. Copyright (c) 2002-2004, 2007 OZAWA Takuma. + +dict.c and dict.h are modified copies that are originally in Kazlib +written by Kaz Kylheku. Copyright is held by Kaz Kylheku, see dict.c +and dict.h for the license. The web page of Kazlib is at +(()). + +== Support + +Bug fixes, suggestions and other feedbacks are welcomed. Please mail +me at (()). + +=end Added: trunk/rbtree-0.2.0/depend =================================================================== --- trunk/rbtree-0.2.0/depend (rev 0) +++ trunk/rbtree-0.2.0/depend 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,2 @@ +dict.o: dict.c dict.h +rbtree.o: rbtree.c dict.h Added: trunk/rbtree-0.2.0/dict.c =================================================================== --- trunk/rbtree-0.2.0/dict.c (rev 0) +++ trunk/rbtree-0.2.0/dict.c 2007-03-09 10:28:46 UTC (rev 236) @@ -0,0 +1,1216 @@ +/* + * Dictionary Abstract Data Type + * Copyright (C) 1997 Kaz Kylheku + * + * Free Software License: + * + * All rights are reserved by the author, with the following exceptions: + * Permission is granted to freely reproduce and distribute this software, + * possibly in exchange for a fee, provided that this copyright notice appears + * intact. Permission is also granted to adapt this software to produce + * derivative works, as long as the modified versions carry this copyright + * notice and additional notices stating that the work has been modified. + * This source code may be translated into executable form and incorporated + * into proprietary software; there is no requirement for such software to + * contain a copyright notice related to this source. + * + * $Id: dict.c,v 1.15 2005/10/06 05:16:35 kuma Exp $ + * $Name: $ + */ + +/* + * Modified for Ruby/RBTree by OZAWA Takuma. + */ + +#include +#include +#include +#include "dict.h" + +#include + +#ifdef KAZLIB_RCSID +static const char rcsid[] = "$Id: dict.c,v 1.15 2005/10/06 05:16:35 kuma Exp $"; +#endif + +/* + * These macros provide short convenient names for structure members, + * which are embellished with dict_ prefixes so that they are + * properly confined to the documented namespace. It's legal for a + * program which uses dict to define, for instance, a macro called ``parent''. + * Such a macro would interfere with the dnode_t struct definition. + * In general, highly portable and reusable C modules which expose their + * structures need to confine structure member names to well-defined spaces. + * The resulting identifiers aren't necessarily convenient to use, nor + * readable, in the implementation, however! + */ + +#define left dict_left +#define right dict_right +#define parent dict_parent +#define color dict_color +#define key dict_key +#define data dict_data + +#define nilnode dict_nilnode +#define nodecount dict_nodecount +#define compare dict_compare +#define allocnode dict_allocnode +#define freenode dict_freenode +#define context dict_context +#define dupes dict_dupes + +#define dictptr dict_dictptr + +#define dict_root(D) ((D)->nilnode.left) +#define dict_nil(D) (&(D)->nilnode) +#define DICT_DEPTH_MAX 64 + +#define COMPARE(dict, key1, key2) dict->compare(key1, key2, dict->context) + +static dnode_t *dnode_alloc(void *context); +static void dnode_free(dnode_t *node, void *context); + +/* + * Perform a ``left rotation'' adjustment on the tree. The given node P and + * its right child C are rearranged so that the P instead becomes the left + * child of C. The left subtree of C is inherited as the new right subtree + * for P. The ordering of the keys within the tree is thus preserved. + */ + +static void rotate_left(dnode_t *upper) +{ + dnode_t *lower, *lowleft, *upparent; + + lower = upper->right; + upper->right = lowleft = lower->left; + lowleft->parent = upper; + + lower->parent = upparent = upper->parent; + + /* don't need to check for root node here because root->parent is + the sentinel nil node, and root->parent->left points back to root */ + + if (upper == upparent->left) { + upparent->left = lower; + } else { + assert (upper == upparent->right); + upparent->right = lower; + } + + lower->left = upper; + upper->parent = lower; +} + +/* + * This operation is the ``mirror'' image of rotate_left. It is + * the same procedure, but with left and right interchanged. + */ + +static void rotate_right(dnode_t *upper) +{ + dnode_t *lower, *lowright, *upparent; + + lower = upper->left; + upper->left = lowright = lower->right; + lowright->parent = upper; + + lower->parent = upparent = upper->parent; + + if (upper == upparent->right) { + upparent->right = lower; + } else { + assert (upper == upparent->left); + upparent->left = lower; + } + + lower->right = upper; + upper->parent = lower; +} + +/* + * Do a postorder traversal of the tree rooted at the specified + * node and free everything under it. Used by dict_free(). + */ + +static void free_nodes(dict_t *dict, dnode_t *node, dnode_t *nil) +{ + if (node == nil) + return; + free_nodes(dict, node->left, nil); + free_nodes(dict, node->right, nil); + dict->freenode(node, dict->context); +} + +/* + * This procedure performs a verification that the given subtree is a binary + * search tree. It performs an inorder traversal of the tree using the + * dict_next() successor function, verifying that the key of each node is + * strictly lower than that of its successor, if duplicates are not allowed, + * or lower or equal if duplicates are allowed. This function is used for + * debugging purposes. + */ + +static int verify_bintree(dict_t *dict) +{ + dnode_t *first, *next; + + first = dict_first(dict); + + if (dict->dupes) { + while (first && (next = dict_next(dict, first))) { + if (COMPARE(dict, first->key, next->key) > 0) + return 0; + first = next; + } + } else { + while (first && (next = dict_next(dict, first))) { + if (COMPARE(dict, first->key, next->key) >= 0) + return 0; + first = next; + } + } + return 1; +} + + +/* + * This function recursively verifies that the given binary subtree satisfies + * three of the red black properties. It checks that every red node has only + * black children. It makes sure that each node is either red or black. And it + * checks that every path has the same count of black nodes from root to leaf. + * It returns the blackheight of the given subtree; this allows blackheights to + * be computed recursively and compared for left and right siblings for + * mismatches. It does not check for every nil node being black, because there + * is only one sentinel nil node. The return value of this function is the + * black height of the subtree rooted at the node ``root'', or zero if the + * subtree is not red-black. + */ + +static unsigned int verify_redblack(dnode_t *nil, dnode_t *root) +{ + unsigned height_left, height_right; + + if (root != nil) { + height_left = verify_redblack(nil, root->left); + height_right = verify_redblack(nil, root->right); + if (height_left == 0 || height_right == 0) + return 0; + if (height_left != height_right) + return 0; + if (root->color == dnode_red) { + if (root->left->color != dnode_black) + return 0; + if (root->right->color != dnode_black) + return 0; + return height_left; + } + if (root->color != dnode_black) + return 0; + return height_left + 1; + } + return 1; +} + +/* + * Compute the actual count of nodes by traversing the tree and + * return it. This could be compared against the stored count to + * detect a mismatch. + */ + +static dictcount_t verify_node_count(dnode_t *nil, dnode_t *root) +{ + if (root == nil) + return 0; + else + return 1 + verify_node_count(nil, root->left) + + verify_node_count(nil, root->right); +} + +/* + * Verify that the tree contains the given node. This is done by + * traversing all of the nodes and comparing their pointers to the + * given pointer. Returns 1 if the node is found, otherwise + * returns zero. It is intended for debugging purposes. + */ + +static int verify_dict_has_node(dnode_t *nil, dnode_t *root, dnode_t *node) +{ + if (root != nil) { + return root == node + || verify_dict_has_node(nil, root->left, node) + || verify_dict_has_node(nil, root->right, node); + } + return 0; +} + + +/* + * Dynamically allocate and initialize a dictionary object. + */ + +dict_t *dict_create(dict_comp_t comp) +{ + dict_t* new = ALLOC(dict_t); + + if (new) { + new->compare = comp; + new->allocnode = dnode_alloc; + new->freenode = dnode_free; + new->context = NULL; + new