Changeset 187

Timestamp:

05/13/07 20:47:11 (1 year ago)

Author:

jk

Message:

Improved DRb server index rebuild handling by keeping index versions

Files:

• trunk/plugin/acts_as_ferret/lib/act_methods.rb (modified) (5 diffs)
• trunk/plugin/acts_as_ferret/lib/acts_as_ferret.rb (modified) (1 diff)
• trunk/plugin/acts_as_ferret/lib/class_methods.rb (modified) (1 diff)
• trunk/plugin/acts_as_ferret/lib/ferret_cap_tasks.rb (added)
• trunk/plugin/acts_as_ferret/lib/ferret_server.rb (modified) (1 diff)
• trunk/plugin/acts_as_ferret/lib/local_index.rb (modified) (4 diffs)
• trunk/plugin/acts_as_ferret/script/ferret_start (modified) (2 diffs)
• trunk/plugin/acts_as_ferret/script/ferret_stop (modified) (1 diff)

trunk/plugin/acts_as_ferret/lib/act_methods.rb

@@ -37 +37 @@
-    #                    this to true. the model class name will be stored in a keyword field 
-    #                    named class_name
+    #
+    # reindex_batch_size:: reindexing is done in batches of this size, default is 1000
-    #
-    # ferret:: Hash of Options that directly influence the way the Ferret engine works. You 
@@ -107 +109 @@
-        :class_name => self.name,
-        :single_index => false,
+        :reindex_batch_size => 1000,
-        :ferret => {},                    # Ferret config Hash
-        :ferret_fields => {}              # list of indexed fields that will be filled later
@@ -130 +133 @@
-      # merge ferret options with those from second parameter hash
-      aaf_configuration[:ferret].update(ferret_options) if ferret_options.is_a?(Hash)
+
+      unless options[:remote]
+        ActsAsFerret::ensure_directory aaf_configuration[:index_dir] 
+        aaf_configuration[:index_base_dir] = aaf_configuration[:index_dir]
+        aaf_configuration[:index_dir] = find_last_index_version(aaf_configuration[:index_dir])
+        logger.debug "using index in #{aaf_configuration[:index_dir]}"
+      end
-
-      # these properties are somewhat vital to the plugin and shouldn't
@@ -146 +156 @@
-        add_fields(aaf_configuration[:additional_fields])
-      end
-
-      ActsAsFerret::ensure_directory aaf_configuration[:index_dir] unless options[:remote]
-
-      # now that all fields have been added, we can initialize the default
@@ -175 +183 @@
-    protected
-    
+    # find the most recent version of an index
+    def find_last_index_version(basedir)
+      # check for versioned index
+      versions = Dir.entries(basedir).select { |f| File.directory?(File.join(basedir, f)) && f =~ /^\d+$/ }
+      if versions.any?
+        # select latest version
+        versions.sort!
+        File.join basedir, versions.last
+      else
+        basedir
+      end
+    end
+
-    # helper that defines a method that adds the given field to a ferret 
-    # document instance

trunk/plugin/acts_as_ferret/lib/acts_as_ferret.rb

@@ -76 +76 @@
-    @@ferret_indexes = Hash.new
-    def self.ferret_indexes; @@ferret_indexes end
+
- 
-  # decorator that adds a total_hits accessor to search result arrays

trunk/plugin/acts_as_ferret/lib/class_methods.rb

@@ -13 +13 @@
-    def rebuild_index(*models)
-      models << self unless models.include?(self)
-(models.map(&:to_s)
+ models.map(&:to_s
-    end                                                            
+
+    # Switches this class to a new index located in dir.
+    # Used by the DRb server when switching to a new index version.
+    def index_dir=(dir)
+      aaf_configuration[:index_dir] = aaf_configuration[:ferret][:path] = dir
+      aaf_index.reopen!
+    end
-    
-    # Retrieve the index instance for this model class. This can either be a

trunk/plugin/acts_as_ferret/lib/ferret_server.rb

@@ -7 +7 @@
-module ActsAsFerret
-
-
+  
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        end
-        {}
-      end
-    end
-  end
-
-
-
-
-
-
-# your needs.
-# - run script/ferret_server (in the plugin directory) via script/runner:
-# RAILS_ENV=production script/runner vendor/plugins/acts_as_ferret/script/ferret_server
-#
-# TODO: automate installation of files to script/ and config/
-
+  
+  
+  
+  
+  
+  # your needs. environments for which no section in the config file exists
+  # will use the index locally (good for unit tests/development mode)
+  # - run script/ferret_server (in the plugin directory) via script/runner:
+  # RAILS_ENV=production script/runner vendor/plugins/acts_as_ferret/script/ferret_server
+  #
+  
-
-
+  
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    end
-
-    def initialize
-      @logger = Logger.new("#{RAILS_ROOT}/log/ferret_server.log")
-    end
-
-    # handles all incoming method calls, and sends them on to the LocalIndex
-    # instance of the correct model class.
-    #
-    # Calls are not queued atm, so this will block until the call returned.
-    # Might throw the occasional LockError, too, which most probably means that you're 
-    # a) rebuilding your index or 
-    # b) have *really* high load. I wasn't able to reproduce this case until
-    # now, if you do, please contact me.
-    #
-    # TODO: rebuild indexes in separate directory so no lock errors in these
-    # cases.
-    def method_missing(name, *args)
-      @logger.debug "\#method_missing(#{name.inspect}, #{args.inspect})"
-      clazz = args.shift.constantize
-      begin
-        clazz.aaf_index.send name, *args
-      rescue NoMethodError
-        @logger.debug "no luck, trying to call class method instead"
-        clazz.send name, *args
-      end
-    rescue
-      @logger.error "ferret server error #{$!}\n#{$!.backtrace.join '\n'}"
-      raise
-    end
-
-    def ferret_index(class_name)
-      # TODO check if in use!
-      class_name.constantize.aaf_index.ferret_index
-    end
-
-    # the main loop taking stuff from the queue and running it...
-    #def run
-    #end
-
-  end
-end
-end

trunk/plugin/acts_as_ferret/lib/local_index.rb

@@ -8 +8 @@
-      super
-      ensure_index_exists
+    end
+
+    def reopen!
+      if @ferret_index
+        @ferret_index.close
+        @ferret_index = nil
+      end
+      logger.debug "reopening index at #{aaf_configuration[:ferret][:path]}"
+      ferret_index
-    end
-
@@ -38 +47 @@
-    # model classes to include in the index
-    def rebuild_index(*models)
-      logger.debug "rebuild index: #{models.inspect}"
-      models << aaf_configuration[:class_name] unless models.include?(aaf_configuration[:class_name])
-      models = models.flatten.uniq.map(&:constantize)
+      logger.debug "rebuild index: #{models.inspect}"
-      index = Ferret::Index::Index.new(aaf_configuration[:ferret].dup.update(:auto_flush => false, 
-                                                                             :field_infos => field_infos(models),
-                                                                             :create => true))
+      do_rebuild_with_index(index, models)
+    end
+
+    def do_rebuild_with_index(index, models)
-      models.each do |model|
-        reindex_model(index, model)
-      end
-      logger.debug("Created Ferret index in: #{aaf_configuration[:index_dir]}")
-      index.flush
+      logger.debug("Created Ferret index in: #{index.options[:path]}. Will now optimize...")
-      index.optimize
-      index.close
-      close_multi_indexes
+      logger.debug("Done.")
-    end
-
@@ -180 +194 @@
-    end
-
-
-    protected
-
-    # returns a MultiIndex instance operating on a MultiReader
-    def multi_index(model_classes)
-      model_classes.sort! { |a, b| a.name <=> b.name }
-      key = model_classes.inject("") { |s, clazz| s + clazz.name }
-      multi_config = aaf_configuration[:ferret].dup
-      multi_config.delete :default_field  # we don't want the default field list of *this* class for multi_searching
-      ActsAsFerret::multi_indexes[key] ||= MultiIndex.new(model_classes, multi_config)
-    end
- 
-    def close_multi_indexes
-      # close combined index readers, just in case
-      # this seems to fix a strange test failure that seems to relate to a
-      # multi_index looking at an old version of the content_base index.
-      ActsAsFerret::multi_indexes.each_pair do |key, index|
-        # puts "#{key} -- #{self.name}"
-        # TODO only close those where necessary (watch inheritance, where
-        # self.name is base class of a class where key is made from)
-        index.close #if key =~ /#{self.name}/
-      end
-      ActsAsFerret::multi_indexes.clear
-    end
-
-    def reindex_model(index, model = aaf_configuration[:class_name].constantize)
-      # index in batches of 1000 to limit memory consumption (fixes #24)
-      # TODO make configurable through options
-      batch_size = 1000
-      model_count = model.count.to_f
-      work_done = 0
-      batch_time = 0
-      logger.info "reindexing model #{model.name}"
-      order = "#{model.primary_key} ASC" # this works around a bug in sqlserver-adapter (where paging only works with an order applied)
-      model.transaction do
-        0.step(model.count, batch_size) do |i|
-          b1 = Time.now.to_f
-          model.find(:all, :limit => batch_size, :offset => i, :order => order).each do |rec|
-            index << rec.to_doc if rec.ferret_enabled?(true)
-          end
-          batch_time = Time.now.to_f - b1
-          work_done = i.to_f / model_count * 100.0 if model_count > 0
-          remaining_time = ( batch_time / batch_size ) * ( model_count - i + batch_size )
-          logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
-        end
-      end
-    end
-
-    # builds a FieldInfos instance for creation of an index containing fields
-    # for the given model classes.
@@ -253 +219 @@
-    end
-
+    protected
+
+    # returns a MultiIndex instance operating on a MultiReader
+    def multi_index(model_classes)
+      model_classes.sort! { |a, b| a.name <=> b.name }
+      key = model_classes.inject("") { |s, clazz| s + clazz.name }
+      multi_config = aaf_configuration[:ferret].dup
+      multi_config.delete :default_field  # we don't want the default field list of *this* class for multi_searching
+      ActsAsFerret::multi_indexes[key] ||= MultiIndex.new(model_classes, multi_config)
+    end
+ 
+    def close_multi_indexes
+      # close combined index readers, just in case
+      # this seems to fix a strange test failure that seems to relate to a
+      # multi_index looking at an old version of the content_base index.
+      ActsAsFerret::multi_indexes.each_pair do |key, index|
+        # puts "#{key} -- #{self.name}"
+        # TODO only close those where necessary (watch inheritance, where
+        # self.name is base class of a class where key is made from)
+        index.close #if key =~ /#{self.name}/
+      end
+      ActsAsFerret::multi_indexes.clear
+    end
+
+    # indexing is done in batches to limit memory consumption (fixes #24).
+    # The default batch size is 1000, this can be changed with the :reindex_batch_size
+    # option of acts_as_ferret.
+    def reindex_model(index, model = aaf_configuration[:class_name].constantize)
+      batch_size = aaf_configuration[:reindex_batch_size]
+      model_count = model.count.to_f
+      work_done = 0
+      batch_time = 0
+      logger.info "reindexing model #{model.name}"
+      order = "#{model.primary_key} ASC" # this works around a bug in sqlserver-adapter (where paging only works with an order applied)
+      model.transaction do
+        0.step(model.count, batch_size) do |i|
+          batch_time = measure_time {
+            model.find(:all, :limit => batch_size, :offset => i, :order => order).each do |rec|
+              index << rec.to_doc if rec.ferret_enabled?(true)
+            end
+          }.to_f
+          work_done = i.to_f / model_count * 100.0 if model_count > 0
+          remaining_time = ( batch_time / batch_size ) * ( model_count - i + batch_size )
+          logger.info "reindex model #{model.name} : #{'%.2f' % work_done}% complete : #{'%.2f' % remaining_time} secs to finish"
+        end
+      end
+    end
+
+    def measure_time
+      t1 = Time.now
+      yield
+      Time.now - t1
+    end
+
+
-  end
-

trunk/plugin/acts_as_ferret/script/ferret_start

@@ -1 @@
-
-
+
-# Ferret DRb server launcher script
-#
@@ -18 +17 @@
-# The server writes a log file in log/ferret_server.log, it's 
-# STDOUT gets redirected to log/ferret_server.out
+
+ENV['FERRET_USE_LOCAL_INDEX'] = 'true'
+require File.dirname(__FILE__) + '/../config/boot'
+require RAILS_ROOT + '/config/environment'
-
-

trunk/plugin/acts_as_ferret/script/ferret_stop

@@ -18 +18 @@
-pid_file = config['pid_file']
-puts "Stopping ferret_server..."
-
+
+
+
+
+
-
-# vim:set filetype=ruby: