To edit pages or tickets please login with username/password: aaf/aaf

Changeset 327

Show
Ignore:
Timestamp:
02/18/08 20:36:07 (8 months ago)
Author:
jk
Message:

rdig fixes

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/plugin/acts_as_ferret/lib/act_methods.rb

    r325 r327  
    6666 
    6767      if options[:rdig] 
     68        cattr_accessor :rdig_configuration 
     69        self.rdig_configuration = options[:rdig] 
    6870        require 'rdig_adapter' 
    6971        include ActsAsFerret::RdigAdapter 
    …… 
    8991 
    9092      index = ActsAsFerret::register_class_with_index(self, index_name, options) 
    91       self.aaf_configuration = index.index_definition 
     93      self.aaf_configuration = index.index_definition.dup 
    9294      logger.debug "configured index for class #{self.name}:\n#{aaf_configuration.inspect}" 
    9395 
    …… 
    117119      method_name = "#{field}_to_ferret" 
    118120      return if instance_methods.include?(method_name) # already defined 
     121      aaf_configuration[:defined_fields] ||= {} 
     122      aaf_configuration[:defined_fields][field] = options 
    119123      dynamic_boost = options[:boost] if options[:boost].is_a?(Symbol) 
    120124      via = options[:via] || field 

  • trunk/plugin/acts_as_ferret/lib/ferret_find_methods.rb

    r323 r327  
    1818      rank   = 0 
    1919      total_hits = find_ids(q, options) do |model, id, score, data| 
    20         #logger.debug "model: #{model}, id: #{id}, data: #{data}" 
     20        logger.debug "model: #{model}, id: #{id}, data: #{data}" 
    2121        result << FerretResult.new(model, id, score, rank += 1, data) 
    2222      end 

  • trunk/plugin/acts_as_ferret/lib/ferret_result.rb

    r317 r327  
    2929 
    3030    def method_missing(method, *args, &block) 
    31       # don't try to fetch attributes from RDig based records 
    3231      if (@ar_record && @use_record) || !@data.has_key?(method) 
    33         RAILS_DEFAULT_LOGGER.debug "miss for key: #{method.inspect} in #{@data.keys.inspect}" 
    3432        to_record.send method, *args, &block 
    3533      else 
    …… 
    4745        @ar_record.ferret_rank  = ferret_rank 
    4846        @ar_record.ferret_score = ferret_score 
    49         @use_record = !@ar_record.class.included_modules.include?(ActsAsFerret::RdigAdapter) 
     47        # don't try to fetch attributes from RDig based records 
     48        @use_record = !@ar_record.class.included_modules.include?(ActsAsFerret::RdigAdapter::InstanceMethods) 
    5049      end 
    5150      @ar_record 

  • trunk/plugin/acts_as_ferret/lib/instance_methods.rb

    r324 r327  
    2929    #                    character. 
    3030    def highlight(query, options = {}) 
    31       self.class.aaf_index.highlight(id, self.class.name, query, options) 
     31      self.class.aaf_index.highlight(self.send(self.class.primary_key), self.class.name, query, options) 
    3232    end 
    3333     
    …… 
    128128       
    129129        # iterate through the fields and add them to the document 
    130         aaf_configuration[:ferret_fields].each_pair do |field, config| 
     130        aaf_configuration[:defined_fields].each_pair do |field, config| 
    131131          doc[field] = self.send("#{field}_to_ferret") unless config[:ignore] 
    132132        end 

  • trunk/plugin/acts_as_ferret/lib/local_index.rb

    r323 r327  
    105105    # highlight search terms for the record with the given id. 
    106106    def highlight(id, class_name, query, options = {}) 
     107      logger.debug("highlight: #{class_name} / #{id} query: #{query}") 
    107108      options.reverse_merge! :num_excerpts => 2, :pre_tag => '<em>', :post_tag => '</em>' 
    108109      highlights = [] 
    109110      ferret_index.synchronize do 
    110111        doc_num = document_number(id, class_name) 
     112 
    111113        if options[:field] 
    112114          highlights << ferret_index.highlight(query, doc_num, options) 
    …… 
    161163      logger.debug "extracting stored fields #{stored_fields.inspect} from document #{doc[:class_name]} / #{doc[:id]}" 
    162164      stored_fields.each do |field| 
    163         logger.debug field 
    164165        if field_cfg = fields[field] 
    165166          data[field_cfg[:via]] = doc[field] 
    166           logger.debug field_cfg.inspect 
    167           logger.debug "#{field} =>>>>> #{doc[field]}" 
    168167        end 
    169168      end if stored_fields  
    170       logger.debug "done." 
    171       data  
     169      logger.debug "done: #{data.inspect}" 
     170      return data  
    172171    end 
    173172 

  • trunk/plugin/acts_as_ferret/lib/rdig_adapter.rb

    r317 r327  
    4646        protected 
    4747        def process_batch 
    48           RAILS_DEFAULT_LOGGER.info "RdigAdapter::Indexer#process_batch: #{@documents.size} docs in queue, offset #{@offset}" 
     48          ActsAsFerret::logger.info "RdigAdapter::Indexer#process_batch: #{@documents.size} docs in queue, offset #{@offset}" 
    4949          @block.call @documents, @offset 
    5050          @offset += @documents.size 
    …… 
    5656        # overriding aaf to return the documents fetched via RDig 
    5757        def records_for_rebuild(batch_size = 1000, &block) 
    58           crawler = RDig::Crawler.new rdig_config, logger 
    5958          indexer = Indexer.new(batch_size, self, &block) 
    60           crawler.instance_variable_set '@indexer', indexer 
    61           crawler.crawl 
     59          configure_rdig do 
     60            crawler = RDig::Crawler.new RDig.configuration, logger 
     61            crawler.instance_variable_set '@indexer', indexer 
     62            crawler.crawl 
     63          end 
     64        rescue => e 
     65          ActsAsFerret::logger.error e 
     66          ActsAsFerret::logger.debug e.backtrace.join("\n") 
    6267        ensure 
    63           indexer.close 
     68          indexer.close if indexer 
    6469        end 
    6570 
    …… 
    7075        end 
    7176 
    72         def rdig_config 
    73           cfg = RDig.configuration.dup 
    74           cfg.index = nil 
    75           aaf_configuration[:rdig][:crawler].each { |k,v| cfg.crawler.send :"#{k}=", v } if aaf_configuration[:rdig][:crawler] 
    76           if aaf_configuration[:rdig][:content_extraction] 
    77             cfg.content_extraction = OpenStruct.new( :hpricot => OpenStruct.new( aaf_configuration[:rdig][:content_extraction] ) ) 
     77        # unfortunately need to modify global RDig.configuration because it's 
     78        # used everywhere in RDig 
     79        def configure_rdig 
     80          # back up original config 
     81          old_cfg = RDig.configuration.dup 
     82          rdig_configuration[:crawler].each { |k,v| RDig.configuration.crawler.send :"#{k}=", v } if rdig_configuration[:crawler] 
     83          if ce_config = rdig_configuration[:content_extraction] 
     84            RDig.configuration.content_extraction = OpenStruct.new( :hpricot => OpenStruct.new( ce_config ) ) 
    7885          end 
    79           cfg 
     86          yield 
     87        ensure 
     88          # restore original config 
     89          RDig.configuration.crawler = old_cfg.crawler 
     90          RDig.configuration.content_extraction = old_cfg.content_extraction 
    8091        end 
    8192 

  • trunk/plugin/acts_as_ferret/lib/without_ar.rb

    r317 r327  
    99    def self.included(target) 
    1010      target.extend ClassMethods 
    11       target.extend ActMethods 
     11      target.extend ActsAsFerret::ActMethods 
    1212      target.send :include, InstanceMethods 
    1313    end 
    …… 
    3333      end 
    3434      def find_for_id(id) 
    35         raise NotImplementedError.new("implement this method in your model class") 
     35        raise NotImplementedError.new("implement find_for_id in class #{self.name}") 
    3636      end 
    3737      def count 

  • trunk/plugin/acts_as_ferret/recipes/aaf_recipes.rb

    r316 r327  
    1414# into current/ when you deploy. 
    1515# 
    16 # In order to use the ferret:index:rebuild task, declare the models you intend to 
    17 # index in config/deploy.rb: 
     16# In order to use the ferret:index:rebuild task, declare the indexes you intend to 
     17# rebuild remotely in config/deploy.rb: 
    1818# 
    19 # for a shared index, do: 
    20 # set :ferret_single_index_models, [ Model, AnotherModel, YetAnotherModel ] 
    21 # This will call Model.rebuild_index( AnotherModel, YetAnotherModel ) 
    22 # 
    23 # for models indexed separately, specify: 
    24 # set :ferret_models, [ Model, AnotherModel ] 
    25 # This will call Model.rebuild_index and AnotherModel.rebuild_index separately. 
    26 # 
    27 # The two methods may be combined if you have a shared index, and some models 
    28 # indexed separately.  
    29 # 
    30 # Like to submit a patch to aaf? Automatically determining the models that use  
    31 # acts_as_ferret so the declaration of these variables in deploy.rb isn't  
    32 # necessary anymore would be really cool ;-) 
    33 # 
     19# set :ferret_indexes, %w( model another_model shared ) 
    3420# 
    3521# HINT: To be very sure that your DRb server and application are always using 
    …… 
    4430# versions of model classes (which might happen otherwise) 
    4531# Downside: Your downtime is a bit longer than with the usual deploy, so be sure to 
    46 # put up some maintenance page for the meantime. 
     32# put up some maintenance page for the meantime. Obviously this won't work if 
     33# your migrations need acts_as_ferret (i.e. if you update model instances which 
     34# would lead to index updates). In this case bring up the DRb server before 
     35# running your migrations: 
     36# 
     37# cap deploy:stop deploy:update ferret:start deploy:migrate ferret:stop deploy:start 
     38# 
     39# Chances are that you're still not safe if your migrations not only modify the index,  
     40# but also change the structure of your models. So just don't do both things in 
     41# one go - I can't think of an easy way to handle this case automatically. 
     42# Suggestions and patches are of course very welcome :-) 
    4743 
    4844namespace :ferret do 
    …… 
    7066    task :rebuild, :roles => :app do 
    7167      rake = fetch(:rake, 'rake') 
    72       single_index_models = fetch(:ferret_single_index_models, nil) 
    73       if single_index_models 
    74         run "cd #{current_path}; RAILS_ENV=#{rails_env} MODEL='#{ferret_single_index_models.join(' ')}' #{rake} ferret:rebuild" 
    75       end 
    76       fetch(:ferret_models, []).each do |m| 
    77         run "cd #{current_path}; RAILS_ENV=#{rails_env} MODEL='#{m}' #{rake} ferret:rebuild" 
     68      indexes = fetch(:ferret_indexes, nil) 
     69      if indexes and indexes.any? 
     70        run "cd #{current_path}; RAILS_ENV=#{rails_env} INDEXES='#{indexes.join(' ')}' #{rake} ferret:rebuild" 
    7871      end 
    7972    end 

  • trunk/plugin/acts_as_ferret/tasks/ferret.rake

    r302 r327  
    11namespace :ferret do 
    22 
    3   # for a shared index, declare all models that should go into this index separated by 
    4   # space: MODEL="MyModel AnotherModel" 
     3  # Rebuild index task. Declare the indexes to be rebuilt with the INDEXES 
     4  # environment variable: 
     5  # 
     6  # INDEXES="my_model shared" rake ferret:rebuild 
    57  desc "Rebuild a Ferret index. Specify what model to rebuild with the MODEL environment variable." 
    68  task :rebuild do 
    79    require File.join(RAILS_ROOT, 'config', 'environment') 
    810 
    9     models = ENV['MODEL'].split.map(&:constantize) 
    10  
    11     start = 1.minute.ago 
    12     models.first.rebuild_index( *models ) 
    13  
    14     # update records that have changed since the rebuild started 
    15     models.each do |m| 
    16       m.records_modified_since(start).each do |object| 
    17         object.ferret_update 
     11    indexes = ENV['INDEXES'].split 
     12    indexes.each do |index_name| 
     13      start = 1.minute.ago 
     14      ActsAsFerret::rebuild_index index_name 
     15      idx = ActsAsFerret::get_index index_name 
     16      # update records that have changed since the rebuild started 
     17      idx.index_definition[:registered_models].each do |m| 
     18        m.records_modified_since(start).each do |object| 
     19          object.ferret_update 
     20        end 
    1821      end 
    1922    end 

To edit pages or tickets please login with username/password: aaf/aaf