Changeset 162

Show
Ignore:
Timestamp:
04/16/08 04:49:48 (7 months ago)
Author:
eric.dumin..@gmail.com
Message:

index creation is multithreaded, index update is still monothreaded.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • branches/oo_indexer/lib/picolena/templates/app/models/indexer.rb

    r161 r162  
    33  @@exclude          = /(Thumbs\.db)/ 
    44  # Number of threads that will be used during indexing process 
    5   @@max_threads_number = 5 
     5  @@max_threads_number = 8 
    66   
    77  class << self 
    …… 
    1919    def index_every_directory(update=true) 
    2020      log :debug => "Indexing every directory" 
    21  
    22  
     21       
     22       
    2323      start=Time.now 
    2424      @update = update 
    …… 
    4444     
    4545    def index_directory_with_multithreads(dir) 
    46       log :debug => "Indexing #{dir}, #{@@max_threads_number} threads" 
     46      # FIXME: Don't know why, but if more than one thread is created while update the index, 
     47      # indexer raises: 
     48      # 
     49      # current thread not owner 
     50      # /usr/lib/ruby/1.8/monitor.rb:278:in `mon_check_owner' 
     51      # /home/www/picolena/lib/picolena/templates/lib/core_exts.rb:32:in `join' 
     52      # ... 
     53      # 
     54      # So Index creation is multithreaded, Index update is monothreaded. 
     55      threads_number = @update ? 1 : @@max_threads_number 
     56      log :debug => "Indexing #{dir}, #{threads_number} thread(s)" 
    4757       
    4858      indexing_list=Dir[File.join(dir,"**/*")].select{|filename| 
    …… 
    5262      # Cutting indexing_list in slices to avoid treating too big a list. 
    5363      # Migth raise a "stack level too deep" otherwise. 
    54       indexing_list.each_slice(100*@@max_threads_number){|indexing_list_chunk| 
    55         log :debug => "NEW CHUNK!!!!!!!!!!" 
    56         @indexing_list_chunk=indexing_list_chunk 
    57         @@max_threads_number.threads{launch_indexing_chain(@indexing_list_chunk)} 
     64      indexing_list_chunks=indexing_list.in_transposed_chunks(threads_number) 
     65       
     66      indexing_list_chunks.each_with_thread{|chunk| 
     67        chunk.each{|filename| 
     68          add_or_update_file(filename) 
     69        } 
    5870      } 
    5971    end 
    60  
     72     
    6173    def add_or_update_file(complete_path) 
    6274      should_be_added = true 
    …… 
    122134    private 
    123135     
    124     def launch_indexing_chain(indexing_list) 
    125       return if indexing_list.empty? 
    126       add_or_update_file(indexing_list.shift) 
    127       launch_indexing_chain(indexing_list) 
    128     end 
    129      
    130136    def log(hash) 
    131137      hash.each{|level,message| 

  • branches/oo_indexer/lib/picolena/templates/lib/core_exts.rb

    r155 r162  
    2323end 
    2424 
    25 class Fixnum 
    26   def threads(&block) 
    27     tds=(1..self).collect{ 
    28       Thread.new { 
    29         block.call 
     25module Enumerable 
     26  def each_with_thread(&block) 
     27    tds=self.collect{|elem| 
     28      Thread.new(elem) {|elem| 
     29        block.call(elem) 
    3030      } 
    3131    } 
    3232    tds.each{|aThread| aThread.join} 
     33  end 
     34end 
     35 
     36class Array 
     37  def in_transposed_chunks(n) 
     38    s=self.size 
     39    i=n-s%n 
     40    (self+[nil]*i).enum_slice(n).to_a.transpose.collect{|e| e.compact} 
    3341  end 
    3442end