build_index (FastRI::FullTextIndexer)

    # File lib/fastri/full_text_indexer.rb, line 62
      def build_index(full_text_IO, suffix_array_IO)
        fulltext = ""
        io = StringIO.new(fulltext)
        io.write MAGIC
        full_text_IO.write MAGIC
        documents.each do |doc|
          data, metadata = @doc_hash[doc]
          io.write(data)
          full_text_IO.write(data)
          meta_txt = Marshal.dump(metadata)
          footer = "\0....#{doc}\0#{meta_txt}\0"
          footer[1,4] = [footer.size - 5].pack("V")
          io.write(footer)
          full_text_IO.write(footer)
        end

        scanner = StringScanner.new(fulltext)
        scanner.scan(Regexp.new(Regexp.escape(MAGIC)))

        count = 0
        suffixes = []
        until scanner.eos?
          count += 1
          start = scanner.pos
          text = scanner.scan_until(/\0/)
          suffixes.concat find_suffixes(text[0..-2], start)
          len = scanner.scan(/..../).unpack("V")[0]
          #puts "LEN: #{len}  #{scanner.pos}  #{scanner.string.size}"
          #puts "#{scanner.string[scanner.pos,20].inspect}"
          scanner.pos += len
          #scanner.terminate if !text
        end
        sorted = suffixes.sort_by{|x| fulltext[x, @max_wordsize]}
        sorted.each_slice(10000){|x| suffix_array_IO.write x.pack("V*")}
        nil
      end