如何自己写一个网络爬虫#

取网页
for each 链接 in 当前网页所有的链接
{
        if(如果本链接是我们想要的 || 这个链接从未访问过)
        {
                处理对本链接
                把本链接设置为已访问
        }
}

require “rubygems”
require “mechanize”

class Crawler < WWW::Mechanize

  attr_accessor :callback
  INDEX = 0
  DOWNLOAD = 1
  PASS = 2

  def initialize
    super
    init
    @first = true
    self.user_agent_alias = “Windows IE 6″
  end

  def init
    @visited = []
  end

  def remember(link)
    @visited << link
  end

  def perform_index(link)
    self.get(link)
    if(self.page.class.to_s == “WWW::Mechanize::Page”)
      links = self.page.links.map {|link| link.href } - @visited
      links.each do |alink|
        start(alink)
      end
    end
  end

  def start(link)
    return if link.nil?
    if([email protected]?(link))
      action = @callback.call(link)
      if(@first)
        @first = false
        perform_index(link)
      end
      case action
        when INDEX
          perform_index(link)
        when DOWNLOAD
          self.get(link).save_as(File.basename(link))
        when PASS
          puts “passing on #{link}”
      end
    end
  end

  def get(site)
    begin
      puts “getting #{site}”
      @visited << site
      super(site)
    rescue
      puts “error getting #{site}”
    end
  end
end

require “crawler”

x = Crawler.new
callback = lambda do |link|
  if(link =~/\\.(zip|rar|gz|pdf|doc)
    x.remember(link)
    return Crawler::PASS
  elsif(link =~/\\.(jpg|jpeg)/)
    return Crawler::DOWNLOAD
  end
  return Crawler::INDEX;
end

x.callback = callback
x.start(”http://somesite.com”)