# File lib/hawlerhelper.rb, line 85 def self.harvest(uri, data) links = Set.new uri = valid_uri(uri) doc = Hpricot(data) tags = Hash[ *%w(a href link href img src script src frame src form action) ] tags.each do |k,v| (doc/"#{k}").each do |t| next if (t.attributes[v].nil? || t.attributes[v].empty?) l = valid_uri(t.attributes[v]) or next next if (offsite?(uri.merge(l), uri)) links << uri.merge(l) end end links.to_a end