require 'rubygems'
require 'scrapi'
require 'open-uri'

scraper = (Scraper.define do
   process_first "span.metaspan", :node=>:element
   result :node
end).scrape(open("http://www.quarkruby.com"){|f| f.read})
# scraper.class
=> "HTML::Tag"

#test if the node is a "span" tag
  node.match :tag => "span"

#test if the node's parent is a "div"
  node.match :parent => { :tag => "div" }
#test if any of the node's ancestors are "table" tags
  node.match :ancestor => { :tag => "table" }

#test if any of the node's immediate children are "em" tags
  node.match :child => { :tag => "em" }
  
#test if any of the node's descendants are "strong" tags
  node.match :descendant => { :tag => "strong" }

#test if the node has between 2 and 4 span tags as immediate children
  node.match :children => { :count => 2..4, :only => { :tag => "span" } }

#get funky: test to see if the node is a "div", has a "ul" ancestor
#and an "li" parent (with "class" = "enum"), and whether or not it has
#a "span" descendant that contains # text matching /hello world/:
  node.match :tag => "div",
             :ancestor => { :tag => "ul" },
             :parent => { :tag => "li",
             :attributes => { :class => "enum" } },
             :descendant => { :tag => "span",
             :child => /hello world/ }

# Sometimes, you can get a HTML::Text node in return
# if its a text node, following are ways of using match on that node.
  node.match (conditions)
# * if +conditions+ is a string, it must be a substring of the node's  content
# * if +conditions+ is a regular expression, it must match the node's content
# * if +conditions+ is a hash, it must contain a <tt>:content</tt> key that
#   is either a string or a regexp, and which is interpreted as described above.

module HTML
   class Tag < Node
      def text
         return @content if respond_to? :content
         data=""
         0.upto(children.size-1) {|i|
             next if children[i].respond_to?:name and children[i].name=="script"
             data += children[i].text
         }
         return data
      end
     
      def inner_html 
         data = ""
         children.each {|e| data += e.to_s}
         data
       end
   end
end

require 'rubygems'
require 'scrapi'
require 'open-uri'

class QScraper
  attr_accessor :document
  attr_accessor :rule_list

  # source can be a node or file
  # Constructor for this class ..
  # Takes a source (file or html::node) as argument.
  # Example : qScraper = QScraper.new("index.html")
  def initialize source, local_file=true
    case source
      when URI, HTML::Node
        @document = source
      when String
        @document = (local_file && File.exists?(source)) ?  File.read(source) : source
      else
        raise ArgumentError, "Can only scrape URI, String or HTML::Node"
    end
  end

  # search string selector in given source and 
  # returns all the matching elements as HTML::Tag object
  # Selector is a css selector 
  # Example:
  # qScraper.search("div.highlight")
  def search selector
    output = (Scraper.define do
      array :elems
      process selector, :elems =>:element
      result :elems
    end).scrape(@document)
    output
  end

  # search the first node matching this selector
  def search_first selector
    output = (Scraper.define do
      process_first selector, :elem=>:element
      result :elem
    end).scrape(@document)
    output
  end

  # Adds a rule for parsing, you can add many rules and 
  # then parse them by making a call to parse_rules
  # Example:
  # q.add_rule({"br#lgpd"=>{:n1ame=>"@id"}})
  def add_rule selector, extractors
    @rule_list ||= {}
    @rule_list.merge!({selector=>extractors})
  end

  # Example:
  # q.add_rules({"table.searchResults2>tbody>tr>td:nth-of-type(3)>a"=>
  #                               {:href=>"@href",:name=>:text},
  #              "table.searchResults2>tr>td:nth-of-type(3)>a.cat"=>
  #                               {:reviewcount=>:text}})
  def add_rules selector_extractors_hash
    @rule_list ||= {}
    @rule_list.merge! (selector_extractors_hash)
  end

  # parses the given rule set .. specified by add_rule methods.
  def parse_rules parser=nil
    result_string = "result "
    @rule_list.values.each do |j| 
        j.keys.each {|i| 
           result_string << ":" << i.to_s << ", "
        }
    end
    result_string = result_string[0..-3]
    array_string = result_string.sub(/result/,"array")
    create_process_string = ""
    @rule_list.keys.each {|i|
      create_process_string << "process \"" << i << "\", " 
      create_process_string << @rule_list[i].inspect.gsub(/[{}]/,"") << "\n"
    }
    res = String.module_eval %{Scraper.define do
      #{array_string}
      #{create_process_string}
      #{result_string}
    end}
    parser ||= :tidy
    f = res.scrape(@document,:parser=>parser)
  end
end

# Small extension to scrapi's HTML::Tag class 
# for "inner_html" method.
#
module HTML
  class Tag < Node
    def inner_html 
      data = ""
      children.each {|e| data += e.to_s}
      data
    end
  end
end

  # Example page to parse : http://www.google.co.in/search?q=scrapi
  doc = QScraper.new(open("http://www.google.co.in/search?q=scrapi"){|f| f.read})

  # Extracting titles elements of search results 
  # returned objects are of HTML::Tag type
  result = doc.search("a.l")

  # now extracting title and link for each of the search results.
  doc.add_rules({"a.l"=>{:url=>"@href",:title=>:text}})
  result = doc.parse_rules
  #=> result.url returns an array of 10 url links
  #=> result.title returns the array of titles