add cookies support

lib/wombat/crawler.rb

@@ -17,7 +17,7 @@ class << self
       self.metadata = DSL::Metadata.new
     end
 
-    def crawl(url = nil, &block)
+    def crawl(url = nil, options = {}, &block)
       if block
         @metadata_dup = self.class.send(:metadata).clone
         instance_eval do
@@ -27,14 +27,14 @@ def method_missing method, *args, &block
           end
         end
         self.instance_eval &block
-        parsed = parse(@metadata_dup, url)
+        parsed = parse(@metadata_dup, url, options)
         instance_eval do
           alias :method_missing :old_method_missing
           remove_instance_variable :@metadata_dup
         end
         parsed
       else
-        parse(self.class.send(:metadata), url)
+        parse(self.class.send(:metadata), url, options)
       end
     end
 

lib/wombat/processing/parser.rb

@@ -3,6 +3,7 @@
 require 'wombat/processing/node_selector'
 require 'mechanize'
 require 'restclient'
+require 'uri'
 
 module Nokogiri
   module XML
@@ -40,30 +41,39 @@ def initialize
         @mechanize.user_agent_alias = Wombat.user_agent_alias if Wombat.user_agent_alias
       end
 
-      def parse(metadata, url=nil)
-        @context = parser_for(metadata, url)
-
+      def parse(metadata, url = nil, options = {})
+        unless options.empty?
+          options = options.each_with_object({}) do |(k, v), memo|
+            memo[k.to_sym] = v.to_s unless v.nil?
+          end
+        end
+        @context = parser_for(metadata, url, options)
+        return nil if @context.nil?
         Wombat::Property::Locators::Factory.locator_for(metadata).locate(@context, @mechanize)
       end
 
       private
-      def parser_for(metadata, url)
+
+      def parser_for(metadata, url, options = {})
         url ||= "#{metadata[:base_url]}#{metadata[:path]}"
-        page = nil
-        parser = nil
-        _method = method_from(metadata[:http_method])
+        return if url.nil?
+        method = method_from(metadata[:http_method])
         data = metadata[:data]
         args = [url, data].compact
         begin
           @page = metadata[:page]
 
           if metadata[:document_format] == :html
-            @page = @mechanize.public_send(_method, *args) unless @page
+            @mechanize.set_proxy(*options[:proxy_args]) if options[:proxy_args]
+            @mechanize.user_agent = options[:user_agent] if options[:user_agent]
+            @mechanize.user_agent_alias = options[:user_agent_alias] if options[:user_agent_alias]
+            update_cookies(url, options[:cookies]) if options[:cookies]
+            @page = @mechanize.public_send(method, *args) unless @page
             parser = @page.parser         # Nokogiri::HTML::Document
             parser.mechanize_page = @page # Mechanize::Page
             parser.headers = @page.header
           else
-            @page = RestClient.public_send(_method, *args) unless @page
+            @page = RestClient.public_send(method, *args) unless @page
             parser = Nokogiri::XML @page
             parser.headers = @page.headers
           end
@@ -79,9 +89,19 @@ def parser_for(metadata, url)
         end
       end
 
-      def method_from(_method)
-        return :get if _method.nil?
-        HTTP_METHODS.detect(->{:get}){ |i| i == _method.downcase.to_sym }
+      def update_cookies(url, cookies)
+        domain = URI.parse(url).host
+        cookies.each do |k, v|
+          cookie = Mechanize::Cookie.new(k.to_s, v.to_s)
+          cookie.domain = domain
+          cookie.path = '/'
+          @mechanize.cookie_jar << cookie
+        end
+      end
+
+      def method_from(method)
+        return :get if method.nil?
+        HTTP_METHODS.detect(-> {:get}){ |i| i == method.downcase.to_sym }
       end
     end
   end

spec/crawler_spec.rb

@@ -134,14 +134,15 @@
     it 'should crawl with url and block' do
       url = 'http://danielinc.com/itens'
 
-      expect(@crawler_instance).to receive(:parse).with(anything, url)
-      @crawler_instance.crawl(url) do
+      opts = {cookies: :cookies}
+      expect(@crawler_instance).to receive(:parse).with(anything, url, opts)
+      @crawler_instance.crawl(url, opts) do
       end
 
       another_instance = @crawler.new
-      expect(another_instance).to receive(:parse).with(anything, url)
+      expect(another_instance).to receive(:parse).with(anything, url, opts)
 
-      another_instance.crawl(url)
+      another_instance.crawl(url, opts)
     end
 
     it 'should remove created method missing' do