Skip to content

Commit

Permalink
added browser option to thor
Browse files Browse the repository at this point in the history
udpated database
  • Loading branch information
jahan-paisley committed Jul 3, 2016
1 parent 8c849d2 commit 5b2858f
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 13 deletions.
Binary file modified data/people_ads.db
Binary file not shown.
36 changes: 24 additions & 12 deletions lib/capybara_config.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,36 @@
require 'selenium-webdriver'

class CapybaraConfig
def self.init proxy=nil
def self.init proxy=nil, browser= "chrome"
Capybara.run_server = false
Capybara.app_host = 'http://www.rahnama.com'
Capybara.default_max_wait_time = 15

Capybara.register_driver :selenium_firefox_nojs do |app|
profile = Selenium::WebDriver::Firefox::Profile.new
profile['javascript.enabled'] = false
profile['permissions.default.image'] = 2
if proxy
profile["network.proxy.type"] = 1
profile["network.proxy.socks"] = proxy.split(':')[0]
profile["network.proxy.socks_port"] = proxy.split(':')[1].to_i
profile["network.proxy.socks_remote_dns"] = true
if browser == "firefox"
Capybara.register_driver :firefox do |app|
profile = Selenium::WebDriver::Chrome::Profile.new
# profile['javascript.enabled'] = false
# profile['permissions.default.image'] = 2
if proxy
profile["network.proxy.type"] = 1
profile["network.proxy.socks"] = proxy.split(':')[0]
profile["network.proxy.socks_port"] = proxy.split(':')[1].to_i
profile["network.proxy.socks_remote_dns"] = true
end
Capybara::Selenium::Driver.new(app, :browser => :firefox)
end
Capybara::Selenium::Driver.new(app, :browser => :firefox, :profile => profile)
Capybara.default_driver = :firefox
else
Capybara.register_driver :chrome do |app|
ENV['HTTP_PROXY'] = ENV['http_proxy'] = nil
options = Hash.new.tap do |opt|
opt[:browser] = :chrome
opt[:proxy] = proxy if proxy
end
Capybara::Selenium::Driver.new(app, options)
end
Capybara.default_driver = :chrome
end
Capybara.default_driver = :selenium_firefox_nojs
end
end

3 changes: 2 additions & 1 deletion rahnama.thor
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ class Rahnama < Thor

desc 'scrap_ads', 'Scrap the Rahnama.com Real Estate Ads based on provided links.txt'
option :proxy
option :browser

def scrap_ads
CapybaraConfig.init options[:proxy]
CapybaraConfig.init options[:proxy], options[:browser]
results= Scrapper.new.start
processor = RawAdProcessor.new results
processor.persist_ads
Expand Down

0 comments on commit 5b2858f

Please sign in to comment.