import planet sources

Gemfile

@@ -21,7 +21,13 @@ gem 'turbolinks'
 gem 'redcarpet', '~> 3.2'
 gem 'paperclip', '~> 4.1'
 
+gem 'feedjira', github: 'feedjira/feedjira'
+gem 'trollop', '~> 2.0'
+
 group :development, :test do
+  gem 'pry'
+  gem 'awesome_print'
+
   gem 'rspec-rails', '~> 3.0'
   gem 'capybara', '~> 2.4.1'
 

Gemfile.lock

@@ -1,3 +1,12 @@
+GIT
+  remote: git://github.com/feedjira/feedjira.git
+  revision: a325b19948cfc3494205abf10665b0f2769230ad
+  specs:
+    feedjira (1.5.0)
+      curb (~> 0.8)
+      loofah (~> 2.0)
+      sax-machine (~> 1.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -28,6 +37,7 @@ GEM
       thread_safe (~> 0.1)
       tzinfo (~> 1.1)
     arel (5.0.1.20140414130214)
+    awesome_print (1.2.0)
     bcrypt (3.1.9)
     bootstrap-sass (3.2.0.2)
       sass (~> 3.2)
@@ -42,13 +52,15 @@ GEM
       activesupport (>= 3.0)
     cocaine (0.5.5)
       climate_control (>= 0.0.3, < 1.0)
+    coderay (1.1.0)
     coffee-rails (4.1.0)
       coffee-script (>= 2.2.0)
       railties (>= 4.0.0, < 5.0)
     coffee-script (2.3.0)
       coffee-script-source
       execjs
     coffee-script-source (1.8.0)
+    curb (0.8.6)
     devise (3.4.1)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -79,8 +91,11 @@ GEM
       actionpack (>= 3.0.0)
       activesupport (>= 3.0.0)
     kgio (2.9.2)
+    loofah (2.0.1)
+      nokogiri (>= 1.5.9)
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
+    method_source (0.8.2)
     mime-types (2.4.3)
     mini_portile (0.6.1)
     minitest (5.4.3)
@@ -95,6 +110,10 @@ GEM
       activesupport (>= 3.0.0)
       cocaine (~> 0.5.3)
       mime-types
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
     rack (1.5.2)
     rack-pjax (0.8.0)
       nokogiri (~> 1.5)
@@ -169,6 +188,8 @@ GEM
       sass (~> 3.2.2)
       sprockets (~> 2.8, < 3.0)
       sprockets-rails (~> 2.0)
+    sax-machine (1.1.1)
+    slop (3.6.0)
     sprockets (2.12.3)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -181,6 +202,7 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.4)
     tilt (1.4.1)
+    trollop (2.0)
     turbolinks (2.5.3)
       coffee-rails
     tzinfo (1.2.2)
@@ -201,21 +223,25 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  awesome_print
   bootstrap-sass (~> 3.2.0)
   capybara (~> 2.4.1)
   coffee-rails (~> 4.0)
   devise (~> 3.4.0)
+  feedjira!
   fuubar (~> 2.0.0)
   jquery-rails
   mysql2
   paperclip (~> 4.1)
+  pry
   rails (~> 4.1.8)
   rails_admin
   rails_locale_detection (~> 2.0.0.pre)
   redcarpet (~> 3.2)
   rspec-instafail (~> 0.2.4)
   rspec-rails (~> 3.0)
   sass-rails (~> 4.0.0)
+  trollop (~> 2.0)
   turbolinks
   uglifier (>= 1.3.0)
   unicorn

bin/planet

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require_relative '../config/environment'
+Planet::CLI.new

config/application.rb

@@ -26,5 +26,7 @@ class Application < Rails::Application
     # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     # config.i18n.default_locale = :de
+
+    config.autoload_paths << Rails.root.join('lib')
   end
 end

config/initializers/planet.rb

@@ -0,0 +1,5 @@
+Planet.configure do |config|
+  config.interval = 10
+  config.log_path = STDOUT
+  config.log_level = :debug
+end

db/migrate/20141210000000_create_planet_tables.rb

@@ -0,0 +1,21 @@
+class CreatePlanetTables < ActiveRecord::Migration
+  def change
+    create_table :planet_feeds do |t|
+      t.string :url, limit: 800
+      t.string :site_url, limit: 800
+      t.string :title
+      t.timestamps
+    end
+
+    create_table :planet_entries do |t|
+      t.string :title
+      t.string :author
+      t.string :url, limit: 800
+      t.string :entry_id, limit: 800
+      t.text :body
+      t.datetime :published_at
+      t.references :feed
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20140602222432) do
+ActiveRecord::Schema.define(version: 20141210000000) do
 
   create_table "assets", force: true do |t|
     t.string   "title"
@@ -30,8 +30,28 @@
     t.string   "title",                      null: false
     t.boolean  "published",  default: false
     t.text     "content",                    null: false
-    t.datetime "created_at",                 null: false
-    t.datetime "updated_at",                 null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "planet_entries", force: true do |t|
+    t.string   "title"
+    t.string   "author"
+    t.string   "url",          limit: 800
+    t.string   "entry_id",     limit: 800
+    t.text     "body"
+    t.datetime "published_at"
+    t.integer  "feed_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "planet_feeds", force: true do |t|
+    t.string   "url",        limit: 800
+    t.string   "site_url",   limit: 800
+    t.string   "title"
+    t.datetime "created_at"
+    t.datetime "updated_at"
   end
 
   create_table "rails_admin_histories", force: true do |t|
@@ -41,8 +61,8 @@
     t.string   "table"
     t.integer  "month",      limit: 2
     t.integer  "year",       limit: 8
-    t.datetime "created_at",           null: false
-    t.datetime "updated_at",           null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
   end
 
   add_index "rails_admin_histories", ["item", "table", "month", "year"], name: "index_rails_admin_histories", using: :btree
@@ -58,8 +78,8 @@
     t.datetime "last_sign_in_at"
     t.string   "current_sign_in_ip"
     t.string   "last_sign_in_ip"
-    t.datetime "created_at",                          null: false
-    t.datetime "updated_at",                          null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
   end
 
   add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree

db/seeds.rb

@@ -7,3 +7,7 @@
 #   Mayor.create(name: 'Emanuel', city: cities.first)
 
 Page.create(title: 'Home', content: 'home', published: true, url: 'home')
+
+[['Diaspora blog', 'http://blog.diasporafoundation.org/feed/atom']].each do |title, url|
+  Planet::Models::Feed.create! title: title, url: url
+end

lib/planet.rb

@@ -0,0 +1,28 @@
+require 'logger'
+
+module Planet
+  require 'planet/cli'
+  require 'planet/config'
+  require 'planet/models'
+  require 'planet/feed_container'
+  require 'planet/fetcher'
+
+  class << self
+    def config
+      (@config ||= Config.new).tap {|config|
+        yield config if block_given?
+      }
+    end
+    alias_method :configure, :config
+
+    def logger
+      (@logger ||= Logger.new config.log_path).tap {|logger|
+        logger.level = config.log_level
+      }
+    end
+  end
+
+  def self.table_name_prefix
+    'planet_'
+  end
+end

lib/planet/cli.rb

@@ -0,0 +1,39 @@
+require 'trollop'
+
+module Planet
+  class CLI
+    def initialize
+      parse_options
+      run
+    end
+
+    private
+
+    def parse_options
+      @options = Trollop.options do
+        banner "Planet fetcher, updates the feeds"
+        opt :interval, "Dispatch update every n minutes", type: :integer
+        opt :logfile, "Path to logfile", type: :string
+        opt :verbosity, "Verbosity (debug, info, warn, error)", type: :string
+      end
+    end
+
+    def run
+      Planet.logger.info "Entering mainloop"
+      loop do
+        fetch_new_entries
+        Planet.logger.info { "Sleeping for #{Planet.config.interval} minutes" }
+        sleep Planet.config.interval*60
+      end
+    end
+
+    def fetch_new_entries
+      Planet.logger.info "Fetching new entries"
+      fetcher.run
+    end
+
+    def fetcher
+      @fetcher ||= Fetcher.new
+    end
+  end
+end

lib/planet/config.rb

@@ -0,0 +1,28 @@
+require 'logger'
+
+module Planet
+  class Config
+    attr_accessor :interval
+    attr_accessor :log_path
+    attr_writer :log_level
+
+    alias_method :configure, :tap
+
+    def initialize
+      self.interval = 10
+      self.log_level = :info
+      self.log_path = STDOUT
+    end
+
+    def log_level
+      @log_level = @log_level.to_sym if @log_level.is_a? String
+      case @log_level
+      when Symbol
+        {debug: Logger::DEBUG, error: Logger::ERROR, fatal: Logger::FATAL,
+         info: Logger::INFO, warn: Logger::WARN}[@log_level]
+      when Fixnum
+        @log_level
+      end
+    end
+  end
+end

lib/planet/feed_container.rb

@@ -0,0 +1,3 @@
+module Planet
+  FeedContainer = Struct.new(:feed, :model)
+end