Adding Research Outputs (DB and model components only) (#2739)

* sym linked necessary files and ignored others

* add research_outputs and mime_types

* add new mime_types Rake task to upgrade task

* reverted sym links

* reverted gitignore changes

* reverted blank lines added to bottom of a few configs

* reverted issue with open_aire config

* minor text change to comment in open_aire initializer to force rubocop fix

* Remove blank line added to open_aire initializer by Atom to fix rubocop

* add research_outputs and mime_types

add new mime_types Rake task to upgrade task

reverted sym links

reverted gitignore changes

reverted blank lines added to bottom of a few configs

reverted issue with open_aire config

minor text change to comment in open_aire initializer to force rubocop fix

Remove blank line added to open_aire initializer by Atom to fix rubocop

* updated schema date in schema.rb

* removed dupplicate copy of open_aire initializer

Author: briri

app/models/mime_type.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: mime_types
+#
+#  id          :bigint           not null, primary key
+#  category    :string           not null
+#  description :string           not null
+#  value       :string           not null
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
+#
+# Indexes
+#
+#  index_mime_types_on_value  (value)
+#
+class MimeType < ApplicationRecord
+
+  include ValidationMessages
+
+  # ================
+  # = Associations =
+  # ================
+
+  has_many :research_outputs
+
+  # ===============
+  # = Validations =
+  # ===============
+
+  validates :category, :description, :value, presence: { message: PRESENCE_MESSAGE }
+
+  # ==========
+  # = Scopes =
+  # ==========
+
+  # Retrieves the unique list of categories
+  scope :categories, -> { pluck(:category).uniq.sort { |a, b| a <=> b } }
+
+end

app/models/research_output.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: research_outputs
+#
+#  id                      :bigint           not null, primary key
+#  abbreviation            :string
+#  access                  :integer          default(0), not null
+#  byte_size               :bigint
+#  coverage_end            :datetime
+#  coverage_region         :string
+#  coverage_start          :datetime
+#  description             :text
+#  display_order           :integer
+#  is_default              :boolean         default("false")
+#  mandatory_attribution   :text
+#  output_type             :integer          default(3), not null
+#  output_type_description :string
+#  personal_data           :boolean
+#  release_date            :datetime
+#  sensitive_data          :boolean
+#  title                   :string           not null
+#  created_at              :datetime         not null
+#  updated_at              :datetime         not null
+#  mime_type_id            :integer
+#  plan_id                 :integer
+#
+# Indexes
+#
+#  index_research_outputs_on_output_type  (output_type)
+#  index_research_outputs_on_plan_id      (plan_id)
+#
+class ResearchOutput < ApplicationRecord
+
+  include Identifiable
+  include ValidationMessages
+
+  enum output_type: %i[audiovisual collection data_paper dataset event image
+                       interactive_resource model_representation physical_object
+                       service software sound text workflow other]
+
+  enum access: %i[open embargoed restricted closed]
+
+  # ================
+  # = Associations =
+  # ================
+
+  belongs_to :plan, optional: true
+  belongs_to :mime_type, optional: true
+
+  # ===============
+  # = Validations =
+  # ===============
+
+  validates_presence_of :output_type, :access, :title, message: PRESENCE_MESSAGE
+  validates_uniqueness_of :title, :abbreviation, scope: :plan_id
+
+  # Ensure presence of the :output_type_description if the user selected 'other'
+  validates_presence_of :output_type_description, if: -> { other? }, message: PRESENCE_MESSAGE
+  # Ensure that :coverage_start comes before :coverage_end
+  validate :end_date_after_start_date
+
+  # ====================
+  # = Instance methods =
+  # ====================
+
+  # :mime_type is only applicable for certain :output_types
+  # This method returns the applicable :mime_types
+  def available_mime_types
+    cat = %w[audio video] if audiovisual? || sound?
+    cat = %w[image] if image?
+    cat = %w[model] if model_representation?
+    cat = %w[text] if data_paper? || dataset? || text?
+
+    cat.present? ? MimeType.where(category: cat).order(:description) : []
+  end
+
+  # TODO: placeholders for once the License, Repository, Metadata Standard and
+  #       Resource Type Lookups feature is built.
+  #
+  #       Be sure to add the scheme in the appropriate upgrade task (and to the
+  #       seed.rb as well)
+  def licenses
+    # scheme = IdentifierScheme.find_by(name: '[name of license scheme]')
+    # return [] unless scheme.present?
+    # identifiers.select { |id| id.identifier_scheme = scheme }
+    []
+  end
+
+  def repositories
+    # scheme = IdentifierScheme.find_by(name: '[name of repository scheme]')
+    # return [] unless scheme.present?
+    # identifiers.select { |id| id.identifier_scheme = scheme }
+    []
+  end
+
+  def metadata_standards
+    # scheme = IdentifierScheme.find_by(name: '[name of openaire scheme]')
+    # return [] unless scheme.present?
+    # identifiers.select { |id| id.identifier_scheme = scheme }
+    []
+  end
+
+  def resource_types
+    # scheme = IdentifierScheme.find_by(name: '[name of resource_type scheme]')
+    # return [] unless scheme.present?
+    # identifiers.select { |id| id.identifier_scheme = scheme }
+    []
+  end
+
+  private
+
+  # Validation to prevent end date from coming before the start date
+  def end_date_after_start_date
+    # allow nil values
+    return true if coverage_end.blank? || coverage_start.blank?
+
+    errors.add(:coverage_end, _("must be after the start date")) if coverage_end < coverage_start
+  end
+
+end

bin/setup

@@ -17,17 +17,21 @@ chdir APP_ROOT do
   system! 'gem install bundler --conservative'
   system('bundle check') || system!('bundle install')
 
-  # Install JavaScript dependencies if using Yarn
-  # system('bin/yarn')
-
-  # puts "\n== Copying sample files =="
-  # unless File.exist?('config/database.yml')
-  #   cp 'config/database.yml.sample', 'config/database.yml'
-  # end
+  puts "\n== Copying sample files =="
+  %w[
+    config/database.yml.sample
+    config/initializers/wicked_pdf.rb.example
+  ].each do |file|
+    new_file .gsub('.sample', '').gsub('.example', '')
+    cp file, new_file unless File.exist?(new_file)
+  end
 
   puts "\n== Preparing database =="
   system! 'bin/rails db:setup'
 
+  puts "\n== Fetching mime types and adding to database =="
+  system! 'bin/rails mime_types:load'
+
   puts "\n== Removing old logs and tempfiles =="
   system! 'bin/rails log:clear tmp:clear'
 

config/initializers/external_apis/open_aire.rb

@@ -2,7 +2,7 @@
 
 # These configuration settings are used to communicate with the
 # Open Aire Research Project Registry API. For more information about
-# the API and to verify that your configuration settings are correct,
+# the API and to verify that your configuration settings are correct.
 Rails.configuration.x.open_aire.api_base_url = "https://api.openaire.eu/"
 # The api_url should contain `%s. This is where the funder is appended!
 Rails.configuration.x.open_aire.search_path = "projects/dspace/%s/ALL/ALL"

db/migrate/20201119201215_create_research_outputs.rb

@@ -0,0 +1,25 @@
+class CreateResearchOutputs < ActiveRecord::Migration[5.2]
+  def change
+    create_table :research_outputs do |t|
+      t.integer  :plan_id, index: true
+      t.integer  :output_type, null: false, index: true, default: 3
+      t.string   :output_type_description
+      t.string   :title, null: false
+      t.string   :abbreviation
+      t.integer  :display_order
+      t.boolean  :is_default
+      t.text     :description
+      t.integer  :mime_type_id
+      t.integer  :access, null: false, default: 0
+      t.datetime :release_date
+      t.boolean  :personal_data
+      t.boolean  :sensitive_data
+      t.bigint   :byte_size
+      t.text     :mandatory_attribution
+      t.datetime :coverage_start
+      t.datetime :coverage_end
+      t.string   :coverage_region
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20201119210343_create_mime_types.rb

@@ -0,0 +1,10 @@
+class CreateMimeTypes < ActiveRecord::Migration[5.2]
+  def change
+    create_table :mime_types do |t|
+      t.string :description, null: false
+      t.string :category, null: false
+      t.string :value, null: false, index: true
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_11_13_174910) do
+ActiveRecord::Schema.define(version: 2020_11_19_210343) do
 
   create_table "annotations", id: :integer, force: :cascade do |t|
     t.integer "question_id"
@@ -157,6 +157,15 @@
     t.boolean "default_language"
   end
 
+  create_table "mime_types", force: :cascade do |t|
+    t.string "description", null: false
+    t.string "category", null: false
+    t.string "value", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["value"], name: "index_mime_types_on_value"
+  end
+
   create_table "notes", id: :integer, force: :cascade do |t|
     t.integer "user_id"
     t.text "text"
@@ -357,6 +366,31 @@
     t.integer "super_region_id"
   end
 
+  create_table "research_outputs", force: :cascade do |t|
+    t.integer "plan_id"
+    t.integer "output_type", default: 3, null: false
+    t.string "output_type_description"
+    t.string "title", null: false
+    t.string "abbreviation"
+    t.integer "display_order"
+    t.boolean "is_default"
+    t.text "description"
+    t.integer "mime_type_id"
+    t.integer "access", default: 0, null: false
+    t.datetime "release_date"
+    t.boolean "personal_data"
+    t.boolean "sensitive_data"
+    t.bigint "byte_size"
+    t.text "mandatory_attribution"
+    t.datetime "coverage_start"
+    t.datetime "coverage_end"
+    t.string "coverage_region"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["output_type"], name: "index_research_outputs_on_output_type"
+    t.index ["plan_id"], name: "index_research_outputs_on_plan_id"
+  end
+
   create_table "roles", id: :integer, force: :cascade do |t|
     t.integer "user_id"
     t.integer "plan_id"

lib/tasks/mime_types.rake

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'httparty'
+
+namespace :mime_types do
+
+  IANA_BASE_URL = "https://www.iana.org/assignments/media-types".freeze
+
+  desc "Fetch all of the latest MIME types and load into the DB"
+  task load: :environment do
+    %w[application.csv audio.csv font.csv image.csv message.csv model.csv
+       multipart.csv text.csv video.csv].each do |file_name|
+      fetch_and_process_mime_type_file(url: "#{IANA_BASE_URL}/#{file_name}")
+    end
+  end
+
+  def fetch_and_process_mime_type_file(url:)
+    p "Processing #{url}"
+    body = fetch_csv_file(url: url)
+    p "  Unable to process the specified URL" unless body.present?
+
+    csv = CSV.parse(body, headers: true, force_quotes: true, encoding: 'iso-8859-1:utf-8')
+    p "  Invalid CSV format. Expecting a 'Name' and 'Template' column" unless csv.headers.include?("Name") &&
+                                                                              csv.headers.include?("Template")
+    process_mime_file(csv: csv)
+    p "  Done"
+  rescue StandardError => e
+    p "  Error processing CSV content - #{e.message}"
+  end
+
+  def process_mime_file(csv:)
+    return unless csv.is_a?(CSV::Table)
+
+    csv.each do |line|
+      next unless line["Template"].present? && line["Name"].present?
+
+      type = MimeType.find_or_initialize_by(value: line["Template"].downcase)
+      type.description = line["Name"]
+      type.category = line["Template"].split("/").first.downcase
+      type.save
+    end
+  end
+
+  def fetch_csv_file(url:)
+    return nil unless url.present?
+
+    payload = HTTParty.get(url, debug: false)
+    return nil unless payload.code == 200
+
+    payload.body
+  end
+
+end

lib/tasks/v3.rake

@@ -7,6 +7,11 @@ namespace :v3 do
     Rake::Task["v3:fix_funder_ids"].execute
   end
 
+  desc "Upgrade from v3.0.0 to v3.1.0"
+  task upgrade_3_1_0: :environment do
+    Rake::Task["mime_types:load"].execute
+  end
+
   # Set any records with a nil `language_id` to the default language
   desc "Change nil language_id entries into the default language"
   task ensure_default_languages: :environment do

spec/factories/mime_types.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: mime_types
+#
+#  id          :bigint           not null, primary key
+#  category    :string           not null
+#  description :string           not null
+#  value       :string           not null
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
+#
+# Indexes
+#
+#  index_mime_types_on_value  (value)
+#
+FactoryBot.define do
+  factory :mime_type do
+    category { %w[application audio audio image text video].sample }
+    description { Faker::Lorem.sentence }
+    value { "#{category}/#{Faker::Lorem.word}" }
+  end
+end