User upload (#67)

* Adding XSV gem version 1.2.0

* Importing xlsx files

* Better UI

* Fixing a counting issue

* Adding a task to clean the Members table

* Adding comments to views

* Better display inside settings

* Better breadcrumb

---------

Co-authored-by: Stephane Paquet <spaquet@up4b.com>

Gemfile

@@ -63,6 +63,10 @@ gem 'argon2', '~> 2.2.0'
 gem 'caxlsx'
 gem 'caxlsx_rails'
 
+# Parse Excel xlsx files
+# [https://github.com/martijn/xsv]
+gem 'xsv'
+
 # Adding OAuth2 support [https://github.com/omniauth/omniauth]
 gem 'omniauth', '~> 2.1.0'
 # Adding Google Sign-in support

Gemfile.lock

@@ -416,6 +416,8 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
+    xsv (1.2.0)
+      rubyzip (>= 1.3, < 3)
     zeitwerk (2.6.7)
 
 PLATFORMS
@@ -477,6 +479,7 @@ DEPENDENCIES
   view_component
   web-console
   webdrivers
+  xsv
 
 RUBY VERSION
    ruby 3.2.2p53

app/controllers/imports_controller.rb

@@ -0,0 +1,21 @@
+class ImportsController < ApplicationController
+  before_action :authenticate_user!
+  before_action :redirect_if_unauthenticated
+
+  def new
+  end
+
+  def create
+    current_user.import_file.attach(params[:file])
+    ProcessExcelImportJob.perform_async(current_user.id)
+    redirect_to imports_path, notice: 'File is being processed. Check back later for the results.'
+  end
+
+  def index
+    @import_results = current_user.import_results.order(created_at: :desc).limit(5)
+  end
+
+  def show
+    @import_result = current_user.import_results.find(params[:id])
+  end
+end

app/controllers/your_questions_controller.rb

@@ -3,8 +3,6 @@ class YourQuestionsController < ApplicationController
   before_action :redirect_if_unauthenticated
 
   def index
-    # TODO: make sure the user is the only one enable to read the question
-    # Pundit ;-)
     @questions = current_user.questions.order(created_at: :desc)
     @count = @questions.count
   end

app/helpers/imports_helper.rb

@@ -0,0 +1,2 @@
+module ImportsHelper
+end

app/mailers/user_mailer.rb

@@ -1,42 +1,42 @@
 class UserMailer < ApplicationMailer
   default from: User::MAILER_FROM_EMAIL
 
-  def confirmation(user, confirmation_token)
-    @user = user
+  def confirmation(user_id, confirmation_token)
+    @user = User.find(user_id)
     @confirmation_token = confirmation_token
 
     mail(
       to: @user.email, 
       subject: 'THEPEW: Account Confirmation Instructions')
   end
 
-  def password_reset(user, password_reset_token)
-    @user = user
+  def password_reset(user_id, password_reset_token)
+    @user = User.find(user_id)
     @password_reset_token = password_reset_token
 
     mail(
       to: @user.email, 
       subject: 'THEPEW: Password Reset Instructions')
   end
 
-  def welcome(user)
-    @user = user
+  def welcome(user_id)
+    @user = User.find(user_id)
     mail(
       to: @user.email,
       subject: 'THEPEW: Welcome'
     )
   end
 
-  def password_change_confirmatiom(user)
-    @user = user
+  def password_change_confirmatiom(user_id)
+    @user = User.find(user_id)
     mail(
       to: @user.email,
       subject: 'THEPEW: Password changed'
     )
   end
 
-  def invite(user, invitation_token)
-    @user = user
+  def invite(user_id, invitation_token)
+    @user = User.find(user_id)
     @invitation_token = invitation_token
     mail(
       to: @user.email,

app/models/import_result.rb

@@ -0,0 +1,28 @@
+# == Schema Information
+#
+# Table name: import_results
+#
+#  id         :uuid             not null, primary key
+#  filename   :string           not null
+#  message    :text
+#  status     :integer          default("uploading"), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  user_id    :uuid             not null
+#
+# Indexes
+#
+#  index_import_results_on_status   (status)
+#  index_import_results_on_user_id  (user_id)
+#
+# Foreign Keys
+#
+#  fk_rails_...  (user_id => users.id)
+#
+class ImportResult < ApplicationRecord
+  belongs_to :user
+
+  validates :filename, presence: true
+
+  enum status: { uploading: 0, processing: 10, success: 20, error: 30 }
+end

app/models/user.rb

@@ -69,6 +69,10 @@ class User < ApplicationRecord
   has_one  :member
   has_one  :organization,    through: :member, required: false
 
+  # To enable bullk upload via an excel spreadsheet
+  has_one_attached :import_file
+  has_many         :import_results, dependent: :destroy
+
   # Connect the user to their Merit
   has_merit
 
@@ -114,7 +118,7 @@ def send_password_reset_email!
     # accepted_invitation_on is different from nil
     if !self.invited || self.accepted_invitation_on != nil
       password_reset_token = signed_id(purpose: :reset_password, expires_in: PASSWORD_RESET_TOKEN_EXPIRATION)
-      UserMailer.password_reset(self, password_reset_token).deliver_later
+      UserMailer.password_reset(self.id, password_reset_token).deliver_later
     end
   end
 
@@ -126,15 +130,15 @@ def send_confirmation_email!
     # join the organization plays the role of a confirmation email.
     if !self.invited
       confirmation_token = signed_id(purpose: :email_confirmation, expires_in: CONFIRMATION_TOKEN_EXPIRATION)
-      UserMailer.confirmation(self, confirmation_token).deliver_later
+      UserMailer.confirmation(self.id, confirmation_token).deliver_later
     end
   end
 
   # Used to send an invite to join an Organization
   # Invitates are valid for 3 days
   def send_invite!
     invitation_token = signed_id(purpose: :invite, expires_in: 3.days)
-    UserMailer.invite(self, invitation_token).deliver_later
+    UserMailer.invite(self.id, invitation_token).deliver_later
   end
 
   def invited!

app/services/excel_import_service.rb

@@ -0,0 +1,95 @@
+require "xsv"
+
+class ExcelImportService
+  def initialize(user)
+    @user = user
+  end
+
+  def process_excel_file(file_path)
+    # Determine the file extension
+    file_extension = File.extname(file_path)
+
+    # Process the Excel file using the appropriate method
+    case file_extension
+    when ".xlsx"
+      success, errors, message = process_xlsx_file(file_path)
+    else
+      return [false, ["Unsupported file format"]]
+    end
+
+    [success, errors, message]
+  end
+
+  private
+
+  def process_xlsx_file(file_path)
+    errors = []
+    processed_records = 0
+    error_records = 0
+
+    sheet = Xsv.open(file_path).sheets[0]  # Assuming the data is in the first sheet
+
+    # Iterate through each row in the sheet, starting from the second row (index 1)
+    sheet.each_with_index do |row, index|
+      next if index.zero? # Skip header row
+
+      email = row[0]
+      # role = row[1]
+
+      # Stop processing when an empty row is encountered
+      break if email.blank?
+
+      # Validate the email and role
+      if email.blank? || !email.match?(URI::MailTo::EMAIL_REGEXP)
+        errors << "Row #{index + 1}: Invalid email | #{email}"
+        error_records += 1
+        processed_records += 1
+        next
+      end
+
+      # if role.blank? || !["admin", ""].include?(role.downcase)
+      #   errors << "Row #{index + 1}: Invalid role"
+      #   error_records += 1
+      #   next
+      # end
+
+      # Create the user
+      user = User.new( email: email,
+          invited_at: Time.current,
+          invited:  true )
+
+      # TODO: Assign role using rollify.
+
+      # Save the user, and add an error message if saving fails
+      unless user.save
+        errors << "Row #{index + 1}: Failed to save user - #{user.errors.full_messages.join(", ")}"
+        error_records += 1
+        processed_records += 1
+        next
+      end
+
+      # Assign the user to the organization
+      member = Member.new
+      member.organization_id = @user.organization.id
+      member.user_id = user.id
+
+      unless member.save
+        errors << "Row #{index + 1}: Failed to save user to organization - #{member.errors.full_messages.join(", ")}"
+        error_records += 1
+        processed_records += 1
+        next
+      end
+
+      # Send email to the user
+      user.send_invite!
+
+      # Incrementing the number of processed records
+      processed_records += 1
+
+    end
+
+    success = errors.empty?
+    message = "Processed records: #{processed_records}, records with errors: #{error_records}"
+    [success, errors, message]
+  end
+end

app/sidekiq/process_excel_import_job.rb

@@ -0,0 +1,44 @@
+class ProcessExcelImportJob
+  include Sidekiq::Job
+
+  # Prevent retry as the file is always removed at the end of this process.
+  sidekiq_options retry: false
+
+  def perform(user_id)
+    user = User.find(user_id)
+    import_file = user.import_file
+
+    # Get the original filename
+    original_file_name = import_file.filename.to_s
+
+    # Create a new ImportResult record with the "processing" status
+    import_result = user.import_results.create(
+      status: :processing,
+      message: "",
+      filename: original_file_name
+    )
+
+    # Open the file using the Active Storage blob
+    import_file.open do |temp_file|
+      # Process the Excel file using the ExcelImportService
+      excel_import_service = ExcelImportService.new(user)
+      success, errors, message = excel_import_service.process_excel_file(temp_file.path)
+
+      # Add error messages if any
+      if !errors.nil?
+        message << "\n"
+        message << errors.join("\n")
+      end
+
+      # Update the import result with the final status and message
+      import_result.update(
+        status: success ? :success : :error,
+        message: message
+      )
+    end
+
+    # Cleanup
+    user.import_file.purge
+  end
+
+end

app/views/imports/_import_result.html.erb

@@ -0,0 +1,38 @@
+<!-- app/views/imports/_import_result.html.erb -->
+<tr class="bg-white border-b dark:bg-gray-800 dark:border-gray-700 hover:bg-gray-50 dark:hover:bg-gray-600">
+  <th scope="row" class="flex items-center px-6 py-4 text-gray-900 whitespace-nowrap dark:text-white">
+    <div class="pl-3">
+      <div class="text-base font-semibold"><%= import_result.filename %></div>
+      <div class="font-normal text-gray-500 text-sm">Uploaded: <%= import_result.created_at.strftime("%Y %b %d at %l%P : %M") %></div>
+      <% if import_result.created_at != import_result.updated_at %>
+        <div class="font-normal text-gray-500 text-sm">Processed: <%= import_result.updated_at.strftime("%Y %b %d at %l%P : %M") %></div>
+      <% end %>
+    </div>
+  </th>
+  <td class="px-6 py-4">
+    <div class="flex items-center">
+      <% if import_result.success? %>
+        <div class="h-2.5 w-2.5 rounded-full bg-green-500 mr-2"></div>
+        Success
+      <% end %>
+      <% if import_result.error? %>
+        <div class="h-2.5 w-2.5 rounded-full bg-red-500 mr-2"></div>
+        Failed
+      <% end %>
+      <% if import_result.processing? %>
+        <div class="h-2.5 w-2.5 rounded-full bg-blue-500 mr-2"></div>
+        Processing
+      <% end %>
+    </div>
+  </td>
+  <td class="px-6 py-4">
+    <%= import_result.message.split("\n")[0] %>
+  </td>
+  <td class="px-6 py-4">
+    <%= link_to import_path(import_result.id) do %>
+      <svg class="w-6 h-6" fill="none" stroke="currentColor" stroke-width="1.5" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
+        <path stroke-linecap="round" stroke-linejoin="round" d="M6.75 12a.75.75 0 11-1.5 0 .75.75 0 011.5 0zM12.75 12a.75.75 0 11-1.5 0 .75.75 0 011.5 0zM18.75 12a.75.75 0 11-1.5 0 .75.75 0 011.5 0z"></path>
+      </svg>
+    <% end %>
+  </td>
+</tr>

app/views/imports/index.html.erb

@@ -0,0 +1,31 @@
+<!-- app/views/imports/index.html.erb -->
+<%= turbo_frame_tag "settings_main" do %>
+  <%= render partial: "settings/breadcrumb", locals: { sub1: "Users" } %>
+  <div class="mb-6">
+    <%= link_to new_import_path, class: "text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:outline-none focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 text-center inline-flex items-center dark:bg-blue-600 dark:hover:bg-blue-700 dark:focus:ring-blue-800" do %>
+      Upload a file
+    <% end %>
+  </div>
+  <div class="relative overflow-x-auto shadow-md sm:rounded-lg">
+    <table class="w-full text-sm text-left text-gray-500 dark:text-gray-400">
+      <thead class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-700 dark:text-gray-400">
+        <tr>
+          <th scope="col" class="px-6 py-3">
+            Filename
+          </th>
+          <th scope="col" class="px-6 py-3">
+            Status
+          </th>
+          <th scope="col" class="px-6 py-3">
+            Message
+          </th>
+          <th scope="col" class="px-6 py-3">
+          </th>
+        </tr>
+      </thead>
+      <tbody>
+        <%= render partial: "import_result", collection: @import_results %>
+      </tbody>
+    </table>
+  </div>
+<% end %>