Store source URL MIME type

apps/cf/lib/sources/fetcher.ex

@@ -41,7 +41,8 @@ defmodule CF.Sources.Fetcher do
           try do
             fetch(url, callback)
           rescue
-            e -> Logger.error("Fetch metadata for #{url} crashed - #{inspect(e)}")
+            e ->
+              Logger.error("Fetch metadata for #{url} crashed - #{inspect(e)}")
           after
             Fetcher.LinkChecker.free_url(url)
           end
@@ -51,14 +52,20 @@ defmodule CF.Sources.Fetcher do
 
   def get_queue, do: Fetcher.LinkChecker.get_queue()
 
+  @url_regex ~r/^https?:\/\/[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&\/\/=]*)/
+
   defp fetch(url, callback) do
-    case do_fetch_source_metadata(url) do
+    without_domain = Regex.replace(@url_regex, url, "\\1")
+
+    case do_fetch_source_metadata(url, MIME.from_path(without_domain)) do
       {:error, _} -> :error
       {:ok, result} -> callback.(result)
     end
   end
 
-  defp do_fetch_source_metadata(url) do
+  @fetchable_mime_types ~w(text/html application/octet-stream application/xhtml+xml application/vnd.ms-htmlhelp)
+
+  defp do_fetch_source_metadata(url, mime_types) when mime_types in @fetchable_mime_types do
     case HTTPoison.get(
            url,
            [],
@@ -80,6 +87,12 @@ defmodule CF.Sources.Fetcher do
     end
   end
 
+  defp do_fetch_source_metadata(url, mime_type) do
+    ext = Path.extname(url)
+    title = Path.basename(url, ext)
+    {:ok, %{file_mime_type: mime_type, title: title}}
+  end
+
   defp source_params_from_tree(tree) do
     head = Floki.find(tree, "head")
 

apps/cf/test/cf/sources/fetcher_test.exs

@@ -28,6 +28,16 @@ defmodule CF.Sources.FetcherTest do
     wait_fetcher()
   end
 
+  test "stores file name and mime type" do
+    # Start web server providing a page with giver metadata
+    Fetcher.fetch_source_metadata("https://CaptainFact.io/great-file.pdf", fn response ->
+      assert response.title == "great-file"
+      assert response.file_mime_type == "application/pdf"
+    end)
+
+    wait_fetcher()
+  end
+
   test "return error if status is not 2xx" do
     # Error 404
     @valid_attributes.url

apps/db/lib/db_schema/source.ex

@@ -8,6 +8,7 @@ defmodule DB.Schema.Source do
     field(:title, :string)
     field(:language, :string)
     field(:site_name, :string)
+    field(:file_mime_type, :string)
 
     timestamps()
   end
@@ -39,7 +40,7 @@ defmodule DB.Schema.Source do
 
   def changeset_fetched(struct, params) do
     struct
-    |> cast(params, [:og_url, :url, :title, :language, :site_name])
+    |> cast(params, [:og_url, :url, :title, :language, :site_name, :file_mime_type])
     |> update_change(:url, &prepare_url/1)
     |> update_change(:og_url, &prepare_url/1)
     |> update_change(:title, &clean_and_truncate/1)
@@ -60,6 +61,15 @@ defmodule DB.Schema.Source do
     |> unique_constraint(:url)
     |> validate_format(:url, @url_regex)
     |> validate_length(:url, min: 10, max: @url_max_length)
+    |> validate_change(:file_mime_type, &validate_file_mime_type/2)
+  end
+
+  defp validate_file_mime_type(:file_mime_type, mime_type) do
+    if MIME.valid?(mime_type) do
+      []
+    else
+      [file_mime_type: "Invalid MIME type"]
+    end
   end
 
   defp clean_and_truncate(str) do

apps/db/mix.exs

@@ -50,6 +50,7 @@ defmodule DB.Mixfile do
       {:burnex, "~> 1.0"},
       {:hashids, "~> 2.0"},
       {:kaur, "~> 1.1"},
+      {:mime, "~> 1.2"},
       {:scrivener_ecto, "~> 1.0"},
 
       # Dev only

apps/db/priv/repo/migrations/20181109233422_add_file_type_to_source.exs

@@ -0,0 +1,9 @@
+defmodule DB.Repo.Migrations.AddFileTypeToSource do
+  use Ecto.Migration
+
+  def change do
+    alter table(:sources) do
+      add(:file_mime_type, :string, null: true)
+    end
+  end
+end

apps/db/test/db_schema/source_test.exs

@@ -5,6 +5,7 @@ defmodule DB.Schema.SourceTest do
 
   @valid_attrs %{
     title: "some content",
+    file_mime_type: "application/pdf",
     url:
       "http://www.lemonde.fr/idees/article/2017/04/24/les-risques-d-une-explosion_5116380_3232.html"
   }
@@ -23,6 +24,11 @@ defmodule DB.Schema.SourceTest do
     refute Source.changeset(%Source{}, %{@valid_attrs | url: "https://xxxxxx"}).valid?
   end
 
+  test "changeset_fetched" do
+    assert Source.changeset_fetched(%Source{}, @valid_attrs).valid?
+    refute Source.changeset_fetched(%Source{}, %{@valid_attrs | file_mime_type: "zzzz"}).valid?
+  end
+
   test "must add https:// if url doesn't start with http:// or https://" do
     changeset = Source.changeset(%Source{}, Map.put(@valid_attrs, :url, "amazing.com/article"))
     assert changeset.changes.url == "https://amazing.com/article"