Tidy moduledoc

README.md

@@ -1,5 +1,25 @@
 # Slot Sync
 
-* Syncs slot data from external provider
+A tool for reading data from a third party via their API and publishing to a broker.
+
+## Overview
+
+* Currently sync's slot data from WIW
+* Runs sync every 5 minutes (see SchedEx for :wiw_sync to change)
 * Caches the data locally in Redis
-* If remote data changed then publishes to Kafka
+* If newly read data has changed from cached version it then publishes to Kafka
+
+## Detail
+```
+(every 5 minutes)
+|> SlotSync.Runner
+|> SlotSync.WIW
+  |> 1 days worth of shifts from today
+  |> get slots on WIW
+|> SlotSync.Dispatcher
+  |> each shift
+|> SlotSync.Processor.Shift (GenServer)
+  |> if matches redis cache then next (cache expires each slot after 1 week)
+  |> if not then save in redis and publish
+|> SlotSync.Publishers.Kafka
+```

lib/slot_sync.ex

@@ -1,9 +1,2 @@
 defmodule SlotSync do
-  @moduledoc """
-  SlotSync keeps the contexts that define your domain
-  and business logic.
-
-  Contexts are also responsible for managing your data, regardless
-  if it comes from the database, an external API or others.
-  """
 end

lib/slot_sync/cache/redis.ex

@@ -1,4 +1,7 @@
 defmodule SlotSync.Cache.Redis do
+  @moduledoc """
+  A interface for caching shift data into redis
+  """
   use GenServer
 
   use Confex, otp_app: :slot_sync

lib/slot_sync/dogstatsd_config.ex

@@ -1,9 +1,8 @@
 defmodule SlotSync.DogStatsdConfig do
-  # require Logger
-
+  @moduledoc """
+  Helper to get DogStatsd setup correctly
+  """
   def configs do
-    # [:host, :port, :namespace, :tags, :max_buffer_size]
-
     host = Confex.get_env(:slot_sync, SlotSync.Datadog)[:host]
     port = Confex.get_env(:slot_sync, SlotSync.Datadog)[:port]
     namespace = Confex.get_env(:slot_sync, SlotSync.Datadog)[:namespace]

lib/slot_sync/kafka_config.ex

@@ -1,4 +1,8 @@
 defmodule SlotSync.KafkaConfig do
+  @moduledoc """
+  Helper to get Kafka setup correctly
+  """
+
   require Logger
 
   def brokers do

lib/slot_sync/publishers/publisher.ex → lib/slot_sync/publishers/kafka.ex

@@ -4,25 +4,20 @@ defmodule SlotSync.Publishers.Kafka do
   """
   use Confex, otp_app: :slot_sync
 
-  alias SlotSync.TrackingLocationsFormatter
   alias KafkaEx.Protocol.Produce.Request
   alias KafkaEx.Protocol.Produce.Message
   require Logger
 
-  # @type key() :: String.t()
-  # @type event() :: String.t()
-  # @type topic_name() :: String.t() | atom()
-  # @type encoded_message() :: String.t()
+  @type key() :: String.t()
+  @type event() :: String.t()
+  @type encoded_message() :: String.t()
 
   @doc """
   This is the main function to trigger a message to be published to Kafka
-
-  ## Examples
-
   """
 
-  # @type message_number() :: integer()
-  # @spec call(map(), integer(), topic_name()) :: {:ok, message_number()} | {:error, String.t()}
+  @type message_number() :: integer()
+  @spec call(map(), integer()) :: {:ok, message_number()} | {:error, String.t()}
   def call(data, id) do
     id
     |> build_event_key()
@@ -33,8 +28,8 @@ defmodule SlotSync.Publishers.Kafka do
 
   defp build_event_key(id), do: [{"id", id}]
 
-  # @spec encode_event_message(key, event, topic_name) ::
-  #         {:encoded_key, encoded_message, encoded_message: encoded_message} | {:error, key}
+  @spec encode_event_message(key, event) ::
+          {:encoded_key, encoded_message, encoded_message: encoded_message} | {:error, key}
   defp encode_event_message(key, event) do
     with {:ok, encoded_key} <- encoder().call(key_schema_name(), key),
          {:ok, encoded_message} <- encoder().call(value_schema_name(), transformed(event)) do
@@ -45,8 +40,6 @@ defmodule SlotSync.Publishers.Kafka do
   end
 
   defp transformed(event) do
-    event["linked_users"] |> IO.inspect(label: "linked_users")
-
     [
       {"account_id", event["account_id"]},
       {"acknowledged", event["acknowledged"]},

lib/slot_sync/runner.ex

@@ -1,6 +1,6 @@
 defmodule SlotSync.Runner do
   @moduledoc """
-  Will schedule running covering a days worth of data on each call
+  Ensures sync peroid covers a rolling amount of data on each call
   """
   @spec start() :: :ok
   def start do

lib/slot_sync/shift_processor.ex

@@ -5,6 +5,8 @@ defmodule SlotSync.Processor.Shift do
   It compares incoming data to that in redis, and updates if changed.
   And then also publishes to Kafka.
 
+                                                    denotes defined on API
+                                                    (others are not listed)
   %{
     "account_id" => 77967,                                  x
     "acknowledged" => 1,                                    x
@@ -19,7 +21,7 @@ defmodule SlotSync.Processor.Shift do
     "end_time" => "Mon, 08 Oct 2018 00:15:00 +0100",
     "id" => 2076303948,                                     x
     "instances" => 0,                                       x
-    "is_open" => false,                                     z
+    "is_open" => false,                                     x
     "linked_users" => nil,
     "location_id" => 3999871,
     "notes" => "",                                          x

lib/slot_sync/wiw.ex

@@ -9,6 +9,8 @@ defmodule SlotSync.WIW do
   W-Token: abc123
 
   https://api.wheniwork.com/2/shifts?start=2018-10-07&end=2018-10-08
+
+  http://dev.wheniwork.com/#shifts
   """
 
   alias SlotSync.Dispatcher