From e58d6e23edd2cfc1da7feab3cfc2b09326b73bff Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Mon, 31 Oct 2022 16:45:31 -0600 Subject: Remove Redis dep, replace w/ native Elixir lib This removes the dependency on Redis for core app functionality. Rather than using the key/value store provided by Redis, Farside now uses a key/val store provided by [cubdb](https://github.com/lucaong/cubdb) for identical functionality but without reliance on a non-Elixir service. This solution was chosen instead of ets, because storing instance data in memory leads to a period of broken functionality whenever the app restarts and hasn't re-populated instance data yet. It was also chosen instead of dets, because the documentation for dets was pretty hard to understand at first glance. Tests and the CI build were updated to reflect the removed dependency on Redis. New environment variable `FARSIDE_DATA_DIR` can be used to point to a directory where the instance data can be stored by cubdb. Co-authored-by: Jason Clark --- lib/farside.ex | 66 +++++++++------------------------------------- lib/farside/application.ex | 7 ++--- lib/farside/instances.ex | 35 +++++------------------- lib/farside/router.ex | 6 ----- 4 files changed, 22 insertions(+), 92 deletions(-) (limited to 'lib') diff --git a/lib/farside.ex b/lib/farside.ex index 8159e82..e680cdc 100644 --- a/lib/farside.ex +++ b/lib/farside.ex @@ -35,15 +35,14 @@ defmodule Farside do } def get_services_map do - {:ok, service_list} = Redix.command(:redix, ["KEYS", "#{@service_prefix}*"]) + service_list = CubDB.select(CubDB) + |> Stream.map(fn {key, _value} -> key end) + |> Stream.filter(fn key -> String.starts_with?(key, @service_prefix) end) + |> Enum.to_list # Match service name to list of available instances Enum.reduce(service_list, %{}, fn service, acc -> - {:ok, instance_list} = - Redix.command( - :redix, - ["LRANGE", service, "0", "-1"] - ) + instance_list = CubDB.get(CubDB, service) Map.put( acc, @@ -58,7 +57,7 @@ defmodule Farside do end def get_service(service) do - # Check if service has an entry in Redis, otherwise try to + # Check if service has an entry in the db, otherwise try to # match against available parent services service_name = cond do !check_service(service) -> @@ -76,42 +75,18 @@ defmodule Farside do def check_service(service) do # Checks to see if a specific service has instances available - # in redis - {:ok, instances} = - Redix.command( - :redix, - [ - "LRANGE", - "#{@service_prefix}#{service}", - "0", - "-1" - ] - ) + instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") - Enum.count(instances) > 0 + instances != nil && Enum.count(instances) > 0 end def last_instance(service) do # Fetches the last selected instance for a particular service - {:ok, previous} = - Redix.command( - :redix, - ["GET", "#{service}#{@previous_suffix}"] - ) - previous + CubDB.get(CubDB, "#{service}#{@previous_suffix}") end def pick_instance(service) do - {:ok, instances} = - Redix.command( - :redix, - [ - "LRANGE", - "#{@service_prefix}#{service}", - "0", - "-1" - ] - ) + instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") # Either pick a random available instance, # or fall back to the default one @@ -127,21 +102,12 @@ defmodule Farside do Enum.filter(instances, &(&1 != last_instance(service))) |> Enum.random() - Redix.command( - :redix, - ["SET", "#{service}#{@previous_suffix}", instance] - ) + CubDB.put(CubDB, "#{service}#{@previous_suffix}", instance) instance end else - {:ok, result} = - Redix.command( - :redix, - ["GET", "#{service}#{@fallback_suffix}"] - ) - - result + CubDB.get(CubDB, "#{service}#{@fallback_suffix}") end instance end @@ -165,12 +131,6 @@ defmodule Farside do end def get_last_updated do - {:ok, last_updated} = - Redix.command( - :redix, - ["GET", "last_updated"] - ) - - last_updated + CubDB.get(CubDB, "last_updated") end end diff --git a/lib/farside/application.ex b/lib/farside/application.ex index 17750e4..cea7871 100644 --- a/lib/farside/application.ex +++ b/lib/farside/application.ex @@ -1,16 +1,13 @@ defmodule Farside.Application do - #@farside_port Application.fetch_env!(:farside, :port) - #@redis_conn Application.fetch_env!(:farside, :redis_conn) @moduledoc false use Application @impl true def start(_type, _args) do - redis_conn = Application.fetch_env!(:farside, :redis_conn) farside_port = Application.fetch_env!(:farside, :port) + data_dir = Application.fetch_env!(:farside, :data_dir) IO.puts "Running on http://localhost:#{farside_port}" - IO.puts "Redis conn: #{redis_conn}" children = [ Plug.Cowboy.child_spec( @@ -21,7 +18,7 @@ defmodule Farside.Application do ] ), {PlugAttack.Storage.Ets, name: Farside.Throttle.Storage, clean_period: 60_000}, - {Redix, {redis_conn, [name: :redix]}}, + {CubDB, [data_dir: data_dir, name: CubDB]}, Farside.Scheduler, Farside.Server ] diff --git a/lib/farside/instances.ex b/lib/farside/instances.ex index f37f306..c358d09 100644 --- a/lib/farside/instances.ex +++ b/lib/farside/instances.ex @@ -12,11 +12,7 @@ defmodule Farside.Instances do update() # Add UTC time of last update - Redix.command(:redix, [ - "SET", - "last_updated", - Calendar.strftime(DateTime.utc_now(), "%c") - ]) + CubDB.put(CubDB, "last_updated", Calendar.strftime(DateTime.utc_now(), "%c")) end def request(url) do @@ -69,41 +65,24 @@ defmodule Farside.Instances do request(request_url) == :good end) - add_to_redis(service, result) + add_to_db(service, result) log_results(service.type, result) end end - def add_to_redis(service, instances) do + def add_to_db(service, instances) do # Remove previous list of instances - Redix.command(:redix, [ - "DEL", - "#{@service_prefix}#{service.type}" - ]) + CubDB.delete(CubDB, "#{@service_prefix}#{service.type}") # Update with new list of available instances - Redix.command( - :redix, - [ - "LPUSH", - "#{@service_prefix}#{service.type}" - ] ++ instances - ) + CubDB.put(CubDB, "#{@service_prefix}#{service.type}", instances) # Set fallback to one of the available instances, # or the default instance if all are "down" if Enum.count(instances) > 0 do - Redix.command(:redix, [ - "SET", - "#{service.type}#{@fallback_suffix}", - Enum.random(instances) - ]) + CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", Enum.random(instances)) else - Redix.command(:redix, [ - "SET", - "#{service.type}#{@fallback_suffix}", - service.fallback - ]) + CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", service.fallback) end end diff --git a/lib/farside/router.ex b/lib/farside/router.ex index ed667e4..a11e07a 100644 --- a/lib/farside/router.ex +++ b/lib/farside/router.ex @@ -29,12 +29,6 @@ defmodule Farside.Router do send_resp(conn, 200, resp) end - get "/ping" do - # Useful for app healthcheck - {:ok, resp} = Redix.command(:redix, ["PING"]) - send_resp(conn, 200, resp) - end - get "/_/:service/*glob" do r_path = String.slice(conn.request_path, 2..-1) -- cgit v1.2.3