From b5bad4defc6c75b9b969658229ce5fd2f3a46107 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Tue, 21 Jan 2025 13:46:29 -0700 Subject: Rewrite project, add daily update of services list The project was rewritten from Elixir to Go, primarily because: - I don't write Elixir anymore and don't want to maintain a project in a language I no longer write - I already write Go for other projects, including my day job, so it's a safer bet for a project that I want to maintain long term - Go allows me to build portable executables that will make it easier for others to run farside on their own machines The Go version of Farsside also has a built in task to fetch the latest services{-full}.json file from the repo and ingest it, which makes running a farside server a lot simpler. It also automatically fetches the latest instance state from https://farside.link unless configured as a primary farside node, which will allow others to use farside without increasing traffic to all instances that are queried by farside (just to the farside node itself). --- lib/farside.ex | 142 --------------------------------------------- lib/farside/application.ex | 29 --------- lib/farside/instances.ex | 134 ------------------------------------------ lib/farside/router.ex | 78 ------------------------- lib/farside/scheduler.ex | 3 - lib/farside/server.ex | 25 -------- lib/farside/throttle.ex | 20 ------- lib/service.ex | 6 -- 8 files changed, 437 deletions(-) delete mode 100644 lib/farside.ex delete mode 100644 lib/farside/application.ex delete mode 100644 lib/farside/instances.ex delete mode 100644 lib/farside/router.ex delete mode 100644 lib/farside/scheduler.ex delete mode 100644 lib/farside/server.ex delete mode 100644 lib/farside/throttle.ex delete mode 100644 lib/service.ex (limited to 'lib') diff --git a/lib/farside.ex b/lib/farside.ex deleted file mode 100644 index 348f77c..0000000 --- a/lib/farside.ex +++ /dev/null @@ -1,142 +0,0 @@ - defmodule Farside do - @service_prefix Application.compile_env!(:farside, :service_prefix) - @fallback_suffix Application.compile_env!(:farside, :fallback_suffix) - @previous_suffix Application.compile_env!(:farside, :previous_suffix) - - # Define relation between available services and their parent service. - # This enables Farside to redirect with links such as: - # farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ - @youtube_regex ~r/youtu(.be|be.com)|invidious|piped/ - @twitter_regex ~r/twitter.com|x.com|nitter/ - @reddit_regex ~r/reddit.com|libreddit|redlib/ - @instagram_regex ~r/instagram.com|proxigram/ - @wikipedia_regex ~r/wikipedia.org|wikiless/ - @medium_regex ~r/medium.com|scribe/ - @odysee_regex ~r/odysee.com|librarian/ - @imgur_regex ~r/imgur.com|rimgo/ - @gtranslate_regex ~r/translate.google.com|lingva/ - @tiktok_regex ~r/tiktok.com|proxitok/ - @imdb_regex ~r/imdb.com|libremdb/ - @quora_regex ~r/quora.com|quetre/ - @gsearch_regex ~r/google.com\/search|whoogle/ - @fandom_regex ~r/fandom.com|breezewiki/ - @github_regex ~r/github.com|gothub/ - @stackoverflow_regex ~r/stackoverflow.com|anonymousoverflow/ - - @parent_services %{ - @youtube_regex => ["invidious", "piped"], - @reddit_regex => ["libreddit", "redlib"], - @instagram_regex => ["proxigram"], - @twitter_regex => ["nitter"], - @wikipedia_regex => ["wikiless"], - @medium_regex => ["scribe"], - @odysee_regex => ["librarian"], - @imgur_regex => ["rimgo"], - @gtranslate_regex => ["lingva"], - @tiktok_regex => ["proxitok"], - @imdb_regex => ["libremdb"], - @quora_regex => ["quetre"], - @gsearch_regex => ["whoogle"], - @fandom_regex => ["breezewiki"], - @github_regex => ["gothub"], - @stackoverflow_regex => ["anonymousoverflow"] - } - - def get_services_map do - service_list = CubDB.select(CubDB) - |> Stream.map(fn {key, _value} -> key end) - |> Stream.filter(fn key -> String.starts_with?(key, @service_prefix) end) - |> Enum.to_list - - # Match service name to list of available instances - Enum.reduce(service_list, %{}, fn service, acc -> - instance_list = CubDB.get(CubDB, service) - - Map.put( - acc, - String.replace_prefix( - service, - @service_prefix, - "" - ), - instance_list - ) - end) - end - - def get_service(service) do - # Check if service has an entry in the db, otherwise try to - # match against available parent services - service_name = cond do - !check_service(service) -> - Enum.find_value( - @parent_services, - fn {k, v} -> - String.match?(service, k) && Enum.random(v) - end) - true -> - service - end - - service_name - end - - def check_service(service) do - # Checks to see if a specific service has instances available - instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") - - instances != nil && Enum.count(instances) > 0 - end - - def last_instance(service) do - # Fetches the last selected instance for a particular service - CubDB.get(CubDB, "#{service}#{@previous_suffix}") - end - - def pick_instance(service) do - instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") - - # Either pick a random available instance, - # or fall back to the default one - instance = - if instances != nil && Enum.count(instances) > 0 do - if Enum.count(instances) == 1 do - # If there's only one instance, just return that one... - List.first(instances) - else - # ...otherwise pick a random one from the list, ensuring - # that the same instance is never picked twice in a row. - instance = - Enum.filter(instances, &(&1 != last_instance(service))) - |> Enum.random() - - CubDB.put(CubDB, "#{service}#{@previous_suffix}", instance) - - instance - end - else - CubDB.get(CubDB, "#{service}#{@fallback_suffix}") - end - instance - end - - def amend_instance(instance, service, path) do - cond do - String.match?(service, @fandom_regex) -> - # Fandom links require the subdomain to be preserved, otherwise the - # requested path won't work. - if String.contains?(service, ".fandom.com") do - wiki = String.replace(service, ".fandom.com", "") - "#{instance}/#{wiki}" - else - instance - end - true -> - instance - end - end - - def get_last_updated do - CubDB.get(CubDB, "last_updated") - end -end diff --git a/lib/farside/application.ex b/lib/farside/application.ex deleted file mode 100644 index 0f11f73..0000000 --- a/lib/farside/application.ex +++ /dev/null @@ -1,29 +0,0 @@ -defmodule Farside.Application do - @moduledoc false - - use Application - - @impl true - def start(_type, _args) do - farside_port = Application.fetch_env!(:farside, :port) - data_dir = Application.fetch_env!(:farside, :data_dir) - IO.puts "Running on http://localhost:#{farside_port}" - - children = [ - Plug.Cowboy.child_spec( - scheme: :http, - plug: Farside.Router, - options: [ - port: String.to_integer(farside_port) - ] - ), - {PlugAttack.Storage.Ets, name: Farside.Throttle.Storage, clean_period: 60_000}, - {CubDB, [data_dir: data_dir, name: CubDB, auto_compact: true]}, - Farside.Scheduler, - Farside.Server - ] - - opts = [strategy: :one_for_one, name: Farside.Supervisor] - Supervisor.start_link(children, opts) - end -end diff --git a/lib/farside/instances.ex b/lib/farside/instances.ex deleted file mode 100644 index d0d26f6..0000000 --- a/lib/farside/instances.ex +++ /dev/null @@ -1,134 +0,0 @@ -defmodule Farside.Instances do - @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix) - @update_file Application.fetch_env!(:farside, :update_file) - @service_prefix Application.fetch_env!(:farside, :service_prefix) - @headers Application.fetch_env!(:farside, :headers) - @queries Application.fetch_env!(:farside, :queries) - @debug_header "======== " - @debug_spacer " " - - # These instance uptimes are inspected as part of the nightly Farside build, - # and should not be included in the constant periodic update. - @skip_service_updates ["searxng", "nitter"] - - def sync() do - File.rename(@update_file, "#{@update_file}-prev") - update() - - # Add UTC time of last update - CubDB.put(CubDB, "last_updated", Calendar.strftime(DateTime.utc_now(), "%c")) - end - - def request(url) do - IO.puts("#{@debug_spacer}#{url}") - - cond do - System.get_env("FARSIDE_TEST") -> - :good - - true -> - HTTPoison.get(url, @headers) - |> then(&elem(&1, 1)) - |> Map.get(:status_code) - |> case do - n when n < 300 -> - IO.puts("#{@debug_spacer}✓ [#{n}]") - :good - - n -> - IO.puts("#{@debug_spacer}x [#{(n && n) || "error"}]") - :bad - end - end - end - - def update() do - services_json = Application.fetch_env!(:farside, :services_json) - {:ok, file} = File.read(services_json) - {:ok, json} = Jason.decode(file) - - # Loop through all instances and check each for availability - for service_json <- json do - service_atom = for {key, val} <- service_json, into: %{} do - {String.to_existing_atom(key), val} - end - - service = struct(%Service{}, service_atom) - - IO.puts("#{@debug_header}#{service.type}") - - result = cond do - Enum.member?(@skip_service_updates, service.type) -> - get_service_vals(service.instances) - true -> - Enum.filter(service.instances, fn instance_url -> - test_url = get_test_val(instance_url) - test_path = get_test_val(service.test_url) - test_request_url = gen_validation_url(test_url, test_path) - - service_url = get_service_val(instance_url) - service_path = get_service_val(service.test_url) - service_request_url = gen_validation_url(service_url, service_path) - - cond do - service_url != test_url -> - service_up = request(service_request_url) - test_up = request(test_request_url) - - service_up == :good && test_up == :good - true -> - request(test_request_url) == :good - end - end) - end - - add_to_db(service, result) - log_results(service.type, result) - end - end - - def add_to_db(service, instances) do - # Ensure only service URLs are inserted, not test URLs (separated by "|") - instances = get_service_vals(instances) - - # Remove previous list of instances - CubDB.delete(CubDB, "#{@service_prefix}#{service.type}") - - # Update with new list of available instances - CubDB.put(CubDB, "#{@service_prefix}#{service.type}", instances) - - # Set fallback to one of the available instances, - # or the default instance if all are "down" - if Enum.count(instances) > 0 do - CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", Enum.random(instances)) - else - CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", service.fallback) - end - end - - def log_results(service_name, results) do - {:ok, file} = File.open(@update_file, [:append, {:delayed_write, 100, 20}]) - IO.write(file, "#{service_name}: #{inspect(results)}\n") - File.close(file) - end - - def gen_validation_url(url, path) do - url <> EEx.eval_string(path, query: Enum.random(@queries)) - end - - def get_service_vals(services) do - Enum.map(services, fn x -> get_service_val(x) end) - end - - def get_service_val(service) do - String.split(service, "|") |> List.first - end - - def get_test_vals(services) do - Enum.map(services, fn x -> get_test_val(x) end) - end - - def get_test_val(service) do - String.split(service, "|") |> List.last - end -end diff --git a/lib/farside/router.ex b/lib/farside/router.ex deleted file mode 100644 index ec49862..0000000 --- a/lib/farside/router.ex +++ /dev/null @@ -1,78 +0,0 @@ -defmodule Farside.Router do - @index Application.fetch_env!(:farside, :index) - @route Application.fetch_env!(:farside, :route) - - use Plug.Router - - plug(RemoteIp) - plug(Farside.Throttle) - plug(:match) - plug(:dispatch) - - def get_query_params(conn) do - cond do - String.length(conn.query_string) > 0 -> - "?#{conn.query_string}" - - true -> - "" - end - end - - match "/" do - resp = - EEx.eval_file( - @index, - last_updated: Farside.get_last_updated(), - services: Farside.get_services_map() - ) - - put_resp_header(conn, "content-type", "text/html") - |> send_resp(200, resp) - end - - match "/_/:service/*glob" do - r_path = String.slice(conn.request_path, 2..-1) - - resp = - EEx.eval_file( - @route, - instance_url: "#{r_path}#{get_query_params(conn)}" - ) - - send_resp(conn, 200, resp) - end - - match "/:service/*glob" do - service_name = cond do - service =~ "http" -> - List.first(glob) - true -> - service - end - - path = cond do - service_name != service -> - Enum.join(Enum.slice(glob, 1..-1), "/") - true -> - Enum.join(glob, "/") - end - - cond do - conn.assigns[:throttle] != nil -> - send_resp(conn, :too_many_requests, "Too many requests - max request rate is 1 per second") - true -> - instance = Farside.get_service(service_name) - |> Farside.pick_instance - |> Farside.amend_instance(service_name, path) - - # Redirect to the available instance - conn - |> Plug.Conn.resp(:found, "") - |> Plug.Conn.put_resp_header( - "location", - "#{instance}/#{path}#{get_query_params(conn)}" - ) - end - end -end diff --git a/lib/farside/scheduler.ex b/lib/farside/scheduler.ex deleted file mode 100644 index 4707624..0000000 --- a/lib/farside/scheduler.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Farside.Scheduler do - use Quantum, otp_app: :farside -end diff --git a/lib/farside/server.ex b/lib/farside/server.ex deleted file mode 100644 index cdb6682..0000000 --- a/lib/farside/server.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Farside.Server do - use GenServer - import Crontab.CronExpression - - def init(init_arg) do - {:ok, init_arg} - end - - def start_link(arg) do - test = System.get_env("FARSIDE_TEST") - cron = System.get_env("FARSIDE_CRON") - - if test == "1" || cron == "0" do - IO.puts("Skipping sync job setup...") - else - Farside.Scheduler.new_job() - |> Quantum.Job.set_name(:sync) - |> Quantum.Job.set_schedule(~e[*/5 * * * *]) - |> Quantum.Job.set_task(fn -> Farside.Instances.sync() end) - |> Farside.Scheduler.add_job() - end - - GenServer.start_link(__MODULE__, arg) - end -end diff --git a/lib/farside/throttle.ex b/lib/farside/throttle.ex deleted file mode 100644 index e2561ab..0000000 --- a/lib/farside/throttle.ex +++ /dev/null @@ -1,20 +0,0 @@ -defmodule Farside.Throttle do - import Plug.Conn - use PlugAttack - - rule "throttle per ip", conn do - # throttle to 1 request per second - throttle(conn.remote_ip, - period: 1_000, - limit: 1, - storage: {PlugAttack.Storage.Ets, Farside.Throttle.Storage} - ) - end - - def allow_action(conn, _data, _opts), do: conn - - def block_action(conn, _data, _opts) do - conn = assign(conn, :throttle, 1) - conn - end -end diff --git a/lib/service.ex b/lib/service.ex deleted file mode 100644 index ae964f6..0000000 --- a/lib/service.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule Service do - defstruct type: nil, - test_url: nil, - fallback: nil, - instances: [] -end -- cgit v1.2.3