From 67baa0ec04b7c1d8b12ed00accd731fc8f6bc3d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bror=20Bojl=C3=A9n?= Date: Wed, 21 Aug 2019 17:47:45 +0200 Subject: [PATCH] add caching to expensive queries --- CHANGELOG.md | 2 + backend/config/config.exs | 5 + backend/lib/backend/application.ex | 3 +- .../controllers/graph_controller.ex | 9 +- .../controllers/instance_controller.ex | 6 +- backend/lib/graph/cache.ex | 93 +++++++++++++++++++ backend/mix.exs | 3 +- backend/mix.lock | 2 + 8 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 backend/lib/graph/cache.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e28a42..2afaf88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add caching to graph + instance endpoints to better handle traffic spikes. + ### Changed ### Deprecated diff --git a/backend/config/config.exs b/backend/config/config.exs index 4d9ab63..528b54c 100644 --- a/backend/config/config.exs +++ b/backend/config/config.exs @@ -34,10 +34,15 @@ config :logger, :console, config :phoenix, :json_library, Jason config :gollum, + # 24 hrs refresh_secs: 86_400, lazy_refresh: true, user_agent: "fediverse.space crawler" +config :backend, Graph.Cache, + # 1 hour + gc_interval: 3600 + config :ex_twilio, account_sid: System.get_env("TWILIO_ACCOUNT_SID"), auth_token: System.get_env("TWILIO_AUTH_TOKEN") diff --git a/backend/lib/backend/application.ex b/backend/lib/backend/application.ex index 0d84678..214c7ca 100644 --- a/backend/lib/backend/application.ex +++ b/backend/lib/backend/application.ex @@ -33,7 +33,8 @@ defmodule Backend.Application do ), Supervisor.child_spec({Task, fn -> HTTPoison.start() end}, id: :start_httpoison), Backend.Scheduler, - Backend.Elasticsearch.Cluster + Backend.Elasticsearch.Cluster, + Graph.Cache ] children = diff --git a/backend/lib/backend_web/controllers/graph_controller.ex b/backend/lib/backend_web/controllers/graph_controller.ex index 87d531f..1271838 100644 --- a/backend/lib/backend_web/controllers/graph_controller.ex +++ b/backend/lib/backend_web/controllers/graph_controller.ex @@ -1,19 +1,16 @@ defmodule BackendWeb.GraphController do use BackendWeb, :controller - - alias Backend.Api + alias Graph.Cache action_fallback BackendWeb.FallbackController def index(conn, _params) do - nodes = Api.list_nodes() - edges = Api.list_edges() + %{nodes: nodes, edges: edges} = Cache.get_graph() render(conn, "index.json", nodes: nodes, edges: edges) end def show(conn, %{"id" => domain}) do - nodes = Api.list_nodes(domain) - edges = Api.list_edges(domain) + %{nodes: nodes, edges: edges} = Cache.get_graph(domain) render(conn, "index.json", nodes: nodes, edges: edges) end end diff --git a/backend/lib/backend_web/controllers/instance_controller.ex b/backend/lib/backend_web/controllers/instance_controller.ex index b83f124..56d5f59 100644 --- a/backend/lib/backend_web/controllers/instance_controller.ex +++ b/backend/lib/backend_web/controllers/instance_controller.ex @@ -2,17 +2,17 @@ defmodule BackendWeb.InstanceController do use BackendWeb, :controller import Backend.Util - alias Backend.Api + alias Graph.Cache action_fallback(BackendWeb.FallbackController) def show(conn, %{"id" => domain}) do - instance = Api.get_instance_with_peers(domain) + instance = Cache.get_instance_with_peers(domain) if instance == nil or instance.opt_out == true do send_resp(conn, 404, "Not found") else - last_crawl = get_last_crawl(domain) + last_crawl = Cache.get_last_crawl(domain) render(conn, "show.json", instance: instance, crawl: last_crawl) end end diff --git a/backend/lib/graph/cache.ex b/backend/lib/graph/cache.ex new file mode 100644 index 0000000..8d05827 --- /dev/null +++ b/backend/lib/graph/cache.ex @@ -0,0 +1,93 @@ +defmodule Graph.Cache do + @moduledoc false + use Nebulex.Cache, + otp_app: :backend, + adapter: Nebulex.Adapters.Local + + alias Backend.{Api, Crawl, Edge, Instance, MostRecentCrawl, Repo} + alias __MODULE__ + require Logger + import Ecto.Query + + @spec get_graph(String.t() | nil) :: %{ + nodes: [Instance.t()], + edges: [Edge.t()] + } + def get_graph(domain \\ nil) do + key = + if domain != nil do + "graph_" <> domain + else + "graph" + end + + case Cache.get(key) do + nil -> + Appsignal.increment_counter("graph_cache.misses", 1) + Logger.debug("Graph cache: miss") + nodes = Api.list_nodes(domain) + edges = Api.list_edges(domain) + # Cache for 10 minutes + Cache.set(key, %{nodes: nodes, edges: edges}, ttl: 600) + %{nodes: nodes, edges: edges} + + data -> + Appsignal.increment_counter("graph_cache.hits", 1) + Logger.debug("Graph cache: hit") + data + end + end + + @spec get_instance_with_peers(String.t()) :: Instance.t() + def get_instance_with_peers(domain) do + key = "instance_" <> domain + + case Cache.get(key) do + nil -> + Appsignal.increment_counter("instance_cache.misses", 1) + Logger.debug("Instance cache: miss") + instance = Api.get_instance_with_peers(domain) + # Cache for one minute + Cache.set(key, instance, ttl: 60) + instance + + data -> + Appsignal.increment_counter("instance_cache.hits", 1) + Logger.debug("Instance cache: hit") + data + end + end + + @spec get_last_crawl(String.t()) :: Crawl.t() | nil + def get_last_crawl(domain) do + key = "most_recent_crawl_" <> domain + + most_recent_crawl_subquery = + MostRecentCrawl + |> select([mrc], %{ + most_recent_id: mrc.crawl_id + }) + |> where([mrc], mrc.instance_domain == ^domain) + + case Cache.get(key) do + nil -> + Appsignal.increment_counter("most_recent_crawl_cache.misses", 1) + Logger.debug("Most recent crawl cache: miss") + + crawl = + Crawl + |> join(:inner, [c], mrc in subquery(most_recent_crawl_subquery), + on: c.id == mrc.most_recent_id + ) + |> Repo.one() + + # Cache for one minute + Cache.set(key, crawl, ttl: 60) + + data -> + Appsignal.increment_counter("most_recent_crawl_cache.hits", 1) + Logger.debug("Most recent crawl cache: hit") + data + end + end +end diff --git a/backend/mix.exs b/backend/mix.exs index f01b3c9..581e4e5 100644 --- a/backend/mix.exs +++ b/backend/mix.exs @@ -63,7 +63,8 @@ defmodule Backend.MixProject do {:ex_twilio, "~> 0.7.0"}, {:elasticsearch, "~> 1.0"}, {:appsignal, "~> 1.10.1"}, - {:credo, "~> 1.1", only: [:dev, :test], runtime: false} + {:credo, "~> 1.1", only: [:dev, :test], runtime: false}, + {:nebulex, "~> 1.1"} ] end diff --git a/backend/mix.lock b/backend/mix.lock index 04bf223..44778e5 100644 --- a/backend/mix.lock +++ b/backend/mix.lock @@ -35,6 +35,7 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, + "nebulex": {:hex, :nebulex, "1.1.0", "be45cc3a2b7d01eb7da05747d38072d336187d05796ad9ef2d9dad9be430f915", [:mix], [{:shards, "~> 0.6", [hex: :shards, repo: "hexpm", optional: false]}], "hexpm"}, "paginator": {:hex, :paginator, "0.6.0", "bc2c01abdd98281ff39b6a7439cf540091122a7927bdaabc167c61d4508f9cbb", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -48,6 +49,7 @@ "public_suffix": {:hex, :public_suffix, "0.6.0", "100cfe86f13f9f6f0cf67e743b1b83c78dd1223a2c422fa03ebf4adff514cbc3", [:mix], [{:idna, ">= 1.2.0 and < 6.0.0", [hex: :idna, repo: "hexpm", optional: false]}], "hexpm"}, "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, + "shards": {:hex, :shards, "0.6.0", "678d292ad74a4598a872930f9b12251f43e97f6050287f1fb712fbfd3d282f75", [:make, :rebar3], [], "hexpm"}, "sobelow": {:hex, :sobelow, "0.8.0", "a3ec73e546dfde19f14818e5000c418e3f305d9edb070e79dd391de0ae1cd1ea", [:mix], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"},