fediverse.space/backend/lib/backend/application.ex

defmodule Backend.Application do
  # See https://hexdocs.pm/elixir/Application.html
  # for more information on OTP Applications
  @moduledoc false

  use Application
  import Backend.Util

  def start(_type, _args) do
    crawl_worker_count = get_config(:crawl_workers)

    children = [
      # Start the Ecto repository
      Backend.Repo,
      # Start the endpoint when the application starts
      BackendWeb.Endpoint,
      # Crawler children
      :hackney_pool.child_spec(:crawler, timeout: 15_000, max_connections: crawl_worker_count),
      Supervisor.child_spec(
        {Task,
         fn ->
           Honeydew.start_queue(:crawl_queue, failure_mode: Honeydew.FailureMode.Abandon)
           Honeydew.start_workers(:crawl_queue, Backend.Crawler, num: crawl_worker_count)
         end},
        id: :start_honeydew
      ),
      Supervisor.child_spec({Task, fn -> HTTPoison.start() end}, id: :start_httpoison),
      Backend.Scheduler,
      Backend.Elasticsearch.Cluster,
      Graph.Cache
    ]

    children =
      case Enum.member?(["true", 1, "1"], System.get_env("SKIP_CRAWL")) do
        true -> children
        false -> children ++ [Backend.Crawler.StaleInstanceManager]
      end

    add_appsignal_probes()

    # See https://hexdocs.pm/elixir/Supervisor.html
    # for other strategies and supported options
    opts = [strategy: :one_for_one, name: Backend.Supervisor]
    Supervisor.start_link(children, opts)
  end

  # Tell Phoenix to update the endpoint configuration
  # whenever the application is updated.
  def config_change(changed, _new, removed) do
    BackendWeb.Endpoint.config_change(changed, removed)
    :ok
  end

  defp add_appsignal_probes do
    Appsignal.Probes.register(:crawler, fn ->
      %{
        queue: %{
          mnesia: mnesia
        }
      } = Honeydew.status(:crawl_queue)

      # How much memory the mnesia queue in using
      memory = mnesia |> Map.get(:"honeydew_:crawl_queue") |> Keyword.get(:memory)
      Appsignal.set_gauge("mnesia_memory", memory)

      # How many jobs are pending in the queue
      queue_length =
        Honeydew.filter(
          :crawl_queue,
          &match?(%Honeydew.Job{completed_at: nil, task: {:run, _}}, &1)
        )
        |> Enum.count()

      Appsignal.set_gauge("queue_length", queue_length)
    end)
  end
end