Fix comments

The YouTube headers are now always added for requests to YouTube.
Previously they were only added for requests going through QUIC.

The session token is now JSON decoded to unescape escaped Unicode characters.

The comment continuation protobuf has been updated and the request now goes
through the YouTube `pbj` JSON API.

src/invidious/comments.cr

@@ -88,7 +88,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
     "cookie" => video.cookie,
   }
 
-  response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req))
+  response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US&pbj=1", headers, form: post_req))
   response = JSON.parse(response.body)
 
   if !response["response"]["continuationContents"]?
@@ -581,13 +581,17 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top")
   object = {
     "2:embedded" => {
       "2:string"    => video_id,
-      "24:varint"   => 1_i64,
-      "25:varint"   => 1_i64,
+      "25:varint"   => 0_i64,
       "28:varint"   => 1_i64,
       "36:embedded" => {
         "5:varint" => -1_i64,
         "8:varint" => 0_i64,
       },
+      "40:embedded" => {
+        "1:varint" => 4_i64,
+        "3:string" => "https://www.youtube.com",
+        "4:string" => "",
+      },
     },
     "3:varint"   => 6_i64,
     "6:embedded" => {

src/invidious/helpers/utils.cr

@@ -83,6 +83,7 @@ def make_client(url : URI, region = nil)
   # TODO: Migrate any applicable endpoints to QUIC
   client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
   client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
+  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 

src/invidious/videos.cr

@@ -830,7 +830,8 @@ def extract_polymer_config(body)
     params["reason"] = JSON::Any.new(reason)
   end
 
-  params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]?)
+  session_token_json_encoded = body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
+  params["sessionToken"] = JSON.parse(%({"key": "#{session_token_json_encoded}"}))["key"]
   params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?)
 
   return params if !player_response