From b49623f90fd17f2dcba5d6471f5857ca936e210e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 14 Oct 2018 11:14:27 -0500 Subject: [PATCH] Revert "Attempt to bypass channel region locks" This reverts commit 95c6747a3eb933b27bd6b4e550652e186590f821. --- src/invidious.cr | 16 ++++++++----- src/invidious/channels.cr | 47 ++------------------------------------- src/invidious/videos.cr | 5 ++++- 3 files changed, 16 insertions(+), 52 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 0c05b1ec4..d655363e1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1487,7 +1487,7 @@ get "/feed/channel/:ucid" do |env| end page = 1 - videos, count = get_60_videos(ucid, page, auto_generated, proxies) + videos, count = get_60_videos(ucid, page, auto_generated) host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?) path = env.request.path @@ -1732,7 +1732,7 @@ get "/channel/:ucid" do |env| end end - videos, count = get_60_videos(ucid, page, auto_generated, proxies) + videos, count = get_60_videos(ucid, page, auto_generated) templated "channel" end @@ -1885,6 +1885,7 @@ get "/api/v1/comments/:id" do |env| proxy_client.read_timeout = 10.seconds proxy_client.connect_timeout = 10.seconds + proxy = list.sample(1)[0] proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy_client.set_proxy(proxy) @@ -1893,10 +1894,13 @@ get "/api/v1/comments/:id" do |env| proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] proxy_html = response.body - if !proxy_html.match(//) + if proxy_html.match(//) + bypass_channel.send(nil) + else bypass_channel.send({proxy_html, proxy_client, proxy_headers}) - break end + + break rescue ex end end @@ -2489,7 +2493,7 @@ get "/api/v1/channels/:ucid" do |env| end page = 1 - videos, count = get_60_videos(ucid, page, auto_generated, proxies) + videos, count = get_60_videos(ucid, page, auto_generated) client = make_client(YT_URL) channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body @@ -2627,7 +2631,7 @@ end halt env, status_code: 404, response: error_message end - videos, count = get_60_videos(ucid, page, auto_generated, proxies) + videos, count = get_60_videos(ucid, page, auto_generated) result = JSON.build do |json| json.array do diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index f7cae7495..0ab8dd9bf 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -176,7 +176,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil) continuation = Base64.urlsafe_encode(continuation) continuation = URI.escape(continuation) - url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + url = "/browse_ajax?continuation=#{continuation}" return url end @@ -216,7 +216,7 @@ def get_about_info(ucid) return {author, ucid, auto_generated, sub_count} end -def get_60_videos(ucid, page, auto_generated, proxies) +def get_60_videos(ucid, page, auto_generated) count = 0 videos = [] of SearchVideo @@ -235,49 +235,6 @@ def get_60_videos(ucid, page, auto_generated, proxies) count += 30 end - if !json["load_more_widget_html"]?.try &.as_s.empty? && nodeset.size < 30 - bypass_channel = Channel(XML::NodeSet | Nil).new - - proxies.each do |region, list| - spawn do - list.each do |proxy| - begin - proxy_client = HTTPClient.new(YT_URL) - proxy_client.read_timeout = 10.seconds - proxy_client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - proxy_client.set_proxy(proxy) - - proxy_response = proxy_client.get(url) - json = JSON.parse(proxy_response.body) - - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - - if nodeset.size == 30 - bypass_channel.send(nodeset) - break - end - rescue ex - end - end - - if nodeset.size != 30 - bypass_channel.send(nil) - end - end - end - - proxies.size.times do - response = bypass_channel.receive - if response - nodeset = response - break - end - end - end - if auto_generated videos += extract_videos(nodeset) else diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 1d823cc4d..ddee94413 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -573,8 +573,11 @@ def fetch_video(id, proxies) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) if !info["reason"]? bypass_channel.send(proxy) - break + else + bypass_channel.send(nil) end + + break rescue ex end end