kopia lustrzana https://github.com/iv-org/invidious
Use fibers to try to speed up importing of channels
rodzic
7951d4c8aa
commit
c2c224b16f
|
@ -1355,7 +1355,7 @@ get "/subscription_manager" do |env|
|
|||
subscriptions = [] of InvidiousChannel
|
||||
user.subscriptions.each do |ucid|
|
||||
begin
|
||||
subscriptions << get_channel(ucid, PG_DB, false)
|
||||
subscriptions << get_channel(ucid, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
|
@ -1475,14 +1475,7 @@ post "/data_control" do |env|
|
|||
end
|
||||
user.subscriptions.uniq!
|
||||
|
||||
user.subscriptions.select! do |ucid|
|
||||
begin
|
||||
get_channel(ucid, PG_DB, false, false)
|
||||
true
|
||||
rescue ex
|
||||
false
|
||||
end
|
||||
end
|
||||
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
|
||||
|
||||
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||
when "import_freetube"
|
||||
|
@ -1491,14 +1484,7 @@ post "/data_control" do |env|
|
|||
end
|
||||
user.subscriptions.uniq!
|
||||
|
||||
user.subscriptions.select! do |ucid|
|
||||
begin
|
||||
get_channel(ucid, PG_DB, false, false)
|
||||
true
|
||||
rescue ex
|
||||
false
|
||||
end
|
||||
end
|
||||
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
|
||||
|
||||
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||
when "import_newpipe_subscriptions"
|
||||
|
@ -1508,13 +1494,7 @@ post "/data_control" do |env|
|
|||
end
|
||||
user.subscriptions.uniq!
|
||||
|
||||
user.subscriptions.each do |ucid|
|
||||
begin
|
||||
get_channel(ucid, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
|
||||
|
||||
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||
when "import_newpipe"
|
||||
|
@ -1533,14 +1513,7 @@ post "/data_control" do |env|
|
|||
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") }
|
||||
user.subscriptions.uniq!
|
||||
|
||||
user.subscriptions.select! do |ucid|
|
||||
begin
|
||||
get_channel(ucid, PG_DB, false, false)
|
||||
true
|
||||
rescue ex
|
||||
false
|
||||
end
|
||||
end
|
||||
user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false)
|
||||
|
||||
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
|
||||
|
||||
|
|
|
@ -21,6 +21,33 @@ class ChannelVideo
|
|||
})
|
||||
end
|
||||
|
||||
def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10)
|
||||
active_threads = 0
|
||||
active_channel = Channel(String | Nil).new
|
||||
|
||||
final = [] of String
|
||||
channels.map do |ucid|
|
||||
if active_threads >= max_threads
|
||||
if response = active_channel.receive
|
||||
active_threads -= 1
|
||||
final << response
|
||||
end
|
||||
end
|
||||
|
||||
active_threads += 1
|
||||
spawn do
|
||||
begin
|
||||
get_channel(ucid, db, refresh, pull_all_videos)
|
||||
active_channel.send(ucid)
|
||||
rescue ex
|
||||
active_channel.send(nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return final
|
||||
end
|
||||
|
||||
def get_channel(id, db, refresh = true, pull_all_videos = true)
|
||||
client = make_client(YT_URL)
|
||||
|
||||
|
|
|
@ -177,19 +177,16 @@ def fetch_user(sid, headers, db)
|
|||
feed = XML.parse_html(feed.body)
|
||||
|
||||
channels = [] of String
|
||||
feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).each do |channel|
|
||||
if !{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
|
||||
channel_id = channel["href"].lstrip("/channel/")
|
||||
|
||||
begin
|
||||
channel = get_channel(channel_id, db, false, false)
|
||||
channels << channel.id
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
|
||||
if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
|
||||
nil
|
||||
else
|
||||
channel["href"].lstrip("/channel/")
|
||||
end
|
||||
end
|
||||
|
||||
channels = get_batch_channels(channels, db, false, false)
|
||||
|
||||
email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
|
||||
if email
|
||||
email = email.content.strip
|
||||
|
|
Ładowanie…
Reference in New Issue