Add internet archive fallback

auto_archive.py

@@ -284,10 +284,12 @@ def update_sheet(wks, row, status, video_data, columns, v):
 
     wks.batch_update(update, value_input_option='USER_ENTERED')
 
+
 def record_stream(url, s3_client, wks, i, columns, v):
     video_data, status = download_vid(url, s3_client)
     update_sheet(wks, i, status, video_data, columns, v)
 
+
 def process_sheet(sheet):
     gc = gspread.service_account()
     sh = gc.open(sheet)
@@ -345,36 +347,56 @@ def process_sheet(sheet):
             v = values[i-1]
 
             if v[url_index] != "" and v[col_to_index(columns['status'])] == "":
-                try:
+
                 if 'http://t.me/' in v[url_index] or 'https://t.me/' in v[url_index]:
-                        video_data, status = download_telegram_video(v[url_index], s3_client, check_if_exists=True)
+                    video_data, status = download_telegram_video(
+                        v[url_index], s3_client, check_if_exists=True)
                     update_sheet(wks, i, status, video_data, columns, v)
                 else:
+                    try:
                         ydl_opts = {
                             'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
                         ydl = youtube_dl.YoutubeDL(ydl_opts)
                         info = ydl.extract_info(v[url_index], download=False)
+                        print(info)
 
                         if 'is_live' in info and info['is_live']:
-                            wks.update(columns['status'] + str(i), 'Recording stream')
+                            wks.update(columns['status'] +
-                            t = threading.Thread(target=record_stream, args=(v[url_index], s3_client, wks, i, columns, v))
+                                       str(i), 'Recording stream')
+                            t = threading.Thread(target=record_stream, args=(
+                                v[url_index], s3_client, wks, i, columns, v))
                             t.start()
                             continue
                         elif 'is_live' not in info or not info['is_live']:
-                            latest_val = wks.acell(columns['status'] + str(i)).value
+                            latest_val = wks.acell(
+                                columns['status'] + str(i)).value
 
                             # check so we don't step on each others' toes
                             if latest_val == '' or latest_val is None:
-                                wks.update(columns['status'] + str(i), 'Archive in progress')
+                                wks.update(
+                                    columns['status'] + str(i), 'Archive in progress')
                                 video_data, status = download_vid(
                                     v[url_index], s3_client, check_if_exists=True)
-                                update_sheet(wks, i, status, video_data, columns, v)
+                                update_sheet(wks, i, status,
+                                             video_data, columns, v)
+                    except:
+                        # i'm sure there's a better way to handle this than nested try/catch blocks
+                        try:
+                            r = requests.get(
+                                'https://web.archive.org/save/' + v[url_index])
 
+                            parsed = BeautifulSoup(r.content, 'html.parser')
+                            title = parsed.find_all('title')[0].text
+
+                            update_sheet(wks, i, str('Internet Archive fallback'), {
+                                         'cdn_url': r.url, 'title': title}, columns, v)
                         except:
                             # if any unexpected errors occured, log these into the Google Sheet
                             t, value, traceback = sys.exc_info()
+
                             update_sheet(wks, i, str(value), {}, columns, v)
 
+
 def main():
     parser = argparse.ArgumentParser(
         description="Automatically use youtube-dl to download media from a Google Sheet")
@@ -385,5 +407,6 @@ def main():
 
     process_sheet(args.sheet)
 
+
 if __name__ == "__main__":
     main()