mirror
/
auto-archiver
kopia lustrzana https://github.com/bellingcat/auto-archiver
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Works with multiple sheets (hardcoded)

pull/9/head
Logan Williams 2021-01-19 10:04:30 +00:00
rodzic f6abb0fb3e
commit e55a4d55dc
1 zmienionych plików z 74 dodań i 68 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

48
update.py
Wyświetl plik

@ -11,27 +11,29 @@ from botocore.errorfactory import ClientError
load_dotenv() load_dotenv()
gc = gspread.service_account() gc = gspread.service_account()
sh = gc.open("Bellingcat media archiver") sh = gc.open("Media Sheet (January 16-20 + People)")
wks = sh.sheet1
values = wks.get_all_values()
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} for ii in range(5):
ydl = youtube_dl.YoutubeDL(ydl_opts) wks = sh.get_worksheet(ii)
values = wks.get_all_values()
s3_client = boto3.client('s3', ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
ydl = youtube_dl.YoutubeDL(ydl_opts)
s3_client = boto3.client('s3',
region_name=os.getenv('DO_SPACES_REGION'), region_name=os.getenv('DO_SPACES_REGION'),
endpoint_url='https://{}.digitaloceanspaces.com'.format(os.getenv('DO_SPACES_REGION')), endpoint_url='https://{}.digitaloceanspaces.com'.format(os.getenv('DO_SPACES_REGION')),
aws_access_key_id=os.getenv('DO_SPACES_KEY'), aws_access_key_id=os.getenv('DO_SPACES_KEY'),
aws_secret_access_key=os.getenv('DO_SPACES_SECRET')) aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
for i in range(2, len(values)+1): for i in range(2, len(values)+1):
v = values[i-1] v = values[i-1]
if v[2] == "": if v[1] != "" and v[10] == "":
print(v[0]) print(v[1])
try: try:
info = ydl.extract_info(v[0], download=False) info = ydl.extract_info(v[1], download=False)
if 'entries' in info: if 'entries' in info:
if len(info['entries']) > 1: if len(info['entries']) > 1:
@ -51,10 +53,10 @@ for i in range(2, len(values)+1):
# file exists # file exists
update = [{ update = [{
'range': 'C' + str(i), 'range': 'K' + str(i),
'values': [['already archived']] 'values': [['already archived']]
}, { }, {
'range': 'D' + str(i), 'range': 'M' + str(i),
'values': [[cdn_url]] 'values': [[cdn_url]]
}] }]
@ -64,29 +66,33 @@ for i in range(2, len(values)+1):
# Not found # Not found
# sometimes this results in a different filename, so do this again # sometimes this results in a different filename, so do this again
info = ydl.extract_info(v[0], download=True) info = ydl.extract_info(v[1], download=True)
if 'entries' in info: if 'entries' in info:
filename = ydl.prepare_filename(info['entries'][0]) filename = ydl.prepare_filename(info['entries'][0])
else: else:
filename = ydl.prepare_filename(info) filename = ydl.prepare_filename(info)
if not os.path.exists(filename):
filename = filename.split('.')[0] + '.mkv'
print(filename) print(filename)
key = filename.split('/')[1] key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
# with open(filename, 'rb') as f: with open(filename, 'rb') as f:
# s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
os.remove(filename) os.remove(filename)
update = [{ update = [{
'range': 'C' + str(i), 'range': 'K' + str(i),
'values': [['successful-desktop']] 'values': [['successful']]
}, { }, {
'range': 'B' + str(i), 'range': 'L' + str(i),
'values': [[datetime.datetime.now().isoformat()]] 'values': [[datetime.datetime.now().isoformat()]]
}, { }, {
'range': 'D' + str(i), 'range': 'M' + str(i),
'values': [[cdn_url]] 'values': [[cdn_url]]
}] }]
@ -95,10 +101,10 @@ for i in range(2, len(values)+1):
t, value, traceback = sys.exc_info() t, value, traceback = sys.exc_info()
update = [{ update = [{
'range': 'C' + str(i), 'range': 'K' + str(i),
'values': [[str(value)]] 'values': [[str(value)]]
}, { }, {
'range': 'B' + str(i), 'range': 'L' + str(i),
'values': [[datetime.datetime.now().isoformat()]] 'values': [[datetime.datetime.now().isoformat()]]
}] }]