From 870cb69fde4d1bc401a464fe28c50847f4665e32 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sat, 21 Jul 2018 22:28:37 +0200 Subject: [PATCH 1/2] Added script to remove unused artifacts from repository --- scripts/clean-unused-artifacts.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 scripts/clean-unused-artifacts.py diff --git a/scripts/clean-unused-artifacts.py b/scripts/clean-unused-artifacts.py new file mode 100644 index 000000000..d38373b30 --- /dev/null +++ b/scripts/clean-unused-artifacts.py @@ -0,0 +1,30 @@ +import gitlab + +TOKEN = "CHANGEME" +CLEAN_BEFORE = "2018-07" +gl = gitlab.Gitlab("https://code.eliotberriot.com", private_token=TOKEN, per_page=100) +project = gl.projects.get("funkwhale/funkwhale") + +jobs = project.jobs.list(as_list=False) +total = jobs.total + +for job in jobs: + if job.attributes["ref"] != "develop": + continue + if job.attributes["status"] != "success": + continue + if job.attributes["tag"] is True: + continue + if job.attributes["name"] not in ["build_api", "build_front", "pages"]: + continue + if job.attributes["created_at"].startswith(CLEAN_BEFORE): + continue + relevant = { + "ref": job.attributes["ref"], + "status": job.attributes["status"], + "tag": job.attributes["tag"], + "name": job.attributes["name"], + "created_at": job.attributes["created_at"], + } + print("Deleting job {}!".format(job.id), relevant) + job.erase() From b3ae2040ec5b4a0851df92373a18e7b422da4b9e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Sun, 22 Jul 2018 11:26:33 +0200 Subject: [PATCH 2/2] Fix #149: better demo setup script, wider catalog --- api/demo/demo-user.py | 7 -- api/demo/load-demo-data.sh | 13 --- api/funkwhale_api/common/preferences.py | 5 ++ demo/README.md | 61 +++++++++++++ demo/download-tracks.sh | 21 ----- demo/env.sample | 6 ++ demo/music.txt | 8 -- demo/setup.sh | 110 +++++++++++++++++++++--- 8 files changed, 169 insertions(+), 62 deletions(-) delete mode 100644 api/demo/demo-user.py delete mode 100755 api/demo/load-demo-data.sh create mode 100644 demo/README.md delete mode 100755 demo/download-tracks.sh create mode 100644 demo/env.sample delete mode 100644 demo/music.txt mode change 100644 => 100755 demo/setup.sh diff --git a/api/demo/demo-user.py b/api/demo/demo-user.py deleted file mode 100644 index 94757d2fa..000000000 --- a/api/demo/demo-user.py +++ /dev/null @@ -1,7 +0,0 @@ -from funkwhale_api.users.models import User - - -u = User.objects.create(email="demo@demo.com", username="demo", is_staff=True) -u.set_password("demo") -u.subsonic_api_token = "demo" -u.save() diff --git a/api/demo/load-demo-data.sh b/api/demo/load-demo-data.sh deleted file mode 100755 index fab0de8b3..000000000 --- a/api/demo/load-demo-data.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /bin/bash - -echo "Loading demo data..." - -python manage.py migrate --noinput - -echo "Creating demo user..." - -cat demo/demo-user.py | python manage.py shell -i python - -echo "Importing demo tracks..." - -python manage.py import_files "/music/**/*.ogg" --recursive --noinput --username demo diff --git a/api/funkwhale_api/common/preferences.py b/api/funkwhale_api/common/preferences.py index acda9a90c..527d00c04 100644 --- a/api/funkwhale_api/common/preferences.py +++ b/api/funkwhale_api/common/preferences.py @@ -14,6 +14,11 @@ def get(pref): return manager[pref] +def set(pref, value): + manager = global_preferences_registry.manager() + manager[pref] = value + + class StringListSerializer(serializers.BaseSerializer): separator = "," sort = True diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 000000000..1fb13b5a1 --- /dev/null +++ b/demo/README.md @@ -0,0 +1,61 @@ +# Setup the demo + +We assume you want to store the demo data in `/srv/funkwhale-demo`. +This setup requires Docker and docker-compose. + +## Create the necessary directories + +`mkdir /srv/funkwhale-demo` + +## Get some music + +You can use your own music (put it in `/usr/share/music`, this is the directory the demo will look into by default). + +If you don't have any music, you can use the repository https://code.eliotberriot.com/funkwhale/catalog, which +requires Git LFS. + +## Create an env file + +Copy the `env.sample` file to ``/srv/funkwhale-demo/.env`. + +Edit the file according to your needs. + +## Copy the setup script + +Copy the `setup.sh` script to ``/srv/funkwhale-demo/setup.sh`. + +Ensure it's executable with `chmod +x setup.sh`. + +## Setup your nginx vhost + +Setup your reverse proxy for the demo as described in https://docs.funkwhale.audio/installation/index.html#nginx. + +This is outside of the scope of this guide, as you will probably want some SSL certificates, however, +ensure you point the vhost configuration to the proper static files: + +- `root` should point to `/srv/funkwhale-demo/demo/front/dist` +- `/media` and `/_protected/media` should point to `/srv/funkwhale-demo/demo/data/media/` +- `/staticfiles` should point to `/srv/funkwhale-demo/demo/data/static` + +## Launch + +Setup the demo: + +``` +cd /srv/funkwhale-demo +sudo ENV_FILE=/srv/funkwhale-demo/.env ./setup.sh +``` + +## Automate + +You'll probaby want to reset the demo every now and then. You can do that +using a cronjob: + +``` +sudo crontab -e +# in the crontab, put this: +SHELL=/bin/bash +0 */3 * * * cd /srv/funkwhale-demo && ENV_FILE=/srv/funkwhale-demo/env ./setup.sh > /srv/funkwhale-demo/crontab.log 2>&1 +``` + +This will reset and restart the demo every 3 hours. diff --git a/demo/download-tracks.sh b/demo/download-tracks.sh deleted file mode 100755 index 448b44592..000000000 --- a/demo/download-tracks.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /bin/bash -set -e -[ -z $1 ] && echo "Path to list file missing" && exit 1 - -echo "This will download tracks from zip archives listed in $1" - -LIST_CONTENT=$(cat $1) -mkdir -p data/music -cd data/music - -echo "Downloading files..." -echo "$LIST_CONTENT" | grep "^[^#;]" | xargs -n 1 curl -LO - -echo "Unzipping archives..." -find . -name "*.zip" | while read filename; do - dirname="${filename%.*}" - mkdir $dirname - unzip -o -d "$dirname" "$filename"; -done; - -echo "Done!" diff --git a/demo/env.sample b/demo/env.sample new file mode 100644 index 000000000..0c9668765 --- /dev/null +++ b/demo/env.sample @@ -0,0 +1,6 @@ +FUNKWHALE_URL=https://demo.funkwhale.audio/ +DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio +FUNKWHALE_API_PORT=5001 +DJANGO_SECRET_KEY=demo +DATABASE_URL=postgresql://postgres@postgres/postgres +CACHE_URL=redis://redis:6379/0 diff --git a/demo/music.txt b/demo/music.txt deleted file mode 100644 index 26a83c4e8..000000000 --- a/demo/music.txt +++ /dev/null @@ -1,8 +0,0 @@ -# You can get a pretty list of open music here: https://archive.org/details/jamendo-albums -https://archive.org/compress/jamendo-069098/formats=OGG%20VORBIS&file=/jamendo-069098.zip -https://archive.org/compress/jamendo-001144/formats=OGG%20VORBIS&file=/jamendo-001144.zip -https://archive.org/compress/jamendo-027690/formats=OGG%20VORBIS&file=/jamendo-027690.zip -https://archive.org/compress/jamendo-001469/formats=OGG%20VORBIS&file=/jamendo-001469.zip -https://archive.org/compress/jamendo-106323/formats=OGG%20VORBIS&file=/jamendo-106323.zip -https://archive.org/compress/jamendo-071149/formats=OGG%20VORBIS&file=/jamendo-071149.zip -https://archive.org/compress/jamendo-085030/formats=OGG%20VORBIS&file=/jamendo-085030.zip diff --git a/demo/setup.sh b/demo/setup.sh old mode 100644 new mode 100755 index e33bdf290..5edbd692c --- a/demo/setup.sh +++ b/demo/setup.sh @@ -1,30 +1,114 @@ #!/bin/bash -eux -version="develop" -music_path="/usr/share/music" -demo_path="/srv/funkwhale-demo/demo" - +version=${VERSION:-develop} +music_path=${MUSIC_PATH:-/usr/share/music} +demo_path=${DEMO_PATH:-/srv/funkwhale-demo/demo} +env_file=${ENV_FILE} echo 'Cleaning everything...' +mkdir -p $demo_path cd $demo_path /usr/local/bin/docker-compose down -v || echo 'Nothing to stop' -rm -rf /srv/funkwhale-demo/demo/* +sudo rm -rf $demo_path/* mkdir -p $demo_path echo 'Downloading demo files...' curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/docker-compose.yml" curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/$version/deploy/env.prod.sample" mkdir data/ -cp -r $music_path data/music - curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/$version/download?job=build_front" unzip front.zip -echo "FUNKWHALE_URL=https://demo.funkwhale.audio/" >> .env -echo "DJANGO_SECRET_KEY=demo" >> .env -echo "DJANGO_ALLOWED_HOSTS=demo.funkwhale.audio" >> .env +cat $env_file >> .env echo "FUNKWHALE_VERSION=$version" >> .env -echo "FUNKWHALE_API_PORT=5001" >> .env -/usr/local/bin/docker-compose pull +echo "MUSIC_DIRECTORY_SERVE_PATH=$music_path" >> .env +echo "MUSIC_DIRECTORY_PATH=$music_path" >> .env +echo "MEDIA_ROOT=$demo_path/data/media/" >> .env +echo "STATIC_ROOT=$demo_path/data/static/" >> .env + +# /usr/local/bin/docker-compose pull /usr/local/bin/docker-compose up -d postgres redis sleep 5 -/usr/local/bin/docker-compose run --rm api demo/load-demo-data.sh +cat .env +cat <