Translated using Weblate (Turkish)

Currently translated at 48.3% (1054 of 2181 strings)

Translation: Funkwhale/Funkwhale Web
Translate-URL: https://translate.funkwhale.audio/projects/funkwhale/front/tr/

.codespellignore

@@ -7,6 +7,7 @@ nd
 readby
 serie
 upto
+afterall
 
 # Names
 nin

.env.dev

@@ -12,10 +12,12 @@ FORWARDED_PROTO=http
 LDAP_ENABLED=False
 FUNKWHALE_SPA_HTML_ROOT=http://nginx/
 PYTHONTRACEMALLOC=0
+MEDIA_ROOT=/data/media
 
 # Uncomment this if you're using traefik/https
 # FORCE_HTTPS_URLS=True
 
 # Customize to your needs
-POSTGRES_VERSION=11
+POSTGRES_VERSION=15
 DEBUG=true
+TYPESENSE_API_KEY="apikey"

.gitignore

@@ -1,3 +1,5 @@
+/dist
+
 ### OSX ###
 .DS_Store
 .AppleDouble
@@ -83,8 +85,12 @@ front/yarn-debug.log*
 front/yarn-error.log*
 front/tests/unit/coverage
 front/tests/e2e/reports
+front/test_results.xml
+front/coverage/
 front/selenium-debug.log
 docs/_build
+#Tauri
+front/tauri/gen
 
 /data/
 .env
@@ -92,7 +98,21 @@ docs/_build
 po/*.po
 docs/swagger
 _build
-front/src/translations.json
-front/src/translations/*.json
-front/locales/en_US/LC_MESSAGES/app.po
 *.prof
+
+# Docker
+docker-bake.*.json
+metadata.json
+
+# Linting
+.eslintcache
+tsconfig.tsbuildinfo
+
+# Vscode
+.vscode/
+
+# Nix
+.direnv/
+.envrc
+flake.nix
+flake.lock

.gitlab-ci.yml

@@ -4,11 +4,15 @@ include:
     file: /templates/pre-commit.yml
   - project: funkwhale/ci
     file: /templates/lychee.yml
+  - project: funkwhale/ci
+    file: /templates/ssh-agent.yml
 
 variables:
   PYTHONDONTWRITEBYTECODE: "true"
 
   PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
+  YARN_CACHE_FOLDER: $CI_PROJECT_DIR/.cache/yarn
+  POETRY_VIRTUALENVS_IN_PROJECT: "true"
 
 .shared_variables:
   # Keep the git files permissions during job setup
@@ -17,7 +21,42 @@ variables:
     GIT_DEPTH: "5"
     FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true"
 
+.shared_caches:
+  # Cache for front related jobs
+  front_cache: &front_cache
+    - key: front-yarn
+      paths: [$YARN_CACHE_FOLDER]
+    - key:
+        prefix: front-node_modules
+        files: [front/yarn.lock]
+      paths: [front/node_modules]
+    - key:
+        prefix: front-lint
+        files:
+          - front/.eslintcache
+          - front/tsconfig.tsbuildinfo
+
+  # Cache for api related jobs
+  # Include the python version to prevent loosing caches in the test matrix
+  api_cache: &api_cache
+    - key: api-pip-$PYTHON_VERSION
+      paths: [$PIP_CACHE_DIR]
+    - key:
+        prefix: api-venv-$PYTHON_VERSION
+        files: [api/poetry.lock]
+      paths: [api/.venv]
+
+  # Cache for docs related jobs
+  docs_cache: &docs_cache
+    - key: docs-pip
+      paths: [$PIP_CACHE_DIR]
+    - key:
+        prefix: docs-venv
+        files: [docs/poetry.lock]
+      paths: [docs/.venv]
+
 default:
+  interruptible: true
   tags:
     - docker
 
@@ -31,7 +70,7 @@ workflow:
           $CI_COMMIT_TAG
         )
     # Run for merge requests from any repo or branches
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_MERGE_REQUEST_ID
 
 stages:
   - review
@@ -41,146 +80,165 @@ stages:
   - publish
 
 review_front:
-  interruptible: true
-  stage: review
-  image: node:18-alpine
-  when: manual
   allow_failure: true
+  stage: review
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+
+  image: $CI_REGISTRY/funkwhale/ci/node-python:18
   variables:
     BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/
     VUE_APP_ROUTER_BASE_URL: /-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/
     VUE_APP_INSTANCE_URL: https://demo.funkwhale.audio
     NODE_ENV: review
     NODE_OPTIONS: --max-old-space-size=4096
+  environment:
+    name: review/front/$CI_COMMIT_REF_NAME
+    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html
+  cache: *front_cache
   before_script:
-    - apk add --no-cache jq bash coreutils python3
-    - rm -rf front-review
     - mkdir front-review
     - cd front
+    - yarn install --frozen-lockfile
   script:
-    - yarn install
-    # this is to ensure we don't have any errors in the output,
-    # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
-    - yarn run build --base ./ | tee /dev/stderr | (! grep -i 'ERROR in')
+    - yarn run build --base ./
     - cp -r dist/* ../front-review
   artifacts:
     expire_in: 2 weeks
     paths:
       - front-review
-  cache:
-    key: "funkwhale__front_dependencies"
-    paths:
-      - front/node_modules
-      - front/yarn.lock
-  environment:
-    name: review/front/$CI_COMMIT_REF_NAME
-    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/front-review/index.html
 
 review_docs:
-  interruptible: true
-  stage: review
   allow_failure: true
-  image: python:3.11
-  variables:
-    BUILD_PATH: "../docs-review"
+  stage: review
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes: [docs/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-docs:3.11
+  environment:
+    name: review/docs/$CI_COMMIT_REF_NAME
+    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
+  cache: *docs_cache
   before_script:
-    - rm -rf docs-review
-    - mkdir docs-review
     - cd docs
-    - apt-get update
-    - apt-get install -y graphviz git
-    - pip install poetry
-    - poetry install
-    - git switch develop && git pull
-    - git switch stable && git pull
-    - git switch $CI_COMMIT_BRANCH && git pull
+    - make install
   script:
-    - poetry run python3 -m sphinx . $BUILD_PATH
-  cache:
-    key: "$CI_PROJECT_ID__sphinx"
-    paths:
-      - "$PIP_CACHE_DIR"
+    - make build BUILD_DIR=../docs-review
   artifacts:
     expire_in: 2 weeks
     paths:
       - docs-review
-  environment:
-    name: review/docs/$CI_COMMIT_REF_NAME
-    url: http://$CI_PROJECT_NAMESPACE.pages.funkwhale.audio/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/docs-review/index.html
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-        - docs/**/*
-    - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH
-      when: manual
 
 find_broken_links:
+  extends: [.lychee]
   allow_failure:
     exit_codes: 2
 
-  extends: [.lychee]
   script:
     - >
       lychee
       --cache
       --no-progress
       --exclude-all-private
-      --exclude-mail
       --exclude 'demo\.funkwhale\.audio'
       --exclude 'nginx\.com'
       --exclude-path 'docs/_templates/'
       -- . || exit $?
 
-changelog_snippet:
-  interruptible: true
-  image: alpine:3.17
+require_changelog:
   stage: lint
-  before_script:
-    - apk add git
-    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
-  script:
-    - git diff --name-only FETCH_HEAD | grep "changes/changelog.d/*"
   rules:
-    - if: $CI_COMMIT_AUTHOR == 'Renovate Bot <bot@dev.funkwhale.audio>'
+    # Don't run on merge request that mention NOCHANGELOG or renovate bot commits
+    - if: >
+        $CI_MERGE_REQUEST_TITLE =~ /NOCHANGELOG/ ||
+        $CI_COMMIT_AUTHOR == "Renovate Bot <bot@dev.funkwhale.audio>"
       when: never
-    - if: $CI_MERGE_REQUEST_TITLE =~ /NOCHANGELOG/
-      when: never
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+  image: python:3.11
+  script:
+    - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA..$CI_COMMIT_SHA | grep "changes/changelog.d/*"
 
 pre-commit:
   extends: [.pre-commit]
 
-eslint:
-  interruptible: true
-  image: node:18-alpine
+lint_api:
+  allow_failure: true
   stage: lint
+  needs: []
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [api/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:3.11
+  before_script:
+    - cd api
+    - make install
+  script:
+    - make lint
+
+lint_front:
+  stage: lint
+  needs: []
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [front/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/node-python:18
+  cache: *front_cache
   before_script:
     - cd front
-    - apk add --no-cache jq bash coreutils python3
-    - yarn install
+    - yarn install --frozen-lockfile
   script:
     - yarn lint --max-warnings 0
     - yarn lint:tsc
-  cache:
-    key: "$CI_PROJECT_ID__eslint_npm_cache"
-    paths:
-      - front/node_modules
+
+test_scripts:
+  stage: test
+  needs: []
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-        - front/**/*
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [scripts/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
+  cache:
+    - key: scripts-pip
+      paths: [$PIP_CACHE_DIR]
+    - key:
+        prefix: scripts-venv
+        files: [scripts/poetry.lock]
+      paths: [scripts/.venv]
+  before_script:
+    - cd scripts
+    - make install
+  script:
+    - make test
 
 test_api:
-  interruptible: true
-  services:
-    - postgres:15-alpine
-    - redis:7-alpine
-  stage: test
   retry: 1
-  cache:
-    key: "$CI_PROJECT_ID__pip_cache"
-    paths:
-      - "$PIP_CACHE_DIR"
+  stage: test
+  needs:
+    - job: lint_api
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [api/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:$PYTHON_VERSION
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+  services:
+    - name: postgres:15-alpine
+      command:
+        - --fsync=off
+        - --full_page_writes=off
+        - --synchronous_commit=off
+    - name: redis:7-alpine
+  cache: *api_cache
   variables:
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
@@ -189,9 +247,16 @@ test_api:
     CACHE_URL: "redis://redis:6379/0"
   before_script:
     - cd api
-    - poetry install --no-root
+    - make install
   script:
-    - poetry run pytest --cov-report xml --cov-report term-missing:skip-covered --cov=funkwhale_api --junitxml=report.xml tests/
+    - >
+      poetry run pytest
+      --junitxml=report.xml
+      --cov
+      --cov-config=pyproject.toml
+      --cov-report=term-missing:skip-covered
+      --cov-report=xml
+      tests
   artifacts:
     expire_in: 2 weeks
     reports:
@@ -199,60 +264,84 @@ test_api:
       coverage_report:
         coverage_format: cobertura
         path: api/coverage.xml
-  parallel:
-    matrix:
-      - PY_VER: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-  image: $CI_REGISTRY/funkwhale/backend-test-docker:$PY_VER
   coverage: '/TOTAL\s*\d*\s*\d*\s*(\d*%)/'
+
+test_front:
+  stage: test
+  needs:
+    - job: lint_front
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
-      changes:
-        - api/**/*
-    - if: $CI_COMMIT_REF_PROTECTED == "true"
-      when: always
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [front/**/*]
 
-# Those tests are disabled for now since no vitest dom emulation is providing
-# AudioContext, which is required for our HTML audio player
-#test_front:
-#  interruptible: true
-#  stage: test
-#  image: node:18-alpine
-#  before_script:
-#    - cd front
-#    - apk add --no-cache jq bash coreutils python3
-#  script:
-#    - yarn install --check-files
-#    - yarn test:unit
-#  cache:
-#    key: "funkwhale__front_dependencies"
-#    paths:
-#      - front/node_modules
-#      - front/yarn.lock
-#  artifacts:
-#    name: "front_${CI_COMMIT_REF_NAME}"
-#    paths:
-#      - front/dist/
-#    reports:
-#      junit: front/coverage/cobertura-coverage.xml
-#  tags:
-#    - docker
-#  rules:
-#    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
-#      changes:
-#        - front/**/*
-#    - if: $CI_COMMIT_REF_PROTECTED == "true"
-#      when: always
+  image: $CI_REGISTRY/funkwhale/ci/node-python:18
+  cache: *front_cache
+  before_script:
+    - cd front
+    - yarn install --frozen-lockfile
+  script:
+    - yarn test:unit
+  artifacts:
+    reports:
+      junit: front/test_results.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: front/coverage/cobertura-coverage.xml
+  coverage: '/All files\s+(?:\|\s+((?:\d+\.)?\d+)\s+){4}.*/'
 
-build_openapi_schema:
+build_metadata:
   stage: build
-  image: $CI_REGISTRY/funkwhale/backend-test-docker:3.11
+
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
+  variables:
+    GIT_FETCH_EXTRA_FLAGS: --prune
+  script:
+    - make build-metadata
+    - make docker-metadata
+  artifacts:
+    reports:
+      dotenv: build_metadata.env
+    paths:
+      - docker-bake.json
+      - docker-bake.api.json
+      - docker-bake.front.json
+
+test_integration:
+  stage: test
+  rules:
+    - if: $RUN_CYPRESS
+  interruptible: true
+
+  image:
+    name: cypress/included:13.6.4
+    entrypoint: [""]
+  cache:
+    - *front_cache
+    - key:
+      paths:
+        - /root/.cache/Cypress
+  before_script:
+    - cd front
+    - yarn install
+  script:
+    - yarn run cypress run
+
+build_api_schema:
+  stage: build
+  needs:
+    - job: test_api
+      optional: true
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [api/**/*]
+    # Add build_docs rules because it depends on the build_api_schema artifact
+    - changes: [docs/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-api:3.11
   services:
     - postgres:15-alpine
     - redis:7-alpine
-  cache:
-    key: "$CI_PROJECT_ID__pip_cache"
-    paths:
-      - "$PIP_CACHE_DIR"
+  cache: *api_cache
   variables:
     DATABASE_URL: "postgresql://postgres@postgres/postgres"
     FUNKWHALE_URL: "https://funkwhale.ci"
@@ -262,8 +351,7 @@ build_openapi_schema:
     API_TYPE: "v1"
   before_script:
     - cd api
-    - pip3 install poetry
-    - poetry install
+    - make install
     - poetry run funkwhale-manage migrate
   script:
     - poetry run funkwhale-manage spectacular --file ../docs/schema.yml
@@ -272,162 +360,192 @@ build_openapi_schema:
     paths:
       - docs/schema.yml
 
-build_documentation:
+build_docs:
   stage: build
-  image: python:3.11
   needs:
-    - job: build_openapi_schema
+    - job: build_api_schema
       artifacts: true
-  variables:
-    BUILD_PATH: "../public"
-    GIT_STRATEGY: clone
-    GIT_DEPTH: 0
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [docs/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python-funkwhale-docs:3.11
+  cache: *docs_cache
   before_script:
     - cd docs
-    - apt-get update
-    - apt-get install -y graphviz
-    - pip install poetry
-    - poetry install
-    - git branch stable --track origin/stable || true
-    - git branch develop --track origin/develop || true
+    - make install
   script:
-    - ./build_docs.sh
-  cache:
-    key: "$CI_PROJECT_ID__sphinx"
-    paths:
-      - "$PIP_CACHE_DIR"
+    - make build-all BUILD_DIR=../public
   artifacts:
     expire_in: 2 weeks
     paths:
       - public
-  rules:
-    - if: $CI_COMMIT_BRANCH == "stable" || $CI_COMMIT_BRANCH == "develop"
-      when: always
-    - changes:
-        - docs/**/*
-      when: always
 
 build_front:
   stage: build
-  image: node:18-alpine
+  needs:
+    # The test_front job is currently disabled
+    # - job: test_front
+    - job: lint_front
+      optional: true
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [front/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/node-python:18
   variables:
     <<: *keep_git_files_permissions
     NODE_OPTIONS: --max-old-space-size=4096
+  cache: *front_cache
   before_script:
-    - apk add --no-cache jq bash coreutils python3
     - cd front
+    - yarn install --frozen-lockfile
   script:
-    - yarn install
-    # this is to ensure we don't have any errors in the output,
-    # cf https://dev.funkwhale.audio/funkwhale/funkwhale/issues/169
-    - yarn run build:deployment | tee /dev/stderr | (! grep -i 'ERROR in')
+    - yarn run build:deployment
   artifacts:
     name: front_${CI_COMMIT_REF_NAME}
     paths:
-      - front/dist/
-  only:
-    - tags@funkwhale/funkwhale
-    - stable@funkwhale/funkwhale
-    - develop@funkwhale/funkwhale
+      - front/dist
 
 build_api:
   stage: build
-  image: bash
+  needs:
+    - job: test_api
+      optional: true
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [api/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
   variables:
     <<: *keep_git_files_permissions
   script:
     - rm -rf api/tests
     - >
-      if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "stable" ]; then
+      if [[ -z "$CI_COMMIT_TAG" ]]; then
         ./scripts/set-api-build-metadata.sh $CI_COMMIT_SHORT_SHA;
       fi
   artifacts:
     name: api_${CI_COMMIT_REF_NAME}
     paths:
       - api
-  only:
-    - tags@funkwhale/funkwhale
-    - stable@funkwhale/funkwhale
-    - develop@funkwhale/funkwhale
 
-deploy_documentation:
-  stage: publish
-  image: alpine
-  needs:
-    - job: build_documentation
-      artifacts: true
+build_tauri:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+    - changes: [front/**/*]
+
+  image: $CI_REGISTRY/funkwhale/ci/node-tauri:18
+  variables:
+    <<: *keep_git_files_permissions
   before_script:
-    - apk add openssh-client rsync
-    - mkdir -p ~/.ssh
-    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-    - eval `ssh-agent -s`
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - source /root/.cargo/env
+    - yarn install
   script:
-    - rsync -r --delete -e "ssh -p 2282" $CI_PROJECT_DIR/public/ docs@docs.funkwhale.audio:/htdocs/$CI_COMMIT_REF_NAME
+    - yarn tauri build --verbose
+  artifacts:
+    name: desktop_${CI_COMMIT_REF_NAME}
+    paths:
+      - front/tauri/target/release/bundle/appimage/*.AppImage
+
+deploy_docs:
+  interruptible: false
+  extends: .ssh-agent
+  stage: publish
+  needs:
+    - job: build_docs
+      artifacts: true
   rules:
     - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
 
-.docker_publish:
-  stage: publish
-  image: egon0/docker-with-buildx-and-git:bash
-  parallel:
-    matrix:
-      - COMPONENT: ["api", "front"]
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - rsync -r --delete -e "ssh -p 2282" $CI_PROJECT_DIR/public/ docs@docs.funkwhale.audio:/htdocs/$CI_COMMIT_REF_NAME
+
+docker:
+  interruptible: false
+  tags: [docker, privileged, multiarch]
+  stage: build
+  needs:
+    - job: build_metadata
+      artifacts: true
+    - job: test_api
+      optional: true
+    - job: test_front
+      optional: true
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        BUILD_ARGS: >
+          --set *.platform=linux/amd64,linux/arm64,linux/arm/v7
+          --no-cache
+          --push
+
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
+      variables:
+        BUILD_ARGS: >
+          --set *.platform=linux/amd64,linux/arm64,linux/arm/v7
+          --set *.cache-from=type=registry,ref=$DOCKER_CACHE_IMAGE:$CI_COMMIT_BRANCH,oci-mediatypes=false
+          --set *.cache-to=type=registry,ref=$DOCKER_CACHE_IMAGE:$CI_COMMIT_BRANCH,mode=max,oci-mediatypes=false
+          --push
+
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_NAMESPACE == "funkwhale"
+      # We don't provide priviledged runners to everyone, so we can only build docker images in the funkwhale group
+      variables:
+        BUILD_ARGS: >
+          --set *.platform=linux/amd64
+          --set *.cache-from=type=registry,ref=$DOCKER_CACHE_IMAGE:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME,oci-mediatypes=false
+
+  image: $CI_REGISTRY/funkwhale/ci/docker:20
+  services:
+    - docker:20-dind
   variables:
     <<: *keep_git_files_permissions
-
-    IMAGE_NAME: funkwhale/$COMPONENT
-    IMAGE: $IMAGE_NAME:$CI_COMMIT_REF_NAME
-    IMAGE_LATEST: $IMAGE_NAME:latest
-
     DOCKER_HOST: tcp://docker:2375/
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
-  tags:
-    - multiarch
-  services:
-    - docker:20-dind
+    BUILDKIT_PROGRESS: plain
+
+    DOCKER_CACHE_IMAGE: $CI_REGISTRY/funkwhale/funkwhale/cache
   before_script:
-    - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
-  cache:
-    key: docker_public_${CI_COMMIT_REF_NAME}
+    - >
+      echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY";
+      if [[ "$BUILD_ARGS" =~ "--push" ]]; then
+        echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_LOGIN" --password-stdin docker.io;
+      fi
+  script:
+    - >
+      if [[ -z "$CI_COMMIT_TAG" ]]; then
+        ./scripts/set-api-build-metadata.sh $CI_COMMIT_SHORT_SHA;
+      fi
+    - docker buildx create --use
+    - make docker-build BUILD_ARGS="--metadata-file metadata.json $BUILD_ARGS"
+    - cat metadata.json
+  artifacts:
+    name: docker_metadata_${CI_COMMIT_REF_NAME}
     paths:
-      - ~/.cargo
+      - metadata.json
 
-docker_publish_stable_release:
-  # Publish a docker image for releases
-  extends: .docker_publish
+package:
+  stage: publish
+  needs:
+    - job: build_metadata
+      artifacts: true
+    - job: build_api
+      artifacts: true
+    - job: build_front
+      artifacts: true
+    - job: build_tauri
+      artifacts: true
   rules:
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^[0-9]+(.[0-9]+){1,2}$/
-  script:
-    # Check if this is the latest release
-    - ./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG - && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=;
-    - export major="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1)"
-    - export minor="$(echo $CI_COMMIT_REF_NAME | cut -d '.' -f 1,2)"
-    - cd $COMPONENT
-    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
-    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE $DOCKER_LATEST_TAG -t $IMAGE_NAME:$major -t $IMAGE_NAME:$minor .
+    - if: $CI_COMMIT_BRANCH =~ /(stable|develop)/
 
-docker_publish_unstable_release:
-  # Publish a docker image for releases
-  extends: .docker_publish
-  rules:
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME !~ /^[0-9]+(.[0-9]+){1,2}$/
+  image: $CI_REGISTRY/funkwhale/ci/python:3.11
+  variables:
+    <<: *keep_git_files_permissions
   script:
-    # Check if this is the latest release
-    - cd $COMPONENT
-    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
-    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
-
-docker_publish_non-release:
-  # Publish a docker image for each commit on develop
-  extends: .docker_publish
-  only:
-    - develop@funkwhale/funkwhale
-    - stable@funkwhale/funkwhale
-  script:
-    - cd $COMPONENT
-    - docker buildx create --use --name A$CI_COMMIT_SHORT_SHA
-    - docker buildx build --platform $BUILD_PLATFORMS --push -t $IMAGE .
+    - make package
+    - scripts/ci-upload-packages.sh

.gitlab/renovate.json

@@ -25,6 +25,16 @@
       "branchConcurrentLimit": 0,
       "prConcurrentLimit": 0
     },
+    {
+      "matchBaseBranches": ["develop"],
+      "matchUpdateTypes": ["major"],
+      "prPriority": 2
+    },
+    {
+      "matchBaseBranches": ["develop"],
+      "matchUpdateTypes": ["minor"],
+      "prPriority": 1
+    },
     {
       "matchUpdateTypes": ["major", "minor"],
       "matchBaseBranches": ["stable"],
@@ -35,12 +45,6 @@
       "matchBaseBranches": ["stable"],
       "enabled": false
     },
-    {
-      "matchUpdateTypes": ["patch", "pin", "digest"],
-      "matchBaseBranches": ["develop"],
-      "automerge": true,
-      "automergeType": "branch"
-    },
     {
       "matchManagers": ["npm"],
       "addLabels": ["Area::Frontend"]
@@ -70,6 +74,10 @@
         ],
         "fileFilters": ["changes/changelog.d/postgres.update"]
       }
+    },
+    {
+      "matchPackageNames": ["python"],
+      "rangeStrategy": "widen"
     }
   ]
 }

.gitpod.yml

@@ -14,11 +14,12 @@ tasks:
       docker-compose up -d
 
       poetry env use python
-      poetry install
+      make install
 
       gp ports await 5432
 
       poetry run funkwhale-manage migrate
+      poetry run funkwhale-manage fw users create --superuser --username gitpod --password funkwhale --email test@example.org
       poetry run funkwhale-manage gitpod init
     command: |
       echo "MEDIA_URL=`gp url 8000`/media/" >> ../.gitpod/.env
@@ -47,49 +48,66 @@ tasks:
       yarn install
     command: yarn dev --host 0.0.0.0 --base ./
 
+  - name: Documentation
+    before: cd docs
+    init: make install
+    command: make dev
+
   - name: Welcome to Funkwhale development!
     env:
       COMPOSE_FILE: /workspace/funkwhale/.gitpod/docker-compose.yml
       ENV_FILE: /workspace/funkwhale/.gitpod/.env
       VUE_EDITOR: code
       DJANGO_SETTINGS_MODULE: config.settings.local
-    init: pre-commit install
+    init: |
+      pre-commit install
+      pre-commit run --all
     command: |
-      pre-commit run --all && clear
       echo ""
       echo -e "    ⠀⠀⠸⣿⣷⣦⣄⣠⣶⣾⣿⠇⠀⠀    You can now start developing Funkwhale with gitpod!"
       echo -e "    ⠀⠀⠀⠈⠉⠻⣿⣿⠟⠉⠁⠀⠀⠀"
       echo -e "    \u1b[34m⣀⠀⢀⡀⢀⣀\u1b[0m⠹⠇\u1b[34m⣀⡀⢀⡀⠀⣀    \u1b[0mTo sign in to the superuser account,"
       echo -e "    \u1b[34m⢻⣇⠘⣧⡈⠻⠶⠶⠟⢁⣾⠃⣸⡟    \u1b[0mplease use these credentials:"
       echo -e "    \u1b[34m⠀⠻⣦⡈⠻⠶⣶⣶⠶⠟⢁⣴⠟⠀"
-      echo -e "    \u1b[34m⠀⠀⠈⠻⠷⣦⣤⣤⣴⠾⠟⠁⠀⠀    gitpod\u1b[0m:\u1b[34mgitpod"
+      echo -e "    \u1b[34m⠀⠀⠈⠻⠷⣦⣤⣤⣴⠾⠟⠁⠀⠀    gitpod\u1b[0m:\u1b[34mfunkwhale"
       echo ""
 
 ports:
-  - port: 8000
+  - name: Funkwhale
+    port: 8000
     visibility: public
     onOpen: notify
 
-  - port: 5000
+  - name: Funkwhale API
+    port: 5000
     visibility: private
     onOpen: ignore
 
-  - port: 5432
+  - name: PostgreSQL
+    port: 5432
     visibility: private
     onOpen: ignore
 
-  - port: 5678
+  - name: Debugpy
+    port: 5678
     visibility: private
     onOpen: ignore
 
-  - port: 6379
+  - name: Redis
+    port: 6379
     visibility: private
     onOpen: ignore
 
-  - port: 8080
+  - name: Frontend
+    port: 8080
     visibility: private
     onOpen: ignore
 
+  - name: Documentation
+    port: 8001
+    visibility: public
+    onOpen: notify
+
 vscode:
   extensions:
     - Vue.volar

.gitpod/Dockerfile

@@ -1,9 +1,13 @@
-FROM gitpod/workspace-full:2022-11-15-17-00-18
+FROM gitpod/workspace-full:2023-10-25-20-43-33
 USER gitpod
 
 RUN sudo apt update -y \
     && sudo apt install libsasl2-dev libldap2-dev libssl-dev ffmpeg gettext -y
 
-RUN pip install poetry pre-commit \
+RUN pyenv install 3.11 && pyenv global 3.11
+
+RUN brew install neovim
+
+RUN pip install poetry pre-commit jinja2 towncrier \
     && poetry config virtualenvs.create true \
     && poetry config virtualenvs.in-project true

.gitpod/docker-compose.yml

@@ -18,7 +18,6 @@ services:
       - 6379:6379
 
   nginx:
-    command: /entrypoint.sh
     env_file:
       - ./.env
     image: nginx
@@ -29,15 +28,16 @@ services:
     environment:
       - "NGINX_MAX_BODY_SIZE=100M"
       - "FUNKWHALE_API_IP=host.docker.internal"
+      - "FUNKWHALE_API_HOST=host.docker.internal"
       - "FUNKWHALE_API_PORT=5000"
       - "FUNKWHALE_FRONT_IP=host.docker.internal"
       - "FUNKWHALE_FRONT_PORT=8080"
       - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-host.docker.internal}"
+      - "FUNKWHALE_PROTOCOL=https"
     volumes:
-      - ../data/media:/protected/media:ro
+      - ../data/media:/workspace/funkwhale/data/media:ro
       - ../data/music:/music:ro
-      - ../data/staticfiles:/staticfiles:ro
+      - ../data/staticfiles:/usr/share/nginx/html/staticfiles/:ro
       - ../deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro
-      - ../docker/nginx/conf.dev:/etc/nginx/nginx.conf.template:ro
-      - ../docker/nginx/entrypoint.sh:/entrypoint.sh:ro
+      - ../docker/nginx/conf.dev:/etc/nginx/templates/default.conf.template:ro
       - ../front:/frontend:ro

.pre-commit-config.yaml

@@ -24,11 +24,22 @@ repos:
       - id: mixed-line-ending
       - id: trailing-whitespace
 
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.5.1
+    hooks:
+      - id: poetry-check
+        files: ^api/pyproject.toml$
+        args: [--directory=api]
+
+      - id: poetry-lock
+        files: ^api/pyproject.toml$
+        args: [--directory=api, --check]
+
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.9.0
     hooks:
       - id: pyupgrade
-        args: [--py37-plus]
+        args: [--py38-plus]
         exclude: ^(api/.*/migrations/.*)
 
   - repo: https://github.com/psf/black
@@ -42,7 +53,7 @@ repos:
       - id: isort
 
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
 

.vscode/launch.json

@@ -1,36 +0,0 @@
-{
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Attach python debugger",
-      "type": "python",
-      "request": "attach",
-      "connect": {
-        "host": "localhost",
-        "port": 5678
-      },
-      "django": true
-    },
-    {
-      "name": "Debug python",
-      "type": "python",
-      "request": "launch",
-      "module": "uvicorn",
-      "cwd": "${workspaceFolder}/api",
-      "envFile": "${workspaceFolder}/.gitpod/.env",
-      "args": [
-        "--reload",
-        "config.asgi:application",
-        "--host",
-        "0.0.0.0",
-        "--port",
-        "5000",
-        "--reload-dir",
-        "config/",
-        "--reload-dir",
-        "funkwhale_api/"
-      ],
-      "django": true
-    }
-  ]
-}

.vscode/markdown.code-snippets

@@ -1,108 +0,0 @@
-{
-	"Callout": {
-		"prefix": "callout",
-		"description": "Add a callout to highlight information",
-		"body": [
-			":::{${1|attention,caution,danger,error,hint,important,note,seealso,tip,warning|}} ${2:Optional title}",
-			"${0:${TM_SELECTED_TEXT}}",
-			":::"
-		],
-	},
-	"Code tabs": {
-		"prefix": "code-tabs",
-		"description": "Insert a set of code tabs",
-		"body": [
-			":::{tab-set-code}",
-			"",
-			"$0",
-			"",
-			":::"
-		]
-	},
-	"Tab set": {
-		"prefix": "tab-set",
-		"description": "Insert a group of generic tabs",
-		"body": [
-			"::::{tab-set}",
-			":::{tab-item} ${1:Tab title}",
-			"$2",
-			":::",
-			"",
-			":::{tab-item} ${3:Tab title}",
-			"$0",
-			":::",
-			"",
-			"::::"
-		]
-	},
-	"Insert fragment": {
-		"prefix": "insert fragment",
-		"description": "Insert reusable text from another file",
-		"body": [
-			":::{include} ${1:full path to file}",
-			":start-after: ${2:the text to start after}",
-			":end-before: ${0:the text to end before}",
-			":::"
-		]
-	},
-	"Dropdown": {
-		"prefix": "insert dropdown",
-		"description": "Insert a dropdown (accordion)",
-		"body": [
-			":::{dropdown} ${1:title}",
-			"${0:${TM_SELECTED_TEXT}}",
-			":::"
-		]
-	},
-	"Versioning": {
-		"prefix": "version change",
-		"description": "Specify when a feature was added, changed, or deprecated",
-		"body": [
-			":::{${1|versionadded,versionchanged,deprecated|}} ${2:v4.32.0}",
-			"${0:${TM_SELECTED_TEXT}}",
-			":::"
-		]
-	},
-	"List table": {
-		"prefix": "list table",
-		"description": "Insert a table defined as a set of lists",
-		"body": [
-			":::{list-table} ${1:Optional title}",
-			":header-rows: ${2:Number of header rows}",
-			"",
-			"* - ${3: First row column 1}",
-			"   - ${4: First row column 2}",
-			"* - ${5: Second row column 1}",
-			"   - ${0: Second row column 2}",
-			":::"
-		]
-	},
-	"Guilabel": {
-		"prefix": "guilabel",
-		"description": "Format text as a GUI label (e.g. a button label or interface label",
-		"body": [
-			"{guilabel}`${0:${TM_SELECTED_TEXT}}`"
-		]
-	},
-	"File": {
-		"prefix": "file",
-		"description": "Format text as a file name or path",
-		"body": [
-			"{file}`${0:${TM_SELECTED_TEXT}}`"
-		]
-	},
-	"Increase indent": {
-		"prefix": "increase indent",
-		"description": "Increase the indentation of all selected colon or backtick fences",
-		"body": [
-			"${TM_SELECTED_TEXT/((?<c>[`:])\\k<c>{2,})/$1$2/gm}"
-		]
-	},
-	"Deprecation warning": {
-		"prefix": "insert deprecation warning",
-		"description": "Inserts an inline deprecation badge. Useful in tables of parameters",
-		"body": [
-			"{bdg-warning}`Deprecated`"
-		]
-	}
-}

.vscode/settings.json

@@ -1,14 +0,0 @@
-{
-  "python.defaultInterpreterPath": "/workspace/funkwhale/api/.venv/bin/python",
-  "python.testing.cwd": "/workspace/funkwhale/api",
-  "python.envFile": "/workspace/funkwhale/.gitpod/.env",
-  "python.testing.pytestArgs": ["--cov=funkwhale_api", "tests/"],
-  "python.testing.unittestEnabled": false,
-  "python.testing.pytestEnabled": true,
-  "vitest.enable": true,
-  "vitest.commandLine": "yarn vitest",
-  "i18n-ally.localesPaths": ["front/src/locales"],
-  "i18n-ally.pathMatcher": "*.json",
-  "i18n-ally.enabledFrameworks": ["vue"],
-  "i18n-ally.keystyle": "nested"
-}

CODEOWNERS

@@ -0,0 +1 @@
+*.md @funkwhale/documentation

Makefile

@@ -0,0 +1,57 @@
+SHELL := bash
+CPU_CORES := $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N )))
+
+BAKE_FILES = \
+	docker-bake.json \
+	docker-bake.api.json \
+	docker-bake.front.json
+
+docker-bake.%.json:
+	./scripts/build_metadata.py --format bake --bake-target $* --bake-image docker.io/funkwhale/$* > $@
+
+docker-metadata: $(BAKE_FILES)
+
+docker-build: docker-metadata
+	docker buildx bake $(foreach FILE,$(BAKE_FILES), --file $(FILE)) --print $(BUILD_ARGS)
+	docker buildx bake $(foreach FILE,$(BAKE_FILES), --file $(FILE)) 		 $(BUILD_ARGS)
+
+build-metadata:
+	./scripts/build_metadata.py --format env | tee build_metadata.env
+
+BUILD_DIR = dist
+package:
+	rm -Rf $(BUILD_DIR)
+	mkdir -p $(BUILD_DIR)
+	tar --create --gunzip --file='$(BUILD_DIR)/funkwhale-api.tar.gz' \
+		--owner='root' \
+		--group='root' \
+		--exclude-vcs \
+		api/config \
+		api/funkwhale_api \
+		api/install_os_dependencies.sh \
+		api/manage.py \
+		api/poetry.lock \
+		api/pyproject.toml \
+		api/Readme.md
+
+	cd '$(BUILD_DIR)' && \
+	tar --extract --gunzip --file='funkwhale-api.tar.gz' && \
+	zip -q 'funkwhale-api.zip' -r api && \
+	rm -Rf api
+
+	tar --create --gunzip --file='$(BUILD_DIR)/funkwhale-front.tar.gz' \
+		--owner='root' \
+		--group='root' \
+		--exclude-vcs \
+		--transform='s/^front\/dist/front/' \
+		front/dist
+
+	cd '$(BUILD_DIR)' && \
+	tar --extract --gunzip --file='funkwhale-front.tar.gz' && \
+	zip -q 'funkwhale-front.zip' -r front && \
+	rm -Rf front
+
+	cd '$(BUILD_DIR)' && \
+	cp ../front/tauri/target/release/bundle/appimage/funkwhale_*.AppImage FunkwhaleDesktop.AppImage
+
+	cd '$(BUILD_DIR)' && sha256sum * > SHA256SUMS

README.md

@@ -23,4 +23,4 @@ If you find a security issue or vulnerability, please report it on our [GitLab i
 
 ## Code of conduct
 
-The Funkwhale collective adheres to a [code of conduct](https://funkwhale.audio/en_US/code-of-conduct) in all our community spaces. Please familiarize yourself with this code and follow it when participating in discussions in our spaces.
+The Funkwhale collective adheres to a [code of conduct](https://funkwhale.audio/code-of-conduct) in all our community spaces. Please familiarize yourself with this code and follow it when participating in discussions in our spaces.

api/Dockerfile

@@ -1,16 +1,18 @@
-FROM alpine:3.17 as pre-build
+FROM alpine:3.19 as requirements
 
-# We need this additional step to avoid having poetrys deps interacting with our
-# dependencies. This is only required until alpine 3.16 is released, since this
-# allows us to install poetry as package.
+RUN set -eux; \
+  apk add --no-cache \
+  poetry \
+  py3-cryptography \
+  py3-pip \
+  python3
 
-RUN apk add --no-cache python3 py3-cryptography py3-pip poetry
 COPY pyproject.toml poetry.lock /
-RUN poetry export --without-hashes > requirements.txt
-RUN poetry export --with dev --without-hashes > dev-requirements.txt
+RUN set -eux; \
+  poetry export --without-hashes --extras typesense > requirements.txt; \
+  poetry export --without-hashes --with dev > dev-requirements.txt;
 
-
-FROM alpine:3.17 as builder
+FROM alpine:3.19 as builder
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
@@ -22,6 +24,7 @@ RUN set -eux; \
   cargo \
   curl \
   gcc \
+  g++ \
   git \
   jpeg-dev \
   libffi-dev \
@@ -34,22 +37,22 @@ RUN set -eux; \
   openssl-dev \
   postgresql-dev \
   zlib-dev \
-  py3-cryptography=38.0.3-r1 \
+  py3-cryptography=41.0.7-r0 \
   py3-lxml=4.9.3-r1 \
-  py3-pillow=9.3.0-r0 \
-  py3-psycopg2=2.9.5-r0 \
-  py3-watchfiles=0.18.1-r0 \
+  py3-pillow=10.3.0-r0 \
+  py3-psycopg2=2.9.9-r0 \
+  py3-watchfiles=0.19.0-r1 \
   python3-dev
 
-# create virtual env for next stage
+# Create virtual env
 RUN python3 -m venv --system-site-packages /venv
-# emulate activation by prefixing PATH
-ENV PATH="/venv/bin:/root/.local/bin:$PATH" VIRTUAL_ENV=/venv
+ENV PATH="/venv/bin:$PATH"
 
-COPY --from=pre-build /requirements.txt /requirements.txt
-COPY --from=pre-build /dev-requirements.txt /dev-requirements.txt
+COPY --from=requirements /requirements.txt /requirements.txt
+COPY --from=requirements /dev-requirements.txt /dev-requirements.txt
 
-RUN set -eux; \
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
   pip3 install --upgrade pip; \
   pip3 install setuptools wheel; \
   # Currently we are unable to relieably build rust-based packages on armv7. This
@@ -58,25 +61,26 @@ RUN set -eux; \
   # to install the deps using pip.
   grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /requirements.txt \
   | pip3 install -r /dev/stdin \
-  cryptography==38.0.3 \
+  cryptography==41.0.7 \
   lxml==4.9.3 \
-  pillow==9.3.0 \
-  psycopg2==2.9.5 \
-  watchfiles==0.18.1
+  pillow==10.2.0 \
+  psycopg2==2.9.9 \
+  watchfiles==0.19.0
 
 ARG install_dev_deps=0
-RUN set -eux; \
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
   if [ "$install_dev_deps" = "1" ] ; then \
     grep -Ev 'cryptography|lxml|pillow|psycopg2|watchfiles' /dev-requirements.txt \
     | pip3 install -r /dev/stdin \
-    cryptography==38.0.3 \
+    cryptography==41.0.7 \
     lxml==4.9.3 \
-    pillow==9.3.0 \
-    psycopg2==2.9.5 \
-    watchfiles==0.18.1; \
+    pillow==10.2.0 \
+    psycopg2==2.9.9 \
+    watchfiles==0.19.0; \
   fi
 
-FROM alpine:3.17 as image
+FROM alpine:3.19 as production
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
@@ -93,21 +97,22 @@ RUN set -eux; \
   libpq \
   libxml2 \
   libxslt \
-  py3-cryptography=38.0.3-r1 \
+  py3-cryptography=41.0.7-r0 \
   py3-lxml=4.9.3-r1 \
-  py3-pillow=9.3.0-r0 \
-  py3-psycopg2=2.9.5-r0 \
-  py3-watchfiles=0.18.1-r0 \
-  python3
+  py3-pillow=10.3.0-r0 \
+  py3-psycopg2=2.9.9-r0 \
+  py3-watchfiles=0.19.0-r1 \
+  python3 \
+  tzdata
 
 COPY --from=builder /venv /venv
-# emulate activation by prefixing PATH
 ENV PATH="/venv/bin:$PATH"
 
 COPY . /app
 WORKDIR /app
 
-RUN set -eux; \
+RUN --mount=type=cache,target=~/.cache/pip; \
+  set -eux; \
   pip3 install --no-deps --editable .
 
 ENV IS_DOCKER_SETUP=true

api/Makefile

@@ -0,0 +1,14 @@
+SHELL := bash
+CPU_CORES := $(shell N=$$(nproc); echo $$(( $$N > 4 ? 4 : $$N )))
+
+.PHONY: install lint
+
+install:
+	poetry install --all-extras
+
+lint:
+	poetry run pylint \
+		--jobs=$(CPU_CORES) \
+		--output-format=colorized \
+		--recursive=true \
+		config funkwhale_api tests

api/config/__init__.py

@@ -1,3 +0,0 @@
-# loads what is required to generate the swagger docs
-# https://matrix.to/#/!nNBDNverFlbfNpReEO:matrix.org/$16579878472182UmZUv:tchncs.de?via=tchncs.de&via=matrix.org&via=juniorjpdj.pl
-import config.schema  # noqa: F401

api/config/api_urls.py

@@ -1,97 +0,0 @@
-from django.conf.urls import include, url
-from rest_framework import routers
-from rest_framework.urlpatterns import format_suffix_patterns
-
-from funkwhale_api.activity import views as activity_views
-from funkwhale_api.audio import views as audio_views
-from funkwhale_api.common import routers as common_routers
-from funkwhale_api.common import views as common_views
-from funkwhale_api.music import views
-from funkwhale_api.playlists import views as playlists_views
-from funkwhale_api.subsonic.views import SubsonicViewSet
-from funkwhale_api.tags import views as tags_views
-
-router = common_routers.OptionalSlashRouter()
-router.register(r"activity", activity_views.ActivityViewSet, "activity")
-router.register(r"tags", tags_views.TagViewSet, "tags")
-router.register(r"plugins", common_views.PluginViewSet, "plugins")
-router.register(r"tracks", views.TrackViewSet, "tracks")
-router.register(r"uploads", views.UploadViewSet, "uploads")
-router.register(r"libraries", views.LibraryViewSet, "libraries")
-router.register(r"listen", views.ListenViewSet, "listen")
-router.register(r"stream", views.StreamViewSet, "stream")
-router.register(r"artists", views.ArtistViewSet, "artists")
-router.register(r"channels", audio_views.ChannelViewSet, "channels")
-router.register(r"subscriptions", audio_views.SubscriptionsViewSet, "subscriptions")
-router.register(r"albums", views.AlbumViewSet, "albums")
-router.register(r"licenses", views.LicenseViewSet, "licenses")
-router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
-router.register(r"mutations", common_views.MutationViewSet, "mutations")
-router.register(r"attachments", common_views.AttachmentViewSet, "attachments")
-v1_patterns = router.urls
-
-subsonic_router = routers.SimpleRouter(trailing_slash=False)
-subsonic_router.register(r"subsonic/rest", SubsonicViewSet, basename="subsonic")
-
-
-v1_patterns += [
-    url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
-    url(
-        r"^instance/",
-        include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
-    ),
-    url(
-        r"^manage/",
-        include(("funkwhale_api.manage.urls", "manage"), namespace="manage"),
-    ),
-    url(
-        r"^moderation/",
-        include(
-            ("funkwhale_api.moderation.urls", "moderation"), namespace="moderation"
-        ),
-    ),
-    url(
-        r"^federation/",
-        include(
-            ("funkwhale_api.federation.api_urls", "federation"), namespace="federation"
-        ),
-    ),
-    url(
-        r"^providers/",
-        include(("funkwhale_api.providers.urls", "providers"), namespace="providers"),
-    ),
-    url(
-        r"^favorites/",
-        include(("funkwhale_api.favorites.urls", "favorites"), namespace="favorites"),
-    ),
-    url(r"^search$", views.Search.as_view(), name="search"),
-    url(
-        r"^radios/",
-        include(("funkwhale_api.radios.urls", "radios"), namespace="radios"),
-    ),
-    url(
-        r"^history/",
-        include(("funkwhale_api.history.urls", "history"), namespace="history"),
-    ),
-    url(
-        r"^",
-        include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
-    ),
-    # XXX: remove if Funkwhale 1.1
-    url(
-        r"^users/",
-        include(("funkwhale_api.users.api_urls", "users"), namespace="users-nested"),
-    ),
-    url(
-        r"^oauth/",
-        include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
-    ),
-    url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
-    url(
-        r"^text-preview/?$", common_views.TextPreviewView.as_view(), name="text-preview"
-    ),
-]
-
-urlpatterns = [
-    url(r"^v1/", include((v1_patterns, "v1"), namespace="v1"))
-] + format_suffix_patterns(subsonic_router.urls, allowed=["view"])

api/config/plugins.py

@@ -303,6 +303,23 @@ LISTENING_CREATED = "listening_created"
 """
 Called when a track is being listened
 """
+LISTENING_SYNC = "listening_sync"
+"""
+Called by the task manager to trigger listening sync
+"""
+FAVORITE_CREATED = "favorite_created"
+"""
+Called when a track is being favorited
+"""
+FAVORITE_DELETED = "favorite_deleted"
+"""
+Called when a favorited track is being unfavorited
+"""
+FAVORITE_SYNC = "favorite_sync"
+"""
+Called by the task manager to trigger favorite sync
+"""
+
 SCAN = "scan"
 """
 

api/config/routing.py

@@ -1,7 +1,7 @@
 from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
-from django.conf.urls import url
 from django.core.asgi import get_asgi_application
+from django.urls import re_path
 
 from funkwhale_api.instance import consumers
 
@@ -10,7 +10,12 @@ application = ProtocolTypeRouter(
         # Empty for now (http->django views is added by default)
         "websocket": AuthMiddlewareStack(
             URLRouter(
-                [url("^api/v1/activity$", consumers.InstanceActivityConsumer.as_asgi())]
+                [
+                    re_path(
+                        "^api/v1/activity$",
+                        consumers.InstanceActivityConsumer.as_asgi(),
+                    )
+                ]
             )
         ),
         "http": get_asgi_application(),

api/config/settings/common.py

@@ -2,7 +2,7 @@ import logging.config
 import sys
 import warnings
 from collections import OrderedDict
-from urllib.parse import urlsplit
+from urllib.parse import urlparse, urlsplit
 
 import environ
 from celery.schedules import crontab
@@ -13,7 +13,29 @@ APPS_DIR = ROOT_DIR.path("funkwhale_api")
 
 env = environ.Env()
 ENV = env
-LOGLEVEL = env("LOGLEVEL", default="info").upper()
+# If DEBUG is `true`, we automatically set the loglevel to "DEBUG"
+# If DEBUG is `false`, we try to read the level from LOGLEVEL environment and default to "INFO"
+LOGLEVEL = (
+    "DEBUG" if env.bool("DEBUG", False) else env("LOGLEVEL", default="info").upper()
+)
+"""
+Default logging level for the Funkwhale processes.
+
+.. note::
+    The `DEBUG` variable overrides the `LOGLEVEL` if it is set to `TRUE`.
+
+    The `LOGLEVEL` value only applies if `DEBUG` is `false` or not present.
+
+Available levels:
+
+- ``debug``
+- ``info``
+- ``warning``
+- ``error``
+- ``critical``
+
+"""
+
 IS_DOCKER_SETUP = env.bool("IS_DOCKER_SETUP", False)
 
 
@@ -35,19 +57,6 @@ if env("FUNKWHALE_SENTRY_DSN", default=None) is not None:
     )
     sentry_sdk.set_tag("instance", env("FUNKWHALE_HOSTNAME"))
 
-"""
-Default logging level for the Funkwhale processes
-
-Available levels:
-
-- ``debug``
-- ``info``
-- ``warning``
-- ``error``
-- ``critical``
-
-"""  # pylint: disable=W0105
-
 LOGGING_CONFIG = None
 logging.config.dictConfig(
     {
@@ -187,9 +196,7 @@ request errors related to this.
 FUNKWHALE_SPA_HTML_CACHE_DURATION = env.int(
     "FUNKWHALE_SPA_HTML_CACHE_DURATION", default=60 * 15
 )
-FUNKWHALE_EMBED_URL = env(
-    "FUNKWHALE_EMBED_URL", default=FUNKWHALE_URL + "/front/embed.html"
-)
+FUNKWHALE_EMBED_URL = env("FUNKWHALE_EMBED_URL", default=FUNKWHALE_URL + "/embed.html")
 FUNKWHALE_SPA_REWRITE_MANIFEST = env.bool(
     "FUNKWHALE_SPA_REWRITE_MANIFEST", default=True
 )
@@ -217,6 +224,13 @@ ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=[]) + [FUNKWHALE_HOSTNA
 List of allowed hostnames for which the Funkwhale server will answer.
 """
 
+CSRF_TRUSTED_ORIGINS = [urlparse(o, FUNKWHALE_PROTOCOL).geturl() for o in ALLOWED_HOSTS]
+"""
+List of origins that are trusted for unsafe requests
+We simply consider all allowed hosts to be trusted origins
+See https://docs.djangoproject.com/en/4.2/ref/settings/#csrf-trusted-origins
+"""
+
 # APP CONFIGURATION
 # ------------------------------------------------------------------------------
 DJANGO_APPS = (
@@ -262,6 +276,7 @@ LOCAL_APPS = (
     # Your stuff: custom apps go here
     "funkwhale_api.instance",
     "funkwhale_api.audio",
+    "funkwhale_api.contrib.listenbrainz",
     "funkwhale_api.music",
     "funkwhale_api.requests",
     "funkwhale_api.favorites",
@@ -272,6 +287,7 @@ LOCAL_APPS = (
     "funkwhale_api.playlists",
     "funkwhale_api.subsonic",
     "funkwhale_api.tags",
+    "funkwhale_api.typesense",
 )
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
@@ -281,9 +297,9 @@ ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
 List of Django apps to load in addition to Funkwhale plugins and apps.
 """
 INSTALLED_APPS = (
-    DJANGO_APPS
+    LOCAL_APPS
+    + DJANGO_APPS
     + THIRD_PARTY_APPS
-    + LOCAL_APPS
     + tuple(ADDITIONAL_APPS)
     + tuple(plugins.trigger_filter(plugins.PLUGINS_APPS, [], enabled=True))
 )
@@ -822,7 +838,7 @@ If you're using password auth (the extra slash is important)
 .. note::
 
     If you want to use Redis over unix sockets, you also need to update
-    :attr:`CELERY_BROKER_URL`, because the scheme differ from the one used by
+    :attr:`CELERY_BROKER_URL`, because the scheme differs from the one used by
     :attr:`CACHE_URL`.
 
 """
@@ -873,7 +889,7 @@ to use a different server or use Redis sockets to connect.
 
 Example:
 
-- ``redis://127.0.0.1:6379/0``
+- ``unix://127.0.0.1:6379/0``
 - ``redis+socket:///run/redis/redis.sock?virtual_host=0``
 
 """
@@ -934,8 +950,25 @@ CELERY_BEAT_SCHEDULE = {
         ),
         "options": {"expires": 60 * 60},
     },
+    "listenbrainz.trigger_listening_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_listening_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
+    "listenbrainz.trigger_favorite_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_favorite_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
 }
 
+if env.str("TYPESENSE_API_KEY", default=None):
+    CELERY_BEAT_SCHEDULE["typesense.build_canonical_index"] = {
+        "task": "typesense.build_canonical_index",
+        "schedule": crontab(day_of_week="*/2", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    }
+
 if env.bool("ADD_ALBUM_TAGS_FROM_TRACKS", default=True):
     CELERY_BEAT_SCHEDULE["music.albums_set_tags_from_tracks"] = {
         "task": "music.albums_set_tags_from_tracks",
@@ -1180,7 +1213,7 @@ if BROWSABLE_API_ENABLED:
         "rest_framework.renderers.BrowsableAPIRenderer",
     )
 
-REST_AUTH_SERIALIZERS = {
+REST_AUTH = {
     "PASSWORD_RESET_SERIALIZER": "funkwhale_api.users.serializers.PasswordResetSerializer",  # noqa
     "PASSWORD_RESET_CONFIRM_SERIALIZER": "funkwhale_api.users.serializers.PasswordResetConfirmSerializer",  # noqa
 }
@@ -1459,3 +1492,22 @@ instead of request header.
 
 HASHING_ALGORITHM = "sha256"
 HASHING_CHUNK_SIZE = 1024 * 100
+
+"""
+Typenses Settings
+"""
+TYPESENSE_API_KEY = env("TYPESENSE_API_KEY", default=None)
+""" Typesense API key. This need to be defined in the .env file for Typenses to be activated."""
+TYPESENSE_PORT = env("TYPESENSE_PORT", default="8108")
+"""Typesense listening port"""
+TYPESENSE_PROTOCOL = env("TYPESENSE_PROTOCOL", default="http")
+"""Typesense listening protocol"""
+TYPESENSE_HOST = env(
+    "TYPESENSE_HOST",
+    default="typesense" if IS_DOCKER_SETUP else "localhost",
+)
+"""
+Typesense hostname. Defaults to `localhost` on non-Docker deployments and to `typesense` on
+Docker deployments.
+"""
+TYPESENSE_NUM_TYPO = env("TYPESENSE_NUM_TYPO", default=5)

api/config/settings/local.py

@@ -76,7 +76,7 @@ DEBUG_TOOLBAR_PANELS = [
 
 # django-extensions
 # ------------------------------------------------------------------------------
-# INSTALLED_APPS += ('django_extensions', )
+INSTALLED_APPS += ("django_extensions",)
 
 INSTALLED_APPS += ("drf_spectacular",)
 
@@ -96,8 +96,6 @@ CELERY_TASK_ALWAYS_EAGER = False
 
 # Your local stuff: Below this line define 3rd party library settings
 
-CSRF_TRUSTED_ORIGINS = [o for o in ALLOWED_HOSTS]
-
 REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "funkwhale_api.schema.CustomAutoSchema"
 SPECTACULAR_SETTINGS = {
     "TITLE": "Funkwhale API",
@@ -149,3 +147,5 @@ MIDDLEWARE = (
     "funkwhale_api.common.middleware.ProfilerMiddleware",
     "funkwhale_api.common.middleware.PymallocMiddleware",
 ) + MIDDLEWARE
+
+TYPESENSE_API_KEY = "apikey"

api/config/settings/production.py

@@ -41,14 +41,6 @@ SECRET_KEY = env("DJANGO_SECRET_KEY")
 # SESSION_COOKIE_HTTPONLY = True
 # SECURE_SSL_REDIRECT = env.bool("DJANGO_SECURE_SSL_REDIRECT", default=True)
 
-# SITE CONFIGURATION
-# ------------------------------------------------------------------------------
-# Hosts/domain names that are valid for this site
-# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
-CSRF_TRUSTED_ORIGINS = ALLOWED_HOSTS
-
-# END SITE CONFIGURATION
-
 # Static Assets
 # ------------------------
 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"

api/config/settings/testing.py

@@ -0,0 +1,9 @@
+import os
+
+os.environ.setdefault("FUNKWHALE_URL", "http://funkwhale.dev")
+
+from .common import *  # noqa
+
+DEBUG = True
+SECRET_KEY = "a_super_secret_key!"
+TYPESENSE_API_KEY = "apikey"

api/config/urls/__init__.py

@@ -1,7 +1,6 @@
 from django.conf import settings
-from django.conf.urls import url
 from django.conf.urls.static import static
-from django.urls import include, path
+from django.urls import include, path, re_path
 from django.views import defaults as default_views
 
 from config import plugins
@@ -10,34 +9,34 @@ from funkwhale_api.common import admin
 plugins_patterns = plugins.trigger_filter(plugins.URLS, [], enabled=True)
 
 api_patterns = [
-    url("v1/", include("config.urls.api")),
-    url("v2/", include("config.urls.api_v2")),
-    url("subsonic/", include("config.urls.subsonic")),
+    re_path("v1/", include("config.urls.api")),
+    re_path("v2/", include("config.urls.api_v2")),
+    re_path("subsonic/", include("config.urls.subsonic")),
 ]
 
 
 urlpatterns = [
     # Django Admin, use {% url 'admin:index' %}
-    url(settings.ADMIN_URL, admin.site.urls),
-    url(r"^api/", include((api_patterns, "api"), namespace="api")),
-    url(
+    re_path(settings.ADMIN_URL, admin.site.urls),
+    re_path(r"^api/", include((api_patterns, "api"), namespace="api")),
+    re_path(
         r"^",
         include(
             ("funkwhale_api.federation.urls", "federation"), namespace="federation"
         ),
     ),
-    url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
-    url(r"^accounts/", include("allauth.urls")),
+    re_path(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
+    re_path(r"^accounts/", include("allauth.urls")),
 ] + plugins_patterns
 
 if settings.DEBUG:
     # This allows the error pages to be debugged during development, just visit
     # these url in browser to see how these error pages look like.
     urlpatterns += [
-        url(r"^400/$", default_views.bad_request),
-        url(r"^403/$", default_views.permission_denied),
-        url(r"^404/$", default_views.page_not_found),
-        url(r"^500/$", default_views.server_error),
+        re_path(r"^400/$", default_views.bad_request),
+        re_path(r"^403/$", default_views.permission_denied),
+        re_path(r"^404/$", default_views.page_not_found),
+        re_path(r"^500/$", default_views.server_error),
     ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
     if "debug_toolbar" in settings.INSTALLED_APPS:
@@ -49,5 +48,5 @@ if settings.DEBUG:
 
     if "silk" in settings.INSTALLED_APPS:
         urlpatterns = [
-            url(r"^api/silk/", include("silk.urls", namespace="silk"))
+            re_path(r"^api/silk/", include("silk.urls", namespace="silk"))
         ] + urlpatterns

api/config/urls/api.py

@@ -1,4 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 
 from funkwhale_api.activity import views as activity_views
 from funkwhale_api.audio import views as audio_views
@@ -28,61 +29,61 @@ router.register(r"attachments", common_views.AttachmentViewSet, "attachments")
 v1_patterns = router.urls
 
 v1_patterns += [
-    url(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
-    url(
+    re_path(r"^oembed/$", views.OembedView.as_view(), name="oembed"),
+    re_path(
         r"^instance/",
         include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
     ),
-    url(
+    re_path(
         r"^manage/",
         include(("funkwhale_api.manage.urls", "manage"), namespace="manage"),
     ),
-    url(
+    re_path(
         r"^moderation/",
         include(
             ("funkwhale_api.moderation.urls", "moderation"), namespace="moderation"
         ),
     ),
-    url(
+    re_path(
         r"^federation/",
         include(
             ("funkwhale_api.federation.api_urls", "federation"), namespace="federation"
         ),
     ),
-    url(
+    re_path(
         r"^providers/",
         include(("funkwhale_api.providers.urls", "providers"), namespace="providers"),
     ),
-    url(
+    re_path(
         r"^favorites/",
         include(("funkwhale_api.favorites.urls", "favorites"), namespace="favorites"),
     ),
-    url(r"^search$", views.Search.as_view(), name="search"),
-    url(
+    re_path(r"^search$", views.Search.as_view(), name="search"),
+    re_path(
         r"^radios/",
         include(("funkwhale_api.radios.urls", "radios"), namespace="radios"),
     ),
-    url(
+    re_path(
         r"^history/",
         include(("funkwhale_api.history.urls", "history"), namespace="history"),
     ),
-    url(
+    re_path(
         r"^",
         include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
     ),
     # XXX: remove if Funkwhale 1.1
-    url(
+    re_path(
         r"^users/",
         include(("funkwhale_api.users.api_urls", "users"), namespace="users-nested"),
     ),
-    url(
+    re_path(
         r"^oauth/",
         include(("funkwhale_api.users.oauth.urls", "oauth"), namespace="oauth"),
     ),
-    url(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
-    url(
+    re_path(r"^rate-limit/?$", common_views.RateLimitView.as_view(), name="rate-limit"),
+    re_path(
         r"^text-preview/?$", common_views.TextPreviewView.as_view(), name="text-preview"
     ),
 ]
 
-urlpatterns = [url("", include((v1_patterns, "v1"), namespace="v1"))]
+urlpatterns = [re_path("", include((v1_patterns, "v1"), namespace="v1"))]

api/config/urls/api_v2.py

@@ -1,4 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 
 from funkwhale_api.common import routers as common_routers
 
@@ -6,10 +7,14 @@ router = common_routers.OptionalSlashRouter()
 v2_patterns = router.urls
 
 v2_patterns += [
-    url(
+    re_path(
         r"^instance/",
-        include(("funkwhale_api.instance.urls", "instance"), namespace="instance"),
+        include(("funkwhale_api.instance.urls_v2", "instance"), namespace="instance"),
+    ),
+    re_path(
+        r"^radios/",
+        include(("funkwhale_api.radios.urls_v2", "radios"), namespace="radios"),
     ),
 ]
 
-urlpatterns = [url("", include((v2_patterns, "v2"), namespace="v2"))]
+urlpatterns = [re_path("", include((v2_patterns, "v2"), namespace="v2"))]

api/config/urls/subsonic.py

@@ -1,4 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 from rest_framework import routers
 from rest_framework.urlpatterns import format_suffix_patterns
 
@@ -8,7 +9,9 @@ subsonic_router = routers.SimpleRouter(trailing_slash=False)
 subsonic_router.register(r"rest", SubsonicViewSet, basename="subsonic")
 
 subsonic_patterns = format_suffix_patterns(subsonic_router.urls, allowed=["view"])
-urlpatterns = [url("", include((subsonic_patterns, "subsonic"), namespace="subsonic"))]
+urlpatterns = [
+    re_path("", include((subsonic_patterns, "subsonic"), namespace="subsonic"))
+]
 
 # urlpatterns = [
 #    url(

api/funkwhale_api/__init__.py

@@ -1,7 +1,4 @@
-__version__ = "1.3.4"
-__version_info__ = tuple(
-    [
-        int(num) if num.isdigit() else num
-        for num in __version__.replace("-", ".", 1).split(".")
-    ]
-)
+from importlib.metadata import version as get_version
+
+version = get_version("funkwhale_api")
+__version__ = version

api/funkwhale_api/activity/utils.py

@@ -48,4 +48,5 @@ def get_activity(user, limit=20):
         ),
     ]
     records = combined_recent(limit=limit, querysets=querysets)
+
     return [r["object"] for r in records]

api/funkwhale_api/audio/serializers.py

@@ -1,10 +1,10 @@
 import datetime
 import logging
+import sys
 import time
 import uuid
 
 import feedparser
-import pytz
 import requests
 from django.conf import settings
 from django.db import transaction
@@ -33,6 +33,11 @@ from funkwhale_api.users import serializers as users_serializers
 
 from . import categories, models
 
+if sys.version_info < (3, 9):
+    from backports.zoneinfo import ZoneInfo
+else:
+    from zoneinfo import ZoneInfo
+
 logger = logging.getLogger(__name__)
 
 
@@ -769,7 +774,7 @@ class RssFeedItemSerializer(serializers.Serializer):
         if "published_parsed" in validated_data:
             track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
                 time.mktime(validated_data["published_parsed"])
-            ).replace(tzinfo=pytz.utc)
+            ).replace(tzinfo=ZoneInfo("UTC"))
 
         upload_defaults = {
             "source": validated_data["links"]["audio"]["source"],

api/funkwhale_api/cli/users.py

@@ -38,7 +38,7 @@ def handler_create_user(
     utils.logger.debug("Creating user…")
     user = serializer.save(request=request)
     utils.logger.debug("Setting permissions and other attributes…")
-    user.is_staff = is_staff
+    user.is_staff = is_staff or is_superuser  # Always set staff if superuser is set
     user.upload_quota = upload_quota
     user.is_superuser = is_superuser
     for permission in permissions:

api/funkwhale_api/common/authentication.py

@@ -1,6 +1,6 @@
-from allauth.account.utils import send_email_confirmation
+from allauth.account.models import EmailAddress
 from django.core.cache import cache
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from oauth2_provider.contrib.rest_framework.authentication import (
     OAuth2Authentication as BaseOAuth2Authentication,
 )
@@ -20,9 +20,13 @@ def resend_confirmation_email(request, user):
     if cache.get(cache_key):
         return False
 
-    done = send_email_confirmation(request, user)
+    # We do the sending of the conformation by hand because we don't want to pass the request down
+    # to the email rendering, which would cause another UnverifiedEmail Exception and restarts the sending
+    # again and again
+    email = EmailAddress.objects.get_for_user(user, user.email)
+    email.send_confirmation()
     cache.set(cache_key, True, THROTTLE_DELAY)
-    return done
+    return True
 
 
 class OAuth2Authentication(BaseOAuth2Authentication):

api/funkwhale_api/common/filters.py

@@ -1,5 +1,6 @@
 from django import forms
 from django.db.models import Q
+from django.db.models.functions import Lower
 from django_filters import rest_framework as filters
 from django_filters import widgets
 from drf_spectacular.utils import extend_schema_field
@@ -239,3 +240,19 @@ class ActorScopeFilter(filters.CharFilter):
             raise EmptyQuerySet()
 
         return Q(**{self.actor_field: actor})
+
+
+class CaseInsensitiveNameOrderingFilter(filters.OrderingFilter):
+    def filter(self, qs, value):
+        order_by = []
+
+        if value is None:
+            return qs
+
+        for param in value:
+            if param == "name":
+                order_by.append(Lower("name"))
+            else:
+                order_by.append(self.get_ordering_value(param))
+
+        return qs.order_by(*order_by)

api/funkwhale_api/common/management/commands/createsuperuser.py

@@ -0,0 +1,22 @@
+import os
+
+from django.contrib.auth.management.commands.createsuperuser import (
+    Command as BaseCommand,
+)
+from django.core.management.base import CommandError
+
+
+class Command(BaseCommand):
+    def handle(self, *apps_label, **options):
+        """
+        Creating Django Superusers would bypass some of our username checks, which can lead to unexpected behaviour.
+        We therefore prohibit the execution of the command.
+        """
+        if not os.environ.get("FORCE") == "1":
+            raise CommandError(
+                "Running createsuperuser on your Funkwhale instance bypasses some of our checks "
+                "which can lead to unexpected behavior of your instance. We therefore suggest to "
+                "run `funkwhale-manage fw users create --superuser` instead."
+            )
+
+        return super().handle(*apps_label, **options)

api/funkwhale_api/common/management/commands/gitpod.py

@@ -36,22 +36,7 @@ class Command(BaseCommand):
         self.stdout.write("")
 
     def init(self):
-        try:
-            user = User.objects.get(username="gitpod")
-        except Exception:
-            call_command(
-                "createsuperuser",
-                username="gitpod",
-                email="gitpod@example.com",
-                no_input=False,
-            )
-            user = User.objects.get(username="gitpod")
-
-        user.set_password("gitpod")
-        if not user.actor:
-            user.create_actor()
-
-        user.save()
+        user = User.objects.get(username="gitpod")
 
         # Allow anonymous access
         preferences.set("common__api_authentication_required", False)

api/funkwhale_api/common/management/commands/inplace_to_s3.py

@@ -0,0 +1,95 @@
+import pathlib
+from argparse import RawTextHelpFormatter
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.music import models
+
+
+class Command(BaseCommand):
+    help = """
+    Update the reference for Uploads that have been imported with --in-place and are now moved to s3.
+
+    Please note: This does not move any file! Make sure you already moved the files to your s3 bucket.
+
+    Specify --source to filter the reference to update to files from a specific in-place directory. If no
+    --source is given, all in-place imported track references will be updated.
+
+    Specify --target to specify a subdirectory in the S3 bucket where you moved the files. If no --target is
+    given, the file is expected to be stored in the same path as before.
+
+    Examples:
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target unset
+
+    All files imported from /music will be updated and expected to be in the same folder structure in the bucket
+
+    Music File: /music/Artist/Album/track.ogg
+    --source: /music
+    --target: /in_place
+
+    The music file is expected to be stored in the bucket in the directory /in_place/Artist/Album/track.ogg
+    """
+
+    def create_parser(self, *args, **kwargs):
+        parser = super().create_parser(*args, **kwargs)
+        parser.formatter_class = RawTextHelpFormatter
+        return parser
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_false",
+            dest="dry_run",
+            default=True,
+            help="Disable dry run mode and apply updates for real on the database",
+        )
+        parser.add_argument(
+            "--source",
+            type=pathlib.Path,
+            required=True,
+            help="Specify the path of the directory where the files originally were stored to update their reference.",
+        )
+        parser.add_argument(
+            "--target",
+            type=pathlib.Path,
+            help="Specify a subdirectory in the S3 bucket where you moved the files to.",
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        if options["dry_run"]:
+            self.stdout.write("Dry-run on, will not touch the database")
+        else:
+            self.stdout.write("Dry-run off, *changing the database*")
+        self.stdout.write("")
+
+        prefix = f"file://{options['source']}"
+
+        to_change = models.Upload.objects.filter(source__startswith=prefix)
+
+        self.stdout.write(f"Found {to_change.count()} uploads to update.")
+
+        target = options.get("target")
+        if target is None:
+            target = options["source"]
+
+        for upl in to_change:
+            upl.audio_file = str(upl.source).replace(str(prefix), str(target))
+            upl.source = None
+            self.stdout.write(f"Upload expected in {upl.audio_file}")
+            if not options["dry_run"]:
+                upl.save()
+
+        self.stdout.write("")
+        if options["dry_run"]:
+            self.stdout.write(
+                "Nothing was updated, rerun this command with --no-dry-run to apply the changes"
+            )
+        else:
+            self.stdout.write("Updating completed!")
+
+        self.stdout.write("")

api/funkwhale_api/common/management/commands/script.py

@@ -26,7 +26,7 @@ class Command(BaseCommand):
             script = available_scripts[name]
         except KeyError:
             raise CommandError(
-                "{} is not a valid script. Run funkwhale-manage for a "
+                "{} is not a valid script. Run funkwhale-manage script for a "
                 "list of available scripts".format(name)
             )
 
@@ -43,7 +43,7 @@ class Command(BaseCommand):
     def show_help(self):
         self.stdout.write("")
         self.stdout.write("Available scripts:")
-        self.stdout.write("Launch with: funkwhale-manage <script_name>")
+        self.stdout.write("Launch with: funkwhale-manage script <script_name>")
         available_scripts = self.get_scripts()
         for name, script in sorted(available_scripts.items()):
             self.stdout.write("")

api/funkwhale_api/common/management/commands/testdata.py

@@ -0,0 +1,43 @@
+from django.core.management.commands.migrate import Command as BaseCommand
+
+from funkwhale_api.federation import factories
+from funkwhale_api.federation.models import Actor
+
+
+class Command(BaseCommand):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.help = "Helper to generate randomized testdata"
+        self.type_choices = {"notifications": self.handle_notifications}
+        self.missing_args_message = f"Please specify one of the following sub-commands: {*self.type_choices.keys(), }"
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest="subcommand")
+
+        notification_parser = subparsers.add_parser("notifications")
+        notification_parser.add_argument(
+            "username", type=str, help="Username to send the notifications to"
+        )
+        notification_parser.add_argument(
+            "--count", type=int, help="Number of elements to create", default=1
+        )
+
+    def handle(self, *args, **options):
+        self.type_choices[options["subcommand"]](options)
+
+    def handle_notifications(self, options):
+        self.stdout.write(
+            f"Create {options['count']} notification(s) for {options['username']}"
+        )
+        try:
+            actor = Actor.objects.get(preferred_username=options["username"])
+        except Actor.DoesNotExist:
+            self.stdout.write(
+                "The user you want to create notifications for does not exist"
+            )
+            return
+
+        follow_activity = factories.ActivityFactory(type="Follow")
+        for _ in range(options["count"]):
+            factories.InboxItemFactory(actor=actor, activity=follow_activity)

api/funkwhale_api/common/middleware.py

@@ -150,7 +150,9 @@ def get_default_head_tags(path):
         {
             "tag": "meta",
             "property": "og:image",
-            "content": utils.join_url(settings.FUNKWHALE_URL, "/front/favicon.png"),
+            "content": utils.join_url(
+                settings.FUNKWHALE_URL, "/android-chrome-512x512.png"
+            ),
         },
         {
             "tag": "meta",

api/funkwhale_api/common/models.py

@@ -60,12 +60,12 @@ class NullsLastSQLCompiler(SQLCompiler):
 class NullsLastQuery(models.sql.query.Query):
     """Use a custom compiler to inject 'NULLS LAST' (for PostgreSQL)."""
 
-    def get_compiler(self, using=None, connection=None):
+    def get_compiler(self, using=None, connection=None, elide_empty=True):
         if using is None and connection is None:
             raise ValueError("Need either using or connection")
         if using:
             connection = connections[using]
-        return NullsLastSQLCompiler(self, connection, using)
+        return NullsLastSQLCompiler(self, connection, using, elide_empty)
 
 
 class NullsLastQuerySet(models.QuerySet):

api/funkwhale_api/common/preferences.py

@@ -2,7 +2,7 @@ import json
 
 from django import forms
 from django.conf import settings
-from django.contrib.postgres.forms import JSONField
+from django.forms import JSONField
 from dynamic_preferences import serializers, types
 from dynamic_preferences.registries import global_preferences_registry
 
@@ -93,7 +93,6 @@ class SerializedPreference(types.BasePreferenceType):
     serializer
     """
 
-    serializer = JSONSerializer
     data_serializer_class = None
     field_class = JSONField
     widget = forms.Textarea

api/funkwhale_api/common/serializers.py

@@ -5,8 +5,8 @@ import os
 import PIL
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files.uploadedfile import SimpleUploadedFile
-from django.utils.encoding import smart_text
-from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import smart_str
+from django.utils.translation import gettext_lazy as _
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
@@ -52,7 +52,7 @@ class RelatedField(serializers.RelatedField):
             self.fail(
                 "does_not_exist",
                 related_field_name=self.related_field_name,
-                value=smart_text(data),
+                value=smart_str(data),
             )
         except (TypeError, ValueError):
             self.fail("invalid")
@@ -349,7 +349,7 @@ class ScopesSerializer(serializers.Serializer):
 
 class IdentSerializer(serializers.Serializer):
     type = serializers.CharField()
-    id = serializers.IntegerField()
+    id = serializers.CharField()
 
 
 class RateLimitSerializer(serializers.Serializer):

api/funkwhale_api/common/signals.py

@@ -1,6 +1,6 @@
 import django.dispatch
 
-mutation_created = django.dispatch.Signal(providing_args=["mutation"])
-mutation_updated = django.dispatch.Signal(
-    providing_args=["mutation", "old_is_approved", "new_is_approved"]
-)
+""" Required args: mutation """
+mutation_created = django.dispatch.Signal()
+""" Required args: mutation, old_is_approved, new_is_approved """
+mutation_updated = django.dispatch.Signal()

api/funkwhale_api/common/throttling.py

@@ -7,7 +7,7 @@ from rest_framework import throttling as rest_throttling
 
 def get_ident(user, request):
     if user and user.is_authenticated:
-        return {"type": "authenticated", "id": user.pk}
+        return {"type": "authenticated", "id": f"{user.pk}"}
     ident = rest_throttling.BaseThrottle().get_ident(request)
 
     return {"type": "anonymous", "id": ident}

api/funkwhale_api/common/utils.py

@@ -477,14 +477,13 @@ def monkey_patch_request_build_absolute_uri():
 def get_file_hash(file, algo=None, chunk_size=None, full_read=False):
     algo = algo or settings.HASHING_ALGORITHM
     chunk_size = chunk_size or settings.HASHING_CHUNK_SIZE
-    handler = getattr(hashlib, algo)
-    hash = handler()
+    hasher = hashlib.new(algo)
     file.seek(0)
     if full_read:
         for byte_block in iter(lambda: file.read(chunk_size), b""):
-            hash.update(byte_block)
+            hasher.update(byte_block)
     else:
         # sometimes, it's useful to only hash the beginning of the file, e.g
         # to avoid a lot of I/O when crawling large libraries
-        hash.update(file.read(chunk_size))
-    return f"{algo}:{hash.hexdigest()}"
+        hasher.update(file.read(chunk_size))
+    return f"{algo}:{hasher.hexdigest()}"

api/funkwhale_api/common/validators.py

@@ -6,7 +6,7 @@ from django.core.exceptions import ValidationError
 from django.core.files.images import get_image_dimensions
 from django.template.defaultfilters import filesizeformat
 from django.utils.deconstruct import deconstructible
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 @deconstructible

api/funkwhale_api/contrib/listenbrainz/client.py

@@ -1,168 +0,0 @@
-# Copyright (c) 2018 Philipp Wolfer <ph.wolfer@gmail.com>
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-import json
-import logging
-import ssl
-import time
-from http.client import HTTPSConnection
-
-HOST_NAME = "api.listenbrainz.org"
-PATH_SUBMIT = "/1/submit-listens"
-SSL_CONTEXT = ssl.create_default_context()
-
-
-class Track:
-    """
-    Represents a single track to submit.
-
-    See https://listenbrainz.readthedocs.io/en/latest/dev/json.html
-    """
-
-    def __init__(self, artist_name, track_name, release_name=None, additional_info={}):
-        """
-        Create a new Track instance
-        @param artist_name as str
-        @param track_name as str
-        @param release_name as str
-        @param additional_info as dict
-        """
-        self.artist_name = artist_name
-        self.track_name = track_name
-        self.release_name = release_name
-        self.additional_info = additional_info
-
-    @staticmethod
-    def from_dict(data):
-        return Track(
-            data["artist_name"],
-            data["track_name"],
-            data.get("release_name", None),
-            data.get("additional_info", {}),
-        )
-
-    def to_dict(self):
-        return {
-            "artist_name": self.artist_name,
-            "track_name": self.track_name,
-            "release_name": self.release_name,
-            "additional_info": self.additional_info,
-        }
-
-    def __repr__(self):
-        return f"Track({self.artist_name}, {self.track_name})"
-
-
-class ListenBrainzClient:
-    """
-    Submit listens to ListenBrainz.org.
-
-    See https://listenbrainz.readthedocs.io/en/latest/dev/api.html
-    """
-
-    def __init__(self, user_token, logger=logging.getLogger(__name__)):
-        self.__next_request_time = 0
-        self.user_token = user_token
-        self.logger = logger
-
-    def listen(self, listened_at, track):
-        """
-        Submit a listen for a track
-        @param listened_at as int
-        @param entry as Track
-        """
-        payload = _get_payload(track, listened_at)
-        return self._submit("single", [payload])
-
-    def playing_now(self, track):
-        """
-        Submit a playing now notification for a track
-        @param track as Track
-        """
-        payload = _get_payload(track)
-        return self._submit("playing_now", [payload])
-
-    def import_tracks(self, tracks):
-        """
-        Import a list of tracks as (listened_at, Track) pairs
-        @param track as [(int, Track)]
-        """
-        payload = _get_payload_many(tracks)
-        return self._submit("import", payload)
-
-    def _submit(self, listen_type, payload, retry=0):
-        self._wait_for_ratelimit()
-        self.logger.debug("ListenBrainz %s: %r", listen_type, payload)
-        data = {"listen_type": listen_type, "payload": payload}
-        headers = {
-            "Authorization": "Token %s" % self.user_token,
-            "Content-Type": "application/json",
-        }
-        body = json.dumps(data)
-        conn = HTTPSConnection(HOST_NAME, context=SSL_CONTEXT)
-        conn.request("POST", PATH_SUBMIT, body, headers)
-        response = conn.getresponse()
-        response_text = response.read()
-        try:
-            response_data = json.loads(response_text)
-        except json.decoder.JSONDecodeError:
-            response_data = response_text
-
-        self._handle_ratelimit(response)
-        log_msg = f"Response {response.status}: {response_data!r}"
-        if response.status == 429 and retry < 5:  # Too Many Requests
-            self.logger.warning(log_msg)
-            return self._submit(listen_type, payload, retry + 1)
-        elif response.status == 200:
-            self.logger.debug(log_msg)
-        else:
-            self.logger.error(log_msg)
-        return response
-
-    def _wait_for_ratelimit(self):
-        now = time.time()
-        if self.__next_request_time > now:
-            delay = self.__next_request_time - now
-            self.logger.debug("Rate limit applies, delay %d", delay)
-            time.sleep(delay)
-
-    def _handle_ratelimit(self, response):
-        remaining = int(response.getheader("X-RateLimit-Remaining", 0))
-        reset_in = int(response.getheader("X-RateLimit-Reset-In", 0))
-        self.logger.debug("X-RateLimit-Remaining: %i", remaining)
-        self.logger.debug("X-RateLimit-Reset-In: %i", reset_in)
-        if remaining == 0:
-            self.__next_request_time = time.time() + reset_in
-
-
-def _get_payload_many(tracks):
-    payload = []
-    for listened_at, track in tracks:
-        data = _get_payload(track, listened_at)
-        payload.append(data)
-    return payload
-
-
-def _get_payload(track, listened_at=None):
-    data = {"track_metadata": track.to_dict()}
-    if listened_at is not None:
-        data["listened_at"] = listened_at
-    return data

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -1,27 +1,31 @@
+import liblistenbrainz
+
 import funkwhale_api
 from config import plugins
+from funkwhale_api.favorites import models as favorites_models
+from funkwhale_api.history import models as history_models
 
-from .client import ListenBrainzClient, Track
+from . import tasks
 from .funkwhale_startup import PLUGIN
 
 
 @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
 def submit_listen(listening, conf, **kwargs):
     user_token = conf["user_token"]
-    if not user_token:
+    if not user_token and not conf["submit_listenings"]:
         return
 
     logger = PLUGIN["logger"]
     logger.info("Submitting listen to ListenBrainz")
-    client = ListenBrainzClient(user_token=user_token, logger=logger)
-    track = get_track(listening.track)
-    client.listen(int(listening.creation_date.timestamp()), track)
+    client = liblistenbrainz.ListenBrainz()
+    client.set_auth_token(user_token)
+    listen = get_lb_listen(listening)
+
+    client.submit_single_listen(listen)
 
 
-def get_track(track):
-    artist = track.artist.name
-    title = track.title
-    album = None
+def get_lb_listen(listening):
+    track = listening.track
     additional_info = {
         "media_player": "Funkwhale",
         "media_player_version": funkwhale_api.__version__,
@@ -36,7 +40,7 @@ def get_track(track):
 
     if track.album:
         if track.album.title:
-            album = track.album.title
+            release_name = track.album.title
         if track.album.mbid:
             additional_info["release_mbid"] = str(track.album.mbid)
 
@@ -47,4 +51,86 @@ def get_track(track):
     if upload:
         additional_info["duration"] = upload.duration
 
-    return Track(artist, title, album, additional_info)
+    return liblistenbrainz.Listen(
+        track_name=track.title,
+        artist_name=track.artist.name,
+        listened_at=listening.creation_date.timestamp(),
+        release_name=release_name,
+        additional_info=additional_info,
+    )
+
+
+@plugins.register_hook(plugins.FAVORITE_CREATED, PLUGIN)
+def submit_favorite_creation(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite to ListenBrainz")
+    client = liblistenbrainz.ListenBrainz()
+    track = track_favorite.track
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz"
+        )
+        return
+    client.submit_user_feedback(1, track.mbid)
+
+
+@plugins.register_hook(plugins.FAVORITE_DELETED, PLUGIN)
+def submit_favorite_deletion(track_favorite, conf, **kwargs):
+    user_token = conf["user_token"]
+    if not user_token or not conf["submit_favorites"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Submitting favorite deletion to ListenBrainz")
+    client = liblistenbrainz.ListenBrainz()
+    track = track_favorite.track
+    if not track.mbid:
+        logger.warning(
+            "This tracks doesn't have a mbid. Feedback will not be submitted to Listenbrainz"
+        )
+        return
+    client.submit_user_feedback(0, track.mbid)
+
+
+@plugins.register_hook(plugins.LISTENING_SYNC, PLUGIN)
+def sync_listenings_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+
+    if not user_name or not conf["sync_listenings"]:
+        return
+    logger = PLUGIN["logger"]
+    logger.info("Getting listenings from ListenBrainz")
+    try:
+        last_ts = (
+            history_models.Listening.objects.filter(user=user)
+            .filter(source="Listenbrainz")
+            .latest("creation_date")
+            .values_list("creation_date", flat=True)
+        ).timestamp()
+    except funkwhale_api.history.models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_listenings(user, user_name, 0)
+        return
+
+    tasks.import_listenbrainz_listenings(user, user_name, last_ts)
+
+
+@plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
+def sync_favorites_from_listenbrainz(user, conf):
+    user_name = conf["user_name"]
+
+    if not user_name or not conf["sync_favorites"]:
+        return
+    try:
+        last_ts = (
+            favorites_models.TrackFavorite.objects.filter(user=user)
+            .filter(source="Listenbrainz")
+            .latest("creation_date")
+            .creation_date.timestamp()
+        )
+    except favorites_models.TrackFavorite.DoesNotExist:
+        tasks.import_listenbrainz_favorites(user, user_name, 0)
+        return
+
+    tasks.import_listenbrainz_favorites(user, user_name, last_ts)

api/funkwhale_api/contrib/listenbrainz/funkwhale_startup.py

@@ -3,7 +3,7 @@ from config import plugins
 PLUGIN = plugins.get_plugin_config(
     name="listenbrainz",
     label="ListenBrainz",
-    description="A plugin that allows you to submit your listens to ListenBrainz.",
+    description="A plugin that allows you to submit or sync your listens and favorites to ListenBrainz.",
     homepage="https://docs.funkwhale.audio/users/builtinplugins.html#listenbrainz-plugin",  # noqa
     version="0.3",
     user=True,
@@ -13,6 +13,45 @@ PLUGIN = plugins.get_plugin_config(
             "type": "text",
             "label": "Your ListenBrainz user token",
             "help": "You can find your user token in your ListenBrainz profile at https://listenbrainz.org/profile/",
-        }
+        },
+        {
+            "name": "user_name",
+            "type": "text",
+            "required": False,
+            "label": "Your ListenBrainz user name.",
+            "help": "Required for importing listenings and favorites with ListenBrainz \
+                but not to send activities",
+        },
+        {
+            "name": "submit_listenings",
+            "type": "boolean",
+            "default": True,
+            "label": "Enable listening submission to ListenBrainz",
+            "help": "If enabled, your listenings from Funkwhale will be imported into ListenBrainz.",
+        },
+        {
+            "name": "sync_listenings",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable listenings sync",
+            "help": "If enabled, your listening from ListenBrainz will be imported into Funkwhale. This means they \
+                will be used along with Funkwhale listenings to filter out recently listened content or \
+                generate recommendations",
+        },
+        {
+            "name": "sync_favorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite sync",
+            "help": "If enabled, your favorites from ListenBrainz will be imported into Funkwhale. This means they \
+                will be used along with Funkwhale favorites (UI display, federation activity)",
+        },
+        {
+            "name": "submit_favorites",
+            "type": "boolean",
+            "default": False,
+            "label": "Enable favorite submission to ListenBrainz services",
+            "help": "If enabled, your favorites from Funkwhale will be submitted to ListenBrainz",
+        },
     ],
 )

api/funkwhale_api/contrib/listenbrainz/tasks.py

@@ -0,0 +1,165 @@
+import datetime
+
+import liblistenbrainz
+from django.utils import timezone
+
+from config import plugins
+from funkwhale_api.favorites import models as favorites_models
+from funkwhale_api.history import models as history_models
+from funkwhale_api.music import models as music_models
+from funkwhale_api.taskapp import celery
+from funkwhale_api.users import models
+
+from .funkwhale_startup import PLUGIN
+
+
+@celery.app.task(name="listenbrainz.trigger_listening_sync_with_listenbrainz")
+def trigger_listening_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.LISTENING_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.trigger_favorite_sync_with_listenbrainz")
+def trigger_favorite_sync_with_listenbrainz():
+    now = timezone.now()
+    active_month = now - datetime.timedelta(days=30)
+    users = (
+        models.User.objects.filter(plugins__code="listenbrainz")
+        .filter(plugins__conf__sync_listenings=True)
+        .filter(last_activity__gte=active_month)
+    )
+    for user in users:
+        plugins.trigger_hook(
+            plugins.FAVORITE_SYNC,
+            user=user,
+            confs=plugins.get_confs(user),
+        )
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_listenings")
+def import_listenbrainz_listenings(user, user_name, since):
+    client = liblistenbrainz.ListenBrainz()
+    response = client.get_listens(username=user_name, min_ts=since, count=100)
+    listens = response["payload"]["listens"]
+    while listens:
+        add_lb_listenings_to_db(listens, user)
+        new_ts = max(
+            listens,
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj.listened_at, timezone.utc
+            ),
+        )
+        response = client.get_listens(username=user_name, min_ts=new_ts, count=100)
+        listens = response["payload"]["listens"]
+
+
+def add_lb_listenings_to_db(listens, user):
+    logger = PLUGIN["logger"]
+    fw_listens = []
+    for listen in listens:
+        if (
+            listen.additional_info.get("submission_client")
+            and listen.additional_info.get("submission_client")
+            == "Funkwhale ListenBrainz plugin"
+            and history_models.Listening.objects.filter(
+                creation_date=datetime.datetime.fromtimestamp(
+                    listen.listened_at, timezone.utc
+                )
+            ).exists()
+        ):
+            logger.info(
+                f"Listen with ts {listen.listened_at} skipped because already in db"
+            )
+            continue
+
+        mbid = (
+            listen.mbid_mapping
+            if hasattr(listen, "mbid_mapping")
+            else listen.recording_mbid
+        )
+
+        if not mbid:
+            logger.info("Received listening that doesn't have a mbid. Skipping...")
+
+        try:
+            track = music_models.Track.objects.get(mbid=mbid)
+        except music_models.Track.DoesNotExist:
+            logger.info(
+                "Received listening that doesn't exist in fw database. Skipping..."
+            )
+            continue
+
+        user = user
+        fw_listen = history_models.Listening(
+            creation_date=datetime.datetime.fromtimestamp(
+                listen.listened_at, timezone.utc
+            ),
+            track=track,
+            user=user,
+            source="Listenbrainz",
+        )
+        fw_listens.append(fw_listen)
+
+    history_models.Listening.objects.bulk_create(fw_listens)
+
+
+@celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
+def import_listenbrainz_favorites(user, user_name, since):
+    client = liblistenbrainz.ListenBrainz()
+    response = client.get_user_feedback(username=user_name)
+    offset = 0
+    while response["feedback"]:
+        count = response["count"]
+        offset = offset + count
+        last_sync = min(
+            response["feedback"],
+            key=lambda obj: datetime.datetime.fromtimestamp(
+                obj["created"], timezone.utc
+            ),
+        )["created"]
+        add_lb_feedback_to_db(response["feedback"], user)
+        if last_sync <= since or count == 0:
+            return
+        response = client.get_user_feedback(username=user_name, offset=offset)
+
+
+def add_lb_feedback_to_db(feedbacks, user):
+    logger = PLUGIN["logger"]
+    for feedback in feedbacks:
+        try:
+            track = music_models.Track.objects.get(mbid=feedback["recording_mbid"])
+        except music_models.Track.DoesNotExist:
+            logger.info(
+                "Received feedback track that doesn't exist in fw database. Skipping..."
+            )
+            continue
+
+        if feedback["score"] == 1:
+            favorites_models.TrackFavorite.objects.get_or_create(
+                user=user,
+                creation_date=datetime.datetime.fromtimestamp(
+                    feedback["created"], timezone.utc
+                ),
+                track=track,
+                source="Listenbrainz",
+            )
+        elif feedback["score"] == 0:
+            try:
+                favorites_models.TrackFavorite.objects.get(
+                    user=user, track=track
+                ).delete()
+            except favorites_models.TrackFavorite.DoesNotExist:
+                continue
+        elif feedback["score"] == -1:
+            logger.info("Funkwhale doesn't support disliked tracks")

api/funkwhale_api/contrib/maloja/funkwhale_ready.py

@@ -17,31 +17,40 @@ def submit_listen(listening, conf, **kwargs):
         return
 
     logger = PLUGIN["logger"]
-    logger.info("Submitting listening to Majola at %s", server_url)
-    payload = get_payload(listening, api_key)
-    logger.debug("Majola payload: %r", payload)
+    logger.info("Submitting listening to Maloja at %s", server_url)
+    payload = get_payload(listening, api_key, conf)
+    logger.debug("Maloja payload: %r", payload)
     url = server_url.rstrip("/") + "/apis/mlj_1/newscrobble"
     session = plugins.get_session()
     response = session.post(url, json=payload)
     response.raise_for_status()
     details = json.loads(response.text)
     if details["status"] == "success":
-        logger.info("Majola listening submitted successfully")
+        logger.info("Maloja listening submitted successfully")
     else:
         raise MalojaException(response.text)
 
 
-def get_payload(listening, api_key):
+def get_payload(listening, api_key, conf):
     track = listening.track
+
+    # See https://github.com/krateng/maloja/blob/master/API.md
     payload = {
         "key": api_key,
         "artists": [track.artist.name],
         "title": track.title,
         "time": int(listening.creation_date.timestamp()),
+        "nofix": bool(conf.get("nofix")),
     }
 
     if track.album:
         if track.album.title:
             payload["album"] = track.album.title
+        if track.album.artist:
+            payload["albumartists"] = [track.album.artist.name]
+
+    upload = track.uploads.filter(duration__gte=0).first()
+    if upload:
+        payload["length"] = upload.duration
 
     return payload

api/funkwhale_api/contrib/maloja/funkwhale_startup.py

@@ -5,10 +5,16 @@ PLUGIN = plugins.get_plugin_config(
     label="Maloja",
     description="A plugin that allows you to submit your listens to your Maloja server.",
     homepage="https://docs.funkwhale.audio/users/builtinplugins.html#maloja-plugin",
-    version="0.1.1",
+    version="0.2",
     user=True,
     conf=[
         {"name": "server_url", "type": "text", "label": "Maloja server URL"},
         {"name": "api_key", "type": "text", "label": "Your Maloja API key"},
+        {
+            "name": "nofix",
+            "type": "boolean",
+            "label": "Skip server-side metadata fixing",
+            "default": False,
+        },
     ],
 )

api/funkwhale_api/favorites/migrations/0002_trackfavorite_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('favorites', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='trackfavorite',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/funkwhale_api/favorites/models.py

@@ -12,6 +12,7 @@ class TrackFavorite(models.Model):
     track = models.ForeignKey(
         Track, related_name="track_favorites", on_delete=models.CASCADE
     )
+    source = models.CharField(max_length=100, null=True, blank=True)
 
     class Meta:
         unique_together = ("track", "user")

api/funkwhale_api/favorites/views.py

@@ -4,6 +4,7 @@ from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
+from config import plugins
 from funkwhale_api.activity import record
 from funkwhale_api.common import fields, permissions
 from funkwhale_api.music import utils as music_utils
@@ -44,6 +45,11 @@ class TrackFavoriteViewSet(
         instance = self.perform_create(serializer)
         serializer = self.get_serializer(instance=instance)
         headers = self.get_success_headers(serializer.data)
+        plugins.trigger_hook(
+            plugins.FAVORITE_CREATED,
+            track_favorite=serializer.instance,
+            confs=plugins.get_confs(self.request.user),
+        )
         record.send(instance)
         return Response(
             serializer.data, status=status.HTTP_201_CREATED, headers=headers
@@ -76,6 +82,11 @@ class TrackFavoriteViewSet(
         except (AttributeError, ValueError, models.TrackFavorite.DoesNotExist):
             return Response({}, status=400)
         favorite.delete()
+        plugins.trigger_hook(
+            plugins.FAVORITE_DELETED,
+            track_favorite=favorite,
+            confs=plugins.get_confs(self.request.user),
+        )
         return Response([], status=status.HTTP_204_NO_CONTENT)
 
     @extend_schema(

api/funkwhale_api/federation/signing.py

@@ -1,8 +1,8 @@
 import datetime
 import logging
+import sys
 
 import cryptography.exceptions
-import pytz
 import requests
 import requests_http_message_signatures
 from django import forms
@@ -11,6 +11,11 @@ from django.utils.http import parse_http_date
 
 from . import exceptions, utils
 
+if sys.version_info < (3, 9):
+    from backports.zoneinfo import ZoneInfo
+else:
+    from zoneinfo import ZoneInfo
+
 logger = logging.getLogger(__name__)
 
 #  the request Date should be between now - 30s and now + 30s
@@ -26,7 +31,7 @@ def verify_date(raw_date):
     except ValueError as e:
         raise forms.ValidationError(str(e))
     dt = datetime.datetime.utcfromtimestamp(ts)
-    dt = dt.replace(tzinfo=pytz.utc)
+    dt = dt.replace(tzinfo=ZoneInfo("UTC"))
     delta = datetime.timedelta(seconds=DATE_HEADER_VALID_FOR)
     now = timezone.now()
     if dt < now - delta or dt > now + delta:

api/funkwhale_api/federation/urls.py

@@ -1,4 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 from rest_framework import routers
 
 from . import views
@@ -23,6 +24,8 @@ music_router.register(r"tracks", views.MusicTrackViewSet, "tracks")
 index_router.register(r"index", views.IndexViewSet, "index")
 
 urlpatterns = router.urls + [
-    url("federation/music/", include((music_router.urls, "music"), namespace="music")),
-    url("federation/", include((index_router.urls, "index"), namespace="index")),
+    re_path(
+        "federation/music/", include((music_router.urls, "music"), namespace="music")
+    ),
+    re_path("federation/", include((index_router.urls, "index"), namespace="index")),
 ]

api/funkwhale_api/history/migrations/0003_listening_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.20 on 2023-12-09 14:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('history', '0002_auto_20180325_1433'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='listening',
+            name='source',
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

api/funkwhale_api/history/models.py

@@ -17,6 +17,7 @@ class Listening(models.Model):
         on_delete=models.CASCADE,
     )
     session_key = models.CharField(max_length=100, null=True, blank=True)
+    source = models.CharField(max_length=100, null=True, blank=True)
 
     class Meta:
         ordering = ("-creation_date",)

api/funkwhale_api/instance/dynamic_preferences_registry.py

@@ -1,3 +1,4 @@
+import pycountry
 from django.core.validators import FileExtensionValidator
 from django.forms import widgets
 from dynamic_preferences import types
@@ -170,3 +171,18 @@ class Banner(ImagePreference):
     default = None
     help_text = "This banner will be displayed on your pod's landing and about page. At least 600x100px recommended."
     field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class Location(types.ChoicePreference):
+    show_in_api = True
+    section = instance
+    name = "location"
+    verbose_name = "Server Location"
+    default = ""
+    choices = [(country.alpha_2, country.name) for country in pycountry.countries]
+    help_text = (
+        "The country or territory in which your server is located. This is displayed in the server's Nodeinfo "
+        "endpoint."
+    )
+    field_kwargs = {"choices": choices, "required": False}

api/funkwhale_api/instance/serializers.py

@@ -12,6 +12,17 @@ class SoftwareSerializer(serializers.Serializer):
         return "funkwhale"
 
 
+class SoftwareSerializer_v2(SoftwareSerializer):
+    repository = serializers.SerializerMethodField()
+    homepage = serializers.SerializerMethodField()
+
+    def get_repository(self, obj):
+        return "https://dev.funkwhale.audio/funkwhale/funkwhale"
+
+    def get_homepage(self, obj):
+        return "https://funkwhale.audio"
+
+
 class ServicesSerializer(serializers.Serializer):
     inbound = serializers.ListField(child=serializers.CharField(), default=[])
     outbound = serializers.ListField(child=serializers.CharField(), default=[])
@@ -31,6 +42,8 @@ class UsersUsageSerializer(serializers.Serializer):
 
 class UsageSerializer(serializers.Serializer):
     users = UsersUsageSerializer()
+    localPosts = serializers.IntegerField(required=False)
+    localComments = serializers.IntegerField(required=False)
 
 
 class TotalCountSerializer(serializers.Serializer):
@@ -92,19 +105,14 @@ class MetadataSerializer(serializers.Serializer):
     private = serializers.SerializerMethodField()
     shortDescription = serializers.SerializerMethodField()
     longDescription = serializers.SerializerMethodField()
-    rules = serializers.SerializerMethodField()
     contactEmail = serializers.SerializerMethodField()
-    terms = serializers.SerializerMethodField()
     nodeName = serializers.SerializerMethodField()
     banner = serializers.SerializerMethodField()
     defaultUploadQuota = serializers.SerializerMethodField()
-    library = serializers.SerializerMethodField()
     supportedUploadExtensions = serializers.ListField(child=serializers.CharField())
     allowList = serializers.SerializerMethodField()
-    reportTypes = ReportTypeSerializer(source="report_types", many=True)
     funkwhaleSupportMessageEnabled = serializers.SerializerMethodField()
     instanceSupportMessage = serializers.SerializerMethodField()
-    endpoints = EndpointsSerializer()
     usage = MetadataUsageSerializer(source="stats", required=False)
 
     def get_private(self, obj) -> bool:
@@ -116,15 +124,9 @@ class MetadataSerializer(serializers.Serializer):
     def get_longDescription(self, obj) -> str:
         return obj["preferences"].get("instance__long_description")
 
-    def get_rules(self, obj) -> str:
-        return obj["preferences"].get("instance__rules")
-
     def get_contactEmail(self, obj) -> str:
         return obj["preferences"].get("instance__contact_email")
 
-    def get_terms(self, obj) -> str:
-        return obj["preferences"].get("instance__terms")
-
     def get_nodeName(self, obj) -> str:
         return obj["preferences"].get("instance__name")
 
@@ -137,15 +139,6 @@ class MetadataSerializer(serializers.Serializer):
     def get_defaultUploadQuota(self, obj) -> int:
         return obj["preferences"].get("users__upload_quota")
 
-    @extend_schema_field(NodeInfoLibrarySerializer)
-    def get_library(self, obj):
-        data = obj["stats"] or {}
-        data["federationEnabled"] = obj["preferences"].get("federation__enabled")
-        data["anonymousCanListen"] = not obj["preferences"].get(
-            "common__api_authentication_required"
-        )
-        return NodeInfoLibrarySerializer(data).data
-
     @extend_schema_field(AllowListStatSerializer)
     def get_allowList(self, obj):
         return AllowListStatSerializer(
@@ -166,6 +159,62 @@ class MetadataSerializer(serializers.Serializer):
         return MetadataUsageSerializer(obj["stats"]).data
 
 
+class Metadata20Serializer(MetadataSerializer):
+    library = serializers.SerializerMethodField()
+    reportTypes = ReportTypeSerializer(source="report_types", many=True)
+    endpoints = EndpointsSerializer()
+    rules = serializers.SerializerMethodField()
+    terms = serializers.SerializerMethodField()
+
+    def get_rules(self, obj) -> str:
+        return obj["preferences"].get("instance__rules")
+
+    def get_terms(self, obj) -> str:
+        return obj["preferences"].get("instance__terms")
+
+    @extend_schema_field(NodeInfoLibrarySerializer)
+    def get_library(self, obj):
+        data = obj["stats"] or {}
+        data["federationEnabled"] = obj["preferences"].get("federation__enabled")
+        data["anonymousCanListen"] = not obj["preferences"].get(
+            "common__api_authentication_required"
+        )
+        return NodeInfoLibrarySerializer(data).data
+
+
+class MetadataContentLocalSerializer(serializers.Serializer):
+    artists = serializers.IntegerField()
+    releases = serializers.IntegerField()
+    recordings = serializers.IntegerField()
+    hoursOfContent = serializers.IntegerField()
+
+
+class MetadataContentCategorySerializer(serializers.Serializer):
+    name = serializers.CharField()
+    count = serializers.IntegerField()
+
+
+class MetadataContentSerializer(serializers.Serializer):
+    local = MetadataContentLocalSerializer()
+    topMusicCategories = MetadataContentCategorySerializer(many=True)
+    topPodcastCategories = MetadataContentCategorySerializer(many=True)
+
+
+class Metadata21Serializer(MetadataSerializer):
+    languages = serializers.ListField(child=serializers.CharField())
+    location = serializers.CharField()
+    content = MetadataContentSerializer()
+    features = serializers.ListField(child=serializers.CharField())
+    codeOfConduct = serializers.SerializerMethodField()
+
+    def get_codeOfConduct(self, obj) -> str:
+        return (
+            full_url("/about/pod#rules")
+            if obj["preferences"].get("instance__rules")
+            else ""
+        )
+
+
 class NodeInfo20Serializer(serializers.Serializer):
     version = serializers.SerializerMethodField()
     software = SoftwareSerializer()
@@ -196,9 +245,36 @@ class NodeInfo20Serializer(serializers.Serializer):
             usage = {"users": {"total": 0, "activeMonth": 0, "activeHalfyear": 0}}
         return UsageSerializer(usage).data
 
-    @extend_schema_field(MetadataSerializer)
+    @extend_schema_field(Metadata20Serializer)
     def get_metadata(self, obj):
-        return MetadataSerializer(obj).data
+        return Metadata20Serializer(obj).data
+
+
+class NodeInfo21Serializer(NodeInfo20Serializer):
+    version = serializers.SerializerMethodField()
+    software = SoftwareSerializer_v2()
+
+    def get_version(self, obj) -> str:
+        return "2.1"
+
+    @extend_schema_field(UsageSerializer)
+    def get_usage(self, obj):
+        usage = None
+        if obj["preferences"]["instance__nodeinfo_stats_enabled"]:
+            usage = obj["stats"]
+            usage["localPosts"] = 0
+            usage["localComments"] = 0
+        else:
+            usage = {
+                "users": {"total": 0, "activeMonth": 0, "activeHalfyear": 0},
+                "localPosts": 0,
+                "localComments": 0,
+            }
+        return UsageSerializer(usage).data
+
+    @extend_schema_field(Metadata21Serializer)
+    def get_metadata(self, obj):
+        return Metadata21Serializer(obj).data
 
 
 class SpaManifestIconSerializer(serializers.Serializer):

api/funkwhale_api/instance/stats.py

@@ -1,6 +1,6 @@
 import datetime
 
-from django.db.models import Sum
+from django.db.models import Count, F, Sum
 from django.utils import timezone
 
 from funkwhale_api.favorites.models import TrackFavorite
@@ -22,6 +22,39 @@ def get():
     }
 
 
+def get_content():
+    return {
+        "local": {
+            "artists": get_artists(),
+            "releases": get_albums(),
+            "recordings": get_tracks(),
+            "hoursOfContent": get_music_duration(),
+        },
+        "topMusicCategories": get_top_music_categories(),
+        "topPodcastCategories": get_top_podcast_categories(),
+    }
+
+
+def get_top_music_categories():
+    return (
+        models.Track.objects.filter(artist__content_category="music")
+        .exclude(tagged_items__tag_id=None)
+        .values(name=F("tagged_items__tag__name"))
+        .annotate(count=Count("name"))
+        .order_by("-count")[:3]
+    )
+
+
+def get_top_podcast_categories():
+    return (
+        models.Track.objects.filter(artist__content_category="podcast")
+        .exclude(tagged_items__tag_id=None)
+        .values(name=F("tagged_items__tag__name"))
+        .annotate(count=Count("name"))
+        .order_by("-count")[:3]
+    )
+
+
 def get_users():
     qs = User.objects.filter(is_active=True)
     now = timezone.now()

api/funkwhale_api/instance/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path
 
 from funkwhale_api.common import routers
 
@@ -8,7 +8,7 @@ admin_router = routers.OptionalSlashRouter()
 admin_router.register(r"admin/settings", views.AdminSettings, "admin-settings")
 
 urlpatterns = [
-    url(r"^nodeinfo/2.0/?$", views.NodeInfo.as_view(), name="nodeinfo-2.0"),
-    url(r"^settings/?$", views.InstanceSettings.as_view(), name="settings"),
-    url(r"^spa-manifest.json", views.SpaManifest.as_view(), name="spa-manifest"),
+    re_path(r"^nodeinfo/2.0/?$", views.NodeInfo20.as_view(), name="nodeinfo-2.0"),
+    re_path(r"^settings/?$", views.InstanceSettings.as_view(), name="settings"),
+    re_path(r"^spa-manifest.json", views.SpaManifest.as_view(), name="spa-manifest"),
 ] + admin_router.urls

api/funkwhale_api/instance/urls_v2.py

@@ -0,0 +1,7 @@
+from django.urls import re_path
+
+from . import views
+
+urlpatterns = [
+    re_path(r"^nodeinfo/2.1/?$", views.NodeInfo21.as_view(), name="nodeinfo-2.1"),
+]

api/funkwhale_api/instance/views.py

@@ -11,6 +11,7 @@ from dynamic_preferences.api import viewsets as preferences_viewsets
 from dynamic_preferences.api.serializers import GlobalPreferenceSerializer
 from dynamic_preferences.registries import global_preferences_registry
 from rest_framework import generics, views
+from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 
 from funkwhale_api import __version__ as funkwhale_version
@@ -58,9 +59,11 @@ class InstanceSettings(generics.GenericAPIView):
 
 
 @method_decorator(ensure_csrf_cookie, name="dispatch")
-class NodeInfo(views.APIView):
+class NodeInfo20(views.APIView):
     permission_classes = []
     authentication_classes = []
+    serializer_class = serializers.NodeInfo20Serializer
+    renderer_classes = (JSONRenderer,)
 
     @extend_schema(
         responses=serializers.NodeInfo20Serializer, operation_id="getNodeInfo20"
@@ -81,6 +84,7 @@ class NodeInfo(views.APIView):
 
         data = {
             "software": {"version": funkwhale_version},
+            "services": {"inbound": ["atom1.0"], "outbound": ["atom1.0"]},
             "preferences": pref,
             "stats": cache_memoize(600, prefix="memoize:instance:stats")(stats.get)()
             if pref["instance__nodeinfo_stats_enabled"]
@@ -112,7 +116,65 @@ class NodeInfo(views.APIView):
                 data["endpoints"]["channels"] = reverse(
                     "federation:index:index-channels"
                 )
-        serializer = serializers.NodeInfo20Serializer(data)
+        serializer = self.serializer_class(data)
+        return Response(
+            serializer.data, status=200, content_type=NODEINFO_2_CONTENT_TYPE
+        )
+
+
+class NodeInfo21(NodeInfo20):
+    serializer_class = serializers.NodeInfo21Serializer
+
+    @extend_schema(
+        responses=serializers.NodeInfo20Serializer, operation_id="getNodeInfo20"
+    )
+    def get(self, request):
+        pref = preferences.all()
+        if (
+            pref["moderation__allow_list_public"]
+            and pref["moderation__allow_list_enabled"]
+        ):
+            allowed_domains = list(
+                Domain.objects.filter(allowed=True)
+                .order_by("name")
+                .values_list("name", flat=True)
+            )
+        else:
+            allowed_domains = None
+
+        data = {
+            "software": {"version": funkwhale_version},
+            "services": {"inbound": ["atom1.0"], "outbound": ["atom1.0"]},
+            "preferences": pref,
+            "stats": cache_memoize(600, prefix="memoize:instance:stats")(stats.get)()
+            if pref["instance__nodeinfo_stats_enabled"]
+            else None,
+            "actorId": get_service_actor().fid,
+            "supportedUploadExtensions": SUPPORTED_EXTENSIONS,
+            "allowed_domains": allowed_domains,
+            "languages": pref.get("moderation__languages"),
+            "location": pref.get("instance__location"),
+            "content": cache_memoize(600, prefix="memoize:instance:content")(
+                stats.get_content
+            )()
+            if pref["instance__nodeinfo_stats_enabled"]
+            else None,
+            "features": [
+                "channels",
+                "podcasts",
+            ],
+        }
+
+        if not pref.get("common__api_authentication_required"):
+            data["features"].append("anonymousCanListen")
+
+        if pref.get("federation__enabled"):
+            data["features"].append("federation")
+
+        if pref.get("music__only_allow_musicbrainz_tagged_files"):
+            data["features"].append("onlyMbidTaggedContent")
+
+        serializer = self.serializer_class(data)
         return Response(
             serializer.data, status=200, content_type=NODEINFO_2_CONTENT_TYPE
         )

api/funkwhale_api/manage/urls.py

@@ -1,4 +1,5 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 
 from funkwhale_api.common import routers
 
@@ -32,14 +33,16 @@ other_router.register(r"channels", views.ManageChannelViewSet, "channels")
 other_router.register(r"tags", views.ManageTagViewSet, "tags")
 
 urlpatterns = [
-    url(
+    re_path(
         r"^federation/",
         include((federation_router.urls, "federation"), namespace="federation"),
     ),
-    url(r"^library/", include((library_router.urls, "instance"), namespace="library")),
-    url(
+    re_path(
+        r"^library/", include((library_router.urls, "instance"), namespace="library")
+    ),
+    re_path(
         r"^moderation/",
         include((moderation_router.urls, "moderation"), namespace="moderation"),
     ),
-    url(r"^users/", include((users_router.urls, "instance"), namespace="users")),
+    re_path(r"^users/", include((users_router.urls, "instance"), namespace="users")),
 ] + other_router.urls

api/funkwhale_api/moderation/dynamic_preferences_registry.py

@@ -1,3 +1,4 @@
+import pycountry
 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 from rest_framework import serializers
@@ -92,3 +93,18 @@ class SignupFormCustomization(common_preferences.SerializedPreference):
     required = False
     default = {}
     data_serializer_class = CustomFormSerializer
+
+
+@global_preferences_registry.register
+class Languages(common_preferences.StringListPreference):
+    show_in_api = True
+    section = moderation
+    name = "languages"
+    default = ["en"]
+    verbose_name = "Moderation languages"
+    help_text = (
+        "The language(s) spoken by the server moderator(s). Set this to inform users "
+        "what languages they should write reports and requests in."
+    )
+    choices = [(lang.alpha_3, lang.name) for lang in pycountry.languages]
+    field_kwargs = {"choices": choices, "required": False}

api/funkwhale_api/moderation/signals.py

@@ -1,3 +1,4 @@
 import django.dispatch
 
-report_created = django.dispatch.Signal(providing_args=["report"])
+""" Required argument: report """
+report_created = django.dispatch.Signal()

api/funkwhale_api/music/dynamic_preferences_registry.py

@@ -32,3 +32,18 @@ class MusicCacheDuration(types.IntPreference):
         "will be erased and retranscoded on the next listening."
     )
     field_kwargs = {"required": False}
+
+
+@global_preferences_registry.register
+class MbidTaggedContent(types.BooleanPreference):
+    show_in_api = True
+    section = music
+    name = "only_allow_musicbrainz_tagged_files"
+    verbose_name = "Only allow Musicbrainz tagged files"
+    help_text = (
+        "Requires uploaded files to be tagged with a MusicBrainz ID. "
+        "Enabling this setting has no impact on previously uploaded files. "
+        "You can use the CLI to clear files that don't contain an MBID or "
+        "or enable quality filtering to hide untagged content from API calls. "
+    )
+    default = False

api/funkwhale_api/music/factories.py

@@ -151,8 +151,9 @@ class TrackFactory(
         if created:
             self.save()
 
-    @factory.post_generation
-    def license(self, created, extracted, **kwargs):
+    # The @factory.post_generation is not used because we must
+    # not redefine the builtin `license` function.
+    def _license_post_generation(self, created, extracted, **kwargs):
         if not created:
             return
 
@@ -160,6 +161,8 @@ class TrackFactory(
             self.license = LicenseFactory(code=extracted)
             self.save()
 
+    license = factory.PostGeneration(_license_post_generation)
+
 
 @registry.register
 class UploadFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):

api/funkwhale_api/music/fake_data.py

@@ -15,7 +15,10 @@ def create_data(count=25):
         )
         for album in albums:
             factories.UploadFactory.create_batch(
-                track__album=album, size=random.randint(3, 18)
+                track__album=album,
+                size=random.randint(3, 18),
+                playable=True,
+                in_place=True,
             )
 
 

api/funkwhale_api/music/filters.py

@@ -104,7 +104,7 @@ class ArtistFilter(
         distinct=True,
         library_field="tracks__uploads__library",
     )
-    ordering = django_filters.OrderingFilter(
+    ordering = common_filters.CaseInsensitiveNameOrderingFilter(
         fields=(
             ("id", "id"),
             ("name", "name"),

api/funkwhale_api/music/licenses.py

@@ -28,7 +28,7 @@ def load(data):
 
     for row in data:
         try:
-            license = existing_by_code[row["code"]]
+            license_ = existing_by_code[row["code"]]
         except KeyError:
             logger.debug("Loading new license: {}".format(row["code"]))
             to_create.append(
@@ -36,15 +36,15 @@ def load(data):
             )
         else:
             logger.debug("Updating license: {}".format(row["code"]))
-            stored = [getattr(license, f) for f in MODEL_FIELDS]
+            stored = [getattr(license_, f) for f in MODEL_FIELDS]
             wanted = [row[f] for f in MODEL_FIELDS]
             if wanted == stored:
                 continue
             # the object in database needs an update
             for f in MODEL_FIELDS:
-                setattr(license, f, row[f])
+                setattr(license_, f, row[f])
 
-            license.save()
+            license_.save()
 
     models.License.objects.bulk_create(to_create)
     return sorted(models.License.objects.all(), key=lambda o: o.code)
@@ -78,12 +78,12 @@ def match(*values):
         else:
             existing = load(LICENSES)
             _cache = existing
-        for license in existing:
-            if license.conf is None:
+        for license_ in existing:
+            if license_.conf is None:
                 continue
-            for i in license.conf["identifiers"]:
+            for i in license_.conf["identifiers"]:
                 if match_urls(url, i):
-                    return license
+                    return license_
 
 
 def match_urls(*urls):

api/funkwhale_api/music/management/commands/generate_typesense_index.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from funkwhale_api.typesense import tasks
+
+
+class Command(BaseCommand):
+    help = """
+    Trigger the generation of a new typesense index for canonical Funkwhale tracks metadata.
+    This is use to resolve Funkwhale tracks to MusicBrainz ids"""
+
+    def handle(self, *args, **kwargs):
+        tasks.build_canonical_index.delay()
+        self.stdout.write("Tasks launched in celery worker.")

api/funkwhale_api/music/management/commands/prune_non_mbid_content.py

@@ -0,0 +1,61 @@
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from funkwhale_api.music import models
+
+
+class Command(BaseCommand):
+    help = """Deletes any tracks not tagged with a MusicBrainz ID from the database. By default, any tracks that
+        have been favorited by a user or added to a playlist are preserved."""
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-dry-run",
+            action="store_true",
+            dest="no_dry_run",
+            default=True,
+            help="Disable dry run mode and apply pruning for real on the database",
+        )
+
+        parser.add_argument(
+            "--include-playlist-content",
+            action="store_true",
+            dest="include_playlist_content",
+            default=False,
+            help="Allow tracks included in playlists to be pruned",
+        )
+
+        parser.add_argument(
+            "--include-favorites-content",
+            action="store_true",
+            dest="include_favorited_content",
+            default=False,
+            help="Allow favorited tracks to be pruned",
+        )
+
+        parser.add_argument(
+            "--include-listened-content",
+            action="store_true",
+            dest="include_listened_content",
+            default=False,
+            help="Allow tracks with listening history to be pruned",
+        )
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        tracks = models.Track.objects.filter(mbid__isnull=True)
+        if not options["include_favorited_content"]:
+            tracks = tracks.filter(track_favorites__isnull=True)
+        if not options["include_playlist_content"]:
+            tracks = tracks.filter(playlist_tracks__isnull=True)
+        if not options["include_listened_content"]:
+            tracks = tracks.filter(listenings__isnull=True)
+
+        pruned_total = tracks.count()
+        total = models.Track.objects.count()
+
+        if options["no_dry_run"]:
+            self.stdout.write(f"Deleting {pruned_total}/{total} tracks…")
+            tracks.delete()
+        else:
+            self.stdout.write(f"Would prune {pruned_total}/{total} tracks")

api/funkwhale_api/music/serializers.py

@@ -226,17 +226,18 @@ class TrackAlbumSerializer(serializers.ModelSerializer):
         )
 
 
-def serialize_upload(upload) -> object:
-    return {
-        "uuid": str(upload.uuid),
-        "listen_url": upload.listen_url,
-        "size": upload.size,
-        "duration": upload.duration,
-        "bitrate": upload.bitrate,
-        "mimetype": upload.mimetype,
-        "extension": upload.extension,
-        "is_local": federation_utils.is_local(upload.fid),
-    }
+class TrackUploadSerializer(serializers.Serializer):
+    uuid = serializers.UUIDField()
+    listen_url = serializers.URLField()
+    size = serializers.IntegerField()
+    duration = serializers.IntegerField()
+    bitrate = serializers.IntegerField()
+    mimetype = serializers.CharField()
+    extension = serializers.CharField()
+    is_local = serializers.SerializerMethodField()
+
+    def get_is_local(self, upload) -> bool:
+        return federation_utils.is_local(upload.fid)
 
 
 def sort_uploads_for_listen(uploads):
@@ -281,11 +282,14 @@ class TrackSerializer(OptionalDescriptionMixin, serializers.Serializer):
     def get_listen_url(self, obj):
         return obj.listen_url
 
-    @extend_schema_field({"type": "array", "items": {"type": "object"}})
+    #  @extend_schema_field({"type": "array", "items": {"type": "object"}})
+    @extend_schema_field(TrackUploadSerializer(many=True))
     def get_uploads(self, obj):
         uploads = getattr(obj, "playable_uploads", [])
         # we put local uploads first
-        uploads = [serialize_upload(u) for u in sort_uploads_for_listen(uploads)]
+        uploads = [
+            TrackUploadSerializer(u).data for u in sort_uploads_for_listen(uploads)
+        ]
         uploads = sorted(uploads, key=lambda u: u["is_local"], reverse=True)
         return list(uploads)
 

api/funkwhale_api/music/signals.py

@@ -1,5 +1,4 @@
 import django.dispatch
 
-upload_import_status_updated = django.dispatch.Signal(
-    providing_args=["old_status", "new_status", "upload"]
-)
+""" Required args: old_status, new_status, upload """
+upload_import_status_updated = django.dispatch.Signal()

api/funkwhale_api/music/tasks.py

@@ -247,6 +247,13 @@ def process_upload(upload, update_denormalization=True):
             return fail_import(
                 upload, "invalid_metadata", detail=detail, file_metadata=metadata_dump
             )
+        check_mbid = preferences.get("music__only_allow_musicbrainz_tagged_files")
+        if check_mbid and not serializer.validated_data.get("mbid"):
+            return fail_import(
+                upload,
+                "Only content tagged with a MusicBrainz ID is permitted on this pod.",
+                detail="You can tag your files with MusicBrainz Picard",
+            )
 
         final_metadata = collections.ChainMap(
             additional_data, serializer.validated_data, internal_config

api/funkwhale_api/music/views.py

@@ -297,8 +297,6 @@ class LibraryViewSet(
         )
         instance.delete()
 
-    follows = action
-
     @extend_schema(
         responses=federation_api_serializers.LibraryFollowSerializer(many=True)
     )
@@ -583,7 +581,7 @@ def handle_serve(
             try:
                 f.download_audio_from_remote(actor=actor)
             except requests.exceptions.RequestException:
-                return Response({"detail": "Remove track is unavailable"}, status=503)
+                return Response({"detail": "Remote track is unavailable"}, status=503)
         data = f.get_audio_data()
         if data:
             f.duration = data["duration"]

api/funkwhale_api/musicbrainz/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path
 
 from funkwhale_api.common import routers
 
@@ -7,22 +7,22 @@ from . import views
 router = routers.OptionalSlashRouter()
 router.register(r"search", views.SearchViewSet, "search")
 urlpatterns = [
-    url(
+    re_path(
         "releases/(?P<uuid>[0-9a-z-]+)/$",
         views.ReleaseDetail.as_view(),
         name="release-detail",
     ),
-    url(
+    re_path(
         "artists/(?P<uuid>[0-9a-z-]+)/$",
         views.ArtistDetail.as_view(),
         name="artist-detail",
     ),
-    url(
+    re_path(
         "release-groups/browse/(?P<artist_uuid>[0-9a-z-]+)/$",
         views.ReleaseGroupBrowse.as_view(),
         name="release-group-browse",
     ),
-    url(
+    re_path(
         "releases/browse/(?P<release_group_uuid>[0-9a-z-]+)/$",
         views.ReleaseBrowse.as_view(),
         name="release-browse",

api/funkwhale_api/providers/urls.py

@@ -1,7 +1,8 @@
-from django.conf.urls import include, url
+from django.conf.urls import include
+from django.urls import re_path
 
 urlpatterns = [
-    url(
+    re_path(
         r"^musicbrainz/",
         include(
             ("funkwhale_api.musicbrainz.urls", "musicbrainz"), namespace="musicbrainz"

api/funkwhale_api/radios/lb_recommendations.py

@@ -0,0 +1,146 @@
+import logging
+import time
+
+import troi
+import troi.core
+from django.core.cache import cache
+from django.core.exceptions import ValidationError
+from django.db.models import Q
+from requests.exceptions import ConnectTimeout
+
+from funkwhale_api.music import models as music_models
+from funkwhale_api.typesense import utils
+
+logger = logging.getLogger(__name__)
+
+
+patches = troi.utils.discover_patches()
+
+SUPPORTED_PATCHES = patches.keys()
+
+
+def run(config, **kwargs):
+    """Validate the received config and run the queryset generation"""
+    candidates = kwargs.pop("candidates", music_models.Track.objects.all())
+    validate(config)
+    return TroiPatch().get_queryset(config, candidates)
+
+
+def validate(config):
+    patch = config.get("patch")
+    if patch not in SUPPORTED_PATCHES:
+        raise ValidationError(
+            'Invalid patch "{}". Supported patches: {}'.format(
+                config["patch"], SUPPORTED_PATCHES
+            )
+        )
+
+    return True
+
+
+def build_radio_queryset(patch, radio_qs):
+    """Take a troi patch, match the missing mbid and then build a radio queryset"""
+
+    start_time = time.time()
+    try:
+        recommendations = patch.generate_playlist()
+    except ConnectTimeout:
+        raise ValueError(
+            "Timed out while connecting to ListenBrainz. No candidates could be retrieved for the radio."
+        )
+    end_time_rec = time.time()
+    logger.info("Troi fetch took :" + str(end_time_rec - start_time))
+
+    if not recommendations:
+        raise ValueError("No candidates found by troi")
+
+    recommended_mbids = [
+        recommended_recording.mbid
+        for recommended_recording in recommendations.playlists[0].recordings
+    ]
+
+    logger.info("Searching for MusicBrainz ID in Funkwhale database")
+
+    qs_recommended = (
+        music_models.Track.objects.all()
+        .filter(mbid__in=recommended_mbids)
+        .order_by("mbid", "pk")
+        .distinct("mbid")
+    )
+    qs_recommended_mbid = [str(i.mbid) for i in qs_recommended]
+
+    recommended_mbids_not_qs = [
+        mbid for mbid in recommended_mbids if mbid not in qs_recommended_mbid
+    ]
+    cached_match = cache.get_many(recommended_mbids_not_qs)
+    cached_match_mbid = [str(i) for i in cached_match.keys()]
+
+    if qs_recommended and cached_match_mbid:
+        logger.info("MusicBrainz IDs found in Funkwhale database and redis")
+        qs_recommended_mbid.extend(cached_match_mbid)
+        mbids_found = qs_recommended_mbid
+    elif qs_recommended and not cached_match_mbid:
+        logger.info("MusicBrainz IDs found in Funkwhale database")
+        mbids_found = qs_recommended_mbid
+    elif not qs_recommended and cached_match_mbid:
+        logger.info("MusicBrainz IDs found in redis cache")
+        mbids_found = cached_match_mbid
+    else:
+        logger.info(
+            "Couldn't find any matches in Funkwhale database. Trying to match all"
+        )
+        mbids_found = []
+
+    recommended_recordings_not_found = [
+        i for i in recommendations.playlists[0].recordings if i.mbid not in mbids_found
+    ]
+
+    logger.info("Matching missing MusicBrainz ID to Funkwhale track")
+
+    start_time_resolv = time.time()
+    utils.resolve_recordings_to_fw_track(recommended_recordings_not_found)
+    end_time_resolv = time.time()
+
+    logger.info(
+        "Resolving "
+        + str(len(recommended_recordings_not_found))
+        + " tracks in "
+        + str(end_time_resolv - start_time_resolv)
+    )
+
+    cached_match = cache.get_many(recommended_mbids)
+
+    if not mbids_found and not cached_match:
+        raise ValueError("No candidates found for troi radio")
+
+    mbids_found_pks = list(
+        music_models.Track.objects.all()
+        .filter(mbid__in=mbids_found)
+        .order_by("mbid", "pk")
+        .distinct("mbid")
+        .values_list("pk", flat=True)
+    )
+
+    mbids_found_pks_unique = [
+        i for i in mbids_found_pks if i not in cached_match.keys()
+    ]
+
+    if mbids_found and cached_match:
+        return radio_qs.filter(
+            Q(pk__in=mbids_found_pks_unique) | Q(pk__in=cached_match.values())
+        )
+    if mbids_found and not cached_match:
+        return radio_qs.filter(pk__in=mbids_found_pks_unique)
+
+    if not mbids_found and cached_match:
+        return radio_qs.filter(pk__in=cached_match.values())
+
+
+class TroiPatch:
+    code = "troi-patch"
+    label = "Troi Patch"
+
+    def get_queryset(self, config, qs):
+        patch_string = config.pop("patch")
+        patch = patches[patch_string]
+        return build_radio_queryset(patch(config), qs)

api/funkwhale_api/radios/models.py

@@ -54,10 +54,6 @@ class RadioSession(models.Model):
     CONFIG_VERSION = 0
     config = JSONField(encoder=DjangoJSONEncoder, blank=True, null=True)
 
-    def save(self, **kwargs):
-        self.radio.clean(self)
-        super().save(**kwargs)
-
     @property
     def next_position(self):
         next_position = 1
@@ -68,16 +64,24 @@ class RadioSession(models.Model):
 
         return next_position
 
-    def add(self, track):
-        new_session_track = RadioSessionTrack.objects.create(
-            track=track, session=self, position=self.next_position
-        )
+    def add(self, tracks):
+        next_position = self.next_position
+        radio_session_tracks = []
+        for i, track in enumerate(tracks):
+            radio_session_track = RadioSessionTrack(
+                track=track, session=self, position=next_position + i
+            )
+            radio_session_tracks.append(radio_session_track)
 
-        return new_session_track
+        new_session_tracks = RadioSessionTrack.objects.bulk_create(radio_session_tracks)
 
-    @property
-    def radio(self):
-        from .registries import registry
+        return new_session_tracks
+
+    def radio(self, api_version):
+        if api_version == 2:
+            from .registries_v2 import registry
+        else:
+            from .registries import registry
 
         return registry[self.radio_type](session=self)
 

api/funkwhale_api/radios/radios.py

@@ -1,4 +1,5 @@
 import datetime
+import json
 import logging
 import random
 from typing import List, Optional, Tuple
@@ -14,7 +15,7 @@ from funkwhale_api.moderation import filters as moderation_filters
 from funkwhale_api.music.models import Artist, Library, Track, Upload
 from funkwhale_api.tags.models import Tag
 
-from . import filters, models
+from . import filters, lb_recommendations, models
 from .registries import registry
 
 logger = logging.getLogger(__name__)
@@ -61,11 +62,19 @@ class SessionRadio(SimpleRadio):
         return self.session
 
     def get_queryset(self, **kwargs):
-        qs = Track.objects.all()
-        if not self.session:
-            return qs
-        if not self.session.user:
-            return qs
+        if not self.session or not self.session.user:
+            return (
+                Track.objects.all()
+                .with_playable_uploads(actor=None)
+                .select_related("artist", "album__artist", "attributed_to")
+            )
+        else:
+            qs = (
+                Track.objects.all()
+                .with_playable_uploads(self.session.user.actor)
+                .select_related("artist", "album__artist", "attributed_to")
+            )
+
         query = moderation_filters.get_filtered_content_query(
             config=moderation_filters.USER_FILTER_CONFIG["TRACK"],
             user=self.session.user,
@@ -75,6 +84,16 @@ class SessionRadio(SimpleRadio):
     def get_queryset_kwargs(self):
         return {}
 
+    def filter_queryset(self, queryset):
+        return queryset
+
+    def filter_from_session(self, queryset):
+        already_played = self.session.session_tracks.all().values_list(
+            "track", flat=True
+        )
+        queryset = queryset.exclude(pk__in=already_played)
+        return queryset
+
     def get_choices(self, **kwargs):
         kwargs.update(self.get_queryset_kwargs())
         queryset = self.get_queryset(**kwargs)
@@ -87,16 +106,6 @@ class SessionRadio(SimpleRadio):
         queryset = self.filter_queryset(queryset)
         return queryset
 
-    def filter_queryset(self, queryset):
-        return queryset
-
-    def filter_from_session(self, queryset):
-        already_played = self.session.session_tracks.all().values_list(
-            "track", flat=True
-        )
-        queryset = queryset.exclude(pk__in=already_played)
-        return queryset
-
     def pick(self, **kwargs):
         return self.pick_many(quantity=1, **kwargs)[0]
 
@@ -104,8 +113,7 @@ class SessionRadio(SimpleRadio):
         choices = self.get_choices(**kwargs)
         picked_choices = super().pick_many(choices=choices, quantity=quantity)
         if self.session:
-            for choice in picked_choices:
-                self.session.add(choice)
+            self.session.add(picked_choices)
         return picked_choices
 
     def validate_session(self, data, **context):
@@ -405,3 +413,58 @@ class RecentlyAdded(SessionRadio):
             Q(artist__content_category="music"),
             Q(creation_date__gt=date),
         )
+
+
+# Use this to experiment on the custom multiple radio with troi
+@registry.register(name="troi")
+class Troi(SessionRadio):
+    """
+    Receive a vuejs generated config and use it to launch a troi radio session.
+    The config data should follow :
+    {"patch": "troi_patch_name", "troi_arg1":"troi_arg_1", "troi_arg2": ...}
+    Validation of the config (args) is done by troi during track fetch.
+    Funkwhale only checks if the patch is implemented
+    """
+
+    config = serializers.JSONField(required=True)
+
+    def append_lb_config(self, data):
+        if self.session.user.settings is None:
+            logger.warning(
+                "No lb_user_name set in user settings. Some troi patches will fail"
+            )
+            return data
+        elif self.session.user.settings.get("lb_user_name") is None:
+            logger.warning(
+                "No lb_user_name set in user settings. Some troi patches will fail"
+            )
+        else:
+            data["user_name"] = self.session.user.settings["lb_user_name"]
+
+        if self.session.user.settings.get("lb_user_token") is None:
+            logger.warning(
+                "No lb_user_token set in user settings. Some troi patch will fail"
+            )
+        else:
+            data["user_token"] = self.session.user.settings["lb_user_token"]
+
+        return data
+
+    def get_queryset_kwargs(self):
+        kwargs = super().get_queryset_kwargs()
+        kwargs["config"] = self.session.config
+        return kwargs
+
+    def validate_session(self, data, **context):
+        data = super().validate_session(data, **context)
+        if data.get("config") is None:
+            raise serializers.ValidationError(
+                "You must provide a configuration for this radio"
+            )
+        return data
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        config = self.append_lb_config(json.loads(kwargs["config"]))
+
+        return lb_recommendations.run(config, candidates=qs)

api/funkwhale_api/radios/radios_v2.py

@@ -0,0 +1,510 @@
+import datetime
+import json
+import logging
+import pickle
+import random
+from typing import List, Optional, Tuple
+
+from django.core.cache import cache
+from django.core.exceptions import ValidationError
+from django.db import connection
+from django.db.models import Q
+from rest_framework import serializers
+
+from funkwhale_api.federation import fields as federation_fields
+from funkwhale_api.federation import models as federation_models
+from funkwhale_api.moderation import filters as moderation_filters
+from funkwhale_api.music.models import Artist, Library, Track, Upload
+from funkwhale_api.tags.models import Tag
+
+from . import filters, lb_recommendations, models
+from .registries_v2 import registry
+
+logger = logging.getLogger(__name__)
+
+
+class SimpleRadio:
+    related_object_field = None
+
+    def clean(self, instance):
+        return
+
+    def weighted_pick(
+        self,
+        choices: List[Tuple[int, int]],
+        previous_choices: Optional[List[int]] = None,
+    ) -> int:
+        total = sum(weight for c, weight in choices)
+        r = random.uniform(0, total)
+        upto = 0
+        for choice, weight in choices:
+            if upto + weight >= r:
+                return choice
+            upto += weight
+
+
+class SessionRadio(SimpleRadio):
+    def __init__(self, session=None):
+        self.session = session
+
+    def start_session(self, user, **kwargs):
+        self.session = models.RadioSession.objects.create(
+            user=user, radio_type=self.radio_type, **kwargs
+        )
+        return self.session
+
+    def get_queryset(self, **kwargs):
+        actor = None
+        try:
+            actor = self.session.user.actor
+        except KeyError:
+            pass  # Maybe logging would be helpful
+
+        qs = (
+            Track.objects.all()
+            .with_playable_uploads(actor=actor)
+            .select_related("artist", "album__artist", "attributed_to")
+        )
+
+        query = moderation_filters.get_filtered_content_query(
+            config=moderation_filters.USER_FILTER_CONFIG["TRACK"],
+            user=self.session.user,
+        )
+        return qs.exclude(query)
+
+    def get_queryset_kwargs(self):
+        return {}
+
+    def filter_queryset(self, queryset):
+        return queryset
+
+    def filter_from_session(self, queryset):
+        already_played = self.session.session_tracks.all().values_list(
+            "track", flat=True
+        )
+        queryset = queryset.exclude(pk__in=already_played)
+        return queryset
+
+    def cache_batch_radio_track(self, **kwargs):
+        BATCH_SIZE = 100
+        # get cached RadioTracks if any
+        try:
+            cached_evaluated_radio_tracks = pickle.loads(
+                cache.get(f"radiotracks{self.session.id}")
+            )
+        except TypeError:
+            cached_evaluated_radio_tracks = None
+
+        # get the queryset and apply filters
+        kwargs.update(self.get_queryset_kwargs())
+        queryset = self.get_queryset(**kwargs)
+        queryset = self.filter_from_session(queryset)
+
+        if kwargs["filter_playable"] is True:
+            queryset = queryset.playable_by(
+                self.session.user.actor if self.session.user else None
+            )
+        queryset = self.filter_queryset(queryset)
+
+        # select a random batch of the qs
+        sliced_queryset = queryset.order_by("?")[:BATCH_SIZE]
+        if len(sliced_queryset) <= 0 and not cached_evaluated_radio_tracks:
+            raise ValueError("No more radio candidates")
+
+        # create the radio session tracks into db in bulk
+        self.session.add(sliced_queryset)
+
+        # evaluate the queryset to save it in cache
+        radio_tracks = list(sliced_queryset)
+
+        if cached_evaluated_radio_tracks is not None:
+            radio_tracks.extend(cached_evaluated_radio_tracks)
+        logger.info(
+            f"Setting redis cache for radio generation with radio id {self.session.id}"
+        )
+        cache.set(f"radiotracks{self.session.id}", pickle.dumps(radio_tracks), 3600)
+        cache.set(f"radioqueryset{self.session.id}", sliced_queryset, 3600)
+
+        return sliced_queryset
+
+    def get_choices(self, quantity, **kwargs):
+        if cache.get(f"radiotracks{self.session.id}"):
+            cached_radio_tracks = pickle.loads(
+                cache.get(f"radiotracks{self.session.id}")
+            )
+            logger.info("Using redis cache for radio generation")
+            radio_tracks = cached_radio_tracks
+            if len(radio_tracks) < quantity:
+                logger.info(
+                    "Not enough radio tracks in cache. Trying to generate new cache"
+                )
+                sliced_queryset = self.cache_batch_radio_track(**kwargs)
+            sliced_queryset = cache.get(f"radioqueryset{self.session.id}")
+        else:
+            sliced_queryset = self.cache_batch_radio_track(**kwargs)
+
+        return sliced_queryset[:quantity]
+
+    def pick_many(self, quantity, **kwargs):
+        if self.session:
+            sliced_queryset = self.get_choices(quantity=quantity, **kwargs)
+        else:
+            logger.info(
+                "No radio session. Can't track user playback. Won't cache queryset results"
+            )
+            sliced_queryset = self.get_choices(quantity=quantity, **kwargs)
+
+        return sliced_queryset
+
+    def validate_session(self, data, **context):
+        return data
+
+
+@registry.register(name="random")
+class RandomRadio(SessionRadio):
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        return qs.filter(artist__content_category="music").order_by("?")
+
+
+@registry.register(name="random_library")
+class RandomLibraryRadio(SessionRadio):
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        tracks_ids = self.session.user.actor.attributed_tracks.all().values_list(
+            "id", flat=True
+        )
+        query = Q(artist__content_category="music") & Q(pk__in=tracks_ids)
+        return qs.filter(query).order_by("?")
+
+
+@registry.register(name="favorites")
+class FavoritesRadio(SessionRadio):
+    def get_queryset_kwargs(self):
+        kwargs = super().get_queryset_kwargs()
+        if self.session:
+            kwargs["user"] = self.session.user
+        return kwargs
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        track_ids = kwargs["user"].track_favorites.all().values_list("track", flat=True)
+        return qs.filter(pk__in=track_ids, artist__content_category="music")
+
+
+@registry.register(name="custom")
+class CustomRadio(SessionRadio):
+    def get_queryset_kwargs(self):
+        kwargs = super().get_queryset_kwargs()
+        kwargs["user"] = self.session.user
+        kwargs["custom_radio"] = self.session.custom_radio
+        return kwargs
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        return filters.run(kwargs["custom_radio"].config, candidates=qs)
+
+    def validate_session(self, data, **context):
+        data = super().validate_session(data, **context)
+        try:
+            user = data["user"]
+        except KeyError:
+            user = context.get("user")
+        try:
+            assert data["custom_radio"].user == user or data["custom_radio"].is_public
+        except KeyError:
+            raise serializers.ValidationError("You must provide a custom radio")
+        except AssertionError:
+            raise serializers.ValidationError("You don't have access to this radio")
+        return data
+
+
+@registry.register(name="custom_multiple")
+class CustomMultiple(SessionRadio):
+    """
+    Receive a vuejs generated config and use it to launch a radio session
+    """
+
+    config = serializers.JSONField(required=True)
+
+    def get_config(self, data):
+        return data["config"]
+
+    def get_queryset_kwargs(self):
+        kwargs = super().get_queryset_kwargs()
+        kwargs["config"] = self.session.config
+        return kwargs
+
+    def validate_session(self, data, **context):
+        data = super().validate_session(data, **context)
+        try:
+            data["config"] is not None
+        except KeyError:
+            raise serializers.ValidationError(
+                "You must provide a configuration for this radio"
+            )
+        return data
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        return filters.run([kwargs["config"]], candidates=qs)
+
+
+class RelatedObjectRadio(SessionRadio):
+    """Abstract radio related to an object (tag, artist, user...)"""
+
+    related_object_field = serializers.IntegerField(required=True)
+
+    def clean(self, instance):
+        super().clean(instance)
+        if not instance.related_object:
+            raise ValidationError(
+                "Cannot start RelatedObjectRadio without related object"
+            )
+        if not isinstance(instance.related_object, self.model):
+            raise ValidationError("Trying to start radio with bad related object")
+
+    def get_related_object(self, pk):
+        return self.model.objects.get(pk=pk)
+
+
+@registry.register(name="tag")
+class TagRadio(RelatedObjectRadio):
+    model = Tag
+    related_object_field = serializers.CharField(required=True)
+
+    def get_related_object(self, name):
+        return self.model.objects.get(name=name)
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        query = (
+            Q(tagged_items__tag=self.session.related_object)
+            | Q(artist__tagged_items__tag=self.session.related_object)
+            | Q(album__tagged_items__tag=self.session.related_object)
+        )
+        return qs.filter(query)
+
+    def get_related_object_id_repr(self, obj):
+        return obj.name
+
+
+def weighted_choice(choices):
+    total = sum(w for c, w in choices)
+    r = random.uniform(0, total)
+    upto = 0
+    for c, w in choices:
+        if upto + w >= r:
+            return c
+        upto += w
+    assert False, "Shouldn't get here"
+
+
+class NextNotFound(Exception):
+    pass
+
+
+@registry.register(name="similar")
+class SimilarRadio(RelatedObjectRadio):
+    model = Track
+
+    def filter_queryset(self, queryset):
+        queryset = super().filter_queryset(queryset)
+        seeds = list(
+            self.session.session_tracks.all()
+            .values_list("track_id", flat=True)
+            .order_by("-id")[:3]
+        ) + [self.session.related_object.pk]
+        for seed in seeds:
+            try:
+                return queryset.filter(pk=self.find_next_id(queryset, seed))
+            except NextNotFound:
+                continue
+
+        return queryset.none()
+
+    def find_next_id(self, queryset, seed):
+        with connection.cursor() as cursor:
+            query = """
+            SELECT next, count(next) AS c
+            FROM (
+                SELECT
+                    track_id,
+                    creation_date,
+                    LEAD(track_id) OVER (
+                        PARTITION by user_id order by creation_date asc
+                    ) AS next
+                FROM history_listening
+                INNER JOIN users_user ON (users_user.id = user_id)
+                WHERE users_user.privacy_level = 'instance' OR users_user.privacy_level = 'everyone' OR user_id = %s
+                ORDER BY creation_date ASC
+            ) t WHERE track_id = %s AND next != %s GROUP BY next ORDER BY c DESC;
+            """
+            cursor.execute(query, [self.session.user_id, seed, seed])
+            next_candidates = list(cursor.fetchall())
+
+        if not next_candidates:
+            raise NextNotFound()
+
+        matching_tracks = list(
+            queryset.filter(pk__in=[c[0] for c in next_candidates]).values_list(
+                "id", flat=True
+            )
+        )
+        next_candidates = [n for n in next_candidates if n[0] in matching_tracks]
+        if not next_candidates:
+            raise NextNotFound()
+        return random.choice([c[0] for c in next_candidates])
+
+
+@registry.register(name="artist")
+class ArtistRadio(RelatedObjectRadio):
+    model = Artist
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        return qs.filter(artist=self.session.related_object)
+
+
+@registry.register(name="less-listened")
+class LessListenedRadio(SessionRadio):
+    def clean(self, instance):
+        instance.related_object = instance.user
+        super().clean(instance)
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        listened = self.session.user.listenings.all().values_list("track", flat=True)
+        return (
+            qs.filter(artist__content_category="music")
+            .exclude(pk__in=listened)
+            .order_by("?")
+        )
+
+
+@registry.register(name="less-listened_library")
+class LessListenedLibraryRadio(SessionRadio):
+    def clean(self, instance):
+        instance.related_object = instance.user
+        super().clean(instance)
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        listened = self.session.user.listenings.all().values_list("track", flat=True)
+        tracks_ids = self.session.user.actor.attributed_tracks.all().values_list(
+            "id", flat=True
+        )
+        query = Q(artist__content_category="music") & Q(pk__in=tracks_ids)
+        return qs.filter(query).exclude(pk__in=listened).order_by("?")
+
+
+@registry.register(name="actor-content")
+class ActorContentRadio(RelatedObjectRadio):
+    """
+    Play content from given actor libraries
+    """
+
+    model = federation_models.Actor
+    related_object_field = federation_fields.ActorRelatedField(required=True)
+
+    def get_related_object(self, value):
+        return value
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        actor_uploads = Upload.objects.filter(
+            library__actor=self.session.related_object,
+        )
+        return qs.filter(pk__in=actor_uploads.values("track"))
+
+    def get_related_object_id_repr(self, obj):
+        return obj.full_username
+
+
+@registry.register(name="library")
+class LibraryRadio(RelatedObjectRadio):
+    """
+    Play content from a given library
+    """
+
+    model = Library
+    related_object_field = serializers.UUIDField(required=True)
+
+    def get_related_object(self, value):
+        return Library.objects.get(uuid=value)
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        actor_uploads = Upload.objects.filter(
+            library=self.session.related_object,
+        )
+        return qs.filter(pk__in=actor_uploads.values("track"))
+
+    def get_related_object_id_repr(self, obj):
+        return obj.uuid
+
+
+@registry.register(name="recently-added")
+class RecentlyAdded(SessionRadio):
+    def get_queryset(self, **kwargs):
+        date = datetime.date.today() - datetime.timedelta(days=30)
+        qs = super().get_queryset(**kwargs)
+        return qs.filter(
+            Q(artist__content_category="music"),
+            Q(creation_date__gt=date),
+        )
+
+
+# Use this to experiment on the custom multiple radio with troi
+@registry.register(name="troi")
+class Troi(SessionRadio):
+    """
+    Receive a vuejs generated config and use it to launch a troi radio session.
+    The config data should follow :
+    {"patch": "troi_patch_name", "troi_arg1":"troi_arg_1", "troi_arg2": ...}
+    Validation of the config (args) is done by troi during track fetch.
+    Funkwhale only checks if the patch is implemented
+    """
+
+    config = serializers.JSONField(required=True)
+
+    def append_lb_config(self, data):
+        if self.session.user.settings is None:
+            logger.warning(
+                "No lb_user_name set in user settings. Some troi patches will fail"
+            )
+            return data
+        elif self.session.user.settings.get("lb_user_name") is None:
+            logger.warning(
+                "No lb_user_name set in user settings. Some troi patches will fail"
+            )
+        else:
+            data["user_name"] = self.session.user.settings["lb_user_name"]
+
+        if self.session.user.settings.get("lb_user_token") is None:
+            logger.warning(
+                "No lb_user_token set in user settings. Some troi patch will fail"
+            )
+        else:
+            data["user_token"] = self.session.user.settings["lb_user_token"]
+
+        return data
+
+    def get_queryset_kwargs(self):
+        kwargs = super().get_queryset_kwargs()
+        kwargs["config"] = self.session.config
+        return kwargs
+
+    def validate_session(self, data, **context):
+        data = super().validate_session(data, **context)
+        if data.get("config") is None:
+            raise serializers.ValidationError(
+                "You must provide a configuration for this radio"
+            )
+        return data
+
+    def get_queryset(self, **kwargs):
+        qs = super().get_queryset(**kwargs)
+        config = self.append_lb_config(json.loads(kwargs["config"]))
+
+        return lb_recommendations.run(config, candidates=qs)

api/funkwhale_api/radios/registries_v2.py

@@ -0,0 +1,10 @@
+import persisting_theory
+
+
+class RadioRegistry_v2(persisting_theory.Registry):
+    def prepare_name(self, data, name=None):
+        setattr(data, "radio_type", name)
+        return name
+
+
+registry = RadioRegistry_v2()

api/funkwhale_api/radios/serializers.py

@@ -40,9 +40,11 @@ class RadioSerializer(serializers.ModelSerializer):
 
 
 class RadioSessionTrackSerializerCreate(serializers.ModelSerializer):
+    count = serializers.IntegerField(required=False, allow_null=True)
+
     class Meta:
         model = models.RadioSessionTrack
-        fields = ("session",)
+        fields = ("session", "count")
 
 
 class RadioSessionTrackSerializer(serializers.ModelSerializer):

api/funkwhale_api/radios/urls.py

@@ -5,7 +5,7 @@ from . import views
 router = routers.OptionalSlashRouter()
 router.register(r"sessions", views.RadioSessionViewSet, "sessions")
 router.register(r"radios", views.RadioViewSet, "radios")
-router.register(r"tracks", views.RadioSessionTrackViewSet, "tracks")
+router.register(r"tracks", views.V1_RadioSessionTrackViewSet, "tracks")
 
 
 urlpatterns = router.urls

api/funkwhale_api/radios/urls_v2.py

@@ -0,0 +1,10 @@
+from funkwhale_api.common import routers
+
+from . import views
+
+router = routers.OptionalSlashRouter()
+
+router.register(r"sessions", views.V2_RadioSessionViewSet, "sessions")
+
+
+urlpatterns = router.urls

api/funkwhale_api/radios/views.py

@@ -1,3 +1,6 @@
+import pickle
+
+from django.core.cache import cache
 from django.db.models import Q
 from drf_spectacular.utils import extend_schema
 from rest_framework import mixins, status, viewsets
@@ -121,7 +124,7 @@ class RadioSessionViewSet(
         return context
 
 
-class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
+class V1_RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
     serializer_class = serializers.RadioSessionTrackSerializer
     queryset = models.RadioSessionTrack.objects.all()
     permission_classes = []
@@ -133,21 +136,19 @@ class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet)
         session = serializer.validated_data["session"]
         if not request.user.is_authenticated and not request.session.session_key:
             self.request.session.create()
-        try:
-            assert (request.user == session.user) or (
-                request.session.session_key == session.session_key
-                and session.session_key
-            )
-        except AssertionError:
+        if not request.user == session.user or (
+            not request.session.session_key == session.session_key
+            and not session.session_key
+        ):
             return Response(status=status.HTTP_403_FORBIDDEN)
+
         try:
-            session.radio.pick()
+            session.radio(api_version=1).pick()
         except ValueError:
             return Response(
                 "Radio doesn't have more candidates", status=status.HTTP_404_NOT_FOUND
             )
         session_track = session.session_tracks.all().latest("id")
-        # self.perform_create(serializer)
         # dirty override here, since we use a different serializer for creation and detail
         serializer = self.serializer_class(
             instance=session_track, context=self.get_serializer_context()
@@ -161,3 +162,99 @@ class RadioSessionTrackViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet)
         if self.action == "create":
             return serializers.RadioSessionTrackSerializerCreate
         return super().get_serializer_class(*args, **kwargs)
+
+
+class V2_RadioSessionViewSet(
+    mixins.CreateModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet
+):
+    """Returns a list of RadioSessions"""
+
+    serializer_class = serializers.RadioSessionSerializer
+    queryset = models.RadioSession.objects.all()
+    permission_classes = []
+
+    @action(detail=True, serializer_class=serializers.RadioSessionTrackSerializerCreate)
+    def tracks(self, request, pk, *args, **kwargs):
+        data = {"session": pk}
+        data["count"] = (
+            request.query_params["count"]
+            if "count" in request.query_params.keys()
+            else 1
+        )
+        serializer = serializers.RadioSessionTrackSerializerCreate(data=data)
+        serializer.is_valid(raise_exception=True)
+        session = serializer.validated_data["session"]
+
+        count = int(data["count"])
+        # this is used for test purpose.
+        filter_playable = (
+            request.query_params["filter_playable"]
+            if "filter_playable" in request.query_params.keys()
+            else True
+        )
+        if not request.user.is_authenticated and not request.session.session_key:
+            self.request.session.create()
+
+        if not request.user == session.user or (
+            not request.session.session_key == session.session_key
+            and not session.session_key
+        ):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+        try:
+            from . import radios_v2  # noqa
+
+            session.radio(api_version=2).pick_many(
+                count, filter_playable=filter_playable
+            )
+        except ValueError:
+            return Response(
+                "Radio doesn't have more candidates", status=status.HTTP_404_NOT_FOUND
+            )
+
+        # dirty override here, since we use a different serializer for creation and detail
+        evaluated_radio_tracks = pickle.loads(cache.get(f"radiotracks{session.id}"))
+        batch = evaluated_radio_tracks[:count]
+        serializer = TrackSerializer(
+            data=batch,
+            many="true",
+        )
+        serializer.is_valid()
+
+        # delete the tracks we sent from the cache
+        new_cached_radiotracks = evaluated_radio_tracks[count:]
+        cache.set(f"radiotracks{session.id}", pickle.dumps(new_cached_radiotracks))
+
+        return Response(
+            serializer.data,
+            status=status.HTTP_201_CREATED,
+        )
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.request.user.is_authenticated:
+            return queryset.filter(
+                Q(user=self.request.user)
+                | Q(session_key=self.request.session.session_key)
+            )
+
+        return queryset.filter(session_key=self.request.session.session_key).exclude(
+            session_key=None
+        )
+
+    def perform_create(self, serializer):
+        if (
+            not self.request.user.is_authenticated
+            and not self.request.session.session_key
+        ):
+            self.request.session.create()
+        return serializer.save(
+            user=self.request.user if self.request.user.is_authenticated else None,
+            session_key=self.request.session.session_key,
+        )
+
+    def get_serializer_context(self):
+        context = super().get_serializer_context()
+        context["user"] = (
+            self.request.user if self.request.user.is_authenticated else None
+        )
+        return context

api/funkwhale_api/subsonic/renderers.py

@@ -6,6 +6,10 @@ from rest_framework import renderers
 import funkwhale_api
 
 
+class TagValue(str):
+    """Use this for string values that must be rendered as tags instead of attributes in XML."""
+
+
 # from https://stackoverflow.com/a/8915039
 # because I want to avoid a lxml dependency just for outputting cdata properly
 # in a RSS feed
@@ -31,10 +35,14 @@ ET._serialize_xml = ET._serialize["xml"] = _serialize_xml
 
 def structure_payload(data):
     payload = {
+        # funkwhaleVersion is deprecated and will be removed in a future
+        # release. Use serverVersion instead.
         "funkwhaleVersion": funkwhale_api.__version__,
+        "serverVersion": funkwhale_api.__version__,
         "status": "ok",
         "type": "funkwhale",
         "version": "1.16.0",
+        "openSubsonic": "true",
     }
     payload.update(data)
     if "detail" in payload:
@@ -81,6 +89,10 @@ def dict_to_xml_tree(root_tag, d, parent=None):
                     el = ET.Element(key)
                     el.text = str(obj)
                 root.append(el)
+        elif isinstance(value, TagValue):
+            el = ET.Element(key)
+            el.text = str(value)
+            root.append(el)
         else:
             if key == "value":
                 root.text = str(value)

api/funkwhale_api/subsonic/serializers.py

@@ -7,6 +7,8 @@ from funkwhale_api.history import models as history_models
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import utils as music_utils
 
+from .renderers import TagValue
+
 
 def to_subsonic_date(date):
     """
@@ -50,6 +52,7 @@ def get_artist_data(artist_values):
         "name": artist_values["name"],
         "albumCount": artist_values["_albums_count"],
         "coverArt": "ar-{}".format(artist_values["id"]),
+        "musicBrainzId": str(artist_values.get("mbid", "")),
     }
 
 
@@ -58,7 +61,7 @@ class GetArtistsSerializer(serializers.Serializer):
         payload = {"ignoredArticles": "", "index": []}
         queryset = queryset.with_albums_count()
         queryset = queryset.order_by(functions.Lower("name"))
-        values = queryset.values("id", "_albums_count", "name")
+        values = queryset.values("id", "_albums_count", "name", "mbid")
 
         first_letter_mapping = collections.defaultdict(list)
         for artist in values:
@@ -102,6 +105,23 @@ class GetArtistSerializer(serializers.Serializer):
         return payload
 
 
+class GetArtistInfo2Serializer(serializers.Serializer):
+    def to_representation(self, artist):
+        payload = {}
+        if artist.mbid:
+            payload["musicBrainzId"] = TagValue(artist.mbid)
+        if artist.attachment_cover:
+            payload["mediumImageUrl"] = TagValue(
+                artist.attachment_cover.download_url_medium_square_crop
+            )
+            payload["largeImageUrl"] = TagValue(
+                artist.attachment_cover.download_url_large_square_crop
+            )
+        if artist.description:
+            payload["biography"] = TagValue(artist.description.rendered)
+        return payload
+
+
 def get_track_data(album, track, upload):
     data = {
         "id": track.pk,
@@ -126,11 +146,13 @@ def get_track_data(album, track, upload):
         "albumId": album.pk if album else "",
         "artistId": album.artist.pk if album else track.artist.pk,
         "type": "music",
+        "mediaType": "song",
+        "musicBrainzId": str(track.mbid or ""),
     }
     if album and album.attachment_cover_id:
         data["coverArt"] = f"al-{album.id}"
     if upload.bitrate:
-        data["bitrate"] = int(upload.bitrate / 1000)
+        data["bitRate"] = int(upload.bitrate / 1000)
     if upload.size:
         data["size"] = upload.size
     if album and album.release_date:
@@ -149,13 +171,17 @@ def get_album2_data(album):
         "created": to_subsonic_date(album.creation_date),
         "duration": album.duration,
         "playCount": album.tracks.aggregate(l=Count("listenings"))["l"] or 0,
+        "mediaType": "album",
+        "musicBrainzId": str(album.mbid or ""),
     }
     if album.attachment_cover_id:
         payload["coverArt"] = f"al-{album.id}"
     if album.tagged_items:
+        genres = [{"name": i.tag.name} for i in album.tagged_items.all()]
         # exposes only first genre since the specification uses singular noun
-        first_genre = album.tagged_items.first()
-        payload["genre"] = first_genre.tag.name if first_genre else ""
+        payload["genre"] = genres[0]["name"] if len(genres) > 0 else ""
+        # OpenSubsonic full genre list
+        payload["genres"] = genres
     if album.release_date:
         payload["year"] = album.release_date.year
     try:
@@ -343,7 +369,7 @@ def get_channel_episode_data(upload, channel_id):
         "genre": "Podcast",
         "size": upload.size if upload.size else "",
         "duration": upload.duration if upload.duration else "",
-        "bitrate": upload.bitrate / 1000 if upload.bitrate else "",
+        "bitRate": upload.bitrate / 1000 if upload.bitrate else "",
         "contentType": upload.mimetype or "audio/mpeg",
         "suffix": upload.extension or "mp3",
         "status": "completed",

api/funkwhale_api/subsonic/views.py

@@ -180,6 +180,19 @@ class SubsonicViewSet(viewsets.GenericViewSet):
         }
         return response.Response(data, status=200)
 
+    @action(
+        detail=False,
+        methods=["get", "post"],
+        url_name="get_open_subsonic_extensions",
+        permission_classes=[],
+        url_path="getOpenSubsonicExtensions",
+    )
+    def get_open_subsonic_extensions(self, request, *args, **kwargs):
+        data = {
+            "openSubsonicExtensions": [{"name": "formPost", "versions": [1]}],
+        }
+        return response.Response(data, status=200)
+
     @action(
         detail=False,
         methods=["get", "post"],
@@ -255,7 +268,9 @@ class SubsonicViewSet(viewsets.GenericViewSet):
     )
     @find_object(music_models.Artist.objects.all(), filter_playable=True)
     def get_artist_info2(self, request, *args, **kwargs):
-        payload = {"artist-info2": {}}
+        artist = kwargs.pop("obj")
+        data = serializers.GetArtistInfo2Serializer(artist).data
+        payload = {"artistInfo2": data}
 
         return response.Response(payload, status=200)
 
@@ -523,7 +538,7 @@ class SubsonicViewSet(viewsets.GenericViewSet):
                 "search_fields": ["name"],
                 "queryset": (
                     music_models.Artist.objects.with_albums_count().values(
-                        "id", "_albums_count", "name"
+                        "id", "_albums_count", "name", "mbid"
                     )
                 ),
                 "serializer": lambda qs: [serializers.get_artist_data(a) for a in qs],

api/funkwhale_api/typesense/factories.py

@@ -0,0 +1,111 @@
+from troi import Artist, Element, Playlist, Recording
+from troi.patch import Patch
+
+recording_list = [
+    Recording(
+        name="I Want It That Way",
+        mbid="87dfa566-21c3-45ed-bc42-1d345b8563fa",
+        artist=Artist(name="artist_name"),
+    ),
+    Recording(name="Untouchable", artist=Artist(name="Another lol")),
+    Recording(
+        name="The Perfect Kiss",
+        mbid="ec0da94e-fbfe-4eb0-968e-024d4c32d1d0",
+        artist=Artist(name="artist_name2"),
+    ),
+    Recording(
+        name="Love Your Voice",
+        mbid="93726547-f8c0-4efd-8e16-d2dee76500f6",
+        artist=Artist(name="artist_name"),
+    ),
+    Recording(
+        name="Hall of Fame",
+        mbid="395bd5a1-79cc-4e04-8869-ca9eabc78d09",
+        artist=Artist(name="artist_name_3"),
+    ),
+]
+
+
+class DummyElement(Element):
+    """Dummy element that returns a fixed playlist for testing"""
+
+    @staticmethod
+    def outputs():
+        return [Playlist]
+
+    def read(self, sources):
+        recordings = recording_list
+
+        return [
+            Playlist(
+                name="Test Export Playlist",
+                description="A playlist to test exporting playlists to spotify",
+                recordings=recordings,
+            )
+        ]
+
+
+class DummyPatch(Patch):
+    """Dummy patch that always returns a fixed set of recordings for testing"""
+
+    @staticmethod
+    def slug():
+        return "test-patch"
+
+    def create(self, inputs):
+        return DummyElement()
+
+    @staticmethod
+    def outputs():
+        return [Recording]
+
+
+recommended_recording_mbids = [
+    "87dfa566-21c3-45ed-bc42-1d345b8563fa",
+    "ec0da94e-fbfe-4eb0-968e-024d4c32d1d0",
+    "93726547-f8c0-4efd-8e16-d2dee76500f6",
+    "395bd5a1-79cc-4e04-8869-ca9eabc78d09",
+]
+
+typesense_search_result = {
+    "facet_counts": [],
+    "found": 1,
+    "out_of": 1,
+    "page": 1,
+    "request_params": {
+        "collection_name": "canonical_fw_data",
+        "per_page": 10,
+        "q": "artist_nameiwantitthatway",
+    },
+    "search_time_ms": 1,
+    "hits": [
+        {
+            "highlights": [
+                {
+                    "field": "combined",
+                    "snippet": "string",
+                    "matched_tokens": ["string"],
+                }
+            ],
+            "document": {
+                "pk": "1",
+                "combined": "artist_nameiwantitthatway",
+            },
+            "text_match": 130916,
+        },
+        {
+            "highlights": [
+                {
+                    "field": "combined",
+                    "snippet": "string",
+                    "matched_tokens": ["string"],
+                }
+            ],
+            "document": {
+                "pk": "2",
+                "combined": "artist_nameiwantitthatway",
+            },
+            "text_match": 130916,
+        },
+    ],
+}

api/funkwhale_api/typesense/tasks.py

@@ -0,0 +1,108 @@
+import logging
+
+from django.conf import settings
+
+from funkwhale_api.music import models as music_models
+from funkwhale_api.taskapp import celery
+
+from . import utils
+
+logger = logging.getLogger(__name__)
+
+
+class TypesenseNotActivate(Exception):
+    pass
+
+
+if not settings.TYPESENSE_API_KEY:
+    logger.info(
+        "Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
+    )
+else:
+    import typesense
+    from typesense.exceptions import ObjectAlreadyExists
+
+
+api_key = settings.TYPESENSE_API_KEY
+host = settings.TYPESENSE_HOST
+port = settings.TYPESENSE_PORT
+protocol = settings.TYPESENSE_PROTOCOL
+
+collection_name = "canonical_fw_data"
+BATCH_SIZE = 10000
+
+
+@celery.app.task(name="typesense.add_tracks_to_index")
+def add_tracks_to_index(tracks_pk):
+    """
+    This will add fw tracks data to the typesense index. It will concatenate the artist name
+    and the track title into one string.
+    """
+
+    client = typesense.Client(
+        {
+            "api_key": api_key,
+            "nodes": [{"host": host, "port": port, "protocol": protocol}],
+            "connection_timeout_seconds": 2,
+        }
+    )
+
+    try:
+        logger.info(f"Updating index {collection_name}")
+        tracks = music_models.Track.objects.all().filter(pk__in=tracks_pk)
+        documents = []
+        for track in tracks:
+            document = dict()
+            document["pk"] = track.pk
+            document["combined"] = utils.delete_non_alnum_characters(
+                track.artist.name + track.title
+            )
+            documents.append(document)
+
+        client.collections[collection_name].documents.import_(
+            documents, {"action": "upsert"}
+        )
+
+    except typesense.exceptions.TypesenseClientError as err:
+        logger.error(f"Can't build index: {str(err)}")
+
+
+@celery.app.task(name="typesense.build_canonical_index")
+def build_canonical_index():
+    if not settings.TYPESENSE_API_KEY:
+        raise TypesenseNotActivate(
+            "Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
+        )
+
+    schema = {
+        "name": collection_name,
+        "fields": [
+            {"name": "combined", "type": "string"},
+            {"name": "pk", "type": "int32"},
+        ],
+        "default_sorting_field": "pk",
+    }
+    client = typesense.Client(
+        {
+            "api_key": api_key,
+            "nodes": [{"host": host, "port": port, "protocol": protocol}],
+            "connection_timeout_seconds": 2,
+        }
+    )
+    try:
+        client.collections.create(schema)
+    except ObjectAlreadyExists:
+        pass
+
+    tracks = music_models.Track.objects.all().values_list("pk", flat=True)
+    total_tracks = tracks.count()
+    total_batches = (total_tracks - 1) // BATCH_SIZE + 1
+
+    for i in range(total_batches):
+        start_index = i * BATCH_SIZE
+        end_index = (i + 1) * (BATCH_SIZE - 1)
+        batch_tracks = tracks[start_index:end_index]
+        logger.info(
+            f"Launching async task to add {str(batch_tracks)} tracks pks to index"
+        )
+        add_tracks_to_index.delay(list(batch_tracks))

api/funkwhale_api/typesense/utils.py

@@ -0,0 +1,91 @@
+import logging
+import re
+
+import unidecode
+from django.conf import settings
+from django.core.cache import cache
+from lb_matching_tools.cleaner import MetadataCleaner
+
+from funkwhale_api.music import models as music_models
+
+logger = logging.getLogger(__name__)
+
+api_key = settings.TYPESENSE_API_KEY
+host = settings.TYPESENSE_HOST
+port = settings.TYPESENSE_PORT
+protocol = settings.TYPESENSE_PROTOCOL
+TYPESENSE_NUM_TYPO = settings.TYPESENSE_NUM_TYPO
+
+
+class TypesenseNotActivate(Exception):
+    pass
+
+
+if not settings.TYPESENSE_API_KEY:
+    logger.info(
+        "Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
+    )
+else:
+    import typesense
+
+
+def delete_non_alnum_characters(text):
+    return unidecode.unidecode(re.sub(r"[^\w]+", "", text).lower())
+
+
+def resolve_recordings_to_fw_track(recordings):
+    """
+    Tries to match a troi recording entity to a fw track using the typesense index.
+    For test purposes : if multiple fw tracks are returned, we log the information
+    but only keep the best result in db to avoid duplicates.
+    """
+
+    if not settings.TYPESENSE_API_KEY:
+        raise TypesenseNotActivate(
+            "Typesense is not activated. You can enable it by setting the TYPESENSE_API_KEY env variable."
+        )
+
+    client = typesense.Client(
+        {
+            "api_key": api_key,
+            "nodes": [{"host": host, "port": port, "protocol": protocol}],
+            "connection_timeout_seconds": 2,
+        }
+    )
+
+    mc = MetadataCleaner()
+
+    for recording in recordings:
+        rec = mc.clean_recording(recording.name)
+        artist = mc.clean_artist(recording.artist.name)
+        canonical_name_for_track = delete_non_alnum_characters(artist + rec)
+
+        logger.debug(f"Trying to resolve : {canonical_name_for_track}")
+
+        search_parameters = {
+            "q": canonical_name_for_track,
+            "query_by": "combined",
+            "num_typos": TYPESENSE_NUM_TYPO,
+            "drop_tokens_threshold": 0,
+        }
+        matches = client.collections["canonical_fw_data"].documents.search(
+            search_parameters
+        )
+
+        if matches["hits"]:
+            hit = matches["hits"][0]
+            pk = hit["document"]["pk"]
+            logger.debug(f"Saving match for track with primary key {pk}")
+            cache.set(recording.mbid, pk)
+
+            if settings.DEBUG and matches["hits"][1]:
+                for hit in matches["hits"][1:]:
+                    pk = hit["document"]["pk"]
+                    fw_track = music_models.Track.objects.get(pk=pk)
+                    logger.info(
+                        f"Duplicate match found for {fw_track.artist.name} {fw_track.title} \
+                                and primary key {pk}. Skipping because of better match."
+                    )
+        else:
+            logger.debug("No match found in fw db")
+    return cache.get_many([rec.mbid for rec in recordings])