From 15673c399fb8e0145713cbedddfc23066dec7974 Mon Sep 17 00:00:00 2001 From: Neeraj P Yetheendran Date: Thu, 15 Feb 2024 16:02:06 +0530 Subject: [PATCH 1/2] mysql and postgres backends limit models for indexing --- .../search/backends/database/mysql/mysql.py | 22 ++++++++----- .../backends/database/postgres/postgres.py | 18 ++++++---- .../management/commands/update_index.py | 33 ++++++++++++++++--- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/wagtail/search/backends/database/mysql/mysql.py b/wagtail/search/backends/database/mysql/mysql.py index 94843edbee..31371158d9 100644 --- a/wagtail/search/backends/database/mysql/mysql.py +++ b/wagtail/search/backends/database/mysql/mysql.py @@ -212,12 +212,16 @@ class Index: ).exclude(object_id__in=existing_pks) stale_entries.delete() - def delete_stale_entries(self): - for model in get_indexed_models(): - # We don’t need to delete stale entries for non-root models, - # since we already delete them by deleting roots. - if not model._meta.parents: + def delete_stale_entries(self, models=None): + if models: + for model in models: self.delete_stale_model_entries(model) + else: + for model in get_indexed_models(): + # We don’t need to delete stale entries for non-root models, + # since we already delete them by deleting roots. + if not model._meta.parents: + self.delete_stale_model_entries(model) def add_item(self, obj): self.add_items(obj._meta.model, [obj]) @@ -491,7 +495,9 @@ class MySQLSearchQueryCompiler(BaseSearchQueryCompiler): ) if not negated: index_entries = index_entries.filter(match_expression) - if self.order_by_relevance: # Only applies to the case where the outermost query is not a Not(), because if it is, the relevance score is always 0 (anything that matches is excluded from the results). + if ( + self.order_by_relevance + ): # Only applies to the case where the outermost query is not a Not(), because if it is, the relevance score is always 0 (anything that matches is excluded from the results). index_entries = index_entries.order_by(score_expression.desc()) else: index_entries = index_entries.exclude(match_expression) @@ -599,8 +605,8 @@ class MySQLSearchRebuilder: def __init__(self, index): self.index = index - def start(self): - self.index.delete_stale_entries() + def start(self, models=None): + self.index.delete_stale_entries(models) return self.index def finish(self): diff --git a/wagtail/search/backends/database/postgres/postgres.py b/wagtail/search/backends/database/postgres/postgres.py index b8758c7b78..a3d5bb832b 100644 --- a/wagtail/search/backends/database/postgres/postgres.py +++ b/wagtail/search/backends/database/postgres/postgres.py @@ -227,12 +227,16 @@ class Index: ).exclude(object_id__in=existing_pks) stale_entries.delete() - def delete_stale_entries(self): - for model in get_indexed_models(): - # We don’t need to delete stale entries for non-root models, - # since we already delete them by deleting roots. - if not model._meta.parents: + def delete_stale_entries(self, models=None): + if models: + for model in models: self.delete_stale_model_entries(model) + else: + for model in get_indexed_models(): + # We don’t need to delete stale entries for non-root models, + # since we already delete them by deleting roots. + if not model._meta.parents: + self.delete_stale_model_entries(model) def add_item(self, obj): self.add_items(obj._meta.model, [obj]) @@ -698,8 +702,8 @@ class PostgresSearchRebuilder: def __init__(self, index): self.index = index - def start(self): - self.index.delete_stale_entries() + def start(self, models=None): + self.index.delete_stale_entries(models) return self.index def finish(self): diff --git a/wagtail/search/management/commands/update_index.py b/wagtail/search/management/commands/update_index.py index 32aa9b0932..4b45db1205 100644 --- a/wagtail/search/management/commands/update_index.py +++ b/wagtail/search/management/commands/update_index.py @@ -70,9 +70,9 @@ class Command(BaseCommand): self.write("Backend '%s' doesn't require rebuilding" % backend_name) return - models_grouped_by_index = group_models_by_index( - backend, get_indexed_models() - ).items() + indexed_models = getattr(self, "only_models", get_indexed_models()) + models_grouped_by_index = group_models_by_index(backend, indexed_models).items() + if not models_grouped_by_index: self.write(backend_name + ": No indices to rebuild") @@ -81,7 +81,11 @@ class Command(BaseCommand): # Start rebuild rebuilder = backend.rebuilder_class(index) - index = rebuilder.start() + + if getattr(self, "only_models", None): + index = rebuilder.start(models) + else: + index = rebuilder.start() # Add models for model in models: @@ -138,6 +142,14 @@ class Command(BaseCommand): type=int, help="Set number of records to be fetched at once for inserting into the index", ) + parser.add_argument( + "--only", + action="store", + dest="only_models", + default=None, + type=str, + help="Only update indexes on certain models. (comma separated, '?' to get list of options)", + ) def handle(self, **options): self.verbosity = options["verbosity"] @@ -153,6 +165,19 @@ class Command(BaseCommand): # index the 'default' backend only backend_names = ["default"] + if options["only_models"]: + all_models = {model._meta.label: model for model in get_indexed_models()} + if options["only_models"] == "?": + self.stdout.write(",".join(all_models.keys())) + return + self.only_models = [] + for model_label in options["only_models"].split(","): + model = all_models.get(model_label, None) + if model: + self.only_models.append(model) + else: + self.stderr.write(f"{model_label} is not a valid model name") + return # Update backends for backend_name in backend_names: self.update_backend( From 9374f47d3702de93d1412e22b22d937b7af8be4f Mon Sep 17 00:00:00 2001 From: Neeraj P Yetheendran Date: Fri, 16 Feb 2024 11:19:54 +0530 Subject: [PATCH 2/2] Reformatted mysql.py --- wagtail/search/backends/database/mysql/mysql.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wagtail/search/backends/database/mysql/mysql.py b/wagtail/search/backends/database/mysql/mysql.py index 31371158d9..58e2a48d7b 100644 --- a/wagtail/search/backends/database/mysql/mysql.py +++ b/wagtail/search/backends/database/mysql/mysql.py @@ -495,9 +495,7 @@ class MySQLSearchQueryCompiler(BaseSearchQueryCompiler): ) if not negated: index_entries = index_entries.filter(match_expression) - if ( - self.order_by_relevance - ): # Only applies to the case where the outermost query is not a Not(), because if it is, the relevance score is always 0 (anything that matches is excluded from the results). + if self.order_by_relevance: # Only applies to the case where the outermost query is not a Not(), because if it is, the relevance score is always 0 (anything that matches is excluded from the results). index_entries = index_entries.order_by(score_expression.desc()) else: index_entries = index_entries.exclude(match_expression)