mirror
/
wagtail
kopia lustrzana https://github.com/wagtail/wagtail
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Merge branch 'kaedroho-searchchanges/bits-and-pieces-2'

stable/0.4.x
Matt Westcott 2014-07-09 23:29:43 +01:00
rodzic b067fad1f9 9b8ee979d2
commit 2cc8629ab4
9 zmienionych plików z 112 dodań i 87 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

6
wagtail/wagtailadmin/taggable.py
Wyświetl plik

@ -21,7 +21,11 @@ class TagSearchable(indexed.Indexed):
@property
def get_tags(self):
return ' '.join([tag.name for tag in self.tags.all()])
return ' '.join([tag.name for tag in self.prefetched_tags()])
@classmethod
def get_indexed_objects(cls):
return super(TagSearchable, cls).get_indexed_objects().prefetch_related('tagged_items__tag')
@classmethod
def search(cls, q, results_per_page=None, page=1, prefetch_tags=False, filters={}):

7
wagtail/wagtailcore/models.py
Wyświetl plik

@ -235,6 +235,9 @@ class PageManager(models.Manager):
def not_public(self):
return self.get_queryset().not_public()
def search(self, query_string, fields=None, backend='default'):
return self.get_queryset().search(query_string, fields=fields, backend=backend)
class PageBase(models.base.ModelBase):
"""Metaclass for Page"""
@ -289,8 +292,12 @@ class Page(six.with_metaclass(PageBase, MP_Node, ClusterableModel, indexed.Index
search_fields = (
indexed.SearchField('title', partial_match=True, boost=100),
indexed.FilterField('id'),
indexed.FilterField('live'),
indexed.FilterField('owner'),
indexed.FilterField('content_type'),
indexed.FilterField('path'),
indexed.FilterField('depth'),
)
def __init__(self, *args, **kwargs):

6
wagtail/wagtailcore/query.py
Wyświetl plik

@ -2,6 +2,8 @@ from django.db.models import Q
from django.contrib.contenttypes.models import ContentType
from treebeard.mp_tree import MP_NodeQuerySet
from wagtail.wagtailsearch.backends import get_search_backend
class PageQuerySet(MP_NodeQuerySet):
"""
@ -121,3 +123,7 @@ class PageQuerySet(MP_NodeQuerySet):
def not_public(self):
return self.exclude(self.public_q())
def search(self, query_string, fields=None, backend='default'):
search_backend = get_search_backend(backend)
return search_backend.search(query_string, self, fields=None)

38
wagtail/wagtailsearch/backends/base.py
Wyświetl plik

@ -1,6 +1,9 @@
from django.db import models
from django.db.models.query import QuerySet
from django.core.exceptions import ImproperlyConfigured
from wagtail.wagtailsearch.indexed import Indexed
from wagtail.wagtailsearch.utils import normalise_query_string
class BaseSearch(object):
@ -32,5 +35,38 @@ class BaseSearch(object):
def delete(self, obj):
return NotImplemented
def search(self, query_string, model, fields=None, filters={}, prefetch_related=[]):
def _search(self, queryset, query_string, fields=None):
return NotImplemented
def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
# Find model/queryset
if isinstance(model_or_queryset, QuerySet):
model = model_or_queryset.model
queryset = model_or_queryset
else:
model = model_or_queryset
queryset = model_or_queryset.objects.all()
# Model must be a descendant of Indexed and be a django model
if not issubclass(model, Indexed) or not issubclass(model, models.Model):
return []
# Normalise query string
if query_string is not None:
query_string = normalise_query_string(query_string)
# Check that theres still a query string after the clean up
if query_string == "":
return []
# Apply filters to queryset
if filters:
queryset = queryset.filter(**filters)
# Prefetch related
if prefetch_related:
for prefetch in prefetch_related:
queryset = queryset.prefetch_related(prefetch)
# Search
return self._search(queryset, query_string, fields=fields)

34
wagtail/wagtailsearch/backends/db.py
Wyświetl plik

@ -2,7 +2,6 @@ from django.db import models
from wagtail.wagtailsearch.backends.base import BaseSearch
from wagtail.wagtailsearch.indexed import Indexed
from wagtail.wagtailsearch.utils import normalise_query_string
class DBSearch(BaseSearch):
@ -27,26 +26,16 @@ class DBSearch(BaseSearch):
def delete(self, obj):
pass # Not needed
def search(self, query_string, model, fields=None, filters=None, prefetch_related=None):
# Get fields
if fields is None:
fields = [field.field_name for field in model.get_searchable_search_fields()]
# Start with all objects
query = model.objects.all()
# Apply filters
if filters:
query = query.filter(**filters)
def _search(self, queryset, query_string, fields=None):
if query_string is not None:
# Normalise query string
query_string = normalise_query_string(query_string)
# Get fields
if fields is None:
fields = [field.field_name for field in queryset.model.get_searchable_search_fields()]
# Get terms
terms = query_string.split()
if not terms:
return model.objects.none()
return queryset.model.objects.none()
# Filter by terms
for term in terms:
@ -54,21 +43,16 @@ class DBSearch(BaseSearch):
for field_name in fields:
# Check if the field exists (this will filter out indexed callables)
try:
model._meta.get_field_by_name(field_name)
queryset.model._meta.get_field_by_name(field_name)
except:
continue
# Filter on this field
term_query |= models.Q(**{'%s__icontains' % field_name: term})
query = query.filter(term_query)
queryset = queryset.filter(term_query)
# Distinct
query = query.distinct()
queryset = queryset.distinct()
# Prefetch related
if prefetch_related:
for prefetch in prefetch_related:
query = query.prefetch_related(prefetch)
return query
return queryset

70
wagtail/wagtailsearch/backends/elasticsearch.py
Wyświetl plik

@ -3,14 +3,13 @@ from __future__ import absolute_import
import json
from django.db import models
from django.db.models.query import QuerySet
from django.db.models.sql.where import SubqueryConstraint
from elasticsearch import Elasticsearch, NotFoundError, RequestError
from elasticsearch.helpers import bulk
from wagtail.wagtailsearch.backends.base import BaseSearch
from wagtail.wagtailsearch.indexed import Indexed, SearchField, FilterField
from wagtail.wagtailsearch.utils import normalise_query_string
class ElasticSearchMapping(object):
@ -122,7 +121,7 @@ class ElasticSearchQuery(object):
def __init__(self, queryset, query_string, fields=None):
self.queryset = queryset
self.query_string = query_string
self.fields = fields or ['_all', '_partials']
self.fields = fields
def _get_filters_from_where(self, where_node):
# Check if this is a leaf node
@ -203,7 +202,16 @@ class ElasticSearchQuery(object):
}
}
raise FilterError('Could not apply filter on ElasticSearch results "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.')
if lookup == 'in':
return {
'terms': {
field_index_name: value,
}
}
raise FilterError('Could not apply filter on ElasticSearch results: "' + field_name + '__' + lookup + ' = ' + unicode(value) + '". Lookup "' + lookup + '"" not recognosed.')
elif isinstance(where_node, SubqueryConstraint):
raise FilterError('Could not apply filter on ElasticSearch results: Subqueries are not allowed.')
# Get child filters
connector = where_node.connector
@ -249,15 +257,21 @@ class ElasticSearchQuery(object):
def to_es(self):
# Query
if self.query_string is not None:
query = {
'query_string': {
'query': self.query_string,
}
}
fields = self.fields or ['_all', '_partials']
# Fields
if self.fields:
query['query_string']['fields'] = self.fields
if len(fields) == 1:
query = {
'match': {
fields[0]: self.query_string,
}
}
else:
query = {
'multi_match': {
'query': self.query_string,
'fields': fields,
}
}
else:
query = {
'match_all': {}
@ -576,35 +590,5 @@ class ElasticSearch(BaseSearch):
except NotFoundError:
pass # Document doesn't exist, ignore this exception
def search(self, query_string, model_or_queryset, fields=None, filters=None, prefetch_related=None):
# Find model/queryset
if isinstance(model_or_queryset, QuerySet):
model = model_or_queryset.model
queryset = model_or_queryset
else:
model = model_or_queryset
queryset = model_or_queryset.objects.all()
# Model must be a descendant of Indexed and be a django model
if not issubclass(model, Indexed) or not issubclass(model, models.Model):
return []
# Normalise query string
if query_string is not None:
query_string = normalise_query_string(query_string)
# Check that theres still a query string after the clean up
if query_string == "":
return []
# Apply filters to queryset
if filters:
queryset = queryset.filter(**filters)
# Prefetch related
if prefetch_related:
for prefetch in prefetch_related:
queryset = queryset.prefetch_related(prefetch)
# Return search results
def _search(self, queryset, query_string, fields=None):
return ElasticSearchResults(self, ElasticSearchQuery(queryset, query_string, fields=fields))

4
wagtail/wagtailsearch/indexed.py
Wyświetl plik

@ -118,6 +118,10 @@ class Indexed(object):
def get_filterable_search_fields(cls):
return filter(lambda field: isinstance(field, FilterField), cls.get_search_fields())
@classmethod
def get_indexed_objects(cls):
return cls.objects.all()
indexed_fields = ()

2
wagtail/wagtailsearch/management/commands/update_index.py
Wyświetl plik

@ -24,7 +24,7 @@ class Command(BaseCommand):
toplevel_content_type = model.indexed_get_toplevel_content_type()
# Loop through objects
for obj in model.objects.all():
for obj in model.get_indexed_objects():
# Get key for this object
key = toplevel_content_type + ':' + str(obj.pk)

32
wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
Wyświetl plik

@ -88,7 +88,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_none_query_string(self):
@ -104,7 +104,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test"), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_and_filter(self):
@ -112,7 +112,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title="Test", live=True), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'and': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
# Make sure field filters are sorted (as they can be in any order which may cause false positives)
query = query.to_es()
@ -131,7 +131,7 @@ class TestElasticSearchQuery(TestCase):
field_filters[:] = sorted(field_filters, key=lambda f: list(f['term'].keys())[0])
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'or': [{'term': {'live_filter': True}}, {'term': {'title_filter': 'Test'}}]}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query, expected_result)
def test_negated_filter(self):
@ -139,7 +139,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.exclude(live=True), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'term': {'live_filter': True}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_fields(self):
@ -147,7 +147,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.all(), "Hello", fields=['title'])
# Check it
expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'query_string': {'query': 'Hello', 'fields': ['title']}}}}
expected_result = {'filtered': {'filter': {'prefix': {'content_type': 'tests_searchtest'}}, 'query': {'match': {'title': 'Hello'}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_exact_lookup(self):
@ -155,7 +155,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__exact="Test"), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'term': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_none_lookup(self):
@ -163,7 +163,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title=None), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_isnull_true_lookup(self):
@ -171,7 +171,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=True), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'missing': {'field': 'title_filter'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_isnull_false_lookup(self):
@ -179,7 +179,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__isnull=False), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'not': {'missing': {'field': 'title_filter'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_startswith_lookup(self):
@ -187,7 +187,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(title__startswith="Test"), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'prefix': {'title_filter': 'Test'}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_gt_lookup(self):
@ -197,7 +197,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gt=datetime.datetime(2014, 4, 29)), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_lt_lookup(self):
@ -205,7 +205,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lt=datetime.datetime(2014, 4, 29)), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lt': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_gte_lookup(self):
@ -213,7 +213,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__gte=datetime.datetime(2014, 4, 29)), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_lte_lookup(self):
@ -221,7 +221,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__lte=datetime.datetime(2014, 4, 29)), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'lte': '2014-04-29'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)
def test_range_lookup(self):
@ -232,7 +232,7 @@ class TestElasticSearchQuery(TestCase):
query = self.ElasticSearchQuery(models.SearchTest.objects.filter(published_date__range=(start_date, end_date)), "Hello")
# Check it
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'query_string': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
expected_result = {'filtered': {'filter': {'and': [{'prefix': {'content_type': 'tests_searchtest'}}, {'range': {'published_date_filter': {'gte': '2014-04-29', 'lte': '2014-08-19'}}}]}, 'query': {'multi_match': {'query': 'Hello', 'fields': ['_all', '_partials']}}}}
self.assertDictEqual(query.to_es(), expected_result)