Move hash generation to actions.

crawlers/mooncrawl/mooncrawl/actions.py

@@ -1,3 +1,6 @@
+from collections import OrderedDict
+import hashlib
+import json
 import logging
 from typing import Any, Dict
 
@@ -41,3 +44,13 @@ def generate_s3_access_links(
     )
 
     return stats_presigned_url
+
+
+def query_parameter_hash(params: Dict[str, Any]) -> str:
+    """
+    Generate a hash of the query parameters
+    """
+
+    hash = hashlib.md5(json.dumps(OrderedDict(params)).encode("utf-8")).hexdigest()
+
+    return hash

crawlers/mooncrawl/mooncrawl/api.py

@@ -5,9 +5,6 @@ import logging
 import time
 from cgi import test
 from datetime import timedelta
-from collections import OrderedDict
-import hashlib
-import json
 from typing import Any, Dict, List
 from uuid import UUID
 
@@ -18,7 +15,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from sqlalchemy import text
 from sqlalchemy.sql.elements import TextClause
 
-from .actions import generate_s3_access_links
+from .actions import generate_s3_access_links, query_parameter_hash
 from . import data
 from .middleware import MoonstreamHTTPException
 from .settings import (
@@ -227,9 +224,7 @@ async def queries_data_update_handler(
             status_code=500, detail="Unmatched amount of applying query parameters"
         )
 
-    params_hash = hashlib.md5(
-        json.dumps(OrderedDict(passed_params)).encode("utf-8")
-    ).hexdigest()
+    params_hash = query_parameter_hash(passed_params)
 
     bucket = MOONSTREAM_S3_QUERIES_BUCKET
     key = f"{MOONSTREAM_S3_QUERIES_BUCKET_PREFIX}/queries/{query_id}/{params_hash}/data.{request_data.file_type}"

crawlers/mooncrawl/mooncrawl/stats_worker/queries.py

@@ -46,10 +46,8 @@ def query_validation(query: str) -> str:
 
 def to_json_types(value):
 
-    if isinstance(value, (str, int, tuple, dict)):
+    if isinstance(value, (str, int, tuple, dict, list)):
         return value
-    if isinstance(value, list):  # psycopg2 issue with list support
-        return tuple(value)
     elif isinstance(value, set):
         return tuple(value)
     else: