kopia lustrzana https://github.com/bugout-dev/moonstream
Push to AWS S3 bucker support
rodzic
c8f4d4efdc
commit
48a1c8fbed
|
@ -0,0 +1,24 @@
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
|
||||||
|
|
||||||
|
def upload_to_aws_s3_bucket(
|
||||||
|
data: str,
|
||||||
|
bucket: str,
|
||||||
|
key: str,
|
||||||
|
metadata: Dict[str, Any] = {},
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Push data to AWS S3 bucket and return URL to object.
|
||||||
|
"""
|
||||||
|
s3 = boto3.client("s3")
|
||||||
|
s3.put_object(
|
||||||
|
Body=data,
|
||||||
|
Bucket=bucket,
|
||||||
|
Key=key,
|
||||||
|
ContentType="application/json",
|
||||||
|
Metadata=metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
return f"{bucket}/{key}"
|
|
@ -3,6 +3,10 @@ from typing import Any, Dict, Union
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
try:
|
||||||
|
from .aws.bucket import upload_to_aws_s3_bucket
|
||||||
|
except Exception as e:
|
||||||
|
pass
|
||||||
from .data import (
|
from .data import (
|
||||||
APISpec,
|
APISpec,
|
||||||
AuthType,
|
AuthType,
|
||||||
|
@ -68,12 +72,12 @@ class Moonstream:
|
||||||
method.value, url=url, timeout=timeout, **kwargs
|
method.value, url=url, timeout=timeout, **kwargs
|
||||||
)
|
)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
except requests.exceptions.RequestException as err:
|
except requests.exceptions.RequestException as e:
|
||||||
r = err.response
|
r = e.response
|
||||||
if not err.response:
|
if not e.response:
|
||||||
# Connection errors, timeouts, etc...
|
# Connection errors, timeouts, etc...
|
||||||
raise MoonstreamResponseException(
|
raise MoonstreamResponseException(
|
||||||
"Network error", status_code=599, detail=str(err)
|
"Network error", status_code=599, detail=str(e)
|
||||||
)
|
)
|
||||||
if r.headers.get("Content-Type") == "application/json":
|
if r.headers.get("Content-Type") == "application/json":
|
||||||
exception_detail = r.json()["detail"]
|
exception_detail = r.json()["detail"]
|
||||||
|
@ -223,6 +227,23 @@ class Moonstream:
|
||||||
|
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
def upload_query_results(
|
||||||
|
self, data: str, bucket: str, key: str, metadata: Dict[str, Any] = {}
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Uploads data to AWS S3 bucket.
|
||||||
|
|
||||||
|
Requirements: "pip install -e .[aws]" with "boto3" module.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
url = upload_to_aws_s3_bucket(
|
||||||
|
data=data, bucket=bucket, key=key, metadata=metadata
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise Exception(str(e))
|
||||||
|
|
||||||
|
return url
|
||||||
|
|
||||||
def delete_query(
|
def delete_query(
|
||||||
self,
|
self,
|
||||||
token: Union[str, uuid.UUID],
|
token: Union[str, uuid.UUID],
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
[mypy]
|
||||||
|
|
||||||
|
[mypy-boto3.*]
|
||||||
|
ignore_missing_imports = True
|
|
@ -13,6 +13,7 @@ setup(
|
||||||
package_data={"moonstream": ["py.typed"]},
|
package_data={"moonstream": ["py.typed"]},
|
||||||
install_requires=["requests", "pydantic", "dataclasses; python_version=='3.6'"],
|
install_requires=["requests", "pydantic", "dataclasses; python_version=='3.6'"],
|
||||||
extras_require={
|
extras_require={
|
||||||
|
"aws": ["boto3"],
|
||||||
"dev": [
|
"dev": [
|
||||||
"black",
|
"black",
|
||||||
"mypy",
|
"mypy",
|
||||||
|
|
Ładowanie…
Reference in New Issue