[add] file storage service
This commit is contained in:
Mark
@@ -76,6 +76,22 @@ class Settings:
|
||||
# File Upload
|
||||
MAX_FILE_SIZE: int = int(os.getenv("MAX_FILE_SIZE", "52428800"))
|
||||
FILE_PATH: str = os.getenv("FILE_PATH", "/files")
|
||||
FILE_URL_EXPIRES: int = int(os.getenv("FILE_URL_EXPIRES", "3600"))
|
||||
|
||||
# Storage Configuration
|
||||
STORAGE_TYPE: str = os.getenv("STORAGE_TYPE", "local")
|
||||
|
||||
# Aliyun OSS Configuration
|
||||
OSS_ENDPOINT: str = os.getenv("OSS_ENDPOINT", "")
|
||||
OSS_ACCESS_KEY_ID: str = os.getenv("OSS_ACCESS_KEY_ID", "")
|
||||
OSS_ACCESS_KEY_SECRET: str = os.getenv("OSS_ACCESS_KEY_SECRET", "")
|
||||
OSS_BUCKET_NAME: str = os.getenv("OSS_BUCKET_NAME", "")
|
||||
|
||||
# AWS S3 Configuration
|
||||
S3_REGION: str = os.getenv("S3_REGION", "")
|
||||
S3_ACCESS_KEY_ID: str = os.getenv("S3_ACCESS_KEY_ID", "")
|
||||
S3_SECRET_ACCESS_KEY: str = os.getenv("S3_SECRET_ACCESS_KEY", "")
|
||||
S3_BUCKET_NAME: str = os.getenv("S3_BUCKET_NAME", "")
|
||||
|
||||
# VOLC ASR settings
|
||||
VOLC_APP_KEY: str = os.getenv("VOLC_APP_KEY", "")
|
||||
|
||||
15
api/app/core/storage/__init__.py
Normal file
15
api/app/core/storage/__init__.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""Storage backend module."""
|
||||
|
||||
from app.core.storage.base import StorageBackend
|
||||
from app.core.storage.factory import StorageFactory
|
||||
from app.core.storage.local import LocalStorage
|
||||
from app.core.storage.oss import OSSStorage
|
||||
from app.core.storage.s3 import S3Storage
|
||||
|
||||
__all__ = [
|
||||
"StorageBackend",
|
||||
"LocalStorage",
|
||||
"OSSStorage",
|
||||
"S3Storage",
|
||||
"StorageFactory",
|
||||
]
|
||||
103
api/app/core/storage/base.py
Normal file
103
api/app/core/storage/base.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
Abstract base class for storage backends.
|
||||
|
||||
This module defines the StorageBackend abstract class that all storage
|
||||
implementations must inherit from. It provides a unified interface for
|
||||
file operations across different storage backends.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
"""
|
||||
Abstract base class for storage backends.
|
||||
|
||||
All storage implementations (local, OSS, S3, etc.) must inherit from this
|
||||
class and implement all abstract methods. All methods are async to support
|
||||
non-blocking I/O operations.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def upload(
|
||||
self,
|
||||
file_key: str,
|
||||
content: bytes,
|
||||
content_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to the storage backend.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
content: File content as bytes.
|
||||
content_type: Optional MIME type of the file.
|
||||
|
||||
Returns:
|
||||
The storage path or URL of the uploaded file.
|
||||
|
||||
Raises:
|
||||
StorageUploadError: If the upload operation fails.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def download(self, file_key: str) -> bytes:
|
||||
"""
|
||||
Download a file from the storage backend.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
File content as bytes.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file does not exist.
|
||||
StorageDownloadError: If the download operation fails.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, file_key: str) -> bool:
|
||||
"""
|
||||
Delete a file from the storage backend.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file was deleted successfully, False if file didn't exist.
|
||||
|
||||
Raises:
|
||||
StorageDeleteError: If the delete operation fails.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def exists(self, file_key: str) -> bool:
|
||||
"""
|
||||
Check if a file exists in the storage backend.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file exists, False otherwise.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_url(self, file_key: str, expires: int = 3600) -> str:
|
||||
"""
|
||||
Get an access URL for the file.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
expires: URL validity period in seconds (default: 1 hour).
|
||||
|
||||
Returns:
|
||||
URL for accessing the file.
|
||||
"""
|
||||
pass
|
||||
103
api/app/core/storage/factory.py
Normal file
103
api/app/core/storage/factory.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
Storage backend factory module.
|
||||
|
||||
This module provides a factory class for creating storage backend instances
|
||||
based on configuration. It implements the singleton pattern to ensure only
|
||||
one storage backend instance exists throughout the application lifecycle.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.storage.base import StorageBackend
|
||||
from app.core.storage_exceptions import StorageConfigError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StorageFactory:
|
||||
"""
|
||||
Factory class for creating storage backend instances.
|
||||
|
||||
This class implements the singleton pattern to ensure only one storage
|
||||
backend instance is created and reused throughout the application.
|
||||
|
||||
Attributes:
|
||||
_instance: The singleton storage backend instance.
|
||||
"""
|
||||
|
||||
_instance: Optional[StorageBackend] = None
|
||||
|
||||
@classmethod
|
||||
def get_storage(cls) -> StorageBackend:
|
||||
"""
|
||||
Get the storage backend instance (singleton).
|
||||
|
||||
Returns:
|
||||
The storage backend instance.
|
||||
|
||||
Raises:
|
||||
ValueError: If the configured storage type is not supported.
|
||||
StorageConfigError: If required configuration is missing.
|
||||
"""
|
||||
if cls._instance is None:
|
||||
cls._instance = cls._create_storage()
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def _create_storage(cls) -> StorageBackend:
|
||||
"""
|
||||
Create a storage backend instance based on configuration.
|
||||
|
||||
Returns:
|
||||
A new storage backend instance.
|
||||
|
||||
Raises:
|
||||
ValueError: If the configured storage type is not supported.
|
||||
StorageConfigError: If required configuration is missing.
|
||||
"""
|
||||
storage_type = settings.STORAGE_TYPE.lower()
|
||||
logger.info(f"Creating storage backend of type: {storage_type}")
|
||||
|
||||
if storage_type == "local":
|
||||
from app.core.storage.local import LocalStorage
|
||||
|
||||
base_path = Path("storage") / settings.FILE_PATH.lstrip("/")
|
||||
return LocalStorage(base_path=str(base_path))
|
||||
|
||||
elif storage_type == "oss":
|
||||
from app.core.storage.oss import OSSStorage
|
||||
|
||||
return OSSStorage(
|
||||
endpoint=settings.OSS_ENDPOINT,
|
||||
access_key_id=settings.OSS_ACCESS_KEY_ID,
|
||||
access_key_secret=settings.OSS_ACCESS_KEY_SECRET,
|
||||
bucket_name=settings.OSS_BUCKET_NAME,
|
||||
)
|
||||
|
||||
elif storage_type == "s3":
|
||||
from app.core.storage.s3 import S3Storage
|
||||
|
||||
return S3Storage(
|
||||
region=settings.S3_REGION,
|
||||
access_key_id=settings.S3_ACCESS_KEY_ID,
|
||||
secret_access_key=settings.S3_SECRET_ACCESS_KEY,
|
||||
bucket_name=settings.S3_BUCKET_NAME,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.error(f"Unsupported storage type: {storage_type}")
|
||||
raise ValueError(f"Unsupported storage type: {storage_type}")
|
||||
|
||||
@classmethod
|
||||
def reset(cls) -> None:
|
||||
"""
|
||||
Reset the singleton instance.
|
||||
|
||||
This method is primarily used for testing purposes to allow
|
||||
creating new storage instances with different configurations.
|
||||
"""
|
||||
cls._instance = None
|
||||
logger.debug("StorageFactory singleton instance reset")
|
||||
196
api/app/core/storage/local.py
Normal file
196
api/app/core/storage/local.py
Normal file
@@ -0,0 +1,196 @@
|
||||
"""
|
||||
Local file system storage backend implementation.
|
||||
|
||||
This module provides a storage backend that stores files on the local
|
||||
file system using async I/O operations via aiofiles.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import aiofiles
|
||||
import aiofiles.os
|
||||
|
||||
from app.core.storage.base import StorageBackend
|
||||
from app.core.storage_exceptions import (
|
||||
StorageDeleteError,
|
||||
StorageDownloadError,
|
||||
StorageUploadError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LocalStorage(StorageBackend):
|
||||
"""
|
||||
Local file system storage implementation.
|
||||
|
||||
This class implements the StorageBackend interface for storing files
|
||||
on the local file system. It uses aiofiles for async file operations.
|
||||
|
||||
Attributes:
|
||||
base_path: The base directory path where files will be stored.
|
||||
"""
|
||||
|
||||
def __init__(self, base_path: str):
|
||||
"""
|
||||
Initialize the LocalStorage backend.
|
||||
|
||||
Args:
|
||||
base_path: The base directory path for file storage.
|
||||
Will be created if it doesn't exist.
|
||||
"""
|
||||
self.base_path = Path(base_path)
|
||||
self.base_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"LocalStorage initialized with base_path: {self.base_path}")
|
||||
|
||||
def _get_full_path(self, file_key: str) -> Path:
|
||||
"""
|
||||
Get the full file system path for a given file key.
|
||||
|
||||
Args:
|
||||
file_key: The unique identifier for the file.
|
||||
|
||||
Returns:
|
||||
The full Path object for the file.
|
||||
"""
|
||||
return self.base_path / file_key
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
file_key: str,
|
||||
content: bytes,
|
||||
content_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to the local file system.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
content: File content as bytes.
|
||||
content_type: Optional MIME type (not used for local storage).
|
||||
|
||||
Returns:
|
||||
The full path of the uploaded file as a string.
|
||||
|
||||
Raises:
|
||||
StorageUploadError: If the upload operation fails.
|
||||
"""
|
||||
full_path = self._get_full_path(file_key)
|
||||
|
||||
try:
|
||||
# Create parent directories if they don't exist
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async with aiofiles.open(full_path, "wb") as f:
|
||||
await f.write(content)
|
||||
|
||||
logger.info(f"File uploaded successfully: {file_key}")
|
||||
return str(full_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file {file_key}: {e}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def download(self, file_key: str) -> bytes:
|
||||
"""
|
||||
Download a file from the local file system.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
File content as bytes.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file does not exist.
|
||||
StorageDownloadError: If the download operation fails.
|
||||
"""
|
||||
full_path = self._get_full_path(file_key)
|
||||
|
||||
if not full_path.exists():
|
||||
logger.warning(f"File not found: {file_key}")
|
||||
raise FileNotFoundError(f"File not found: {file_key}")
|
||||
|
||||
try:
|
||||
async with aiofiles.open(full_path, "rb") as f:
|
||||
content = await f.read()
|
||||
|
||||
logger.info(f"File downloaded successfully: {file_key}")
|
||||
return content
|
||||
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download file {file_key}: {e}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def delete(self, file_key: str) -> bool:
|
||||
"""
|
||||
Delete a file from the local file system.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file was deleted, False if it didn't exist.
|
||||
|
||||
Raises:
|
||||
StorageDeleteError: If the delete operation fails.
|
||||
"""
|
||||
full_path = self._get_full_path(file_key)
|
||||
|
||||
if not full_path.exists():
|
||||
logger.info(f"File does not exist, nothing to delete: {file_key}")
|
||||
return False
|
||||
|
||||
try:
|
||||
await aiofiles.os.remove(full_path)
|
||||
logger.info(f"File deleted successfully: {file_key}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file {file_key}: {e}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def exists(self, file_key: str) -> bool:
|
||||
"""
|
||||
Check if a file exists in the local file system.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file exists, False otherwise.
|
||||
"""
|
||||
full_path = self._get_full_path(file_key)
|
||||
return full_path.exists()
|
||||
|
||||
async def get_url(self, file_key: str, expires: int = 3600) -> str:
|
||||
"""
|
||||
Get an access URL for the file.
|
||||
|
||||
For local storage, this returns a relative path that can be used
|
||||
by the API layer to serve the file.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
expires: URL validity period in seconds (not used for local storage).
|
||||
|
||||
Returns:
|
||||
A relative URL path for accessing the file.
|
||||
"""
|
||||
return f"/files/{file_key}"
|
||||
233
api/app/core/storage/oss.py
Normal file
233
api/app/core/storage/oss.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""
|
||||
Aliyun OSS storage backend implementation.
|
||||
|
||||
This module provides a storage backend that stores files on Aliyun Object
|
||||
Storage Service (OSS) using the oss2 SDK.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import oss2
|
||||
from oss2.exceptions import NoSuchKey, OssError
|
||||
|
||||
from app.core.storage.base import StorageBackend
|
||||
from app.core.storage_exceptions import (
|
||||
StorageConfigError,
|
||||
StorageConnectionError,
|
||||
StorageDeleteError,
|
||||
StorageDownloadError,
|
||||
StorageUploadError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OSSStorage(StorageBackend):
|
||||
"""
|
||||
Aliyun OSS storage implementation.
|
||||
|
||||
This class implements the StorageBackend interface for storing files
|
||||
on Aliyun Object Storage Service (OSS).
|
||||
|
||||
Attributes:
|
||||
bucket: The oss2.Bucket instance for OSS operations.
|
||||
bucket_name: The name of the OSS bucket.
|
||||
endpoint: The OSS endpoint URL.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
access_key_id: str,
|
||||
access_key_secret: str,
|
||||
bucket_name: str,
|
||||
):
|
||||
"""
|
||||
Initialize the OSSStorage backend.
|
||||
|
||||
Args:
|
||||
endpoint: The OSS endpoint URL (e.g., 'https://oss-cn-hangzhou.aliyuncs.com').
|
||||
access_key_id: The Aliyun access key ID.
|
||||
access_key_secret: The Aliyun access key secret.
|
||||
bucket_name: The name of the OSS bucket.
|
||||
|
||||
Raises:
|
||||
StorageConfigError: If any required configuration is missing.
|
||||
"""
|
||||
# Validate required configuration
|
||||
if not endpoint:
|
||||
raise StorageConfigError(message="OSS endpoint is required")
|
||||
if not access_key_id:
|
||||
raise StorageConfigError(message="OSS access_key_id is required")
|
||||
if not access_key_secret:
|
||||
raise StorageConfigError(message="OSS access_key_secret is required")
|
||||
if not bucket_name:
|
||||
raise StorageConfigError(message="OSS bucket_name is required")
|
||||
|
||||
self.endpoint = endpoint
|
||||
self.bucket_name = bucket_name
|
||||
|
||||
try:
|
||||
auth = oss2.Auth(access_key_id, access_key_secret)
|
||||
self.bucket = oss2.Bucket(auth, endpoint, bucket_name)
|
||||
logger.info(
|
||||
f"OSSStorage initialized with endpoint: {endpoint}, bucket: {bucket_name}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize OSS client: {e}")
|
||||
raise StorageConnectionError(
|
||||
message=f"Failed to initialize OSS client: {e}",
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
file_key: str,
|
||||
content: bytes,
|
||||
content_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to OSS.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
content: File content as bytes.
|
||||
content_type: Optional MIME type of the file.
|
||||
|
||||
Returns:
|
||||
The file key of the uploaded file.
|
||||
|
||||
Raises:
|
||||
StorageUploadError: If the upload operation fails.
|
||||
"""
|
||||
try:
|
||||
headers = {}
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
|
||||
self.bucket.put_object(file_key, content, headers=headers if headers else None)
|
||||
logger.info(f"File uploaded to OSS successfully: {file_key}")
|
||||
return file_key
|
||||
|
||||
except OssError as e:
|
||||
logger.error(f"OSS error uploading file {file_key}: {e}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file to OSS: {e.message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to OSS {file_key}: {e}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file to OSS: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def download(self, file_key: str) -> bytes:
|
||||
"""
|
||||
Download a file from OSS.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
File content as bytes.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file does not exist.
|
||||
StorageDownloadError: If the download operation fails.
|
||||
"""
|
||||
try:
|
||||
result = self.bucket.get_object(file_key)
|
||||
content = result.read()
|
||||
logger.info(f"File downloaded from OSS successfully: {file_key}")
|
||||
return content
|
||||
|
||||
except NoSuchKey:
|
||||
logger.warning(f"File not found in OSS: {file_key}")
|
||||
raise FileNotFoundError(f"File not found: {file_key}")
|
||||
except OssError as e:
|
||||
logger.error(f"OSS error downloading file {file_key}: {e}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file from OSS: {e.message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download file from OSS {file_key}: {e}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file from OSS: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def delete(self, file_key: str) -> bool:
|
||||
"""
|
||||
Delete a file from OSS.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file was deleted successfully.
|
||||
|
||||
Raises:
|
||||
StorageDeleteError: If the delete operation fails.
|
||||
"""
|
||||
try:
|
||||
self.bucket.delete_object(file_key)
|
||||
logger.info(f"File deleted from OSS successfully: {file_key}")
|
||||
return True
|
||||
|
||||
except OssError as e:
|
||||
logger.error(f"OSS error deleting file {file_key}: {e}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file from OSS: {e.message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file from OSS {file_key}: {e}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file from OSS: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def exists(self, file_key: str) -> bool:
|
||||
"""
|
||||
Check if a file exists in OSS.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file exists, False otherwise.
|
||||
"""
|
||||
try:
|
||||
return self.bucket.object_exists(file_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check file existence in OSS {file_key}: {e}")
|
||||
return False
|
||||
|
||||
async def get_url(self, file_key: str, expires: int = 3600) -> str:
|
||||
"""
|
||||
Get a presigned URL for accessing the file.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
expires: URL validity period in seconds (default: 1 hour).
|
||||
|
||||
Returns:
|
||||
A presigned URL for accessing the file.
|
||||
"""
|
||||
try:
|
||||
url = self.bucket.sign_url("GET", file_key, expires)
|
||||
logger.debug(f"Generated presigned URL for {file_key}, expires in {expires}s")
|
||||
return url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate presigned URL for {file_key}: {e}")
|
||||
# Return a basic URL format as fallback
|
||||
return f"https://{self.bucket_name}.{self.endpoint.replace('https://', '').replace('http://', '')}/{file_key}"
|
||||
299
api/app/core/storage/s3.py
Normal file
299
api/app/core/storage/s3.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
AWS S3 storage backend implementation.
|
||||
|
||||
This module provides a storage backend that stores files on AWS S3
|
||||
using the boto3 SDK.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, NoCredentialsError, BotoCoreError
|
||||
|
||||
from app.core.storage.base import StorageBackend
|
||||
from app.core.storage_exceptions import (
|
||||
StorageConfigError,
|
||||
StorageConnectionError,
|
||||
StorageDeleteError,
|
||||
StorageDownloadError,
|
||||
StorageUploadError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3Storage(StorageBackend):
|
||||
"""
|
||||
AWS S3 storage implementation.
|
||||
|
||||
This class implements the StorageBackend interface for storing files
|
||||
on AWS Simple Storage Service (S3).
|
||||
|
||||
Attributes:
|
||||
client: The boto3 S3 client instance.
|
||||
bucket_name: The name of the S3 bucket.
|
||||
region: The AWS region.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
region: str,
|
||||
access_key_id: str,
|
||||
secret_access_key: str,
|
||||
bucket_name: str,
|
||||
):
|
||||
"""
|
||||
Initialize the S3Storage backend.
|
||||
|
||||
Args:
|
||||
region: The AWS region (e.g., 'us-east-1').
|
||||
access_key_id: The AWS access key ID.
|
||||
secret_access_key: The AWS secret access key.
|
||||
bucket_name: The name of the S3 bucket.
|
||||
|
||||
Raises:
|
||||
StorageConfigError: If any required configuration is missing.
|
||||
StorageConnectionError: If connection to S3 fails.
|
||||
"""
|
||||
# Validate required configuration
|
||||
if not region:
|
||||
raise StorageConfigError(message="S3 region is required")
|
||||
if not access_key_id:
|
||||
raise StorageConfigError(message="S3 access_key_id is required")
|
||||
if not secret_access_key:
|
||||
raise StorageConfigError(message="S3 secret_access_key is required")
|
||||
if not bucket_name:
|
||||
raise StorageConfigError(message="S3 bucket_name is required")
|
||||
|
||||
self.region = region
|
||||
self.bucket_name = bucket_name
|
||||
|
||||
try:
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
region_name=region,
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
)
|
||||
logger.info(
|
||||
f"S3Storage initialized with region: {region}, bucket: {bucket_name}"
|
||||
)
|
||||
except NoCredentialsError as e:
|
||||
logger.error(f"Invalid AWS credentials: {e}")
|
||||
raise StorageConfigError(
|
||||
message=f"Invalid AWS credentials: {e}",
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize S3 client: {e}")
|
||||
raise StorageConnectionError(
|
||||
message=f"Failed to initialize S3 client: {e}",
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
file_key: str,
|
||||
content: bytes,
|
||||
content_type: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to S3.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
content: File content as bytes.
|
||||
content_type: Optional MIME type of the file.
|
||||
|
||||
Returns:
|
||||
The file key of the uploaded file.
|
||||
|
||||
Raises:
|
||||
StorageUploadError: If the upload operation fails.
|
||||
"""
|
||||
try:
|
||||
extra_args = {}
|
||||
if content_type:
|
||||
extra_args["ContentType"] = content_type
|
||||
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=file_key,
|
||||
Body=content,
|
||||
**extra_args,
|
||||
)
|
||||
logger.info(f"File uploaded to S3 successfully: {file_key}")
|
||||
return file_key
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code", "Unknown")
|
||||
error_message = e.response.get("Error", {}).get("Message", str(e))
|
||||
logger.error(f"S3 ClientError uploading file {file_key}: {error_message}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file to S3 ({error_code}): {error_message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except BotoCoreError as e:
|
||||
logger.error(f"S3 BotoCoreError uploading file {file_key}: {e}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file to S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to S3 {file_key}: {e}")
|
||||
raise StorageUploadError(
|
||||
message=f"Failed to upload file to S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def download(self, file_key: str) -> bytes:
|
||||
"""
|
||||
Download a file from S3.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
File content as bytes.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the file does not exist.
|
||||
StorageDownloadError: If the download operation fails.
|
||||
"""
|
||||
try:
|
||||
response = self.client.get_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=file_key,
|
||||
)
|
||||
content = response["Body"].read()
|
||||
logger.info(f"File downloaded from S3 successfully: {file_key}")
|
||||
return content
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code", "Unknown")
|
||||
if error_code in ("NoSuchKey", "404"):
|
||||
logger.warning(f"File not found in S3: {file_key}")
|
||||
raise FileNotFoundError(f"File not found: {file_key}")
|
||||
error_message = e.response.get("Error", {}).get("Message", str(e))
|
||||
logger.error(f"S3 ClientError downloading file {file_key}: {error_message}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file from S3 ({error_code}): {error_message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except BotoCoreError as e:
|
||||
logger.error(f"S3 BotoCoreError downloading file {file_key}: {e}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file from S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download file from S3 {file_key}: {e}")
|
||||
raise StorageDownloadError(
|
||||
message=f"Failed to download file from S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def delete(self, file_key: str) -> bool:
|
||||
"""
|
||||
Delete a file from S3.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file was deleted successfully.
|
||||
|
||||
Raises:
|
||||
StorageDeleteError: If the delete operation fails.
|
||||
"""
|
||||
try:
|
||||
self.client.delete_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=file_key,
|
||||
)
|
||||
logger.info(f"File deleted from S3 successfully: {file_key}")
|
||||
return True
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code", "Unknown")
|
||||
error_message = e.response.get("Error", {}).get("Message", str(e))
|
||||
logger.error(f"S3 ClientError deleting file {file_key}: {error_message}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file from S3 ({error_code}): {error_message}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except BotoCoreError as e:
|
||||
logger.error(f"S3 BotoCoreError deleting file {file_key}: {e}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file from S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file from S3 {file_key}: {e}")
|
||||
raise StorageDeleteError(
|
||||
message=f"Failed to delete file from S3: {e}",
|
||||
file_key=file_key,
|
||||
cause=e,
|
||||
)
|
||||
|
||||
async def exists(self, file_key: str) -> bool:
|
||||
"""
|
||||
Check if a file exists in S3.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
|
||||
Returns:
|
||||
True if the file exists, False otherwise.
|
||||
"""
|
||||
try:
|
||||
self.client.head_object(
|
||||
Bucket=self.bucket_name,
|
||||
Key=file_key,
|
||||
)
|
||||
return True
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code", "Unknown")
|
||||
if error_code in ("404", "NoSuchKey"):
|
||||
return False
|
||||
logger.error(f"Failed to check file existence in S3 {file_key}: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check file existence in S3 {file_key}: {e}")
|
||||
return False
|
||||
|
||||
async def get_url(self, file_key: str, expires: int = 3600) -> str:
|
||||
"""
|
||||
Get a presigned URL for accessing the file.
|
||||
|
||||
Args:
|
||||
file_key: Unique identifier for the file in the storage system.
|
||||
expires: URL validity period in seconds (default: 1 hour).
|
||||
|
||||
Returns:
|
||||
A presigned URL for accessing the file.
|
||||
"""
|
||||
try:
|
||||
url = self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": self.bucket_name,
|
||||
"Key": file_key,
|
||||
},
|
||||
ExpiresIn=expires,
|
||||
)
|
||||
logger.debug(f"Generated presigned URL for {file_key}, expires in {expires}s")
|
||||
return url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate presigned URL for {file_key}: {e}")
|
||||
# Return a basic URL format as fallback
|
||||
return f"https://{self.bucket_name}.s3.{self.region}.amazonaws.com/{file_key}"
|
||||
101
api/app/core/storage/url_signer.py
Normal file
101
api/app/core/storage/url_signer.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
URL signing utilities for local file storage.
|
||||
|
||||
This module provides functions to generate and verify signed URLs
|
||||
with expiration time for local file access.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
from typing import Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
def generate_signed_url(
|
||||
file_id: str,
|
||||
expires: int,
|
||||
base_url: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a signed URL for local file access.
|
||||
|
||||
Args:
|
||||
file_id: The file UUID as string.
|
||||
expires: URL validity period in seconds.
|
||||
base_url: Base URL prefix (default: http://localhost:8000/api).
|
||||
|
||||
Returns:
|
||||
A signed URL with expiration timestamp and signature.
|
||||
|
||||
Example:
|
||||
>>> generate_signed_url("abc-123", 3600)
|
||||
'http://localhost:8000/api/storage/public/abc-123?expires=1234567890&signature=xxx'
|
||||
"""
|
||||
if base_url is None:
|
||||
# Use SERVER_IP or default to localhost
|
||||
server_url = f"http://{settings.SERVER_IP}:8000/api"
|
||||
base_url = server_url
|
||||
|
||||
# Calculate expiration timestamp
|
||||
expires_at = int(time.time()) + expires
|
||||
|
||||
# Generate signature
|
||||
signature = _generate_signature(file_id, expires_at)
|
||||
|
||||
# Build URL with query parameters
|
||||
params = urlencode({
|
||||
"expires": expires_at,
|
||||
"signature": signature,
|
||||
})
|
||||
|
||||
return f"{base_url}/storage/public/{file_id}?{params}"
|
||||
|
||||
|
||||
def verify_signed_url(
|
||||
file_id: str,
|
||||
expires_at: int,
|
||||
signature: str,
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Verify a signed URL.
|
||||
|
||||
Args:
|
||||
file_id: The file UUID as string.
|
||||
expires_at: The expiration timestamp.
|
||||
signature: The signature to verify.
|
||||
|
||||
Returns:
|
||||
A tuple of (is_valid, error_message).
|
||||
If valid, error_message is None.
|
||||
"""
|
||||
# Check expiration
|
||||
if time.time() > expires_at:
|
||||
return False, "URL has expired"
|
||||
|
||||
# Verify signature
|
||||
expected_signature = _generate_signature(file_id, expires_at)
|
||||
if not hmac.compare_digest(signature, expected_signature):
|
||||
return False, "Invalid signature"
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def _generate_signature(file_id: str, expires_at: int) -> str:
|
||||
"""
|
||||
Generate HMAC signature for URL parameters.
|
||||
|
||||
Args:
|
||||
file_id: The file UUID as string.
|
||||
expires_at: The expiration timestamp.
|
||||
|
||||
Returns:
|
||||
Hex-encoded HMAC-SHA256 signature.
|
||||
"""
|
||||
secret_key = settings.SECRET_KEY.encode()
|
||||
message = f"{file_id}:{expires_at}".encode()
|
||||
|
||||
signature = hmac.new(secret_key, message, hashlib.sha256).hexdigest()
|
||||
return signature
|
||||
59
api/app/core/storage_exceptions.py
Normal file
59
api/app/core/storage_exceptions.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Custom exceptions for storage operations.
|
||||
|
||||
This module defines a hierarchy of exceptions for handling storage-related errors,
|
||||
including configuration errors, connection errors, and operation-specific errors.
|
||||
"""
|
||||
|
||||
|
||||
class StorageError(Exception):
|
||||
"""Base exception for all storage operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
file_key: str | None = None,
|
||||
cause: Exception | None = None,
|
||||
):
|
||||
self.message = message
|
||||
self.file_key = file_key
|
||||
self.cause = cause
|
||||
super().__init__(self.message)
|
||||
|
||||
def __str__(self) -> str:
|
||||
parts = [self.message]
|
||||
if self.file_key:
|
||||
parts.append(f"file_key={self.file_key}")
|
||||
if self.cause:
|
||||
parts.append(f"cause={self.cause}")
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
class StorageConfigError(StorageError):
|
||||
"""Exception raised when storage configuration is invalid or missing."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class StorageConnectionError(StorageError):
|
||||
"""Exception raised when connection to storage backend fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class StorageUploadError(StorageError):
|
||||
"""Exception raised when file upload operation fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class StorageDownloadError(StorageError):
|
||||
"""Exception raised when file download operation fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class StorageDeleteError(StorageError):
|
||||
"""Exception raised when file delete operation fails."""
|
||||
|
||||
pass
|
||||
Reference in New Issue
Block a user