959 lines
30 KiB
Python
959 lines
30 KiB
Python
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
# may not use this file except in compliance with the License. A copy of
|
|
# the License is located at
|
|
#
|
|
# https://aws.amazon.com/apache2.0/
|
|
#
|
|
# or in the "license" file accompanying this file. This file is
|
|
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
# ANY KIND, either express or implied. See the License for the specific
|
|
# language governing permissions and limitations under the License.
|
|
import copy as python_copy
|
|
import logging
|
|
from functools import partial
|
|
|
|
from botocore.exceptions import ClientError
|
|
|
|
from boto3 import utils
|
|
from boto3.compat import is_append_mode
|
|
from boto3.s3.transfer import (
|
|
ProgressCallbackInvoker,
|
|
S3Transfer,
|
|
TransferConfig,
|
|
create_transfer_manager,
|
|
)
|
|
|
|
try:
|
|
from botocore.context import with_current_context
|
|
except ImportError:
|
|
from functools import wraps
|
|
|
|
def with_current_context(hook=None):
|
|
def decorator(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapper
|
|
|
|
return decorator
|
|
|
|
|
|
try:
|
|
from botocore.useragent import register_feature_id
|
|
except ImportError:
|
|
|
|
def register_feature_id(feature_id):
|
|
pass
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def inject_s3_transfer_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'upload_file', upload_file)
|
|
utils.inject_attribute(class_attributes, 'download_file', download_file)
|
|
utils.inject_attribute(class_attributes, 'copy', copy)
|
|
utils.inject_attribute(class_attributes, 'upload_fileobj', upload_fileobj)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', download_fileobj
|
|
)
|
|
|
|
|
|
def inject_bucket_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'load', bucket_load)
|
|
utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_file', bucket_download_file
|
|
)
|
|
utils.inject_attribute(class_attributes, 'copy', bucket_copy)
|
|
utils.inject_attribute(
|
|
class_attributes, 'upload_fileobj', bucket_upload_fileobj
|
|
)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', bucket_download_fileobj
|
|
)
|
|
|
|
|
|
def inject_object_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'upload_file', object_upload_file)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_file', object_download_file
|
|
)
|
|
utils.inject_attribute(class_attributes, 'copy', object_copy)
|
|
utils.inject_attribute(
|
|
class_attributes, 'upload_fileobj', object_upload_fileobj
|
|
)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', object_download_fileobj
|
|
)
|
|
|
|
|
|
def inject_object_summary_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'load', object_summary_load)
|
|
|
|
|
|
def bucket_load(self, *args, **kwargs):
|
|
"""
|
|
Calls s3.Client.list_buckets() to update the attributes of the Bucket
|
|
resource.
|
|
"""
|
|
# The docstring above is phrased this way to match what the autogenerated
|
|
# docs produce.
|
|
|
|
# We can't actually get the bucket's attributes from a HeadBucket,
|
|
# so we need to use a ListBuckets and search for our bucket.
|
|
# However, we may fail if we lack permissions to ListBuckets
|
|
# or the bucket is in another account. In which case, creation_date
|
|
# will be None.
|
|
self.meta.data = {}
|
|
try:
|
|
response = self.meta.client.list_buckets()
|
|
for bucket_data in response['Buckets']:
|
|
if bucket_data['Name'] == self.name:
|
|
self.meta.data = bucket_data
|
|
break
|
|
except ClientError as e:
|
|
if not e.response.get('Error', {}).get('Code') == 'AccessDenied':
|
|
raise
|
|
|
|
|
|
def object_summary_load(self, *args, **kwargs):
|
|
"""
|
|
Calls s3.Client.head_object to update the attributes of the ObjectSummary
|
|
resource.
|
|
"""
|
|
response = self.meta.client.head_object(
|
|
Bucket=self.bucket_name, Key=self.key
|
|
)
|
|
if 'ContentLength' in response:
|
|
response['Size'] = response.pop('ContentLength')
|
|
self.meta.data = response
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def upload_file(
|
|
self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
s3.upload_file('/tmp/hello.txt', 'amzn-s3-demo-bucket', 'hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method, except that
|
|
argument names are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to upload to.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
with S3Transfer(self, Config) as transfer:
|
|
return transfer.upload_file(
|
|
filename=Filename,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
callback=Callback,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def download_file(
|
|
self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
s3.download_file('amzn-s3-demo-bucket', 'hello.txt', '/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to download from.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
with S3Transfer(self, Config) as transfer:
|
|
return transfer.download_file(
|
|
bucket=Bucket,
|
|
key=Key,
|
|
filename=Filename,
|
|
extra_args=ExtraArgs,
|
|
callback=Callback,
|
|
)
|
|
|
|
|
|
def bucket_upload_file(
|
|
self, Filename, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Bucket('amzn-s3-demo-bucket').upload_file('/tmp/hello.txt', 'hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.upload_file(
|
|
Filename=Filename,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def bucket_download_file(
|
|
self, Key, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Bucket('amzn-s3-demo-bucket').download_file('hello.txt', '/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.download_file(
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
Filename=Filename,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_upload_file(
|
|
self, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Object('amzn-s3-demo-bucket', 'hello.txt').upload_file('/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.upload_file(
|
|
Filename=Filename,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_download_file(
|
|
self, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Object('amzn-s3-demo-bucket', 'hello.txt').download_file('/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.download_file(
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
Filename=Filename,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def copy(
|
|
self,
|
|
CopySource,
|
|
Bucket,
|
|
Key,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to another.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
s3.meta.client.copy(copy_source, 'amzn-s3-demo-bucket2', 'otherkey')
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to copy to
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to copy to
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operations that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the
|
|
client for the source object. The current client still
|
|
requires IAM permissions to access both buckets.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
# copy is not supported in the CRT
|
|
new_config = python_copy.copy(config)
|
|
new_config.preferred_transfer_client = "classic"
|
|
|
|
with create_transfer_manager(self, new_config) as manager:
|
|
future = manager.copy(
|
|
copy_source=CopySource,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
source_client=SourceClient,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_copy(
|
|
self,
|
|
CopySource,
|
|
Key,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to an object in this bucket.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket2')
|
|
bucket.copy(copy_source, 'otherkey')
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to copy to
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operations that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the
|
|
client for the source object. The current client still
|
|
requires IAM permissions to access both buckets.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
return self.meta.client.copy(
|
|
CopySource=CopySource,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
SourceClient=SourceClient,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_copy(
|
|
self,
|
|
CopySource,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to this object.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket2')
|
|
obj = bucket.Object('otherkey')
|
|
obj.copy(copy_source)
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operations that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the
|
|
client for the source object. The current client still
|
|
requires IAM permissions to access both buckets.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
return self.meta.client.copy(
|
|
CopySource=CopySource,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
SourceClient=SourceClient,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def upload_fileobj(
|
|
self, Fileobj, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to S3.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
|
|
with open('filename', 'rb') as data:
|
|
s3.upload_fileobj(data, 'amzn-s3-demo-bucket', 'mykey')
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to upload to.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
if not hasattr(Fileobj, 'read'):
|
|
raise ValueError('Fileobj must implement read')
|
|
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
with create_transfer_manager(self, config) as manager:
|
|
future = manager.upload(
|
|
fileobj=Fileobj,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_upload_fileobj(
|
|
self, Fileobj, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to this bucket.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
|
|
with open('filename', 'rb') as data:
|
|
bucket.upload_fileobj(data, 'mykey')
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
return self.meta.client.upload_fileobj(
|
|
Fileobj=Fileobj,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_upload_fileobj(
|
|
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to this object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
obj = bucket.Object('mykey')
|
|
|
|
with open('filename', 'rb') as data:
|
|
obj.upload_fileobj(data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
return self.meta.client.upload_fileobj(
|
|
Fileobj=Fileobj,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def disable_threading_if_append_mode(config, fileobj):
|
|
"""Set `TransferConfig.use_threads` to `False` if file-like
|
|
object is in append mode.
|
|
|
|
:type config: boto3.s3.transfer.TransferConfig
|
|
:param config: The transfer configuration to be used when performing the
|
|
download.
|
|
|
|
:type fileobj: A file-like object
|
|
:param fileobj: A file-like object to inspect for append mode.
|
|
"""
|
|
if is_append_mode(fileobj):
|
|
config.use_threads = False
|
|
logger.warning(
|
|
'A single thread will be used because the provided file object '
|
|
'is in append mode. Writes may always be appended to the end of '
|
|
'the file regardless of seek position, so a single thread must be '
|
|
'used to ensure sequential writes.'
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def download_fileobj(
|
|
self, Bucket, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an object from S3 to a file-like object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
|
|
with open('filename', 'wb') as data:
|
|
s3.download_fileobj('amzn-s3-demo-bucket', 'mykey', data)
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to download from.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
if not hasattr(Fileobj, 'write'):
|
|
raise ValueError('Fileobj must implement write')
|
|
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
new_config = python_copy.copy(config)
|
|
disable_threading_if_append_mode(new_config, Fileobj)
|
|
|
|
with create_transfer_manager(self, new_config) as manager:
|
|
future = manager.download(
|
|
bucket=Bucket,
|
|
key=Key,
|
|
fileobj=Fileobj,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_download_fileobj(
|
|
self, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an object from this bucket to a file-like-object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
|
|
with open('filename', 'wb') as data:
|
|
bucket.download_fileobj('mykey', data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
return self.meta.client.download_fileobj(
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
Fileobj=Fileobj,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_download_fileobj(
|
|
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download this object from S3 to a file-like object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
obj = bucket.Object('mykey')
|
|
|
|
with open('filename', 'wb') as data:
|
|
obj.download_fileobj(data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
return self.meta.client.download_fileobj(
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
Fileobj=Fileobj,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|