/usr/lib/python2.6/site-packages/boto/glacier
Edit: /usr/lib/python2.6/site-packages/boto/glacier/job.py (7999B)
# -*- coding: utf-8 -*-
# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
import math
import socket
from boto.glacier.exceptions import TreeHashDoesNotMatchError, \
DownloadArchiveError
from boto.glacier.utils import tree_hash_from_str
class Job(object):
DefaultPartSize = 4 * 1024 * 1024
ResponseDataElements = (('Action', 'action', None),
('ArchiveId', 'archive_id', None),
('ArchiveSizeInBytes', 'archive_size', 0),
('Completed', 'completed', False),
('CompletionDate', 'completion_date', None),
('CreationDate', 'creation_date', None),
('InventorySizeInBytes', 'inventory_size', 0),
('JobDescription', 'description', None),
('JobId', 'id', None),
('SHA256TreeHash', 'sha256_treehash', None),
('SNSTopic', 'sns_topic', None),
('StatusCode', 'status_code', None),
('StatusMessage', 'status_message', None),
('VaultARN', 'arn', None))
def __init__(self, vault, response_data=None):
self.vault = vault
if response_data:
for response_name, attr_name, default in self.ResponseDataElements:
setattr(self, attr_name, response_data[response_name])
else:
for response_name, attr_name, default in self.ResponseDataElements:
setattr(self, attr_name, default)
def __repr__(self):
return 'Job(%s)' % self.arn
def get_output(self, byte_range=None, validate_checksum=False):
"""
This operation downloads the output of the job. Depending on
the job type you specified when you initiated the job, the
output will be either the content of an archive or a vault
inventory.
You can download all the job output or download a portion of
the output by specifying a byte range. In the case of an
archive retrieval job, depending on the byte range you
specify, Amazon Glacier returns the checksum for the portion
of the data. You can compute the checksum on the client and
verify that the values match to ensure the portion you
downloaded is the correct data.
:type byte_range: tuple
:param range: A tuple of integer specifying the slice (in bytes)
of the archive you want to receive
:type validate_checksum: bool
:param validate_checksum: Specify whether or not to validate
the associate tree hash. If the response does not contain
a TreeHash, then no checksum will be verified.
"""
response = self.vault.layer1.get_job_output(self.vault.name,
self.id,
byte_range)
if validate_checksum and 'TreeHash' in response:
data = response.read()
actual_tree_hash = tree_hash_from_str(data)
if response['TreeHash'] != actual_tree_hash:
raise TreeHashDoesNotMatchError(
"The calculated tree hash %s does not match the "
"expected tree hash %s for the byte range %s" % (
actual_tree_hash, response['TreeHash'], byte_range))
return response
def _calc_num_chunks(self, chunk_size):
return int(math.ceil(self.archive_size / float(chunk_size)))
def download_to_file(self, filename, chunk_size=DefaultPartSize,
verify_hashes=True, retry_exceptions=(socket.error,)):
"""Download an archive to a file by name.
:type filename: str
:param filename: The name of the file where the archive
contents will be saved.
:type chunk_size: int
:param chunk_size: The chunk size to use when downloading
the archive.
:type verify_hashes: bool
:param verify_hashes: Indicates whether or not to verify
the tree hashes for each downloaded chunk.
"""
num_chunks = self._calc_num_chunks(chunk_size)
with open(filename, 'wb') as output_file:
self._download_to_fileob(output_file, num_chunks, chunk_size,
verify_hashes, retry_exceptions)
def download_to_fileobj(self, output_file, chunk_size=DefaultPartSize,
verify_hashes=True,
retry_exceptions=(socket.error,)):
"""Download an archive to a file object.
:type output_file: file
:param output_file: The file object where the archive
contents will be saved.
:type chunk_size: int
:param chunk_size: The chunk size to use when downloading
the archive.
:type verify_hashes: bool
:param verify_hashes: Indicates whether or not to verify
the tree hashes for each downloaded chunk.
"""
num_chunks = self._calc_num_chunks(chunk_size)
self._download_to_fileob(output_file, num_chunks, chunk_size,
verify_hashes, retry_exceptions)
def _download_to_fileob(self, fileobj, num_chunks, chunk_size, verify_hashes,
retry_exceptions):
for i in range(num_chunks):
byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1)
data, expected_tree_hash = self._download_byte_range(
byte_range, retry_exceptions)
if verify_hashes:
actual_tree_hash = tree_hash_from_str(data)
if expected_tree_hash != actual_tree_hash:
raise TreeHashDoesNotMatchError(
"The calculated tree hash %s does not match the "
"expected tree hash %s for the byte range %s" % (
actual_tree_hash, expected_tree_hash, byte_range))
fileobj.write(data)
def _download_byte_range(self, byte_range, retry_exceptions):
# You can occasionally get socket.errors when downloading
# chunks from Glacier, so each chunk can be retried up
# to 5 times.
for _ in range(5):
try:
response = self.get_output(byte_range)
data = response.read()
expected_tree_hash = response['TreeHash']
return data, expected_tree_hash
except retry_exceptions as e:
continue
else:
raise DownloadArchiveError("There was an error downloading"
"byte range %s: %s" % (byte_range,
e))