Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding a gzip-ed text file for requests download system test. #35

Merged
merged 2 commits into from
Oct 20, 2017
Merged

Adding a gzip-ed text file for requests download system test. #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec45424
Re-factor `system/requests/test_download.py`.
dhermes Oct 20, 2017
c215a04
Adding a `gzip`-ed text file for requests download system test.
dhermes Oct 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions tests/data/file.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
abcdefghijklmnopqrstuvwxyz0123456789
Binary file added tests/data/file.txt.gz
View file
Open in desktop
Binary file not shown.
176 changes: 110 additions & 66 deletions tests/system/requests/test_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

import base64
import copy
import hashlib
import io
import os
Expand All @@ -31,11 +32,41 @@

CURR_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(CURR_DIR, u'..', u'..', u'data')
IMG_FILES = (
os.path.realpath(os.path.join(DATA_DIR, u'image1.jpg')),
os.path.realpath(os.path.join(DATA_DIR, u'image2.jpg')),
)
PLAIN_TEXT = u'text/plain'
IMAGE_JPEG = u'image/jpeg'
ALL_FILES = (
{
u'path': os.path.realpath(os.path.join(DATA_DIR, u'image1.jpg')),
u'content_type': IMAGE_JPEG,
u'checksum': u'1bsd83IYNug8hd+V1ING3Q==',
u'slices': (
slice(1024, 16386, None), # obj[1024:16386]
slice(None, 8192, None), # obj[:8192]
slice(-256, None, None), # obj[-256:]
slice(262144, None, None), # obj[262144:]
),
}, {
u'path': os.path.realpath(os.path.join(DATA_DIR, u'image2.jpg')),
u'content_type': IMAGE_JPEG,
u'checksum': u'gdLXJltiYAMP9WZZFEQI1Q==',
u'slices': (
slice(1024, 16386, None), # obj[1024:16386]
slice(None, 8192, None), # obj[:8192]
slice(-256, None, None), # obj[-256:]
slice(262144, None, None), # obj[262144:]
),
}, {
u'path': os.path.realpath(os.path.join(DATA_DIR, u'file.txt.gz')),
u'uncompressed': os.path.realpath(os.path.join(DATA_DIR, u'file.txt')),
u'content_type': PLAIN_TEXT,
# NOTE: This **should** be u'KHRs/+ZSrc/FuuR4qz/PZQ=='.
u'checksum': u'XHSHAr/SpIeZtZbjgQ4nGw==',
u'slices': (),

This comment was marked as spam.

Sign in to view

u'metadata': {
u'contentEncoding': u'gzip',
},
},
)
ENCRYPTED_ERR = (
b'The target object is encrypted by a customer-supplied encryption key.')
NO_BODY_ERR = (
Expand Down Expand Up @@ -90,19 +121,44 @@ def delete_blob(transport, blob_name):
assert response.status_code == http_client.NO_CONTENT


def _get_contents_for_upload(info):
with open(info[u'path'], u'rb') as file_obj:
return file_obj.read()


def _get_contents(info):
full_path = info.get(u'uncompressed', info[u'path'])
with open(full_path, u'rb') as file_obj:
return file_obj.read()


def _get_blob_name(info):
full_path = info.get(u'uncompressed', info[u'path'])
return os.path.basename(full_path)


@pytest.fixture(scope=u'module')
def add_files(authorized_transport):
blob_names = []
for img_file in IMG_FILES:
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()
for info in ALL_FILES:
to_upload = _get_contents_for_upload(info)
blob_name = _get_blob_name(info)

blob_name = os.path.basename(img_file)
blob_names.append(blob_name)
upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
upload = resumable_requests.SimpleUpload(upload_url)
response = upload.transmit(
authorized_transport, actual_contents, u'image/jpeg')
if u'metadata' in info:
upload = resumable_requests.MultipartUpload(utils.MULTIPART_UPLOAD)
metadata = copy.deepcopy(info[u'metadata'])
metadata[u'name'] = blob_name
response = upload.transmit(
authorized_transport, to_upload,
metadata, info[u'content_type'])
else:
upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(
blob_name=blob_name)
upload = resumable_requests.SimpleUpload(upload_url)
response = upload.transmit(
authorized_transport, to_upload, info[u'content_type'])

assert response.status_code == http_client.OK

yield
Expand All @@ -125,11 +181,9 @@ def check_tombstoned(download, transport):


def test_download_full(add_files, authorized_transport):
for img_file in IMG_FILES:
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()

blob_name = os.path.basename(img_file)
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)

# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
Expand All @@ -142,11 +196,9 @@ def test_download_full(add_files, authorized_transport):


def test_download_to_stream(add_files, authorized_transport):
for img_file in IMG_FILES:
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()

blob_name = os.path.basename(img_file)
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)

# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
Expand All @@ -165,12 +217,8 @@ def test_download_to_stream(add_files, authorized_transport):


def test_corrupt_download(add_files, corrupting_transport):
actual_checksums = {
u'image1.jpg': u'1bsd83IYNug8hd+V1ING3Q==',
u'image2.jpg': u'gdLXJltiYAMP9WZZFEQI1Q==',
}
for img_file in IMG_FILES:
blob_name = os.path.basename(img_file)
for info in ALL_FILES:
blob_name = _get_blob_name(info)

# Create the actual download object.
media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
Expand All @@ -183,7 +231,7 @@ def test_corrupt_download(add_files, corrupting_transport):
assert download.finished
msg = download_mod._CHECKSUM_MISMATCH.format(
download.media_url, CorruptingAuthorizedSession.EMPTY_HASH,
actual_checksums[blob_name])
info[u'checksum'])
assert exc_info.value.args == (msg,)


Expand Down Expand Up @@ -280,28 +328,25 @@ def test_bad_range(simple_file, authorized_transport):
check_tombstoned(download, authorized_transport)


def _download_slice(media_url, slice_):
assert slice_.step is None

end = None
if slice_.stop is not None:
end = slice_.stop - 1

return resumable_requests.Download(
media_url, start=slice_.start, end=end)


def test_download_partial(add_files, authorized_transport):
slices = (
slice(1024, 16386, None), # obj[1024:16386]
slice(None, 8192, None), # obj[:8192]
slice(-256, None, None), # obj[-256:]
slice(262144, None, None), # obj[262144:]
)
for img_file in IMG_FILES:
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()

blob_name = os.path.basename(img_file)

# Create the multiple download "slices".
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)

media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
downloads = (
resumable_requests.Download(media_url, start=1024, end=16385),
resumable_requests.Download(media_url, end=8191),
resumable_requests.Download(media_url, start=-256),
resumable_requests.Download(media_url, start=262144),
)
for download, slice_ in zip(downloads, slices):
for slice_ in info[u'slices']:
download = _download_slice(media_url, slice_)
response = download.consume(authorized_transport)
assert response.status_code == http_client.PARTIAL_CONTENT
assert response.content == actual_contents[slice_]
Expand Down Expand Up @@ -346,10 +391,9 @@ def consume_chunks(download, authorized_transport,


def test_chunked_download(add_files, authorized_transport):
for img_file in IMG_FILES:
blob_name = os.path.basename(img_file)
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)

total_bytes = len(actual_contents)
num_chunks, chunk_size = get_chunk_size(7, total_bytes)
Expand All @@ -373,18 +417,18 @@ def test_chunked_download(add_files, authorized_transport):


def test_chunked_download_partial(add_files, authorized_transport):
slices = (
slice(1024, 16386, None), # obj[1024:16386]
slice(0, 8192, None), # obj[0:8192]
slice(262144, None, None), # obj[262144:]
)
for img_file in IMG_FILES:
with open(img_file, u'rb') as file_obj:
actual_contents = file_obj.read()

blob_name = os.path.basename(img_file)
for info in ALL_FILES:
actual_contents = _get_contents(info)
blob_name = _get_blob_name(info)

media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
for slice_ in slices:
for slice_ in info[u'slices']:
# Manually replace a missing start with 0.
start = 0 if slice_.start is None else slice_.start
# Chunked downloads don't support a negative index.
if start < 0:
continue

# First determine how much content is in the slice and
# use it to determine a chunking strategy.
total_bytes = len(actual_contents)
Expand All @@ -398,11 +442,11 @@ def test_chunked_download_partial(add_files, authorized_transport):
end = end_byte

num_chunks, chunk_size = get_chunk_size(
7, end_byte - slice_.start + 1)
7, end_byte - start + 1)
# Create the actual download object.
stream = io.BytesIO()
download = resumable_requests.ChunkedDownload(
media_url, chunk_size, stream, start=slice_.start, end=end)
media_url, chunk_size, stream, start=start, end=end)
# Consume the resource in chunks.
num_responses, last_response = consume_chunks(
download, authorized_transport, total_bytes, actual_contents)
Expand Down