Left: | ||
Right: |
OLD | NEW |
---|---|
1 # -*- coding: utf-8 -*- | 1 # -*- coding: utf-8 -*- |
2 # Copyright 2013 Google Inc. All Rights Reserved. | 2 # Copyright 2013 Google Inc. All Rights Reserved. |
3 # | 3 # |
4 # Licensed under the Apache License, Version 2.0 (the "License"); | 4 # Licensed under the Apache License, Version 2.0 (the "License"); |
5 # you may not use this file except in compliance with the License. | 5 # you may not use this file except in compliance with the License. |
6 # You may obtain a copy of the License at | 6 # You may obtain a copy of the License at |
7 # | 7 # |
8 # http://www.apache.org/licenses/LICENSE-2.0 | 8 # http://www.apache.org/licenses/LICENSE-2.0 |
9 # | 9 # |
10 # Unless required by applicable law or agreed to in writing, software | 10 # Unless required by applicable law or agreed to in writing, software |
11 # distributed under the License is distributed on an "AS IS" BASIS, | 11 # distributed under the License is distributed on an "AS IS" BASIS, |
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 # See the License for the specific language governing permissions and | 13 # See the License for the specific language governing permissions and |
14 # limitations under the License. | 14 # limitations under the License. |
15 """Integration tests for cp command.""" | 15 """Integration tests for cp command.""" |
16 | 16 |
17 from __future__ import absolute_import | 17 from __future__ import absolute_import |
18 | 18 |
19 import base64 | 19 import base64 |
20 import binascii | 20 import binascii |
21 import datetime | 21 import datetime |
22 import gzip | |
22 from hashlib import md5 | 23 from hashlib import md5 |
23 import httplib | 24 import httplib |
24 import logging | 25 import logging |
25 import os | 26 import os |
26 import pickle | 27 import pickle |
27 import pkgutil | 28 import pkgutil |
28 import random | 29 import random |
29 import re | 30 import re |
30 import string | 31 import string |
31 import sys | 32 import sys |
32 import threading | 33 import threading |
33 | 34 |
34 from apitools.base.py import exceptions as apitools_exceptions | 35 from apitools.base.py import exceptions as apitools_exceptions |
35 import boto | 36 import boto |
36 from boto import storage_uri | 37 from boto import storage_uri |
37 from boto.exception import ResumableTransferDisposition | 38 from boto.exception import ResumableTransferDisposition |
38 from boto.exception import StorageResponseError | 39 from boto.exception import StorageResponseError |
39 from boto.storage_uri import BucketStorageUri | 40 from boto.storage_uri import BucketStorageUri |
40 import crcmod | 41 import crcmod |
41 | |
42 from gslib.cloud_api import ResumableUploadStartOverException | 42 from gslib.cloud_api import ResumableUploadStartOverException |
43 from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD | 43 from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD |
44 from gslib.cs_api_map import ApiSelector | 44 from gslib.cs_api_map import ApiSelector |
45 from gslib.discard_messages_queue import DiscardMessagesQueue | 45 from gslib.discard_messages_queue import DiscardMessagesQueue |
46 from gslib.gcs_json_api import GcsJsonApi | 46 from gslib.gcs_json_api import GcsJsonApi |
47 from gslib.parallel_tracker_file import ObjectFromTracker | 47 from gslib.parallel_tracker_file import ObjectFromTracker |
48 from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile | 48 from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile |
49 from gslib.project_id import PopulateProjectId | 49 from gslib.project_id import PopulateProjectId |
50 from gslib.storage_url import StorageUrlFromString | 50 from gslib.storage_url import StorageUrlFromString |
51 from gslib.tests.rewrite_helper import EnsureRewriteResumeCallbackHandler | 51 from gslib.tests.rewrite_helper import EnsureRewriteResumeCallbackHandler |
(...skipping 21 matching lines...) Expand all Loading... | |
73 from gslib.tests.util import TEST_ENCRYPTION_KEY1 | 73 from gslib.tests.util import TEST_ENCRYPTION_KEY1 |
74 from gslib.tests.util import TEST_ENCRYPTION_KEY1_SHA256_B64 | 74 from gslib.tests.util import TEST_ENCRYPTION_KEY1_SHA256_B64 |
75 from gslib.tests.util import TEST_ENCRYPTION_KEY2 | 75 from gslib.tests.util import TEST_ENCRYPTION_KEY2 |
76 from gslib.tests.util import TEST_ENCRYPTION_KEY3 | 76 from gslib.tests.util import TEST_ENCRYPTION_KEY3 |
77 from gslib.tests.util import unittest | 77 from gslib.tests.util import unittest |
78 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_m essages | 78 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_m essages |
79 from gslib.tracker_file import DeleteTrackerFile | 79 from gslib.tracker_file import DeleteTrackerFile |
80 from gslib.tracker_file import GetRewriteTrackerFilePath | 80 from gslib.tracker_file import GetRewriteTrackerFilePath |
81 from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths | 81 from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths |
82 from gslib.ui_controller import BytesToFixedWidthString | 82 from gslib.ui_controller import BytesToFixedWidthString |
83 from gslib.utils import hashing_helper | |
83 from gslib.utils.boto_util import UsingCrcmodExtension | 84 from gslib.utils.boto_util import UsingCrcmodExtension |
84 from gslib.utils.constants import START_CALLBACK_PER_BYTES | 85 from gslib.utils.constants import START_CALLBACK_PER_BYTES |
85 from gslib.utils.constants import UTF8 | 86 from gslib.utils.constants import UTF8 |
86 from gslib.utils.copy_helper import GetTrackerFilePath | 87 from gslib.utils.copy_helper import GetTrackerFilePath |
87 from gslib.utils.copy_helper import PARALLEL_UPLOAD_STATIC_SALT | 88 from gslib.utils.copy_helper import PARALLEL_UPLOAD_STATIC_SALT |
88 from gslib.utils.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE | 89 from gslib.utils.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE |
89 from gslib.utils.copy_helper import TrackerFileType | 90 from gslib.utils.copy_helper import TrackerFileType |
90 from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents | 91 from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents |
91 from gslib.utils.hashing_helper import CalculateMd5FromContents | 92 from gslib.utils.hashing_helper import CalculateMd5FromContents |
92 from gslib.utils.posix_util import GID_ATTR | 93 from gslib.utils.posix_util import GID_ATTR |
(...skipping 2847 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2940 self.assertTrue(os.path.isfile(tracker_filename)) | 2941 self.assertTrue(os.path.isfile(tracker_filename)) |
2941 self.assertTrue(os.path.isfile('%s_.gztmp' % fpath2)) | 2942 self.assertTrue(os.path.isfile('%s_.gztmp' % fpath2)) |
2942 stderr = self.RunGsUtil(['cp', suri(object_uri), suri(fpath2)], | 2943 stderr = self.RunGsUtil(['cp', suri(object_uri), suri(fpath2)], |
2943 return_stderr=True) | 2944 return_stderr=True) |
2944 self.assertIn('Resuming download', stderr) | 2945 self.assertIn('Resuming download', stderr) |
2945 with open(fpath2, 'r') as f: | 2946 with open(fpath2, 'r') as f: |
2946 self.assertEqual(f.read(), contents, 'File contents did not match.') | 2947 self.assertEqual(f.read(), contents, 'File contents did not match.') |
2947 self.assertFalse(os.path.isfile(tracker_filename)) | 2948 self.assertFalse(os.path.isfile(tracker_filename)) |
2948 self.assertFalse(os.path.isfile('%s_.gztmp' % fpath2)) | 2949 self.assertFalse(os.path.isfile('%s_.gztmp' % fpath2)) |
2949 | 2950 |
2951 def test_cp_download_transfer_encoded(self): | |
2952 """Tests chunked transfer encoded download handling. | |
2953 | |
2954 Tests that download works correctly with a gzipped chunkd transfer- | |
houglum
2018/10/05 00:59:13
Nit: This second paragraph shouldn't be indented t
houglum
2018/10/05 00:59:13
s/chunkd/chunked/
Mike Schwartz
2018/10/05 15:21:53
Done.
Mike Schwartz
2018/10/05 15:21:53
Done.
| |
2955 encoded object (which therefore lacks Content-Length) of a size that gets | |
2956 fetched in a single chunk (exercising downloading of objects lacking a | |
2957 length response header). | |
2958 """ | |
2959 # Upload a file / content-encoding / content-type that triggers this flow. | |
2960 # Note: We need to use the file with pre-zipped format and manually set the | |
2961 # content-encoding and content-type because the Python gzip module (used by | |
2962 # gsutil cp -Z) won't reproduce the bytes that trigger this problem. | |
2963 bucket_uri = self.CreateBucket() | |
2964 object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo') | |
2965 input_filename = 'gslib/data/favicon.ico.gz' | |
houglum
2018/10/05 00:59:13
This only works if your CWD is the root of the gsu
Mike Schwartz
2018/10/05 15:21:53
Done.
| |
2966 self.RunGsUtil(['-h', 'Content-Encoding:gzip', | |
2967 '-h', 'Content-Type:image/x-icon', | |
2968 'cp', suri(input_filename), suri(object_uri)]) | |
2969 # Compute the crc32c of the uncompressed bytes. | |
houglum
2018/10/05 00:59:13
Do we want to perform this test even `UsingCrcmodE
Mike Schwartz
2018/10/05 15:21:53
It's a small file so performance wouldn't be a pro
| |
2970 with gzip.open(input_filename) as fp: | |
2971 hash_dict = {} | |
2972 hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') | |
2973 hashing_helper.CalculateHashesFromContents(fp, hash_dict) | |
2974 in_file_crc32c = hash_dict['crc32c'].crcValue | |
2975 | |
2976 # Downloading this file triggers the flow. | |
2977 fpath2 = self.CreateTempFile() | |
2978 self.RunGsUtil(['cp', suri(object_uri), suri(fpath2)]) | |
2979 with open(fpath2, 'rb') as fp: | |
2980 hash_dict = {} | |
2981 hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') | |
2982 hashing_helper.CalculateHashesFromContents(fp, hash_dict) | |
2983 out_file_crc32c = hash_dict['crc32c'].crcValue | |
2984 | |
2985 self.assertEqual(in_file_crc32c, out_file_crc32c) | |
2986 | |
2950 @SequentialAndParallelTransfer | 2987 @SequentialAndParallelTransfer |
2951 def test_cp_resumable_download_check_hashes_never(self): | 2988 def test_cp_resumable_download_check_hashes_never(self): |
2952 """Tests that resumble downloads work with check_hashes = never.""" | 2989 """Tests that resumble downloads work with check_hashes = never.""" |
2953 bucket_uri = self.CreateBucket() | 2990 bucket_uri = self.CreateBucket() |
2954 contents = 'abcd' * self.halt_size | 2991 contents = 'abcd' * self.halt_size |
2955 object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', | 2992 object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', |
2956 contents=contents) | 2993 contents=contents) |
2957 fpath = self.CreateTempFile() | 2994 fpath = self.CreateTempFile() |
2958 test_callback_file = self.CreateTempFile( | 2995 test_callback_file = self.CreateTempFile( |
2959 contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) | 2996 contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) |
(...skipping 855 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
3815 bucket_uri = self.CreateBucket() | 3852 bucket_uri = self.CreateBucket() |
3816 fpath = self.CreateTempFile(contents='abcd') | 3853 fpath = self.CreateTempFile(contents='abcd') |
3817 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): | 3854 with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): |
3818 log_handler = self.RunCommand('cp', [fpath, suri(bucket_uri)], | 3855 log_handler = self.RunCommand('cp', [fpath, suri(bucket_uri)], |
3819 return_log_handler=True) | 3856 return_log_handler=True) |
3820 warning_messages = log_handler.messages['warning'] | 3857 warning_messages = log_handler.messages['warning'] |
3821 self.assertEquals(1, len(warning_messages)) | 3858 self.assertEquals(1, len(warning_messages)) |
3822 self.assertIn('Found no hashes to validate object upload', | 3859 self.assertIn('Found no hashes to validate object upload', |
3823 warning_messages[0]) | 3860 warning_messages[0]) |
3824 | 3861 |
OLD | NEW |