diff --git a/src/azure-cli/azure/cli/command_modules/storage/_validators.py b/src/azure-cli/azure/cli/command_modules/storage/_validators.py index 77d083091dd..fe50c875264 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/_validators.py +++ b/src/azure-cli/azure/cli/command_modules/storage/_validators.py @@ -298,8 +298,8 @@ def process_blob_source_uri(cmd, namespace): if not sas: prefix = cmd.command_kwargs['resource_type'].value[0] if is_storagev2(prefix): - sas = create_short_lived_blob_sas_v2(cmd, source_account_name, source_account_key, container, - blob) + sas = create_short_lived_blob_sas_v2(cmd, source_account_name, container, + blob, account_key=source_account_key) else: sas = create_short_lived_blob_sas(cmd, source_account_name, source_account_key, container, blob) query_params = [] @@ -409,8 +409,8 @@ def validate_source_uri(cmd, namespace): # pylint: disable=too-many-statements dir_name, file_name) elif valid_blob_source and (ns.get('share_name', None) or not same_account): if is_storagev2(prefix): - source_sas = create_short_lived_blob_sas_v2(cmd, source_account_name, source_account_key, container, - blob) + source_sas = create_short_lived_blob_sas_v2(cmd, source_account_name, container, + blob, account_key=source_account_key) else: source_sas = create_short_lived_blob_sas(cmd, source_account_name, source_account_key, container, blob) @@ -435,7 +435,8 @@ def validate_source_uri(cmd, namespace): # pylint: disable=too-many-statements def validate_source_url(cmd, namespace): # pylint: disable=too-many-statements, too-many-locals - from .util import create_short_lived_blob_sas, create_short_lived_blob_sas_v2, create_short_lived_file_sas + from .util import create_short_lived_blob_sas, create_short_lived_blob_sas_v2, create_short_lived_file_sas, \ + create_short_lived_file_sas_v2 from azure.cli.core.azclierror import InvalidArgumentValueError, RequiredArgumentMissingError, \ MutuallyExclusiveArgumentError usage_string = \ @@ -463,6 +464,8 @@ def validate_source_url(cmd, namespace): # pylint: disable=too-many-statements, source_account_name = ns.pop('source_account_name', None) source_account_key = ns.pop('source_account_key', None) source_sas = ns.pop('source_sas', None) + token_credential = ns.get('token_credential') + is_oauth = token_credential is not None # source in the form of an uri uri = ns.get('source_url', None) @@ -499,7 +502,7 @@ def validate_source_url(cmd, namespace): # pylint: disable=too-many-statements, # determine if the copy will happen in the same storage account same_account = False - if not source_account_key and not source_sas: + if not source_account_key and not source_sas and not is_oauth: if source_account_name == ns.get('account_name', None): same_account = True source_account_key = ns.get('account_key', None) @@ -511,20 +514,41 @@ def validate_source_url(cmd, namespace): # pylint: disable=too-many-statements, except ValueError: raise RequiredArgumentMissingError('Source storage account {} not found.'.format(source_account_name)) + # if oauth, use user delegation key to generate sas + source_user_delegation_key = None + if is_oauth: + client_kwargs = {'account_name': source_account_name, + 'token_credential': token_credential} + if valid_blob_source: + client = cf_blob_service(cmd.cli_ctx, client_kwargs) + + from datetime import datetime, timedelta + start = datetime.utcnow() + expiry = datetime.utcnow() + timedelta(days=1) + source_user_delegation_key = client.get_user_delegation_key(start, expiry) + # Both source account name and either key or sas (or both) are now available if not source_sas: + prefix = cmd.command_kwargs['resource_type'].value[0] # generate a sas token even in the same account when the source and destination are not the same kind. if valid_file_source and (ns.get('container_name', None) or not same_account): dir_name, file_name = os.path.split(path) if path else (None, '') - source_sas = create_short_lived_file_sas(cmd, source_account_name, source_account_key, share, - dir_name, file_name) + if dir_name == '': + dir_name = None + if is_storagev2(prefix): + source_sas = create_short_lived_file_sas_v2(cmd, source_account_name, source_account_key, share, + dir_name, file_name) + else: + source_sas = create_short_lived_file_sas(cmd, source_account_name, source_account_key, share, + dir_name, file_name) elif valid_blob_source and (ns.get('share_name', None) or not same_account): prefix = cmd.command_kwargs['resource_type'].value[0] # is_storagev2() is used to distinguish if the command is in track2 SDK # If yes, we will use get_login_credentials() as token credential if is_storagev2(prefix): - source_sas = create_short_lived_blob_sas_v2(cmd, source_account_name, source_account_key, container, - blob) + source_sas = create_short_lived_blob_sas_v2(cmd, source_account_name, container, blob, + account_key=source_account_key, + user_delegation_key=source_user_delegation_key) else: source_sas = create_short_lived_blob_sas(cmd, source_account_name, source_account_key, container, blob) @@ -1069,6 +1093,8 @@ def get_source_file_or_blob_service_client_track2(cmd, namespace): source_sas = ns.get('source_sas', None) source_container = ns.get('source_container', None) source_share = ns.get('source_share', None) + token_credential = ns.get('token_credential') + is_oauth = token_credential is not None if source_uri and source_account: raise ValueError(usage_string) @@ -1090,13 +1116,13 @@ def get_source_file_or_blob_service_client_track2(cmd, namespace): source_account, source_key, source_sas = ns['account_name'], ns['account_key'], ns['sas_token'] - if source_account: + if source_account and not is_oauth: if not (source_key or source_sas): # when neither storage account key nor SAS is given, try to fetch the key in the current # subscription source_key = _query_account_key(cmd.cli_ctx, source_account) - elif source_uri: + elif source_uri and not is_oauth: if source_key or source_container or source_share: raise ValueError(usage_string) @@ -1125,7 +1151,7 @@ def get_source_file_or_blob_service_client_track2(cmd, namespace): ns['source_container'] = source_container ns['source_share'] = source_share # get sas token for source - if not source_sas: + if not source_sas and not is_oauth: from .util import create_short_lived_container_sas_track2, create_short_lived_share_sas_track2 if source_container: source_sas = create_short_lived_container_sas_track2(cmd, account_name=source_account, @@ -1139,6 +1165,8 @@ def get_source_file_or_blob_service_client_track2(cmd, namespace): client_kwargs = {'account_name': ns['source_account_name'], 'account_key': ns['source_account_key'], 'sas_token': ns['source_sas']} + if is_oauth: + client_kwargs.update({'token_credential': token_credential}) if source_container: ns['source_client'] = cf_blob_service(cmd.cli_ctx, client_kwargs) if source_share: diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py b/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py index d515c2c1f4e..c7e7e457792 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py +++ b/src/azure-cli/azure/cli/command_modules/storage/operations/blob.py @@ -908,6 +908,14 @@ def create_blob_url(client, container_name, blob_name, snapshot, protocol='https def _copy_blob_to_blob_container(cmd, blob_service, source_blob_service, destination_container, destination_path, source_container, source_blob_name, source_sas, **kwargs): t_blob_client = cmd.get_models('_blob_client#BlobClient') + # generate sas for oauth copy source + if not source_sas: + from ..util import create_short_lived_blob_sas_v2 + start = datetime.utcnow() + expiry = datetime.utcnow() + timedelta(hours=1) + source_user_delegation_key = source_blob_service.get_user_delegation_key(start, expiry) + source_sas = create_short_lived_blob_sas_v2(cmd, source_blob_service.account_name, source_container, + source_blob_name, user_delegation_key=source_user_delegation_key) source_client = t_blob_client(account_url=source_blob_service.url, container_name=source_container, blob_name=source_blob_name, credential=source_sas) source_blob_url = source_client.url @@ -931,7 +939,10 @@ def _copy_file_to_blob_container(blob_service, source_file_service, destination_ source_share, source_sas, source_file_dir, source_file_name): t_share_client = source_file_service.get_share_client(source_share) t_file_client = t_share_client.get_file_client(os.path.join(source_file_dir, source_file_name)) - source_file_url = '{}?{}'.format(t_file_client.url, source_sas) + if '?' not in t_file_client.url: + source_file_url = '{}?{}'.format(t_file_client.url, source_sas) + else: + source_file_url = t_file_client.url source_path = os.path.join(source_file_dir, source_file_name) if source_file_dir else source_file_name destination_blob_name = normalize_blob_file_path(destination_path, source_path) diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_copy_rehydrate_priority.yaml b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_copy_rehydrate_priority.yaml index bc8fc1fe3a8..8c22db0bf24 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_copy_rehydrate_priority.yaml +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/recordings/test_storage_blob_copy_rehydrate_priority.yaml @@ -15,12 +15,12 @@ interactions: ParameterSetName: - -n -g --query -o User-Agent: - - AZURECLI/2.61.0 azsdk-python-core/1.28.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-core/1.28.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) method: POST uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clitest.rg000001/providers/Microsoft.Storage/storageAccounts/clitest000002/listKeys?api-version=2023-05-01&$expand=kerb response: body: - string: '{"keys":[{"creationTime":"2024-06-19T11:17:11.9946193Z","keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"creationTime":"2024-06-19T11:17:11.9946193Z","keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}' + string: '{"keys":[{"creationTime":"2024-09-26T06:54:13.8367221Z","keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"creationTime":"2024-09-26T06:54:13.8367221Z","keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}' headers: cache-control: - no-cache @@ -29,7 +29,7 @@ interactions: content-type: - application/json date: - - Wed, 19 Jun 2024 11:17:37 GMT + - Thu, 26 Sep 2024 06:54:39 GMT expires: - '-1' pragma: @@ -41,9 +41,9 @@ interactions: x-content-type-options: - nosniff x-ms-operation-identifier: - - tenantId=54826b22-38d6-4fb2-bad9-b7b93a3e9c5a,objectId=a7250e3a-0e5e-48e2-9a34-45f1f5e1a91e/eastus2euap/29e2ebd8-cd86-424e-a5a2-d2c2b11d8ce6 + - tenantId=54826b22-38d6-4fb2-bad9-b7b93a3e9c5a,objectId=a7250e3a-0e5e-48e2-9a34-45f1f5e1a91e/eastus2euap/d14d947f-a22b-43b0-97ea-03ab56047374 x-ms-ratelimit-remaining-subscription-resource-requests: - - '11982' + - '11997' status: code: 200 message: OK @@ -63,9 +63,9 @@ interactions: ParameterSetName: - -n --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-date: - - Wed, 19 Jun 2024 11:17:38 GMT + - Thu, 26 Sep 2024 06:54:40 GMT x-ms-version: - '2022-11-02' method: PUT @@ -77,11 +77,11 @@ interactions: content-length: - '0' date: - - Wed, 19 Jun 2024 11:17:39 GMT + - Thu, 26 Sep 2024 06:54:40 GMT etag: - - '"0x8DC90516E4F83AF"' + - '"0x8DCDDF8186A07A3"' last-modified: - - Wed, 19 Jun 2024 11:17:39 GMT + - Thu, 26 Sep 2024 06:54:41 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -105,9 +105,9 @@ interactions: ParameterSetName: - -n --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-date: - - Wed, 19 Jun 2024 11:17:39 GMT + - Thu, 26 Sep 2024 06:54:41 GMT x-ms-version: - '2022-11-02' method: PUT @@ -119,11 +119,11 @@ interactions: content-length: - '0' date: - - Wed, 19 Jun 2024 11:17:40 GMT + - Thu, 26 Sep 2024 06:54:41 GMT etag: - - '"0x8DC90516EFCE5C0"' + - '"0x8DCDDF8192DE19F"' last-modified: - - Wed, 19 Jun 2024 11:17:40 GMT + - Thu, 26 Sep 2024 06:54:42 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -151,11 +151,11 @@ interactions: ParameterSetName: - -c -f -n --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-blob-type: - BlockBlob x-ms-date: - - Wed, 19 Jun 2024 11:17:41 GMT + - Thu, 26 Sep 2024 06:54:42 GMT x-ms-version: - '2022-11-02' method: PUT @@ -169,11 +169,11 @@ interactions: content-md5: - zjOP5omXeKrPwoQU8tlJiw== date: - - Wed, 19 Jun 2024 11:17:41 GMT + - Thu, 26 Sep 2024 06:54:43 GMT etag: - - '"0x8DC90516FCF7C98"' + - '"0x8DCDDF81A2E18DD"' last-modified: - - Wed, 19 Jun 2024 11:17:42 GMT + - Thu, 26 Sep 2024 06:54:44 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-content-crc64: @@ -201,11 +201,11 @@ interactions: ParameterSetName: - -c -n --tier --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-access-tier: - Archive x-ms-date: - - Wed, 19 Jun 2024 11:17:42 GMT + - Thu, 26 Sep 2024 06:54:44 GMT x-ms-rehydrate-priority: - Standard x-ms-version: @@ -219,7 +219,7 @@ interactions: content-length: - '0' date: - - Wed, 19 Jun 2024 11:17:43 GMT + - Thu, 26 Sep 2024 06:54:44 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -241,9 +241,9 @@ interactions: ParameterSetName: - -c -n --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-date: - - Wed, 19 Jun 2024 11:17:43 GMT + - Thu, 26 Sep 2024 06:54:45 GMT x-ms-version: - '2022-11-02' method: HEAD @@ -261,21 +261,21 @@ interactions: content-type: - application/octet-stream date: - - Wed, 19 Jun 2024 11:17:43 GMT + - Thu, 26 Sep 2024 06:54:46 GMT etag: - - '"0x8DC90516FCF7C98"' + - '"0x8DCDDF81A2E18DD"' last-modified: - - Wed, 19 Jun 2024 11:17:42 GMT + - Thu, 26 Sep 2024 06:54:44 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-access-tier: - Archive x-ms-access-tier-change-time: - - Wed, 19 Jun 2024 11:17:43 GMT + - Thu, 26 Sep 2024 06:54:45 GMT x-ms-blob-type: - BlockBlob x-ms-creation-time: - - Wed, 19 Jun 2024 11:17:42 GMT + - Thu, 26 Sep 2024 06:54:44 GMT x-ms-lease-state: - available x-ms-lease-status: @@ -303,13 +303,13 @@ interactions: ParameterSetName: - -b -c --source-uri --tier -r --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-access-tier: - Cool x-ms-copy-source: - - https://clitests5g5dgvy4legk3oi2.blob.core.windows.net/cont55vdihupicjxx6k5pynw/src + - https://clitestp4eii7zykjbavd2cz.blob.core.windows.net/contkwfpqzoedn3e2ue5xrgn/src x-ms-date: - - Wed, 19 Jun 2024 11:17:44 GMT + - Thu, 26 Sep 2024 06:54:46 GMT x-ms-rehydrate-priority: - High x-ms-version: @@ -323,15 +323,15 @@ interactions: content-length: - '0' date: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT etag: - - '"0x8DC905171ED2A1C"' + - '"0x8DCDDF81C589C26"' last-modified: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-copy-id: - - 99a088e9-f188-4f05-a042-7aaaecb881d4 + - 22973a57-57bd-478f-b2e9-a0edc15d2b44 x-ms-copy-status: - success x-ms-version: @@ -353,9 +353,9 @@ interactions: ParameterSetName: - -c -n --account-name --account-key User-Agent: - - AZURECLI/2.61.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) + - AZURECLI/2.64.0 azsdk-python-storage-blob/12.16.0 Python/3.9.13 (Windows-10-10.0.19045-SP0) x-ms-date: - - Wed, 19 Jun 2024 11:17:46 GMT + - Thu, 26 Sep 2024 06:54:47 GMT x-ms-version: - '2022-11-02' method: HEAD @@ -373,35 +373,35 @@ interactions: content-type: - application/octet-stream date: - - Wed, 19 Jun 2024 11:17:46 GMT + - Thu, 26 Sep 2024 06:54:48 GMT etag: - - '"0x8DC905171ED2A1C"' + - '"0x8DCDDF81C589C26"' last-modified: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-access-tier: - Archive x-ms-access-tier-change-time: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT x-ms-archive-status: - rehydrate-pending-to-cool x-ms-blob-type: - BlockBlob x-ms-copy-completion-time: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT x-ms-copy-id: - - 99a088e9-f188-4f05-a042-7aaaecb881d4 + - 22973a57-57bd-478f-b2e9-a0edc15d2b44 x-ms-copy-progress: - 16384/16384 x-ms-copy-source: - - https://clitests5g5dgvy4legk3oi2.blob.core.windows.net/cont55vdihupicjxx6k5pynw/src + - https://clitestp4eii7zykjbavd2cz.blob.core.windows.net/contkwfpqzoedn3e2ue5xrgn/src x-ms-copy-status: - success x-ms-copy-status-description: - pending x-ms-creation-time: - - Wed, 19 Jun 2024 11:17:45 GMT + - Thu, 26 Sep 2024 06:54:47 GMT x-ms-lease-state: - available x-ms-lease-status: diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py index 6a876769fda..77590974d6b 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/latest/test_storage_blob_copy_scenarios.py @@ -80,14 +80,14 @@ def test_storage_blob_copy_same_account_sas(self, resource_group, storage_accoun from datetime import datetime, timedelta start = datetime.utcnow().strftime('%Y-%m-%dT%H:%MZ') - expiry = (datetime.utcnow() + timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%MZ') + expiry = (datetime.utcnow() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:%MZ') sas = self.storage_cmd('storage blob generate-sas -c {} -n src --permissions r --start {}' ' --expiry {}', account_info, source_container, start, expiry).output.strip() self.storage_cmd('storage blob copy start -b dst -c {} --source-blob src --sas-token {} --source-container {} ' - '--source-if-unmodified-since "2021-06-29T06:32Z" --destination-if-modified-since ' - '"2020-06-29T06:32Z" ', account_info, target_container, sas, source_container) + '--source-if-unmodified-since {} --destination-if-modified-since {}', + account_info, target_container, sas, source_container, expiry, expiry) from time import sleep, time start = time() @@ -540,3 +540,76 @@ def test_storage_blob_show_with_copy_in_progress(self, resource_group, source_ac '--destination-container {} --destination-blob {}', target_account_info, source_account, source_container, source_blob, target_container, target_blob) self.storage_cmd('storage blob show -n {} -c {}', target_account_info, target_blob, target_container) + + @ResourceGroupPreparer() + @StorageAccountPreparer(parameter_name='source_account', allow_shared_key_access=False) + @StorageAccountPreparer(parameter_name='target_account', allow_shared_key_access=False) + def test_storage_blob_copy_oauth(self, resource_group, source_account, target_account): + source_file = self.create_temp_file(16, full_random=True) + source_account_info = self.get_account_info(resource_group, source_account) + target_account_info = self.get_account_info(resource_group, target_account) + + with open(source_file, 'rb') as f: + expect_content = f.read() + + source_container = self.create_container(source_account_info, oauth=True) + target_container = self.create_container(target_account_info, oauth=True) + source_blob = 'srcblob' + target_blob = 'dstblob' + + self.oauth_cmd('storage blob upload -c {} -f "{}" -n {} --account-name {}'.format( + source_container, source_file, source_blob, source_account)) + + self.oauth_cmd('storage blob copy start -b {} -c {} --source-blob {} --source-account-name {} ' + '--source-container {} --account-name {}'.format( + target_blob, target_container, source_blob, source_account, source_container, target_account)) + + from time import sleep, time + start = time() + while True: + # poll until copy has succeeded + blob = self.oauth_cmd('storage blob show -c {} -n {} --account-name {}'.format( + target_container, target_blob, target_account)).get_output_in_json() + if blob["properties"]["copy"]["status"] == "success" or time() - start > 10: + break + sleep(1) + + target_file = self.create_temp_file(1) + self.oauth_cmd('storage blob download -c {} -n {} -f "{}" --account-name {}'.format( + target_container, target_blob, target_file, target_account)) + + with open(target_file, 'rb') as f: + actual_content = f.read() + + self.assertEqual(expect_content, actual_content) + + @ResourceGroupPreparer() + @StorageAccountPreparer(parameter_name='account1', kind='StorageV2', allow_shared_key_access=False, hns=True) + @StorageAccountPreparer(parameter_name='account2', kind='StorageV2', allow_shared_key_access=False) + def test_storage_blob_copy_batch_oauth(self, resource_group, account1, account2): + source_file = self.create_temp_file(16, full_random=False) + for src_account, dst_account in [(account1, account1), (account2, account2), (account1, account2), + (account2, account1)]: + src_account_info = self.get_account_info(resource_group, src_account) + dst_account_info = self.get_account_info(resource_group, dst_account) + src_container = self.create_container(src_account_info, oauth=True) + dst_container = self.create_container(dst_account_info, oauth=True) + dst_container_2 = self.create_container(dst_account_info, oauth=True) + + blobs = ['blobğşŞ', 'blogÉ®'] + for blob_name in blobs: + self.oauth_cmd('storage blob upload -c {} -f "{}" -n {} --account-name {}', + src_container, source_file, blob_name, src_account) + + self.oauth_cmd('storage fs directory create -f {} -n newdir --account-name {}', src_container, src_account) + + # empty dir will be skipped when copy from hns to hns + copied_file = 2 if (src_account, dst_account) == (account1, account1) else 3 + self.oauth_cmd('storage blob copy start-batch --destination-container {} --source-container {} ' + '--source-account-name {} --account-name {}', dst_container, src_container, + src_account, dst_account).assert_with_checks( + JMESPathCheck('length(@)', copied_file)) + self.oauth_cmd('storage blob copy start-batch --destination-container {} --pattern "blob*" ' + '--source-container {} --source-account-name {} --account-name {}', + dst_container_2, src_container, src_account, dst_account).assert_with_checks( + JMESPathCheck('length(@)', 1)) diff --git a/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py b/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py index 6228f2acb27..820d5aca6f3 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py +++ b/src/azure-cli/azure/cli/command_modules/storage/tests/storage_test_util.py @@ -59,9 +59,12 @@ def storage_cmd_negative(self, cmd, account_info, *args): cmd = '{} --account-name {} --account-key {}'.format(cmd, *account_info) return self.cmd(cmd, expect_failure=True) - def create_container(self, account_info, prefix='cont', length=24): + def create_container(self, account_info, prefix='cont', length=24, oauth=False): container_name = self.create_random_name(prefix=prefix, length=length) - self.storage_cmd('storage container create -n {}', account_info, container_name) + if oauth: + self.oauth_cmd('storage container create -n {} --account-name {}', container_name, account_info[0]) + else: + self.storage_cmd('storage container create -n {}', account_info, container_name) return container_name def create_share(self, account_info, prefix='share', length=24): diff --git a/src/azure-cli/azure/cli/command_modules/storage/util.py b/src/azure-cli/azure/cli/command_modules/storage/util.py index 67ccae7ba88..abb05045c60 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/util.py +++ b/src/azure-cli/azure/cli/command_modules/storage/util.py @@ -164,7 +164,7 @@ def create_short_lived_blob_sas(cmd, account_name, account_key, container, blob) return sas.generate_blob(container, blob, permission=t_blob_permissions(read=True), expiry=expiry, protocol='https') -def create_short_lived_blob_sas_v2(cmd, account_name, account_key, container, blob): +def create_short_lived_blob_sas_v2(cmd, account_name, container, blob, account_key=None, user_delegation_key=None): from datetime import timedelta t_sas = cmd.get_models('_shared_access_signature#BlobSharedAccessSignature', @@ -172,7 +172,12 @@ def create_short_lived_blob_sas_v2(cmd, account_name, account_key, container, bl t_blob_permissions = cmd.get_models('_models#BlobSasPermissions', resource_type=ResourceType.DATA_STORAGE_BLOB) expiry = (datetime.utcnow() + timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ') - sas = t_sas(account_name, account_key) + if account_key: + sas = t_sas(account_name, account_key=account_key) + elif user_delegation_key: + sas = t_sas(account_name, user_delegation_key=user_delegation_key) + else: + raise ValueError("Either account key or user delegation key need to be provided.") return sas.generate_blob(container, blob, permission=t_blob_permissions(read=True), expiry=expiry, protocol='https')