From 428f296ca7f594f334b50dcbc726a1c7c0d95652 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:02:35 +1100 Subject: [PATCH 01/13] Enable Photon by default Preserve the requested Prefix in Arrow flat-list responses. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 4 ++++ .../azure-storage-blobs/src/blob_container_client.cpp | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 66f9d87a90..74565883ca 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -6,6 +6,10 @@ ### Breaking Changes +- Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to + Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue + requesting XML responses. + ### Bugs Fixed - Fixed CMake dependency resolution when FlatCC provides only a shared runtime library. diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index d555c2bd1c..f9111f3d48 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -1484,7 +1484,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto expectResponseFormat = options.ResponseFormat; if (expectResponseFormat == StorageResponseFormat::Auto) { - expectResponseFormat = StorageResponseFormat::Xml; + expectResponseFormat = StorageResponseFormat::Arrow; } if (expectResponseFormat == StorageResponseFormat::Arrow) { @@ -1505,12 +1505,14 @@ namespace Azure { namespace Storage { namespace Blobs { { actualResponseFormat = StorageResponseFormat::Arrow; } - (void)actualResponseFormat; - ListBlobsPagedResponse pagedResponse; pagedResponse.ServiceEndpoint = std::move(response.Value.ServiceEndpoint); pagedResponse.BlobContainerName = std::move(response.Value.BlobContainerName); pagedResponse.Prefix = std::move(response.Value.Prefix); + if (actualResponseFormat == StorageResponseFormat::Arrow && options.Prefix.HasValue()) + { + pagedResponse.Prefix = options.Prefix.Value(); + } for (auto& i : response.Value.Items) { pagedResponse.Blobs.push_back(BlobItemConversion(i)); @@ -1541,7 +1543,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto expectResponseFormat = options.ResponseFormat; if (expectResponseFormat == StorageResponseFormat::Auto) { - expectResponseFormat = StorageResponseFormat::Xml; + expectResponseFormat = StorageResponseFormat::Arrow; } if (expectResponseFormat == StorageResponseFormat::Arrow) { From a109dca08d6c80e31123f661c139d25c22811fe6 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:12:36 +1100 Subject: [PATCH 02/13] Bump Files Shares API version Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-storage-files-shares/CHANGELOG.md | 1 + .../storage/files/shares/rest_client.hpp | 2 +- .../src/rest_client.cpp | 100 +++++++++--------- .../swagger/README.md | 6 +- 4 files changed, 55 insertions(+), 54 deletions(-) diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index ac3dc98246..a70d080493 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features Added +- Bumped up API version to `2027-03-07`. ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index 96bc601993..7a49a68676 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -32,7 +32,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2026-10-06"; + constexpr static const char* ApiVersion = "2027-03-07"; } // namespace _detail namespace Models { /** diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index e5fee88cc6..0612daf59e 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -378,7 +378,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -403,7 +403,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -698,7 +698,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { _internal::UrlEncodeQueryParameter( ListSharesIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1157,7 +1157,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "userdelegationkey"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1291,7 +1291,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-access-tier", options.AccessTier.Value().ToString()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.EnabledProtocols.HasValue() && !options.EnabledProtocols.Value().ToString().empty()) { @@ -1390,7 +1390,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -1547,7 +1547,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.DeleteSnapshots.HasValue() && !options.DeleteSnapshots.Value().ToString().empty()) { request.SetHeader("x-ms-delete-snapshots", options.DeleteSnapshots.Value().ToString()); @@ -1598,7 +1598,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1637,7 +1637,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1679,7 +1679,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1718,7 +1718,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1761,7 +1761,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1799,7 +1799,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1842,7 +1842,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "filepermission"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1878,7 +1878,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-file-permission-format", options.FilePermissionFormat.Value().ToString()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1913,7 +1913,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Quota.HasValue()) { request.SetHeader("x-ms-share-quota", std::to_string(options.Quota.Value())); @@ -2044,7 +2044,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2075,7 +2075,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "acl"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2227,7 +2227,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "acl"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2259,7 +2259,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "stats"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2350,7 +2350,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FilePermission.HasValue() && !options.FilePermission.Value().empty()) { request.SetHeader("x-ms-file-permission", options.FilePermission.Value()); @@ -2484,7 +2484,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -2571,7 +2571,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -2595,7 +2595,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "directory"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FilePermission.HasValue() && !options.FilePermission.Value().empty()) { request.SetHeader("x-ms-file-permission", options.FilePermission.Value()); @@ -2718,7 +2718,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -2772,7 +2772,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "maxresults", std::to_string(options.MaxResults.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Include.HasValue() && !ListFilesIncludeFlagsToString(options.Include.Value()).empty()) { @@ -3165,7 +3165,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-recursive", options.Recursive.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -3378,7 +3378,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-recursive", options.Recursive.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -3416,7 +3416,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "directory"); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (!options.RenameSource.empty()) { request.SetHeader("x-ms-file-rename-source", options.RenameSource); @@ -3531,7 +3531,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); request.SetHeader("x-ms-content-length", std::to_string(options.FileContentLength)); request.SetHeader("x-ms-type", "file"); if (options.FileContentType.HasValue() && !options.FileContentType.Value().empty()) @@ -3720,7 +3720,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Range.HasValue() && !options.Range.Value().empty()) { request.SetHeader("x-ms-range", options.Range.Value()); @@ -3929,7 +3929,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4092,7 +4092,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4123,7 +4123,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileContentLength.HasValue()) { request.SetHeader("x-ms-content-length", std::to_string(options.FileContentLength.Value())); @@ -4284,7 +4284,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4328,7 +4328,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4366,7 +4366,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4407,7 +4407,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4445,7 +4445,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4492,7 +4492,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("Content-MD5", Core::Convert::Base64Encode(options.ContentMD5.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4589,7 +4589,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-source-if-none-match-crc64", Core::Convert::Base64Encode(options.SourceIfNoneMatchCrc64.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4665,7 +4665,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "prevsharesnapshot", _internal::UrlEncodeQueryParameter(options.Prevsharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Range.HasValue() && !options.Range.Value().empty()) { request.SetHeader("x-ms-range", options.Range.Value()); @@ -4813,7 +4813,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Core::Context& context) { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); for (const auto& p : options.Metadata) { request.SetHeader("x-ms-meta-" + p.first, p.second); @@ -4940,7 +4940,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "copyid", _internal::UrlEncodeQueryParameter(options.CopyId)); } request.SetHeader("x-ms-copy-action", "abort"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4987,7 +4987,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -5196,7 +5196,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-handle-id", options.HandleId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -5233,7 +5233,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (!options.RenameSource.empty()) { request.SetHeader("x-ms-file-rename-source", options.RenameSource); @@ -5347,7 +5347,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "symboliclink"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); for (const auto& p : options.Metadata) { request.SetHeader("x-ms-meta-" + p.first, p.second); @@ -5432,7 +5432,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -5460,7 +5460,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "hardlink"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); request.SetHeader("x-ms-type", "file"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index 15867f8c86..26e8e6daf5 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-files-shares namespace: Azure::Storage::Files::Shares output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.FileStorage/stable/2026-12-06/file.json +input-file: https://raw.githubusercontent.com/Jinming-Hu/azure-rest-api-specs/refs/heads/stg105-file-codegen/specification/storage/data-plane/Microsoft.FileStorage/stable/2027-03-07/file.json ``` ## ModelFour Options @@ -79,12 +79,12 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2026-10-06"] + "enum": ["2027-03-07"] }; - from: swagger-document where: $.parameters transform: > - $.ApiVersionParameter.enum = ["2026-10-06"]; + $.ApiVersionParameter.enum = ["2027-03-07"]; ``` ### Rename Operations From fe163a16c6f22b20e6cbb3244b965527294f74d5 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:13:40 +1100 Subject: [PATCH 03/13] Add share creation time support Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-storage-files-shares/CHANGELOG.md | 3 +++ .../storage/files/shares/rest_client.hpp | 8 ++++++ .../src/rest_client.cpp | 16 +++++++++++ .../swagger/README.md | 27 +++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index a70d080493..792ca07e4a 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -5,6 +5,9 @@ ### Features Added - Bumped up API version to `2027-03-07`. +- Added the share creation time to `ShareClient::GetProperties` and + `ShareServiceClient::ListShares` responses. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index 7a49a68676..c97d1ec9fd 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -330,6 +330,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The date and time the share was last modified. */ DateTime LastModified; + /** + * The date and time the share was created. + */ + Nullable CreatedOn; /** * The ETag contains a value which represents the version of the share, in quotes. */ @@ -637,6 +641,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * the last modified time of the share. */ DateTime LastModified; + /** + * Returns the date and time the share was created. + */ + Nullable CreatedOn; /** * Returns the current share quota in GB. */ diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index 0612daf59e..211ee2df40 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -731,6 +731,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { kMetadata, kProperties, kLastModified, + kCreationTime, kEtag, kQuota, kProvisionedIops, @@ -772,6 +773,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { {"Metadata", XmlTagEnum::kMetadata}, {"Properties", XmlTagEnum::kProperties}, {"Last-Modified", XmlTagEnum::kLastModified}, + {"Creation-Time", XmlTagEnum::kCreationTime}, {"Etag", XmlTagEnum::kEtag}, {"Quota", XmlTagEnum::kQuota}, {"ProvisionedIops", XmlTagEnum::kProvisionedIops}, @@ -886,6 +888,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { vectorElement1.Details.LastModified = DateTime::Parse(node.Value, Azure::DateTime::DateFormat::Rfc1123); } + else if ( + xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kShares && xmlPath[2] == XmlTagEnum::kShare + && xmlPath[3] == XmlTagEnum::kProperties && xmlPath[4] == XmlTagEnum::kCreationTime) + { + vectorElement1.Details.CreatedOn + = DateTime::Parse(node.Value, Azure::DateTime::DateFormat::Rfc1123); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kShares && xmlPath[2] == XmlTagEnum::kShare @@ -1416,6 +1426,12 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); response.LastModified = DateTime::Parse( pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + if (pRawResponse->GetHeaders().count("x-ms-share-creation-time") != 0) + { + response.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-share-creation-time"), + Azure::DateTime::DateFormat::Rfc1123); + } response.Quota = std::stoll(pRawResponse->GetHeaders().at("x-ms-share-quota")); if (pRawResponse->GetHeaders().count("x-ms-share-provisioned-iops") != 0) { diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index 26e8e6daf5..df0858a227 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -482,6 +482,20 @@ directive: $.ShareItemInternal.properties["Details"] = {"$ref": "#/definitions/ShareItemDetails", "x-ms-xml": {"name": "Properties"}}; $.ShareItemInternal["x-ms-client-name"] = "ShareItem"; $.ShareItemDetails.properties["ProvisionedBandwidthMiBps"]["x-ms-client-name"] = "ProvisionedBandwidthMBps"; + $.ShareItemDetails.properties["CreationTime"]["x-ms-client-name"] = "CreatedOn"; + $.ShareItemDetails.properties["CreationTime"]["x-ms-xml"] = {"name": "Creation-Time"}; + $.ShareItemDetails.properties["CreationTime"]["x-nullable"] = true; + $.ShareItemDetails.properties["CreationTime"].description = "The date and time the share was created."; + const creationTime = $.ShareItemDetails.properties["CreationTime"]; + delete $.ShareItemDetails.properties["CreationTime"]; + const properties = {}; + for (const name in $.ShareItemDetails.properties) { + properties[name] = $.ShareItemDetails.properties[name]; + if (name === "Last-Modified") { + properties["CreationTime"] = creationTime; + } + } + $.ShareItemDetails.properties = properties; delete $.ShareItemDetails.properties["EnableSmbDirectoryLease"]; delete $.ShareItemInternal.properties["Properties"]; delete $.ShareItemInternal.required; @@ -571,6 +585,19 @@ directive: $["x-ms-share-max-burst-credits-for-iops"]["x-nullable"] = true; $["x-ms-share-next-allowed-provisioned-iops-downgrade-time"]["x-nullable"] = true; $["x-ms-share-next-allowed-provisioned-bandwidth-downgrade-time"]["x-nullable"] = true; + $["x-ms-share-creation-time"]["x-ms-client-name"] = "CreatedOn"; + $["x-ms-share-creation-time"]["x-nullable"] = true; + $["x-ms-share-creation-time"].description = "Returns the date and time the share was created."; + const creationTime = $["x-ms-share-creation-time"]; + delete $["x-ms-share-creation-time"]; + const headers = {}; + for (const name in $) { + headers[name] = $[name]; + if (name === "Last-Modified") { + headers["x-ms-share-creation-time"] = creationTime; + } + } + $ = headers; - from: swagger-document where: $["x-ms-paths"]["/{shareName}?restype=share"].get.responses["200"] transform: > From 60308fd11e69aa182e823dc1fd0b1398514a9921 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:13:41 +1100 Subject: [PATCH 04/13] Add Files Change Feed properties Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-storage-files-shares/CHANGELOG.md | 1 + .../storage/files/shares/rest_client.hpp | 16 ++++ .../storage/files/shares/share_options.hpp | 25 ++++++ .../src/rest_client.cpp | 32 ++++++++ .../src/share_client.cpp | 16 +++- .../swagger/README.md | 77 +++++++++++++++++++ 6 files changed, 166 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index 792ca07e4a..61a1a0128d 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -7,6 +7,7 @@ - Bumped up API version to `2027-03-07`. - Added the share creation time to `ShareClient::GetProperties` and `ShareServiceClient::ListShares` responses. +- Added support for enabling and configuring Files Change Feed when creating or updating a share. ### Breaking Changes diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index c97d1ec9fd..59d70619d1 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -734,6 +734,18 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * Returns the current share next allowed provisioned bandwidth downgrade time. */ Nullable NextAllowedProvisionedBandwidthDowngradeTime; + /** + * Specifies whether change feed is enabled on the share. + */ + Nullable IsChangeFeedEnabled; + /** + * The number of days that change feed records are retained. + */ + Nullable ChangeFeedRetentionInDays; + /** + * The name of the blob container where change feed records are stored. + */ + Nullable ChangeFeedBlobContainerName; }; /** * @brief Specifies the option include to delete the base share and all of its snapshots. @@ -2570,6 +2582,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { Nullable FileRequestIntent; Nullable ShareProvisionedIops; Nullable ShareProvisionedBandwidthMibps; + Nullable EnableChangeFeed; + Nullable ChangeFeedRetentionInDays; }; static Response Create( Core::Http::_internal::HttpPipeline& pipeline, @@ -2701,6 +2715,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { Nullable FileRequestIntent; Nullable ShareProvisionedIops; Nullable ShareProvisionedBandwidthMibps; + Nullable EnableChangeFeed; + Nullable ChangeFeedRetentionInDays; }; static Response SetProperties( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index f1138235f6..af41789143 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -491,6 +491,18 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * throughput is 60 MiB/sec and maximum is 5,125 MiB/sec. */ Nullable ProvisionedMaxBandwidthMibps; + + /** + * Optional. Version 2026-06-06 and newer. Specifies whether change feed is enabled on the + * share. + */ + Nullable EnableChangeFeed; + + /** + * Optional. Version 2026-06-06 and newer. Specifies the number of days that change feed + * records are retained. Valid values are between 1 and 365. The default is 7 days. + */ + Nullable ChangeFeedRetentionInDays; }; /** @@ -581,6 +593,19 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * bandwidth is 60 MiB/sec and max is 5,120 MiB/sec. */ Nullable ProvisionedMaxBandwidthMibps; + + /** + * Optional. Version 2026-06-06 and newer. Specifies whether change feed is enabled on the + * share. + */ + Nullable EnableChangeFeed; + + /** + * Optional. Version 2026-06-06 and newer. Specifies the number of days that change feed + * records are retained. Valid values are between 1 and 365. Once change feed is enabled, this + * value can be updated independently. + */ + Nullable ChangeFeedRetentionInDays; }; /** diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index 211ee2df40..67d2e3b65c 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -1351,6 +1351,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-provisioned-bandwidth-mibps", std::to_string(options.ShareProvisionedBandwidthMibps.Value())); } + if (options.EnableChangeFeed.HasValue()) + { + request.SetHeader( + "x-ms-file-enable-change-feed", options.EnableChangeFeed.Value() ? "true" : "false"); + } + if (options.ChangeFeedRetentionInDays.HasValue()) + { + request.SetHeader( + "x-ms-file-change-feed-retention-in-days", + std::to_string(options.ChangeFeedRetentionInDays.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -1548,6 +1559,16 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-next-allowed-provisioned-bandwidth-downgrade-time"), Azure::DateTime::DateFormat::Rfc1123); } + if (pRawResponse->GetHeaders().count("x-ms-file-change-feed-retention-in-days") != 0) + { + response.ChangeFeedRetentionInDays + = std::stoi(pRawResponse->GetHeaders().at("x-ms-file-change-feed-retention-in-days")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-blob-container-for-xfiles-change-feed") != 0) + { + response.ChangeFeedBlobContainerName + = pRawResponse->GetHeaders().at("x-ms-file-blob-container-for-xfiles-change-feed"); + } return Response(std::move(response), std::move(pRawResponse)); } Response ShareClient::Delete( @@ -1986,6 +2007,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-provisioned-bandwidth-mibps", std::to_string(options.ShareProvisionedBandwidthMibps.Value())); } + if (options.EnableChangeFeed.HasValue()) + { + request.SetHeader( + "x-ms-file-enable-change-feed", options.EnableChangeFeed.Value() ? "true" : "false"); + } + if (options.ChangeFeedRetentionInDays.HasValue()) + { + request.SetHeader( + "x-ms-file-change-feed-retention-in-days", + std::to_string(options.ChangeFeedRetentionInDays.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) diff --git a/sdk/storage/azure-storage-files-shares/src/share_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_client.cpp index 00ae769c8e..6fb9797149 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_client.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -150,6 +151,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { protocolLayerOptions.PaidBurstingMaxBandwidthMibps = options.PaidBurstingMaxBandwidthMibps; protocolLayerOptions.ShareProvisionedIops = options.ProvisionedMaxIops; protocolLayerOptions.ShareProvisionedBandwidthMibps = options.ProvisionedMaxBandwidthMibps; + protocolLayerOptions.EnableChangeFeed = options.EnableChangeFeed; + protocolLayerOptions.ChangeFeedRetentionInDays = options.ChangeFeedRetentionInDays; auto result = _detail::ShareClient::Create(*m_pipeline, m_shareUrl, protocolLayerOptions, context); Models::CreateShareResult ret; @@ -244,8 +247,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; auto protocolLayerOptions = _detail::ShareClient::GetSharePropertiesOptions(); protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; - return _detail::ShareClient::GetProperties( + auto response = _detail::ShareClient::GetProperties( *m_pipeline, m_shareUrl, protocolLayerOptions, context); + const auto& headers = response.RawResponse->GetHeaders(); + auto changeFeedEnabled = headers.find("x-ms-file-enable-change-feed"); + if (changeFeedEnabled != headers.end()) + { + response.Value.IsChangeFeedEnabled + = Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + changeFeedEnabled->second, "true"); + } + return response; } Azure::Response ShareClient::SetProperties( @@ -264,6 +276,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { protocolLayerOptions.PaidBurstingMaxBandwidthMibps = options.PaidBurstingMaxBandwidthMibps; protocolLayerOptions.ShareProvisionedIops = options.ProvisionedMaxIops; protocolLayerOptions.ShareProvisionedBandwidthMibps = options.ProvisionedMaxBandwidthMibps; + protocolLayerOptions.EnableChangeFeed = options.EnableChangeFeed; + protocolLayerOptions.ChangeFeedRetentionInDays = options.ChangeFeedRetentionInDays; return _detail::ShareClient::SetProperties( *m_pipeline, m_shareUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index df0858a227..51c612816d 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -552,6 +552,60 @@ directive: $ = $.filter(p => !p["$ref"] || !p["$ref"].endsWith("#/parameters/EnableSmbDirectoryLease")); ``` +### Share Change Feed + +```yaml +directive: + - from: swagger-document + where: $.parameters + transform: > + $.EnableChangeFeed = { + "name": "x-ms-file-enable-change-feed", + "in": "header", + "type": "boolean", + "required": false, + "x-ms-client-name": "EnableChangeFeed" + }; + $.ChangeFeedRetentionInDays = { + "name": "x-ms-file-change-feed-retention-in-days", + "in": "header", + "type": "integer", + "format": "int32", + "required": false, + "x-ms-client-name": "ChangeFeedRetentionInDays" + }; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + for (const path of ["/{shareName}?restype=share", "/{shareName}?restype=share&comp=properties"]) { + $[path].put.parameters.push( + {"$ref": "#/parameters/EnableChangeFeed"}, + {"$ref": "#/parameters/ChangeFeedRetentionInDays"}); + } + - from: swagger-document + where: $["x-ms-paths"]["/{shareName}?restype=share"].get.responses["200"].headers + transform: > + $["x-ms-file-enable-change-feed"] = { + "type": "boolean", + "x-ms-client-name": "IsChangeFeedEnabled", + "x-nullable": true, + "description": "Specifies whether change feed is enabled on the share." + }; + $["x-ms-file-change-feed-retention-in-days"] = { + "type": "integer", + "format": "int32", + "x-ms-client-name": "ChangeFeedRetentionInDays", + "x-nullable": true, + "description": "The number of days that change feed records are retained." + }; + $["x-ms-file-blob-container-for-xfiles-change-feed"] = { + "type": "string", + "x-ms-client-name": "ChangeFeedBlobContainerName", + "x-nullable": true, + "description": "The name of the blob container where change feed records are stored." + }; +``` + ### GetShareProperties ```yaml @@ -610,6 +664,29 @@ directive: "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true, "x-ms-xml": {"name": ""}} } }; + $.schema.required = []; + for (const headerName in $.headers) { + const header = $.headers[headerName]; + const clientName = header["x-ms-client-name"] + || headerName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + const propertyName = clientName[0].toUpperCase() + clientName.slice(1); + if (propertyName !== "AccessTier") { + const property = JSON.parse(JSON.stringify(header)); + delete property["x-ms-client-name"]; + property["x-ms-xml"] = {"name": ""}; + $.schema.properties[propertyName] = property; + } + if (!$.schema.properties[propertyName]["x-nullable"]) { + $.schema.required.push(propertyName); + } + } + $.schema.properties.Metadata = { + "$ref": "#/definitions/Metadata", + "description": $.schema.properties.Metadata.description, + "x-ms-xml": {"name": ""} + }; + $.schema.properties.ProvisionedIops.description = "Returns the current share provisioned IOPS."; + delete $.headers["x-ms-file-enable-change-feed"]; ``` ### GetShareAccessPolicy From dc267770996a47887d121a3dd038e2e598daa0c6 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:14:21 +1100 Subject: [PATCH 05/13] Add Files APIs by file ID Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure-storage-files-shares/CHANGELOG.md | 2 + .../storage/files/shares/rest_client.hpp | 47 ++++ .../storage/files/shares/share_client.hpp | 14 + .../storage/files/shares/share_constants.hpp | 1 + .../files/shares/share_directory_client.hpp | 14 +- .../files/shares/share_file_client.hpp | 24 +- .../storage/files/shares/share_options.hpp | 15 ++ .../storage/files/shares/share_responses.hpp | 38 +++ .../src/rest_client.cpp | 252 ++++++++++++++++++ .../src/share_client.cpp | 25 ++ .../src/share_directory_client.cpp | 31 +++ .../src/share_file_client.cpp | 99 +++++++ .../src/share_lease_client.cpp | 5 + .../src/share_options.cpp | 34 +++ .../swagger/README.md | 99 +++++++ 15 files changed, 690 insertions(+), 10 deletions(-) diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index 61a1a0128d..c4d594b350 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -8,6 +8,8 @@ - Added the share creation time to `ShareClient::GetProperties` and `ShareServiceClient::ListShares` responses. - Added support for enabling and configuring Files Change Feed when creating or updating a share. +- Added support for addressing files and directories by file ID, retrieving their properties, and + listing a file's hard links. ### Breaking Changes diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index 59d70619d1..3c6b9cd572 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -1327,6 +1327,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * NFS only. Type of the file or directory. */ Nullable NfsFileType; + /** + * The name of the directory. + */ + Nullable FileName; }; } // namespace _detail /** @@ -1998,6 +2002,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * NFS only. Type of the file or directory. */ Nullable NfsFileType; + /** + * The name of the file. + */ + Nullable FileName; }; } // namespace _detail /** @@ -2519,6 +2527,34 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ Models::NfsFileType NfsFileType; }; + /** + * @brief A hard link to a file. + */ + struct HardLink final + { + /** + * The name of the hard link. + */ + StringEncoded FileName; + /** + * The fileId of the parent directory of the hard link. + */ + std::string ParentId; + }; + /** + * @brief Response type for #Azure::Storage::Files::Shares::FileClient::GetFileLinks. + */ + struct GetFileLinksResult final + { + /** + * Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetProperties. + */ + FileProperties Properties; + /** + * Array of HardLink. + */ + std::vector HardLinks; + }; } // namespace _detail } // namespace Models namespace _detail { @@ -3256,6 +3292,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Core::Url& url, const CreateFileHardLinkOptions& options, const Core::Context& context); + struct GetFileFileLinksOptions final + { + Nullable LeaseId; + Nullable AllowTrailingDot; + Nullable FileRequestIntent; + }; + static Response GetFileLinks( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetFileFileLinksOptions& options, + const Core::Context& context); }; } // namespace _detail }}}} // namespace Azure::Storage::Files::Shares \ No newline at end of file diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp index 37a6d0ae32..abb599d13c 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp @@ -91,6 +91,20 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ ShareDirectoryClient GetRootDirectoryClient() const; + /** + * @brief Creates a file client that addresses a file by its file ID. + * @param fileId The file ID of the file. + * @return A client that addresses the file by its file ID. + */ + ShareFileClient GetFileClientByFileId(const std::string& fileId) const; + + /** + * @brief Creates a directory client that addresses a directory by its file ID. + * @param fileId The file ID of the directory. + * @return A client that addresses the directory by its file ID. + */ + ShareDirectoryClient GetDirectoryClientByFileId(const std::string& fileId) const; + /** * @brief Creates the file share. * @param options Optional parameters to create this file share. diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp index 73e0d67863..02409362f4 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp @@ -14,6 +14,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { namespace _detail { constexpr static const char* ShareSnapshotQueryParameter = "sharesnapshot"; + constexpr static const char* FileIdQueryParameter = "fileid"; // Error codes: constexpr static const char* ParentNotFound = "ParentNotFound"; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp index 4e22eed048..d85e027c58 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp @@ -78,6 +78,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ std::string GetUrl() const { return m_shareDirectoryUrl.GetAbsoluteUrl(); } + /** + * @brief Gets the file ID when this client addresses a directory by ID. + * + * @return The file ID, or an empty string when this client addresses a directory by path. + */ + std::string GetFileId() const; + /** * @brief Create a ShareDirectoryClient that's a sub directory of the current * ShareDirectoryClient @@ -274,17 +281,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { private: Azure::Core::Url m_shareDirectoryUrl; + Azure::Nullable m_fileId; std::shared_ptr m_pipeline; _detail::ShareClientConfiguration m_clientConfiguration; explicit ShareDirectoryClient( Azure::Core::Url shareDirectoryUrl, std::shared_ptr pipeline, - _detail::ShareClientConfiguration clientConfiguration) - : m_shareDirectoryUrl(std::move(shareDirectoryUrl)), m_pipeline(std::move(pipeline)), - m_clientConfiguration(std::move(clientConfiguration)) - { - } + _detail::ShareClientConfiguration clientConfiguration); friend class ShareClient; }; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp index 3c0b9a0607..f6ad203f6f 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp @@ -77,6 +77,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ std::string GetUrl() const { return m_shareFileUrl.GetAbsoluteUrl(); } + /** + * @brief Gets the file ID when this client addresses a file by ID. + * + * @return The file ID, or an empty string when this client addresses a file by path. + */ + std::string GetFileId() const; + /** * @brief Initializes a new instance of the ShareFileClient class with an identical URL * source but the specified share snapshot timestamp. @@ -446,19 +453,26 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateHardLinkOptions& options = CreateHardLinkOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Returns the hard links and properties of a file addressed by file ID. + * @param options Optional parameters for getting the file links. + * @param context Context for cancelling the operation. + * @return The file properties and hard links. + */ + Azure::Response GetFileLinks( + const GetFileLinksOptions& options = GetFileLinksOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: Azure::Core::Url m_shareFileUrl; + Azure::Nullable m_fileId; std::shared_ptr m_pipeline; _detail::ShareClientConfiguration m_clientConfiguration; explicit ShareFileClient( Azure::Core::Url shareFileUrl, std::shared_ptr pipeline, - _detail::ShareClientConfiguration clientConfiguration) - : m_shareFileUrl(std::move(shareFileUrl)), m_pipeline(std::move(pipeline)), - m_clientConfiguration(std::move(clientConfiguration)) - { - } + _detail::ShareClientConfiguration clientConfiguration); friend class ShareClient; friend class ShareDirectoryClient; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index af41789143..972eab2bda 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -272,6 +273,9 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { }; namespace _detail { + Azure::Nullable GetFileIdFromUrl(const Azure::Core::Url& url); + void AssertPathAddressed(const Azure::Nullable& fileId, const char* operationName); + struct ShareClientConfiguration { /** @@ -1180,6 +1184,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { LeaseAccessConditions AccessConditions; }; + /** + * @brief Optional parameters for #Azure::Storage::Files::Shares::ShareFileClient::GetFileLinks. + */ + struct GetFileLinksOptions final + { + /** + * The operation will only succeed if the access condition is met. + */ + LeaseAccessConditions AccessConditions; + }; + /** * @brief Optional parameters for #Azure::Storage::Files::Shares::ShareFileClient::SetProperties. */ diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp index 5b6ff648c6..39255b6919 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp @@ -526,6 +526,40 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The NFS related properties for the file. */ FilePosixProperties PosixProperties; + /** + * The file name. This is populated when the file is addressed by file ID. + */ + Nullable FileName; + }; + + /** + * @brief A hard link to a file. + */ + struct FileLink final + { + /** + * The name of the hard link. + */ + std::string Name; + /** + * The file ID of the hard link's parent directory. + */ + std::string ParentId; + }; + + /** + * @brief Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetFileLinks. + */ + struct GetFileLinksResult final + { + /** + * The properties of the file. + */ + FileProperties Properties; + /** + * The hard links of the file. + */ + std::vector Links; }; /** @@ -737,6 +771,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The NFS related properties for the file. */ FilePosixProperties PosixProperties; + /** + * The directory name. This is populated when the directory is addressed by file ID. + */ + Nullable FileName; }; /** diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index 67d2e3b65c..859b8d94c8 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -2603,6 +2603,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.NfsFileType = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } return Response( std::move(response), std::move(pRawResponse)); } @@ -4125,6 +4129,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.NfsFileType = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } return Response( std::move(response), std::move(pRawResponse)); } @@ -5562,5 +5570,249 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Response( std::move(response), std::move(pRawResponse)); } + Response FileClient::GetFileLinks( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetFileFileLinksOptions& options, + const Core::Context& context) + { + auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); + request.GetUrl().AppendQueryParameter("comp", "hardlinks"); + request.SetHeader("x-ms-version", "2027-03-07"); + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.AllowTrailingDot.HasValue()) + { + request.SetHeader( + "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); + } + if (options.FileRequestIntent.HasValue() + && !options.FileRequestIntent.Value().ToString().empty()) + { + request.SetHeader("x-ms-file-request-intent", options.FileRequestIntent.Value().ToString()); + } + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (httpStatusCode != Core::Http::HttpStatusCode::Ok) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::_detail::GetFileLinksResult response; + { + const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + enum class XmlTagEnum + { + kUnknown, + kHardLinks, + kHardLink, + kFileName, + kParentId, + }; + const std::unordered_map XmlTagEnumMap{ + {"HardLinks", XmlTagEnum::kHardLinks}, + {"HardLink", XmlTagEnum::kHardLink}, + {"FileName", XmlTagEnum::kFileName}, + {"ParentId", XmlTagEnum::kParentId}, + }; + std::vector xmlPath; + Models::_detail::HardLink vectorElement1; + while (true) + { + auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + else if (node.Type == _internal::XmlNodeType::StartTag) + { + auto ite = XmlTagEnumMap.find(node.Name); + xmlPath.push_back(ite == XmlTagEnumMap.end() ? XmlTagEnum::kUnknown : ite->second); + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kFileName) + { + vectorElement1.FileName.Content = node.Value; + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kParentId) + { + vectorElement1.ParentId = node.Value; + } + } + else if (node.Type == _internal::XmlNodeType::Attribute) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kFileName + && node.Name == "Encoded") + { + vectorElement1.FileName.Encoded = node.Value == std::string("true"); + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + if (xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink) + { + response.HardLinks.push_back(std::move(vectorElement1)); + vectorElement1 = Models::_detail::HardLink(); + } + xmlPath.pop_back(); + } + } + } + response.Properties.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + for (auto i = pRawResponse->GetHeaders().lower_bound("x-ms-meta-"); + i != pRawResponse->GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; + ++i) + { + response.Properties.Metadata.emplace(i->first.substr(10), i->second); + } + response.Properties.FileSize + = std::stoll(pRawResponse->GetHeaders().at("x-ms-content-length")); + if (pRawResponse->GetHeaders().count("x-ms-content-type") != 0) + { + response.Properties.HttpHeaders.ContentType + = pRawResponse->GetHeaders().at("x-ms-content-type"); + } + response.Properties.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + if (pRawResponse->GetHeaders().count("x-ms-content-md5") != 0) + { + response.Properties.HttpHeaders.ContentHash.Value + = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-md5")); + response.Properties.HttpHeaders.ContentHash.Algorithm = HashAlgorithm::Md5; + } + if (pRawResponse->GetHeaders().count("x-ms-content-encoding") != 0) + { + response.Properties.HttpHeaders.ContentEncoding + = pRawResponse->GetHeaders().at("x-ms-content-encoding"); + } + if (pRawResponse->GetHeaders().count("x-ms-cache-control") != 0) + { + response.Properties.HttpHeaders.CacheControl + = pRawResponse->GetHeaders().at("x-ms-cache-control"); + } + if (pRawResponse->GetHeaders().count("x-ms-content-disposition") != 0) + { + response.Properties.HttpHeaders.ContentDisposition + = pRawResponse->GetHeaders().at("x-ms-content-disposition"); + } + if (pRawResponse->GetHeaders().count("x-ms-content-language") != 0) + { + response.Properties.HttpHeaders.ContentLanguage + = pRawResponse->GetHeaders().at("x-ms-content-language"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-completion-time") != 0) + { + response.Properties.CopyCompletedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-copy-completion-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status-description") != 0) + { + response.Properties.CopyStatusDescription + = pRawResponse->GetHeaders().at("x-ms-copy-status-description"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-id") != 0) + { + response.Properties.CopyId = pRawResponse->GetHeaders().at("x-ms-copy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-progress") != 0) + { + response.Properties.CopyProgress = pRawResponse->GetHeaders().at("x-ms-copy-progress"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-source") != 0) + { + response.Properties.CopySource = pRawResponse->GetHeaders().at("x-ms-copy-source"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status") != 0) + { + response.Properties.CopyStatus + = Models::CopyStatus(pRawResponse->GetHeaders().at("x-ms-copy-status")); + } + response.Properties.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + if (pRawResponse->GetHeaders().count("x-ms-file-attributes") != 0) + { + response.Properties.SmbProperties.Attributes + = Models::FileAttributes(pRawResponse->GetHeaders().at("x-ms-file-attributes")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-creation-time") != 0) + { + response.Properties.SmbProperties.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-creation-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-last-write-time") != 0) + { + response.Properties.SmbProperties.LastWrittenOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-last-write-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-change-time") != 0) + { + response.Properties.SmbProperties.ChangedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-change-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-permission-key") != 0) + { + response.Properties.SmbProperties.PermissionKey + = pRawResponse->GetHeaders().at("x-ms-file-permission-key"); + } + response.Properties.SmbProperties.FileId = pRawResponse->GetHeaders().at("x-ms-file-id"); + response.Properties.SmbProperties.ParentFileId + = pRawResponse->GetHeaders().at("x-ms-file-parent-id"); + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.Properties.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0) + { + response.Properties.LeaseState + = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0) + { + response.Properties.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-mode") != 0) + { + response.Properties.FileMode = pRawResponse->GetHeaders().at("x-ms-mode"); + } + if (pRawResponse->GetHeaders().count("x-ms-owner") != 0) + { + response.Properties.Owner = pRawResponse->GetHeaders().at("x-ms-owner"); + } + if (pRawResponse->GetHeaders().count("x-ms-group") != 0) + { + response.Properties.Group = pRawResponse->GetHeaders().at("x-ms-group"); + } + if (pRawResponse->GetHeaders().count("x-ms-link-count") != 0) + { + response.Properties.LinkCount + = std::stoll(pRawResponse->GetHeaders().at("x-ms-link-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-file-type") != 0) + { + response.Properties.NfsFileType + = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.Properties.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } + return Response( + std::move(response), std::move(pRawResponse)); + } } // namespace _detail }}}} // namespace Azure::Storage::Files::Shares \ No newline at end of file diff --git a/sdk/storage/azure-storage-files-shares/src/share_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_client.cpp index 6fb9797149..d1f7c2c738 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_client.cpp @@ -3,6 +3,7 @@ #include "azure/storage/files/shares/share_client.hpp" +#include "azure/storage/files/shares/share_constants.hpp" #include "azure/storage/files/shares/share_directory_client.hpp" #include "azure/storage/files/shares/share_file_client.hpp" #include "private/package_version.hpp" @@ -117,6 +118,30 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return ShareDirectoryClient(m_shareUrl, m_pipeline, m_clientConfiguration); } + ShareFileClient ShareClient::GetFileClientByFileId(const std::string& fileId) const + { + if (fileId.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + Core::Url fileUrl(m_shareUrl); + fileUrl.AppendQueryParameter( + _detail::FileIdQueryParameter, _internal::UrlEncodeQueryParameter(fileId)); + return ShareFileClient(std::move(fileUrl), m_pipeline, m_clientConfiguration); + } + + ShareDirectoryClient ShareClient::GetDirectoryClientByFileId(const std::string& fileId) const + { + if (fileId.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + Core::Url directoryUrl(m_shareUrl); + directoryUrl.AppendQueryParameter( + _detail::FileIdQueryParameter, _internal::UrlEncodeQueryParameter(fileId)); + return ShareDirectoryClient(std::move(directoryUrl), m_pipeline, m_clientConfiguration); + } + ShareClient ShareClient::WithSnapshot(const std::string& snapshot) const { ShareClient newClient(*this); diff --git a/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp index f659ede92b..8c40536eab 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp @@ -3,6 +3,7 @@ #include "azure/storage/files/shares/share_directory_client.hpp" +#include "azure/storage/files/shares/share_constants.hpp" #include "azure/storage/files/shares/share_file_client.hpp" #include "private/package_version.hpp" @@ -17,6 +18,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { + std::string ShareDirectoryClient::GetFileId() const { return m_fileId.ValueOr(std::string()); } + ShareDirectoryClient ShareDirectoryClient::CreateFromConnectionString( const std::string& connectionString, const std::string& shareName, @@ -39,12 +42,23 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { } } + ShareDirectoryClient::ShareDirectoryClient( + Core::Url shareDirectoryUrl, + std::shared_ptr pipeline, + _detail::ShareClientConfiguration clientConfiguration) + : m_shareDirectoryUrl(std::move(shareDirectoryUrl)), + m_fileId(_detail::GetFileIdFromUrl(m_shareDirectoryUrl)), m_pipeline(std::move(pipeline)), + m_clientConfiguration(std::move(clientConfiguration)) + { + } + ShareDirectoryClient::ShareDirectoryClient( const std::string& shareDirectoryUrl, std::shared_ptr credential, const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -68,6 +82,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -99,6 +114,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -118,6 +134,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ShareDirectoryClient ShareDirectoryClient::GetSubdirectoryClient( const std::string& subdirectoryName) const { + _detail::AssertPathAddressed(m_fileId, "GetSubdirectoryClient"); auto builder = m_shareDirectoryUrl; builder.AppendPath(_internal::UrlEncodePath(subdirectoryName)); return ShareDirectoryClient(builder, m_pipeline, m_clientConfiguration); @@ -125,6 +142,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ShareFileClient ShareDirectoryClient::GetFileClient(const std::string& fileName) const { + _detail::AssertPathAddressed(m_fileId, "GetFileClient"); auto builder = m_shareDirectoryUrl; builder.AppendPath(_internal::UrlEncodePath(fileName)); return ShareFileClient(builder, m_pipeline, m_clientConfiguration); @@ -150,6 +168,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Create"); auto protocolLayerOptions = _detail::DirectoryClient::CreateDirectoryOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -217,6 +236,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateIfNotExists"); try { return Create(options, context); @@ -239,6 +259,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const RenameFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "RenameFile"); auto sourceFileUrl = m_shareDirectoryUrl; sourceFileUrl.AppendPath(_internal::UrlEncodePath(fileName)); @@ -300,6 +321,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const RenameDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "RenameSubdirectory"); auto sourceDirectoryUrl = m_shareDirectoryUrl; sourceDirectoryUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName)); @@ -359,6 +381,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Delete"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::DeleteDirectoryOptions(); protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; @@ -375,6 +398,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DeleteIfExists"); try { return Delete(options, context); @@ -417,6 +441,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ret.PosixProperties.Owner = std::move(response.Value.Owner); ret.PosixProperties.Group = std::move(response.Value.Group); ret.PosixProperties.NfsFileType = std::move(response.Value.NfsFileType); + ret.FileName = std::move(response.Value.FileName); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } @@ -426,6 +451,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetDirectoryPropertiesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetProperties"); auto protocolLayerOptions = _detail::DirectoryClient::SetDirectoryPropertiesOptions(); protocolLayerOptions.FileAttributes = smbProperties.Attributes.ToString(); if (smbProperties.CreatedOn.HasValue()) @@ -483,6 +509,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetDirectoryMetadataOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetMetadata"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::SetDirectoryMetadataOptions(); protocolLayerOptions.Metadata @@ -497,6 +524,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListFilesAndDirectoriesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListFilesAndDirectories"); auto protocolLayerOptions = _detail::DirectoryClient::ListDirectoryFilesAndDirectoriesSegmentOptions(); protocolLayerOptions.Prefix = options.Prefix; @@ -572,6 +600,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListDirectoryHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListHandles"); auto protocolLayerOptions = _detail::DirectoryClient::ListDirectoryHandlesOptions(); protocolLayerOptions.Marker = options.ContinuationToken; protocolLayerOptions.MaxResults = options.PageSizeHint; @@ -631,6 +660,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseDirectoryHandleOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseHandle"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::ForceDirectoryCloseHandlesOptions(); protocolLayerOptions.HandleId = handleId; @@ -647,6 +677,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseAllDirectoryHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseAllHandles"); auto protocolLayerOptions = _detail::DirectoryClient::ForceDirectoryCloseHandlesOptions(); protocolLayerOptions.HandleId = FileAllHandles; protocolLayerOptions.Marker = options.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp index 05efc80168..ef3b411aae 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp @@ -27,6 +27,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { constexpr static const int32_t DefaultListAllRangesPageSizeHint = 10000; + std::string ShareFileClient::GetFileId() const { return m_fileId.ValueOr(std::string()); } + ShareFileClient ShareFileClient::CreateFromConnectionString( const std::string& connectionString, const std::string& shareName, @@ -49,12 +51,23 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { } } + ShareFileClient::ShareFileClient( + Core::Url shareFileUrl, + std::shared_ptr pipeline, + _detail::ShareClientConfiguration clientConfiguration) + : m_shareFileUrl(std::move(shareFileUrl)), + m_fileId(_detail::GetFileIdFromUrl(m_shareFileUrl)), m_pipeline(std::move(pipeline)), + m_clientConfiguration(std::move(clientConfiguration)) + { + } + ShareFileClient::ShareFileClient( const std::string& shareFileUrl, std::shared_ptr credential, const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -78,6 +91,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -109,6 +123,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -145,6 +160,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Create"); auto protocolLayerOptions = _detail::FileClient::CreateFileOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -283,6 +299,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Delete"); auto protocolLayerOptions = _detail::FileClient::DeleteFileOptions(); protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; @@ -299,6 +316,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DeleteIfExists"); try { return Delete(options, context); @@ -320,6 +338,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Download"); bool isStructuredMessage = false; auto protocolLayerOptions = _detail::FileClient::DownloadFileOptions(); if (options.Range.HasValue()) @@ -495,6 +514,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const StartFileCopyOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "StartCopy"); auto protocolLayerOptions = _detail::FileClient::StartFileCopyOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -661,6 +681,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const AbortFileCopyOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "AbortCopy"); auto protocolLayerOptions = _detail::FileClient::AbortFileCopyOptions(); protocolLayerOptions.CopyId = std::move(copyId); protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; @@ -706,6 +727,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ret.PosixProperties.Group = std::move(response.Value.Group); ret.PosixProperties.NfsFileType = std::move(response.Value.NfsFileType); ret.PosixProperties.LinkCount = std::move(response.Value.LinkCount); + ret.FileName = std::move(response.Value.FileName); return Azure::Response(std::move(ret), std::move(response.RawResponse)); } @@ -715,6 +737,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetFilePropertiesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetProperties"); auto protocolLayerOptions = _detail::FileClient::SetFileHttpHeadersOptions(); protocolLayerOptions.FileAttributes = smbProperties.Attributes.ToString(); if (smbProperties.CreatedOn.HasValue()) @@ -798,6 +821,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetFileMetadataOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetMetadata"); auto protocolLayerOptions = _detail::FileClient::SetFileMetadataOptions(); protocolLayerOptions.Metadata = std::map(metadata.begin(), metadata.end()); @@ -814,6 +838,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileRangeOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadRange"); auto protocolLayerOptions = _detail::FileClient::UploadFileRangeOptions(); protocolLayerOptions.FileRangeWrite = "update"; protocolLayerOptions.Range = std::string("bytes=") + std::to_string(offset) + std::string("-") @@ -863,6 +888,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ClearFileRangeOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ClearRange"); auto protocolLayerOptions = _detail::FileClient::UploadFileRangeOptions(); protocolLayerOptions.FileRangeWrite = "clear"; protocolLayerOptions.Range = std::string("bytes=") + std::to_string(offset) + std::string("-") @@ -890,6 +916,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetRangeList"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -934,6 +961,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetRangeListDiff"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -979,6 +1007,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetAllRangeList"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -1028,6 +1057,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetAllRangeListDiff"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -1079,6 +1109,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListFileHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListHandles"); auto protocolLayerOptions = _detail::FileClient::ListFileHandlesOptions(); protocolLayerOptions.Marker = options.ContinuationToken; protocolLayerOptions.MaxResults = options.PageSizeHint; @@ -1137,6 +1168,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseFileHandleOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseHandle"); (void)options; auto protocolLayerOptions = _detail::FileClient::ForceFileCloseHandlesOptions(); protocolLayerOptions.HandleId = handleId; @@ -1152,6 +1184,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseAllFileHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseAllHandles"); auto protocolLayerOptions = _detail::FileClient::ForceFileCloseHandlesOptions(); protocolLayerOptions.HandleId = FileAllHandles; protocolLayerOptions.Marker = options.ContinuationToken; @@ -1179,6 +1212,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileToOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DownloadTo"); // Just start downloading using an initial chunk. If it's a small file, we'll get the whole // thing in one shot. If it's a large file, we'll get its full size in Content-Range and can // keep downloading it in chunks. @@ -1292,6 +1326,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileToOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DownloadTo"); // Just start downloading using an initial chunk. If it's a small file, we'll get the whole // thing in one shot. If it's a large file, we'll get its full size in Content-Range and can // keep downloading it in chunks. @@ -1413,6 +1448,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileFromOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadFrom"); _detail::FileClient::CreateFileOptions protocolLayerOptions; protocolLayerOptions.FileContentLength = bufferSize; protocolLayerOptions.FileAttributes = options.SmbProperties.Attributes.ToString(); @@ -1526,6 +1562,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileFromOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadFrom"); _internal::FileReader fileReader(fileName); _detail::FileClient::CreateFileOptions protocolLayerOptions; @@ -1643,6 +1680,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileRangeFromUriOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadRangeFromUri"); AZURE_ASSERT_MSG(sourceRange.Length.HasValue(), "Source length cannot be null."); int64_t rangeLength = sourceRange.Length.Value(); @@ -1696,6 +1734,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateSymbolicLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateSymbolicLink"); _detail::FileClient::CreateFileSymbolicLinkOptions protocolLayerOptions; protocolLayerOptions.LinkText = linkText; if (options.CreatedOn.HasValue()) @@ -1736,6 +1775,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetSymbolicLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetSymbolicLink"); (void)options; _detail::FileClient::GetFileSymbolicLinkOptions protocolLayerOptions; protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; @@ -1755,6 +1795,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateHardLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateHardLink"); _detail::FileClient::CreateFileHardLinkOptions protocolLayerOptions; protocolLayerOptions.TargetFile = targetFile; protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; @@ -1775,4 +1816,62 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); } + + Azure::Response ShareFileClient::GetFileLinks( + const GetFileLinksOptions& options, + const Azure::Core::Context& context) const + { + if (!m_fileId.HasValue()) + { + throw std::logic_error("GetFileLinks requires a file ID-addressed client."); + } + + _detail::FileClient::GetFileFileLinksOptions protocolLayerOptions; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; + protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; + auto response = _detail::FileClient::GetFileLinks( + *m_pipeline, m_shareFileUrl, protocolLayerOptions, context); + + Models::GetFileLinksResult result; + auto& properties = result.Properties; + properties.CopyCompletedOn = std::move(response.Value.Properties.CopyCompletedOn); + properties.CopyId = std::move(response.Value.Properties.CopyId); + properties.CopyProgress = std::move(response.Value.Properties.CopyProgress); + properties.CopySource = std::move(response.Value.Properties.CopySource); + properties.CopyStatus = std::move(response.Value.Properties.CopyStatus); + properties.CopyStatusDescription = std::move(response.Value.Properties.CopyStatusDescription); + properties.ETag = std::move(response.Value.Properties.ETag); + properties.FileSize = response.Value.Properties.FileSize; + properties.LastModified = std::move(response.Value.Properties.LastModified); + properties.SmbProperties = std::move(response.Value.Properties.SmbProperties); + properties.HttpHeaders = std::move(response.Value.Properties.HttpHeaders); + properties.Metadata = std::move(response.Value.Properties.Metadata); + properties.IsServerEncrypted = response.Value.Properties.IsServerEncrypted; + properties.LeaseDuration = std::move(response.Value.Properties.LeaseDuration); + properties.LeaseState = std::move(response.Value.Properties.LeaseState); + properties.LeaseStatus = std::move(response.Value.Properties.LeaseStatus); + properties.FileName = std::move(response.Value.Properties.FileName); + properties.PosixProperties.Owner = std::move(response.Value.Properties.Owner); + properties.PosixProperties.Group = std::move(response.Value.Properties.Group); + properties.PosixProperties.LinkCount = std::move(response.Value.Properties.LinkCount); + properties.PosixProperties.NfsFileType = std::move(response.Value.Properties.NfsFileType); + if (response.Value.Properties.FileMode.HasValue()) + { + properties.PosixProperties.FileMode + = Models::NfsFileMode::ParseOctalFileMode(response.Value.Properties.FileMode.Value()); + } + + result.Links.reserve(response.Value.HardLinks.size()); + for (auto& hardLink : response.Value.HardLinks) + { + Models::FileLink link; + link.Name = hardLink.FileName.Encoded ? Core::Url::Decode(hardLink.FileName.Content) + : std::move(hardLink.FileName.Content); + link.ParentId = std::move(hardLink.ParentId); + result.Links.emplace_back(std::move(link)); + } + return Azure::Response( + std::move(result), std::move(response.RawResponse)); + } }}}} // namespace Azure::Storage::Files::Shares diff --git a/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp index 025051fcfa..93b4921ba0 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp @@ -22,6 +22,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "AcquireLease"); _detail::FileClient::AcquireFileLeaseOptions protocolLayerOptions; protocolLayerOptions.ProposedLeaseId = GetLeaseId(); protocolLayerOptions.Duration = static_cast(duration.count()); @@ -79,6 +80,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "RenewLease"); AZURE_ASSERT_MSG(false, "File lease doesn't support renew"); AZURE_NOT_IMPLEMENTED(); } @@ -116,6 +118,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "ReleaseLease"); _detail::FileClient::ReleaseFileLeaseOptions protocolLayerOptions; protocolLayerOptions.LeaseId = GetLeaseId(); protocolLayerOptions.AllowTrailingDot @@ -170,6 +173,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "ChangeLease"); _detail::FileClient::ChangeFileLeaseOptions protocolLayerOptions; protocolLayerOptions.LeaseId = GetLeaseId(); protocolLayerOptions.ProposedLeaseId = proposedLeaseId; @@ -237,6 +241,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "BreakLease"); _detail::FileClient::BreakFileLeaseOptions protocolLayerOptions; protocolLayerOptions.AllowTrailingDot = m_fileClient.Value().m_clientConfiguration.AllowTrailingDot; diff --git a/sdk/storage/azure-storage-files-shares/src/share_options.cpp b/sdk/storage/azure-storage-files-shares/src/share_options.cpp index 345d5ba23c..b99c34746d 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_options.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_options.cpp @@ -3,10 +3,44 @@ #include "azure/storage/files/shares/share_options.hpp" +#include "azure/storage/files/shares/share_constants.hpp" + +#include + +#include + namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareAudience ShareAudience::DefaultAudience(_internal::StorageDefaultAudience); + namespace _detail { + Azure::Nullable GetFileIdFromUrl(const Azure::Core::Url& url) + { + for (const auto& queryParameter : url.GetQueryParameters()) + { + if (Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + queryParameter.first, FileIdQueryParameter)) + { + if (queryParameter.second.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + return queryParameter.second; + } + } + return Azure::Nullable(); + } + + void AssertPathAddressed(const Azure::Nullable& fileId, const char* operationName) + { + if (fileId.HasValue()) + { + throw std::logic_error( + std::string(operationName) + " is not supported for a file ID-addressed client."); + } + } + } // namespace _detail + namespace Models { namespace { RolePermissions ParseOctal(char c) diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index 51c612816d..240dea708b 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -62,6 +62,8 @@ directive: where: $["x-ms-paths"] transform: > delete $["/{shareName}?restype=share&comp=undelete"].put; + delete $["/{shareName}?restype=directory"].get; + delete $["/{shareName}"].head; ``` ### API Version @@ -911,6 +913,12 @@ directive: $.headers["x-ms-owner"]["x-nullable"] = true; $.headers["x-ms-group"]["x-nullable"] = true; $.headers["x-ms-file-file-type"]["x-nullable"] = true; + $.headers["x-ms-file-name"]["x-ms-client-name"] = "FileName"; + $.headers["x-ms-file-name"]["x-nullable"] = true; + $.headers["x-ms-file-name"].description = "The name of the directory."; + const fileName = $.headers["x-ms-file-name"]; + delete $.headers["x-ms-file-name"]; + $.headers["x-ms-file-name"] = fileName; $.schema = { "type": "object", "x-ms-client-name": "DirectoryProperties", @@ -1030,6 +1038,12 @@ directive: $.headers["x-ms-group"]["x-nullable"] = true; $.headers["x-ms-file-file-type"]["x-nullable"] = true; $.headers["x-ms-link-count"]["x-nullable"] = true; + $.headers["x-ms-file-name"]["x-ms-client-name"] = "FileName"; + $.headers["x-ms-file-name"]["x-nullable"] = true; + $.headers["x-ms-file-name"].description = "The name of the file."; + const fileName = $.headers["x-ms-file-name"]; + delete $.headers["x-ms-file-name"]; + $.headers["x-ms-file-name"] = fileName; delete $.headers["x-ms-type"]; $.schema = { "type": "object", @@ -1349,6 +1363,91 @@ directive: }; ``` +### GetFileLinks + +```yaml +directive: + - from: swagger-document + where: $.definitions + transform: > + $.HardLink["x-namespace"] = "_detail"; + const propertiesResponse = $doc["x-ms-paths"]["/{shareName}/{directory}/{fileName}"].head.responses["200"]; + const propertiesSchema = JSON.parse(JSON.stringify(propertiesResponse.schema)); + propertiesSchema.required = ["SmbProperties", "HttpHeaders"]; + for (const headerName in propertiesResponse.headers) { + const header = propertiesResponse.headers[headerName]; + const propertyPath = header["x-ms-client-path"] + || header["x-ms-client-name"] + || headerName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + if (propertyPath.includes(".")) { + continue; + } + const propertyName = propertyPath[0].toUpperCase() + propertyPath.slice(1); + const property = JSON.parse(JSON.stringify(header)); + delete property["x-ms-client-name"]; + delete property["x-ms-client-path"]; + property["x-ms-xml"] = {"name": ""}; + propertiesSchema.properties[propertyName] = property; + if (!property["x-nullable"]) { + propertiesSchema.required.push(propertyName); + } + } + propertiesSchema.properties.Metadata = { + "$ref": "#/definitions/Metadata", + "description": propertiesSchema.properties.Metadata.description, + "x-ms-xml": {"name": ""} + }; + propertiesSchema.description = "Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetProperties."; + $.FileProperties = propertiesSchema; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + const propertiesResponse = $["/{shareName}/{directory}/{fileName}"].head.responses["200"]; + propertiesResponse.schema = {"$ref": "#/definitions/FileProperties"}; + + const operation = $["/{shareName}?comp=hardlinks"].get; + operation.operationId = "File_GetFileLinks"; + operation.parameters = operation.parameters.filter(p => + p.name !== "fileid" && p["$ref"] !== "#/parameters/ShareSnapshot"); + operation.responses["200"].schema = { + "type": "object", + "x-ms-client-name": "GetFileLinksResult", + "x-ms-sealed": false, + "x-namespace": "_detail", + "xml": {"name": "HardLinks"}, + "required": ["Properties", "HardLinks"], + "properties": { + "Properties": { + "$ref": "#/definitions/FileProperties", + "x-ms-xml": {"name": ""} + }, + "HardLinks": { + "type": "array", + "x-ms-xml": {"name": "."}, + "items": {"$ref": "#/definitions/HardLink"} + } + } + }; + + const response = operation.responses["200"]; + const headers = {}; + for (const sourceHeaderName in propertiesResponse.headers) { + let headerName = sourceHeaderName; + if (["Content-Length", "Content-Type", "Content-MD5", "Content-Encoding", + "Content-Language", "Cache-Control", "Content-Disposition"].includes(headerName)) { + headerName = "x-ms-" + headerName.toLowerCase(); + } + const header = JSON.parse(JSON.stringify(propertiesResponse.headers[sourceHeaderName])); + const propertyPath = header["x-ms-client-path"] + || header["x-ms-client-name"] + || sourceHeaderName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + delete header["x-ms-client-name"]; + header["x-ms-client-path"] = "Properties." + propertyPath[0].toUpperCase() + propertyPath.slice(1); + headers[headerName] = header; + } + response.headers = headers; +``` + ### Description ```yaml From 939e5d9fa4383c68724c8759b2d01c41e994ca0c Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:14:22 +1100 Subject: [PATCH 06/13] Add persisted session authentication for Blob Storage Normalize hostname casing for service/account inference without changing URL casing. Use CMake automatic DLL exports for session providers without exporting STL members. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 2 + .../azure-storage-blobs/CMakeLists.txt | 4 + .../inc/azure/storage/blobs/blob_options.hpp | 110 ++++++ .../azure-storage-blobs/src/blob_client.cpp | 8 +- .../src/blob_container_client.cpp | 8 +- .../azure-storage-blobs/src/blob_options.cpp | 17 + .../src/blob_service_client.cpp | 8 +- .../private/session_authentication_policy.cpp | 165 +++++++++ .../private/session_authentication_policy.hpp | 24 ++ .../token_credential_session_provider.cpp | 315 ++++++++++++++++++ .../token_credential_session_provider.hpp | 64 ++++ .../azure-storage-common/CMakeLists.txt | 2 + .../common/internal/shared_key_policy.hpp | 11 +- .../internal/storage_bearer_token_auth.hpp | 16 +- .../storage/common/internal/storage_url.hpp | 24 ++ .../src/shared_key_policy.cpp | 9 +- ...age_bearer_token_authentication_policy.cpp | 9 +- .../azure-storage-common/src/storage_url.cpp | 115 +++++++ .../azure-storage-files-datalake/CHANGELOG.md | 2 + .../files/datalake/datalake_options.hpp | 6 + .../src/datalake_directory_client.cpp | 53 +-- .../src/datalake_file_system_client.cpp | 50 +-- .../src/datalake_utilities.cpp | 1 + 23 files changed, 908 insertions(+), 115 deletions(-) create mode 100644 sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp create mode 100644 sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp create mode 100644 sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp create mode 100644 sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp create mode 100644 sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp create mode 100644 sdk/storage/azure-storage-common/src/storage_url.cpp diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 74565883ca..e06660cf58 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added support for persisted session authentication for eligible blob download requests. + ### Breaking Changes - Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 5e7ff0cc3f..8eee037487 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -76,6 +76,10 @@ set( src/private/avro_parser.cpp src/private/avro_parser.hpp src/private/package_version.hpp + src/private/session_authentication_policy.cpp + src/private/session_authentication_policy.hpp + src/private/token_credential_session_provider.cpp + src/private/token_credential_session_provider.hpp src/rest_client.cpp ) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index aa9434d246..d9ae656deb 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -5,6 +5,9 @@ #include "azure/storage/blobs/rest_client.hpp" +#include +#include +#include #include #include #include @@ -23,6 +26,69 @@ namespace Azure { namespace Storage { namespace Blobs { + class ContainerSessionProvider; + class SessionProvider; + + namespace _detail { + class SessionAuthenticationPolicy; + class TokenCredentialSessionProvider; + } // namespace _detail + + /** + * @brief Specifies how persisted session authentication is used. + */ + enum class SessionMode + { + /** + * @brief Lets the SDK determine when to use sessions. + * + * @note Currently behaves the same as #Disabled. + */ + Auto, + + /** + * @brief Disables session authentication. + */ + Disabled, + + /** + * @brief Prefers session authentication for eligible requests. + * + * @note The client can fall back to bearer authentication while acquiring a session, when + * session creation is temporarily unavailable, or when the service rejects a session. + */ + Enabled, + }; + + /** + * @brief Configures persisted session authentication. + */ + struct SessionOptions final + { + /** + * @brief The session authentication mode. + * + * In Auto mode, clients currently use bearer authentication. + */ + SessionMode Mode = SessionMode::Auto; + + /** + * @brief Account name used when signing requests. Required when it cannot be inferred from URL. + * + * For custom endpoints, both AccountName and Provider must be specified. + */ + std::string AccountName; + + /** + * @brief Optional provider whose session cache can be shared by multiple clients. + * + * When unset, each independently constructed client creates its own provider. Clients derived + * from that client share its provider and cache. For custom endpoints, both Provider and + * AccountName must be specified. + */ + std::shared_ptr Provider; + }; + /** * @brief Audiences available for blob service * @@ -287,6 +353,50 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable DownloadValidationOptions; + + /** + * @brief Configures persisted session authentication. + */ + Blobs::SessionOptions Session; + }; + + /** + * @brief Base type for providers that own and cache persisted sessions. + * + * Session providers can be shared by multiple clients through #SessionOptions::Provider. + * Use #ContainerSessionProvider for TokenCredential-based session authentication. + */ + class SessionProvider { + public: + virtual ~SessionProvider(); + + private: + friend class ContainerSessionProvider; + friend class _detail::SessionAuthenticationPolicy; + + SessionProvider(); + + std::shared_ptr<_detail::TokenCredentialSessionProvider> m_provider; + }; + + /** + * @brief Creates and caches container-scoped sessions using a TokenCredential. + */ + class ContainerSessionProvider final : public SessionProvider { + public: + /** + * @brief Creates a provider backed by a TokenCredential. + * + * @param serviceUrl The Blob service URL. + * @param credential The credential used to create sessions. + * @param options Client options used by Create Session requests. + */ + ContainerSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options); + + ~ContainerSessionProvider() override; }; /** diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 13aa2c82a4..2d020b3ac6 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -7,6 +7,7 @@ #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include @@ -88,9 +89,10 @@ namespace Azure { namespace Storage { namespace Blobs { options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) : _internal::StorageScope); - pipelineOptions.TokenAuthPolicy - = std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>( - credential, tokenContext, options.EnableTenantDiscovery); + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + blobUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); } m_pipeline = std::make_shared( diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index f9111f3d48..4092dfe7c2 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -8,6 +8,7 @@ #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include @@ -1285,8 +1286,8 @@ namespace Azure { namespace Storage { namespace Blobs { m_blobContainerUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); std::unique_ptr tokenAuthPolicy; + Azure::Core::Credentials::TokenRequestContext tokenContext; { - Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) @@ -1310,7 +1311,10 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; - pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy); + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + blobContainerUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); diff --git a/sdk/storage/azure-storage-blobs/src/blob_options.cpp b/sdk/storage/azure-storage-blobs/src/blob_options.cpp index 13143f6139..4a75133774 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_options.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_options.cpp @@ -3,8 +3,25 @@ #include "azure/storage/blobs/blob_options.hpp" +#include "private/token_credential_session_provider.hpp" + namespace Azure { namespace Storage { namespace Blobs { + SessionProvider::SessionProvider() = default; + + SessionProvider::~SessionProvider() = default; + + ContainerSessionProvider::ContainerSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options) + { + m_provider = std::make_shared<_detail::TokenCredentialSessionProvider>( + serviceUrl, std::move(credential), options); + } + + ContainerSessionProvider::~ContainerSessionProvider() = default; + const BlobAudience BlobAudience::DefaultAudience(_internal::StorageDefaultAudience); BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateCsvTextOptions( diff --git a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp index a7e4fd2068..c55fd9a13a 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp @@ -5,6 +5,7 @@ #include "azure/storage/blobs/blob_batch.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include @@ -85,8 +86,8 @@ namespace Azure { namespace Storage { namespace Blobs { m_serviceUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); std::unique_ptr tokenAuthPolicy; + Azure::Core::Credentials::TokenRequestContext tokenContext; { - Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) @@ -110,7 +111,10 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; - pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy); + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + serviceUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); diff --git a/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp new file mode 100644 index 0000000000..eae34eb64e --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "session_authentication_policy.hpp" + +#include "token_credential_session_provider.hpp" + +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + namespace { + const Azure::Core::Context::Key SessionContextKey; + } + + class SessionSigningPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + explicit SessionSigningPolicy(std::string accountName) : m_accountName(std::move(accountName)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + SessionTokenInfo session; + if (context.TryGetValue(SessionContextKey, session)) + { + request.RemoveHeader(Azure::Storage::_internal::HttpHeaderAuthorization); + request.SetHeader( + Azure::Storage::_internal::HttpHeaderAuthorization, + "Session " + session.Token + ":" + + Azure::Storage::_internal::SharedKeyPolicy::GetSignature( + request, m_accountName, session.Key)); + } + return nextPolicy.Send(request, context); + } + + private: + std::string m_accountName; + }; + + class SessionAuthenticationPolicy final + : public Azure::Storage::_internal::StorageBearerTokenAuthenticationPolicy { + public: + SessionAuthenticationPolicy( + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + std::shared_ptr provider) + : StorageBearerTokenAuthenticationPolicy( + std::move(credential), + std::move(tokenRequestContext), + enableTenantDiscovery), + m_provider(provider->m_provider) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + private: + std::shared_ptr m_provider; + + std::unique_ptr AuthorizeAndSendRequest( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, + Azure::Core::Context const& context) const override + { + if (!m_provider->IsRequestEligible(request)) + { + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + + auto session = m_provider->GetSession(request, context); + if (!session.HasValue()) + { + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + + auto sessionContext = context.WithValue(SessionContextKey, SessionTokenInfo(session.Value())); + auto response = nextPolicy.Send(request, sessionContext); + if (response->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Unauthorized) + { + return response; + } + + m_provider->InvalidateSession(request, session.Value()); + request.RemoveHeader(Azure::Storage::_internal::HttpHeaderAuthorization); + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + }; + + TokenAuthenticationPolicies CreateTokenAuthenticationPolicies( + const std::string& clientUrl, + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + const BlobClientOptions& options) + { + const auto createBearerPolicy = [&credential, + &tokenRequestContext, + enableTenantDiscovery]() -> TokenAuthenticationPolicies { + TokenAuthenticationPolicies policies; + policies.TokenAuthPolicy + = std::make_unique( + std::move(credential), std::move(tokenRequestContext), enableTenantDiscovery); + return policies; + }; + + if (options.Session.Mode != SessionMode::Enabled) + { + return createBearerPolicy(); + } + + auto provider = options.Session.Provider; + auto accountName = options.Session.AccountName; + if (!provider || accountName.empty()) + { + auto urlParts = Azure::Storage::_internal::ParseStorageUrl(Azure::Core::Url(clientUrl)); + if (!urlParts.HasValue()) + { + throw std::invalid_argument( + "SessionOptions.AccountName and the Blob service URL could not be determined from the " + "client URL. Specify both SessionOptions.AccountName and SessionOptions.Provider for " + "custom endpoints."); + } + if (!accountName.empty() && accountName != urlParts.Value().AccountName) + { + throw std::invalid_argument( + "SessionOptions.AccountName does not match the account name in the client URL."); + } + accountName = std::move(urlParts.Value().AccountName); + if (!provider) + { + provider = std::make_shared( + urlParts.Value().ServiceUrl, credential, options); + } + } + TokenAuthenticationPolicies policies; + policies.TokenAuthPolicy = std::make_unique( + std::move(credential), + std::move(tokenRequestContext), + enableTenantDiscovery, + std::move(provider)); + policies.FinalAuthPolicy = std::make_unique(std::move(accountName)); + return policies; + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp new file mode 100644 index 0000000000..2662ba1116 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + struct TokenAuthenticationPolicies final + { + std::unique_ptr TokenAuthPolicy; + std::unique_ptr FinalAuthPolicy; + }; + + TokenAuthenticationPolicies CreateTokenAuthenticationPolicies( + const std::string& clientUrl, + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + const BlobClientOptions& options); + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp new file mode 100644 index 0000000000..b43994692b --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "token_credential_session_provider.hpp" + +#include "package_version.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { +constexpr auto SessionRefreshLeadTime = std::chrono::seconds(30); +constexpr auto SessionRefreshSuppression = std::chrono::seconds(30); +constexpr auto SessionFailureCooldown = std::chrono::minutes(5); + +} // namespace + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + std::string TokenCredentialSessionProvider::GetContainerUrl( + const Azure::Core::Http::Request& request) const + { + auto url = request.GetUrl(); + const auto& path = url.GetPath(); + const auto& servicePath = m_serviceUrl.GetPath(); + size_t containerStart = 0; + if (!servicePath.empty() && path.compare(0, servicePath.size(), servicePath) == 0) + { + containerStart = servicePath.size(); + } + const auto getFirstPathSegment = [](const std::string& value, size_t start) { + while (start < value.size() && value[start] == '/') + { + ++start; + } + const auto end = value.find('/', start); + return value.substr(start, end - start); + }; + const auto container = getFirstPathSegment(path, containerStart); + if (container.empty()) + { + return {}; + } + auto containerUrl = m_serviceUrl; + containerUrl.AppendPath(container); + return containerUrl.GetAbsoluteUrl(); + } + + std::shared_ptr + TokenCredentialSessionProvider::GetCacheEntry(const std::string& containerUrl) + { + std::lock_guard lock(m_cacheMutex); + auto& entry = m_cache[containerUrl]; + if (!entry) + { + entry = std::make_shared(); + } + return entry; + } + + SessionTokenInfo TokenCredentialSessionProvider::CreateSession( + const std::string& containerUrl, + const Azure::Core::Context& context) + { + _internal::XmlWriter writer; + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "CreateSessionRequest"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "AuthenticationType", "HMAC"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::End}); + auto xmlBody = writer.GetDocument(); + + Azure::Core::IO::MemoryBodyStream body( + reinterpret_cast(xmlBody.data()), xmlBody.size()); + Azure::Core::Url url(containerUrl); + url.AppendQueryParameter("restype", "container"); + url.AppendQueryParameter("comp", "session"); + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, url, &body); + request.SetHeader(_internal::HttpHeaderContentType, "application/xml; charset=UTF-8"); + request.SetHeader(_internal::HttpHeaderContentLength, std::to_string(body.Length())); + + auto response = m_pipeline->Send(request, context); + if (response->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Created) + { + throw StorageException::CreateFromResponse(std::move(response)); + } + + SessionTokenInfo session; + const auto& responseBody = response->GetBody(); + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + std::string currentTag; + bool expirationFound = false; + while (true) + { + const auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + if (node.Type == _internal::XmlNodeType::StartTag) + { + currentTag = node.Name; + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (currentTag == "SessionToken") + { + session.Token = node.Value; + } + else if (currentTag == "SessionKey") + { + session.Key = node.Value; + } + else if (currentTag == "Expiration") + { + session.ExpiresOn = Azure::DateTime::Parse( + node.Value, + node.Value.find(',') != std::string::npos ? Azure::DateTime::DateFormat::Rfc1123 + : Azure::DateTime::DateFormat::Rfc3339); + expirationFound = true; + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + currentTag.clear(); + } + } + if (session.Token.empty() || session.Key.empty() || !expirationFound) + { + throw std::runtime_error("Create Session response was missing required session fields."); + } + return session; + } + + TokenCredentialSessionProvider::TokenCredentialSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options) + { + if (!options.Session.AccountName.empty()) + { + m_serviceUrl = Azure::Core::Url(serviceUrl); + } + else + { + auto urlParts = Azure::Storage::_internal::ParseStorageUrl(Azure::Core::Url(serviceUrl)); + if (!urlParts.HasValue()) + { + throw std::invalid_argument( + "SessionOptions.AccountName could not be determined from the provided service URL."); + } + m_serviceUrl = Azure::Core::Url(urlParts.Value().ServiceUrl); + } + _internal::BuildStoragePipelineOptions pipelineOptions; + pipelineOptions.PackageName = _internal::BlobServicePackageName; + pipelineOptions.PackageVersion = PackageVersion::ToString(); + pipelineOptions.ApiVersion = Blobs::_detail::ApiVersion; + Azure::Core::Credentials::TokenRequestContext tokenContext; + tokenContext.Scopes.emplace_back( + options.Audience.HasValue() + ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) + : _internal::StorageScope); + pipelineOptions.TokenAuthPolicy + = std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>( + std::move(credential), tokenContext, options.EnableTenantDiscovery); + m_pipeline = std::make_shared( + _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); + } + + bool TokenCredentialSessionProvider::IsRequestEligible( + const Azure::Core::Http::Request& request) const + { + auto requestHost = request.GetUrl().GetHost(); + const auto headers = request.GetHeaders(); + const auto hostHeader = headers.find("Host"); + if (hostHeader != headers.end()) + { + requestHost + = Azure::Core::Url(request.GetUrl().GetScheme() + "://" + hostHeader->second).GetHost(); + } + if (requestHost != m_serviceUrl.GetHost()) + { + return false; + } + if (request.GetMethod() != Azure::Core::Http::HttpMethod::Get) + { + return false; + } + const auto query = request.GetUrl().GetQueryParameters(); + if (query.count("comp") != 0 || query.count("restype") != 0) + { + return false; + } + if (request.GetHeaders().count("x-ms-structured-body") != 0) + { + return false; + } + const auto containerUrl = GetContainerUrl(request); + return !containerUrl.empty() + && request.GetUrl().GetPath().size() > Azure::Core::Url(containerUrl).GetPath().size(); + } + + Azure::Nullable TokenCredentialSessionProvider::GetSession( + const Azure::Core::Http::Request& request, + const Azure::Core::Context& context) + { + const auto containerUrl = GetContainerUrl(request); + if (containerUrl.empty()) + { + return {}; + } + auto entry = GetCacheEntry(containerUrl); + auto now = Azure::DateTime::clock::now(); + { + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() + && now < entry->Current.Value().ExpiresOn - SessionRefreshLeadTime) + { + return entry->Current; + } + if (entry->CooldownUntil > now) + { + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + return {}; + } + if (entry->LastRefreshStarted + SessionRefreshSuppression > now) + { + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + return {}; + } + entry->LastRefreshStarted = now; + } + + try + { + auto session = CreateSession(containerUrl, context); + std::lock_guard lock(entry->Mutex); + if (!entry->Current.HasValue() || entry->Current.Value().ExpiresOn < session.ExpiresOn) + { + entry->Current = session; + } + return session; + } + catch (const StorageException& e) + { + const auto statusCode = static_cast(e.StatusCode); + const bool softFailure = statusCode >= 500 || statusCode == 403 + || (statusCode == 400 && e.ErrorCode == "FeatureNotEnabled"); + now = Azure::DateTime::clock::now(); + std::lock_guard lock(entry->Mutex); + if (softFailure) + { + const auto cooldownUntil = now + SessionFailureCooldown; + if (entry->CooldownUntil < cooldownUntil) + { + entry->CooldownUntil = cooldownUntil; + } + } + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + if (!softFailure) + { + throw; + } + return {}; + } + catch (const std::exception&) + { + now = Azure::DateTime::clock::now(); + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + throw; + } + } + + void TokenCredentialSessionProvider::InvalidateSession( + const Azure::Core::Http::Request& request, + const SessionTokenInfo& session) + { + const auto containerUrl = GetContainerUrl(request); + if (containerUrl.empty()) + { + return; + } + auto entry = GetCacheEntry(containerUrl); + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() && entry->Current.Value().Token == session.Token) + { + entry->Current.Reset(); + } + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp new file mode 100644 index 0000000000..9292fa13cc --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace Azure { namespace Core { namespace Http { namespace _internal { + class HttpPipeline; +}}}} // namespace Azure::Core::Http::_internal + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + struct SessionTokenInfo final + { + std::string Token; + std::string Key; + Azure::DateTime ExpiresOn; + }; + + class TokenCredentialSessionProvider final { + public: + TokenCredentialSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options); + + Azure::Nullable GetSession( + const Azure::Core::Http::Request& request, + const Azure::Core::Context& context); + + void InvalidateSession( + const Azure::Core::Http::Request& request, + const SessionTokenInfo& session); + + bool IsRequestEligible(const Azure::Core::Http::Request& request) const; + + private: + struct CacheEntry + { + std::mutex Mutex; + Azure::Nullable Current; + Azure::DateTime CooldownUntil; + Azure::DateTime LastRefreshStarted; + }; + + Azure::Core::Url m_serviceUrl; + std::shared_ptr m_pipeline; + std::mutex m_cacheMutex; + std::map> m_cache; + + std::string GetContainerUrl(const Azure::Core::Http::Request& request) const; + std::shared_ptr GetCacheEntry(const std::string& containerUrl); + SessionTokenInfo CreateSession( + const std::string& containerUrl, + const Azure::Core::Context& context); + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index ceed2c7422..0aca417c06 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -59,6 +59,7 @@ set( inc/azure/storage/common/internal/storage_retry_policy.hpp inc/azure/storage/common/internal/storage_service_version_policy.hpp inc/azure/storage/common/internal/storage_switch_to_secondary_policy.hpp + inc/azure/storage/common/internal/storage_url.hpp inc/azure/storage/common/internal/structured_message_decoding_stream.hpp inc/azure/storage/common/internal/structured_message_encoding_stream.hpp inc/azure/storage/common/internal/structured_message_helper.hpp @@ -85,6 +86,7 @@ set( src/storage_pipeline.cpp src/storage_retry_policy.cpp src/storage_switch_to_secondary_policy.cpp + src/storage_url.cpp src/structured_message_decoding_stream.cpp src/structured_message_encoding_stream.cpp src/structured_message_helper.cpp diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp index 3a0f608953..bd18eb39be 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp @@ -32,13 +32,18 @@ namespace Azure { namespace Storage { namespace _internal { Core::Context const& context) const override { request.SetHeader( - "Authorization", "SharedKey " + m_credential->AccountName + ":" + GetSignature(request)); + "Authorization", + "SharedKey " + m_credential->AccountName + ":" + + GetSignature(request, m_credential->AccountName, m_credential->GetAccountKey())); return nextPolicy.Send(request, context); } - private: - std::string GetSignature(const Core::Http::Request& request) const; + static std::string GetSignature( + const Core::Http::Request& request, + const std::string& accountName, + const std::string& signingKey); + private: std::shared_ptr m_credential; }; diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp index ca32557526..7d5e11585e 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp @@ -10,7 +10,7 @@ namespace Azure { namespace Storage { namespace _internal { - class StorageBearerTokenAuthenticationPolicy final + class StorageBearerTokenAuthenticationPolicy : public Core::Http::Policies::_internal::BearerTokenAuthenticationPolicy { public: /** @@ -37,6 +37,15 @@ namespace Azure { namespace Storage { namespace _internal { return std::unique_ptr(new StorageBearerTokenAuthenticationPolicy(*this)); } + protected: + void AuthorizeRequest(Azure::Core::Http::Request& request, Azure::Core::Context const& context) + const; + + std::unique_ptr AuthorizeAndSendRequest( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, + Azure::Core::Context const& context) const override; + private: struct SafeTenantId { @@ -66,11 +75,6 @@ namespace Azure { namespace Storage { namespace _internal { mutable SafeTenantId m_safeTenantId; bool m_enableTenantDiscovery; - std::unique_ptr AuthorizeAndSendRequest( - Azure::Core::Http::Request& request, - Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, - Azure::Core::Context const& context) const override; - bool AuthorizeRequestOnChallenge( std::string const& challenge, Azure::Core::Http ::Request& request, diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp new file mode 100644 index 0000000000..f8233d572a --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include + +namespace Azure { namespace Storage { namespace _internal { + + struct StorageUrlParts final + { + std::string Service; + std::string AccountName; + std::string ServiceUrl; + Azure::Nullable ContainerName; + Azure::Nullable ContainerUrl; + }; + + Azure::Nullable ParseStorageUrl(const Azure::Core::Url& url); + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/shared_key_policy.cpp b/sdk/storage/azure-storage-common/src/shared_key_policy.cpp index 3a8d7e0b37..265e7d6011 100644 --- a/sdk/storage/azure-storage-common/src/shared_key_policy.cpp +++ b/sdk/storage/azure-storage-common/src/shared_key_policy.cpp @@ -92,7 +92,10 @@ bool comparator(const std::string& lhs, const std::string& rhs) namespace Azure { namespace Storage { namespace _internal { - std::string SharedKeyPolicy::GetSignature(const Core::Http::Request& request) const + std::string SharedKeyPolicy::GetSignature( + const Core::Http::Request& request, + const std::string& accountName, + const std::string& signingKey) { std::string string_to_sign; string_to_sign += request.GetMethod().ToString() + "\n"; @@ -146,7 +149,7 @@ namespace Azure { namespace Storage { namespace _internal { ordered_kv.clear(); // canonicalized resource - string_to_sign += "/" + m_credential->AccountName + "/" + request.GetUrl().GetPath() + "\n"; + string_to_sign += "/" + accountName + "/" + request.GetUrl().GetPath() + "\n"; for (const auto& query : request.GetUrl().GetQueryParameters()) { std::string key = Azure::Core::_internal::StringExtensions::ToLower(query.first); @@ -164,6 +167,6 @@ namespace Azure { namespace Storage { namespace _internal { return Azure::Core::Convert::Base64Encode(_internal::HmacSha256( std::vector(string_to_sign.begin(), string_to_sign.end()), - Azure::Core::Convert::Base64Decode(m_credential->GetAccountKey()))); + Azure::Core::Convert::Base64Decode(signingKey))); } }}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp b/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp index 9f39fb1e71..cd617e5c8f 100644 --- a/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp +++ b/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp @@ -13,6 +13,14 @@ namespace Azure { namespace Storage { namespace _internal { Azure::Core::Http::Request& request, Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, Azure::Core::Context const& context) const + { + AuthorizeRequest(request, context); + return nextPolicy.Send(request, context); + } + + void StorageBearerTokenAuthenticationPolicy::AuthorizeRequest( + Azure::Core::Http::Request& request, + Azure::Core::Context const& context) const { std::string tenantId = m_safeTenantId.Get(); if (!tenantId.empty() || !m_enableTenantDiscovery) @@ -22,7 +30,6 @@ namespace Azure { namespace Storage { namespace _internal { tokenRequestContext.TenantId = tenantId; AuthenticateAndAuthorizeRequest(request, tokenRequestContext, context); } - return nextPolicy.Send(request, context); } bool StorageBearerTokenAuthenticationPolicy::AuthorizeRequestOnChallenge( diff --git a/sdk/storage/azure-storage-common/src/storage_url.cpp b/sdk/storage/azure-storage-common/src/storage_url.cpp new file mode 100644 index 0000000000..2af0a6e9da --- /dev/null +++ b/sdk/storage/azure-storage-common/src/storage_url.cpp @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/storage_url.hpp" + +#include + +namespace { + +std::string StripAccountSuffix(std::string accountName) +{ + while (true) + { + const auto originalSize = accountName.size(); + for (const std::string suffix : + {"-ipv6", "-dualstack", "-secondary", "-internetrouting", "-microsoftrouting"}) + { + if (accountName.size() > suffix.size() + && accountName.compare(accountName.size() - suffix.size(), suffix.size(), suffix) == 0) + { + accountName.resize(accountName.size() - suffix.size()); + break; + } + } + if (accountName.size() == originalSize) + { + break; + } + } + return accountName; +} + +} // namespace + +namespace Azure { namespace Storage { namespace _internal { + + Azure::Nullable ParseStorageUrl(const Azure::Core::Url& url) + { + const auto hostname = Azure::Core::_internal::StringExtensions::ToLower(url.GetHost()); + std::string endpoint; + for (const std::string candidate : + {".preprod.core.windows.net", + ".core.windows.net", + ".core.chinacloudapi.cn", + ".core.usgovcloudapi.net"}) + { + if (hostname.find(candidate) != std::string::npos) + { + endpoint = candidate; + break; + } + } + std::string service; + std::string accountName; + if (!endpoint.empty()) + { + for (const std::string candidate : {"blob", "dfs", "file", "queue", "table", "web"}) + { + const auto suffix = "." + candidate + endpoint; + if (hostname.size() > suffix.size() + && hostname.compare(hostname.size() - suffix.size(), suffix.size(), suffix) == 0) + { + service = candidate; + accountName = hostname.substr(0, hostname.size() - suffix.size()); + break; + } + } + } + else + { + for (const std::string candidate : {"blob", "dfs", "file", "queue", "table", "web"}) + { + const auto suffix = "." + candidate + ".storage.azure.net"; + if (hostname.size() <= suffix.size() + || hostname.compare(hostname.size() - suffix.size(), suffix.size(), suffix) != 0) + { + continue; + } + + service = candidate; + accountName = hostname.substr(0, hostname.find('.')); + break; + } + } + + accountName = StripAccountSuffix(std::move(accountName)); + if (service.empty() || accountName.empty()) + { + return {}; + } + + StorageUrlParts result; + result.Service = std::move(service); + result.AccountName = std::move(accountName); + + auto serviceUrl = url; + serviceUrl.SetPath({}); + serviceUrl.SetQueryParameters({}); + result.ServiceUrl = serviceUrl.GetAbsoluteUrl(); + + const auto& path = url.GetPath(); + if (!path.empty()) + { + const auto containerNameEnd = path.find('/', 1); + result.ContainerName + = containerNameEnd == std::string::npos ? path : path.substr(0, containerNameEnd); + + auto containerUrl = serviceUrl; + containerUrl.SetPath(result.ContainerName.Value()); + result.ContainerUrl = containerUrl.GetAbsoluteUrl(); + } + return result; + } + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index 5616dc067f..ad3092c6c3 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -4,6 +4,8 @@ ### Features Added +- Added support for persisted session authentication for eligible file download requests. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index ed6247bcfd..f40c9fd73d 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -87,6 +87,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using GetServicePropertiesOptions = Blobs::GetServicePropertiesOptions; using SetServicePropertiesOptions = Blobs::SetServicePropertiesOptions; using EncryptionKey = Blobs::EncryptionKey; + using SessionOptions = Blobs::SessionOptions; /** * Configures whether to do content validation for file uploads and downloads. @@ -223,6 +224,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable DownloadValidationOptions; + + /** + * @brief Configures persisted session authentication for Blob endpoint download requests. + */ + SessionOptions Session; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 0422c0f6d2..525a0a3eeb 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -121,30 +121,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - - auto renamedBlobClient = Blobs::BlobClient( - _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + auto renamedBlobClient = m_blobClient; + renamedBlobClient.m_blobUrl = _detail::GetBlobUrlFromUrl(destinationDfsUrl); auto renamedFileClient = DataLakeFileClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), @@ -194,31 +172,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - - auto renamedBlobClient = Blobs::BlobClient( - _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + auto renamedBlobClient = m_blobClient; + renamedBlobClient.m_blobUrl = _detail::GetBlobUrlFromUrl(destinationDfsUrl); auto renamedDirectoryClient = DataLakeDirectoryClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index 04085d3884..4cbce13e8b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -384,31 +384,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - auto renamedBlobClient = Blobs::BlobClient( _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + m_blobContainerClient.m_pipeline, + m_blobContainerClient.m_clientConfiguration); auto renamedFileClient = DataLakeFileClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), @@ -458,31 +437,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - auto renamedBlobClient = Blobs::BlobClient( _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + m_blobContainerClient.m_pipeline, + m_blobContainerClient.m_clientConfiguration); auto renamedDirectoryClient = DataLakeDirectoryClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 3e39529e2a..5a555b3665 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -109,6 +109,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam blobOptions.ApiVersion = options.ApiVersion; blobOptions.CustomerProvidedKey = options.CustomerProvidedKey; blobOptions.EnableTenantDiscovery = options.EnableTenantDiscovery; + blobOptions.Session = options.Session; if (options.Audience.HasValue()) { blobOptions.Audience = Blobs::BlobAudience(options.Audience.Value().ToString()); From c2cbcff4ceb285d2d134aa0c245b3bb9fed37cca Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:14:24 +1100 Subject: [PATCH 07/13] Improve partitioned transfers and add data locality support Reduce default initial transfer sizes to 4 MiB and overlap remaining downloads with consumption of the initial response for Blob and Files transfers. Add Blob and DataLake layout APIs, download hints, and locality routing. Keep managed locality routing disabled by default while supporting explicit Auto and Enabled selection. Include both features' changelogs and layout member documentation. Expose the layout ranges directly as vectors in BlobLayoutInfo and FileLayoutInfo, without redundant range-container wrapper types. Preserve the explicit false legal-hold initializer through the generation customization and regenerated BlobProperties declaration. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 11 + .../azure-storage-blobs/CMakeLists.txt | 2 + .../inc/azure/storage/blobs/blob_client.hpp | 11 + .../inc/azure/storage/blobs/blob_options.hpp | 60 ++- .../azure/storage/blobs/blob_responses.hpp | 53 ++ .../inc/azure/storage/blobs/rest_client.hpp | 98 ++++ .../azure-storage-blobs/src/blob_client.cpp | 419 ++++++++++++---- .../src/blob_container_client.cpp | 3 + .../src/blob_responses.cpp | 7 + .../src/blob_service_client.cpp | 3 + .../src/private/data_locality.cpp | 163 +++++++ .../src/private/data_locality.hpp | 80 +++ .../azure-storage-blobs/src/rest_client.cpp | 454 ++++++++++++++++++ .../azure-storage-blobs/swagger/README.md | 191 +++++++- .../azure-storage-common/CMakeLists.txt | 2 + .../internal/storage_data_locality_policy.hpp | 37 ++ .../common/internal/storage_pipeline.hpp | 1 + .../src/storage_data_locality_policy.cpp | 39 ++ .../src/storage_pipeline.cpp | 5 + .../azure-storage-files-datalake/CHANGELOG.md | 11 + .../files/datalake/datalake_file_client.hpp | 11 + .../files/datalake/datalake_options.hpp | 33 +- .../files/datalake/datalake_responses.hpp | 48 ++ .../src/datalake_file_client.cpp | 30 ++ .../src/datalake_path_client.cpp | 57 +-- .../src/datalake_responses.cpp | 9 + .../src/datalake_utilities.cpp | 62 +++ .../src/private/datalake_utilities.hpp | 5 + .../src/share_file_client.cpp | 105 ++-- 29 files changed, 1838 insertions(+), 172 deletions(-) create mode 100644 sdk/storage/azure-storage-blobs/src/private/data_locality.cpp create mode 100644 sdk/storage/azure-storage-blobs/src/private/data_locality.hpp create mode 100644 sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp create mode 100644 sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index e06660cf58..1213ed93cf 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -5,6 +5,13 @@ ### Features Added - Added support for persisted session authentication for eligible blob download requests. +- Added `BlobClient::GetLayout` and related options and response types for retrieving locality + endpoints, byte ranges, and blob properties. Ranges are exposed directly through + `Models::BlobLayoutInfo::Ranges` as a vector of `Models::BlobLayoutRange`. +- Added `DownloadBlobOptions::LayoutEndpoint` for one-shot download routing and + `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for + managed downloads. +- Added `Models::DownloadHint` and `Models::DownloadBlobDetails::DownloadHint`. ### Breaking Changes @@ -18,6 +25,10 @@ ### Other Changes +- Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to + 4 MiB, matching the default subsequent chunk size. Explicitly configured + `InitialChunkSize` and `SingleUploadThreshold` values are still respected. + ## 12.19.0 (2026-09-29) ### Features Added diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 8eee037487..7b1f5203b8 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -75,6 +75,8 @@ set( src/page_blob_client.cpp src/private/avro_parser.cpp src/private/avro_parser.hpp + src/private/data_locality.cpp + src/private/data_locality.hpp src/private/package_version.hpp src/private/session_authentication_policy.cpp src/private/session_authentication_policy.hpp diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index 2a1e90ad64..ec3511e048 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -425,6 +425,17 @@ namespace Azure { namespace Storage { namespace Blobs { const GetAccountInfoOptions& options = GetAccountInfoOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Gets the data locality layout for this blob. + * + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A paged response containing the blob's data locality layout. + */ + BlobLayoutPagedResponse GetLayout( + const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + protected: /** @brief Blob Url */ Azure::Core::Url m_blobUrl; diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index d9ae656deb..1693edf2e7 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -89,6 +89,27 @@ namespace Azure { namespace Storage { namespace Blobs { std::shared_ptr Provider; }; + /** + * @brief Determines whether locality-aware routing is used for managed downloads. + */ + enum class LayoutAwareRouting + { + /** + * @brief The client library determines whether locality-aware routing is enabled. + */ + Auto, + + /** + * @brief Locality-aware routing is disabled. + */ + Disabled, + + /** + * @brief Locality-aware routing is enabled. + */ + Enabled, + }; + /** * @brief Audiences available for blob service * @@ -911,6 +932,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief Optional. The layout endpoint to use for this download. + * + * When set, the request is sent to this endpoint while preserving the client endpoint in the + * Host header. + */ + std::string LayoutEndpoint; }; /** @@ -933,7 +962,7 @@ namespace Azure { namespace Storage { namespace Blobs { * downloaded in a single request. Blobs larger than this limit will continue being downloaded * in chunks of size ChunkSize. */ - int64_t InitialChunkSize = 256 * 1024 * 1024; + int64_t InitialChunkSize = 4 * 1024 * 1024; /** * @brief The maximum number of bytes in a single request. @@ -950,6 +979,12 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief Determines whether locality-aware routing is used for the parallel range requests. + * Disabled by default. + */ + Blobs::LayoutAwareRouting LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; }; /** @@ -1179,7 +1214,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Blob smaller than this will be uploaded with a single upload operation. This value * cannot be larger than 5000 MiB. */ - int64_t SingleUploadThreshold = 256 * 1024 * 1024; + int64_t SingleUploadThreshold = 4 * 1024 * 1024; /** * @brief The maximum number of bytes in a single request. This value cannot be larger than @@ -1922,6 +1957,27 @@ namespace Azure { namespace Storage { namespace Blobs { { }; + /** + * @brief Optional parameters for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + struct GetBlobLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + BlobAccessConditions AccessConditions; + }; + namespace _detail { inline std::string TagsToString(const std::map& tags) { diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index de54ac6a48..4521c85d1e 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -29,6 +29,38 @@ namespace Azure { namespace Storage { namespace Models { + /** + * @brief Describes a range in a blob's data locality layout. + */ + struct BlobLayoutRange final + { + /** + * @brief The range of bytes served by the endpoint. + */ + Azure::Core::Http::HttpRange Range; + + /** + * @brief The host and port of the endpoint serving this range. + */ + std::string Endpoint; + }; + + /** + * @brief A page of blob layout information and the blob properties returned with it. + */ + struct BlobLayoutInfo final + { + /** + * @brief The ranges in this page of the blob's data locality layout. + */ + std::vector Ranges; + + /** + * @brief The blob properties returned with the layout. + */ + BlobProperties Properties; + }; + /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::DownloadTo. */ @@ -482,6 +514,27 @@ namespace Azure { namespace Storage { friend class Azure::Core::PagedResponse; }; + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + class BlobLayoutPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and blob properties in this page. + */ + Models::BlobLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_blobClient; + GetBlobLayoutOptions m_operationOptions; + + friend class Blobs::BlobClient; + friend class Azure::Core::PagedResponse; + }; + namespace _detail { void ParseListBlobsResult(Models::_detail::ListBlobsResult& result); } // namespace _detail diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index c70f815c71..804072aa25 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -1450,6 +1450,19 @@ namespace Azure { namespace Storage { namespace Blobs { std::string ContentType; }; } // namespace _detail + /** + * @brief Indicates the download hint for the blob. + */ + class DownloadHint final : public Core::_internal::ExtendableEnumeration { + public: + /** Constructs a new DownloadHint instance */ + DownloadHint() = default; + /** Constructs a new DownloadHint from a string. */ + explicit DownloadHint(std::string value) : ExtendableEnumeration(std::move(value)) {} + + /** Constant value of type DownloadHint: Layout */ + AZ_STORAGE_BLOBS_DLLEXPORT const static DownloadHint Layout; + }; /** * @brief Detailed information of the downloaded blob. */ @@ -1614,6 +1627,10 @@ namespace Azure { namespace Storage { namespace Blobs { * The underlying tier of a smart tier blob. Only returned if the blob is in Smart tier. */ Nullable SmartAccessTier; + /** + * Indicates the download hint for the blob. + */ + Nullable DownloadHint; }; /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::Download. @@ -1880,6 +1897,65 @@ namespace Azure { namespace Storage { namespace Blobs { */ bool Deleted = true; }; + namespace _detail { + struct BlobLayoutRangesItem final + { + /** + * The start byte offset of the range. + */ + std::int64_t Start = std::int64_t(); + /** + * The end byte offset of the range. + */ + std::int64_t End = std::int64_t(); + /** + * Index into the Endpoints array indicating which endpoint serves this range. + */ + std::int32_t EndpointIndex = std::int32_t(); + }; + struct BlobLayoutEndpointsItem final + { + /** + * The index of the endpoint, referenced by Range elements. + */ + std::int32_t Index = std::int32_t(); + /** + * The host:port of the endpoint. + */ + std::string Value; + }; + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + struct BlobLayout final + { + /** + * Array of BlobLayout-RangesItem. + */ + std::vector Ranges; + /** + * Array of BlobLayout-EndpointsItem. + */ + std::vector Endpoints; + /** + * The continuation marker used for this request. + */ + std::string Marker; + /** + * If the number of ranges exceeds MaxResults, a NextMarker is returned for use in + * subsequent requests to continue listing. + */ + std::string NextMarker; + /** + * The maximum number of ranges to return per request. + */ + std::int32_t MaxResults = std::int32_t(); + /** + * Properties of a blob. + */ + BlobProperties Properties; + }; + } // namespace _detail /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::Undelete. */ @@ -3658,6 +3734,28 @@ namespace Azure { namespace Storage { namespace Blobs { const Core::Url& url, const DeleteBlobOptions& options, const Core::Context& context); + struct GetBlobLayoutOptions final + { + Nullable Snapshot; + Nullable VersionId; + Nullable Marker; + Nullable MaxResults; + Nullable Range; + Nullable LeaseId; + Nullable IfTags; + Nullable IfModifiedSince; + Nullable IfUnmodifiedSince; + ETag IfMatch; + ETag IfNoneMatch; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; + }; + static Response GetLayout( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetBlobLayoutOptions& options, + const Core::Context& context); struct UndeleteBlobOptions final { }; diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 2d020b3ac6..8bbb13ab20 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -6,11 +6,13 @@ #include "azure/storage/blobs/append_blob_client.hpp" #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" +#include "private/data_locality.hpp" #include "private/package_version.hpp" #include "private/session_authentication_policy.hpp" #include #include +#include #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +28,10 @@ #include #include +#include +#include +#include +#include namespace Azure { namespace Storage { namespace Blobs { @@ -32,6 +39,127 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context::Key const DataLakeInteroperabilityExtraOptionsKey; } + namespace { + bool HasDataLocalityHint(const Models::DownloadBlobDetails& details) + { + return details.DownloadHint.HasValue() + && Azure::Core::_internal::StringExtensions::ToLower( + details.DownloadHint.Value().ToString()) + == Models::DownloadHint::Layout.ToString(); + } + + bool IsLayoutAwareRoutingEnabled(LayoutAwareRouting value) + { + return value != LayoutAwareRouting::Disabled; + } + + void CompleteBlobPropertiesFromResponse( + Models::BlobProperties& properties, + const Azure::Core::Http::RawResponse& rawResponse) + { + if (properties.AccessTier.HasValue() && !properties.IsAccessTierInferred.HasValue()) + { + properties.IsAccessTierInferred = false; + } + if (properties.VersionId.HasValue() && !properties.IsCurrentVersion.HasValue()) + { + properties.IsCurrentVersion = false; + } + if (properties.CopyStatus.HasValue() && !properties.IsIncrementalCopy.HasValue()) + { + properties.IsIncrementalCopy = false; + } + if (properties.BlobType == Models::BlobType::AppendBlob && !properties.IsSealed.HasValue()) + { + properties.IsSealed = false; + } + + std::map> orPropertiesMap; + for (auto i = rawResponse.GetHeaders().lower_bound("x-ms-or-"); + i != rawResponse.GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; + ++i) + { + const std::string& header = i->first; + auto underscorePos = header.find('_', 8); + if (underscorePos == std::string::npos) + { + continue; + } + std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); + Models::ObjectReplicationRule rule; + rule.RuleId = header.substr(underscorePos + 1); + rule.ReplicationStatus = Models::ObjectReplicationStatus(i->second); + orPropertiesMap[policyId].emplace_back(std::move(rule)); + } + for (auto& property : orPropertiesMap) + { + Models::ObjectReplicationPolicy policy; + policy.PolicyId = property.first; + policy.Rules = std::move(property.second); + properties.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); + } + } + + std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( + BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context) + { + auto refresher = [range = std::move(range), + eTag = std::move(eTag), + context = std::move(context), + blobClient = std::move(blobClient)]() { + try + { + GetBlobLayoutOptions options; + options.Range = range; + options.AccessConditions.IfMatch = eTag; + auto page = blobClient.GetLayout(options, context); + _detail::DataLocalityLayout layout; + for (; page.HasPage(); page.MoveToNextPage(context)) + { + if (!layout.ETag.HasValue()) + { + layout.ETag = page.Layout.Properties.ETag; + } + if (layout.BlobSize == 0) + { + layout.BlobSize = page.Layout.Properties.BlobSize; + } + for (auto& layoutRange : page.Layout.Ranges) + { + if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 + || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + layout.Ranges.push_back(std::move(layoutRange)); + } + } + std::sort( + layout.Ranges.begin(), + layout.Ranges.end(), + [](const Models::BlobLayoutRange& lhs, const Models::BlobLayoutRange& rhs) { + return lhs.Range.Offset < rhs.Range.Offset; + }); + return layout; + } + catch (const StorageException& exception) + { + if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest + || static_cast(exception.StatusCode) >= 500) + { + return _detail::DataLocalityLayout(); + } + throw; + } + }; + + return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); + } + } // namespace + BlobClient BlobClient::CreateFromConnectionString( const std::string& connectionString, const std::string& blobContainerName, @@ -65,6 +193,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::make_unique<_internal::SharedKeyPolicy>(credential); m_pipeline = std::make_shared( @@ -83,6 +212,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; { Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( @@ -113,6 +243,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); @@ -216,8 +347,14 @@ namespace Azure { namespace Storage { namespace Blobs { = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); } + std::string dataLocalityEndpoint; + const auto downloadContext = !options.LayoutEndpoint.empty() + ? _internal::WithDataLocalityEndpoint(context, options.LayoutEndpoint) + : context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) + ? context + : _internal::WithReplicaStatus(context); auto downloadResponse = _detail::BlobClient::Download( - *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + *m_pipeline, m_blobUrl, protocolLayerOptions, downloadContext); int64_t structuredContentLength = 0; if (isStructuredMessage) @@ -351,6 +488,83 @@ namespace Azure { namespace Storage { namespace Blobs { return downloadResponse; } + BlobLayoutPagedResponse BlobClient::GetLayout( + const GetBlobLayoutOptions& options, + const Azure::Core::Context& context) const + { + _detail::BlobClient::GetBlobLayoutOptions protocolLayerOptions; + protocolLayerOptions.Marker = options.ContinuationToken; + if (options.Range.HasValue()) + { + std::string range = "bytes=" + std::to_string(options.Range.Value().Offset) + "-"; + if (options.Range.Value().Length.HasValue()) + { + range += std::to_string( + options.Range.Value().Offset + options.Range.Value().Length.Value() - 1); + } + protocolLayerOptions.Range = std::move(range); + } + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + if (m_clientConfiguration.CustomerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 + = m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm + = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); + } + + auto response + = _detail::BlobClient::GetLayout(*m_pipeline, m_blobUrl, protocolLayerOptions, context); + BlobLayoutPagedResponse pagedResponse; + std::map endpoints; + for (const auto& endpoint : response.Value.Endpoints) + { + if (endpoint.Index < 0 || endpoint.Value.empty() + || !endpoints.emplace(endpoint.Index, endpoint.Value).second) + { + throw Azure::Core::RequestFailedException("Invalid blob layout endpoint."); + } + } + for (const auto& range : response.Value.Ranges) + { + const auto endpoint = endpoints.find(range.EndpointIndex); + if (range.Start < 0 || range.End < range.Start || endpoint == endpoints.end()) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + const auto rangeLength + = static_cast(range.End) - static_cast(range.Start) + 1; + if (rangeLength > static_cast((std::numeric_limits::max)())) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + Models::BlobLayoutRange layoutRange; + layoutRange.Range.Offset = range.Start; + layoutRange.Range.Length = static_cast(rangeLength); + layoutRange.Endpoint = endpoint->second; + pagedResponse.Layout.Ranges.push_back(std::move(layoutRange)); + } + pagedResponse.Layout.Properties = std::move(response.Value.Properties); + if (!pagedResponse.Layout.Properties.ETag.HasValue()) + { + pagedResponse.Layout.Properties.ETag = options.AccessConditions.IfMatch; + } + CompleteBlobPropertiesFromResponse(pagedResponse.Layout.Properties, *response.RawResponse); + + pagedResponse.m_blobClient = std::make_shared(*this); + pagedResponse.m_operationOptions = options; + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.NextMarker; + pagedResponse.RawResponse = std::move(response.RawResponse); + return pagedResponse; + } + Azure::Response BlobClient::DownloadTo( uint8_t* buffer, size_t bufferSize, @@ -393,6 +607,8 @@ namespace Azure { namespace Storage { namespace Blobs { blobRangeSize = blobSize; } firstChunkLength = (std::min)(firstChunkLength, blobRangeSize); + const int64_t remainingOffset = firstChunkOffset + firstChunkLength; + const int64_t remainingSize = blobRangeSize - firstChunkLength; if (static_cast(blobRangeSize) > (std::numeric_limits::max)() || static_cast(blobRangeSize) > bufferSize) @@ -401,13 +617,20 @@ namespace Azure { namespace Storage { namespace Blobs { "Buffer is not big enough, blob range size is " + std::to_string(blobRangeSize) + "."); } - int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( - buffer, static_cast(firstChunkLength), context); - if (bytesRead != firstChunkLength) + std::unique_ptr<_detail::DataLocalityLayoutState> dataLocalityState; + if (remainingSize > 0 && IsLayoutAwareRoutingEnabled(options.LayoutAwareRouting) + && HasDataLocalityHint(firstChunk.Value.Details)) { - throw Azure::Core::RequestFailedException("Error when reading body stream."); + Core::Http::HttpRange remainingRange; + remainingRange.Offset = remainingOffset; + remainingRange.Length = remainingSize; + dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState->WaitForInitialLayout(); + if (!dataLocalityState->HasLayout()) + { + dataLocalityState.reset(); + } } - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadBlobToResult ret; @@ -419,9 +642,8 @@ namespace Azure { namespace Storage { namespace Blobs { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadBlobOptions chunkOptions; @@ -430,11 +652,14 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - auto chunk = Download(chunkOptions, context); + const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; + auto chunk = Download(chunkOptions, chunkContext); int64_t bytesRead = chunk.Value.BodyStream->ReadToCount( buffer + (offset - firstChunkOffset), static_cast(chunkOptions.Range.Value().Length.Value()), - context); + chunkContext); if (bytesRead != chunkOptions.Range.Value().Length.Value()) { throw Azure::Core::RequestFailedException("Error when reading body stream."); @@ -442,20 +667,52 @@ namespace Azure { namespace Storage { namespace Blobs { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); - ret.Value.TransactionalContentHash.Reset(); + lastChunkResponse = std::move(chunk); } }; - int64_t remainingOffset = firstChunkOffset + firstChunkLength; - int64_t remainingSize = blobRangeSize - firstChunkLength; + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( + buffer, static_cast(firstChunkLength), context); + if (bytesRead != firstChunkLength) + { + throw Azure::Core::RequestFailedException("Error when reading body stream."); + } + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); + if (lastChunkResponse.HasValue()) + { + ret.Value.TransactionalContentHash.Reset(); + } ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = blobRangeSize; return ret; @@ -502,6 +759,23 @@ namespace Azure { namespace Storage { namespace Blobs { blobRangeSize = blobSize; } firstChunkLength = (std::min)(firstChunkLength, blobRangeSize); + const int64_t remainingOffset = firstChunkOffset + firstChunkLength; + const int64_t remainingSize = blobRangeSize - firstChunkLength; + + std::unique_ptr<_detail::DataLocalityLayoutState> dataLocalityState; + if (remainingSize > 0 && IsLayoutAwareRoutingEnabled(options.LayoutAwareRouting) + && HasDataLocalityHint(firstChunk.Value.Details)) + { + Core::Http::HttpRange remainingRange; + remainingRange.Offset = remainingOffset; + remainingRange.Length = remainingSize; + dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState->WaitForInitialLayout(); + if (!dataLocalityState->HasLayout()) + { + dataLocalityState.reset(); + } + } auto bodyStreamToFile = [](Azure::Core::IO::BodyStream& stream, _internal::FileWriter& fileWriter, @@ -525,8 +799,6 @@ namespace Azure { namespace Storage { namespace Blobs { }; _internal::FileWriter fileWriter(fileName); - bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadBlobToResult ret; @@ -538,9 +810,8 @@ namespace Azure { namespace Storage { namespace Blobs { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadBlobOptions chunkOptions; @@ -549,30 +820,60 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - auto chunk = Download(chunkOptions, context); + const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; + auto chunk = Download(chunkOptions, chunkContext); bodyStreamToFile( *(chunk.Value.BodyStream), fileWriter, offset - firstChunkOffset, chunkOptions.Range.Value().Length.Value(), - context); + chunkContext); if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); - ret.Value.TransactionalContentHash.Reset(); + lastChunkResponse = std::move(chunk); } }; - int64_t remainingOffset = firstChunkOffset + firstChunkLength; - int64_t remainingSize = blobRangeSize - firstChunkLength; + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); + firstChunk.Value.BodyStream.reset(); - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); + if (lastChunkResponse.HasValue()) + { + ret.Value.TransactionalContentHash.Reset(); + } ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = blobRangeSize; return ret; @@ -607,51 +908,7 @@ namespace Azure { namespace Storage { namespace Blobs { } auto response = _detail::BlobClient::GetProperties( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); - if (response.Value.AccessTier.HasValue() && !response.Value.IsAccessTierInferred.HasValue()) - { - response.Value.IsAccessTierInferred = false; - } - if (response.Value.VersionId.HasValue() && !response.Value.IsCurrentVersion.HasValue()) - { - response.Value.IsCurrentVersion = false; - } - if (response.Value.CopyStatus.HasValue() && !response.Value.IsIncrementalCopy.HasValue()) - { - response.Value.IsIncrementalCopy = false; - } - if (response.Value.BlobType == Models::BlobType::AppendBlob - && !response.Value.IsSealed.HasValue()) - { - response.Value.IsSealed = false; - } - { - std::map> orPropertiesMap; - for (auto i = response.RawResponse->GetHeaders().lower_bound("x-ms-or-"); - i != response.RawResponse->GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; - ++i) - { - const std::string& header = i->first; - auto underscorePos = header.find('_', 8); - if (underscorePos == std::string::npos) - { - continue; - } - std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); - std::string ruleId = header.substr(underscorePos + 1); - - Models::ObjectReplicationRule rule; - rule.RuleId = std::move(ruleId); - rule.ReplicationStatus = Models::ObjectReplicationStatus(i->second); - orPropertiesMap[policyId].emplace_back(std::move(rule)); - } - for (auto& property : orPropertiesMap) - { - Models::ObjectReplicationPolicy policy; - policy.PolicyId = property.first; - policy.Rules = std::move(property.second); - response.Value.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); - } - } + CompleteBlobPropertiesFromResponse(response.Value, *response.RawResponse); return response; } diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index 4092dfe7c2..6ccd5e28c1 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -1268,6 +1268,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyAuthPolicy); m_pipeline = std::make_shared( @@ -1311,6 +1312,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; auto authPolicies = _detail::CreateTokenAuthenticationPolicies( blobContainerUrl, credential, tokenContext, options.EnableTenantDiscovery, options); pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); @@ -1349,6 +1351,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 0a1c08457b..3f7279681d 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -10,6 +10,13 @@ namespace Azure { namespace Storage { namespace Blobs { + void BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + m_operationOptions.ContinuationToken = NextPageToken; + m_operationOptions.AccessConditions.IfMatch = Layout.Properties.ETag; + *this = m_blobClient->GetLayout(m_operationOptions, context); + } + std::unique_ptr StartBlobCopyOperation::PollInternal( const Azure::Core::Context&) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp index c55fd9a13a..58aae744f9 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp @@ -68,6 +68,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyPolicy); m_pipeline = std::make_shared( @@ -111,6 +112,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; auto authPolicies = _detail::CreateTokenAuthenticationPolicies( serviceUrl, credential, tokenContext, options.EnableTenantDiscovery, options); pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); @@ -149,6 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp new file mode 100644 index 0000000000..698bcf0b22 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "data_locality.hpp" + +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + std::string GetIdealDataLocalityEndpoint( + int64_t offset, + int64_t length, + const DataLocalityLayout& layout) + { + if (offset < 0 || length <= 0) + { + return std::string(); + } + + const int64_t requestEnd = length - 1 > (std::numeric_limits::max)() - offset + ? (std::numeric_limits::max)() + : offset + (length - 1); + auto range = std::lower_bound( + layout.Ranges.begin(), + layout.Ranges.end(), + offset, + [](const Models::BlobLayoutRange& value, int64_t target) { + return value.Range.Offset <= target + && value.Range.Length.Value() <= target - value.Range.Offset; + }); + std::unordered_map endpointOverlaps; + for (; range != layout.Ranges.end() && range->Range.Offset <= requestEnd; ++range) + { + const int64_t rangeEnd = range->Range.Offset + (range->Range.Length.Value() - 1); + const int64_t overlapStart = (std::max)(offset, range->Range.Offset); + const int64_t overlapEnd = (std::min)(requestEnd, rangeEnd); + if (overlapStart > overlapEnd) + { + continue; + } + auto endpoint = endpointOverlaps.emplace(range->Endpoint, 0); + endpoint.first->second += static_cast(overlapEnd - overlapStart) + 1; + } + + std::string idealEndpoint; + uint64_t largestOverlap = 0; + for (const auto& endpoint : endpointOverlaps) + { + if (endpoint.second > largestOverlap) + { + idealEndpoint = endpoint.first; + largestOverlap = endpoint.second; + } + } + return idealEndpoint; + } + + DataLocalityLayoutState::DataLocalityLayoutState(std::function refresh) + : m_refresh(std::move(refresh)), m_lastRefresh(std::chrono::steady_clock::now()), + m_lastRefreshAttempt(m_lastRefresh), m_initialRefresh(true) + { + StartRefresh(); + } + + std::string DataLocalityLayoutState::GetEndpoint(int64_t offset, int64_t length) + { + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(m_mutex); + if (m_refreshFuture.valid() + && m_refreshFuture.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + { + m_refreshFuture.get(); + } + const auto age = now - m_lastRefresh; + if (age > DataLocalityLayoutValidity && !m_refreshFuture.valid() + && now - m_lastRefreshAttempt > DataLocalityLayoutValidity) + { + StartRefresh(); + } + if (age > DataLocalityLayoutValidity + DataLocalityLayoutExtendedExpiry) + { + return std::string(); + } + return GetIdealDataLocalityEndpoint(offset, length, m_layout); + } + + void DataLocalityLayoutState::WaitForInitialLayout() + { + WaitForRefreshTask(); + + std::exception_ptr initialRefreshException; + { + std::lock_guard lock(m_mutex); + if (m_initialRefresh) + { + m_initialRefresh = false; + initialRefreshException = std::move(m_initialRefreshException); + } + } + if (initialRefreshException) + { + std::rethrow_exception(initialRefreshException); + } + } + + void DataLocalityLayoutState::WaitForRefreshTask() + { + std::future refreshFuture; + { + std::lock_guard lock(m_mutex); + if (m_refreshFuture.valid()) + { + refreshFuture = std::move(m_refreshFuture); + } + } + if (refreshFuture.valid()) + { + refreshFuture.get(); + } + } + + void DataLocalityLayoutState::StartRefresh() + { + m_lastRefreshAttempt = std::chrono::steady_clock::now(); + try + { + m_refreshFuture = std::async(std::launch::async, [this]() { + DataLocalityLayout refreshedLayout; + bool refreshed = false; + std::exception_ptr refreshException; + try + { + refreshedLayout = m_refresh(); + refreshed = !refreshedLayout.Ranges.empty(); + } + catch (...) + { + refreshException = std::current_exception(); + } + + std::lock_guard lock(m_mutex); + if (refreshed) + { + m_layout = std::move(refreshedLayout); + m_lastRefresh = std::chrono::steady_clock::now(); + } + else if (m_initialRefresh) + { + m_initialRefreshException = std::move(refreshException); + } + }); + } + catch (...) + { + // If a refresh task cannot be created, continue with stale or primary routing. + } + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp new file mode 100644 index 0000000000..4871ca9df6 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + constexpr std::chrono::minutes DataLocalityLayoutValidity{5}; + constexpr std::chrono::minutes DataLocalityLayoutExtendedExpiry{5}; + + struct DataLocalityLayout final + { + std::vector Ranges; + int64_t BlobSize = 0; + Azure::ETag ETag; + }; + + std::string GetIdealDataLocalityEndpoint( + int64_t offset, + int64_t length, + const DataLocalityLayout& layout); + + class DataLocalityLayoutState final { + public: + explicit DataLocalityLayoutState(std::function refresh); + + ~DataLocalityLayoutState() { WaitForRefreshTask(); } + + DataLocalityLayoutState(const DataLocalityLayoutState&) = delete; + DataLocalityLayoutState& operator=(const DataLocalityLayoutState&) = delete; + + std::string GetEndpoint(int64_t offset, int64_t length); + + void WaitForInitialLayout(); + + bool HasLayout() const + { + std::lock_guard lock(m_mutex); + return !m_layout.Ranges.empty(); + } + + int64_t GetBlobSize() const + { + std::lock_guard lock(m_mutex); + return m_layout.BlobSize; + } + + Azure::ETag GetETag() const + { + std::lock_guard lock(m_mutex); + return m_layout.ETag; + } + + private: + void WaitForRefreshTask(); + + void StartRefresh(); + + DataLocalityLayout m_layout; + std::function m_refresh; + std::chrono::steady_clock::time_point m_lastRefresh; + std::chrono::steady_clock::time_point m_lastRefreshAttempt; + mutable std::mutex m_mutex; + std::future m_refreshFuture; + std::exception_ptr m_initialRefreshException; + bool m_initialRefresh = false; + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 904db11b9d..e35fafdc98 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -166,6 +166,7 @@ namespace Azure { namespace Storage { namespace Blobs { const BlobType BlobType::BlockBlob("BlockBlob"); const BlobType BlobType::PageBlob("PageBlob"); const BlobType BlobType::AppendBlob("AppendBlob"); + const DownloadHint DownloadHint::Layout("layout"); const DeleteSnapshotsOption DeleteSnapshotsOption::IncludeSnapshots("include"); const DeleteSnapshotsOption DeleteSnapshotsOption::OnlySnapshots("only"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::NeverExpire("NeverExpire"); @@ -411,6 +412,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobServiceProperties response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -707,6 +709,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::ServiceStatistics response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -806,6 +809,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::ListBlobContainersResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1098,6 +1102,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::UserDelegationKey response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1287,6 +1292,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::FindBlobsByTagsResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1604,6 +1610,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobContainerAccessPolicy response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1897,6 +1904,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::FindBlobsByTagsResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -2653,6 +2661,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.Details.SmartAccessTier = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-smart-access-tier")); } + if (pRawResponse->GetHeaders().count("x-ms-download-hint") != 0) + { + response.Details.DownloadHint + = Models::DownloadHint(pRawResponse->GetHeaders().at("x-ms-download-hint")); + } if (httpStatusCode == Core::Http::HttpStatusCode::Ok) { if (pRawResponse->GetHeaders().count("Content-MD5") != 0) @@ -3058,6 +3071,443 @@ namespace Azure { namespace Storage { namespace Blobs { Models::DeleteBlobResult response; return Response(std::move(response), std::move(pRawResponse)); } + Response BlobClient::GetLayout( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetBlobLayoutOptions& options, + const Core::Context& context) + { + auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); + request.GetUrl().AppendQueryParameter("comp", "layout"); + if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "snapshot", _internal::UrlEncodeQueryParameter(options.Snapshot.Value())); + } + if (options.VersionId.HasValue() && !options.VersionId.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "versionid", _internal::UrlEncodeQueryParameter(options.VersionId.Value())); + } + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } + if (options.Range.HasValue() && !options.Range.Value().empty()) + { + request.SetHeader("x-ms-range", options.Range.Value()); + } + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.IfTags.HasValue() && !options.IfTags.Value().empty()) + { + request.SetHeader("x-ms-if-tags", options.IfTags.Value()); + } + if (options.IfModifiedSince.HasValue()) + { + request.SetHeader( + "If-Modified-Since", + options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfUnmodifiedSince.HasValue()) + { + request.SetHeader( + "If-Unmodified-Since", + options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfMatch.HasValue() && !options.IfMatch.ToString().empty()) + { + request.SetHeader("If-Match", options.IfMatch.ToString()); + } + if (options.IfNoneMatch.HasValue() && !options.IfNoneMatch.ToString().empty()) + { + request.SetHeader("If-None-Match", options.IfNoneMatch.ToString()); + } + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } + request.SetHeader("x-ms-version", "2026-10-06"); + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok + || httpStatusCode == Core::Http::HttpStatusCode::NoContent)) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::_detail::BlobLayout response; + { + const auto& responseBody = pRawResponse->GetBody(); + if (!responseBody.empty()) + { + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + enum class XmlTagEnum + { + kUnknown, + kBlobLayout, + kRanges, + kRange, + kEndpoints, + kEndpoint, + kMarker, + kNextMarker, + kMaxResults, + }; + const std::unordered_map XmlTagEnumMap{ + {"BlobLayout", XmlTagEnum::kBlobLayout}, + {"Ranges", XmlTagEnum::kRanges}, + {"Range", XmlTagEnum::kRange}, + {"Endpoints", XmlTagEnum::kEndpoints}, + {"Endpoint", XmlTagEnum::kEndpoint}, + {"Marker", XmlTagEnum::kMarker}, + {"NextMarker", XmlTagEnum::kNextMarker}, + {"MaxResults", XmlTagEnum::kMaxResults}, + }; + std::vector xmlPath; + Models::_detail::BlobLayoutRangesItem vectorElement1; + Models::_detail::BlobLayoutEndpointsItem vectorElement2; + while (true) + { + auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + else if (node.Type == _internal::XmlNodeType::StartTag) + { + auto ite = XmlTagEnumMap.find(node.Name); + xmlPath.push_back(ite == XmlTagEnumMap.end() ? XmlTagEnum::kUnknown : ite->second); + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kMarker) + { + response.Marker = node.Value; + } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.NextMarker = node.Value; + } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kMaxResults) + { + response.MaxResults = std::stoi(node.Value); + } + } + else if (node.Type == _internal::XmlNodeType::Attribute) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "Start") + { + vectorElement1.Start = std::stoll(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "End") + { + vectorElement1.End = std::stoll(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "EndpointIndex") + { + vectorElement1.EndpointIndex = std::stoi(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint + && node.Name == "Index") + { + vectorElement2.Index = std::stoi(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint + && node.Name == "Value") + { + vectorElement2.Value = node.Value; + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange) + { + response.Ranges.push_back(std::move(vectorElement1)); + vectorElement1 = Models::_detail::BlobLayoutRangesItem(); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint) + { + response.Endpoints.push_back(std::move(vectorElement2)); + vectorElement2 = Models::_detail::BlobLayoutEndpointsItem(); + } + xmlPath.pop_back(); + } + } + } + } + if (pRawResponse->GetHeaders().count("Last-Modified") != 0) + { + response.Properties.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-length") != 0) + { + response.Properties.BlobSize + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-content-length")); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-type") != 0) + { + response.Properties.HttpHeaders.ContentType + = pRawResponse->GetHeaders().at("x-ms-blob-content-type"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-encoding") != 0) + { + response.Properties.HttpHeaders.ContentEncoding + = pRawResponse->GetHeaders().at("x-ms-blob-content-encoding"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-md5") != 0) + { + response.Properties.HttpHeaders.ContentHash.Value + = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-blob-content-md5")); + response.Properties.HttpHeaders.ContentHash.Algorithm = HashAlgorithm::Md5; + } + if (pRawResponse->GetHeaders().count("x-ms-creation-time") != 0) + { + response.Properties.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-creation-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + for (auto i = pRawResponse->GetHeaders().lower_bound("x-ms-meta-"); + i != pRawResponse->GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; + ++i) + { + response.Properties.Metadata.emplace(i->first.substr(10), i->second); + } + if (pRawResponse->GetHeaders().count("x-ms-or-policy-id") != 0) + { + response.Properties.ObjectReplicationDestinationPolicyId + = pRawResponse->GetHeaders().at("x-ms-or-policy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-type") != 0) + { + response.Properties.BlobType + = Models::BlobType(pRawResponse->GetHeaders().at("x-ms-blob-type")); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-completion-time") != 0) + { + response.Properties.CopyCompletedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-copy-completion-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status-description") != 0) + { + response.Properties.CopyStatusDescription + = pRawResponse->GetHeaders().at("x-ms-copy-status-description"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-id") != 0) + { + response.Properties.CopyId = pRawResponse->GetHeaders().at("x-ms-copy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-progress") != 0) + { + response.Properties.CopyProgress = pRawResponse->GetHeaders().at("x-ms-copy-progress"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-source") != 0) + { + response.Properties.CopySource = pRawResponse->GetHeaders().at("x-ms-copy-source"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status") != 0) + { + response.Properties.CopyStatus + = Models::CopyStatus(pRawResponse->GetHeaders().at("x-ms-copy-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-incremental-copy") != 0) + { + response.Properties.IsIncrementalCopy + = pRawResponse->GetHeaders().at("x-ms-incremental-copy") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-destination-snapshot") != 0) + { + response.Properties.IncrementalCopyDestinationSnapshot + = pRawResponse->GetHeaders().at("x-ms-copy-destination-snapshot"); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.Properties.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0) + { + response.Properties.LeaseState + = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0) + { + response.Properties.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + } + if (pRawResponse->GetHeaders().count("ETag") != 0) + { + response.Properties.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + } + if (pRawResponse->GetHeaders().count("Content-Disposition") != 0) + { + response.Properties.HttpHeaders.ContentDisposition + = pRawResponse->GetHeaders().at("Content-Disposition"); + } + if (pRawResponse->GetHeaders().count("Content-Language") != 0) + { + response.Properties.HttpHeaders.ContentLanguage + = pRawResponse->GetHeaders().at("Content-Language"); + } + if (pRawResponse->GetHeaders().count("Cache-Control") != 0) + { + response.Properties.HttpHeaders.CacheControl + = pRawResponse->GetHeaders().at("Cache-Control"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.Properties.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-committed-block-count") != 0) + { + response.Properties.CommittedBlockCount + = std::stoi(pRawResponse->GetHeaders().at("x-ms-blob-committed-block-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-server-encrypted") != 0) + { + response.Properties.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) + { + response.Properties.EncryptionKeySha256 = Core::Convert::Base64Decode( + pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256")); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-scope") != 0) + { + response.Properties.EncryptionScope + = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier") != 0) + { + response.Properties.AccessTier + = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-access-tier")); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier-inferred") != 0) + { + response.Properties.IsAccessTierInferred + = pRawResponse->GetHeaders().at("x-ms-access-tier-inferred") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-smart-access-tier") != 0) + { + response.Properties.SmartAccessTier + = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-smart-access-tier")); + } + if (pRawResponse->GetHeaders().count("x-ms-archive-status") != 0) + { + response.Properties.ArchiveStatus + = Models::ArchiveStatus(pRawResponse->GetHeaders().at("x-ms-archive-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier-change-time") != 0) + { + response.Properties.AccessTierChangedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-access-tier-change-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-version-id") != 0) + { + response.Properties.VersionId = pRawResponse->GetHeaders().at("x-ms-version-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-is-current-version") != 0) + { + response.Properties.IsCurrentVersion + = pRawResponse->GetHeaders().at("x-ms-is-current-version") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-tag-count") != 0) + { + response.Properties.TagCount = std::stoi(pRawResponse->GetHeaders().at("x-ms-tag-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-expiry-time") != 0) + { + response.Properties.ExpiresOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-expiry-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-sealed") != 0) + { + response.Properties.IsSealed + = pRawResponse->GetHeaders().at("x-ms-blob-sealed") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-rehydrate-priority") != 0) + { + response.Properties.RehydratePriority + = Models::RehydratePriority(pRawResponse->GetHeaders().at("x-ms-rehydrate-priority")); + } + if (pRawResponse->GetHeaders().count("x-ms-last-access-time") != 0) + { + response.Properties.LastAccessedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-last-access-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-immutability-policy-until-date") != 0) + { + if (!response.Properties.ImmutabilityPolicy.HasValue()) + { + response.Properties.ImmutabilityPolicy = Models::BlobImmutabilityPolicy(); + } + response.Properties.ImmutabilityPolicy.Value().ExpiresOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-immutability-policy-until-date"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-immutability-policy-mode") != 0) + { + if (!response.Properties.ImmutabilityPolicy.HasValue()) + { + response.Properties.ImmutabilityPolicy = Models::BlobImmutabilityPolicy(); + } + response.Properties.ImmutabilityPolicy.Value().PolicyMode + = Models::BlobImmutabilityPolicyMode( + pRawResponse->GetHeaders().at("x-ms-immutability-policy-mode")); + } + if (pRawResponse->GetHeaders().count("x-ms-legal-hold") != 0) + { + response.Properties.HasLegalHold + = pRawResponse->GetHeaders().at("x-ms-legal-hold") == std::string("true"); + } + return Response(std::move(response), std::move(pRawResponse)); + } Response BlobClient::Undelete( Core::Http::_internal::HttpPipeline& pipeline, const Core::Url& url, @@ -4534,6 +4984,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::map response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -5352,6 +5803,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::GetPageRangesResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -5530,6 +5982,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::GetPageRangesDiffResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -7193,6 +7646,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::GetBlockListResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 9bfa6f1355..05ef91ce67 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-blobs namespace: Azure::Storage::Blobs output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-12-06/blob.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/refs/heads/feature/storage/stg105base-swagger/specification/storage/data-plane/Microsoft.BlobStorage/stable/2027-03-07/blob.json ``` ## ModelFour Options @@ -26,6 +26,43 @@ modelerfour: See the [AutoRest samples](https://github.com/Azure/autorest/tree/master/Samples/3b-custom-transformations) for more about how we're customizing things. +### Generate Get Blob Layout responses + +```yaml +directive: + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=layout"].get + transform: > + $.responses["204"].schema = $.responses["200"].schema; + for (const statusCode of ["200", "204"]) { + for (const headerName in $.responses[statusCode].headers) { + if (!["x-ms-meta", "x-ms-or"].includes(headerName)) { + $.responses[statusCode].headers[headerName]["x-nullable"] = true; + } + } + } + - from: swagger-document + where: $.definitions.BlobLayout + transform: > + const rangeItem = $.properties.Ranges.properties.Range.items; + const endpointItem = $.properties.Endpoints.properties.Endpoint.items; + $["x-ms-sealed"] = false; + $["x-namespace"] = "_detail"; + rangeItem["x-namespace"] = "_detail"; + endpointItem["x-namespace"] = "_detail"; + $.properties.Ranges = { + "type": "array", + "xml": {"name": "Ranges", "wrapped": true}, + "items": rangeItem + }; + $.properties.Endpoints = { + "type": "array", + "xml": {"name": "Endpoints", "wrapped": true}, + "items": endpointItem + }; + $.properties.Properties = {"$ref": "#/definitions/BlobProperties"}; +``` + ### Fix Generator Warnings ```yaml @@ -75,6 +112,7 @@ directive: delete $["/{filesystem}/{path}?FileRename"]; delete $["/{containerName}?restype=container&comp=list&flat&arrow"]; delete $["/{containerName}?restype=container&comp=list&hierarchy&arrow"]; + delete $["/{containerName}?restype=container&comp=session"]; for (const operation in $) { for (const verb in $[operation]) { @@ -967,6 +1005,12 @@ directive: - from: swagger-document where: $.definitions transform: > + $.DownloadHint = { + "type": "string", + "description": "Indicates the download hint for the blob.", + "enum": ["layout"], + "x-ms-enum": {"name": "DownloadHint", "modelAsString": true} + }; $.DownloadBlobDetails = { "type": "object", "required": ["ETag", "LastModified", "CreatedOn", "HttpHeaders", "IsServerEncrypted", "HasLegalHold"], @@ -1003,7 +1047,8 @@ directive: "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, "IsAccessTierInferred": {"type": "boolean", "x-nullable": true}, "AccessTierChangedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, - "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true} + "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "DownloadHint": {"$ref": "#/definitions/DownloadHint", "x-nullable": true} } }; $.DownloadBlobResult = { @@ -1072,6 +1117,8 @@ directive: $[status_code].headers["x-ms-access-tier-inferred"]["x-ms-client-path"] = "Details.IsAccessTierInferred"; $[status_code].headers["x-ms-access-tier-change-time"]["x-ms-client-path"] = "Details.AccessTierChangedOn"; $[status_code].headers["x-ms-smart-access-tier"]["x-ms-client-path"] = "Details.SmartAccessTier"; + $[status_code].headers["x-ms-download-hint"]["x-ms-client-path"] = "Details.DownloadHint"; + $[status_code].headers["x-ms-download-hint"]["x-nullable"] = true; delete $[status_code].headers["Accept-Ranges"]; delete $[status_code].headers["Content-Length"]; delete $[status_code].headers["Content-Range"]; @@ -1098,18 +1145,81 @@ directive: transform: > $.push({"$ref": "#/parameters/UserPrincipalName"}); - from: swagger-document - where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses + where: $.definitions transform: > - $["200"].schema = { + $.BlobProperties = { "type": "object", "x-ms-client-name": "BlobProperties", "x-ms-sealed": false, + "required": [ + "LastModified", + "CreatedOn", + "Metadata", + "BlobType", + "BlobSize", + "ETag", + "IsServerEncrypted", + "HttpHeaders", + "HasLegalHold" + ], "properties": { "ObjectReplicationSourceProperties": {"type": "array", "items": {"$ref": "#/definitions/ObjectReplicationPolicy"}, "x-ms-xml": {"name": ""}}, "ImmutabilityPolicy": {"$ref": "#/definitions/BlobImmutabilityPolicy", "x-nullable": true, "x-ms-xml": {"name": ""}}, - "HttpHeaders": {"$ref": "#/definitions/BlobHttpHeaders", "x-ms-xml": {"name": ""}} + "HttpHeaders": {"$ref": "#/definitions/BlobHttpHeaders", "x-ms-xml": {"name": ""}}, + "LastModified": {"type": "string", "format": "date-time-rfc1123"}, + "CreatedOn": {"type": "string", "format": "date-time-rfc1123"}, + "Metadata": {"$ref": "#/definitions/Metadata"}, + "ObjectReplicationDestinationPolicyId": {"type": "string", "x-nullable": true}, + "BlobType": { + "type": "string", + "enum": ["BlockBlob", "PageBlob", "AppendBlob"], + "x-ms-enum": {"name": "BlobType", "modelAsString": false} + }, + "CopyCompletedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "CopyStatusDescription": {"type": "string", "x-nullable": true}, + "CopyId": {"type": "string", "x-nullable": true}, + "CopyProgress": {"type": "string", "x-nullable": true}, + "CopySource": {"type": "string", "x-nullable": true}, + "CopyStatus": {"$ref": "#/definitions/CopyStatus", "x-nullable": true}, + "IsIncrementalCopy": {"type": "boolean", "x-nullable": true}, + "IncrementalCopyDestinationSnapshot": {"type": "string", "x-nullable": true}, + "LeaseDuration": { + "type": "string", + "enum": ["infinite", "fixed"], + "x-ms-enum": {"name": "LeaseDurationType", "modelAsString": false}, + "x-nullable": true + }, + "LeaseState": {"$ref": "#/definitions/LeaseState", "x-nullable": true}, + "LeaseStatus": {"$ref": "#/definitions/LeaseStatus", "x-nullable": true}, + "BlobSize": {"type": "integer", "format": "int64"}, + "ETag": {"type": "string", "format": "etag"}, + "SequenceNumber": {"type": "integer", "format": "int64", "x-nullable": true}, + "CommittedBlockCount": {"type": "integer", "format": "int32", "x-nullable": true}, + "IsServerEncrypted": {"type": "boolean"}, + "EncryptionKeySha256": {"type": "string", "format": "byte", "x-nullable": true}, + "EncryptionScope": {"type": "string", "x-nullable": true}, + "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "IsAccessTierInferred": {"type": "boolean", "x-nullable": true}, + "ArchiveStatus": {"$ref": "#/definitions/ArchiveStatus", "x-nullable": true}, + "AccessTierChangedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "VersionId": {"type": "string", "x-nullable": true}, + "IsCurrentVersion": {"type": "boolean", "x-nullable": true}, + "TagCount": {"type": "integer", "format": "int32", "x-nullable": true}, + "ExpiresOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "IsSealed": {"type": "boolean", "x-nullable": true}, + "RehydratePriority": {"$ref": "#/definitions/RehydratePriority", "x-nullable": true}, + "LastAccessedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "HasLegalHold": {"type": "boolean", "x-ms-client-default": false} } }; + for (const propertyName in $.BlobProperties.properties) { + $.BlobProperties.properties[propertyName]["x-ms-xml"] = {"name": ""}; + } + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses["200"] + transform: > + $.schema = {"$ref": "#/definitions/BlobProperties"}; - from: swagger-document where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses["200"].headers transform: > @@ -1182,6 +1292,77 @@ directive: $["x-ms-blob-type"]["x-ms-client-default"] = ""; delete $["Accept-Ranges"]; delete $["x-ms-or"]; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + const propertiesHeaders = $["/{containerName}/{blob}"].head.responses["200"].headers; + const layoutResponses = $["/{containerName}/{blob}?comp=layout"].get.responses; + const mappings = { + "Last-Modified": ["Last-Modified", "LastModified"], + "x-ms-creation-time": ["x-ms-creation-time", "CreatedOn"], + "x-ms-meta": ["x-ms-meta", "Metadata"], + "x-ms-or-policy-id": ["x-ms-or-policy-id", "ObjectReplicationDestinationPolicyId"], + "x-ms-blob-type": ["x-ms-blob-type", "BlobType"], + "x-ms-copy-completion-time": ["x-ms-copy-completion-time", "CopyCompletedOn"], + "x-ms-copy-status-description": ["x-ms-copy-status-description", "CopyStatusDescription"], + "x-ms-copy-id": ["x-ms-copy-id", "CopyId"], + "x-ms-copy-progress": ["x-ms-copy-progress", "CopyProgress"], + "x-ms-copy-source": ["x-ms-copy-source", "CopySource"], + "x-ms-copy-status": ["x-ms-copy-status", "CopyStatus"], + "x-ms-incremental-copy": ["x-ms-incremental-copy", "IsIncrementalCopy"], + "x-ms-copy-destination-snapshot": ["x-ms-copy-destination-snapshot", "IncrementalCopyDestinationSnapshot"], + "x-ms-lease-duration": ["x-ms-lease-duration", "LeaseDuration"], + "x-ms-lease-state": ["x-ms-lease-state", "LeaseState"], + "x-ms-lease-status": ["x-ms-lease-status", "LeaseStatus"], + "x-ms-blob-content-length": ["Content-Length", "BlobSize"], + "ETag": ["ETag", "ETag"], + "x-ms-blob-content-type": ["Content-Type", "HttpHeaders.ContentType"], + "x-ms-blob-content-md5": ["Content-MD5", "HttpHeaders.ContentHash"], + "x-ms-blob-content-encoding": ["Content-Encoding", "HttpHeaders.ContentEncoding"], + "Content-Disposition": ["Content-Disposition", "HttpHeaders.ContentDisposition"], + "Content-Language": ["Content-Language", "HttpHeaders.ContentLanguage"], + "Cache-Control": ["Cache-Control", "HttpHeaders.CacheControl"], + "x-ms-blob-sequence-number": ["x-ms-blob-sequence-number", "SequenceNumber"], + "x-ms-blob-committed-block-count": ["x-ms-blob-committed-block-count", "CommittedBlockCount"], + "x-ms-server-encrypted": ["x-ms-server-encrypted", "IsServerEncrypted"], + "x-ms-encryption-key-sha256": ["x-ms-encryption-key-sha256", "EncryptionKeySha256"], + "x-ms-encryption-scope": ["x-ms-encryption-scope", "EncryptionScope"], + "x-ms-access-tier": ["x-ms-access-tier", "AccessTier"], + "x-ms-smart-access-tier": ["x-ms-smart-access-tier", "SmartAccessTier"], + "x-ms-access-tier-inferred": ["x-ms-access-tier-inferred", "IsAccessTierInferred"], + "x-ms-archive-status": ["x-ms-archive-status", "ArchiveStatus"], + "x-ms-access-tier-change-time": ["x-ms-access-tier-change-time", "AccessTierChangedOn"], + "x-ms-version-id": ["x-ms-version-id", "VersionId"], + "x-ms-is-current-version": ["x-ms-is-current-version", "IsCurrentVersion"], + "x-ms-tag-count": ["x-ms-tag-count", "TagCount"], + "x-ms-expiry-time": ["x-ms-expiry-time", "ExpiresOn"], + "x-ms-blob-sealed": ["x-ms-blob-sealed", "IsSealed"], + "x-ms-rehydrate-priority": ["x-ms-rehydrate-priority", "RehydratePriority"], + "x-ms-last-access-time": ["x-ms-last-access-time", "LastAccessedOn"], + "x-ms-immutability-policy-mode": ["x-ms-immutability-policy-mode", "ImmutabilityPolicy.PolicyMode"], + "x-ms-immutability-policy-until-date": ["x-ms-immutability-policy-until-date", "ImmutabilityPolicy.ExpiresOn"], + "x-ms-legal-hold": ["x-ms-legal-hold", "HasLegalHold"] + }; + for (const statusCode of ["200", "204"]) { + const headers = layoutResponses[statusCode].headers; + for (const headerName in headers) { + if (!mappings[headerName]) { + delete headers[headerName]; + } + } + for (const headerName in mappings) { + if (!headers[headerName]) { + continue; + } + const [sourceHeaderName, propertyPath] = mappings[headerName]; + headers[headerName] = JSON.parse(JSON.stringify(propertiesHeaders[sourceHeaderName])); + delete headers[headerName]["x-ms-client-name"]; + headers[headerName]["x-ms-client-path"] = "Properties." + propertyPath; + if (headerName !== "x-ms-meta") { + headers[headerName]["x-nullable"] = true; + } + } + } ``` ### DeleteBlob diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index 0aca417c06..d2e3d724e6 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -54,6 +54,7 @@ set( inc/azure/storage/common/internal/shared_key_policy.hpp inc/azure/storage/common/internal/storage_bearer_token_auth.hpp inc/azure/storage/common/internal/storage_bearer_token_authentication_policy.hpp + inc/azure/storage/common/internal/storage_data_locality_policy.hpp inc/azure/storage/common/internal/storage_per_retry_policy.hpp inc/azure/storage/common/internal/storage_pipeline.hpp inc/azure/storage/common/internal/storage_retry_policy.hpp @@ -81,6 +82,7 @@ set( src/shared_key_policy.cpp src/storage_bearer_token_authentication_policy.cpp src/storage_credential.cpp + src/storage_data_locality_policy.cpp src/storage_exception.cpp src/storage_per_retry_policy.cpp src/storage_pipeline.cpp diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp new file mode 100644 index 0000000000..e6176798bb --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/common/dll_import_export.hpp" + +#include + +#include +#include + +namespace Azure { namespace Storage { namespace _internal { + + AZ_STORAGE_COMMON_DLLEXPORT extern const Azure::Core::Context::Key DataLocalityEndpointKey; + + inline Azure::Core::Context WithDataLocalityEndpoint( + const Azure::Core::Context& context, + const std::string& endpoint) + { + return endpoint.empty() ? context : context.WithValue(DataLocalityEndpointKey, endpoint); + } + + class StorageDataLocalityPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + const Azure::Core::Context& context) const override; + }; + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp index 172b788660..38fab95724 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp @@ -19,6 +19,7 @@ namespace Azure { namespace Storage { namespace _internal { std::string PrimaryHost; std::string SecondaryHost; std::string ApiVersion; + bool AddDataLocalityPolicy = false; std::unique_ptr TokenAuthPolicy; std::unique_ptr SharedKeyAuthPolicy; }; diff --git a/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp b/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp new file mode 100644 index 0000000000..5674fe54c8 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +namespace Azure { namespace Storage { namespace _internal { + + const Azure::Core::Context::Key DataLocalityEndpointKey; + + std::unique_ptr StorageDataLocalityPolicy::Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + const Azure::Core::Context& context) const + { + std::string endpoint; + if (!context.TryGetValue(DataLocalityEndpointKey, endpoint) || endpoint.empty()) + { + return nextPolicy.Send(request, context); + } + + auto& requestUrl = request.GetUrl(); + std::string originalAuthority = requestUrl.GetHost(); + if (requestUrl.GetPort() != 0) + { + originalAuthority += ":" + std::to_string(requestUrl.GetPort()); + } + + // The Storage service returns endpoint values as absolute URLs with a scheme and port. + Azure::Core::Url endpointUrl( + endpoint.find("://") == std::string::npos ? requestUrl.GetScheme() + "://" + endpoint + : endpoint); + request.SetHeader("Host", originalAuthority); + requestUrl.SetScheme(endpointUrl.GetScheme()); + requestUrl.SetHost(endpointUrl.GetHost()); + requestUrl.SetPort(endpointUrl.GetPort()); + return nextPolicy.Send(request, context); + } + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp index 339286b830..c33886d499 100644 --- a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp +++ b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp @@ -4,6 +4,7 @@ #include "azure/storage/common/internal/storage_pipeline.hpp" #include "azure/storage/common/internal/constants.hpp" +#include "azure/storage/common/internal/storage_data_locality_policy.hpp" #include "azure/storage/common/internal/storage_per_retry_policy.hpp" #include "azure/storage/common/internal/storage_retry_policy.hpp" #include "azure/storage/common/internal/storage_service_version_policy.hpp" @@ -41,6 +42,10 @@ namespace Azure { namespace Storage { namespace _internal { { policies.emplace_back(policy->Clone()); } + if (storagePipelineOptions.AddDataLocalityPolicy) + { + policies.emplace_back(std::make_unique()); + } // Retry policy policies.emplace_back(std::make_unique(clientOptions.Retry)); diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index ad3092c6c3..b33a93452f 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -5,6 +5,13 @@ ### Features Added - Added support for persisted session authentication for eligible file download requests. +- Added `DataLakeFileClient::GetLayout` and related options and response types for retrieving + locality endpoints, byte ranges, and file properties. Ranges are exposed directly through + `Models::FileLayoutInfo::Ranges` as a vector of `Models::FileLayoutRange`. +- Added `DownloadFileOptions::LayoutEndpoint` for one-shot download routing and + `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for + managed downloads. +- Added the `Models::DownloadHint` alias and `Models::DownloadFileDetails::DownloadHint`. ### Breaking Changes @@ -15,6 +22,10 @@ ### Other Changes +- Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to + 4 MiB, matching the default subsequent chunk size. Explicitly configured + `InitialChunkSize` and `SingleUploadThreshold` values are still respected. + ## 12.17.0 (2026-09-29) ### Other Changes diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index 9743563c1f..e3af80df50 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -189,6 +189,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DownloadFileOptions& options = DownloadFileOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Gets the data locality layout for this file. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A paged response containing the file's data locality layout. + * @remark This request is sent to the blob endpoint. + */ + FileLayoutPagedResponse GetLayout( + const GetFileLayoutOptions& options = GetFileLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + /** * @brief Creates a new file, or updates the content of an existing file. Updating * an existing file overwrites any existing metadata on the file. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index f40c9fd73d..266eb0b768 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -83,6 +83,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; + /** + * @brief Determines whether locality-aware routing is used for managed file downloads. + */ + using LayoutAwareRouting = Blobs::LayoutAwareRouting; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; using GetServicePropertiesOptions = Blobs::GetServicePropertiesOptions; using SetServicePropertiesOptions = Blobs::SetServicePropertiesOptions; @@ -886,6 +890,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; + /** + * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. + */ + struct GetFileLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + PathAccessConditions AccessConditions; + }; + /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. @@ -924,6 +949,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief The locality endpoint to use for this download. An empty value uses the normal + * endpoint. + */ + std::string LayoutEndpoint; }; /** @@ -1037,7 +1068,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * File smaller than this will be uploaded with a single upload operation. This value * cannot be larger than 5000 MiB. */ - int64_t SingleUploadThreshold = 256 * 1024 * 1024; + int64_t SingleUploadThreshold = 4 * 1024 * 1024; /** * The maximum number of bytes in a single request. This value cannot be larger than diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 1a434048d2..d0660cf788 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -17,6 +17,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { class DataLakeFileSystemClient; class DataLakePathClient; class DataLakeDirectoryClient; + class DataLakeFileClient; namespace Models { @@ -462,6 +463,26 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; + /** + * @brief A file range and its resolved locality endpoint. + */ + using FileLayoutRange = Blobs::Models::BlobLayoutRange; + + /** + * @brief A page of file layout information and the file properties returned with it. + */ + struct FileLayoutInfo final + { + /** + * @brief The ranges and locality endpoints in this page. + */ + std::vector Ranges; + /** + * @brief The file properties returned with this page. + */ + PathProperties Properties; + }; + /** * @brief The access control list of a path. */ @@ -553,6 +574,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using UploadFileFromResult = Blobs::Models::UploadBlockBlobResult; using ScheduleFileDeletionResult = Blobs::Models::SetBlobExpiryResult; using CopyStatus = Blobs::Models::CopyStatus; + using DownloadHint = Blobs::Models::DownloadHint; /** * @brief Response type for #Azure::Storage::Files::DataLake::DataLakeFileClient::Query. @@ -731,6 +753,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The permissions of the file. */ Azure::Nullable Permissions; + + /** + * @brief The service hint for locality-aware downloads. + */ + Azure::Nullable DownloadHint; }; /** @@ -796,6 +823,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models + /** + * @brief Response type for + * #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. + */ + class FileLayoutPagedResponse final : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and file properties in this page. + */ + Models::FileLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_fileClient; + GetFileLayoutOptions m_operationOptions; + + friend class DataLakeFileClient; + friend class Azure::Core::PagedResponse; + }; + /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 0057174fc3..6b957c633b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -204,6 +204,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { blobOptions.Range = options.Range; blobOptions.RangeHashAlgorithm = options.RangeHashAlgorithm; blobOptions.Range = options.Range; + blobOptions.LayoutEndpoint = options.LayoutEndpoint; blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; @@ -253,6 +254,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); auto& headers = response.RawResponse->GetHeaders(); auto encryptionContext = headers.find(_detail::EncryptionContextHeaderName); if (encryptionContext != headers.end()) @@ -283,6 +285,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } + FileLayoutPagedResponse DataLakeFileClient::GetLayout( + const GetFileLayoutOptions& options, + const Azure::Core::Context& context) const + { + Blobs::GetBlobLayoutOptions blobOptions; + blobOptions.Range = options.Range; + blobOptions.ContinuationToken = options.ContinuationToken; + blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; + blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; + auto response = m_blobClient.GetLayout(blobOptions, context); + + FileLayoutPagedResponse result; + result.Layout.Ranges = std::move(response.Layout.Ranges); + result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( + std::move(response.Layout.Properties), *response.RawResponse); + result.CurrentPageToken = std::move(response.CurrentPageToken); + result.NextPageToken = std::move(response.NextPageToken); + result.RawResponse = std::move(response.RawResponse); + result.m_fileClient = std::make_shared(*this); + result.m_operationOptions = options; + return result; + } + Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, @@ -364,6 +392,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } @@ -404,6 +433,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 4e5b4953d5..5abf39f303 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -343,61 +343,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, options.IncludeUserPrincipalName.Value()) : context); - Models::PathProperties ret; - ret.ETag = std::move(response.Value.ETag); - ret.LastModified = std::move(response.Value.LastModified); - ret.CreatedOn = std::move(response.Value.CreatedOn); - ret.Metadata = std::move(response.Value.Metadata); - ret.LeaseDuration = std::move(response.Value.LeaseDuration); - ret.LeaseState = std::move(response.Value.LeaseState); - ret.LeaseStatus = std::move(response.Value.LeaseStatus); - ret.HttpHeaders = std::move(response.Value.HttpHeaders); - ret.IsServerEncrypted = response.Value.IsServerEncrypted; - ret.EncryptionKeySha256 = std::move(response.Value.EncryptionKeySha256); - ret.EncryptionScope = std::move(response.Value.EncryptionScope); - ret.CopyId = std::move(response.Value.CopyId); - ret.CopySource = std::move(response.Value.CopySource); - ret.CopyStatus = std::move(response.Value.CopyStatus); - ret.CopyProgress = std::move(response.Value.CopyProgress); - ret.CopyCompletedOn = std::move(response.Value.CopyCompletedOn); - ret.ExpiresOn = std::move(response.Value.ExpiresOn); - ret.LastAccessedOn = std::move(response.Value.LastAccessedOn); - ret.FileSize = response.Value.BlobSize; - ret.ArchiveStatus = std::move(response.Value.ArchiveStatus); - ret.RehydratePriority = std::move(response.Value.RehydratePriority); - ret.CopyStatusDescription = std::move(response.Value.CopyStatusDescription); - ret.IsIncrementalCopy = std::move(response.Value.IsIncrementalCopy); - ret.IncrementalCopyDestinationSnapshot - = std::move(response.Value.IncrementalCopyDestinationSnapshot); - ret.VersionId = std::move(response.Value.VersionId); - ret.IsCurrentVersion = std::move(response.Value.IsCurrentVersion); - ret.IsDirectory = _detail::MetadataIndicatesIsDirectory(ret.Metadata); - auto& headers = response.RawResponse->GetHeaders(); - auto encryptionContext = headers.find(_detail::EncryptionContextHeaderName); - if (encryptionContext != headers.end()) - { - ret.EncryptionContext = encryptionContext->second; - } - auto acl = headers.find(_detail::AclHeaderName); - if (acl != headers.end()) - { - ret.Acls = Models::Acl::DeserializeAcls(acl->second); - } - auto owner = headers.find(_detail::OwnerHeaderName); - if (owner != headers.end()) - { - ret.Owner = owner->second; - } - auto group = headers.find(_detail::GroupHeaderName); - if (group != headers.end()) - { - ret.Group = group->second; - } - auto permissions = headers.find(_detail::PermissionsHeaderName); - if (permissions != headers.end()) - { - ret.Permissions = permissions->second; - } + auto ret = _detail::PathPropertiesFromBlobProperties( + std::move(response.Value), *response.RawResponse); return Azure::Response(std::move(ret), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index c6fa0aa1c5..3652ac5891 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -4,11 +4,20 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" #include "azure/storage/files/datalake/datalake_directory_client.hpp" +#include "azure/storage/files/datalake/datalake_file_client.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { + void FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + auto options = m_operationOptions; + options.ContinuationToken = NextPageToken; + options.AccessConditions.IfMatch = Layout.Properties.ETag; + *this = m_fileClient->GetLayout(options, context); + } + namespace Models { Acl Acl::FromString(const std::string& aclString) diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 5a555b3665..f6538da505 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -23,6 +23,68 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam return Win32FileTimeConverter::Win32FileTimeToDateTime(std::stoll(value)); } + Models::PathProperties PathPropertiesFromBlobProperties( + Blobs::Models::BlobProperties properties, + const Azure::Core::Http::RawResponse& response) + { + Models::PathProperties ret; + ret.ETag = std::move(properties.ETag); + ret.LastModified = std::move(properties.LastModified); + ret.CreatedOn = std::move(properties.CreatedOn); + ret.Metadata = std::move(properties.Metadata); + ret.LeaseDuration = std::move(properties.LeaseDuration); + ret.LeaseState = std::move(properties.LeaseState); + ret.LeaseStatus = std::move(properties.LeaseStatus); + ret.HttpHeaders = std::move(properties.HttpHeaders); + ret.IsServerEncrypted = properties.IsServerEncrypted; + ret.EncryptionKeySha256 = std::move(properties.EncryptionKeySha256); + ret.EncryptionScope = std::move(properties.EncryptionScope); + ret.CopyId = std::move(properties.CopyId); + ret.CopySource = std::move(properties.CopySource); + ret.CopyStatus = std::move(properties.CopyStatus); + ret.CopyProgress = std::move(properties.CopyProgress); + ret.CopyCompletedOn = std::move(properties.CopyCompletedOn); + ret.ExpiresOn = std::move(properties.ExpiresOn); + ret.LastAccessedOn = std::move(properties.LastAccessedOn); + ret.FileSize = properties.BlobSize; + ret.ArchiveStatus = std::move(properties.ArchiveStatus); + ret.RehydratePriority = std::move(properties.RehydratePriority); + ret.CopyStatusDescription = std::move(properties.CopyStatusDescription); + ret.IsIncrementalCopy = std::move(properties.IsIncrementalCopy); + ret.IncrementalCopyDestinationSnapshot + = std::move(properties.IncrementalCopyDestinationSnapshot); + ret.VersionId = std::move(properties.VersionId); + ret.IsCurrentVersion = std::move(properties.IsCurrentVersion); + ret.IsDirectory = MetadataIndicatesIsDirectory(ret.Metadata); + const auto& headers = response.GetHeaders(); + auto encryptionContext = headers.find(EncryptionContextHeaderName); + if (encryptionContext != headers.end()) + { + ret.EncryptionContext = encryptionContext->second; + } + auto acl = headers.find(AclHeaderName); + if (acl != headers.end()) + { + ret.Acls = Models::Acl::DeserializeAcls(acl->second); + } + auto owner = headers.find(OwnerHeaderName); + if (owner != headers.end()) + { + ret.Owner = owner->second; + } + auto group = headers.find(GroupHeaderName); + if (group != headers.end()) + { + ret.Group = group->second; + } + auto permissions = headers.find(PermissionsHeaderName); + if (permissions != headers.end()) + { + ret.Permissions = permissions->second; + } + return ret; + } + Azure::Core::Url GetBlobUrlFromUrl(const Azure::Core::Url& url) { std::string host = url.GetHost(); diff --git a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp index b2d9116e7a..9ebbdb01dd 100644 --- a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp @@ -4,6 +4,7 @@ #pragma once #include "azure/storage/files/datalake/datalake_options.hpp" +#include "azure/storage/files/datalake/datalake_responses.hpp" #include #include @@ -31,6 +32,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam DateTime ParseFileTimeString(const std::string& value); + Models::PathProperties PathPropertiesFromBlobProperties( + Blobs::Models::BlobProperties properties, + const Azure::Core::Http::RawResponse& response); + /** * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. * diff --git a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp index ef3b411aae..bec68a5cf3 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp @@ -23,6 +23,8 @@ #include #include +#include + namespace Azure { namespace Storage { namespace Files { namespace Shares { constexpr static const int32_t DefaultListAllRangesPageSizeHint = 10000; @@ -1260,14 +1262,6 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "Buffer is not big enough, file range size is " + std::to_string(fileRangeSize) + "."); } - int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( - buffer, static_cast(firstChunkLength), context); - if (bytesRead != firstChunkLength) - { - throw Azure::Core::RequestFailedException("Error when reading body stream."); - } - firstChunk.Value.BodyStream.reset(); - auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadFileToResult ret; ret.FileSize = response.Value.FileSize; @@ -1276,9 +1270,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadFileOptions chunkOptions; @@ -1303,19 +1296,51 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); + lastChunkResponse = std::move(chunk); } }; int64_t remainingOffset = firstChunkOffset + firstChunkLength; int64_t remainingSize = fileRangeSize - firstChunkLength; - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( + buffer, static_cast(firstChunkLength), context); + if (bytesRead != firstChunkLength) + { + throw Azure::Core::RequestFailedException("Error when reading body stream."); + } + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = fileRangeSize; return ret; @@ -1388,8 +1413,6 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { }; _internal::FileWriter fileWriter(fileName); - bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadFileToResult ret; @@ -1399,9 +1422,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadFileOptions chunkOptions; @@ -1424,19 +1446,46 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); + lastChunkResponse = std::move(chunk); } }; int64_t remainingOffset = firstChunkOffset + firstChunkLength; int64_t remainingSize = fileRangeSize - firstChunkLength; - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = fileRangeSize; return ret; From d2769f5611d39d167c08d6391f1038fd03ac5a4d Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 13:14:25 +1100 Subject: [PATCH 08/13] Update Storage comments and documentation Collect standalone deletion remarks, existing API descriptions, and generated-model documentation without changing executable behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../storage/blobs/blob_container_client.hpp | 16 +++ .../inc/azure/storage/blobs/blob_options.hpp | 2 +- .../storage/blobs/blob_service_client.hpp | 8 ++ .../inc/azure/storage/blobs/rest_client.hpp | 119 ++++++------------ .../azure-storage-blobs/src/blob_client.cpp | 4 + .../azure-storage-blobs/swagger/README.md | 45 +++++++ .../src/storage_pipeline.cpp | 2 +- .../datalake/datalake_file_system_client.hpp | 18 +++ .../storage/files/shares/rest_client.hpp | 2 +- .../swagger/README.md | 3 + 10 files changed, 134 insertions(+), 85 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp index 12899e479e..a8eb25e6f0 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp @@ -153,6 +153,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified container for deletion. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. * @return A DeleteBlobContainerResult if successful. @@ -165,6 +173,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified container for deletion if it exists. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. * @return A DeleteBlobContainerResult if the container exists. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index 1693edf2e7..923b6fd4ab 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -609,7 +609,7 @@ namespace Azure { namespace Storage { namespace Blobs { enum class StorageResponseFormat { /** - * @brief Let the service choose the response format (default). + * @brief Use the default response format, which is currently Apache Arrow. */ Auto, /** diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp index bc81ae9b13..5a5380de3d 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp @@ -206,6 +206,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified blob container for deletion. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param blobContainerName The name of the container to delete. * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index 804072aa25..b3adf763ee 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -1663,12 +1663,12 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobType BlobType; }; /** - * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetProperties. + * @brief Properties of a blob. */ struct BlobProperties final { /** - * Array of ObjectReplicationPolicy. + * Object replication policies associated with the source blob. */ std::vector ObjectReplicationSourceProperties; /** @@ -1676,89 +1676,67 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable ImmutabilityPolicy; /** - * Standard HTTP properties supported by containers and blobs. + * Standard HTTP properties supported by the blob. */ BlobHttpHeaders HttpHeaders; /** - * Returns the date and time the blob was last modified. Any operation that modifies the blob, - * including an update of the blob's metadata or properties, changes the last-modified time of - * the blob. + * The date and time when the blob was last modified. */ DateTime LastModified; /** - * Returns the date and time the blob was created. + * The date and time when the blob was created. */ DateTime CreatedOn; /** - * A set of name-value pair associated with this blob. + * A set of name-value pairs associated with the blob. */ Core::CaseInsensitiveMap Metadata; /** - * Optional. Only valid when Object Replication is enabled for the storage container and on - * the destination blob of the replication. + * The object replication policy identifier for the destination blob. */ Nullable ObjectReplicationDestinationPolicyId; /** - * The blob's type. + * The type of the blob. */ Models::BlobType BlobType; /** - * Conclusion time of the last attempted Copy Blob operation where this blob was the - * destination blob. This value can specify the time of a completed, aborted, or failed copy - * attempt. This header does not appear if a copy is pending, if this blob has never been the - * destination in a Copy Blob operation, or if this blob has been modified after a concluded - * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List. + * The completion time of the last attempted copy operation. */ Nullable CopyCompletedOn; /** - * Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last - * fatal or non-fatal copy operation failure. This header does not appear if this blob has - * never been the destination in a Copy Blob operation, or if this blob has been modified - * after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block - * List. + * A description of the last copy operation status. */ Nullable CopyStatusDescription; /** - * String identifier for this copy operation. Use with Get Blob Properties to check the status - * of this copy operation, or pass to Abort Copy Blob to abort a pending copy. + * The identifier of the last copy operation. */ Nullable CopyId; /** - * Contains the number of bytes copied and the total bytes in the source in the last attempted - * Copy Blob operation where this blob was the destination blob. Can show between 0 and - * Content-Length bytes copied. This header does not appear if this blob has never been the - * destination in a Copy Blob operation, or if this blob has been modified after a concluded - * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List. + * The progress of the last copy operation. */ Nullable CopyProgress; /** - * URL up to 2 KB in length that specifies the source blob or file used in the last attempted - * Copy Blob operation where this blob was the destination blob. This header does not appear - * if this blob has never been the destination in a Copy Blob operation, or if this blob has - * been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or - * Put Block List. + * The source of the last copy operation. */ Nullable CopySource; /** - * State of the copy operation identified by x-ms-copy-id. + * The status of the last copy operation. */ Nullable CopyStatus; /** - * Included if the blob is incremental copy blob. + * Indicates whether the blob is an incremental copy. */ Nullable IsIncrementalCopy; /** - * Included if the blob is incremental copy blob or incremental copy snapshot, if - * x-ms-copy-status is success. Snapshot time of the last successful incremental copy snapshot - * for this blob. + * The destination snapshot of the last incremental copy. */ Nullable IncrementalCopyDestinationSnapshot; /** - * When a blob is leased, specifies whether the lease is of infinite or fixed duration. + * The duration of the blob lease. */ Nullable LeaseDuration; /** - * Lease state of the blob. + * The current lease state of the blob. */ Nullable LeaseState; /** @@ -1766,80 +1744,59 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable LeaseStatus; /** - * Size of the blob in bytes. + * The size of the blob in bytes. */ std::int64_t BlobSize = std::int64_t(); /** - * The ETag contains a value that you can use to perform operations conditionally. If the - * request version is 2011-08-18 or newer, the ETag value will be in quotes. + * The entity tag of the blob. */ Azure::ETag ETag; /** - * The current sequence number for a page blob. This header is not returned for block blobs or - * append blobs. + * The current sequence number of a page blob. */ Nullable SequenceNumber; /** - * The number of committed blocks present in the blob. This header is returned only for append - * blobs. + * The number of committed blocks in an append blob. */ Nullable CommittedBlockCount; /** - * The value of this header is set to true if the blob data and application metadata are - * completely encrypted using the specified algorithm. Otherwise, the value is set to false - * (when the blob is unencrypted, or if only parts of the blob/application metadata are - * encrypted). + * Indicates whether the blob is encrypted by the service. */ bool IsServerEncrypted = bool(); /** - * The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only - * returned when the metadata was encrypted with a customer-provided key. + * The SHA-256 hash of the customer-provided encryption key. */ Nullable> EncryptionKeySha256; /** - * Returns the name of the encryption scope used to encrypt the blob contents and application - * metadata. Note that the absence of this header implies use of the default account - * encryption scope. + * The encryption scope used to encrypt the blob. */ Nullable EncryptionScope; /** - * The tier of page blob on a premium storage account or tier of block blob on blob storage - * LRS accounts. For a list of allowed premium page blob tiers, see - * https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd. For blob - * storage LRS accounts, valid values are Hot/Cool/Archive. + * The access tier of the blob. */ Nullable AccessTier; /** - * For page blobs on a premium storage account only. If the access tier is not explicitly set - * on the blob, the tier is inferred based on its content length and this header will be - * returned with true value. + * Indicates whether the access tier was inferred. */ Nullable IsAccessTierInferred; /** - * For blob storage LRS accounts, valid values are - * rehydrate-pending-to-hot/rehydrate-pending-to-cool. If the blob is being rehydrated and is - * not complete then this header is returned indicating that rehydrate is pending and also - * tells the destination tier. + * The archive rehydration status of the blob. */ Nullable ArchiveStatus; /** - * The time the tier was changed on the object. This is only returned if the tier on the block - * blob was ever set. + * The time when the access tier was last changed. */ Nullable AccessTierChangedOn; /** - * The underlying tier of a smart tier blob. Only returned if the blob is in Smart tier. + * The underlying access tier of a smart-tier blob. */ Nullable SmartAccessTier; /** - * A DateTime value returned by the service that uniquely identifies the blob. The value of - * this header indicates the blob version, and may be used in subsequent requests to access - * this version of the blob. + * The version identifier of the blob. */ Nullable VersionId; /** - * The value of this header indicates whether version of this blob is a current version, see - * also x-ms-version-id header. + * Indicates whether this is the current blob version. */ Nullable IsCurrentVersion; /** @@ -1847,25 +1804,23 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable TagCount; /** - * The time this blob will expire. + * The time when the blob will expire. */ Nullable ExpiresOn; /** - * If this blob has been sealed. + * Indicates whether the append blob is sealed. */ Nullable IsSealed; /** - * If an object is in rehydrate pending state then this header is returned with priority of - * rehydrate. Valid values are High and Standard. + * The priority of an archive rehydration operation. */ Nullable RehydratePriority; /** - * UTC date/time value generated by the service that indicates the time at which the blob was - * last read or written to. + * The time when the blob was last read or written. */ Nullable LastAccessedOn; /** - * Indicates if a legal hold is present on the blob. + * Indicates whether the blob has a legal hold. */ bool HasLegalHold = false; }; diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 8bbb13ab20..afcda50efd 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -583,6 +583,8 @@ namespace Azure { namespace Storage { namespace Blobs { DownloadBlobOptions firstChunkOptions; firstChunkOptions.Range = options.Range; + // Do not synthesize a range when the caller did not provide one. A ranged GET fails with 416 + // for an empty blob; an unbounded response is still consumed only up to firstChunkLength. if (firstChunkOptions.Range.HasValue()) { firstChunkOptions.Range.Value().Length = firstChunkLength; @@ -735,6 +737,8 @@ namespace Azure { namespace Storage { namespace Blobs { DownloadBlobOptions firstChunkOptions; firstChunkOptions.Range = options.Range; + // Do not synthesize a range when the caller did not provide one. A ranged GET fails with 416 + // for an empty blob; an unbounded response is still consumed only up to firstChunkLength. if (firstChunkOptions.Range.HasValue()) { firstChunkOptions.Range.Value().Length = firstChunkLength; diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 05ef91ce67..2cd7b49b10 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -1216,6 +1216,51 @@ directive: for (const propertyName in $.BlobProperties.properties) { $.BlobProperties.properties[propertyName]["x-ms-xml"] = {"name": ""}; } + $.BlobProperties.description = "Properties of a blob."; + const descriptions = { + "ObjectReplicationSourceProperties": "Object replication policies associated with the source blob.", + "ImmutabilityPolicy": "Immutability policy associated with the blob.", + "HttpHeaders": "Standard HTTP properties supported by the blob.", + "LastModified": "The date and time when the blob was last modified.", + "CreatedOn": "The date and time when the blob was created.", + "Metadata": "A set of name-value pairs associated with the blob.", + "ObjectReplicationDestinationPolicyId": "The object replication policy identifier for the destination blob.", + "BlobType": "The type of the blob.", + "CopyCompletedOn": "The completion time of the last attempted copy operation.", + "CopyStatusDescription": "A description of the last copy operation status.", + "CopyId": "The identifier of the last copy operation.", + "CopyProgress": "The progress of the last copy operation.", + "CopySource": "The source of the last copy operation.", + "CopyStatus": "The status of the last copy operation.", + "IsIncrementalCopy": "Indicates whether the blob is an incremental copy.", + "IncrementalCopyDestinationSnapshot": "The destination snapshot of the last incremental copy.", + "LeaseDuration": "The duration of the blob lease.", + "LeaseState": "The current lease state of the blob.", + "LeaseStatus": "The current lease status of the blob.", + "BlobSize": "The size of the blob in bytes.", + "ETag": "The entity tag of the blob.", + "SequenceNumber": "The current sequence number of a page blob.", + "CommittedBlockCount": "The number of committed blocks in an append blob.", + "IsServerEncrypted": "Indicates whether the blob is encrypted by the service.", + "EncryptionKeySha256": "The SHA-256 hash of the customer-provided encryption key.", + "EncryptionScope": "The encryption scope used to encrypt the blob.", + "AccessTier": "The access tier of the blob.", + "IsAccessTierInferred": "Indicates whether the access tier was inferred.", + "ArchiveStatus": "The archive rehydration status of the blob.", + "AccessTierChangedOn": "The time when the access tier was last changed.", + "SmartAccessTier": "The underlying access tier of a smart-tier blob.", + "VersionId": "The version identifier of the blob.", + "IsCurrentVersion": "Indicates whether this is the current blob version.", + "TagCount": "The number of tags associated with the blob.", + "ExpiresOn": "The time when the blob will expire.", + "IsSealed": "Indicates whether the append blob is sealed.", + "RehydratePriority": "The priority of an archive rehydration operation.", + "LastAccessedOn": "The time when the blob was last read or written.", + "HasLegalHold": "Indicates whether the blob has a legal hold." + }; + for (const propertyName in descriptions) { + $.BlobProperties.properties[propertyName].description = descriptions[propertyName]; + } - from: swagger-document where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses["200"] transform: > diff --git a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp index c33886d499..d3fa8463e1 100644 --- a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp +++ b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp @@ -71,7 +71,7 @@ namespace Azure { namespace Storage { namespace _internal { policies.emplace_back(policy->Clone()); } - // SharedKey/SAS auth runs last among per-retry policies so the signature covers + // Request-signing auth runs last among per-retry policies so the signature covers // the final, fully-modified request. if (storagePipelineOptions.SharedKeyAuthPolicy) { diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index d7bfd9a560..16d63cf000 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -118,6 +118,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * @brief Deletes the file system. + * + * @remark When a file system is deleted, a file system with the same name cannot be created for + * at least 30 seconds. The file system might not be available for more than 30 seconds if the + * service is still processing the request. While the file system is being deleted, attempts to + * create a file system with the same name fail with status code 409 (Conflict). Operations on + * paths within the file system, including reads and writes, may continue to succeed for up to + * 30 seconds after the delete request is accepted. After this period, all operations on the + * file system and its paths fail with status code 404 (Not Found). + * * @param options Optional parameters to delete this file system. * @param context Context for cancelling long running operations. * @return Azure::Response containing the @@ -130,6 +139,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * @brief Deletes the file system if it exists. + * + * @remark When a file system is deleted, a file system with the same name cannot be created for + * at least 30 seconds. The file system might not be available for more than 30 seconds if the + * service is still processing the request. While the file system is being deleted, attempts to + * create a file system with the same name fail with status code 409 (Conflict). Operations on + * paths within the file system, including reads and writes, may continue to succeed for up to + * 30 seconds after the delete request is accepted. After this period, all operations on the + * file system and its paths fail with status code 404 (Not Found). + * * @param options Optional parameters to delete this file system. * @param context Context for cancelling long running operations. * @return Azure::Response containing the diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index 3c6b9cd572..0be9e9c959 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -1220,7 +1220,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { AZ_STORAGE_FILES_SHARES_DLLEXPORT const static FilePropertySemantics Restore; }; /** - * @brief NFS only. Type of the file or directory. + * @brief Type of the file. */ class NfsFileType final : public Core::_internal::ExtendableEnumeration { public: diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index 240dea708b..c6f72363b1 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -610,6 +610,9 @@ directive: ### GetShareProperties +The change-feed enablement header is parsed case-insensitively by the handwritten +`ShareClient::GetProperties` implementation, while the public field remains `Nullable`. + ```yaml directive: - from: swagger-document From b526ea6fec69f6582fa117e23805a355c13c1b9a Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 19:54:37 +1100 Subject: [PATCH 09/13] Allow Storage service identifiers in spelling checks Recognize the sovereign-cloud endpoint suffixes and service-defined xfiles header. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .vscode/cspell.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index a973802240..4555949b78 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -98,6 +98,7 @@ "cgmanifest", "Chelnokov", "chewi", + "chinacloudapi", "Chukwu", "cmake", "CMOCKA", @@ -291,6 +292,7 @@ "unskipped", "UPNs", "usgov", + "usgovcloudapi", "usgoviowa", "usgovvirginia", "vcpkg", @@ -315,6 +317,7 @@ "Xclang", "xcode", "xcodebuild", + "xfiles", "XSMB", "zulu" ], From 3a8e3c709d8c83aed79586b40ba7bf1a434a4b3f Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Tue, 6 Oct 2026 17:48:25 +1100 Subject: [PATCH 10/13] Add Storage feature tests and recordings Collect feature tests, fixture migrations, regression assertions, shared test helpers, test build definitions, and the published recording pin. Include default-disabled locality coverage and playback-only Files cases pending production enablement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/assets.json | 2 +- .../test/ut/CMakeLists.txt | 2 + .../test/ut/blob_container_client_test.cpp | 20 +- .../test/ut/block_blob_client_test.cpp | 12 + .../test/ut/data_locality_test.cpp | 858 ++++++++++++++++++ .../test/ut/session_authentication_test.cpp | 666 ++++++++++++++ .../test/ut/storage_retry_policy_test.cpp | 1 + .../test/ut/storage_timeout_test.cpp | 25 - .../test/ut/CMakeLists.txt | 1 + .../test/ut/storage_url_test.cpp | 127 +++ .../test/ut/test_base.hpp | 25 + .../test/ut/CMakeLists.txt | 2 + .../test/ut/data_locality_test.cpp | 395 ++++++++ .../test/ut/datalake_file_client_test.cpp | 16 +- .../test/ut/session_authentication_test.cpp | 244 +++++ .../test/ut/share_client_test.cpp | 179 +++- .../test/ut/share_file_client_test.cpp | 54 +- .../test/ut/share_service_client_test.cpp | 8 +- 18 files changed, 2587 insertions(+), 50 deletions(-) create mode 100644 sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp create mode 100644 sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp create mode 100644 sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index 818eca0575..9f08d82d12 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_6016ecf9a6" + "Tag": "cpp/storage_01f138ad9c" } diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index ad37169887..46474f1672 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -29,9 +29,11 @@ add_executable ( block_blob_client_test.cpp block_blob_client_test.hpp connection_reuse_test.cpp + data_locality_test.cpp macro_guard.cpp page_blob_client_test.cpp page_blob_client_test.hpp + session_authentication_test.cpp simplified_header_test.cpp storage_retry_policy_test.cpp storage_timeout_test.cpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 707c3100b1..6eb1d6869d 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -103,6 +103,7 @@ namespace Azure { namespace Storage { namespace Test { Blobs::Models::ListBlobsIncludeFlags include) { Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = blobName; options.Include = include; for (auto page = containerClient.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) @@ -239,7 +240,7 @@ namespace Azure { namespace Storage { namespace Test { } for (const auto format : - {Blobs::StorageResponseFormat::Arrow, Blobs::StorageResponseFormat::Xml}) + {Blobs::StorageResponseFormat::Auto, Blobs::StorageResponseFormat::Xml}) { Azure::Storage::Blobs::ListBlobsOptions options; @@ -256,11 +257,18 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE( pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); + const auto& contentType + = pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderContentType); if (format == Blobs::StorageResponseFormat::Xml) { + EXPECT_NE(contentType.find(_internal::ContentTypeXml), std::string::npos); EXPECT_FALSE(pageResult.ServiceEndpoint.empty()); EXPECT_EQ(pageResult.BlobContainerName, m_containerName); } + else + { + EXPECT_NE(contentType.find(_internal::ContentTypeApacheArrowStream), std::string::npos); + } for (const auto& blob : pageResult.Blobs) { EXPECT_FALSE(blob.Name.empty()); @@ -308,6 +316,7 @@ namespace Azure { namespace Storage { namespace Test { for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + EXPECT_EQ(pageResult.Prefix, prefix1); for (const auto& blob : pageResult.Blobs) { listBlobs.insert(blob.Name); @@ -343,6 +352,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.PageSizeHint = 3; options.StartFrom = startFrom; std::set listBlobs; @@ -458,6 +468,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = prefix; std::set items; for (auto pageResult = containerClient.ListBlobsByHierarchy(delimiter, options); @@ -530,6 +541,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = prefix; options.StartFrom = startFrom; std::set items; @@ -591,6 +603,7 @@ namespace Azure { namespace Storage { namespace Test { blobClient.AppendBlock(contentStream); Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = blobName; options.Include = Blobs::Models::ListBlobsIncludeFlags::Snapshots | Blobs::Models::ListBlobsIncludeFlags::Versions @@ -1666,6 +1679,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); Blobs::Models::BlobItem blobItem; Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = "aaaaa"; for (auto page = containerClient.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) { @@ -1679,7 +1693,9 @@ namespace Azure { namespace Storage { namespace Test { } EXPECT_EQ(blobItem.Name, blobName); bool found = false; - for (auto page = containerClient.ListBlobsByHierarchy("/"); page.HasPage(); + Blobs::ListBlobsOptions hierarchyOptions; + hierarchyOptions.ResponseFormat = Blobs::StorageResponseFormat::Xml; + for (auto page = containerClient.ListBlobsByHierarchy("/", hierarchyOptions); page.HasPage(); page.MoveToNextPage()) { for (auto& p : page.BlobPrefixes) diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index 3f4be369a2..58cf27dd3c 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -32,6 +32,18 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { + TEST(BlockBlobClientOptionsTest, DefaultTransferSizes) + { + Blobs::DownloadBlobToOptions downloadOptions; + EXPECT_EQ( + downloadOptions.TransferOptions.InitialChunkSize, + downloadOptions.TransferOptions.ChunkSize); + EXPECT_EQ(downloadOptions.TransferOptions.InitialChunkSize, 4_MB); + + Blobs::UploadBlockBlobFromOptions uploadOptions; + EXPECT_EQ(uploadOptions.TransferOptions.SingleUploadThreshold, 4_MB); + } + void BlockBlobClientTest::SetUp() { BlobContainerClientTest::SetUp(); diff --git a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp new file mode 100644 index 0000000000..2e5e7c15ed --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp @@ -0,0 +1,858 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "../../src/private/data_locality.hpp" +#include "test/ut/test_base.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + struct LocalityRequest final + { + std::string Host; + std::string HostHeader; + std::string IfMatch; + std::string Range; + bool IsLayout = false; + }; + + struct LocalityState final + { + std::mutex Mutex; + std::vector Requests; + Core::Http::HttpStatusCode LayoutStatus = Core::Http::HttpStatusCode::Ok; + bool PaginateLayout = false; + bool ReturnDownloadHint = true; + std::string DownloadHint = "layout"; + }; + + class LocalityTransportPolicy final : public Core::Http::Policies::HttpPolicy { + public: + LocalityTransportPolicy( + std::shared_ptr state, + std::shared_ptr data) + : m_state(std::move(state)), m_data(std::move(data)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + Core::Context const& context) const override + { + (void)nextPolicy; + (void)context; + + const auto query = request.GetUrl().GetQueryParameters(); + const bool isLayout = query.find("comp") != query.end() && query.at("comp") == "layout"; + const auto& headers = request.GetHeaders(); + const auto hostHeader = headers.find("host"); + const auto ifMatchHeader = headers.find("if-match"); + const auto rangeHeader = headers.find("x-ms-range"); + + { + std::lock_guard lock(m_state->Mutex); + m_state->Requests.push_back(LocalityRequest{ + request.GetUrl().GetHost(), + hostHeader == headers.end() ? std::string() : hostHeader->second, + ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, + rangeHeader == headers.end() ? std::string() : rangeHeader->second, + isLayout}); + } + + if (isLayout) + { + auto setLayoutPropertyHeaders = [&](Core::Http::RawResponse& response) { + response.SetHeader("etag", "\"locality-etag\""); + response.SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response.SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response.SetHeader("x-ms-meta-layout", "metadata"); + response.SetHeader("x-ms-blob-content-length", std::to_string(m_data->size())); + response.SetHeader("x-ms-blob-content-type", "application/octet-stream"); + response.SetHeader("x-ms-blob-content-encoding", "identity"); + response.SetHeader("x-ms-blob-type", "BlockBlob"); + response.SetHeader("x-ms-server-encrypted", "true"); + }; + if (m_state->LayoutStatus == Core::Http::HttpStatusCode::NoContent) + { + auto response = std::make_unique( + 1, 1, Core::Http::HttpStatusCode::NoContent, "No Content"); + setLayoutPropertyHeaders(*response); + return response; + } + if (m_state->LayoutStatus != Core::Http::HttpStatusCode::Ok) + { + const std::string errorBody + = "MockLayoutErrorlayout error"; + auto response = std::make_unique( + 1, 1, m_state->LayoutStatus, "layout error"); + response->SetBody(std::vector(errorBody.begin(), errorBody.end())); + response->SetHeader("content-type", "application/xml"); + response->SetHeader("x-ms-error-code", "MockLayoutError"); + return response; + } + + const bool secondPage + = query.find("marker") != query.end() && query.at("marker") == "page2"; + std::string ranges; + std::string nextMarker; + if (!m_state->PaginateLayout) + { + ranges = "" + "" + "" + ""; + } + else if (!secondPage) + { + ranges = "" + ""; + nextMarker = "page2"; + } + else + { + ranges = "" + ""; + } + const std::string layout = "" + ranges + + "" + "" + "" + "" + + nextMarker + "4"; + auto response = std::make_unique( + 1, 1, Core::Http::HttpStatusCode::Ok, "OK"); + response->SetBody(std::vector(layout.begin(), layout.end())); + response->SetHeader("content-type", "application/xml"); + setLayoutPropertyHeaders(*response); + return response; + } + + int64_t offset = 0; + int64_t length = static_cast(m_data->size()); + if (rangeHeader != headers.end()) + { + const auto separator = rangeHeader->second.find('-'); + offset = std::stoll(rangeHeader->second.substr(6, separator - 6)); + const int64_t end = std::stoll(rangeHeader->second.substr(separator + 1)); + length = (std::min)(end - offset + 1, static_cast(m_data->size()) - offset); + } + + auto response = std::make_unique( + 1, + 1, + rangeHeader == headers.end() ? Core::Http::HttpStatusCode::Ok + : Core::Http::HttpStatusCode::PartialContent, + "OK"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(m_data->data() + offset), + static_cast(length))); + response->SetHeader("content-length", std::to_string(length)); + if (rangeHeader != headers.end()) + { + response->SetHeader( + "content-range", + "bytes " + std::to_string(offset) + "-" + std::to_string(offset + length - 1) + "/" + + std::to_string(m_data->size())); + } + response->SetHeader("etag", "\"locality-etag\""); + response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response->SetHeader("x-ms-blob-content-length", std::to_string(m_data->size())); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + if (m_state->ReturnDownloadHint) + { + response->SetHeader("x-ms-download-hint", m_state->DownloadHint); + } + return response; + } + + private: + std::shared_ptr m_state; + std::shared_ptr m_data; + }; + + Blobs::BlobClient CreateLocalityClient( + const std::shared_ptr& state, + const std::shared_ptr& data) + { + Blobs::BlobClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state, data)); + return Blobs::BlobClient("https://primary.test/container/blob", options); + } + + Blobs::BlobClient CreateLocalityClientFromService( + const std::shared_ptr& state, + const std::shared_ptr& data) + { + Blobs::BlobClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state, data)); + return Blobs::BlobServiceClient("https://primary.test", options) + .GetBlobContainerClient("container") + .GetBlobClient("blob"); + } + + Blobs::DownloadBlobToOptions CreateDownloadOptions() + { + Blobs::DownloadBlobToOptions options; + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Enabled; + options.TransferOptions.InitialChunkSize = 128 * 1024; + options.TransferOptions.ChunkSize = 64 * 1024; + options.TransferOptions.Concurrency = 4; + return options; + } + + Blobs::Models::BlobLayoutRange MakeLayoutRange( + int64_t offset, + int64_t length, + std::string endpoint) + { + Blobs::Models::BlobLayoutRange range; + range.Range.Offset = offset; + range.Range.Length = length; + range.Endpoint = std::move(endpoint); + return range; + } + + Blobs::_detail::DataLocalityLayout CreateLayout(const std::string& endpoint) + { + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = {MakeLayoutRange(0, 1024 * 1024, endpoint)}; + return layout; + } + } // namespace + + class DataLocalityRecordedTest : public StorageTest { + }; + + TEST(DataLocalityTest, SelectsIdealEndpoint) + { + using Range = Blobs::Models::BlobLayoutRange; + struct TestCase final + { + const char* Name; + int64_t Offset; + int64_t Length; + std::vector Ranges; + std::string ExpectedEndpoint; + }; + + const auto maxOffset = (std::numeric_limits::max)(); + const std::vector testCases{ + {"NegativeOffset", -1, 1, {MakeLayoutRange(0, 10, "A")}, ""}, + {"ZeroLength", 0, 0, {MakeLayoutRange(0, 10, "A")}, ""}, + {"NegativeLength", 0, -1, {MakeLayoutRange(0, 10, "A")}, ""}, + {"EmptyLayout", 0, 1, {}, ""}, + {"BeforeLayout", 0, 5, {MakeLayoutRange(10, 10, "A")}, ""}, + {"AfterLayout", 20, 1, {MakeLayoutRange(10, 10, "A")}, ""}, + {"RequestStartsInGap", 5, 10, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"ExactRangeStart", 10, 1, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"ExactRangeEnd", 19, 1, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"SkipsEarlierRanges", + 100, + 1, + {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(100, 10, "B")}, + "B"}, + {"LargestOverlap", + 200 * 1024, + 400 * 1024, + {MakeLayoutRange(0, 256 * 1024, "A"), + MakeLayoutRange(256 * 1024, 256 * 1024, "B"), + MakeLayoutRange(512 * 1024, 256 * 1024, "A")}, + "B"}, + {"AggregatesDisjointRanges", + 5, + 25, + {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(10, 10, "B"), MakeLayoutRange(20, 10, "A")}, + "A"}, + {"IgnoresLayoutGaps", + 0, + 30, + {MakeLayoutRange(0, 5, "A"), MakeLayoutRange(20, 10, "B")}, + "B"}, + {"RequestEndAtMax", maxOffset - 1, 2, {MakeLayoutRange(maxOffset - 9, 10, "A")}, "A"}, + {"SingleByteAtMax", maxOffset, 1, {MakeLayoutRange(maxOffset, 1, "A")}, "A"}, + {"RequestEndOverflow", maxOffset - 4, 10, {MakeLayoutRange(maxOffset - 9, 10, "A")}, "A"}, + }; + + for (const auto& testCase : testCases) + { + SCOPED_TRACE(testCase.Name); + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = testCase.Ranges; + EXPECT_EQ( + Blobs::_detail::GetIdealDataLocalityEndpoint(testCase.Offset, testCase.Length, layout), + testCase.ExpectedEndpoint); + } + } + + TEST(DataLocalityTest, SelectsEitherEndpointForEqualOverlap) + { + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(10, 10, "B")}; + + const auto endpoint = Blobs::_detail::GetIdealDataLocalityEndpoint(5, 10, layout); + EXPECT_TRUE(endpoint == "A" || endpoint == "B"); + } + + TEST(DataLocalityTest, FreshLayoutDoesNotRefresh) + { + std::atomic refreshCount{0}; + Blobs::_detail::DataLocalityLayoutState state([&]() { + ++refreshCount; + return CreateLayout("A"); + }); + + state.WaitForInitialLayout(); + EXPECT_EQ(refreshCount, 1); + EXPECT_EQ(state.GetEndpoint(0, 1), "A"); + EXPECT_EQ(refreshCount, 1); + } + + TEST(DataLocalityTest, GetLayoutNoContentReturnsBlobProperties) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; + auto data = std::make_shared(1024 * 1024, 'n'); + auto client = CreateLocalityClient(state, data); + + auto page = client.GetLayout(); + + EXPECT_TRUE(page.Layout.Ranges.empty()); + EXPECT_EQ(page.Layout.Properties.ETag.ToString(), "\"locality-etag\""); + EXPECT_EQ(page.Layout.Properties.BlobSize, static_cast(data->size())); + EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); + EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); + } + + TEST_F(DataLocalityRecordedTest, GetLayoutMultipleEndpoints_PLAYBACKONLY_) + { + auto clientOptions = InitStorageClientOptions(); + auto blobClient = Blobs::BlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), + "data-locality-test", + "data-locality-multi-endpoint.bin", + clientOptions); + + std::vector ranges; + std::set endpoints; + auto page = blobClient.GetLayout(); + EXPECT_EQ(page.Layout.Properties.BlobSize, 64 * 1024 * 1024); + EXPECT_EQ(page.Layout.Properties.BlobType, Blobs::Models::BlobType::BlockBlob); + EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); + EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted); + for (; page.HasPage(); page.MoveToNextPage()) + { + for (const auto& range : page.Layout.Ranges) + { + EXPECT_GE(range.Range.Offset, 0); + ASSERT_TRUE(range.Range.Length.HasValue()); + EXPECT_GT(range.Range.Length.Value(), 0); + EXPECT_FALSE(range.Endpoint.empty()); + ranges.push_back(range); + endpoints.insert(range.Endpoint); + } + } + + if (m_testContext.IsLiveMode() && endpoints.size() < 2) + { + GTEST_SKIP() << "The configured live account does not return a multi-endpoint layout."; + } + ASSERT_GT(ranges.size(), 1U); + EXPECT_GE(endpoints.size(), 2U); + + const auto differentEndpoint + = std::adjacent_find(ranges.begin(), ranges.end(), [](auto const& left, auto const& right) { + return left.Endpoint != right.Endpoint; + }); + ASSERT_NE(differentEndpoint, ranges.end()); + const auto nextRange = differentEndpoint + 1; + ASSERT_GE(nextRange->Range.Offset, 2); + + constexpr int64_t downloadLength = 5; + const int64_t downloadOffset = nextRange->Range.Offset - 2; + Blobs::DownloadBlobOptions baselineOptions; + baselineOptions.Range = Core::Http::HttpRange(); + baselineOptions.Range.Value().Offset = downloadOffset; + baselineOptions.Range.Value().Length = downloadLength; + auto downloadResponse = blobClient.Download(baselineOptions); + auto content = downloadResponse.Value.BodyStream->ReadToEnd(); + ASSERT_EQ(content.size(), static_cast(downloadLength)); + ASSERT_TRUE(downloadResponse.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ( + downloadResponse.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + + TEST_F(DataLocalityRecordedTest, GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_) + { + auto clientOptions = InitStorageClientOptions(); + auto client = Blobs::BlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), + "data-locality-test", + "data-locality-multi-endpoint.bin", + clientOptions); + constexpr int64_t blobSize = 64 * 1024 * 1024; + for (bool bounded : {true, false}) + { + SCOPED_TRACE(bounded); + Blobs::GetBlobLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : blobSize - 16; + if (bounded) + { + options.Range.Value().Length = 5; + } + const auto offset = options.Range.Value().Offset; + bool coversOffset = false; + size_t rangeCount = 0; + for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) + { + EXPECT_EQ(page.Layout.Properties.BlobSize, blobSize); + EXPECT_GT(page.Layout.Properties.BlobSize, options.Range.Value().Length.ValueOr(16)); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + ASSERT_GT(range.Range.Length.Value(), 0); + ASSERT_GE(range.Range.Offset, 0); + ASSERT_LE(range.Range.Offset, blobSize); + ASSERT_LE(range.Range.Length.Value(), blobSize - range.Range.Offset); + EXPECT_FALSE(range.Endpoint.empty()); + coversOffset = coversOffset + || (range.Range.Offset <= offset + && offset - range.Range.Offset < range.Range.Length.Value()); + ++rangeCount; + } + } + EXPECT_GT(rangeCount, 0U); + EXPECT_TRUE(coversOffset); + } + } + + TEST(DataLocalityTest, RoutesDownloadChunks) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, '\0'); + for (size_t i = 0; i < data->size(); ++i) + { + (*data)[i] = static_cast(i % 127); + } + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + auto response = client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + + EXPECT_EQ(response.Value.BlobSize, static_cast(data->size())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 16U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Host, "primary.test"); + EXPECT_TRUE(state->Requests[0].Range.empty()); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_EQ(state->Requests[1].Range, "bytes=131072-1048575"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + for (auto request = state->Requests.begin() + 2; request != state->Requests.end(); ++request) + { + EXPECT_FALSE(request->IsLayout); + EXPECT_TRUE(request->Host == "locality0.test" || request->Host == "locality1.test"); + EXPECT_EQ(request->HostHeader, "primary.test"); + EXPECT_EQ(request->IfMatch, "\"locality-etag\""); + EXPECT_FALSE(request->Range.empty()); + const auto separator = request->Range.find('-'); + const auto offset = std::stoll(request->Range.substr(6, separator - 6)); + EXPECT_EQ( + request->Host, (offset / (256 * 1024)) % 2 == 0 ? "locality0.test" : "locality1.test"); + } + } + + TEST(DataLocalityTest, RoutesFileDownloadChunks) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'f'); + auto client = CreateLocalityClientFromService(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + + auto response = client.DownloadTo(fileName, CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, static_cast(data->size())); + + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_EQ(downloaded, *data); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 16U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Host, "primary.test"); + EXPECT_TRUE(state->Requests[0].Range.empty()); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (auto request = state->Requests.begin() + 2; request != state->Requests.end(); ++request) + { + EXPECT_TRUE(request->Host == "locality0.test" || request->Host == "locality1.test"); + EXPECT_EQ(request->HostHeader, "primary.test"); + EXPECT_EQ(request->IfMatch, "\"locality-etag\""); + } + } + + TEST(DataLocalityTest, CompletesSmallBufferDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 's'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + + TEST(DataLocalityTest, DownloadsEmptyBlobWithoutRange) + { + auto data = std::make_shared(); + + { + auto state = std::make_shared(); + auto client = CreateLocalityClient(state, data); + std::vector buffer; + + auto response = client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, 0); + EXPECT_EQ(response.Value.ContentRange.Length.Value(), 0); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + + { + auto state = std::make_shared(); + auto client = CreateLocalityClient(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + + auto response = client.DownloadTo(fileName, CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, 0); + EXPECT_EQ(response.Value.ContentRange.Length.Value(), 0); + + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_TRUE(downloaded.empty()); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + } + + TEST(DataLocalityTest, CompletesSingleRangeDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + constexpr int64_t offset = 200 * 1024; + constexpr int64_t length = 400 * 1024; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = length; + options.TransferOptions.InitialChunkSize = length; + + std::vector buffer(length); + client.DownloadTo(buffer.data(), buffer.size(), options); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin() + offset)); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Range, "bytes=204800-614399"); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + } + + TEST(DataLocalityTest, CompletesSmallFileDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 'f'); + auto client = CreateLocalityClient(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 0; + options.Range.Value().Length = static_cast(data->size()); + + client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_EQ(downloaded, *data); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + } + + TEST(DataLocalityTest, DoesNotFetchLayoutWithoutDownloadHint) + { + auto state = std::make_shared(); + state->ReturnDownloadHint = false; + auto data = std::make_shared(1024 * 1024, 'h'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (const auto& request : state->Requests) + { + EXPECT_FALSE(request.IsLayout); + EXPECT_EQ(request.Host, "primary.test"); + } + } + + TEST(DataLocalityTest, AutoEnablesLayoutAwareRouting) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'a'); + auto client = CreateLocalityClient(state, data); + auto options = CreateDownloadOptions(); + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Auto; + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), options); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_TRUE(state->Requests[1].IsLayout); + } + + TEST(DataLocalityTest, DefaultAndDisabledSkipLayoutAwareRouting) + { + for (bool useDefault : {true, false}) + { + for (bool toFile : {false, true}) + { + SCOPED_TRACE(useDefault); + SCOPED_TRACE(toFile); + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'd'); + auto client = CreateLocalityClient(state, data); + Blobs::DownloadBlobToOptions options; + EXPECT_EQ(options.LayoutAwareRouting, Blobs::LayoutAwareRouting::Disabled); + options.TransferOptions = CreateDownloadOptions().TransferOptions; + if (!useDefault) + { + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; + } + + std::vector content(data->size()); + if (toFile) + { + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + content.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + file.close(); + std::remove(fileName.c_str()); + } + else + { + client.DownloadTo(content.data(), content.size(), options); + } + EXPECT_TRUE(std::equal(content.begin(), content.end(), data->begin(), data->end())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (const auto& request : state->Requests) + { + EXPECT_FALSE(request.IsLayout); + EXPECT_EQ(request.Host, "primary.test"); + } + } + } + } + + TEST(DataLocalityTest, AcceptsCaseInsensitiveDownloadHint) + { + auto state = std::make_shared(); + state->DownloadHint = "LAYOUT"; + auto data = std::make_shared(1024 * 1024, 'h'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + } + + TEST(DataLocalityTest, RoutesToEndpointWithLargestOverlap) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + constexpr int64_t offset = 200 * 1024; + constexpr int64_t length = 400 * 1024; + constexpr int64_t initialLength = 64 * 1024; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = length; + options.TransferOptions.InitialChunkSize = initialLength; + options.TransferOptions.ChunkSize = length - initialLength; + + std::vector buffer(length); + client.DownloadTo(buffer.data(), buffer.size(), options); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin() + offset)); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 3U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Range, "bytes=204800-270335"); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_EQ(state->Requests[1].Range, "bytes=270336-614399"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_FALSE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests[2].Range, "bytes=270336-614399"); + EXPECT_EQ(state->Requests[2].Host, "locality1.test"); + } + + TEST(DataLocalityTest, ReadsAllLayoutPages) + { + auto state = std::make_shared(); + state->PaginateLayout = true; + auto data = std::make_shared(1024 * 1024, 'p'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 17U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_TRUE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_EQ(state->Requests[2].IfMatch, "\"locality-etag\""); + } + + TEST(DataLocalityTest, RoutesOneShotDownloadToLayoutEndpoint) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + + Blobs::DownloadBlobOptions options; + options.LayoutEndpoint = "locality0.test:443"; + auto response = client.Download(options); + + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "locality0.test"); + EXPECT_EQ(state->Requests.front().HostHeader, "primary.test"); + } + + TEST(DataLocalityTest, FallsBackOnUnsupportedLayout) + { + for (const auto status : + {Core::Http::HttpStatusCode::BadRequest, Core::Http::HttpStatusCode::InternalServerError}) + { + auto state = std::make_shared(); + state->LayoutStatus = status; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_NO_THROW(client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (auto request = state->Requests.begin(); request != state->Requests.end(); ++request) + { + if (!request->IsLayout) + { + EXPECT_EQ(request->Host, "primary.test"); + EXPECT_TRUE(request->HostHeader.empty()); + } + } + } + } + + TEST(DataLocalityTest, PropagatesNonFallbackLayoutError) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NotFound; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_THROW( + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()), StorageException); + } + + TEST(DataLocalityTest, FallsBackWhenBlobHasNoLayout) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_NO_THROW(client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (auto request = state->Requests.begin() + 1; request != state->Requests.end(); ++request) + { + EXPECT_EQ(request->Host, "primary.test"); + } + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp new file mode 100644 index 0000000000..92a339c943 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp @@ -0,0 +1,666 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "blob_container_client_test.hpp" + +#include +#include + +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + class SessionTestCredential final : public Azure::Core::Credentials::TokenCredential { + public: + SessionTestCredential() : TokenCredential("SessionTestCredential") {} + + mutable int GetTokenCount = 0; + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext&, + const Azure::Core::Context&) const override + { + ++GetTokenCount; + return {"bearer-token", Azure::DateTime::clock::now() + std::chrono::hours(1)}; + } + }; + + class CountingTokenCredential final : public Azure::Core::Credentials::TokenCredential { + public: + explicit CountingTokenCredential( + std::shared_ptr credential) + : TokenCredential("CountingTokenCredential"), m_credential(std::move(credential)) + { + } + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext& tokenRequestContext, + const Azure::Core::Context& context) const override + { + ++m_getTokenCount; + return m_credential->GetToken(tokenRequestContext, context); + } + + int GetTokenCount() const { return m_getTokenCount.load(); } + + private: + std::shared_ptr m_credential; + mutable std::atomic m_getTokenCount{0}; + }; + + struct SessionTestState final + { + bool RejectFirstSessionRequest = false; + bool RejectedSessionRequest = false; + bool RejectCreateSession = false; + bool RejectCreateSessionWithNotFound = false; + bool RejectDownloadWithNotFound = false; + bool FailFirstDownloadWithServerError = false; + bool FailedDownloadWithServerError = false; + std::vector CreateSessionContainers; + std::vector CreateSessionVersions; + std::vector AuthorizationHeaders; + std::vector DownloadHosts; + std::vector SessionSignaturesValid; + bool MutatedDownloadObserved = false; + std::list> ResponseBodies; + }; + + class SessionTestTransport final : public Azure::Core::Http::HttpTransport { + public: + explicit SessionTestTransport(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Context const&) override + { + std::lock_guard lock(m_mutex); + const auto authorization = request.GetHeader("Authorization"); + m_state->AuthorizationHeaders.emplace_back( + authorization.HasValue() ? authorization.Value() : std::string()); + + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post && query.count("comp") != 0 + && query.at("comp") == "session") + { + m_state->CreateSessionContainers.emplace_back(request.GetUrl().GetPath()); + const auto version = request.GetHeader("x-ms-version"); + m_state->CreateSessionVersions.emplace_back( + version.HasValue() ? version.Value() : std::string()); + if (m_state->RejectCreateSession) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::Forbidden, "Forbidden"); + } + if (m_state->RejectCreateSessionWithNotFound) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + } + const auto token = "token-" + request.GetUrl().GetPath(); + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const std::string body + = "id" + "2099-01-01T00:00:00Z" + "HMAC" + + token + "" + key + + ""; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::Created, + "Created", + std::vector(body.begin(), body.end())); + } + + if (authorization.HasValue() && authorization.Value().find("Session ") == 0) + { + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const auto signatureSeparator = authorization.Value().find(':'); + const auto expectedAuthorization = authorization.Value().substr(0, signatureSeparator + 1) + + Azure::Storage::_internal::SharedKeyPolicy::GetSignature(request, "account", key); + m_state->SessionSignaturesValid.emplace_back( + signatureSeparator != std::string::npos + && authorization.Value() == expectedAuthorization); + } + if (query.count("session-test") != 0 && request.GetHeader("x-ms-session-test").HasValue()) + { + m_state->MutatedDownloadObserved = true; + } + m_state->DownloadHosts.emplace_back(request.GetUrl().GetHost()); + + if (m_state->RejectFirstSessionRequest && !m_state->RejectedSessionRequest + && authorization.HasValue() && authorization.Value().find("Session ") == 0) + { + m_state->RejectedSessionRequest = true; + auto response + = CreateResponse(Azure::Core::Http::HttpStatusCode::Unauthorized, "Unauthorized"); + response->SetHeader("WWW-Authenticate", "Session error=session_expired"); + return response; + } + + if (m_state->FailFirstDownloadWithServerError && !m_state->FailedDownloadWithServerError) + { + m_state->FailedDownloadWithServerError = true; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::InternalServerError, "Internal Server Error"); + } + + if (m_state->RejectDownloadWithNotFound) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + } + + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("x-ms-creation-time", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-server-encrypted", "true"); + return response; + } + + private: + std::unique_ptr CreateResponse( + Azure::Core::Http::HttpStatusCode statusCode, + const std::string& reasonPhrase, + std::vector body = {}) const + { + m_state->ResponseBodies.emplace_back(std::move(body)); + auto response + = std::make_unique(1, 1, statusCode, reasonPhrase); + response->SetBodyStream( + std::make_unique(m_state->ResponseBodies.back())); + return response; + } + + std::shared_ptr m_state; + mutable std::mutex m_mutex; + }; + + class SessionRequestMutationPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + std::unique_ptr Clone() const override + { + return std::make_unique(); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + request.GetUrl().AppendQueryParameter("session-test", "value"); + request.SetHeader("x-ms-session-test", "value"); + return nextPolicy.Send(request, context); + } + }; + + class DataLocalityTestPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + DataLocalityTestPolicy() : m_downloadCount(std::make_shared>(0)) {} + + explicit DataLocalityTestPolicy(std::shared_ptr> downloadCount) + : m_downloadCount(std::move(downloadCount)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(m_downloadCount); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Get && query.count("comp") == 0) + { + request.SetHeader("Host", request.GetUrl().GetHost()); + request.GetUrl().SetHost( + ++*m_downloadCount == 1 ? "tenant1.blob.core.windows.net" + : "tenant2.blob.core.windows.net"); + } + return nextPolicy.Send(request, context); + } + + private: + std::shared_ptr> m_downloadCount; + }; + + std::unique_ptr CreateSessionRequestCountingPolicy( + std::shared_ptr> sessionRequestCount) + { + return std::make_unique( + [sessionRequestCount + = std::move(sessionRequestCount)](const Azure::Core::Http::Request& request) { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post + && query.count("comp") != 0 && query.at("comp") == "session") + { + ++*sessionRequestCount; + } + }); + } + + void AddSessionTestTransport( + Blobs::BlobClientOptions& options, + const std::shared_ptr& state) + { + options.Transport.Transport = std::make_shared(state); + } + + void EnableSessions( + Blobs::BlobClientOptions& options, + const std::shared_ptr& state) + { + AddSessionTestTransport(options, state); + options.Session.Mode = Blobs::SessionMode::Enabled; + } + } // namespace + + TEST(SessionAuthenticationTest, SignsEligibleDownload) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + ASSERT_EQ(state->SessionSignaturesValid.size(), 1U); + EXPECT_TRUE(state->SessionSignaturesValid[0]); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, SignsAfterUserPerRetryPolicies) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.PerRetryPolicies.emplace_back(std::make_unique()); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->MutatedDownloadObserved); + ASSERT_EQ(state->SessionSignaturesValid.size(), 1U); + EXPECT_TRUE(state->SessionSignaturesValid[0]); + } + + TEST(SessionAuthenticationTest, FallsBackToBearerAfter401) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectFirstSessionRequest = true; + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), std::string::npos); + EXPECT_EQ(credential->GetTokenCount, 2); + } + + TEST(SessionAuthenticationTest, UsesBearerOnSecondaryHost) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->FailFirstDownloadWithServerError = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 1; + options.Retry.RetryDelay = std::chrono::milliseconds(0); + options.Retry.MaxRetryDelay = std::chrono::milliseconds(0); + options.SecondaryHostForRetryReads = "account-secondary.blob.core.windows.net"; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + ASSERT_EQ(state->DownloadHosts.size(), 3U); + EXPECT_EQ(state->DownloadHosts[0], "account.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[1], "account-secondary.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[2], "account.blob.core.windows.net"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 4U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Bearer "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[3].find("Session "), 0U); + EXPECT_EQ(credential->GetTokenCount, 2); + } + + TEST(SessionAuthenticationTest, ReusesSessionAcrossDataLocalityTenants) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + options.PerOperationPolicies.emplace_back(std::make_unique()); + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + ASSERT_EQ(state->DownloadHosts.size(), 2U); + EXPECT_EQ(state->DownloadHosts[0], "tenant1.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[1], "tenant2.blob.core.windows.net"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, SharedProviderCachesPerContainer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions providerOptions; + EnableSessions(providerOptions, state); + providerOptions.Session.AccountName = "account"; + auto provider = std::make_shared( + "https://account.blob.core.windows.net", credential, providerOptions); + + auto clientOptions = providerOptions; + clientOptions.Session.Provider = provider; + Blobs::BlobServiceClient serviceClient( + "https://account.blob.core.windows.net", credential, clientOptions); + auto first = serviceClient.GetBlobContainerClient("container1").GetBlobClient("blob1"); + auto second = serviceClient.GetBlobContainerClient("container1").GetBlobClient("blob2"); + auto third = serviceClient.GetBlobContainerClient("container2").GetBlobClient("blob"); + + EXPECT_NO_THROW(first.Download()); + EXPECT_NO_THROW(second.Download()); + EXPECT_NO_THROW(third.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 2U); + EXPECT_EQ(state->CreateSessionContainers[0], "container1"); + EXPECT_EQ(state->CreateSessionContainers[1], "container2"); + EXPECT_EQ(state->CreateSessionVersions[0], Blobs::_detail::ApiVersion); + EXPECT_EQ(state->CreateSessionVersions[1], Blobs::_detail::ApiVersion); + } + + TEST(SessionAuthenticationTest, SharedProviderSupportsIndependentClients) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions providerOptions; + EnableSessions(providerOptions, state); + providerOptions.Session.AccountName = "account"; + auto provider = std::make_shared( + "https://account.blob.core.windows.net", credential, providerOptions); + + auto clientOptions = providerOptions; + clientOptions.Session.Provider = provider; + Blobs::BlobClient first( + "https://account.blob.core.windows.net/container/blob1", credential, clientOptions); + Blobs::BlobClient second( + "https://account.blob.core.windows.net/container/blob2", credential, clientOptions); + + EXPECT_NO_THROW(first.Download()); + EXPECT_NO_THROW(second.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), 0U); + } + + TEST(SessionAuthenticationTest, ExplicitProviderSupportsCustomEndpoint) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + options.Session.Provider = std::make_shared( + "https://storage.contoso.com", credential, options); + + Blobs::BlobClient client("https://storage.contoso.com/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + } + + TEST(SessionAuthenticationTest, ExplicitProviderSupportsPathStyleEndpoint) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + options.Session.Provider = std::make_shared( + "https://localhost/account", credential, options); + + Blobs::BlobClient client("https://localhost/account/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "account/container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-account/container:"), 0U); + } + + TEST(SessionAuthenticationTest, CustomEndpointRequiresAccountNameAndProvider) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Enabled; + options.Session.AccountName = "account"; + + EXPECT_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options), + std::invalid_argument); + } + + TEST(SessionAuthenticationTest, AutoModeUsesBearer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + AddSessionTestTransport(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->CreateSessionContainers.empty()); + ASSERT_EQ(state->AuthorizationHeaders.size(), 1U); + EXPECT_EQ(state->AuthorizationHeaders[0].find("Bearer "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, AutoModeAllowsCustomEndpointConstruction) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + + EXPECT_NO_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options)); + } + + TEST(SessionAuthenticationTest, EnabledModeRejectsCustomEndpointWithoutConfiguration) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Enabled; + + EXPECT_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options), + std::invalid_argument); + } + + TEST(SessionAuthenticationTest, DefaultProviderCachesSoftFailure) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectCreateSession = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + } + + TEST(SessionAuthenticationTest, PropagatesCreateSessionNotFound) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectCreateSessionWithNotFound = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/missing-container/blob", credential, options); + + EXPECT_THROW(client.Download(), StorageException); + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "missing-container"); + } + + TEST(SessionAuthenticationTest, PropagatesDownloadNotFound) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectDownloadWithNotFound = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/missing-blob", credential, options); + + EXPECT_THROW(client.Download(), StorageException); + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + } + + TEST(SessionAuthenticationTest, DisabledModeUsesBearer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + AddSessionTestTransport(options, state); + options.Session.Mode = Blobs::SessionMode::Disabled; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->CreateSessionContainers.empty()); + ASSERT_EQ(state->AuthorizationHeaders.size(), 1U); + EXPECT_EQ(state->AuthorizationHeaders[0].find("Bearer "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, DisabledModeAllowsCustomEndpointConstruction) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Disabled; + + EXPECT_NO_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options)); + } + + class SessionAuthenticationRecordedTest : public BlobContainerClientTest { + }; + + TEST_F(SessionAuthenticationRecordedTest, DownloadsWithDefaultSessionProvider_PLAYBACKONLY_) + { + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + const std::vector content{'s', 'e', 's', 's', 'i', 'o', 'n'}; + blobClient.UploadFrom(content.data(), content.size()); + + auto credential = std::make_shared(GetTestCredential()); + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + Blobs::BlobClient sessionClient(blobClient.GetUrl(), credential, options); + + for (int i = 0; i < 3; ++i) + { + EXPECT_NO_THROW(sessionClient.Download().Value.BodyStream->ReadToEnd()); + } + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_EQ(sessionRequestCount->load(), 1); + + Blobs::Models::BlobHttpHeaders headers; + headers.ContentType = "application/octet-stream"; + EXPECT_NO_THROW(sessionClient.GetProperties()); + EXPECT_NO_THROW(sessionClient.SetMetadata({{"key", "value"}})); + EXPECT_NO_THROW(sessionClient.SetHttpHeaders(headers)); + EXPECT_EQ(credential->GetTokenCount(), 2); + EXPECT_EQ(sessionRequestCount->load(), 1); + } + + TEST_F( + SessionAuthenticationRecordedTest, + IndependentClientsShareExplicitSessionProvider_PLAYBACKONLY_) + { + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + const std::vector content{'s', 'e', 's', 's', 'i', 'o', 'n'}; + blobClient.UploadFrom(content.data(), content.size()); + + auto credential = std::make_shared(GetTestCredential()); + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + options.Session.Provider = std::make_shared( + GetBlobServiceUrl(), credential, options); + options.Session.AccountName = m_accountName; + Blobs::BlobClient firstClient(blobClient.GetUrl(), credential, options); + Blobs::BlobClient secondClient(blobClient.GetUrl(), credential, options); + + EXPECT_NO_THROW(firstClient.Download().Value.BodyStream->ReadToEnd()); + EXPECT_NO_THROW(secondClient.Download().Value.BodyStream->ReadToEnd()); + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_EQ(sessionRequestCount->load(), 1); + + Blobs::Models::BlobHttpHeaders headers; + headers.ContentType = "application/octet-stream"; + EXPECT_NO_THROW(firstClient.GetProperties()); + EXPECT_NO_THROW(firstClient.SetMetadata({{"key", "value"}})); + EXPECT_NO_THROW(firstClient.SetHttpHeaders(headers)); + EXPECT_EQ(credential->GetTokenCount(), 2); + EXPECT_EQ(sessionRequestCount->load(), 1); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp index df22348619..6300e0e0d2 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp @@ -435,6 +435,7 @@ namespace Azure { namespace Storage { namespace Test { options.TransferOptions.InitialChunkSize = 2; options.TransferOptions.ChunkSize = 2; options.TransferOptions.Concurrency = 1; + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; blobClient.DownloadTo( reinterpret_cast(&downloadBuffer[0]), static_cast(downloadBuffer.size()), diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp index 244ca8da31..1ab2a0b12f 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp @@ -11,31 +11,6 @@ namespace Azure { namespace Storage { namespace Test { - class PeekHttpRequestPolicy final : public Core::Http::Policies::HttpPolicy { - public: - PeekHttpRequestPolicy(std::function callback) - : m_callback(std::move(callback)) - { - } - - std::unique_ptr Send( - Core::Http::Request& request, - Core::Http::Policies::NextHttpPolicy nextPolicy, - Core::Context const& context) const override - { - m_callback(request); - return nextPolicy.Send(request, context); - } - - std::unique_ptr Clone() const override - { - return std::make_unique(*this); - } - - private: - std::function m_callback; - }; - TEST_F(StorageTest, StoragetimeoutTestBasic_LIVEONLY_) { Azure::Nullable timeout; diff --git a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt index bbe25d9e2f..ccce4f50f9 100644 --- a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable ( crypt_functions_test.cpp metadata_test.cpp storage_credential_test.cpp + storage_url_test.cpp structured_message_test.cpp test_base.cpp test_base.hpp diff --git a/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp b/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp new file mode 100644 index 0000000000..4018ac44d4 --- /dev/null +++ b/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + TEST(StorageUrlTest, ParsesStandardEndpoint) + { + auto result = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.blob.core.windows.net/container/blob?versionid=version")); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + EXPECT_EQ(result.Value().ServiceUrl, "https://account.blob.core.windows.net"); + ASSERT_TRUE(result.Value().ContainerName.HasValue()); + EXPECT_EQ(result.Value().ContainerName.Value(), "container"); + EXPECT_EQ( + result.Value().ContainerUrl.Value(), "https://account.blob.core.windows.net/container"); + } + + TEST(StorageUrlTest, ParsesMixedCaseEndpoints) + { + for (const std::string hostname : + {"Account.BLOB.CORE.WINDOWS.NET", + "ACCOUNT.blob.core.windows.net", + "Account-DuAlStAcK-SeCoNdArY.BlOb.CoRe.WiNdOwS.NeT", + "Account.BlOb.PrEpRoD.CoRe.WiNdOwS.NeT", + "Account.BlOb.CoRe.ChInAcLoUdApI.Cn", + "Account.BlOb.CoRe.UsGoVcLoUdApI.NeT", + "Account-SeCoNdArY.Z50.BlOb.StOrAgE.AzUrE.NeT", + "Account-SeCoNdArY.CuStOm.BlOb.StOrAgE.AzUrE.NeT"}) + { + SCOPED_TRACE(hostname); + const auto serviceUrl = "https://" + hostname; + const auto absoluteUrl = serviceUrl + "/Container/Blob?versionid=MiXeD"; + const Azure::Core::Url url(absoluteUrl); + auto result = _internal::ParseStorageUrl(url); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + EXPECT_EQ(result.Value().ServiceUrl, serviceUrl); + ASSERT_TRUE(result.Value().ContainerName.HasValue()); + EXPECT_EQ(result.Value().ContainerName.Value(), "Container"); + EXPECT_EQ(result.Value().ContainerUrl.Value(), serviceUrl + "/Container"); + EXPECT_EQ(url.GetAbsoluteUrl(), absoluteUrl); + } + } + + TEST(StorageUrlTest, RejectsNonStorageEndpoints) + { + EXPECT_FALSE(_internal::ParseStorageUrl(Azure::Core::Url("https://example.com")).HasValue()); + EXPECT_FALSE(_internal::ParseStorageUrl(Azure::Core::Url("https://localhost/account/container")) + .HasValue()); + EXPECT_FALSE( + _internal::ParseStorageUrl(Azure::Core::Url("https://storage.contoso.com/container")) + .HasValue()); + } + + TEST(StorageUrlTest, ParsesSovereignCloudEndpoints) + { + auto china = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.blob.core.chinacloudapi.cn/container/blob")); + ASSERT_TRUE(china.HasValue()); + EXPECT_EQ(china.Value().Service, "blob"); + EXPECT_EQ(china.Value().AccountName, "account"); + EXPECT_EQ(china.Value().ServiceUrl, "https://account.blob.core.chinacloudapi.cn"); + + auto government = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.file.core.usgovcloudapi.net/share/file")); + ASSERT_TRUE(government.HasValue()); + EXPECT_EQ(government.Value().Service, "file"); + EXPECT_EQ(government.Value().AccountName, "account"); + EXPECT_EQ(government.Value().ServiceUrl, "https://account.file.core.usgovcloudapi.net"); + } + + TEST(StorageUrlTest, StripsAccountSuffixesRepeatedly) + { + auto network = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-dualstack-secondary.blob.core.windows.net")); + + ASSERT_TRUE(network.HasValue()); + EXPECT_EQ(network.Value().AccountName, "account"); + + auto internetRouting = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-internetrouting-secondary.blob.core.windows.net")); + ASSERT_TRUE(internetRouting.HasValue()); + EXPECT_EQ(internetRouting.Value().AccountName, "account"); + + auto microsoftRouting = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-microsoftrouting-secondary.dfs.core.windows.net")); + ASSERT_TRUE(microsoftRouting.HasValue()); + EXPECT_EQ(microsoftRouting.Value().AccountName, "account"); + } + + TEST(StorageUrlTest, ParsesAzureDnsZoneEndpoints) + { + auto blob = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.z1.blob.storage.azure.net/container/blob")); + ASSERT_TRUE(blob.HasValue()); + EXPECT_EQ(blob.Value().Service, "blob"); + EXPECT_EQ(blob.Value().AccountName, "account"); + EXPECT_EQ(blob.Value().ServiceUrl, "https://account.z1.blob.storage.azure.net"); + + auto dfs = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-secondary.z50.dfs.storage.azure.net/filesystem/file")); + ASSERT_TRUE(dfs.HasValue()); + EXPECT_EQ(dfs.Value().Service, "dfs"); + EXPECT_EQ(dfs.Value().AccountName, "account"); + EXPECT_EQ(dfs.Value().ServiceUrl, "https://account-secondary.z50.dfs.storage.azure.net"); + } + + TEST(StorageUrlTest, ParsesFlexibleAzureDnsZoneEndpoints) + { + auto result = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-secondary.custom.blob.storage.azure.net/container")); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.hpp b/sdk/storage/azure-storage-common/test/ut/test_base.hpp index be9dfe5fd3..1aa3abdc20 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.hpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.hpp @@ -57,6 +57,31 @@ namespace Azure { namespace Storage { ResponseFactory m_responseFactory; }; + class PeekHttpRequestPolicy final : public Core::Http::Policies::HttpPolicy { + public: + explicit PeekHttpRequestPolicy(std::function callback) + : m_callback(std::move(callback)) + { + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + Core::Context const& context) const override + { + m_callback(request); + return nextPolicy.Send(request, context); + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + private: + std::function m_callback; + }; + class StorageTest : public Azure::Core::Test::TestBase { public: StorageTest() { TestBase::SetUpTestSuiteLocal(AZURE_TEST_ASSETS_DIR); } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt index e79263f17b..08a345d5b4 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt @@ -16,6 +16,7 @@ SetUpTestProxy("sdk/storage") add_executable ( azure-storage-files-datalake-test + data_locality_test.cpp datalake_directory_client_test.cpp datalake_directory_client_test.hpp datalake_file_client_test.cpp @@ -29,6 +30,7 @@ add_executable ( datalake_service_client_test.cpp datalake_service_client_test.hpp macro_guard.cpp + session_authentication_test.cpp simplified_header_test.cpp # Include shared test source code ${CMAKE_CURRENT_SOURCE_DIR}/../../../azure-storage-common/test/ut/test_base.cpp diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp new file mode 100644 index 0000000000..899b4191e3 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp @@ -0,0 +1,395 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "test/ut/test_base.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + namespace DataLake = Files::DataLake; + + struct LocalityRequest final + { + std::string Host; + std::string HostHeader; + std::string Range; + std::string IfMatch; + std::string Marker; + bool IsLayout; + }; + + struct LocalityState final + { + std::vector Requests; + bool NoLayout = false; + }; + + class LocalityTransportPolicy final : public Core::Http::Policies::HttpPolicy { + public: + explicit LocalityTransportPolicy(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy, + Core::Context const&) const override + { + const auto& query = request.GetUrl().GetQueryParameters(); + const auto& headers = request.GetHeaders(); + const auto header = [&](const char* name) { + auto value = headers.find(name); + return value == headers.end() ? std::string() : value->second; + }; + const bool isLayout = query.count("comp") && query.at("comp") == "layout"; + const auto marker = query.count("marker") ? query.at("marker") : std::string(); + m_state->Requests.push_back(LocalityRequest{ + request.GetUrl().GetHost(), + header("host"), + header("x-ms-range"), + header("if-match"), + marker, + isLayout}); + + auto response = std::make_unique( + 1, + 1, + isLayout + ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent + : Core::Http::HttpStatusCode::Ok) + : Core::Http::HttpStatusCode::PartialContent, + "OK"); + response->SetHeader("etag", "\"layout-etag\""); + response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetHeader("x-ms-blob-content-length", "12"); + response->SetHeader("x-ms-blob-content-type", "application/octet-stream"); + response->SetHeader("x-ms-meta-layout", "metadata"); + response->SetHeader("x-ms-owner", "owner"); + response->SetHeader("x-ms-group", "group"); + response->SetHeader("x-ms-permissions", "rw-r-----"); + response->SetHeader("x-ms-acl", "user::rw-,group::r--,other::---"); + response->SetHeader("x-ms-encryption-context", "context"); + if (isLayout) + { + if (!m_state->NoLayout) + { + const std::string xml = marker.empty() + ? "" + "" + "page2" + : "" + "" + ""; + response->SetBody(std::vector(xml.begin(), xml.end())); + response->SetHeader("content-type", "application/xml"); + } + return response; + } + + const auto range = header("x-ms-range"); + const auto separator = range.find('-'); + const auto offset = std::stoll(range.substr(6, separator - 6)); + const auto end = std::stoll(range.substr(separator + 1)); + static const std::string data = "abcdefghijkl"; + response->SetHeader("content-length", std::to_string(end - offset + 1)); + response->SetHeader( + "content-range", "bytes " + std::to_string(offset) + "-" + std::to_string(end) + "/12"); + response->SetHeader("x-ms-download-hint", "layout"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(data.data() + offset), + static_cast(end - offset + 1))); + return response; + } + + private: + std::shared_ptr m_state; + }; + + DataLake::DataLakeFileClient CreateClient( + const std::shared_ptr& state, + int origin) + { + DataLake::DataLakeClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state)); + const std::string url = "https://account.dfs.core.windows.net"; + switch (origin) + { + case 0: + return DataLake::DataLakeFileClient(url + "/filesystem/file", options); + case 1: + return DataLake::DataLakeServiceClient(url, options) + .GetFileSystemClient("filesystem") + .GetFileClient("file"); + case 2: + return DataLake::DataLakeFileSystemClient(url + "/filesystem", options) + .GetFileClient("file"); + default: + return DataLake::DataLakeDirectoryClient(url + "/filesystem/directory", options) + .GetFileClient("file"); + } + } + } // namespace + + class DataLakeLocalityRecordedTest : public StorageTest { + }; + + TEST_F(DataLakeLocalityRecordedTest, GetLayoutAndDownloadHint_PLAYBACKONLY_) + { + if (!m_testContext.IsPlaybackMode()) + { + GTEST_SKIP() << "Reuses the Blob multi-endpoint recording."; + } + m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) + + "/../../../azure-storage-blobs/test/ut/recordings"; + m_testContext.RenameTest( + "DataLocalityRecordedTest", "GetLayoutMultipleEndpoints_PLAYBACKONLY_"); + auto options = InitStorageClientOptions(); + auto client = DataLake::DataLakeFileClient( + "https://" + StandardStorageAccountName() + + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", + GetTestCredential(), + options); + auto page = client.GetLayout(); + EXPECT_EQ(page.Layout.Properties.FileSize, 64 * 1024 * 1024); + EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); + ASSERT_TRUE(page.Layout.Properties.IsServerEncrypted.HasValue()); + EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted.Value()); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + EXPECT_FALSE(page.Layout.Properties.IsDirectory); + std::set endpoints; + std::vector ranges; + for (; page.HasPage(); page.MoveToNextPage()) + { + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + EXPECT_GT(range.Range.Length.Value(), 0); + EXPECT_FALSE(range.Endpoint.empty()); + ranges.push_back(range); + endpoints.insert(range.Endpoint); + } + } + EXPECT_GE(endpoints.size(), 2U); + ASSERT_GT(ranges.size(), 1U); + DataLake::DownloadFileOptions downloadOptions; + downloadOptions.Range = Core::Http::HttpRange(); + downloadOptions.Range.Value().Offset = ranges[1].Range.Offset - 2; + downloadOptions.Range.Value().Length = 5; + auto response = client.Download(downloadOptions); + EXPECT_EQ(response.Value.Body->ReadToEnd().size(), 5U); + EXPECT_EQ(response.Value.FileSize, 64 * 1024 * 1024); + ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + + TEST_F(DataLakeLocalityRecordedTest, GetLayoutRangeReturnsFullFileSize_PLAYBACKONLY_) + { + if (!m_testContext.IsPlaybackMode()) + { + GTEST_SKIP() << "Reuses the Blob ranged layout recording."; + } + m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) + + "/../../../azure-storage-blobs/test/ut/recordings"; + m_testContext.RenameTest( + "DataLocalityRecordedTest", "GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_"); + auto clientOptions = InitStorageClientOptions(); + auto client = DataLake::DataLakeFileClient( + "https://" + StandardStorageAccountName() + + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", + GetTestCredential(), + clientOptions); + constexpr int64_t fileSize = 64 * 1024 * 1024; + for (bool bounded : {true, false}) + { + SCOPED_TRACE(bounded); + DataLake::GetFileLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : fileSize - 16; + if (bounded) + { + options.Range.Value().Length = 5; + } + const auto offset = options.Range.Value().Offset; + bool coversOffset = false; + size_t rangeCount = 0; + for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) + { + EXPECT_EQ(page.Layout.Properties.FileSize, fileSize); + EXPECT_GT(page.Layout.Properties.FileSize, options.Range.Value().Length.ValueOr(16)); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + ASSERT_GT(range.Range.Length.Value(), 0); + ASSERT_GE(range.Range.Offset, 0); + ASSERT_LE(range.Range.Offset, fileSize); + ASSERT_LE(range.Range.Length.Value(), fileSize - range.Range.Offset); + EXPECT_FALSE(range.Endpoint.empty()); + coversOffset = coversOffset + || (range.Range.Offset <= offset + && offset - range.Range.Offset < range.Range.Length.Value()); + ++rangeCount; + } + } + EXPECT_GT(rangeCount, 0U); + EXPECT_TRUE(coversOffset); + } + } + + TEST(DataLakeLocalityTest, GetLayoutMapsPropertiesAndPages) + { + auto state = std::make_shared(); + auto client = CreateClient(state, 0); + DataLake::GetFileLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 2; + options.Range.Value().Length = 10; + options.AccessConditions.IfMatch = ETag("\"layout-etag\""); + auto page = client.GetLayout(options); + EXPECT_EQ(page.CurrentPageToken, ""); + EXPECT_EQ(page.NextPageToken.Value(), "page2"); + ASSERT_EQ(page.Layout.Ranges.size(), 1U); + EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality0.test:443"); + EXPECT_EQ(page.Layout.Properties.FileSize, 12); + EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); + EXPECT_EQ(page.Layout.Properties.Owner.Value(), "owner"); + EXPECT_EQ(page.Layout.Properties.Group.Value(), "group"); + EXPECT_EQ(page.Layout.Properties.Permissions.Value(), "rw-r-----"); + EXPECT_EQ(page.Layout.Properties.EncryptionContext.Value(), "context"); + ASSERT_TRUE(page.Layout.Properties.Acls.HasValue()); + EXPECT_EQ(page.Layout.Properties.Acls.Value().size(), 3U); + page.MoveToNextPage(); + EXPECT_EQ(page.CurrentPageToken, "page2"); + ASSERT_EQ(page.Layout.Ranges.size(), 1U); + EXPECT_EQ(page.Layout.Ranges[0].Range.Offset, 6); + EXPECT_EQ(page.Layout.Ranges[0].Range.Length.Value(), 6); + EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality1.test:443"); + page.MoveToNextPage(); + EXPECT_FALSE(page.HasPage()); + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Range, "bytes=2-11"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"layout-etag\""); + EXPECT_EQ(state->Requests[1].Marker, "page2"); + } + + TEST(DataLakeLocalityTest, GetLayoutNoContent) + { + auto state = std::make_shared(); + state->NoLayout = true; + auto page = CreateClient(state, 0).GetLayout(); + EXPECT_TRUE(page.Layout.Ranges.empty()); + EXPECT_EQ(page.Layout.Properties.FileSize, 12); + EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); + page.MoveToNextPage(); + EXPECT_FALSE(page.HasPage()); + } + + TEST(DataLakeLocalityTest, RoutesOneShotDownloadForAllClientOrigins) + { + for (int origin = 0; origin != 4; ++origin) + { + SCOPED_TRACE(origin); + auto state = std::make_shared(); + auto client = CreateClient(state, origin); + DataLake::DownloadFileOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 6; + options.Range.Value().Length = 1; + options.LayoutEndpoint = "locality1.test:443"; + auto response = client.Download(options); + EXPECT_EQ(response.Value.Body->ReadToEnd(), std::vector{'g'}); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_EQ(state->Requests[0].Host, "locality1.test"); + EXPECT_EQ(state->Requests[0].HostHeader, "account.blob.core.windows.net"); + ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + } + + TEST(DataLakeLocalityTest, RoutesManagedDownloadsForAllClientOrigins) + { + using Routing = DataLake::LayoutAwareRouting; + for (int origin = 0; origin != 4; ++origin) + { + for (const auto& routing : + {Azure::Nullable(), + Azure::Nullable(Routing::Auto), + Azure::Nullable(Routing::Enabled), + Azure::Nullable(Routing::Disabled)}) + { + for (bool toFile : {false, true}) + { + SCOPED_TRACE(origin); + SCOPED_TRACE(toFile); + SCOPED_TRACE(routing.HasValue() ? static_cast(routing.Value()) : -1); + auto state = std::make_shared(); + auto client = CreateClient(state, origin); + DataLake::DownloadFileToOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 5; + options.Range.Value().Length = 7; + EXPECT_EQ(options.LayoutAwareRouting, DataLake::LayoutAwareRouting::Disabled); + if (routing.HasValue()) + { + options.LayoutAwareRouting = routing.Value(); + } + options.TransferOptions.InitialChunkSize = 1; + options.TransferOptions.ChunkSize = 6; + options.TransferOptions.Concurrency = 1; + std::vector content(7); + if (toFile) + { + const auto fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + auto response = client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + content.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + file.close(); + std::remove(fileName.c_str()); + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + } + else + { + auto response = client.DownloadTo(content.data(), content.size(), options); + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + } + EXPECT_EQ(content, (std::vector{'f', 'g', 'h', 'i', 'j', 'k', 'l'})); + if (options.LayoutAwareRouting == DataLake::LayoutAwareRouting::Disabled) + { + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests.back().Host, "account.blob.core.windows.net"); + EXPECT_FALSE(state->Requests.back().IsLayout); + } + else + { + ASSERT_EQ(state->Requests.size(), 4U); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_TRUE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests.back().Host, "locality1.test"); + EXPECT_EQ(state->Requests.back().HostHeader, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests.back().IfMatch, "\"layout-etag\""); + } + } + } + } + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index 5e03711982..c49615ed65 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -26,6 +26,18 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { + TEST(DataLakeFileClientOptionsTest, DefaultTransferSizes) + { + Files::DataLake::DownloadFileToOptions downloadOptions; + EXPECT_EQ( + downloadOptions.TransferOptions.InitialChunkSize, + downloadOptions.TransferOptions.ChunkSize); + EXPECT_EQ(downloadOptions.TransferOptions.InitialChunkSize, 4_MB); + + Files::DataLake::UploadFileFromOptions uploadOptions; + EXPECT_EQ(uploadOptions.TransferOptions.SingleUploadThreshold, 4_MB); + } + void DataLakeFileClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); @@ -820,7 +832,9 @@ namespace Azure { namespace Storage { namespace Test { { auto fileClient = *m_fileClient; const auto blobContent = RandomBuffer(static_cast(8_MB)); - fileClient.UploadFrom(blobContent.data(), blobContent.size()); + Files::DataLake::UploadFileFromOptions uploadOptions; + uploadOptions.TransferOptions.SingleUploadThreshold = blobContent.size(); + fileClient.UploadFrom(blobContent.data(), blobContent.size(), uploadOptions); auto testDownloadToBuffer = [&](int concurrency, int64_t downloadSize, diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp new file mode 100644 index 0000000000..8dc10f5840 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "datalake_file_client_test.hpp" + +#include + +#include +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + struct SessionRequest final + { + Azure::Core::Http::HttpMethod Method; + std::string Host; + std::string Authorization; + std::string Component; + }; + + struct SessionTestState final + { + std::vector Requests; + std::list> ResponseBodies; + }; + + std::unique_ptr CreateSessionRequestCountingPolicy( + std::shared_ptr> sessionRequestCount) + { + return std::make_unique( + [sessionRequestCount + = std::move(sessionRequestCount)](const Azure::Core::Http::Request& request) { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post + && query.count("comp") != 0 && query.at("comp") == "session") + { + ++*sessionRequestCount; + } + }); + } + + class SessionTestCredential final : public Azure::Core::Credentials::TokenCredential { + public: + SessionTestCredential() : TokenCredential("SessionTestCredential") {} + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext&, + const Azure::Core::Context&) const override + { + return {"bearer-token", Azure::DateTime::clock::now() + std::chrono::hours(1)}; + } + }; + + class CountingTokenCredential final : public Azure::Core::Credentials::TokenCredential { + public: + explicit CountingTokenCredential( + std::shared_ptr credential) + : TokenCredential("CountingTokenCredential"), m_credential(std::move(credential)) + { + } + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext& tokenRequestContext, + const Azure::Core::Context& context) const override + { + ++m_getTokenCount; + return m_credential->GetToken(tokenRequestContext, context); + } + + int GetTokenCount() const { return m_getTokenCount.load(); } + + private: + std::shared_ptr m_credential; + mutable std::atomic m_getTokenCount{0}; + }; + + class SessionTestTransport final : public Azure::Core::Http::HttpTransport { + public: + explicit SessionTestTransport(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Context const&) override + { + std::lock_guard lock(m_mutex); + const auto authorization = request.GetHeader("Authorization"); + const auto query = request.GetUrl().GetQueryParameters(); + const auto component = query.find("comp"); + m_state->Requests.push_back( + {request.GetMethod(), + request.GetUrl().GetHost(), + authorization.HasValue() ? authorization.Value() : std::string(), + component == query.end() ? std::string() : component->second}); + + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post && component != query.end() + && component->second == "session") + { + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const std::string body = "id" + "2099-01-01T00:00:00Z" + "HMAC" + "session-token" + + key + ""; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::Created, + "Created", + std::vector(body.begin(), body.end())); + } + + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Put) + { + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Created, "Created"); + response->SetHeader("ETag", "\"etag\""); + response->SetHeader("Last-Modified", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-request-server-encrypted", "true"); + return response; + } + + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("x-ms-creation-time", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-server-encrypted", "true"); + return response; + } + + private: + std::unique_ptr CreateResponse( + Azure::Core::Http::HttpStatusCode statusCode, + const std::string& reasonPhrase, + std::vector body = {}) + { + m_state->ResponseBodies.emplace_back(std::move(body)); + auto response + = std::make_unique(1, 1, statusCode, reasonPhrase); + response->SetBodyStream( + std::make_unique(m_state->ResponseBodies.back())); + return response; + } + + std::shared_ptr m_state; + std::mutex m_mutex; + }; + + Files::DataLake::DataLakeClientOptions CreateSessionOptions( + const std::shared_ptr& state) + { + Files::DataLake::DataLakeClientOptions options; + options.Transport.Transport = std::make_shared(state); + options.Session.Mode = Blobs::SessionMode::Enabled; + options.Session.AccountName = "account"; + return options; + } + } // namespace + + TEST(DataLakeSessionAuthenticationTest, DownloadUsesBlobSessionPipeline) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileClient client( + "https://account.dfs.core.windows.net/filesystem/file", + std::make_shared(), + CreateSessionOptions(state)); + + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Post); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[0].Component, "session"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + EXPECT_EQ(state->Requests[1].Method, Azure::Core::Http::HttpMethod::Get); + EXPECT_EQ(state->Requests[1].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Authorization.find("Session "), 0U); + } + + TEST(DataLakeSessionAuthenticationTest, DfsOperationsUseBearer) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileClient client( + "https://account.dfs.core.windows.net/filesystem/file", + std::make_shared(), + CreateSessionOptions(state)); + + EXPECT_NO_THROW(client.Create()); + + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Put); + EXPECT_EQ(state->Requests[0].Host, "account.dfs.core.windows.net"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + } + + TEST(DataLakeSessionAuthenticationTest, RenamedClientRetainsBlobSessionPipeline) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileSystemClient client( + "https://account.dfs.core.windows.net/filesystem", + std::make_shared(), + CreateSessionOptions(state)); + + auto renamedClient = client.RenameFile("source", "destination").Value; + EXPECT_NO_THROW(renamedClient.Download()); + + ASSERT_EQ(state->Requests.size(), 3U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Put); + EXPECT_EQ(state->Requests[0].Host, "account.dfs.core.windows.net"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + EXPECT_EQ(state->Requests[1].Method, Azure::Core::Http::HttpMethod::Post); + EXPECT_EQ(state->Requests[1].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Component, "session"); + EXPECT_EQ(state->Requests[2].Method, Azure::Core::Http::HttpMethod::Get); + EXPECT_EQ(state->Requests[2].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[2].Authorization.find("Session "), 0U); + } + + class DataLakeSessionAuthenticationRecordedTest : public DataLakeFileClientTest { + }; + + TEST_F(DataLakeSessionAuthenticationRecordedTest, DownloadUsesSessionAuthentication_PLAYBACKONLY_) + { + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + auto credential = std::make_shared(GetTestCredential()); + Files::DataLake::DataLakeFileClient client(m_fileClient->GetUrl(), credential, options); + + EXPECT_NO_THROW(client.Download().Value.Body->ReadToEnd()); + EXPECT_NO_THROW(client.Download().Value.Body->ReadToEnd()); + EXPECT_EQ(sessionRequestCount->load(), 1); + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_NO_THROW(client.GetProperties()); + EXPECT_EQ(sessionRequestCount->load(), 1); + EXPECT_EQ(credential->GetTokenCount(), 2); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp index 5a364a4ad4..f20d43abe1 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp @@ -28,6 +28,133 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { names namespace Azure { namespace Storage { namespace Test { + TEST(FileIdAddressedClientTest, ClientStateAndValidation) + { + Files::Shares::ShareClient shareClient( + "https://account.file.core.windows.net/share?sharesnapshot=snapshot&sig=signature"); + + EXPECT_THROW(shareClient.GetFileClientByFileId(""), std::invalid_argument); + EXPECT_THROW(shareClient.GetDirectoryClientByFileId(""), std::invalid_argument); + EXPECT_THROW( + Files::Shares::ShareFileClient("https://account.file.core.windows.net/share?fileid="), + std::invalid_argument); + EXPECT_THROW( + Files::Shares::ShareDirectoryClient("https://account.file.core.windows.net/share?fileid="), + std::invalid_argument); + + auto fileClient = shareClient.GetFileClientByFileId("12345"); + EXPECT_EQ(fileClient.GetFileId(), "12345"); + auto fileUrl = Core::Url(fileClient.GetUrl()); + EXPECT_EQ(fileUrl.GetQueryParameters().at("sharesnapshot"), "snapshot"); + EXPECT_EQ(fileUrl.GetQueryParameters().at("sig"), "signature"); + EXPECT_EQ(fileUrl.GetQueryParameters().at("fileid"), "12345"); + auto snapshotFileClient = fileClient.WithShareSnapshot("new-snapshot"); + EXPECT_EQ(snapshotFileClient.GetFileId(), "12345"); + EXPECT_EQ( + Core::Url(snapshotFileClient.GetUrl()).GetQueryParameters().at("sharesnapshot"), + "new-snapshot"); + + auto directoryClient = shareClient.GetDirectoryClientByFileId("67890"); + EXPECT_EQ(directoryClient.GetFileId(), "67890"); + auto snapshotDirectoryClient = directoryClient.WithShareSnapshot("new-snapshot"); + EXPECT_EQ(snapshotDirectoryClient.GetFileId(), "67890"); + + Files::Shares::ShareFileClient uppercaseFileIdClient( + "https://account.file.core.windows.net/share?FILEID=12345"); + EXPECT_EQ(uppercaseFileIdClient.GetFileId(), "12345"); + + Files::Shares::ShareDirectoryClient uppercaseDirectoryFileIdClient( + "https://account.file.core.windows.net/share?FILEID=67890"); + EXPECT_EQ(uppercaseDirectoryFileIdClient.GetFileId(), "67890"); + + Files::Shares::ShareFileClient pathClient("https://account.file.core.windows.net/share/path"); + EXPECT_TRUE(pathClient.GetFileId().empty()); + EXPECT_THROW(pathClient.GetFileLinks(), std::logic_error); + } + + TEST(FileIdAddressedClientTest, FileUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto fileClient = shareClient.GetFileClientByFileId("12345"); + std::vector content(1); + Core::IO::MemoryBodyStream contentStream(content); + Core::Http::HttpRange sourceRange; + + EXPECT_THROW(fileClient.Create(1), std::logic_error); + EXPECT_THROW(fileClient.Delete(), std::logic_error); + EXPECT_THROW(fileClient.DeleteIfExists(), std::logic_error); + EXPECT_THROW(fileClient.Download(), std::logic_error); + EXPECT_THROW(fileClient.DownloadTo(content.data(), content.size()), std::logic_error); + EXPECT_THROW(fileClient.DownloadTo("file-that-must-not-be-created"), std::logic_error); + EXPECT_THROW(fileClient.UploadFrom(content.data(), content.size()), std::logic_error); + EXPECT_THROW(fileClient.UploadFrom("file-that-must-not-be-opened"), std::logic_error); + EXPECT_THROW( + fileClient.StartCopy("https://account.file.core.windows.net/share/source"), + std::logic_error); + EXPECT_THROW(fileClient.AbortCopy("copy-id"), std::logic_error); + EXPECT_THROW( + fileClient.SetProperties( + Files::Shares::Models::FileHttpHeaders(), Files::Shares::Models::FileSmbProperties()), + std::logic_error); + EXPECT_THROW(fileClient.SetMetadata(Storage::Metadata()), std::logic_error); + EXPECT_THROW(fileClient.UploadRange(0, contentStream), std::logic_error); + EXPECT_THROW(fileClient.ClearRange(0, 1), std::logic_error); + EXPECT_THROW(fileClient.GetRangeList(), std::logic_error); + EXPECT_THROW(fileClient.GetRangeListDiff("snapshot"), std::logic_error); + EXPECT_THROW(fileClient.GetAllRangeList(), std::logic_error); + EXPECT_THROW(fileClient.GetAllRangeListDiff("snapshot"), std::logic_error); + EXPECT_THROW(fileClient.ListHandles(), std::logic_error); + EXPECT_THROW(fileClient.ForceCloseHandle("handle-id"), std::logic_error); + EXPECT_THROW(fileClient.ForceCloseAllHandles(), std::logic_error); + EXPECT_THROW( + fileClient.UploadRangeFromUri( + 0, "https://account.file.core.windows.net/share/source", sourceRange), + std::logic_error); + EXPECT_THROW(fileClient.CreateSymbolicLink("target"), std::logic_error); + EXPECT_THROW(fileClient.GetSymbolicLink(), std::logic_error); + EXPECT_THROW(fileClient.CreateHardLink("target"), std::logic_error); + } + + TEST(FileIdAddressedClientTest, DirectoryUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto directoryClient = shareClient.GetDirectoryClientByFileId("67890"); + + EXPECT_THROW(directoryClient.GetSubdirectoryClient("child"), std::logic_error); + EXPECT_THROW(directoryClient.GetFileClient("file"), std::logic_error); + EXPECT_THROW(directoryClient.Create(), std::logic_error); + EXPECT_THROW(directoryClient.CreateIfNotExists(), std::logic_error); + EXPECT_THROW(directoryClient.RenameFile("file", "destination"), std::logic_error); + EXPECT_THROW(directoryClient.RenameSubdirectory("directory", "destination"), std::logic_error); + EXPECT_THROW(directoryClient.Delete(), std::logic_error); + EXPECT_THROW(directoryClient.DeleteIfExists(), std::logic_error); + EXPECT_THROW( + directoryClient.SetProperties(Files::Shares::Models::FileSmbProperties()), + std::logic_error); + EXPECT_THROW(directoryClient.SetMetadata(Storage::Metadata()), std::logic_error); + EXPECT_THROW(directoryClient.ListFilesAndDirectories(), std::logic_error); + EXPECT_THROW(directoryClient.ListHandles(), std::logic_error); + EXPECT_THROW(directoryClient.ForceCloseHandle("handle-id"), std::logic_error); + EXPECT_THROW(directoryClient.ForceCloseAllHandles(), std::logic_error); + } + + TEST(FileIdAddressedClientTest, LeaseUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto fileClient = shareClient.GetFileClientByFileId("12345"); + Files::Shares::ShareLeaseClient leaseClient( + fileClient, Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); + EXPECT_THROW( + leaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration), + std::logic_error); + EXPECT_THROW(leaseClient.Renew(), std::logic_error); + EXPECT_THROW(leaseClient.Release(), std::logic_error); + EXPECT_THROW( + leaseClient.Change(Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()), + std::logic_error); + EXPECT_THROW(leaseClient.Break(), std::logic_error); + } + void FileShareClientTest::SetUp() { FileShareServiceClientTest::SetUp(); @@ -205,7 +332,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareClientTest, ShareProperties) + TEST_F(FileShareClientTest, ShareProperties_PLAYBACKONLY_) { const int32_t quota32GB = 32; const int32_t quota64GB = 64; @@ -218,6 +345,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(m_shareClient->SetProperties(options)); auto result = m_shareClient->GetProperties(); EXPECT_EQ(quota32GB, result.Value.Quota); + ASSERT_TRUE(result.Value.CreatedOn.HasValue()); + EXPECT_TRUE(IsValidTime(result.Value.CreatedOn.Value())); options.ShareQuotaInGiB = quota64GB; EXPECT_NO_THROW(m_shareClient->SetProperties(options)); result = m_shareClient->GetProperties(); @@ -827,6 +956,54 @@ namespace Azure { namespace Storage { namespace Test { && !properties.EnableSnapshotVirtualDirectoryAccess.Value()); } + TEST_F(FileShareClientTest, ChangeFeed_PLAYBACKONLY_) + { + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + + Files::Shares::CreateShareOptions createOptions; + createOptions.EnableChangeFeed = true; + createOptions.ChangeFeedRetentionInDays = 3; + shareClient.Create(createOptions); + + auto properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(3, properties.ChangeFeedRetentionInDays.Value()); + ASSERT_TRUE(properties.ChangeFeedBlobContainerName.HasValue()); + EXPECT_EQ(0U, properties.ChangeFeedBlobContainerName.Value().find("$fileschangefeed-")); + + Files::Shares::SetSharePropertiesOptions setPropertiesOptions; + setPropertiesOptions.EnableChangeFeed = true; + setPropertiesOptions.ChangeFeedRetentionInDays = 5; + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(5, properties.ChangeFeedRetentionInDays.Value()); + EXPECT_TRUE(properties.ChangeFeedBlobContainerName.HasValue()); + + setPropertiesOptions.EnableChangeFeed.Reset(); + setPropertiesOptions.ChangeFeedRetentionInDays = 30; + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(30, properties.ChangeFeedRetentionInDays.Value()); + + setPropertiesOptions.EnableChangeFeed = false; + setPropertiesOptions.ChangeFeedRetentionInDays.Reset(); + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_FALSE(properties.IsChangeFeedEnabled.Value()); + } + TEST_F(FileShareClientTest, FilePermissionFormat) { auto sddlPermission diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp index 3539fb6f45..3fb8112d50 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp @@ -30,6 +30,34 @@ namespace Azure { namespace Storage { namespace Test { m_fileClient->Create(1024); } + TEST_F(FileShareFileClientTest, FileIdAddressing_PLAYBACKONLY_) + { + auto pathProperties = m_fileClient->GetProperties().Value; + ASSERT_FALSE(pathProperties.SmbProperties.FileId.empty()); + ASSERT_FALSE(pathProperties.SmbProperties.ParentFileId.empty()); + + auto fileIdClient = m_shareClient->GetFileClientByFileId(pathProperties.SmbProperties.FileId); + auto fileIdProperties = fileIdClient.GetProperties().Value; + ASSERT_TRUE(fileIdProperties.FileName.HasValue()); + EXPECT_EQ(fileIdProperties.FileName.Value(), m_fileName); + EXPECT_EQ(fileIdProperties.SmbProperties.FileId, pathProperties.SmbProperties.FileId); + + auto links = fileIdClient.GetFileLinks().Value; + ASSERT_EQ(links.Links.size(), 1); + EXPECT_EQ(links.Links[0].Name, m_fileName); + EXPECT_EQ(links.Links[0].ParentId, pathProperties.SmbProperties.ParentFileId); + + auto directoryPathProperties = m_fileShareDirectoryClient->GetProperties().Value; + ASSERT_FALSE(directoryPathProperties.SmbProperties.FileId.empty()); + auto directoryIdClient + = m_shareClient->GetDirectoryClientByFileId(directoryPathProperties.SmbProperties.FileId); + auto directoryIdProperties = directoryIdClient.GetProperties().Value; + ASSERT_TRUE(directoryIdProperties.FileName.HasValue()); + EXPECT_EQ(directoryIdProperties.FileName.Value(), m_directoryName); + EXPECT_EQ( + directoryIdProperties.SmbProperties.FileId, directoryPathProperties.SmbProperties.FileId); + } + TEST_F(FileShareFileClientTest, CreateDeleteFiles) { { @@ -2428,16 +2456,9 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareFileClientTest, PremiumPosixProperties) + TEST_F(NfsFileShareClientTest, PremiumPosixProperties) { - auto shareServiceClient = *m_premiumShareServiceClient; - - auto shareName = LowercaseRandomString(); - auto shareClient = GetPremiumShareClientForTest(shareName); - Files::Shares::CreateShareOptions shareOptions; - shareOptions.EnabledProtocols = Files::Shares::Models::ShareProtocols::Nfs; - EXPECT_NO_THROW(shareClient.Create(shareOptions)); - auto otherProperties = m_fileClient->GetProperties().Value; + auto& shareClient = *m_shareClient; auto fileName = LowercaseRandomString(); auto fileClient = shareClient.GetRootDirectoryClient().GetFileClient(fileName); @@ -2554,8 +2575,8 @@ namespace Azure { namespace Storage { namespace Test { auto symbolicLinkClient = shareClient.GetRootDirectoryClient().GetFileClient(LowercaseRandomString()); Files::Shares::CreateSymbolicLinkOptions createSymbolicLinkOptions; - createSymbolicLinkOptions.CreatedOn = otherProperties.SmbProperties.CreatedOn; - createSymbolicLinkOptions.LastWrittenOn = otherProperties.SmbProperties.LastWrittenOn; + createSymbolicLinkOptions.CreatedOn = properties.SmbProperties.CreatedOn; + createSymbolicLinkOptions.LastWrittenOn = properties.SmbProperties.LastWrittenOn; createSymbolicLinkOptions.Metadata = RandomMetadata(); createSymbolicLinkOptions.Group = "123"; @@ -2668,6 +2689,7 @@ namespace Azure { namespace Storage { namespace Test { // From file fileClient = shareClient.GetRootDirectoryClient().GetFileClient(LowercaseRandomString()); EXPECT_NO_THROW(fileClient.UploadFrom(tempFilename, uploadOptions)); + DeleteFile(tempFilename); properties = fileClient.GetProperties().Value; EXPECT_TRUE(properties.PosixProperties.FileMode.HasValue()); EXPECT_EQ(properties.PosixProperties.FileMode.Value().ToOctalFileMode(), octalMode); @@ -2684,15 +2706,9 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::Models::NfsFileType::Regular); } - TEST_F(FileShareFileClientTest, PremiumPosixPropertiesForCopy) + TEST_F(NfsFileShareClientTest, PremiumPosixPropertiesForCopy) { - auto shareServiceClient = *m_premiumShareServiceClient; - - auto shareName = LowercaseRandomString(); - auto shareClient = GetPremiumShareClientForTest(shareName); - Files::Shares::CreateShareOptions shareOptions; - shareOptions.EnabledProtocols = Files::Shares::Models::ShareProtocols::Nfs; - EXPECT_NO_THROW(shareClient.Create(shareOptions)); + auto& shareClient = *m_shareClient; auto sourceName = LowercaseRandomString(); auto sourceClient = shareClient.GetRootDirectoryClient().GetFileClient(sourceName); diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp index f2ff8c2a8b..cb91a62dde 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp @@ -111,7 +111,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareServiceClientTest, ListShares) + TEST_F(FileShareServiceClientTest, ListShares_PLAYBACKONLY_) { std::string prefix1 = LowercaseRandomString(); std::string prefix2 = LowercaseRandomString(); @@ -136,6 +136,12 @@ namespace Azure { namespace Storage { namespace Test { for (const auto& share : page.Shares) { result.insert(share.Name); + if (shareSet1.find(share.Name) != shareSet1.end() + || shareSet2.find(share.Name) != shareSet2.end()) + { + ASSERT_TRUE(share.Details.CreatedOn.HasValue()); + EXPECT_TRUE(IsValidTime(share.Details.CreatedOn.Value())); + } } } for (const auto& name : shareSet1) From 3ecb51adc1627c8c963d1f81914c370cfd128d9c Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Wed, 14 Oct 2026 18:43:32 +1100 Subject: [PATCH 11/13] Fix Arrow blob listing immutability policy parsing Initialize the nullable policy before assigning either Arrow immutability column without clearing a value parsed from the other column. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/blob_container_client.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index 6ccd5e28c1..4816aaf256 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -427,12 +427,22 @@ namespace Azure { namespace Storage { namespace Blobs { } else if (columnName == "ImmutabilityPolicyUntilDate") { - result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy.Value().ExpiresOn + auto& policy = result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy; + if (!policy.HasValue()) + { + policy = Models::BlobImmutabilityPolicy(); + } + policy.Value().ExpiresOn = std::chrono::system_clock::time_point(std::chrono::seconds(parsedIntValue)); } else if (columnName == "ImmutabilityPolicyMode") { - result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy.Value().PolicyMode + auto& policy = result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy; + if (!policy.HasValue()) + { + policy = Models::BlobImmutabilityPolicy(); + } + policy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(std::move(parsedStringValue)); } else if (columnName == "VersionId") From 6079cfd47e5d5ce08f0193054cdb6f80e4208810 Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Wed, 14 Oct 2026 19:20:22 +1100 Subject: [PATCH 12/13] Reclassify Arrow listing default as other changes Move the default XML-to-Arrow listing response format note out of Breaking Changes because the public blob listing behavior is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/storage/azure-storage-blobs/CHANGELOG.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 1213ed93cf..899b43734a 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -13,18 +13,15 @@ managed downloads. - Added `Models::DownloadHint` and `Models::DownloadBlobDetails::DownloadHint`. -### Breaking Changes - -- Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to - Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue - requesting XML responses. - ### Bugs Fixed - Fixed CMake dependency resolution when FlatCC provides only a shared runtime library. ### Other Changes +- Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to + Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue + requesting XML responses. - Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to 4 MiB, matching the default subsequent chunk size. Explicitly configured `InitialChunkSize` and `SingleUploadThreshold` values are still respected. From 068e1cee9a421edf01452503de1620ce5f8ae4bd Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Thu, 15 Oct 2026 11:39:04 +1100 Subject: [PATCH 13/13] Use new APIView parser without production submissions for testing Combine parser pin from PR #7464 with the exact PR #7448 Storage head. Keep APIReview token generation and packages publication, but disable both production revision detection steps via Condition: false. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/archetype-sdk-client.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 4b91f1a9a5..b5ac0a3fa1 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -86,7 +86,7 @@ jobs: image: $(WINDOWSVMIMAGE) os: windows variables: - apiviewParserVersion: "20240927.5" + apiviewParserVersion: "20261009.3" VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' Package.EnableSBOMSigning: true steps: @@ -334,6 +334,8 @@ jobs: - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory)\packages + # Test-only: keep JSON artifacts without submitting revisions to production APIView. + Condition: false - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: