diff --git a/.vscode/cspell.json b/.vscode/cspell.json index a973802240..4555949b78 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -98,6 +98,7 @@ "cgmanifest", "Chelnokov", "chewi", + "chinacloudapi", "Chukwu", "cmake", "CMOCKA", @@ -291,6 +292,7 @@ "unskipped", "UPNs", "usgov", + "usgovcloudapi", "usgoviowa", "usgovvirginia", "vcpkg", @@ -315,6 +317,7 @@ "Xclang", "xcode", "xcodebuild", + "xfiles", "XSMB", "zulu" ], diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 4b91f1a9a5..b5ac0a3fa1 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -86,7 +86,7 @@ jobs: image: $(WINDOWSVMIMAGE) os: windows variables: - apiviewParserVersion: "20240927.5" + apiviewParserVersion: "20261009.3" VCPKG_DEFAULT_TRIPLET: 'x64-windows-static' Package.EnableSBOMSigning: true steps: @@ -334,6 +334,8 @@ jobs: - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml parameters: ArtifactPath: $(Build.ArtifactStagingDirectory)\packages + # Test-only: keep JSON artifacts without submitting revisions to production APIView. + Condition: false - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: diff --git a/sdk/storage/assets.json b/sdk/storage/assets.json index 818eca0575..9f08d82d12 100644 --- a/sdk/storage/assets.json +++ b/sdk/storage/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "cpp", "TagPrefix": "cpp/storage", - "Tag": "cpp/storage_6016ecf9a6" + "Tag": "cpp/storage_01f138ad9c" } diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 66f9d87a90..899b43734a 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -4,7 +4,14 @@ ### Features Added -### Breaking Changes +- Added support for persisted session authentication for eligible blob download requests. +- Added `BlobClient::GetLayout` and related options and response types for retrieving locality + endpoints, byte ranges, and blob properties. Ranges are exposed directly through + `Models::BlobLayoutInfo::Ranges` as a vector of `Models::BlobLayoutRange`. +- Added `DownloadBlobOptions::LayoutEndpoint` for one-shot download routing and + `DownloadBlobToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for + managed downloads. +- Added `Models::DownloadHint` and `Models::DownloadBlobDetails::DownloadHint`. ### Bugs Fixed @@ -12,6 +19,13 @@ ### Other Changes +- Changed the default response format for `ListBlobs` and `ListBlobsByHierarchy` from XML to + Apache Arrow. Set `ListBlobsOptions::ResponseFormat` to `StorageResponseFormat::Xml` to continue + requesting XML responses. +- Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to + 4 MiB, matching the default subsequent chunk size. Explicitly configured + `InitialChunkSize` and `SingleUploadThreshold` values are still respected. + ## 12.19.0 (2026-09-29) ### Features Added diff --git a/sdk/storage/azure-storage-blobs/CMakeLists.txt b/sdk/storage/azure-storage-blobs/CMakeLists.txt index 5e7ff0cc3f..7b1f5203b8 100644 --- a/sdk/storage/azure-storage-blobs/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/CMakeLists.txt @@ -75,7 +75,13 @@ set( src/page_blob_client.cpp src/private/avro_parser.cpp src/private/avro_parser.hpp + src/private/data_locality.cpp + src/private/data_locality.hpp src/private/package_version.hpp + src/private/session_authentication_policy.cpp + src/private/session_authentication_policy.hpp + src/private/token_credential_session_provider.cpp + src/private/token_credential_session_provider.hpp src/rest_client.cpp ) diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp index 2a1e90ad64..ec3511e048 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_client.hpp @@ -425,6 +425,17 @@ namespace Azure { namespace Storage { namespace Blobs { const GetAccountInfoOptions& options = GetAccountInfoOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Gets the data locality layout for this blob. + * + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A paged response containing the blob's data locality layout. + */ + BlobLayoutPagedResponse GetLayout( + const GetBlobLayoutOptions& options = GetBlobLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + protected: /** @brief Blob Url */ Azure::Core::Url m_blobUrl; diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp index 12899e479e..a8eb25e6f0 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp @@ -153,6 +153,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified container for deletion. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. * @return A DeleteBlobContainerResult if successful. @@ -165,6 +173,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified container for deletion if it exists. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. * @return A DeleteBlobContainerResult if the container exists. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp index aa9434d246..923b6fd4ab 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_options.hpp @@ -5,6 +5,9 @@ #include "azure/storage/blobs/rest_client.hpp" +#include +#include +#include #include #include #include @@ -23,6 +26,90 @@ namespace Azure { namespace Storage { namespace Blobs { + class ContainerSessionProvider; + class SessionProvider; + + namespace _detail { + class SessionAuthenticationPolicy; + class TokenCredentialSessionProvider; + } // namespace _detail + + /** + * @brief Specifies how persisted session authentication is used. + */ + enum class SessionMode + { + /** + * @brief Lets the SDK determine when to use sessions. + * + * @note Currently behaves the same as #Disabled. + */ + Auto, + + /** + * @brief Disables session authentication. + */ + Disabled, + + /** + * @brief Prefers session authentication for eligible requests. + * + * @note The client can fall back to bearer authentication while acquiring a session, when + * session creation is temporarily unavailable, or when the service rejects a session. + */ + Enabled, + }; + + /** + * @brief Configures persisted session authentication. + */ + struct SessionOptions final + { + /** + * @brief The session authentication mode. + * + * In Auto mode, clients currently use bearer authentication. + */ + SessionMode Mode = SessionMode::Auto; + + /** + * @brief Account name used when signing requests. Required when it cannot be inferred from URL. + * + * For custom endpoints, both AccountName and Provider must be specified. + */ + std::string AccountName; + + /** + * @brief Optional provider whose session cache can be shared by multiple clients. + * + * When unset, each independently constructed client creates its own provider. Clients derived + * from that client share its provider and cache. For custom endpoints, both Provider and + * AccountName must be specified. + */ + std::shared_ptr Provider; + }; + + /** + * @brief Determines whether locality-aware routing is used for managed downloads. + */ + enum class LayoutAwareRouting + { + /** + * @brief The client library determines whether locality-aware routing is enabled. + */ + Auto, + + /** + * @brief Locality-aware routing is disabled. + */ + Disabled, + + /** + * @brief Locality-aware routing is enabled. + */ + Enabled, + }; + /** * @brief Audiences available for blob service * @@ -287,6 +374,50 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable DownloadValidationOptions; + + /** + * @brief Configures persisted session authentication. + */ + Blobs::SessionOptions Session; + }; + + /** + * @brief Base type for providers that own and cache persisted sessions. + * + * Session providers can be shared by multiple clients through #SessionOptions::Provider. + * Use #ContainerSessionProvider for TokenCredential-based session authentication. + */ + class SessionProvider { + public: + virtual ~SessionProvider(); + + private: + friend class ContainerSessionProvider; + friend class _detail::SessionAuthenticationPolicy; + + SessionProvider(); + + std::shared_ptr<_detail::TokenCredentialSessionProvider> m_provider; + }; + + /** + * @brief Creates and caches container-scoped sessions using a TokenCredential. + */ + class ContainerSessionProvider final : public SessionProvider { + public: + /** + * @brief Creates a provider backed by a TokenCredential. + * + * @param serviceUrl The Blob service URL. + * @param credential The credential used to create sessions. + * @param options Client options used by Create Session requests. + */ + ContainerSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options); + + ~ContainerSessionProvider() override; }; /** @@ -478,7 +609,7 @@ namespace Azure { namespace Storage { namespace Blobs { enum class StorageResponseFormat { /** - * @brief Let the service choose the response format (default). + * @brief Use the default response format, which is currently Apache Arrow. */ Auto, /** @@ -801,6 +932,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief Optional. The layout endpoint to use for this download. + * + * When set, the request is sent to this endpoint while preserving the client endpoint in the + * Host header. + */ + std::string LayoutEndpoint; }; /** @@ -823,7 +962,7 @@ namespace Azure { namespace Storage { namespace Blobs { * downloaded in a single request. Blobs larger than this limit will continue being downloaded * in chunks of size ChunkSize. */ - int64_t InitialChunkSize = 256 * 1024 * 1024; + int64_t InitialChunkSize = 4 * 1024 * 1024; /** * @brief The maximum number of bytes in a single request. @@ -840,6 +979,12 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Optional. Configures whether to do content validation for blob downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief Determines whether locality-aware routing is used for the parallel range requests. + * Disabled by default. + */ + Blobs::LayoutAwareRouting LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; }; /** @@ -1069,7 +1214,7 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Blob smaller than this will be uploaded with a single upload operation. This value * cannot be larger than 5000 MiB. */ - int64_t SingleUploadThreshold = 256 * 1024 * 1024; + int64_t SingleUploadThreshold = 4 * 1024 * 1024; /** * @brief The maximum number of bytes in a single request. This value cannot be larger than @@ -1812,6 +1957,27 @@ namespace Azure { namespace Storage { namespace Blobs { { }; + /** + * @brief Optional parameters for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + struct GetBlobLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + BlobAccessConditions AccessConditions; + }; + namespace _detail { inline std::string TagsToString(const std::map& tags) { diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp index de54ac6a48..4521c85d1e 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_responses.hpp @@ -29,6 +29,38 @@ namespace Azure { namespace Storage { namespace Models { + /** + * @brief Describes a range in a blob's data locality layout. + */ + struct BlobLayoutRange final + { + /** + * @brief The range of bytes served by the endpoint. + */ + Azure::Core::Http::HttpRange Range; + + /** + * @brief The host and port of the endpoint serving this range. + */ + std::string Endpoint; + }; + + /** + * @brief A page of blob layout information and the blob properties returned with it. + */ + struct BlobLayoutInfo final + { + /** + * @brief The ranges in this page of the blob's data locality layout. + */ + std::vector Ranges; + + /** + * @brief The blob properties returned with the layout. + */ + BlobProperties Properties; + }; + /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::DownloadTo. */ @@ -482,6 +514,27 @@ namespace Azure { namespace Storage { friend class Azure::Core::PagedResponse; }; + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + class BlobLayoutPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and blob properties in this page. + */ + Models::BlobLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_blobClient; + GetBlobLayoutOptions m_operationOptions; + + friend class Blobs::BlobClient; + friend class Azure::Core::PagedResponse; + }; + namespace _detail { void ParseListBlobsResult(Models::_detail::ListBlobsResult& result); } // namespace _detail diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp index bc81ae9b13..5a5380de3d 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_service_client.hpp @@ -206,6 +206,14 @@ namespace Azure { namespace Storage { namespace Blobs { * @brief Marks the specified blob container for deletion. The container and any blobs * contained within it are later deleted during garbage collection. * + * @remark When a container is deleted, a container with the same name cannot be created for at + * least 30 seconds. The container might not be available for more than 30 seconds if the + * service is still processing the request. While the container is being deleted, attempts to + * create a container with the same name fail with status code 409 (Conflict). Operations on + * blobs within the container, including reads and writes, may continue to succeed for up to 30 + * seconds after the delete request is accepted. After this period, all operations on the + * container and its blobs fail with status code 404 (Not Found). + * * @param blobContainerName The name of the container to delete. * @param options Optional parameters to execute this function. * @param context Context for cancelling long running operations. diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index c70f815c71..b3adf763ee 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -1450,6 +1450,19 @@ namespace Azure { namespace Storage { namespace Blobs { std::string ContentType; }; } // namespace _detail + /** + * @brief Indicates the download hint for the blob. + */ + class DownloadHint final : public Core::_internal::ExtendableEnumeration { + public: + /** Constructs a new DownloadHint instance */ + DownloadHint() = default; + /** Constructs a new DownloadHint from a string. */ + explicit DownloadHint(std::string value) : ExtendableEnumeration(std::move(value)) {} + + /** Constant value of type DownloadHint: Layout */ + AZ_STORAGE_BLOBS_DLLEXPORT const static DownloadHint Layout; + }; /** * @brief Detailed information of the downloaded blob. */ @@ -1614,6 +1627,10 @@ namespace Azure { namespace Storage { namespace Blobs { * The underlying tier of a smart tier blob. Only returned if the blob is in Smart tier. */ Nullable SmartAccessTier; + /** + * Indicates the download hint for the blob. + */ + Nullable DownloadHint; }; /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::Download. @@ -1646,12 +1663,12 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobType BlobType; }; /** - * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetProperties. + * @brief Properties of a blob. */ struct BlobProperties final { /** - * Array of ObjectReplicationPolicy. + * Object replication policies associated with the source blob. */ std::vector ObjectReplicationSourceProperties; /** @@ -1659,89 +1676,67 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable ImmutabilityPolicy; /** - * Standard HTTP properties supported by containers and blobs. + * Standard HTTP properties supported by the blob. */ BlobHttpHeaders HttpHeaders; /** - * Returns the date and time the blob was last modified. Any operation that modifies the blob, - * including an update of the blob's metadata or properties, changes the last-modified time of - * the blob. + * The date and time when the blob was last modified. */ DateTime LastModified; /** - * Returns the date and time the blob was created. + * The date and time when the blob was created. */ DateTime CreatedOn; /** - * A set of name-value pair associated with this blob. + * A set of name-value pairs associated with the blob. */ Core::CaseInsensitiveMap Metadata; /** - * Optional. Only valid when Object Replication is enabled for the storage container and on - * the destination blob of the replication. + * The object replication policy identifier for the destination blob. */ Nullable ObjectReplicationDestinationPolicyId; /** - * The blob's type. + * The type of the blob. */ Models::BlobType BlobType; /** - * Conclusion time of the last attempted Copy Blob operation where this blob was the - * destination blob. This value can specify the time of a completed, aborted, or failed copy - * attempt. This header does not appear if a copy is pending, if this blob has never been the - * destination in a Copy Blob operation, or if this blob has been modified after a concluded - * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List. + * The completion time of the last attempted copy operation. */ Nullable CopyCompletedOn; /** - * Only appears when x-ms-copy-status is failed or pending. Describes the cause of the last - * fatal or non-fatal copy operation failure. This header does not appear if this blob has - * never been the destination in a Copy Blob operation, or if this blob has been modified - * after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or Put Block - * List. + * A description of the last copy operation status. */ Nullable CopyStatusDescription; /** - * String identifier for this copy operation. Use with Get Blob Properties to check the status - * of this copy operation, or pass to Abort Copy Blob to abort a pending copy. + * The identifier of the last copy operation. */ Nullable CopyId; /** - * Contains the number of bytes copied and the total bytes in the source in the last attempted - * Copy Blob operation where this blob was the destination blob. Can show between 0 and - * Content-Length bytes copied. This header does not appear if this blob has never been the - * destination in a Copy Blob operation, or if this blob has been modified after a concluded - * Copy Blob operation using Set Blob Properties, Put Blob, or Put Block List. + * The progress of the last copy operation. */ Nullable CopyProgress; /** - * URL up to 2 KB in length that specifies the source blob or file used in the last attempted - * Copy Blob operation where this blob was the destination blob. This header does not appear - * if this blob has never been the destination in a Copy Blob operation, or if this blob has - * been modified after a concluded Copy Blob operation using Set Blob Properties, Put Blob, or - * Put Block List. + * The source of the last copy operation. */ Nullable CopySource; /** - * State of the copy operation identified by x-ms-copy-id. + * The status of the last copy operation. */ Nullable CopyStatus; /** - * Included if the blob is incremental copy blob. + * Indicates whether the blob is an incremental copy. */ Nullable IsIncrementalCopy; /** - * Included if the blob is incremental copy blob or incremental copy snapshot, if - * x-ms-copy-status is success. Snapshot time of the last successful incremental copy snapshot - * for this blob. + * The destination snapshot of the last incremental copy. */ Nullable IncrementalCopyDestinationSnapshot; /** - * When a blob is leased, specifies whether the lease is of infinite or fixed duration. + * The duration of the blob lease. */ Nullable LeaseDuration; /** - * Lease state of the blob. + * The current lease state of the blob. */ Nullable LeaseState; /** @@ -1749,80 +1744,59 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable LeaseStatus; /** - * Size of the blob in bytes. + * The size of the blob in bytes. */ std::int64_t BlobSize = std::int64_t(); /** - * The ETag contains a value that you can use to perform operations conditionally. If the - * request version is 2011-08-18 or newer, the ETag value will be in quotes. + * The entity tag of the blob. */ Azure::ETag ETag; /** - * The current sequence number for a page blob. This header is not returned for block blobs or - * append blobs. + * The current sequence number of a page blob. */ Nullable SequenceNumber; /** - * The number of committed blocks present in the blob. This header is returned only for append - * blobs. + * The number of committed blocks in an append blob. */ Nullable CommittedBlockCount; /** - * The value of this header is set to true if the blob data and application metadata are - * completely encrypted using the specified algorithm. Otherwise, the value is set to false - * (when the blob is unencrypted, or if only parts of the blob/application metadata are - * encrypted). + * Indicates whether the blob is encrypted by the service. */ bool IsServerEncrypted = bool(); /** - * The SHA-256 hash of the encryption key used to encrypt the metadata. This header is only - * returned when the metadata was encrypted with a customer-provided key. + * The SHA-256 hash of the customer-provided encryption key. */ Nullable> EncryptionKeySha256; /** - * Returns the name of the encryption scope used to encrypt the blob contents and application - * metadata. Note that the absence of this header implies use of the default account - * encryption scope. + * The encryption scope used to encrypt the blob. */ Nullable EncryptionScope; /** - * The tier of page blob on a premium storage account or tier of block blob on blob storage - * LRS accounts. For a list of allowed premium page blob tiers, see - * https://learn.microsoft.com/azure/virtual-machines/disks-types#premium-ssd. For blob - * storage LRS accounts, valid values are Hot/Cool/Archive. + * The access tier of the blob. */ Nullable AccessTier; /** - * For page blobs on a premium storage account only. If the access tier is not explicitly set - * on the blob, the tier is inferred based on its content length and this header will be - * returned with true value. + * Indicates whether the access tier was inferred. */ Nullable IsAccessTierInferred; /** - * For blob storage LRS accounts, valid values are - * rehydrate-pending-to-hot/rehydrate-pending-to-cool. If the blob is being rehydrated and is - * not complete then this header is returned indicating that rehydrate is pending and also - * tells the destination tier. + * The archive rehydration status of the blob. */ Nullable ArchiveStatus; /** - * The time the tier was changed on the object. This is only returned if the tier on the block - * blob was ever set. + * The time when the access tier was last changed. */ Nullable AccessTierChangedOn; /** - * The underlying tier of a smart tier blob. Only returned if the blob is in Smart tier. + * The underlying access tier of a smart-tier blob. */ Nullable SmartAccessTier; /** - * A DateTime value returned by the service that uniquely identifies the blob. The value of - * this header indicates the blob version, and may be used in subsequent requests to access - * this version of the blob. + * The version identifier of the blob. */ Nullable VersionId; /** - * The value of this header indicates whether version of this blob is a current version, see - * also x-ms-version-id header. + * Indicates whether this is the current blob version. */ Nullable IsCurrentVersion; /** @@ -1830,25 +1804,23 @@ namespace Azure { namespace Storage { namespace Blobs { */ Nullable TagCount; /** - * The time this blob will expire. + * The time when the blob will expire. */ Nullable ExpiresOn; /** - * If this blob has been sealed. + * Indicates whether the append blob is sealed. */ Nullable IsSealed; /** - * If an object is in rehydrate pending state then this header is returned with priority of - * rehydrate. Valid values are High and Standard. + * The priority of an archive rehydration operation. */ Nullable RehydratePriority; /** - * UTC date/time value generated by the service that indicates the time at which the blob was - * last read or written to. + * The time when the blob was last read or written. */ Nullable LastAccessedOn; /** - * Indicates if a legal hold is present on the blob. + * Indicates whether the blob has a legal hold. */ bool HasLegalHold = false; }; @@ -1880,6 +1852,65 @@ namespace Azure { namespace Storage { namespace Blobs { */ bool Deleted = true; }; + namespace _detail { + struct BlobLayoutRangesItem final + { + /** + * The start byte offset of the range. + */ + std::int64_t Start = std::int64_t(); + /** + * The end byte offset of the range. + */ + std::int64_t End = std::int64_t(); + /** + * Index into the Endpoints array indicating which endpoint serves this range. + */ + std::int32_t EndpointIndex = std::int32_t(); + }; + struct BlobLayoutEndpointsItem final + { + /** + * The index of the endpoint, referenced by Range elements. + */ + std::int32_t Index = std::int32_t(); + /** + * The host:port of the endpoint. + */ + std::string Value; + }; + /** + * @brief Response type for #Azure::Storage::Blobs::BlobClient::GetLayout. + */ + struct BlobLayout final + { + /** + * Array of BlobLayout-RangesItem. + */ + std::vector Ranges; + /** + * Array of BlobLayout-EndpointsItem. + */ + std::vector Endpoints; + /** + * The continuation marker used for this request. + */ + std::string Marker; + /** + * If the number of ranges exceeds MaxResults, a NextMarker is returned for use in + * subsequent requests to continue listing. + */ + std::string NextMarker; + /** + * The maximum number of ranges to return per request. + */ + std::int32_t MaxResults = std::int32_t(); + /** + * Properties of a blob. + */ + BlobProperties Properties; + }; + } // namespace _detail /** * @brief Response type for #Azure::Storage::Blobs::BlobClient::Undelete. */ @@ -3658,6 +3689,28 @@ namespace Azure { namespace Storage { namespace Blobs { const Core::Url& url, const DeleteBlobOptions& options, const Core::Context& context); + struct GetBlobLayoutOptions final + { + Nullable Snapshot; + Nullable VersionId; + Nullable Marker; + Nullable MaxResults; + Nullable Range; + Nullable LeaseId; + Nullable IfTags; + Nullable IfModifiedSince; + Nullable IfUnmodifiedSince; + ETag IfMatch; + ETag IfNoneMatch; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; + }; + static Response GetLayout( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetBlobLayoutOptions& options, + const Core::Context& context); struct UndeleteBlobOptions final { }; diff --git a/sdk/storage/azure-storage-blobs/src/blob_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_client.cpp index 13aa2c82a4..afcda50efd 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_client.cpp @@ -6,10 +6,13 @@ #include "azure/storage/blobs/append_blob_client.hpp" #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" +#include "private/data_locality.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include +#include #include #include #include @@ -17,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +28,10 @@ #include #include +#include +#include +#include +#include namespace Azure { namespace Storage { namespace Blobs { @@ -31,6 +39,127 @@ namespace Azure { namespace Storage { namespace Blobs { Azure::Core::Context::Key const DataLakeInteroperabilityExtraOptionsKey; } + namespace { + bool HasDataLocalityHint(const Models::DownloadBlobDetails& details) + { + return details.DownloadHint.HasValue() + && Azure::Core::_internal::StringExtensions::ToLower( + details.DownloadHint.Value().ToString()) + == Models::DownloadHint::Layout.ToString(); + } + + bool IsLayoutAwareRoutingEnabled(LayoutAwareRouting value) + { + return value != LayoutAwareRouting::Disabled; + } + + void CompleteBlobPropertiesFromResponse( + Models::BlobProperties& properties, + const Azure::Core::Http::RawResponse& rawResponse) + { + if (properties.AccessTier.HasValue() && !properties.IsAccessTierInferred.HasValue()) + { + properties.IsAccessTierInferred = false; + } + if (properties.VersionId.HasValue() && !properties.IsCurrentVersion.HasValue()) + { + properties.IsCurrentVersion = false; + } + if (properties.CopyStatus.HasValue() && !properties.IsIncrementalCopy.HasValue()) + { + properties.IsIncrementalCopy = false; + } + if (properties.BlobType == Models::BlobType::AppendBlob && !properties.IsSealed.HasValue()) + { + properties.IsSealed = false; + } + + std::map> orPropertiesMap; + for (auto i = rawResponse.GetHeaders().lower_bound("x-ms-or-"); + i != rawResponse.GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; + ++i) + { + const std::string& header = i->first; + auto underscorePos = header.find('_', 8); + if (underscorePos == std::string::npos) + { + continue; + } + std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); + Models::ObjectReplicationRule rule; + rule.RuleId = header.substr(underscorePos + 1); + rule.ReplicationStatus = Models::ObjectReplicationStatus(i->second); + orPropertiesMap[policyId].emplace_back(std::move(rule)); + } + for (auto& property : orPropertiesMap) + { + Models::ObjectReplicationPolicy policy; + policy.PolicyId = property.first; + policy.Rules = std::move(property.second); + properties.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); + } + } + + std::unique_ptr<_detail::DataLocalityLayoutState> CreateDataLocalityLayoutState( + BlobClient blobClient, + Azure::Nullable range, + Azure::ETag eTag, + Azure::Core::Context context) + { + auto refresher = [range = std::move(range), + eTag = std::move(eTag), + context = std::move(context), + blobClient = std::move(blobClient)]() { + try + { + GetBlobLayoutOptions options; + options.Range = range; + options.AccessConditions.IfMatch = eTag; + auto page = blobClient.GetLayout(options, context); + _detail::DataLocalityLayout layout; + for (; page.HasPage(); page.MoveToNextPage(context)) + { + if (!layout.ETag.HasValue()) + { + layout.ETag = page.Layout.Properties.ETag; + } + if (layout.BlobSize == 0) + { + layout.BlobSize = page.Layout.Properties.BlobSize; + } + for (auto& layoutRange : page.Layout.Ranges) + { + if (layoutRange.Endpoint.empty() || layoutRange.Range.Offset < 0 + || !layoutRange.Range.Length.HasValue() || layoutRange.Range.Length.Value() <= 0) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + layout.Ranges.push_back(std::move(layoutRange)); + } + } + std::sort( + layout.Ranges.begin(), + layout.Ranges.end(), + [](const Models::BlobLayoutRange& lhs, const Models::BlobLayoutRange& rhs) { + return lhs.Range.Offset < rhs.Range.Offset; + }); + return layout; + } + catch (const StorageException& exception) + { + if (exception.StatusCode == Core::Http::HttpStatusCode::BadRequest + || static_cast(exception.StatusCode) >= 500) + { + return _detail::DataLocalityLayout(); + } + throw; + } + }; + + return std::make_unique<_detail::DataLocalityLayoutState>(std::move(refresher)); + } + } // namespace + BlobClient BlobClient::CreateFromConnectionString( const std::string& connectionString, const std::string& blobContainerName, @@ -64,6 +193,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::make_unique<_internal::SharedKeyPolicy>(credential); m_pipeline = std::make_shared( @@ -82,15 +212,17 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; { Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) : _internal::StorageScope); - pipelineOptions.TokenAuthPolicy - = std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>( - credential, tokenContext, options.EnableTenantDiscovery); + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + blobUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); } m_pipeline = std::make_shared( @@ -111,6 +243,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); @@ -214,8 +347,14 @@ namespace Azure { namespace Storage { namespace Blobs { = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); } + std::string dataLocalityEndpoint; + const auto downloadContext = !options.LayoutEndpoint.empty() + ? _internal::WithDataLocalityEndpoint(context, options.LayoutEndpoint) + : context.TryGetValue(_internal::DataLocalityEndpointKey, dataLocalityEndpoint) + ? context + : _internal::WithReplicaStatus(context); auto downloadResponse = _detail::BlobClient::Download( - *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); + *m_pipeline, m_blobUrl, protocolLayerOptions, downloadContext); int64_t structuredContentLength = 0; if (isStructuredMessage) @@ -349,6 +488,83 @@ namespace Azure { namespace Storage { namespace Blobs { return downloadResponse; } + BlobLayoutPagedResponse BlobClient::GetLayout( + const GetBlobLayoutOptions& options, + const Azure::Core::Context& context) const + { + _detail::BlobClient::GetBlobLayoutOptions protocolLayerOptions; + protocolLayerOptions.Marker = options.ContinuationToken; + if (options.Range.HasValue()) + { + std::string range = "bytes=" + std::to_string(options.Range.Value().Offset) + "-"; + if (options.Range.Value().Length.HasValue()) + { + range += std::to_string( + options.Range.Value().Offset + options.Range.Value().Length.Value() - 1); + } + protocolLayerOptions.Range = std::move(range); + } + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.IfTags = options.AccessConditions.TagConditions; + protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch; + protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + if (m_clientConfiguration.CustomerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_clientConfiguration.CustomerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 + = m_clientConfiguration.CustomerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm + = m_clientConfiguration.CustomerProvidedKey.Value().Algorithm.ToString(); + } + + auto response + = _detail::BlobClient::GetLayout(*m_pipeline, m_blobUrl, protocolLayerOptions, context); + BlobLayoutPagedResponse pagedResponse; + std::map endpoints; + for (const auto& endpoint : response.Value.Endpoints) + { + if (endpoint.Index < 0 || endpoint.Value.empty() + || !endpoints.emplace(endpoint.Index, endpoint.Value).second) + { + throw Azure::Core::RequestFailedException("Invalid blob layout endpoint."); + } + } + for (const auto& range : response.Value.Ranges) + { + const auto endpoint = endpoints.find(range.EndpointIndex); + if (range.Start < 0 || range.End < range.Start || endpoint == endpoints.end()) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + const auto rangeLength + = static_cast(range.End) - static_cast(range.Start) + 1; + if (rangeLength > static_cast((std::numeric_limits::max)())) + { + throw Azure::Core::RequestFailedException("Invalid blob layout range."); + } + Models::BlobLayoutRange layoutRange; + layoutRange.Range.Offset = range.Start; + layoutRange.Range.Length = static_cast(rangeLength); + layoutRange.Endpoint = endpoint->second; + pagedResponse.Layout.Ranges.push_back(std::move(layoutRange)); + } + pagedResponse.Layout.Properties = std::move(response.Value.Properties); + if (!pagedResponse.Layout.Properties.ETag.HasValue()) + { + pagedResponse.Layout.Properties.ETag = options.AccessConditions.IfMatch; + } + CompleteBlobPropertiesFromResponse(pagedResponse.Layout.Properties, *response.RawResponse); + + pagedResponse.m_blobClient = std::make_shared(*this); + pagedResponse.m_operationOptions = options; + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = response.Value.NextMarker; + pagedResponse.RawResponse = std::move(response.RawResponse); + return pagedResponse; + } + Azure::Response BlobClient::DownloadTo( uint8_t* buffer, size_t bufferSize, @@ -367,6 +583,8 @@ namespace Azure { namespace Storage { namespace Blobs { DownloadBlobOptions firstChunkOptions; firstChunkOptions.Range = options.Range; + // Do not synthesize a range when the caller did not provide one. A ranged GET fails with 416 + // for an empty blob; an unbounded response is still consumed only up to firstChunkLength. if (firstChunkOptions.Range.HasValue()) { firstChunkOptions.Range.Value().Length = firstChunkLength; @@ -391,6 +609,8 @@ namespace Azure { namespace Storage { namespace Blobs { blobRangeSize = blobSize; } firstChunkLength = (std::min)(firstChunkLength, blobRangeSize); + const int64_t remainingOffset = firstChunkOffset + firstChunkLength; + const int64_t remainingSize = blobRangeSize - firstChunkLength; if (static_cast(blobRangeSize) > (std::numeric_limits::max)() || static_cast(blobRangeSize) > bufferSize) @@ -399,13 +619,20 @@ namespace Azure { namespace Storage { namespace Blobs { "Buffer is not big enough, blob range size is " + std::to_string(blobRangeSize) + "."); } - int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( - buffer, static_cast(firstChunkLength), context); - if (bytesRead != firstChunkLength) + std::unique_ptr<_detail::DataLocalityLayoutState> dataLocalityState; + if (remainingSize > 0 && IsLayoutAwareRoutingEnabled(options.LayoutAwareRouting) + && HasDataLocalityHint(firstChunk.Value.Details)) { - throw Azure::Core::RequestFailedException("Error when reading body stream."); + Core::Http::HttpRange remainingRange; + remainingRange.Offset = remainingOffset; + remainingRange.Length = remainingSize; + dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState->WaitForInitialLayout(); + if (!dataLocalityState->HasLayout()) + { + dataLocalityState.reset(); + } } - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadBlobToResult ret; @@ -417,9 +644,8 @@ namespace Azure { namespace Storage { namespace Blobs { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadBlobOptions chunkOptions; @@ -428,11 +654,14 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - auto chunk = Download(chunkOptions, context); + const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; + auto chunk = Download(chunkOptions, chunkContext); int64_t bytesRead = chunk.Value.BodyStream->ReadToCount( buffer + (offset - firstChunkOffset), static_cast(chunkOptions.Range.Value().Length.Value()), - context); + chunkContext); if (bytesRead != chunkOptions.Range.Value().Length.Value()) { throw Azure::Core::RequestFailedException("Error when reading body stream."); @@ -440,20 +669,52 @@ namespace Azure { namespace Storage { namespace Blobs { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); - ret.Value.TransactionalContentHash.Reset(); + lastChunkResponse = std::move(chunk); } }; - int64_t remainingOffset = firstChunkOffset + firstChunkLength; - int64_t remainingSize = blobRangeSize - firstChunkLength; + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( + buffer, static_cast(firstChunkLength), context); + if (bytesRead != firstChunkLength) + { + throw Azure::Core::RequestFailedException("Error when reading body stream."); + } + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); + if (lastChunkResponse.HasValue()) + { + ret.Value.TransactionalContentHash.Reset(); + } ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = blobRangeSize; return ret; @@ -476,6 +737,8 @@ namespace Azure { namespace Storage { namespace Blobs { DownloadBlobOptions firstChunkOptions; firstChunkOptions.Range = options.Range; + // Do not synthesize a range when the caller did not provide one. A ranged GET fails with 416 + // for an empty blob; an unbounded response is still consumed only up to firstChunkLength. if (firstChunkOptions.Range.HasValue()) { firstChunkOptions.Range.Value().Length = firstChunkLength; @@ -500,6 +763,23 @@ namespace Azure { namespace Storage { namespace Blobs { blobRangeSize = blobSize; } firstChunkLength = (std::min)(firstChunkLength, blobRangeSize); + const int64_t remainingOffset = firstChunkOffset + firstChunkLength; + const int64_t remainingSize = blobRangeSize - firstChunkLength; + + std::unique_ptr<_detail::DataLocalityLayoutState> dataLocalityState; + if (remainingSize > 0 && IsLayoutAwareRoutingEnabled(options.LayoutAwareRouting) + && HasDataLocalityHint(firstChunk.Value.Details)) + { + Core::Http::HttpRange remainingRange; + remainingRange.Offset = remainingOffset; + remainingRange.Length = remainingSize; + dataLocalityState = CreateDataLocalityLayoutState(*this, remainingRange, eTag, context); + dataLocalityState->WaitForInitialLayout(); + if (!dataLocalityState->HasLayout()) + { + dataLocalityState.reset(); + } + } auto bodyStreamToFile = [](Azure::Core::IO::BodyStream& stream, _internal::FileWriter& fileWriter, @@ -523,8 +803,6 @@ namespace Azure { namespace Storage { namespace Blobs { }; _internal::FileWriter fileWriter(fileName); - bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadBlobToResult ret; @@ -536,9 +814,8 @@ namespace Azure { namespace Storage { namespace Blobs { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadBlobOptions chunkOptions; @@ -547,30 +824,60 @@ namespace Azure { namespace Storage { namespace Blobs { chunkOptions.Range.Value().Length = length; chunkOptions.AccessConditions.IfMatch = eTag; chunkOptions.ValidationOptions = options.ValidationOptions; - auto chunk = Download(chunkOptions, context); + const auto chunkContext = dataLocalityState ? _internal::WithDataLocalityEndpoint( + context, dataLocalityState->GetEndpoint(offset, length)) + : context; + auto chunk = Download(chunkOptions, chunkContext); bodyStreamToFile( *(chunk.Value.BodyStream), fileWriter, offset - firstChunkOffset, chunkOptions.Range.Value().Length.Value(), - context); + chunkContext); if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); - ret.Value.TransactionalContentHash.Reset(); + lastChunkResponse = std::move(chunk); } }; - int64_t remainingOffset = firstChunkOffset + firstChunkLength; - int64_t remainingSize = blobRangeSize - firstChunkLength; + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); + firstChunk.Value.BodyStream.reset(); - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); + if (lastChunkResponse.HasValue()) + { + ret.Value.TransactionalContentHash.Reset(); + } ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = blobRangeSize; return ret; @@ -605,51 +912,7 @@ namespace Azure { namespace Storage { namespace Blobs { } auto response = _detail::BlobClient::GetProperties( *m_pipeline, m_blobUrl, protocolLayerOptions, _internal::WithReplicaStatus(context)); - if (response.Value.AccessTier.HasValue() && !response.Value.IsAccessTierInferred.HasValue()) - { - response.Value.IsAccessTierInferred = false; - } - if (response.Value.VersionId.HasValue() && !response.Value.IsCurrentVersion.HasValue()) - { - response.Value.IsCurrentVersion = false; - } - if (response.Value.CopyStatus.HasValue() && !response.Value.IsIncrementalCopy.HasValue()) - { - response.Value.IsIncrementalCopy = false; - } - if (response.Value.BlobType == Models::BlobType::AppendBlob - && !response.Value.IsSealed.HasValue()) - { - response.Value.IsSealed = false; - } - { - std::map> orPropertiesMap; - for (auto i = response.RawResponse->GetHeaders().lower_bound("x-ms-or-"); - i != response.RawResponse->GetHeaders().end() && i->first.substr(0, 8) == "x-ms-or-"; - ++i) - { - const std::string& header = i->first; - auto underscorePos = header.find('_', 8); - if (underscorePos == std::string::npos) - { - continue; - } - std::string policyId = std::string(header.begin() + 8, header.begin() + underscorePos); - std::string ruleId = header.substr(underscorePos + 1); - - Models::ObjectReplicationRule rule; - rule.RuleId = std::move(ruleId); - rule.ReplicationStatus = Models::ObjectReplicationStatus(i->second); - orPropertiesMap[policyId].emplace_back(std::move(rule)); - } - for (auto& property : orPropertiesMap) - { - Models::ObjectReplicationPolicy policy; - policy.PolicyId = property.first; - policy.Rules = std::move(property.second); - response.Value.ObjectReplicationSourceProperties.emplace_back(std::move(policy)); - } - } + CompleteBlobPropertiesFromResponse(response.Value, *response.RawResponse); return response; } diff --git a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp index d555c2bd1c..4816aaf256 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_container_client.cpp @@ -8,6 +8,7 @@ #include "azure/storage/blobs/block_blob_client.hpp" #include "azure/storage/blobs/page_blob_client.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include @@ -426,12 +427,22 @@ namespace Azure { namespace Storage { namespace Blobs { } else if (columnName == "ImmutabilityPolicyUntilDate") { - result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy.Value().ExpiresOn + auto& policy = result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy; + if (!policy.HasValue()) + { + policy = Models::BlobImmutabilityPolicy(); + } + policy.Value().ExpiresOn = std::chrono::system_clock::time_point(std::chrono::seconds(parsedIntValue)); } else if (columnName == "ImmutabilityPolicyMode") { - result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy.Value().PolicyMode + auto& policy = result.Items[batchRowStartOffset + r].Details.ImmutabilityPolicy; + if (!policy.HasValue()) + { + policy = Models::BlobImmutabilityPolicy(); + } + policy.Value().PolicyMode = Models::BlobImmutabilityPolicyMode(std::move(parsedStringValue)); } else if (columnName == "VersionId") @@ -1267,6 +1278,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyAuthPolicy); m_pipeline = std::make_shared( @@ -1285,8 +1297,8 @@ namespace Azure { namespace Storage { namespace Blobs { m_blobContainerUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); std::unique_ptr tokenAuthPolicy; + Azure::Core::Credentials::TokenRequestContext tokenContext; { - Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) @@ -1310,7 +1322,11 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; - pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy); + pipelineOptions.AddDataLocalityPolicy = true; + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + blobContainerUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); @@ -1345,6 +1361,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_blobContainerUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); @@ -1484,7 +1501,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto expectResponseFormat = options.ResponseFormat; if (expectResponseFormat == StorageResponseFormat::Auto) { - expectResponseFormat = StorageResponseFormat::Xml; + expectResponseFormat = StorageResponseFormat::Arrow; } if (expectResponseFormat == StorageResponseFormat::Arrow) { @@ -1505,12 +1522,14 @@ namespace Azure { namespace Storage { namespace Blobs { { actualResponseFormat = StorageResponseFormat::Arrow; } - (void)actualResponseFormat; - ListBlobsPagedResponse pagedResponse; pagedResponse.ServiceEndpoint = std::move(response.Value.ServiceEndpoint); pagedResponse.BlobContainerName = std::move(response.Value.BlobContainerName); pagedResponse.Prefix = std::move(response.Value.Prefix); + if (actualResponseFormat == StorageResponseFormat::Arrow && options.Prefix.HasValue()) + { + pagedResponse.Prefix = options.Prefix.Value(); + } for (auto& i : response.Value.Items) { pagedResponse.Blobs.push_back(BlobItemConversion(i)); @@ -1541,7 +1560,7 @@ namespace Azure { namespace Storage { namespace Blobs { auto expectResponseFormat = options.ResponseFormat; if (expectResponseFormat == StorageResponseFormat::Auto) { - expectResponseFormat = StorageResponseFormat::Xml; + expectResponseFormat = StorageResponseFormat::Arrow; } if (expectResponseFormat == StorageResponseFormat::Arrow) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_options.cpp b/sdk/storage/azure-storage-blobs/src/blob_options.cpp index 13143f6139..4a75133774 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_options.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_options.cpp @@ -3,8 +3,25 @@ #include "azure/storage/blobs/blob_options.hpp" +#include "private/token_credential_session_provider.hpp" + namespace Azure { namespace Storage { namespace Blobs { + SessionProvider::SessionProvider() = default; + + SessionProvider::~SessionProvider() = default; + + ContainerSessionProvider::ContainerSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options) + { + m_provider = std::make_shared<_detail::TokenCredentialSessionProvider>( + serviceUrl, std::move(credential), options); + } + + ContainerSessionProvider::~ContainerSessionProvider() = default; + const BlobAudience BlobAudience::DefaultAudience(_internal::StorageDefaultAudience); BlobQueryInputTextOptions BlobQueryInputTextOptions::CreateCsvTextOptions( diff --git a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp index 0a1c08457b..3f7279681d 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_responses.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_responses.cpp @@ -10,6 +10,13 @@ namespace Azure { namespace Storage { namespace Blobs { + void BlobLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + m_operationOptions.ContinuationToken = NextPageToken; + m_operationOptions.AccessConditions.IfMatch = Layout.Properties.ETag; + *this = m_blobClient->GetLayout(m_operationOptions, context); + } + std::unique_ptr StartBlobCopyOperation::PollInternal( const Azure::Core::Context&) { diff --git a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp index a7e4fd2068..58aae744f9 100644 --- a/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp @@ -5,6 +5,7 @@ #include "azure/storage/blobs/blob_batch.hpp" #include "private/package_version.hpp" +#include "private/session_authentication_policy.hpp" #include #include @@ -67,6 +68,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; pipelineOptions.SharedKeyAuthPolicy = std::move(sharedKeyPolicy); m_pipeline = std::make_shared( @@ -85,8 +87,8 @@ namespace Azure { namespace Storage { namespace Blobs { m_serviceUrl.GetHost(), options.SecondaryHostForRetryReads)); perRetryPolicies.emplace_back(std::make_unique<_internal::StoragePerRetryPolicy>()); std::unique_ptr tokenAuthPolicy; + Azure::Core::Credentials::TokenRequestContext tokenContext; { - Azure::Core::Credentials::TokenRequestContext tokenContext; tokenContext.Scopes.emplace_back( options.Audience.HasValue() ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) @@ -110,7 +112,11 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; - pipelineOptions.TokenAuthPolicy = std::move(tokenAuthPolicy); + pipelineOptions.AddDataLocalityPolicy = true; + auto authPolicies = _detail::CreateTokenAuthenticationPolicies( + serviceUrl, credential, tokenContext, options.EnableTenantDiscovery, options); + pipelineOptions.TokenAuthPolicy = std::move(authPolicies.TokenAuthPolicy); + pipelineOptions.SharedKeyAuthPolicy = std::move(authPolicies.FinalAuthPolicy); m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); @@ -145,6 +151,7 @@ namespace Azure { namespace Storage { namespace Blobs { pipelineOptions.PrimaryHost = m_serviceUrl.GetHost(); pipelineOptions.SecondaryHost = options.SecondaryHostForRetryReads; pipelineOptions.ApiVersion = options.ApiVersion; + pipelineOptions.AddDataLocalityPolicy = true; m_pipeline = std::make_shared( _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp new file mode 100644 index 0000000000..698bcf0b22 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.cpp @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "data_locality.hpp" + +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + std::string GetIdealDataLocalityEndpoint( + int64_t offset, + int64_t length, + const DataLocalityLayout& layout) + { + if (offset < 0 || length <= 0) + { + return std::string(); + } + + const int64_t requestEnd = length - 1 > (std::numeric_limits::max)() - offset + ? (std::numeric_limits::max)() + : offset + (length - 1); + auto range = std::lower_bound( + layout.Ranges.begin(), + layout.Ranges.end(), + offset, + [](const Models::BlobLayoutRange& value, int64_t target) { + return value.Range.Offset <= target + && value.Range.Length.Value() <= target - value.Range.Offset; + }); + std::unordered_map endpointOverlaps; + for (; range != layout.Ranges.end() && range->Range.Offset <= requestEnd; ++range) + { + const int64_t rangeEnd = range->Range.Offset + (range->Range.Length.Value() - 1); + const int64_t overlapStart = (std::max)(offset, range->Range.Offset); + const int64_t overlapEnd = (std::min)(requestEnd, rangeEnd); + if (overlapStart > overlapEnd) + { + continue; + } + auto endpoint = endpointOverlaps.emplace(range->Endpoint, 0); + endpoint.first->second += static_cast(overlapEnd - overlapStart) + 1; + } + + std::string idealEndpoint; + uint64_t largestOverlap = 0; + for (const auto& endpoint : endpointOverlaps) + { + if (endpoint.second > largestOverlap) + { + idealEndpoint = endpoint.first; + largestOverlap = endpoint.second; + } + } + return idealEndpoint; + } + + DataLocalityLayoutState::DataLocalityLayoutState(std::function refresh) + : m_refresh(std::move(refresh)), m_lastRefresh(std::chrono::steady_clock::now()), + m_lastRefreshAttempt(m_lastRefresh), m_initialRefresh(true) + { + StartRefresh(); + } + + std::string DataLocalityLayoutState::GetEndpoint(int64_t offset, int64_t length) + { + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lock(m_mutex); + if (m_refreshFuture.valid() + && m_refreshFuture.wait_for(std::chrono::seconds(0)) == std::future_status::ready) + { + m_refreshFuture.get(); + } + const auto age = now - m_lastRefresh; + if (age > DataLocalityLayoutValidity && !m_refreshFuture.valid() + && now - m_lastRefreshAttempt > DataLocalityLayoutValidity) + { + StartRefresh(); + } + if (age > DataLocalityLayoutValidity + DataLocalityLayoutExtendedExpiry) + { + return std::string(); + } + return GetIdealDataLocalityEndpoint(offset, length, m_layout); + } + + void DataLocalityLayoutState::WaitForInitialLayout() + { + WaitForRefreshTask(); + + std::exception_ptr initialRefreshException; + { + std::lock_guard lock(m_mutex); + if (m_initialRefresh) + { + m_initialRefresh = false; + initialRefreshException = std::move(m_initialRefreshException); + } + } + if (initialRefreshException) + { + std::rethrow_exception(initialRefreshException); + } + } + + void DataLocalityLayoutState::WaitForRefreshTask() + { + std::future refreshFuture; + { + std::lock_guard lock(m_mutex); + if (m_refreshFuture.valid()) + { + refreshFuture = std::move(m_refreshFuture); + } + } + if (refreshFuture.valid()) + { + refreshFuture.get(); + } + } + + void DataLocalityLayoutState::StartRefresh() + { + m_lastRefreshAttempt = std::chrono::steady_clock::now(); + try + { + m_refreshFuture = std::async(std::launch::async, [this]() { + DataLocalityLayout refreshedLayout; + bool refreshed = false; + std::exception_ptr refreshException; + try + { + refreshedLayout = m_refresh(); + refreshed = !refreshedLayout.Ranges.empty(); + } + catch (...) + { + refreshException = std::current_exception(); + } + + std::lock_guard lock(m_mutex); + if (refreshed) + { + m_layout = std::move(refreshedLayout); + m_lastRefresh = std::chrono::steady_clock::now(); + } + else if (m_initialRefresh) + { + m_initialRefreshException = std::move(refreshException); + } + }); + } + catch (...) + { + // If a refresh task cannot be created, continue with stale or primary routing. + } + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp new file mode 100644 index 0000000000..4871ca9df6 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/data_locality.hpp @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + constexpr std::chrono::minutes DataLocalityLayoutValidity{5}; + constexpr std::chrono::minutes DataLocalityLayoutExtendedExpiry{5}; + + struct DataLocalityLayout final + { + std::vector Ranges; + int64_t BlobSize = 0; + Azure::ETag ETag; + }; + + std::string GetIdealDataLocalityEndpoint( + int64_t offset, + int64_t length, + const DataLocalityLayout& layout); + + class DataLocalityLayoutState final { + public: + explicit DataLocalityLayoutState(std::function refresh); + + ~DataLocalityLayoutState() { WaitForRefreshTask(); } + + DataLocalityLayoutState(const DataLocalityLayoutState&) = delete; + DataLocalityLayoutState& operator=(const DataLocalityLayoutState&) = delete; + + std::string GetEndpoint(int64_t offset, int64_t length); + + void WaitForInitialLayout(); + + bool HasLayout() const + { + std::lock_guard lock(m_mutex); + return !m_layout.Ranges.empty(); + } + + int64_t GetBlobSize() const + { + std::lock_guard lock(m_mutex); + return m_layout.BlobSize; + } + + Azure::ETag GetETag() const + { + std::lock_guard lock(m_mutex); + return m_layout.ETag; + } + + private: + void WaitForRefreshTask(); + + void StartRefresh(); + + DataLocalityLayout m_layout; + std::function m_refresh; + std::chrono::steady_clock::time_point m_lastRefresh; + std::chrono::steady_clock::time_point m_lastRefreshAttempt; + mutable std::mutex m_mutex; + std::future m_refreshFuture; + std::exception_ptr m_initialRefreshException; + bool m_initialRefresh = false; + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp new file mode 100644 index 0000000000..eae34eb64e --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.cpp @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "session_authentication_policy.hpp" + +#include "token_credential_session_provider.hpp" + +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + namespace { + const Azure::Core::Context::Key SessionContextKey; + } + + class SessionSigningPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + explicit SessionSigningPolicy(std::string accountName) : m_accountName(std::move(accountName)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + SessionTokenInfo session; + if (context.TryGetValue(SessionContextKey, session)) + { + request.RemoveHeader(Azure::Storage::_internal::HttpHeaderAuthorization); + request.SetHeader( + Azure::Storage::_internal::HttpHeaderAuthorization, + "Session " + session.Token + ":" + + Azure::Storage::_internal::SharedKeyPolicy::GetSignature( + request, m_accountName, session.Key)); + } + return nextPolicy.Send(request, context); + } + + private: + std::string m_accountName; + }; + + class SessionAuthenticationPolicy final + : public Azure::Storage::_internal::StorageBearerTokenAuthenticationPolicy { + public: + SessionAuthenticationPolicy( + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + std::shared_ptr provider) + : StorageBearerTokenAuthenticationPolicy( + std::move(credential), + std::move(tokenRequestContext), + enableTenantDiscovery), + m_provider(provider->m_provider) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + private: + std::shared_ptr m_provider; + + std::unique_ptr AuthorizeAndSendRequest( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, + Azure::Core::Context const& context) const override + { + if (!m_provider->IsRequestEligible(request)) + { + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + + auto session = m_provider->GetSession(request, context); + if (!session.HasValue()) + { + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + + auto sessionContext = context.WithValue(SessionContextKey, SessionTokenInfo(session.Value())); + auto response = nextPolicy.Send(request, sessionContext); + if (response->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Unauthorized) + { + return response; + } + + m_provider->InvalidateSession(request, session.Value()); + request.RemoveHeader(Azure::Storage::_internal::HttpHeaderAuthorization); + return StorageBearerTokenAuthenticationPolicy::AuthorizeAndSendRequest( + request, nextPolicy, context); + } + }; + + TokenAuthenticationPolicies CreateTokenAuthenticationPolicies( + const std::string& clientUrl, + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + const BlobClientOptions& options) + { + const auto createBearerPolicy = [&credential, + &tokenRequestContext, + enableTenantDiscovery]() -> TokenAuthenticationPolicies { + TokenAuthenticationPolicies policies; + policies.TokenAuthPolicy + = std::make_unique( + std::move(credential), std::move(tokenRequestContext), enableTenantDiscovery); + return policies; + }; + + if (options.Session.Mode != SessionMode::Enabled) + { + return createBearerPolicy(); + } + + auto provider = options.Session.Provider; + auto accountName = options.Session.AccountName; + if (!provider || accountName.empty()) + { + auto urlParts = Azure::Storage::_internal::ParseStorageUrl(Azure::Core::Url(clientUrl)); + if (!urlParts.HasValue()) + { + throw std::invalid_argument( + "SessionOptions.AccountName and the Blob service URL could not be determined from the " + "client URL. Specify both SessionOptions.AccountName and SessionOptions.Provider for " + "custom endpoints."); + } + if (!accountName.empty() && accountName != urlParts.Value().AccountName) + { + throw std::invalid_argument( + "SessionOptions.AccountName does not match the account name in the client URL."); + } + accountName = std::move(urlParts.Value().AccountName); + if (!provider) + { + provider = std::make_shared( + urlParts.Value().ServiceUrl, credential, options); + } + } + TokenAuthenticationPolicies policies; + policies.TokenAuthPolicy = std::make_unique( + std::move(credential), + std::move(tokenRequestContext), + enableTenantDiscovery, + std::move(provider)); + policies.FinalAuthPolicy = std::make_unique(std::move(accountName)); + return policies; + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp new file mode 100644 index 0000000000..2662ba1116 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/session_authentication_policy.hpp @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + struct TokenAuthenticationPolicies final + { + std::unique_ptr TokenAuthPolicy; + std::unique_ptr FinalAuthPolicy; + }; + + TokenAuthenticationPolicies CreateTokenAuthenticationPolicies( + const std::string& clientUrl, + std::shared_ptr credential, + Azure::Core::Credentials::TokenRequestContext tokenRequestContext, + bool enableTenantDiscovery, + const BlobClientOptions& options); + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp new file mode 100644 index 0000000000..b43994692b --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.cpp @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "token_credential_session_provider.hpp" + +#include "package_version.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { +constexpr auto SessionRefreshLeadTime = std::chrono::seconds(30); +constexpr auto SessionRefreshSuppression = std::chrono::seconds(30); +constexpr auto SessionFailureCooldown = std::chrono::minutes(5); + +} // namespace + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + std::string TokenCredentialSessionProvider::GetContainerUrl( + const Azure::Core::Http::Request& request) const + { + auto url = request.GetUrl(); + const auto& path = url.GetPath(); + const auto& servicePath = m_serviceUrl.GetPath(); + size_t containerStart = 0; + if (!servicePath.empty() && path.compare(0, servicePath.size(), servicePath) == 0) + { + containerStart = servicePath.size(); + } + const auto getFirstPathSegment = [](const std::string& value, size_t start) { + while (start < value.size() && value[start] == '/') + { + ++start; + } + const auto end = value.find('/', start); + return value.substr(start, end - start); + }; + const auto container = getFirstPathSegment(path, containerStart); + if (container.empty()) + { + return {}; + } + auto containerUrl = m_serviceUrl; + containerUrl.AppendPath(container); + return containerUrl.GetAbsoluteUrl(); + } + + std::shared_ptr + TokenCredentialSessionProvider::GetCacheEntry(const std::string& containerUrl) + { + std::lock_guard lock(m_cacheMutex); + auto& entry = m_cache[containerUrl]; + if (!entry) + { + entry = std::make_shared(); + } + return entry; + } + + SessionTokenInfo TokenCredentialSessionProvider::CreateSession( + const std::string& containerUrl, + const Azure::Core::Context& context) + { + _internal::XmlWriter writer; + writer.Write(_internal::XmlNode{_internal::XmlNodeType::StartTag, "CreateSessionRequest"}); + writer.Write( + _internal::XmlNode{_internal::XmlNodeType::StartTag, "AuthenticationType", "HMAC"}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::EndTag}); + writer.Write(_internal::XmlNode{_internal::XmlNodeType::End}); + auto xmlBody = writer.GetDocument(); + + Azure::Core::IO::MemoryBodyStream body( + reinterpret_cast(xmlBody.data()), xmlBody.size()); + Azure::Core::Url url(containerUrl); + url.AppendQueryParameter("restype", "container"); + url.AppendQueryParameter("comp", "session"); + Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Post, url, &body); + request.SetHeader(_internal::HttpHeaderContentType, "application/xml; charset=UTF-8"); + request.SetHeader(_internal::HttpHeaderContentLength, std::to_string(body.Length())); + + auto response = m_pipeline->Send(request, context); + if (response->GetStatusCode() != Azure::Core::Http::HttpStatusCode::Created) + { + throw StorageException::CreateFromResponse(std::move(response)); + } + + SessionTokenInfo session; + const auto& responseBody = response->GetBody(); + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + std::string currentTag; + bool expirationFound = false; + while (true) + { + const auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + if (node.Type == _internal::XmlNodeType::StartTag) + { + currentTag = node.Name; + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (currentTag == "SessionToken") + { + session.Token = node.Value; + } + else if (currentTag == "SessionKey") + { + session.Key = node.Value; + } + else if (currentTag == "Expiration") + { + session.ExpiresOn = Azure::DateTime::Parse( + node.Value, + node.Value.find(',') != std::string::npos ? Azure::DateTime::DateFormat::Rfc1123 + : Azure::DateTime::DateFormat::Rfc3339); + expirationFound = true; + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + currentTag.clear(); + } + } + if (session.Token.empty() || session.Key.empty() || !expirationFound) + { + throw std::runtime_error("Create Session response was missing required session fields."); + } + return session; + } + + TokenCredentialSessionProvider::TokenCredentialSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options) + { + if (!options.Session.AccountName.empty()) + { + m_serviceUrl = Azure::Core::Url(serviceUrl); + } + else + { + auto urlParts = Azure::Storage::_internal::ParseStorageUrl(Azure::Core::Url(serviceUrl)); + if (!urlParts.HasValue()) + { + throw std::invalid_argument( + "SessionOptions.AccountName could not be determined from the provided service URL."); + } + m_serviceUrl = Azure::Core::Url(urlParts.Value().ServiceUrl); + } + _internal::BuildStoragePipelineOptions pipelineOptions; + pipelineOptions.PackageName = _internal::BlobServicePackageName; + pipelineOptions.PackageVersion = PackageVersion::ToString(); + pipelineOptions.ApiVersion = Blobs::_detail::ApiVersion; + Azure::Core::Credentials::TokenRequestContext tokenContext; + tokenContext.Scopes.emplace_back( + options.Audience.HasValue() + ? _internal::GetDefaultScopeForAudience(options.Audience.Value().ToString()) + : _internal::StorageScope); + pipelineOptions.TokenAuthPolicy + = std::make_unique<_internal::StorageBearerTokenAuthenticationPolicy>( + std::move(credential), tokenContext, options.EnableTenantDiscovery); + m_pipeline = std::make_shared( + _internal::BuildHttpPipelinePolicies(options, std::move(pipelineOptions))); + } + + bool TokenCredentialSessionProvider::IsRequestEligible( + const Azure::Core::Http::Request& request) const + { + auto requestHost = request.GetUrl().GetHost(); + const auto headers = request.GetHeaders(); + const auto hostHeader = headers.find("Host"); + if (hostHeader != headers.end()) + { + requestHost + = Azure::Core::Url(request.GetUrl().GetScheme() + "://" + hostHeader->second).GetHost(); + } + if (requestHost != m_serviceUrl.GetHost()) + { + return false; + } + if (request.GetMethod() != Azure::Core::Http::HttpMethod::Get) + { + return false; + } + const auto query = request.GetUrl().GetQueryParameters(); + if (query.count("comp") != 0 || query.count("restype") != 0) + { + return false; + } + if (request.GetHeaders().count("x-ms-structured-body") != 0) + { + return false; + } + const auto containerUrl = GetContainerUrl(request); + return !containerUrl.empty() + && request.GetUrl().GetPath().size() > Azure::Core::Url(containerUrl).GetPath().size(); + } + + Azure::Nullable TokenCredentialSessionProvider::GetSession( + const Azure::Core::Http::Request& request, + const Azure::Core::Context& context) + { + const auto containerUrl = GetContainerUrl(request); + if (containerUrl.empty()) + { + return {}; + } + auto entry = GetCacheEntry(containerUrl); + auto now = Azure::DateTime::clock::now(); + { + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() + && now < entry->Current.Value().ExpiresOn - SessionRefreshLeadTime) + { + return entry->Current; + } + if (entry->CooldownUntil > now) + { + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + return {}; + } + if (entry->LastRefreshStarted + SessionRefreshSuppression > now) + { + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + return {}; + } + entry->LastRefreshStarted = now; + } + + try + { + auto session = CreateSession(containerUrl, context); + std::lock_guard lock(entry->Mutex); + if (!entry->Current.HasValue() || entry->Current.Value().ExpiresOn < session.ExpiresOn) + { + entry->Current = session; + } + return session; + } + catch (const StorageException& e) + { + const auto statusCode = static_cast(e.StatusCode); + const bool softFailure = statusCode >= 500 || statusCode == 403 + || (statusCode == 400 && e.ErrorCode == "FeatureNotEnabled"); + now = Azure::DateTime::clock::now(); + std::lock_guard lock(entry->Mutex); + if (softFailure) + { + const auto cooldownUntil = now + SessionFailureCooldown; + if (entry->CooldownUntil < cooldownUntil) + { + entry->CooldownUntil = cooldownUntil; + } + } + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + if (!softFailure) + { + throw; + } + return {}; + } + catch (const std::exception&) + { + now = Azure::DateTime::clock::now(); + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() && now < entry->Current.Value().ExpiresOn) + { + return entry->Current; + } + throw; + } + } + + void TokenCredentialSessionProvider::InvalidateSession( + const Azure::Core::Http::Request& request, + const SessionTokenInfo& session) + { + const auto containerUrl = GetContainerUrl(request); + if (containerUrl.empty()) + { + return; + } + auto entry = GetCacheEntry(containerUrl); + std::lock_guard lock(entry->Mutex); + if (entry->Current.HasValue() && entry->Current.Value().Token == session.Token) + { + entry->Current.Reset(); + } + } + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp new file mode 100644 index 0000000000..9292fa13cc --- /dev/null +++ b/sdk/storage/azure-storage-blobs/src/private/token_credential_session_provider.hpp @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace Azure { namespace Core { namespace Http { namespace _internal { + class HttpPipeline; +}}}} // namespace Azure::Core::Http::_internal + +namespace Azure { namespace Storage { namespace Blobs { namespace _detail { + + struct SessionTokenInfo final + { + std::string Token; + std::string Key; + Azure::DateTime ExpiresOn; + }; + + class TokenCredentialSessionProvider final { + public: + TokenCredentialSessionProvider( + const std::string& serviceUrl, + std::shared_ptr credential, + const BlobClientOptions& options); + + Azure::Nullable GetSession( + const Azure::Core::Http::Request& request, + const Azure::Core::Context& context); + + void InvalidateSession( + const Azure::Core::Http::Request& request, + const SessionTokenInfo& session); + + bool IsRequestEligible(const Azure::Core::Http::Request& request) const; + + private: + struct CacheEntry + { + std::mutex Mutex; + Azure::Nullable Current; + Azure::DateTime CooldownUntil; + Azure::DateTime LastRefreshStarted; + }; + + Azure::Core::Url m_serviceUrl; + std::shared_ptr m_pipeline; + std::mutex m_cacheMutex; + std::map> m_cache; + + std::string GetContainerUrl(const Azure::Core::Http::Request& request) const; + std::shared_ptr GetCacheEntry(const std::string& containerUrl); + SessionTokenInfo CreateSession( + const std::string& containerUrl, + const Azure::Core::Context& context); + }; + +}}}} // namespace Azure::Storage::Blobs::_detail diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 904db11b9d..e35fafdc98 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -166,6 +166,7 @@ namespace Azure { namespace Storage { namespace Blobs { const BlobType BlobType::BlockBlob("BlockBlob"); const BlobType BlobType::PageBlob("PageBlob"); const BlobType BlobType::AppendBlob("AppendBlob"); + const DownloadHint DownloadHint::Layout("layout"); const DeleteSnapshotsOption DeleteSnapshotsOption::IncludeSnapshots("include"); const DeleteSnapshotsOption DeleteSnapshotsOption::OnlySnapshots("only"); const ScheduleBlobExpiryOriginType ScheduleBlobExpiryOriginType::NeverExpire("NeverExpire"); @@ -411,6 +412,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobServiceProperties response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -707,6 +709,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::ServiceStatistics response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -806,6 +809,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::ListBlobContainersResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1098,6 +1102,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::UserDelegationKey response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1287,6 +1292,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::FindBlobsByTagsResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1604,6 +1610,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::BlobContainerAccessPolicy response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -1897,6 +1904,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::FindBlobsByTagsResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -2653,6 +2661,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.Details.SmartAccessTier = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-smart-access-tier")); } + if (pRawResponse->GetHeaders().count("x-ms-download-hint") != 0) + { + response.Details.DownloadHint + = Models::DownloadHint(pRawResponse->GetHeaders().at("x-ms-download-hint")); + } if (httpStatusCode == Core::Http::HttpStatusCode::Ok) { if (pRawResponse->GetHeaders().count("Content-MD5") != 0) @@ -3058,6 +3071,443 @@ namespace Azure { namespace Storage { namespace Blobs { Models::DeleteBlobResult response; return Response(std::move(response), std::move(pRawResponse)); } + Response BlobClient::GetLayout( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetBlobLayoutOptions& options, + const Core::Context& context) + { + auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); + request.GetUrl().AppendQueryParameter("comp", "layout"); + if (options.Snapshot.HasValue() && !options.Snapshot.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "snapshot", _internal::UrlEncodeQueryParameter(options.Snapshot.Value())); + } + if (options.VersionId.HasValue() && !options.VersionId.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "versionid", _internal::UrlEncodeQueryParameter(options.VersionId.Value())); + } + if (options.Marker.HasValue() && !options.Marker.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "marker", _internal::UrlEncodeQueryParameter(options.Marker.Value())); + } + if (options.MaxResults.HasValue()) + { + request.GetUrl().AppendQueryParameter( + "maxresults", std::to_string(options.MaxResults.Value())); + } + if (options.Range.HasValue() && !options.Range.Value().empty()) + { + request.SetHeader("x-ms-range", options.Range.Value()); + } + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.IfTags.HasValue() && !options.IfTags.Value().empty()) + { + request.SetHeader("x-ms-if-tags", options.IfTags.Value()); + } + if (options.IfModifiedSince.HasValue()) + { + request.SetHeader( + "If-Modified-Since", + options.IfModifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfUnmodifiedSince.HasValue()) + { + request.SetHeader( + "If-Unmodified-Since", + options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); + } + if (options.IfMatch.HasValue() && !options.IfMatch.ToString().empty()) + { + request.SetHeader("If-Match", options.IfMatch.ToString()); + } + if (options.IfNoneMatch.HasValue() && !options.IfNoneMatch.ToString().empty()) + { + request.SetHeader("If-None-Match", options.IfNoneMatch.ToString()); + } + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } + request.SetHeader("x-ms-version", "2026-10-06"); + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (!(httpStatusCode == Core::Http::HttpStatusCode::Ok + || httpStatusCode == Core::Http::HttpStatusCode::NoContent)) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::_detail::BlobLayout response; + { + const auto& responseBody = pRawResponse->GetBody(); + if (!responseBody.empty()) + { + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + enum class XmlTagEnum + { + kUnknown, + kBlobLayout, + kRanges, + kRange, + kEndpoints, + kEndpoint, + kMarker, + kNextMarker, + kMaxResults, + }; + const std::unordered_map XmlTagEnumMap{ + {"BlobLayout", XmlTagEnum::kBlobLayout}, + {"Ranges", XmlTagEnum::kRanges}, + {"Range", XmlTagEnum::kRange}, + {"Endpoints", XmlTagEnum::kEndpoints}, + {"Endpoint", XmlTagEnum::kEndpoint}, + {"Marker", XmlTagEnum::kMarker}, + {"NextMarker", XmlTagEnum::kNextMarker}, + {"MaxResults", XmlTagEnum::kMaxResults}, + }; + std::vector xmlPath; + Models::_detail::BlobLayoutRangesItem vectorElement1; + Models::_detail::BlobLayoutEndpointsItem vectorElement2; + while (true) + { + auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + else if (node.Type == _internal::XmlNodeType::StartTag) + { + auto ite = XmlTagEnumMap.find(node.Name); + xmlPath.push_back(ite == XmlTagEnumMap.end() ? XmlTagEnum::kUnknown : ite->second); + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kMarker) + { + response.Marker = node.Value; + } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kNextMarker) + { + response.NextMarker = node.Value; + } + else if ( + xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kMaxResults) + { + response.MaxResults = std::stoi(node.Value); + } + } + else if (node.Type == _internal::XmlNodeType::Attribute) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "Start") + { + vectorElement1.Start = std::stoll(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "End") + { + vectorElement1.End = std::stoll(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange + && node.Name == "EndpointIndex") + { + vectorElement1.EndpointIndex = std::stoi(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint + && node.Name == "Index") + { + vectorElement2.Index = std::stoi(node.Value); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint + && node.Name == "Value") + { + vectorElement2.Value = node.Value; + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kRanges && xmlPath[2] == XmlTagEnum::kRange) + { + response.Ranges.push_back(std::move(vectorElement1)); + vectorElement1 = Models::_detail::BlobLayoutRangesItem(); + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kBlobLayout + && xmlPath[1] == XmlTagEnum::kEndpoints && xmlPath[2] == XmlTagEnum::kEndpoint) + { + response.Endpoints.push_back(std::move(vectorElement2)); + vectorElement2 = Models::_detail::BlobLayoutEndpointsItem(); + } + xmlPath.pop_back(); + } + } + } + } + if (pRawResponse->GetHeaders().count("Last-Modified") != 0) + { + response.Properties.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-length") != 0) + { + response.Properties.BlobSize + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-content-length")); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-type") != 0) + { + response.Properties.HttpHeaders.ContentType + = pRawResponse->GetHeaders().at("x-ms-blob-content-type"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-encoding") != 0) + { + response.Properties.HttpHeaders.ContentEncoding + = pRawResponse->GetHeaders().at("x-ms-blob-content-encoding"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-content-md5") != 0) + { + response.Properties.HttpHeaders.ContentHash.Value + = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-blob-content-md5")); + response.Properties.HttpHeaders.ContentHash.Algorithm = HashAlgorithm::Md5; + } + if (pRawResponse->GetHeaders().count("x-ms-creation-time") != 0) + { + response.Properties.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-creation-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + for (auto i = pRawResponse->GetHeaders().lower_bound("x-ms-meta-"); + i != pRawResponse->GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; + ++i) + { + response.Properties.Metadata.emplace(i->first.substr(10), i->second); + } + if (pRawResponse->GetHeaders().count("x-ms-or-policy-id") != 0) + { + response.Properties.ObjectReplicationDestinationPolicyId + = pRawResponse->GetHeaders().at("x-ms-or-policy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-type") != 0) + { + response.Properties.BlobType + = Models::BlobType(pRawResponse->GetHeaders().at("x-ms-blob-type")); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-completion-time") != 0) + { + response.Properties.CopyCompletedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-copy-completion-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status-description") != 0) + { + response.Properties.CopyStatusDescription + = pRawResponse->GetHeaders().at("x-ms-copy-status-description"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-id") != 0) + { + response.Properties.CopyId = pRawResponse->GetHeaders().at("x-ms-copy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-progress") != 0) + { + response.Properties.CopyProgress = pRawResponse->GetHeaders().at("x-ms-copy-progress"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-source") != 0) + { + response.Properties.CopySource = pRawResponse->GetHeaders().at("x-ms-copy-source"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status") != 0) + { + response.Properties.CopyStatus + = Models::CopyStatus(pRawResponse->GetHeaders().at("x-ms-copy-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-incremental-copy") != 0) + { + response.Properties.IsIncrementalCopy + = pRawResponse->GetHeaders().at("x-ms-incremental-copy") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-destination-snapshot") != 0) + { + response.Properties.IncrementalCopyDestinationSnapshot + = pRawResponse->GetHeaders().at("x-ms-copy-destination-snapshot"); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.Properties.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0) + { + response.Properties.LeaseState + = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0) + { + response.Properties.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + } + if (pRawResponse->GetHeaders().count("ETag") != 0) + { + response.Properties.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + } + if (pRawResponse->GetHeaders().count("Content-Disposition") != 0) + { + response.Properties.HttpHeaders.ContentDisposition + = pRawResponse->GetHeaders().at("Content-Disposition"); + } + if (pRawResponse->GetHeaders().count("Content-Language") != 0) + { + response.Properties.HttpHeaders.ContentLanguage + = pRawResponse->GetHeaders().at("Content-Language"); + } + if (pRawResponse->GetHeaders().count("Cache-Control") != 0) + { + response.Properties.HttpHeaders.CacheControl + = pRawResponse->GetHeaders().at("Cache-Control"); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.Properties.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-committed-block-count") != 0) + { + response.Properties.CommittedBlockCount + = std::stoi(pRawResponse->GetHeaders().at("x-ms-blob-committed-block-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-server-encrypted") != 0) + { + response.Properties.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) + { + response.Properties.EncryptionKeySha256 = Core::Convert::Base64Decode( + pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256")); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-scope") != 0) + { + response.Properties.EncryptionScope + = pRawResponse->GetHeaders().at("x-ms-encryption-scope"); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier") != 0) + { + response.Properties.AccessTier + = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-access-tier")); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier-inferred") != 0) + { + response.Properties.IsAccessTierInferred + = pRawResponse->GetHeaders().at("x-ms-access-tier-inferred") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-smart-access-tier") != 0) + { + response.Properties.SmartAccessTier + = Models::AccessTier(pRawResponse->GetHeaders().at("x-ms-smart-access-tier")); + } + if (pRawResponse->GetHeaders().count("x-ms-archive-status") != 0) + { + response.Properties.ArchiveStatus + = Models::ArchiveStatus(pRawResponse->GetHeaders().at("x-ms-archive-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-access-tier-change-time") != 0) + { + response.Properties.AccessTierChangedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-access-tier-change-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-version-id") != 0) + { + response.Properties.VersionId = pRawResponse->GetHeaders().at("x-ms-version-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-is-current-version") != 0) + { + response.Properties.IsCurrentVersion + = pRawResponse->GetHeaders().at("x-ms-is-current-version") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-tag-count") != 0) + { + response.Properties.TagCount = std::stoi(pRawResponse->GetHeaders().at("x-ms-tag-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-expiry-time") != 0) + { + response.Properties.ExpiresOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-expiry-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-blob-sealed") != 0) + { + response.Properties.IsSealed + = pRawResponse->GetHeaders().at("x-ms-blob-sealed") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-rehydrate-priority") != 0) + { + response.Properties.RehydratePriority + = Models::RehydratePriority(pRawResponse->GetHeaders().at("x-ms-rehydrate-priority")); + } + if (pRawResponse->GetHeaders().count("x-ms-last-access-time") != 0) + { + response.Properties.LastAccessedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-last-access-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-immutability-policy-until-date") != 0) + { + if (!response.Properties.ImmutabilityPolicy.HasValue()) + { + response.Properties.ImmutabilityPolicy = Models::BlobImmutabilityPolicy(); + } + response.Properties.ImmutabilityPolicy.Value().ExpiresOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-immutability-policy-until-date"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-immutability-policy-mode") != 0) + { + if (!response.Properties.ImmutabilityPolicy.HasValue()) + { + response.Properties.ImmutabilityPolicy = Models::BlobImmutabilityPolicy(); + } + response.Properties.ImmutabilityPolicy.Value().PolicyMode + = Models::BlobImmutabilityPolicyMode( + pRawResponse->GetHeaders().at("x-ms-immutability-policy-mode")); + } + if (pRawResponse->GetHeaders().count("x-ms-legal-hold") != 0) + { + response.Properties.HasLegalHold + = pRawResponse->GetHeaders().at("x-ms-legal-hold") == std::string("true"); + } + return Response(std::move(response), std::move(pRawResponse)); + } Response BlobClient::Undelete( Core::Http::_internal::HttpPipeline& pipeline, const Core::Url& url, @@ -4534,6 +4984,7 @@ namespace Azure { namespace Storage { namespace Blobs { std::map response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -5352,6 +5803,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::GetPageRangesResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -5530,6 +5982,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::_detail::GetPageRangesDiffResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum @@ -7193,6 +7646,7 @@ namespace Azure { namespace Storage { namespace Blobs { Models::GetBlockListResult response; { const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( reinterpret_cast(responseBody.data()), responseBody.size()); enum class XmlTagEnum diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 9bfa6f1355..2cd7b49b10 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-blobs namespace: Azure::Storage::Blobs output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-12-06/blob.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/refs/heads/feature/storage/stg105base-swagger/specification/storage/data-plane/Microsoft.BlobStorage/stable/2027-03-07/blob.json ``` ## ModelFour Options @@ -26,6 +26,43 @@ modelerfour: See the [AutoRest samples](https://github.com/Azure/autorest/tree/master/Samples/3b-custom-transformations) for more about how we're customizing things. +### Generate Get Blob Layout responses + +```yaml +directive: + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}?comp=layout"].get + transform: > + $.responses["204"].schema = $.responses["200"].schema; + for (const statusCode of ["200", "204"]) { + for (const headerName in $.responses[statusCode].headers) { + if (!["x-ms-meta", "x-ms-or"].includes(headerName)) { + $.responses[statusCode].headers[headerName]["x-nullable"] = true; + } + } + } + - from: swagger-document + where: $.definitions.BlobLayout + transform: > + const rangeItem = $.properties.Ranges.properties.Range.items; + const endpointItem = $.properties.Endpoints.properties.Endpoint.items; + $["x-ms-sealed"] = false; + $["x-namespace"] = "_detail"; + rangeItem["x-namespace"] = "_detail"; + endpointItem["x-namespace"] = "_detail"; + $.properties.Ranges = { + "type": "array", + "xml": {"name": "Ranges", "wrapped": true}, + "items": rangeItem + }; + $.properties.Endpoints = { + "type": "array", + "xml": {"name": "Endpoints", "wrapped": true}, + "items": endpointItem + }; + $.properties.Properties = {"$ref": "#/definitions/BlobProperties"}; +``` + ### Fix Generator Warnings ```yaml @@ -75,6 +112,7 @@ directive: delete $["/{filesystem}/{path}?FileRename"]; delete $["/{containerName}?restype=container&comp=list&flat&arrow"]; delete $["/{containerName}?restype=container&comp=list&hierarchy&arrow"]; + delete $["/{containerName}?restype=container&comp=session"]; for (const operation in $) { for (const verb in $[operation]) { @@ -967,6 +1005,12 @@ directive: - from: swagger-document where: $.definitions transform: > + $.DownloadHint = { + "type": "string", + "description": "Indicates the download hint for the blob.", + "enum": ["layout"], + "x-ms-enum": {"name": "DownloadHint", "modelAsString": true} + }; $.DownloadBlobDetails = { "type": "object", "required": ["ETag", "LastModified", "CreatedOn", "HttpHeaders", "IsServerEncrypted", "HasLegalHold"], @@ -1003,7 +1047,8 @@ directive: "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, "IsAccessTierInferred": {"type": "boolean", "x-nullable": true}, "AccessTierChangedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, - "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true} + "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "DownloadHint": {"$ref": "#/definitions/DownloadHint", "x-nullable": true} } }; $.DownloadBlobResult = { @@ -1072,6 +1117,8 @@ directive: $[status_code].headers["x-ms-access-tier-inferred"]["x-ms-client-path"] = "Details.IsAccessTierInferred"; $[status_code].headers["x-ms-access-tier-change-time"]["x-ms-client-path"] = "Details.AccessTierChangedOn"; $[status_code].headers["x-ms-smart-access-tier"]["x-ms-client-path"] = "Details.SmartAccessTier"; + $[status_code].headers["x-ms-download-hint"]["x-ms-client-path"] = "Details.DownloadHint"; + $[status_code].headers["x-ms-download-hint"]["x-nullable"] = true; delete $[status_code].headers["Accept-Ranges"]; delete $[status_code].headers["Content-Length"]; delete $[status_code].headers["Content-Range"]; @@ -1098,18 +1145,126 @@ directive: transform: > $.push({"$ref": "#/parameters/UserPrincipalName"}); - from: swagger-document - where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses + where: $.definitions transform: > - $["200"].schema = { + $.BlobProperties = { "type": "object", "x-ms-client-name": "BlobProperties", "x-ms-sealed": false, + "required": [ + "LastModified", + "CreatedOn", + "Metadata", + "BlobType", + "BlobSize", + "ETag", + "IsServerEncrypted", + "HttpHeaders", + "HasLegalHold" + ], "properties": { "ObjectReplicationSourceProperties": {"type": "array", "items": {"$ref": "#/definitions/ObjectReplicationPolicy"}, "x-ms-xml": {"name": ""}}, "ImmutabilityPolicy": {"$ref": "#/definitions/BlobImmutabilityPolicy", "x-nullable": true, "x-ms-xml": {"name": ""}}, - "HttpHeaders": {"$ref": "#/definitions/BlobHttpHeaders", "x-ms-xml": {"name": ""}} + "HttpHeaders": {"$ref": "#/definitions/BlobHttpHeaders", "x-ms-xml": {"name": ""}}, + "LastModified": {"type": "string", "format": "date-time-rfc1123"}, + "CreatedOn": {"type": "string", "format": "date-time-rfc1123"}, + "Metadata": {"$ref": "#/definitions/Metadata"}, + "ObjectReplicationDestinationPolicyId": {"type": "string", "x-nullable": true}, + "BlobType": { + "type": "string", + "enum": ["BlockBlob", "PageBlob", "AppendBlob"], + "x-ms-enum": {"name": "BlobType", "modelAsString": false} + }, + "CopyCompletedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "CopyStatusDescription": {"type": "string", "x-nullable": true}, + "CopyId": {"type": "string", "x-nullable": true}, + "CopyProgress": {"type": "string", "x-nullable": true}, + "CopySource": {"type": "string", "x-nullable": true}, + "CopyStatus": {"$ref": "#/definitions/CopyStatus", "x-nullable": true}, + "IsIncrementalCopy": {"type": "boolean", "x-nullable": true}, + "IncrementalCopyDestinationSnapshot": {"type": "string", "x-nullable": true}, + "LeaseDuration": { + "type": "string", + "enum": ["infinite", "fixed"], + "x-ms-enum": {"name": "LeaseDurationType", "modelAsString": false}, + "x-nullable": true + }, + "LeaseState": {"$ref": "#/definitions/LeaseState", "x-nullable": true}, + "LeaseStatus": {"$ref": "#/definitions/LeaseStatus", "x-nullable": true}, + "BlobSize": {"type": "integer", "format": "int64"}, + "ETag": {"type": "string", "format": "etag"}, + "SequenceNumber": {"type": "integer", "format": "int64", "x-nullable": true}, + "CommittedBlockCount": {"type": "integer", "format": "int32", "x-nullable": true}, + "IsServerEncrypted": {"type": "boolean"}, + "EncryptionKeySha256": {"type": "string", "format": "byte", "x-nullable": true}, + "EncryptionScope": {"type": "string", "x-nullable": true}, + "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "IsAccessTierInferred": {"type": "boolean", "x-nullable": true}, + "ArchiveStatus": {"$ref": "#/definitions/ArchiveStatus", "x-nullable": true}, + "AccessTierChangedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "SmartAccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true}, + "VersionId": {"type": "string", "x-nullable": true}, + "IsCurrentVersion": {"type": "boolean", "x-nullable": true}, + "TagCount": {"type": "integer", "format": "int32", "x-nullable": true}, + "ExpiresOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "IsSealed": {"type": "boolean", "x-nullable": true}, + "RehydratePriority": {"$ref": "#/definitions/RehydratePriority", "x-nullable": true}, + "LastAccessedOn": {"type": "string", "format": "date-time-rfc1123", "x-nullable": true}, + "HasLegalHold": {"type": "boolean", "x-ms-client-default": false} } }; + for (const propertyName in $.BlobProperties.properties) { + $.BlobProperties.properties[propertyName]["x-ms-xml"] = {"name": ""}; + } + $.BlobProperties.description = "Properties of a blob."; + const descriptions = { + "ObjectReplicationSourceProperties": "Object replication policies associated with the source blob.", + "ImmutabilityPolicy": "Immutability policy associated with the blob.", + "HttpHeaders": "Standard HTTP properties supported by the blob.", + "LastModified": "The date and time when the blob was last modified.", + "CreatedOn": "The date and time when the blob was created.", + "Metadata": "A set of name-value pairs associated with the blob.", + "ObjectReplicationDestinationPolicyId": "The object replication policy identifier for the destination blob.", + "BlobType": "The type of the blob.", + "CopyCompletedOn": "The completion time of the last attempted copy operation.", + "CopyStatusDescription": "A description of the last copy operation status.", + "CopyId": "The identifier of the last copy operation.", + "CopyProgress": "The progress of the last copy operation.", + "CopySource": "The source of the last copy operation.", + "CopyStatus": "The status of the last copy operation.", + "IsIncrementalCopy": "Indicates whether the blob is an incremental copy.", + "IncrementalCopyDestinationSnapshot": "The destination snapshot of the last incremental copy.", + "LeaseDuration": "The duration of the blob lease.", + "LeaseState": "The current lease state of the blob.", + "LeaseStatus": "The current lease status of the blob.", + "BlobSize": "The size of the blob in bytes.", + "ETag": "The entity tag of the blob.", + "SequenceNumber": "The current sequence number of a page blob.", + "CommittedBlockCount": "The number of committed blocks in an append blob.", + "IsServerEncrypted": "Indicates whether the blob is encrypted by the service.", + "EncryptionKeySha256": "The SHA-256 hash of the customer-provided encryption key.", + "EncryptionScope": "The encryption scope used to encrypt the blob.", + "AccessTier": "The access tier of the blob.", + "IsAccessTierInferred": "Indicates whether the access tier was inferred.", + "ArchiveStatus": "The archive rehydration status of the blob.", + "AccessTierChangedOn": "The time when the access tier was last changed.", + "SmartAccessTier": "The underlying access tier of a smart-tier blob.", + "VersionId": "The version identifier of the blob.", + "IsCurrentVersion": "Indicates whether this is the current blob version.", + "TagCount": "The number of tags associated with the blob.", + "ExpiresOn": "The time when the blob will expire.", + "IsSealed": "Indicates whether the append blob is sealed.", + "RehydratePriority": "The priority of an archive rehydration operation.", + "LastAccessedOn": "The time when the blob was last read or written.", + "HasLegalHold": "Indicates whether the blob has a legal hold." + }; + for (const propertyName in descriptions) { + $.BlobProperties.properties[propertyName].description = descriptions[propertyName]; + } + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses["200"] + transform: > + $.schema = {"$ref": "#/definitions/BlobProperties"}; - from: swagger-document where: $["x-ms-paths"]["/{containerName}/{blob}"].head.responses["200"].headers transform: > @@ -1182,6 +1337,77 @@ directive: $["x-ms-blob-type"]["x-ms-client-default"] = ""; delete $["Accept-Ranges"]; delete $["x-ms-or"]; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + const propertiesHeaders = $["/{containerName}/{blob}"].head.responses["200"].headers; + const layoutResponses = $["/{containerName}/{blob}?comp=layout"].get.responses; + const mappings = { + "Last-Modified": ["Last-Modified", "LastModified"], + "x-ms-creation-time": ["x-ms-creation-time", "CreatedOn"], + "x-ms-meta": ["x-ms-meta", "Metadata"], + "x-ms-or-policy-id": ["x-ms-or-policy-id", "ObjectReplicationDestinationPolicyId"], + "x-ms-blob-type": ["x-ms-blob-type", "BlobType"], + "x-ms-copy-completion-time": ["x-ms-copy-completion-time", "CopyCompletedOn"], + "x-ms-copy-status-description": ["x-ms-copy-status-description", "CopyStatusDescription"], + "x-ms-copy-id": ["x-ms-copy-id", "CopyId"], + "x-ms-copy-progress": ["x-ms-copy-progress", "CopyProgress"], + "x-ms-copy-source": ["x-ms-copy-source", "CopySource"], + "x-ms-copy-status": ["x-ms-copy-status", "CopyStatus"], + "x-ms-incremental-copy": ["x-ms-incremental-copy", "IsIncrementalCopy"], + "x-ms-copy-destination-snapshot": ["x-ms-copy-destination-snapshot", "IncrementalCopyDestinationSnapshot"], + "x-ms-lease-duration": ["x-ms-lease-duration", "LeaseDuration"], + "x-ms-lease-state": ["x-ms-lease-state", "LeaseState"], + "x-ms-lease-status": ["x-ms-lease-status", "LeaseStatus"], + "x-ms-blob-content-length": ["Content-Length", "BlobSize"], + "ETag": ["ETag", "ETag"], + "x-ms-blob-content-type": ["Content-Type", "HttpHeaders.ContentType"], + "x-ms-blob-content-md5": ["Content-MD5", "HttpHeaders.ContentHash"], + "x-ms-blob-content-encoding": ["Content-Encoding", "HttpHeaders.ContentEncoding"], + "Content-Disposition": ["Content-Disposition", "HttpHeaders.ContentDisposition"], + "Content-Language": ["Content-Language", "HttpHeaders.ContentLanguage"], + "Cache-Control": ["Cache-Control", "HttpHeaders.CacheControl"], + "x-ms-blob-sequence-number": ["x-ms-blob-sequence-number", "SequenceNumber"], + "x-ms-blob-committed-block-count": ["x-ms-blob-committed-block-count", "CommittedBlockCount"], + "x-ms-server-encrypted": ["x-ms-server-encrypted", "IsServerEncrypted"], + "x-ms-encryption-key-sha256": ["x-ms-encryption-key-sha256", "EncryptionKeySha256"], + "x-ms-encryption-scope": ["x-ms-encryption-scope", "EncryptionScope"], + "x-ms-access-tier": ["x-ms-access-tier", "AccessTier"], + "x-ms-smart-access-tier": ["x-ms-smart-access-tier", "SmartAccessTier"], + "x-ms-access-tier-inferred": ["x-ms-access-tier-inferred", "IsAccessTierInferred"], + "x-ms-archive-status": ["x-ms-archive-status", "ArchiveStatus"], + "x-ms-access-tier-change-time": ["x-ms-access-tier-change-time", "AccessTierChangedOn"], + "x-ms-version-id": ["x-ms-version-id", "VersionId"], + "x-ms-is-current-version": ["x-ms-is-current-version", "IsCurrentVersion"], + "x-ms-tag-count": ["x-ms-tag-count", "TagCount"], + "x-ms-expiry-time": ["x-ms-expiry-time", "ExpiresOn"], + "x-ms-blob-sealed": ["x-ms-blob-sealed", "IsSealed"], + "x-ms-rehydrate-priority": ["x-ms-rehydrate-priority", "RehydratePriority"], + "x-ms-last-access-time": ["x-ms-last-access-time", "LastAccessedOn"], + "x-ms-immutability-policy-mode": ["x-ms-immutability-policy-mode", "ImmutabilityPolicy.PolicyMode"], + "x-ms-immutability-policy-until-date": ["x-ms-immutability-policy-until-date", "ImmutabilityPolicy.ExpiresOn"], + "x-ms-legal-hold": ["x-ms-legal-hold", "HasLegalHold"] + }; + for (const statusCode of ["200", "204"]) { + const headers = layoutResponses[statusCode].headers; + for (const headerName in headers) { + if (!mappings[headerName]) { + delete headers[headerName]; + } + } + for (const headerName in mappings) { + if (!headers[headerName]) { + continue; + } + const [sourceHeaderName, propertyPath] = mappings[headerName]; + headers[headerName] = JSON.parse(JSON.stringify(propertiesHeaders[sourceHeaderName])); + delete headers[headerName]["x-ms-client-name"]; + headers[headerName]["x-ms-client-path"] = "Properties." + propertyPath; + if (headerName !== "x-ms-meta") { + headers[headerName]["x-nullable"] = true; + } + } + } ``` ### DeleteBlob diff --git a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt index ad37169887..46474f1672 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-blobs/test/ut/CMakeLists.txt @@ -29,9 +29,11 @@ add_executable ( block_blob_client_test.cpp block_blob_client_test.hpp connection_reuse_test.cpp + data_locality_test.cpp macro_guard.cpp page_blob_client_test.cpp page_blob_client_test.hpp + session_authentication_test.cpp simplified_header_test.cpp storage_retry_policy_test.cpp storage_timeout_test.cpp diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp index 707c3100b1..6eb1d6869d 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_container_client_test.cpp @@ -103,6 +103,7 @@ namespace Azure { namespace Storage { namespace Test { Blobs::Models::ListBlobsIncludeFlags include) { Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = blobName; options.Include = include; for (auto page = containerClient.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) @@ -239,7 +240,7 @@ namespace Azure { namespace Storage { namespace Test { } for (const auto format : - {Blobs::StorageResponseFormat::Arrow, Blobs::StorageResponseFormat::Xml}) + {Blobs::StorageResponseFormat::Auto, Blobs::StorageResponseFormat::Xml}) { Azure::Storage::Blobs::ListBlobsOptions options; @@ -256,11 +257,18 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_FALSE(pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderDate).empty()); EXPECT_FALSE( pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderXMsVersion).empty()); + const auto& contentType + = pageResult.RawResponse->GetHeaders().at(_internal::HttpHeaderContentType); if (format == Blobs::StorageResponseFormat::Xml) { + EXPECT_NE(contentType.find(_internal::ContentTypeXml), std::string::npos); EXPECT_FALSE(pageResult.ServiceEndpoint.empty()); EXPECT_EQ(pageResult.BlobContainerName, m_containerName); } + else + { + EXPECT_NE(contentType.find(_internal::ContentTypeApacheArrowStream), std::string::npos); + } for (const auto& blob : pageResult.Blobs) { EXPECT_FALSE(blob.Name.empty()); @@ -308,6 +316,7 @@ namespace Azure { namespace Storage { namespace Test { for (auto pageResult = containerClient.ListBlobs(options); pageResult.HasPage(); pageResult.MoveToNextPage()) { + EXPECT_EQ(pageResult.Prefix, prefix1); for (const auto& blob : pageResult.Blobs) { listBlobs.insert(blob.Name); @@ -343,6 +352,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.PageSizeHint = 3; options.StartFrom = startFrom; std::set listBlobs; @@ -458,6 +468,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = prefix; std::set items; for (auto pageResult = containerClient.ListBlobsByHierarchy(delimiter, options); @@ -530,6 +541,7 @@ namespace Azure { namespace Storage { namespace Test { } Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = prefix; options.StartFrom = startFrom; std::set items; @@ -591,6 +603,7 @@ namespace Azure { namespace Storage { namespace Test { blobClient.AppendBlock(contentStream); Azure::Storage::Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = blobName; options.Include = Blobs::Models::ListBlobsIncludeFlags::Snapshots | Blobs::Models::ListBlobsIncludeFlags::Versions @@ -1666,6 +1679,7 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(blobUrl, containerClient.GetUrl() + "/" + _internal::UrlEncodePath(blobName)); Blobs::Models::BlobItem blobItem; Blobs::ListBlobsOptions options; + options.ResponseFormat = Blobs::StorageResponseFormat::Xml; options.Prefix = "aaaaa"; for (auto page = containerClient.ListBlobs(options); page.HasPage(); page.MoveToNextPage()) { @@ -1679,7 +1693,9 @@ namespace Azure { namespace Storage { namespace Test { } EXPECT_EQ(blobItem.Name, blobName); bool found = false; - for (auto page = containerClient.ListBlobsByHierarchy("/"); page.HasPage(); + Blobs::ListBlobsOptions hierarchyOptions; + hierarchyOptions.ResponseFormat = Blobs::StorageResponseFormat::Xml; + for (auto page = containerClient.ListBlobsByHierarchy("/", hierarchyOptions); page.HasPage(); page.MoveToNextPage()) { for (auto& p : page.BlobPrefixes) diff --git a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp index 3f4be369a2..58cf27dd3c 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/block_blob_client_test.cpp @@ -32,6 +32,18 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { + TEST(BlockBlobClientOptionsTest, DefaultTransferSizes) + { + Blobs::DownloadBlobToOptions downloadOptions; + EXPECT_EQ( + downloadOptions.TransferOptions.InitialChunkSize, + downloadOptions.TransferOptions.ChunkSize); + EXPECT_EQ(downloadOptions.TransferOptions.InitialChunkSize, 4_MB); + + Blobs::UploadBlockBlobFromOptions uploadOptions; + EXPECT_EQ(uploadOptions.TransferOptions.SingleUploadThreshold, 4_MB); + } + void BlockBlobClientTest::SetUp() { BlobContainerClientTest::SetUp(); diff --git a/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp new file mode 100644 index 0000000000..2e5e7c15ed --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/data_locality_test.cpp @@ -0,0 +1,858 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "../../src/private/data_locality.hpp" +#include "test/ut/test_base.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + struct LocalityRequest final + { + std::string Host; + std::string HostHeader; + std::string IfMatch; + std::string Range; + bool IsLayout = false; + }; + + struct LocalityState final + { + std::mutex Mutex; + std::vector Requests; + Core::Http::HttpStatusCode LayoutStatus = Core::Http::HttpStatusCode::Ok; + bool PaginateLayout = false; + bool ReturnDownloadHint = true; + std::string DownloadHint = "layout"; + }; + + class LocalityTransportPolicy final : public Core::Http::Policies::HttpPolicy { + public: + LocalityTransportPolicy( + std::shared_ptr state, + std::shared_ptr data) + : m_state(std::move(state)), m_data(std::move(data)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + Core::Context const& context) const override + { + (void)nextPolicy; + (void)context; + + const auto query = request.GetUrl().GetQueryParameters(); + const bool isLayout = query.find("comp") != query.end() && query.at("comp") == "layout"; + const auto& headers = request.GetHeaders(); + const auto hostHeader = headers.find("host"); + const auto ifMatchHeader = headers.find("if-match"); + const auto rangeHeader = headers.find("x-ms-range"); + + { + std::lock_guard lock(m_state->Mutex); + m_state->Requests.push_back(LocalityRequest{ + request.GetUrl().GetHost(), + hostHeader == headers.end() ? std::string() : hostHeader->second, + ifMatchHeader == headers.end() ? std::string() : ifMatchHeader->second, + rangeHeader == headers.end() ? std::string() : rangeHeader->second, + isLayout}); + } + + if (isLayout) + { + auto setLayoutPropertyHeaders = [&](Core::Http::RawResponse& response) { + response.SetHeader("etag", "\"locality-etag\""); + response.SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response.SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response.SetHeader("x-ms-meta-layout", "metadata"); + response.SetHeader("x-ms-blob-content-length", std::to_string(m_data->size())); + response.SetHeader("x-ms-blob-content-type", "application/octet-stream"); + response.SetHeader("x-ms-blob-content-encoding", "identity"); + response.SetHeader("x-ms-blob-type", "BlockBlob"); + response.SetHeader("x-ms-server-encrypted", "true"); + }; + if (m_state->LayoutStatus == Core::Http::HttpStatusCode::NoContent) + { + auto response = std::make_unique( + 1, 1, Core::Http::HttpStatusCode::NoContent, "No Content"); + setLayoutPropertyHeaders(*response); + return response; + } + if (m_state->LayoutStatus != Core::Http::HttpStatusCode::Ok) + { + const std::string errorBody + = "MockLayoutErrorlayout error"; + auto response = std::make_unique( + 1, 1, m_state->LayoutStatus, "layout error"); + response->SetBody(std::vector(errorBody.begin(), errorBody.end())); + response->SetHeader("content-type", "application/xml"); + response->SetHeader("x-ms-error-code", "MockLayoutError"); + return response; + } + + const bool secondPage + = query.find("marker") != query.end() && query.at("marker") == "page2"; + std::string ranges; + std::string nextMarker; + if (!m_state->PaginateLayout) + { + ranges = "" + "" + "" + ""; + } + else if (!secondPage) + { + ranges = "" + ""; + nextMarker = "page2"; + } + else + { + ranges = "" + ""; + } + const std::string layout = "" + ranges + + "" + "" + "" + "" + + nextMarker + "4"; + auto response = std::make_unique( + 1, 1, Core::Http::HttpStatusCode::Ok, "OK"); + response->SetBody(std::vector(layout.begin(), layout.end())); + response->SetHeader("content-type", "application/xml"); + setLayoutPropertyHeaders(*response); + return response; + } + + int64_t offset = 0; + int64_t length = static_cast(m_data->size()); + if (rangeHeader != headers.end()) + { + const auto separator = rangeHeader->second.find('-'); + offset = std::stoll(rangeHeader->second.substr(6, separator - 6)); + const int64_t end = std::stoll(rangeHeader->second.substr(separator + 1)); + length = (std::min)(end - offset + 1, static_cast(m_data->size()) - offset); + } + + auto response = std::make_unique( + 1, + 1, + rangeHeader == headers.end() ? Core::Http::HttpStatusCode::Ok + : Core::Http::HttpStatusCode::PartialContent, + "OK"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(m_data->data() + offset), + static_cast(length))); + response->SetHeader("content-length", std::to_string(length)); + if (rangeHeader != headers.end()) + { + response->SetHeader( + "content-range", + "bytes " + std::to_string(offset) + "-" + std::to_string(offset + length - 1) + "/" + + std::to_string(m_data->size())); + } + response->SetHeader("etag", "\"locality-etag\""); + response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response->SetHeader("x-ms-blob-content-length", std::to_string(m_data->size())); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + if (m_state->ReturnDownloadHint) + { + response->SetHeader("x-ms-download-hint", m_state->DownloadHint); + } + return response; + } + + private: + std::shared_ptr m_state; + std::shared_ptr m_data; + }; + + Blobs::BlobClient CreateLocalityClient( + const std::shared_ptr& state, + const std::shared_ptr& data) + { + Blobs::BlobClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state, data)); + return Blobs::BlobClient("https://primary.test/container/blob", options); + } + + Blobs::BlobClient CreateLocalityClientFromService( + const std::shared_ptr& state, + const std::shared_ptr& data) + { + Blobs::BlobClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state, data)); + return Blobs::BlobServiceClient("https://primary.test", options) + .GetBlobContainerClient("container") + .GetBlobClient("blob"); + } + + Blobs::DownloadBlobToOptions CreateDownloadOptions() + { + Blobs::DownloadBlobToOptions options; + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Enabled; + options.TransferOptions.InitialChunkSize = 128 * 1024; + options.TransferOptions.ChunkSize = 64 * 1024; + options.TransferOptions.Concurrency = 4; + return options; + } + + Blobs::Models::BlobLayoutRange MakeLayoutRange( + int64_t offset, + int64_t length, + std::string endpoint) + { + Blobs::Models::BlobLayoutRange range; + range.Range.Offset = offset; + range.Range.Length = length; + range.Endpoint = std::move(endpoint); + return range; + } + + Blobs::_detail::DataLocalityLayout CreateLayout(const std::string& endpoint) + { + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = {MakeLayoutRange(0, 1024 * 1024, endpoint)}; + return layout; + } + } // namespace + + class DataLocalityRecordedTest : public StorageTest { + }; + + TEST(DataLocalityTest, SelectsIdealEndpoint) + { + using Range = Blobs::Models::BlobLayoutRange; + struct TestCase final + { + const char* Name; + int64_t Offset; + int64_t Length; + std::vector Ranges; + std::string ExpectedEndpoint; + }; + + const auto maxOffset = (std::numeric_limits::max)(); + const std::vector testCases{ + {"NegativeOffset", -1, 1, {MakeLayoutRange(0, 10, "A")}, ""}, + {"ZeroLength", 0, 0, {MakeLayoutRange(0, 10, "A")}, ""}, + {"NegativeLength", 0, -1, {MakeLayoutRange(0, 10, "A")}, ""}, + {"EmptyLayout", 0, 1, {}, ""}, + {"BeforeLayout", 0, 5, {MakeLayoutRange(10, 10, "A")}, ""}, + {"AfterLayout", 20, 1, {MakeLayoutRange(10, 10, "A")}, ""}, + {"RequestStartsInGap", 5, 10, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"ExactRangeStart", 10, 1, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"ExactRangeEnd", 19, 1, {MakeLayoutRange(10, 10, "A")}, "A"}, + {"SkipsEarlierRanges", + 100, + 1, + {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(100, 10, "B")}, + "B"}, + {"LargestOverlap", + 200 * 1024, + 400 * 1024, + {MakeLayoutRange(0, 256 * 1024, "A"), + MakeLayoutRange(256 * 1024, 256 * 1024, "B"), + MakeLayoutRange(512 * 1024, 256 * 1024, "A")}, + "B"}, + {"AggregatesDisjointRanges", + 5, + 25, + {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(10, 10, "B"), MakeLayoutRange(20, 10, "A")}, + "A"}, + {"IgnoresLayoutGaps", + 0, + 30, + {MakeLayoutRange(0, 5, "A"), MakeLayoutRange(20, 10, "B")}, + "B"}, + {"RequestEndAtMax", maxOffset - 1, 2, {MakeLayoutRange(maxOffset - 9, 10, "A")}, "A"}, + {"SingleByteAtMax", maxOffset, 1, {MakeLayoutRange(maxOffset, 1, "A")}, "A"}, + {"RequestEndOverflow", maxOffset - 4, 10, {MakeLayoutRange(maxOffset - 9, 10, "A")}, "A"}, + }; + + for (const auto& testCase : testCases) + { + SCOPED_TRACE(testCase.Name); + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = testCase.Ranges; + EXPECT_EQ( + Blobs::_detail::GetIdealDataLocalityEndpoint(testCase.Offset, testCase.Length, layout), + testCase.ExpectedEndpoint); + } + } + + TEST(DataLocalityTest, SelectsEitherEndpointForEqualOverlap) + { + Blobs::_detail::DataLocalityLayout layout; + layout.Ranges = {MakeLayoutRange(0, 10, "A"), MakeLayoutRange(10, 10, "B")}; + + const auto endpoint = Blobs::_detail::GetIdealDataLocalityEndpoint(5, 10, layout); + EXPECT_TRUE(endpoint == "A" || endpoint == "B"); + } + + TEST(DataLocalityTest, FreshLayoutDoesNotRefresh) + { + std::atomic refreshCount{0}; + Blobs::_detail::DataLocalityLayoutState state([&]() { + ++refreshCount; + return CreateLayout("A"); + }); + + state.WaitForInitialLayout(); + EXPECT_EQ(refreshCount, 1); + EXPECT_EQ(state.GetEndpoint(0, 1), "A"); + EXPECT_EQ(refreshCount, 1); + } + + TEST(DataLocalityTest, GetLayoutNoContentReturnsBlobProperties) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; + auto data = std::make_shared(1024 * 1024, 'n'); + auto client = CreateLocalityClient(state, data); + + auto page = client.GetLayout(); + + EXPECT_TRUE(page.Layout.Ranges.empty()); + EXPECT_EQ(page.Layout.Properties.ETag.ToString(), "\"locality-etag\""); + EXPECT_EQ(page.Layout.Properties.BlobSize, static_cast(data->size())); + EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); + EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); + } + + TEST_F(DataLocalityRecordedTest, GetLayoutMultipleEndpoints_PLAYBACKONLY_) + { + auto clientOptions = InitStorageClientOptions(); + auto blobClient = Blobs::BlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), + "data-locality-test", + "data-locality-multi-endpoint.bin", + clientOptions); + + std::vector ranges; + std::set endpoints; + auto page = blobClient.GetLayout(); + EXPECT_EQ(page.Layout.Properties.BlobSize, 64 * 1024 * 1024); + EXPECT_EQ(page.Layout.Properties.BlobType, Blobs::Models::BlobType::BlockBlob); + EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); + EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted); + for (; page.HasPage(); page.MoveToNextPage()) + { + for (const auto& range : page.Layout.Ranges) + { + EXPECT_GE(range.Range.Offset, 0); + ASSERT_TRUE(range.Range.Length.HasValue()); + EXPECT_GT(range.Range.Length.Value(), 0); + EXPECT_FALSE(range.Endpoint.empty()); + ranges.push_back(range); + endpoints.insert(range.Endpoint); + } + } + + if (m_testContext.IsLiveMode() && endpoints.size() < 2) + { + GTEST_SKIP() << "The configured live account does not return a multi-endpoint layout."; + } + ASSERT_GT(ranges.size(), 1U); + EXPECT_GE(endpoints.size(), 2U); + + const auto differentEndpoint + = std::adjacent_find(ranges.begin(), ranges.end(), [](auto const& left, auto const& right) { + return left.Endpoint != right.Endpoint; + }); + ASSERT_NE(differentEndpoint, ranges.end()); + const auto nextRange = differentEndpoint + 1; + ASSERT_GE(nextRange->Range.Offset, 2); + + constexpr int64_t downloadLength = 5; + const int64_t downloadOffset = nextRange->Range.Offset - 2; + Blobs::DownloadBlobOptions baselineOptions; + baselineOptions.Range = Core::Http::HttpRange(); + baselineOptions.Range.Value().Offset = downloadOffset; + baselineOptions.Range.Value().Length = downloadLength; + auto downloadResponse = blobClient.Download(baselineOptions); + auto content = downloadResponse.Value.BodyStream->ReadToEnd(); + ASSERT_EQ(content.size(), static_cast(downloadLength)); + ASSERT_TRUE(downloadResponse.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ( + downloadResponse.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + + TEST_F(DataLocalityRecordedTest, GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_) + { + auto clientOptions = InitStorageClientOptions(); + auto client = Blobs::BlobClient::CreateFromConnectionString( + StandardStorageConnectionString(), + "data-locality-test", + "data-locality-multi-endpoint.bin", + clientOptions); + constexpr int64_t blobSize = 64 * 1024 * 1024; + for (bool bounded : {true, false}) + { + SCOPED_TRACE(bounded); + Blobs::GetBlobLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : blobSize - 16; + if (bounded) + { + options.Range.Value().Length = 5; + } + const auto offset = options.Range.Value().Offset; + bool coversOffset = false; + size_t rangeCount = 0; + for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) + { + EXPECT_EQ(page.Layout.Properties.BlobSize, blobSize); + EXPECT_GT(page.Layout.Properties.BlobSize, options.Range.Value().Length.ValueOr(16)); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + ASSERT_GT(range.Range.Length.Value(), 0); + ASSERT_GE(range.Range.Offset, 0); + ASSERT_LE(range.Range.Offset, blobSize); + ASSERT_LE(range.Range.Length.Value(), blobSize - range.Range.Offset); + EXPECT_FALSE(range.Endpoint.empty()); + coversOffset = coversOffset + || (range.Range.Offset <= offset + && offset - range.Range.Offset < range.Range.Length.Value()); + ++rangeCount; + } + } + EXPECT_GT(rangeCount, 0U); + EXPECT_TRUE(coversOffset); + } + } + + TEST(DataLocalityTest, RoutesDownloadChunks) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, '\0'); + for (size_t i = 0; i < data->size(); ++i) + { + (*data)[i] = static_cast(i % 127); + } + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + auto response = client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + + EXPECT_EQ(response.Value.BlobSize, static_cast(data->size())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 16U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Host, "primary.test"); + EXPECT_TRUE(state->Requests[0].Range.empty()); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_EQ(state->Requests[1].Range, "bytes=131072-1048575"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + for (auto request = state->Requests.begin() + 2; request != state->Requests.end(); ++request) + { + EXPECT_FALSE(request->IsLayout); + EXPECT_TRUE(request->Host == "locality0.test" || request->Host == "locality1.test"); + EXPECT_EQ(request->HostHeader, "primary.test"); + EXPECT_EQ(request->IfMatch, "\"locality-etag\""); + EXPECT_FALSE(request->Range.empty()); + const auto separator = request->Range.find('-'); + const auto offset = std::stoll(request->Range.substr(6, separator - 6)); + EXPECT_EQ( + request->Host, (offset / (256 * 1024)) % 2 == 0 ? "locality0.test" : "locality1.test"); + } + } + + TEST(DataLocalityTest, RoutesFileDownloadChunks) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'f'); + auto client = CreateLocalityClientFromService(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + + auto response = client.DownloadTo(fileName, CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, static_cast(data->size())); + + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_EQ(downloaded, *data); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 16U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Host, "primary.test"); + EXPECT_TRUE(state->Requests[0].Range.empty()); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (auto request = state->Requests.begin() + 2; request != state->Requests.end(); ++request) + { + EXPECT_TRUE(request->Host == "locality0.test" || request->Host == "locality1.test"); + EXPECT_EQ(request->HostHeader, "primary.test"); + EXPECT_EQ(request->IfMatch, "\"locality-etag\""); + } + } + + TEST(DataLocalityTest, CompletesSmallBufferDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 's'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + + TEST(DataLocalityTest, DownloadsEmptyBlobWithoutRange) + { + auto data = std::make_shared(); + + { + auto state = std::make_shared(); + auto client = CreateLocalityClient(state, data); + std::vector buffer; + + auto response = client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, 0); + EXPECT_EQ(response.Value.ContentRange.Length.Value(), 0); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + + { + auto state = std::make_shared(); + auto client = CreateLocalityClient(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + + auto response = client.DownloadTo(fileName, CreateDownloadOptions()); + EXPECT_EQ(response.Value.BlobSize, 0); + EXPECT_EQ(response.Value.ContentRange.Length.Value(), 0); + + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_TRUE(downloaded.empty()); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_TRUE(state->Requests.front().Range.empty()); + } + } + + TEST(DataLocalityTest, CompletesSingleRangeDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + constexpr int64_t offset = 200 * 1024; + constexpr int64_t length = 400 * 1024; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = length; + options.TransferOptions.InitialChunkSize = length; + + std::vector buffer(length); + client.DownloadTo(buffer.data(), buffer.size(), options); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin() + offset)); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Range, "bytes=204800-614399"); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + } + + TEST(DataLocalityTest, CompletesSmallFileDownloadBeforeLayout) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 'f'); + auto client = CreateLocalityClient(state, data); + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 0; + options.Range.Value().Length = static_cast(data->size()); + + client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + const std::string downloaded((std::istreambuf_iterator(file)), {}); + file.close(); + std::remove(fileName.c_str()); + EXPECT_EQ(downloaded, *data); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "primary.test"); + } + + TEST(DataLocalityTest, DoesNotFetchLayoutWithoutDownloadHint) + { + auto state = std::make_shared(); + state->ReturnDownloadHint = false; + auto data = std::make_shared(1024 * 1024, 'h'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (const auto& request : state->Requests) + { + EXPECT_FALSE(request.IsLayout); + EXPECT_EQ(request.Host, "primary.test"); + } + } + + TEST(DataLocalityTest, AutoEnablesLayoutAwareRouting) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'a'); + auto client = CreateLocalityClient(state, data); + auto options = CreateDownloadOptions(); + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Auto; + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), options); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_TRUE(state->Requests[1].IsLayout); + } + + TEST(DataLocalityTest, DefaultAndDisabledSkipLayoutAwareRouting) + { + for (bool useDefault : {true, false}) + { + for (bool toFile : {false, true}) + { + SCOPED_TRACE(useDefault); + SCOPED_TRACE(toFile); + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'd'); + auto client = CreateLocalityClient(state, data); + Blobs::DownloadBlobToOptions options; + EXPECT_EQ(options.LayoutAwareRouting, Blobs::LayoutAwareRouting::Disabled); + options.TransferOptions = CreateDownloadOptions().TransferOptions; + if (!useDefault) + { + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; + } + + std::vector content(data->size()); + if (toFile) + { + const std::string fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + content.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + file.close(); + std::remove(fileName.c_str()); + } + else + { + client.DownloadTo(content.data(), content.size(), options); + } + EXPECT_TRUE(std::equal(content.begin(), content.end(), data->begin(), data->end())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (const auto& request : state->Requests) + { + EXPECT_FALSE(request.IsLayout); + EXPECT_EQ(request.Host, "primary.test"); + } + } + } + } + + TEST(DataLocalityTest, AcceptsCaseInsensitiveDownloadHint) + { + auto state = std::make_shared(); + state->DownloadHint = "LAYOUT"; + auto data = std::make_shared(1024 * 1024, 'h'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + } + + TEST(DataLocalityTest, RoutesToEndpointWithLargestOverlap) + { + auto state = std::make_shared(); + auto data = std::make_shared(1024 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + constexpr int64_t offset = 200 * 1024; + constexpr int64_t length = 400 * 1024; + constexpr int64_t initialLength = 64 * 1024; + auto options = CreateDownloadOptions(); + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = offset; + options.Range.Value().Length = length; + options.TransferOptions.InitialChunkSize = initialLength; + options.TransferOptions.ChunkSize = length - initialLength; + + std::vector buffer(length); + client.DownloadTo(buffer.data(), buffer.size(), options); + EXPECT_TRUE(std::equal(buffer.begin(), buffer.end(), data->begin() + offset)); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 3U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_EQ(state->Requests[0].Range, "bytes=204800-270335"); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_EQ(state->Requests[1].Range, "bytes=270336-614399"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_FALSE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests[2].Range, "bytes=270336-614399"); + EXPECT_EQ(state->Requests[2].Host, "locality1.test"); + } + + TEST(DataLocalityTest, ReadsAllLayoutPages) + { + auto state = std::make_shared(); + state->PaginateLayout = true; + auto data = std::make_shared(1024 * 1024, 'p'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 17U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_TRUE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests[1].IfMatch, "\"locality-etag\""); + EXPECT_EQ(state->Requests[2].IfMatch, "\"locality-etag\""); + } + + TEST(DataLocalityTest, RoutesOneShotDownloadToLayoutEndpoint) + { + auto state = std::make_shared(); + auto data = std::make_shared(4 * 1024, 'o'); + auto client = CreateLocalityClient(state, data); + + Blobs::DownloadBlobOptions options; + options.LayoutEndpoint = "locality0.test:443"; + auto response = client.Download(options); + + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + + std::lock_guard lock(state->Mutex); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests.front().IsLayout); + EXPECT_EQ(state->Requests.front().Host, "locality0.test"); + EXPECT_EQ(state->Requests.front().HostHeader, "primary.test"); + } + + TEST(DataLocalityTest, FallsBackOnUnsupportedLayout) + { + for (const auto status : + {Core::Http::HttpStatusCode::BadRequest, Core::Http::HttpStatusCode::InternalServerError}) + { + auto state = std::make_shared(); + state->LayoutStatus = status; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_NO_THROW(client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + EXPECT_FALSE(state->Requests[0].IsLayout); + EXPECT_TRUE(state->Requests[1].IsLayout); + for (auto request = state->Requests.begin(); request != state->Requests.end(); ++request) + { + if (!request->IsLayout) + { + EXPECT_EQ(request->Host, "primary.test"); + EXPECT_TRUE(request->HostHeader.empty()); + } + } + } + } + + TEST(DataLocalityTest, PropagatesNonFallbackLayoutError) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NotFound; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_THROW( + client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions()), StorageException); + } + + TEST(DataLocalityTest, FallsBackWhenBlobHasNoLayout) + { + auto state = std::make_shared(); + state->LayoutStatus = Core::Http::HttpStatusCode::NoContent; + auto data = std::make_shared(1024 * 1024, 'x'); + auto client = CreateLocalityClient(state, data); + + std::vector buffer(data->size()); + EXPECT_NO_THROW(client.DownloadTo(buffer.data(), buffer.size(), CreateDownloadOptions())); + EXPECT_TRUE(std::equal(data->begin(), data->end(), buffer.begin())); + + std::lock_guard lock(state->Mutex); + ASSERT_GT(state->Requests.size(), 1U); + for (auto request = state->Requests.begin() + 1; request != state->Requests.end(); ++request) + { + EXPECT_EQ(request->Host, "primary.test"); + } + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp new file mode 100644 index 0000000000..92a339c943 --- /dev/null +++ b/sdk/storage/azure-storage-blobs/test/ut/session_authentication_test.cpp @@ -0,0 +1,666 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "blob_container_client_test.hpp" + +#include +#include + +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + class SessionTestCredential final : public Azure::Core::Credentials::TokenCredential { + public: + SessionTestCredential() : TokenCredential("SessionTestCredential") {} + + mutable int GetTokenCount = 0; + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext&, + const Azure::Core::Context&) const override + { + ++GetTokenCount; + return {"bearer-token", Azure::DateTime::clock::now() + std::chrono::hours(1)}; + } + }; + + class CountingTokenCredential final : public Azure::Core::Credentials::TokenCredential { + public: + explicit CountingTokenCredential( + std::shared_ptr credential) + : TokenCredential("CountingTokenCredential"), m_credential(std::move(credential)) + { + } + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext& tokenRequestContext, + const Azure::Core::Context& context) const override + { + ++m_getTokenCount; + return m_credential->GetToken(tokenRequestContext, context); + } + + int GetTokenCount() const { return m_getTokenCount.load(); } + + private: + std::shared_ptr m_credential; + mutable std::atomic m_getTokenCount{0}; + }; + + struct SessionTestState final + { + bool RejectFirstSessionRequest = false; + bool RejectedSessionRequest = false; + bool RejectCreateSession = false; + bool RejectCreateSessionWithNotFound = false; + bool RejectDownloadWithNotFound = false; + bool FailFirstDownloadWithServerError = false; + bool FailedDownloadWithServerError = false; + std::vector CreateSessionContainers; + std::vector CreateSessionVersions; + std::vector AuthorizationHeaders; + std::vector DownloadHosts; + std::vector SessionSignaturesValid; + bool MutatedDownloadObserved = false; + std::list> ResponseBodies; + }; + + class SessionTestTransport final : public Azure::Core::Http::HttpTransport { + public: + explicit SessionTestTransport(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Context const&) override + { + std::lock_guard lock(m_mutex); + const auto authorization = request.GetHeader("Authorization"); + m_state->AuthorizationHeaders.emplace_back( + authorization.HasValue() ? authorization.Value() : std::string()); + + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post && query.count("comp") != 0 + && query.at("comp") == "session") + { + m_state->CreateSessionContainers.emplace_back(request.GetUrl().GetPath()); + const auto version = request.GetHeader("x-ms-version"); + m_state->CreateSessionVersions.emplace_back( + version.HasValue() ? version.Value() : std::string()); + if (m_state->RejectCreateSession) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::Forbidden, "Forbidden"); + } + if (m_state->RejectCreateSessionWithNotFound) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + } + const auto token = "token-" + request.GetUrl().GetPath(); + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const std::string body + = "id" + "2099-01-01T00:00:00Z" + "HMAC" + + token + "" + key + + ""; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::Created, + "Created", + std::vector(body.begin(), body.end())); + } + + if (authorization.HasValue() && authorization.Value().find("Session ") == 0) + { + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const auto signatureSeparator = authorization.Value().find(':'); + const auto expectedAuthorization = authorization.Value().substr(0, signatureSeparator + 1) + + Azure::Storage::_internal::SharedKeyPolicy::GetSignature(request, "account", key); + m_state->SessionSignaturesValid.emplace_back( + signatureSeparator != std::string::npos + && authorization.Value() == expectedAuthorization); + } + if (query.count("session-test") != 0 && request.GetHeader("x-ms-session-test").HasValue()) + { + m_state->MutatedDownloadObserved = true; + } + m_state->DownloadHosts.emplace_back(request.GetUrl().GetHost()); + + if (m_state->RejectFirstSessionRequest && !m_state->RejectedSessionRequest + && authorization.HasValue() && authorization.Value().find("Session ") == 0) + { + m_state->RejectedSessionRequest = true; + auto response + = CreateResponse(Azure::Core::Http::HttpStatusCode::Unauthorized, "Unauthorized"); + response->SetHeader("WWW-Authenticate", "Session error=session_expired"); + return response; + } + + if (m_state->FailFirstDownloadWithServerError && !m_state->FailedDownloadWithServerError) + { + m_state->FailedDownloadWithServerError = true; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::InternalServerError, "Internal Server Error"); + } + + if (m_state->RejectDownloadWithNotFound) + { + return CreateResponse(Azure::Core::Http::HttpStatusCode::NotFound, "Not Found"); + } + + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("x-ms-creation-time", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-server-encrypted", "true"); + return response; + } + + private: + std::unique_ptr CreateResponse( + Azure::Core::Http::HttpStatusCode statusCode, + const std::string& reasonPhrase, + std::vector body = {}) const + { + m_state->ResponseBodies.emplace_back(std::move(body)); + auto response + = std::make_unique(1, 1, statusCode, reasonPhrase); + response->SetBodyStream( + std::make_unique(m_state->ResponseBodies.back())); + return response; + } + + std::shared_ptr m_state; + mutable std::mutex m_mutex; + }; + + class SessionRequestMutationPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + std::unique_ptr Clone() const override + { + return std::make_unique(); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + request.GetUrl().AppendQueryParameter("session-test", "value"); + request.SetHeader("x-ms-session-test", "value"); + return nextPolicy.Send(request, context); + } + }; + + class DataLocalityTestPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + DataLocalityTestPolicy() : m_downloadCount(std::make_shared>(0)) {} + + explicit DataLocalityTestPolicy(std::shared_ptr> downloadCount) + : m_downloadCount(std::move(downloadCount)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(m_downloadCount); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + Azure::Core::Context const& context) const override + { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Get && query.count("comp") == 0) + { + request.SetHeader("Host", request.GetUrl().GetHost()); + request.GetUrl().SetHost( + ++*m_downloadCount == 1 ? "tenant1.blob.core.windows.net" + : "tenant2.blob.core.windows.net"); + } + return nextPolicy.Send(request, context); + } + + private: + std::shared_ptr> m_downloadCount; + }; + + std::unique_ptr CreateSessionRequestCountingPolicy( + std::shared_ptr> sessionRequestCount) + { + return std::make_unique( + [sessionRequestCount + = std::move(sessionRequestCount)](const Azure::Core::Http::Request& request) { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post + && query.count("comp") != 0 && query.at("comp") == "session") + { + ++*sessionRequestCount; + } + }); + } + + void AddSessionTestTransport( + Blobs::BlobClientOptions& options, + const std::shared_ptr& state) + { + options.Transport.Transport = std::make_shared(state); + } + + void EnableSessions( + Blobs::BlobClientOptions& options, + const std::shared_ptr& state) + { + AddSessionTestTransport(options, state); + options.Session.Mode = Blobs::SessionMode::Enabled; + } + } // namespace + + TEST(SessionAuthenticationTest, SignsEligibleDownload) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + ASSERT_EQ(state->SessionSignaturesValid.size(), 1U); + EXPECT_TRUE(state->SessionSignaturesValid[0]); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, SignsAfterUserPerRetryPolicies) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.PerRetryPolicies.emplace_back(std::make_unique()); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->MutatedDownloadObserved); + ASSERT_EQ(state->SessionSignaturesValid.size(), 1U); + EXPECT_TRUE(state->SessionSignaturesValid[0]); + } + + TEST(SessionAuthenticationTest, FallsBackToBearerAfter401) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectFirstSessionRequest = true; + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), std::string::npos); + EXPECT_EQ(credential->GetTokenCount, 2); + } + + TEST(SessionAuthenticationTest, UsesBearerOnSecondaryHost) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->FailFirstDownloadWithServerError = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 1; + options.Retry.RetryDelay = std::chrono::milliseconds(0); + options.Retry.MaxRetryDelay = std::chrono::milliseconds(0); + options.SecondaryHostForRetryReads = "account-secondary.blob.core.windows.net"; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + ASSERT_EQ(state->DownloadHosts.size(), 3U); + EXPECT_EQ(state->DownloadHosts[0], "account.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[1], "account-secondary.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[2], "account.blob.core.windows.net"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 4U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Bearer "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[3].find("Session "), 0U); + EXPECT_EQ(credential->GetTokenCount, 2); + } + + TEST(SessionAuthenticationTest, ReusesSessionAcrossDataLocalityTenants) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + options.PerOperationPolicies.emplace_back(std::make_unique()); + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + ASSERT_EQ(state->DownloadHosts.size(), 2U); + EXPECT_EQ(state->DownloadHosts[0], "tenant1.blob.core.windows.net"); + EXPECT_EQ(state->DownloadHosts[1], "tenant2.blob.core.windows.net"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, SharedProviderCachesPerContainer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions providerOptions; + EnableSessions(providerOptions, state); + providerOptions.Session.AccountName = "account"; + auto provider = std::make_shared( + "https://account.blob.core.windows.net", credential, providerOptions); + + auto clientOptions = providerOptions; + clientOptions.Session.Provider = provider; + Blobs::BlobServiceClient serviceClient( + "https://account.blob.core.windows.net", credential, clientOptions); + auto first = serviceClient.GetBlobContainerClient("container1").GetBlobClient("blob1"); + auto second = serviceClient.GetBlobContainerClient("container1").GetBlobClient("blob2"); + auto third = serviceClient.GetBlobContainerClient("container2").GetBlobClient("blob"); + + EXPECT_NO_THROW(first.Download()); + EXPECT_NO_THROW(second.Download()); + EXPECT_NO_THROW(third.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 2U); + EXPECT_EQ(state->CreateSessionContainers[0], "container1"); + EXPECT_EQ(state->CreateSessionContainers[1], "container2"); + EXPECT_EQ(state->CreateSessionVersions[0], Blobs::_detail::ApiVersion); + EXPECT_EQ(state->CreateSessionVersions[1], Blobs::_detail::ApiVersion); + } + + TEST(SessionAuthenticationTest, SharedProviderSupportsIndependentClients) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions providerOptions; + EnableSessions(providerOptions, state); + providerOptions.Session.AccountName = "account"; + auto provider = std::make_shared( + "https://account.blob.core.windows.net", credential, providerOptions); + + auto clientOptions = providerOptions; + clientOptions.Session.Provider = provider; + Blobs::BlobClient first( + "https://account.blob.core.windows.net/container/blob1", credential, clientOptions); + Blobs::BlobClient second( + "https://account.blob.core.windows.net/container/blob2", credential, clientOptions); + + EXPECT_NO_THROW(first.Download()); + EXPECT_NO_THROW(second.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 3U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session "), 0U); + EXPECT_EQ(state->AuthorizationHeaders[2].find("Session "), 0U); + } + + TEST(SessionAuthenticationTest, ExplicitProviderSupportsCustomEndpoint) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + options.Session.Provider = std::make_shared( + "https://storage.contoso.com", credential, options); + + Blobs::BlobClient client("https://storage.contoso.com/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-container:"), 0U); + } + + TEST(SessionAuthenticationTest, ExplicitProviderSupportsPathStyleEndpoint) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + EnableSessions(options, state); + options.Session.AccountName = "account"; + options.Session.Provider = std::make_shared( + "https://localhost/account", credential, options); + + Blobs::BlobClient client("https://localhost/account/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "account/container"); + ASSERT_EQ(state->AuthorizationHeaders.size(), 2U); + EXPECT_EQ(state->AuthorizationHeaders[1].find("Session token-account/container:"), 0U); + } + + TEST(SessionAuthenticationTest, CustomEndpointRequiresAccountNameAndProvider) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Enabled; + options.Session.AccountName = "account"; + + EXPECT_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options), + std::invalid_argument); + } + + TEST(SessionAuthenticationTest, AutoModeUsesBearer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + AddSessionTestTransport(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->CreateSessionContainers.empty()); + ASSERT_EQ(state->AuthorizationHeaders.size(), 1U); + EXPECT_EQ(state->AuthorizationHeaders[0].find("Bearer "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, AutoModeAllowsCustomEndpointConstruction) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + + EXPECT_NO_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options)); + } + + TEST(SessionAuthenticationTest, EnabledModeRejectsCustomEndpointWithoutConfiguration) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Enabled; + + EXPECT_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options), + std::invalid_argument); + } + + TEST(SessionAuthenticationTest, DefaultProviderCachesSoftFailure) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectCreateSession = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + options.Session.AccountName = "account"; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + } + + TEST(SessionAuthenticationTest, PropagatesCreateSessionNotFound) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectCreateSessionWithNotFound = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/missing-container/blob", credential, options); + + EXPECT_THROW(client.Download(), StorageException); + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "missing-container"); + } + + TEST(SessionAuthenticationTest, PropagatesDownloadNotFound) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + state->RejectDownloadWithNotFound = true; + Blobs::BlobClientOptions options; + options.Retry.MaxRetries = 0; + EnableSessions(options, state); + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/missing-blob", credential, options); + + EXPECT_THROW(client.Download(), StorageException); + ASSERT_EQ(state->CreateSessionContainers.size(), 1U); + EXPECT_EQ(state->CreateSessionContainers[0], "container"); + } + + TEST(SessionAuthenticationTest, DisabledModeUsesBearer) + { + auto credential = std::make_shared(); + auto state = std::make_shared(); + Blobs::BlobClientOptions options; + AddSessionTestTransport(options, state); + options.Session.Mode = Blobs::SessionMode::Disabled; + + Blobs::BlobClient client( + "https://account.blob.core.windows.net/container/blob", credential, options); + EXPECT_NO_THROW(client.Download()); + + EXPECT_TRUE(state->CreateSessionContainers.empty()); + ASSERT_EQ(state->AuthorizationHeaders.size(), 1U); + EXPECT_EQ(state->AuthorizationHeaders[0].find("Bearer "), 0U); + EXPECT_EQ(credential->GetTokenCount, 1); + } + + TEST(SessionAuthenticationTest, DisabledModeAllowsCustomEndpointConstruction) + { + auto credential = std::make_shared(); + Blobs::BlobClientOptions options; + options.Session.Mode = Blobs::SessionMode::Disabled; + + EXPECT_NO_THROW( + Blobs::BlobClient("https://storage.contoso.com/container/blob", credential, options)); + } + + class SessionAuthenticationRecordedTest : public BlobContainerClientTest { + }; + + TEST_F(SessionAuthenticationRecordedTest, DownloadsWithDefaultSessionProvider_PLAYBACKONLY_) + { + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + const std::vector content{'s', 'e', 's', 's', 'i', 'o', 'n'}; + blobClient.UploadFrom(content.data(), content.size()); + + auto credential = std::make_shared(GetTestCredential()); + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + Blobs::BlobClient sessionClient(blobClient.GetUrl(), credential, options); + + for (int i = 0; i < 3; ++i) + { + EXPECT_NO_THROW(sessionClient.Download().Value.BodyStream->ReadToEnd()); + } + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_EQ(sessionRequestCount->load(), 1); + + Blobs::Models::BlobHttpHeaders headers; + headers.ContentType = "application/octet-stream"; + EXPECT_NO_THROW(sessionClient.GetProperties()); + EXPECT_NO_THROW(sessionClient.SetMetadata({{"key", "value"}})); + EXPECT_NO_THROW(sessionClient.SetHttpHeaders(headers)); + EXPECT_EQ(credential->GetTokenCount(), 2); + EXPECT_EQ(sessionRequestCount->load(), 1); + } + + TEST_F( + SessionAuthenticationRecordedTest, + IndependentClientsShareExplicitSessionProvider_PLAYBACKONLY_) + { + auto blobClient = m_blobContainerClient->GetBlockBlobClient(RandomString()); + const std::vector content{'s', 'e', 's', 's', 'i', 'o', 'n'}; + blobClient.UploadFrom(content.data(), content.size()); + + auto credential = std::make_shared(GetTestCredential()); + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + options.Session.Provider = std::make_shared( + GetBlobServiceUrl(), credential, options); + options.Session.AccountName = m_accountName; + Blobs::BlobClient firstClient(blobClient.GetUrl(), credential, options); + Blobs::BlobClient secondClient(blobClient.GetUrl(), credential, options); + + EXPECT_NO_THROW(firstClient.Download().Value.BodyStream->ReadToEnd()); + EXPECT_NO_THROW(secondClient.Download().Value.BodyStream->ReadToEnd()); + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_EQ(sessionRequestCount->load(), 1); + + Blobs::Models::BlobHttpHeaders headers; + headers.ContentType = "application/octet-stream"; + EXPECT_NO_THROW(firstClient.GetProperties()); + EXPECT_NO_THROW(firstClient.SetMetadata({{"key", "value"}})); + EXPECT_NO_THROW(firstClient.SetHttpHeaders(headers)); + EXPECT_EQ(credential->GetTokenCount(), 2); + EXPECT_EQ(sessionRequestCount->load(), 1); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp index df22348619..6300e0e0d2 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_retry_policy_test.cpp @@ -435,6 +435,7 @@ namespace Azure { namespace Storage { namespace Test { options.TransferOptions.InitialChunkSize = 2; options.TransferOptions.ChunkSize = 2; options.TransferOptions.Concurrency = 1; + options.LayoutAwareRouting = Blobs::LayoutAwareRouting::Disabled; blobClient.DownloadTo( reinterpret_cast(&downloadBuffer[0]), static_cast(downloadBuffer.size()), diff --git a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp index 244ca8da31..1ab2a0b12f 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/storage_timeout_test.cpp @@ -11,31 +11,6 @@ namespace Azure { namespace Storage { namespace Test { - class PeekHttpRequestPolicy final : public Core::Http::Policies::HttpPolicy { - public: - PeekHttpRequestPolicy(std::function callback) - : m_callback(std::move(callback)) - { - } - - std::unique_ptr Send( - Core::Http::Request& request, - Core::Http::Policies::NextHttpPolicy nextPolicy, - Core::Context const& context) const override - { - m_callback(request); - return nextPolicy.Send(request, context); - } - - std::unique_ptr Clone() const override - { - return std::make_unique(*this); - } - - private: - std::function m_callback; - }; - TEST_F(StorageTest, StoragetimeoutTestBasic_LIVEONLY_) { Azure::Nullable timeout; diff --git a/sdk/storage/azure-storage-common/CMakeLists.txt b/sdk/storage/azure-storage-common/CMakeLists.txt index ceed2c7422..d2e3d724e6 100644 --- a/sdk/storage/azure-storage-common/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/CMakeLists.txt @@ -54,11 +54,13 @@ set( inc/azure/storage/common/internal/shared_key_policy.hpp inc/azure/storage/common/internal/storage_bearer_token_auth.hpp inc/azure/storage/common/internal/storage_bearer_token_authentication_policy.hpp + inc/azure/storage/common/internal/storage_data_locality_policy.hpp inc/azure/storage/common/internal/storage_per_retry_policy.hpp inc/azure/storage/common/internal/storage_pipeline.hpp inc/azure/storage/common/internal/storage_retry_policy.hpp inc/azure/storage/common/internal/storage_service_version_policy.hpp inc/azure/storage/common/internal/storage_switch_to_secondary_policy.hpp + inc/azure/storage/common/internal/storage_url.hpp inc/azure/storage/common/internal/structured_message_decoding_stream.hpp inc/azure/storage/common/internal/structured_message_encoding_stream.hpp inc/azure/storage/common/internal/structured_message_helper.hpp @@ -80,11 +82,13 @@ set( src/shared_key_policy.cpp src/storage_bearer_token_authentication_policy.cpp src/storage_credential.cpp + src/storage_data_locality_policy.cpp src/storage_exception.cpp src/storage_per_retry_policy.cpp src/storage_pipeline.cpp src/storage_retry_policy.cpp src/storage_switch_to_secondary_policy.cpp + src/storage_url.cpp src/structured_message_decoding_stream.cpp src/structured_message_encoding_stream.cpp src/structured_message_helper.cpp diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp index 3a0f608953..bd18eb39be 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/shared_key_policy.hpp @@ -32,13 +32,18 @@ namespace Azure { namespace Storage { namespace _internal { Core::Context const& context) const override { request.SetHeader( - "Authorization", "SharedKey " + m_credential->AccountName + ":" + GetSignature(request)); + "Authorization", + "SharedKey " + m_credential->AccountName + ":" + + GetSignature(request, m_credential->AccountName, m_credential->GetAccountKey())); return nextPolicy.Send(request, context); } - private: - std::string GetSignature(const Core::Http::Request& request) const; + static std::string GetSignature( + const Core::Http::Request& request, + const std::string& accountName, + const std::string& signingKey); + private: std::shared_ptr m_credential; }; diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp index ca32557526..7d5e11585e 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_bearer_token_auth.hpp @@ -10,7 +10,7 @@ namespace Azure { namespace Storage { namespace _internal { - class StorageBearerTokenAuthenticationPolicy final + class StorageBearerTokenAuthenticationPolicy : public Core::Http::Policies::_internal::BearerTokenAuthenticationPolicy { public: /** @@ -37,6 +37,15 @@ namespace Azure { namespace Storage { namespace _internal { return std::unique_ptr(new StorageBearerTokenAuthenticationPolicy(*this)); } + protected: + void AuthorizeRequest(Azure::Core::Http::Request& request, Azure::Core::Context const& context) + const; + + std::unique_ptr AuthorizeAndSendRequest( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, + Azure::Core::Context const& context) const override; + private: struct SafeTenantId { @@ -66,11 +75,6 @@ namespace Azure { namespace Storage { namespace _internal { mutable SafeTenantId m_safeTenantId; bool m_enableTenantDiscovery; - std::unique_ptr AuthorizeAndSendRequest( - Azure::Core::Http::Request& request, - Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, - Azure::Core::Context const& context) const override; - bool AuthorizeRequestOnChallenge( std::string const& challenge, Azure::Core::Http ::Request& request, diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp new file mode 100644 index 0000000000..e6176798bb --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_data_locality_policy.hpp @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include "azure/storage/common/dll_import_export.hpp" + +#include + +#include +#include + +namespace Azure { namespace Storage { namespace _internal { + + AZ_STORAGE_COMMON_DLLEXPORT extern const Azure::Core::Context::Key DataLocalityEndpointKey; + + inline Azure::Core::Context WithDataLocalityEndpoint( + const Azure::Core::Context& context, + const std::string& endpoint) + { + return endpoint.empty() ? context : context.WithValue(DataLocalityEndpointKey, endpoint); + } + + class StorageDataLocalityPolicy final : public Azure::Core::Http::Policies::HttpPolicy { + public: + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + const Azure::Core::Context& context) const override; + }; + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp index 172b788660..38fab95724 100644 --- a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_pipeline.hpp @@ -19,6 +19,7 @@ namespace Azure { namespace Storage { namespace _internal { std::string PrimaryHost; std::string SecondaryHost; std::string ApiVersion; + bool AddDataLocalityPolicy = false; std::unique_ptr TokenAuthPolicy; std::unique_ptr SharedKeyAuthPolicy; }; diff --git a/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp new file mode 100644 index 0000000000..f8233d572a --- /dev/null +++ b/sdk/storage/azure-storage-common/inc/azure/storage/common/internal/storage_url.hpp @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include + +namespace Azure { namespace Storage { namespace _internal { + + struct StorageUrlParts final + { + std::string Service; + std::string AccountName; + std::string ServiceUrl; + Azure::Nullable ContainerName; + Azure::Nullable ContainerUrl; + }; + + Azure::Nullable ParseStorageUrl(const Azure::Core::Url& url); + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/shared_key_policy.cpp b/sdk/storage/azure-storage-common/src/shared_key_policy.cpp index 3a8d7e0b37..265e7d6011 100644 --- a/sdk/storage/azure-storage-common/src/shared_key_policy.cpp +++ b/sdk/storage/azure-storage-common/src/shared_key_policy.cpp @@ -92,7 +92,10 @@ bool comparator(const std::string& lhs, const std::string& rhs) namespace Azure { namespace Storage { namespace _internal { - std::string SharedKeyPolicy::GetSignature(const Core::Http::Request& request) const + std::string SharedKeyPolicy::GetSignature( + const Core::Http::Request& request, + const std::string& accountName, + const std::string& signingKey) { std::string string_to_sign; string_to_sign += request.GetMethod().ToString() + "\n"; @@ -146,7 +149,7 @@ namespace Azure { namespace Storage { namespace _internal { ordered_kv.clear(); // canonicalized resource - string_to_sign += "/" + m_credential->AccountName + "/" + request.GetUrl().GetPath() + "\n"; + string_to_sign += "/" + accountName + "/" + request.GetUrl().GetPath() + "\n"; for (const auto& query : request.GetUrl().GetQueryParameters()) { std::string key = Azure::Core::_internal::StringExtensions::ToLower(query.first); @@ -164,6 +167,6 @@ namespace Azure { namespace Storage { namespace _internal { return Azure::Core::Convert::Base64Encode(_internal::HmacSha256( std::vector(string_to_sign.begin(), string_to_sign.end()), - Azure::Core::Convert::Base64Decode(m_credential->GetAccountKey()))); + Azure::Core::Convert::Base64Decode(signingKey))); } }}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp b/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp index 9f39fb1e71..cd617e5c8f 100644 --- a/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp +++ b/sdk/storage/azure-storage-common/src/storage_bearer_token_authentication_policy.cpp @@ -13,6 +13,14 @@ namespace Azure { namespace Storage { namespace _internal { Azure::Core::Http::Request& request, Azure::Core::Http::Policies::NextHttpPolicy& nextPolicy, Azure::Core::Context const& context) const + { + AuthorizeRequest(request, context); + return nextPolicy.Send(request, context); + } + + void StorageBearerTokenAuthenticationPolicy::AuthorizeRequest( + Azure::Core::Http::Request& request, + Azure::Core::Context const& context) const { std::string tenantId = m_safeTenantId.Get(); if (!tenantId.empty() || !m_enableTenantDiscovery) @@ -22,7 +30,6 @@ namespace Azure { namespace Storage { namespace _internal { tokenRequestContext.TenantId = tenantId; AuthenticateAndAuthorizeRequest(request, tokenRequestContext, context); } - return nextPolicy.Send(request, context); } bool StorageBearerTokenAuthenticationPolicy::AuthorizeRequestOnChallenge( diff --git a/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp b/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp new file mode 100644 index 0000000000..5674fe54c8 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/storage_data_locality_policy.cpp @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +namespace Azure { namespace Storage { namespace _internal { + + const Azure::Core::Context::Key DataLocalityEndpointKey; + + std::unique_ptr StorageDataLocalityPolicy::Send( + Azure::Core::Http::Request& request, + Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, + const Azure::Core::Context& context) const + { + std::string endpoint; + if (!context.TryGetValue(DataLocalityEndpointKey, endpoint) || endpoint.empty()) + { + return nextPolicy.Send(request, context); + } + + auto& requestUrl = request.GetUrl(); + std::string originalAuthority = requestUrl.GetHost(); + if (requestUrl.GetPort() != 0) + { + originalAuthority += ":" + std::to_string(requestUrl.GetPort()); + } + + // The Storage service returns endpoint values as absolute URLs with a scheme and port. + Azure::Core::Url endpointUrl( + endpoint.find("://") == std::string::npos ? requestUrl.GetScheme() + "://" + endpoint + : endpoint); + request.SetHeader("Host", originalAuthority); + requestUrl.SetScheme(endpointUrl.GetScheme()); + requestUrl.SetHost(endpointUrl.GetHost()); + requestUrl.SetPort(endpointUrl.GetPort()); + return nextPolicy.Send(request, context); + } + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp index 339286b830..d3fa8463e1 100644 --- a/sdk/storage/azure-storage-common/src/storage_pipeline.cpp +++ b/sdk/storage/azure-storage-common/src/storage_pipeline.cpp @@ -4,6 +4,7 @@ #include "azure/storage/common/internal/storage_pipeline.hpp" #include "azure/storage/common/internal/constants.hpp" +#include "azure/storage/common/internal/storage_data_locality_policy.hpp" #include "azure/storage/common/internal/storage_per_retry_policy.hpp" #include "azure/storage/common/internal/storage_retry_policy.hpp" #include "azure/storage/common/internal/storage_service_version_policy.hpp" @@ -41,6 +42,10 @@ namespace Azure { namespace Storage { namespace _internal { { policies.emplace_back(policy->Clone()); } + if (storagePipelineOptions.AddDataLocalityPolicy) + { + policies.emplace_back(std::make_unique()); + } // Retry policy policies.emplace_back(std::make_unique(clientOptions.Retry)); @@ -66,7 +71,7 @@ namespace Azure { namespace Storage { namespace _internal { policies.emplace_back(policy->Clone()); } - // SharedKey/SAS auth runs last among per-retry policies so the signature covers + // Request-signing auth runs last among per-retry policies so the signature covers // the final, fully-modified request. if (storagePipelineOptions.SharedKeyAuthPolicy) { diff --git a/sdk/storage/azure-storage-common/src/storage_url.cpp b/sdk/storage/azure-storage-common/src/storage_url.cpp new file mode 100644 index 0000000000..2af0a6e9da --- /dev/null +++ b/sdk/storage/azure-storage-common/src/storage_url.cpp @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "azure/storage/common/internal/storage_url.hpp" + +#include + +namespace { + +std::string StripAccountSuffix(std::string accountName) +{ + while (true) + { + const auto originalSize = accountName.size(); + for (const std::string suffix : + {"-ipv6", "-dualstack", "-secondary", "-internetrouting", "-microsoftrouting"}) + { + if (accountName.size() > suffix.size() + && accountName.compare(accountName.size() - suffix.size(), suffix.size(), suffix) == 0) + { + accountName.resize(accountName.size() - suffix.size()); + break; + } + } + if (accountName.size() == originalSize) + { + break; + } + } + return accountName; +} + +} // namespace + +namespace Azure { namespace Storage { namespace _internal { + + Azure::Nullable ParseStorageUrl(const Azure::Core::Url& url) + { + const auto hostname = Azure::Core::_internal::StringExtensions::ToLower(url.GetHost()); + std::string endpoint; + for (const std::string candidate : + {".preprod.core.windows.net", + ".core.windows.net", + ".core.chinacloudapi.cn", + ".core.usgovcloudapi.net"}) + { + if (hostname.find(candidate) != std::string::npos) + { + endpoint = candidate; + break; + } + } + std::string service; + std::string accountName; + if (!endpoint.empty()) + { + for (const std::string candidate : {"blob", "dfs", "file", "queue", "table", "web"}) + { + const auto suffix = "." + candidate + endpoint; + if (hostname.size() > suffix.size() + && hostname.compare(hostname.size() - suffix.size(), suffix.size(), suffix) == 0) + { + service = candidate; + accountName = hostname.substr(0, hostname.size() - suffix.size()); + break; + } + } + } + else + { + for (const std::string candidate : {"blob", "dfs", "file", "queue", "table", "web"}) + { + const auto suffix = "." + candidate + ".storage.azure.net"; + if (hostname.size() <= suffix.size() + || hostname.compare(hostname.size() - suffix.size(), suffix.size(), suffix) != 0) + { + continue; + } + + service = candidate; + accountName = hostname.substr(0, hostname.find('.')); + break; + } + } + + accountName = StripAccountSuffix(std::move(accountName)); + if (service.empty() || accountName.empty()) + { + return {}; + } + + StorageUrlParts result; + result.Service = std::move(service); + result.AccountName = std::move(accountName); + + auto serviceUrl = url; + serviceUrl.SetPath({}); + serviceUrl.SetQueryParameters({}); + result.ServiceUrl = serviceUrl.GetAbsoluteUrl(); + + const auto& path = url.GetPath(); + if (!path.empty()) + { + const auto containerNameEnd = path.find('/', 1); + result.ContainerName + = containerNameEnd == std::string::npos ? path : path.substr(0, containerNameEnd); + + auto containerUrl = serviceUrl; + containerUrl.SetPath(result.ContainerName.Value()); + result.ContainerUrl = containerUrl.GetAbsoluteUrl(); + } + return result; + } + +}}} // namespace Azure::Storage::_internal diff --git a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt index bbe25d9e2f..ccce4f50f9 100644 --- a/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-common/test/ut/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable ( crypt_functions_test.cpp metadata_test.cpp storage_credential_test.cpp + storage_url_test.cpp structured_message_test.cpp test_base.cpp test_base.hpp diff --git a/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp b/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp new file mode 100644 index 0000000000..4018ac44d4 --- /dev/null +++ b/sdk/storage/azure-storage-common/test/ut/storage_url_test.cpp @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + TEST(StorageUrlTest, ParsesStandardEndpoint) + { + auto result = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.blob.core.windows.net/container/blob?versionid=version")); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + EXPECT_EQ(result.Value().ServiceUrl, "https://account.blob.core.windows.net"); + ASSERT_TRUE(result.Value().ContainerName.HasValue()); + EXPECT_EQ(result.Value().ContainerName.Value(), "container"); + EXPECT_EQ( + result.Value().ContainerUrl.Value(), "https://account.blob.core.windows.net/container"); + } + + TEST(StorageUrlTest, ParsesMixedCaseEndpoints) + { + for (const std::string hostname : + {"Account.BLOB.CORE.WINDOWS.NET", + "ACCOUNT.blob.core.windows.net", + "Account-DuAlStAcK-SeCoNdArY.BlOb.CoRe.WiNdOwS.NeT", + "Account.BlOb.PrEpRoD.CoRe.WiNdOwS.NeT", + "Account.BlOb.CoRe.ChInAcLoUdApI.Cn", + "Account.BlOb.CoRe.UsGoVcLoUdApI.NeT", + "Account-SeCoNdArY.Z50.BlOb.StOrAgE.AzUrE.NeT", + "Account-SeCoNdArY.CuStOm.BlOb.StOrAgE.AzUrE.NeT"}) + { + SCOPED_TRACE(hostname); + const auto serviceUrl = "https://" + hostname; + const auto absoluteUrl = serviceUrl + "/Container/Blob?versionid=MiXeD"; + const Azure::Core::Url url(absoluteUrl); + auto result = _internal::ParseStorageUrl(url); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + EXPECT_EQ(result.Value().ServiceUrl, serviceUrl); + ASSERT_TRUE(result.Value().ContainerName.HasValue()); + EXPECT_EQ(result.Value().ContainerName.Value(), "Container"); + EXPECT_EQ(result.Value().ContainerUrl.Value(), serviceUrl + "/Container"); + EXPECT_EQ(url.GetAbsoluteUrl(), absoluteUrl); + } + } + + TEST(StorageUrlTest, RejectsNonStorageEndpoints) + { + EXPECT_FALSE(_internal::ParseStorageUrl(Azure::Core::Url("https://example.com")).HasValue()); + EXPECT_FALSE(_internal::ParseStorageUrl(Azure::Core::Url("https://localhost/account/container")) + .HasValue()); + EXPECT_FALSE( + _internal::ParseStorageUrl(Azure::Core::Url("https://storage.contoso.com/container")) + .HasValue()); + } + + TEST(StorageUrlTest, ParsesSovereignCloudEndpoints) + { + auto china = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.blob.core.chinacloudapi.cn/container/blob")); + ASSERT_TRUE(china.HasValue()); + EXPECT_EQ(china.Value().Service, "blob"); + EXPECT_EQ(china.Value().AccountName, "account"); + EXPECT_EQ(china.Value().ServiceUrl, "https://account.blob.core.chinacloudapi.cn"); + + auto government = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.file.core.usgovcloudapi.net/share/file")); + ASSERT_TRUE(government.HasValue()); + EXPECT_EQ(government.Value().Service, "file"); + EXPECT_EQ(government.Value().AccountName, "account"); + EXPECT_EQ(government.Value().ServiceUrl, "https://account.file.core.usgovcloudapi.net"); + } + + TEST(StorageUrlTest, StripsAccountSuffixesRepeatedly) + { + auto network = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-dualstack-secondary.blob.core.windows.net")); + + ASSERT_TRUE(network.HasValue()); + EXPECT_EQ(network.Value().AccountName, "account"); + + auto internetRouting = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-internetrouting-secondary.blob.core.windows.net")); + ASSERT_TRUE(internetRouting.HasValue()); + EXPECT_EQ(internetRouting.Value().AccountName, "account"); + + auto microsoftRouting = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-microsoftrouting-secondary.dfs.core.windows.net")); + ASSERT_TRUE(microsoftRouting.HasValue()); + EXPECT_EQ(microsoftRouting.Value().AccountName, "account"); + } + + TEST(StorageUrlTest, ParsesAzureDnsZoneEndpoints) + { + auto blob = _internal::ParseStorageUrl( + Azure::Core::Url("https://account.z1.blob.storage.azure.net/container/blob")); + ASSERT_TRUE(blob.HasValue()); + EXPECT_EQ(blob.Value().Service, "blob"); + EXPECT_EQ(blob.Value().AccountName, "account"); + EXPECT_EQ(blob.Value().ServiceUrl, "https://account.z1.blob.storage.azure.net"); + + auto dfs = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-secondary.z50.dfs.storage.azure.net/filesystem/file")); + ASSERT_TRUE(dfs.HasValue()); + EXPECT_EQ(dfs.Value().Service, "dfs"); + EXPECT_EQ(dfs.Value().AccountName, "account"); + EXPECT_EQ(dfs.Value().ServiceUrl, "https://account-secondary.z50.dfs.storage.azure.net"); + } + + TEST(StorageUrlTest, ParsesFlexibleAzureDnsZoneEndpoints) + { + auto result = _internal::ParseStorageUrl( + Azure::Core::Url("https://account-secondary.custom.blob.storage.azure.net/container")); + + ASSERT_TRUE(result.HasValue()); + EXPECT_EQ(result.Value().Service, "blob"); + EXPECT_EQ(result.Value().AccountName, "account"); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-common/test/ut/test_base.hpp b/sdk/storage/azure-storage-common/test/ut/test_base.hpp index be9dfe5fd3..1aa3abdc20 100644 --- a/sdk/storage/azure-storage-common/test/ut/test_base.hpp +++ b/sdk/storage/azure-storage-common/test/ut/test_base.hpp @@ -57,6 +57,31 @@ namespace Azure { namespace Storage { ResponseFactory m_responseFactory; }; + class PeekHttpRequestPolicy final : public Core::Http::Policies::HttpPolicy { + public: + explicit PeekHttpRequestPolicy(std::function callback) + : m_callback(std::move(callback)) + { + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy nextPolicy, + Core::Context const& context) const override + { + m_callback(request); + return nextPolicy.Send(request, context); + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + private: + std::function m_callback; + }; + class StorageTest : public Azure::Core::Test::TestBase { public: StorageTest() { TestBase::SetUpTestSuiteLocal(AZURE_TEST_ASSETS_DIR); } diff --git a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md index 5616dc067f..b33a93452f 100644 --- a/sdk/storage/azure-storage-files-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-datalake/CHANGELOG.md @@ -4,6 +4,15 @@ ### Features Added +- Added support for persisted session authentication for eligible file download requests. +- Added `DataLakeFileClient::GetLayout` and related options and response types for retrieving + locality endpoints, byte ranges, and file properties. Ranges are exposed directly through + `Models::FileLayoutInfo::Ranges` as a vector of `Models::FileLayoutRange`. +- Added `DownloadFileOptions::LayoutEndpoint` for one-shot download routing and + `DownloadFileToOptions::LayoutAwareRouting` (`Auto`, `Disabled`, `Enabled`; default `Disabled`) for + managed downloads. +- Added the `Models::DownloadHint` alias and `Models::DownloadFileDetails::DownloadHint`. + ### Breaking Changes ### Bugs Fixed @@ -13,6 +22,10 @@ ### Other Changes +- Changed the default initial transfer size for partitioned uploads and downloads from 256 MiB to + 4 MiB, matching the default subsequent chunk size. Explicitly configured + `InitialChunkSize` and `SingleUploadThreshold` values are still respected. + ## 12.17.0 (2026-09-29) ### Other Changes diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index 9743563c1f..e3af80df50 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -189,6 +189,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DownloadFileOptions& options = DownloadFileOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Gets the data locality layout for this file. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A paged response containing the file's data locality layout. + * @remark This request is sent to the blob endpoint. + */ + FileLayoutPagedResponse GetLayout( + const GetFileLayoutOptions& options = GetFileLayoutOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + /** * @brief Creates a new file, or updates the content of an existing file. Updating * an existing file overwrites any existing metadata on the file. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index d7bfd9a560..16d63cf000 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -118,6 +118,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * @brief Deletes the file system. + * + * @remark When a file system is deleted, a file system with the same name cannot be created for + * at least 30 seconds. The file system might not be available for more than 30 seconds if the + * service is still processing the request. While the file system is being deleted, attempts to + * create a file system with the same name fail with status code 409 (Conflict). Operations on + * paths within the file system, including reads and writes, may continue to succeed for up to + * 30 seconds after the delete request is accepted. After this period, all operations on the + * file system and its paths fail with status code 404 (Not Found). + * * @param options Optional parameters to delete this file system. * @param context Context for cancelling long running operations. * @return Azure::Response containing the @@ -130,6 +139,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * @brief Deletes the file system if it exists. + * + * @remark When a file system is deleted, a file system with the same name cannot be created for + * at least 30 seconds. The file system might not be available for more than 30 seconds if the + * service is still processing the request. While the file system is being deleted, attempts to + * create a file system with the same name fail with status code 409 (Conflict). Operations on + * paths within the file system, including reads and writes, may continue to succeed for up to + * 30 seconds after the delete request is accepted. After this period, all operations on the + * file system and its paths fail with status code 404 (Not Found). + * * @param options Optional parameters to delete this file system. * @param context Context for cancelling long running operations. * @return Azure::Response containing the diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index ed6247bcfd..266eb0b768 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -83,10 +83,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; + /** + * @brief Determines whether locality-aware routing is used for managed file downloads. + */ + using LayoutAwareRouting = Blobs::LayoutAwareRouting; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; using GetServicePropertiesOptions = Blobs::GetServicePropertiesOptions; using SetServicePropertiesOptions = Blobs::SetServicePropertiesOptions; using EncryptionKey = Blobs::EncryptionKey; + using SessionOptions = Blobs::SessionOptions; /** * Configures whether to do content validation for file uploads and downloads. @@ -223,6 +228,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable DownloadValidationOptions; + + /** + * @brief Configures persisted session authentication for Blob endpoint download requests. + */ + SessionOptions Session; }; /** @@ -880,6 +890,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IncludeUserPrincipalName; }; + /** + * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. + */ + struct GetFileLayoutOptions final + { + /** + * @brief The range of bytes for which to retrieve the layout. + */ + Azure::Nullable Range; + + /** + * @brief The continuation token for retrieving the next page. + */ + Azure::Nullable ContinuationToken; + + /** + * @brief Access conditions for the operation. + */ + PathAccessConditions AccessConditions; + }; + /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Download. * @remark Some optional parameter is mandatory in certain combination. @@ -918,6 +949,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * @brief Optional. Configures whether to do content validation for file downloads. */ Azure::Nullable ValidationOptions; + + /** + * @brief The locality endpoint to use for this download. An empty value uses the normal + * endpoint. + */ + std::string LayoutEndpoint; }; /** @@ -1031,7 +1068,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * File smaller than this will be uploaded with a single upload operation. This value * cannot be larger than 5000 MiB. */ - int64_t SingleUploadThreshold = 256 * 1024 * 1024; + int64_t SingleUploadThreshold = 4 * 1024 * 1024; /** * The maximum number of bytes in a single request. This value cannot be larger than diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 1a434048d2..d0660cf788 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -17,6 +17,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { class DataLakeFileSystemClient; class DataLakePathClient; class DataLakeDirectoryClient; + class DataLakeFileClient; namespace Models { @@ -462,6 +463,26 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable Permissions; }; + /** + * @brief A file range and its resolved locality endpoint. + */ + using FileLayoutRange = Blobs::Models::BlobLayoutRange; + + /** + * @brief A page of file layout information and the file properties returned with it. + */ + struct FileLayoutInfo final + { + /** + * @brief The ranges and locality endpoints in this page. + */ + std::vector Ranges; + /** + * @brief The file properties returned with this page. + */ + PathProperties Properties; + }; + /** * @brief The access control list of a path. */ @@ -553,6 +574,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using UploadFileFromResult = Blobs::Models::UploadBlockBlobResult; using ScheduleFileDeletionResult = Blobs::Models::SetBlobExpiryResult; using CopyStatus = Blobs::Models::CopyStatus; + using DownloadHint = Blobs::Models::DownloadHint; /** * @brief Response type for #Azure::Storage::Files::DataLake::DataLakeFileClient::Query. @@ -731,6 +753,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The permissions of the file. */ Azure::Nullable Permissions; + + /** + * @brief The service hint for locality-aware downloads. + */ + Azure::Nullable DownloadHint; }; /** @@ -796,6 +823,27 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } // namespace Models + /** + * @brief Response type for + * #Azure::Storage::Files::DataLake::DataLakeFileClient::GetLayout. + */ + class FileLayoutPagedResponse final : public Azure::Core::PagedResponse { + public: + /** + * @brief The layout and file properties in this page. + */ + Models::FileLayoutInfo Layout; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_fileClient; + GetFileLayoutOptions m_operationOptions; + + friend class DataLakeFileClient; + friend class Azure::Core::PagedResponse; + }; + /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakeServiceClient::ListFileSystems. diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 0422c0f6d2..525a0a3eeb 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -121,30 +121,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - - auto renamedBlobClient = Blobs::BlobClient( - _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + auto renamedBlobClient = m_blobClient; + renamedBlobClient.m_blobUrl = _detail::GetBlobUrlFromUrl(destinationDfsUrl); auto renamedFileClient = DataLakeFileClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), @@ -194,31 +172,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - - auto renamedBlobClient = Blobs::BlobClient( - _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + auto renamedBlobClient = m_blobClient; + renamedBlobClient.m_blobUrl = _detail::GetBlobUrlFromUrl(destinationDfsUrl); auto renamedDirectoryClient = DataLakeDirectoryClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 0057174fc3..6b957c633b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -204,6 +204,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { blobOptions.Range = options.Range; blobOptions.RangeHashAlgorithm = options.RangeHashAlgorithm; blobOptions.Range = options.Range; + blobOptions.LayoutEndpoint = options.LayoutEndpoint; blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; @@ -253,6 +254,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); auto& headers = response.RawResponse->GetHeaders(); auto encryptionContext = headers.find(_detail::EncryptionContextHeaderName); if (encryptionContext != headers.end()) @@ -283,6 +285,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(ret), std::move(response.RawResponse)); } + FileLayoutPagedResponse DataLakeFileClient::GetLayout( + const GetFileLayoutOptions& options, + const Azure::Core::Context& context) const + { + Blobs::GetBlobLayoutOptions blobOptions; + blobOptions.Range = options.Range; + blobOptions.ContinuationToken = options.ContinuationToken; + blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; + blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; + auto response = m_blobClient.GetLayout(blobOptions, context); + + FileLayoutPagedResponse result; + result.Layout.Ranges = std::move(response.Layout.Ranges); + result.Layout.Properties = _detail::PathPropertiesFromBlobProperties( + std::move(response.Layout.Properties), *response.RawResponse); + result.CurrentPageToken = std::move(response.CurrentPageToken); + result.NextPageToken = std::move(response.NextPageToken); + result.RawResponse = std::move(response.RawResponse); + result.m_fileClient = std::make_shared(*this); + result.m_operationOptions = options; + return result; + } + Azure::Response DataLakeFileClient::UploadFrom( const std::string& fileName, const UploadFileFromOptions& options, @@ -364,6 +392,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } @@ -404,6 +433,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256); ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope); ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted; + ret.Details.DownloadHint = std::move(response.Value.Details.DownloadHint); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index 04085d3884..4cbce13e8b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -384,31 +384,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - auto renamedBlobClient = Blobs::BlobClient( _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + m_blobContainerClient.m_pipeline, + m_blobContainerClient.m_clientConfiguration); auto renamedFileClient = DataLakeFileClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), @@ -458,31 +437,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - Azure::Nullable uploadValidationOptions; - if (m_clientConfiguration.UploadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.UploadValidationOptions.Value().Algorithm; - uploadValidationOptions = blobOptions; - } - - Azure::Nullable downloadValidationOptions; - if (m_clientConfiguration.DownloadValidationOptions.HasValue()) - { - Blobs::TransferValidationOptions blobOptions; - blobOptions.Algorithm = m_clientConfiguration.DownloadValidationOptions.Value().Algorithm; - downloadValidationOptions = blobOptions; - } - - Blobs::_detail::BlobClientConfiguration blobClientConfiguration; - blobClientConfiguration.CustomerProvidedKey = m_clientConfiguration.CustomerProvidedKey; - blobClientConfiguration.UploadValidationOptions = std::move(uploadValidationOptions); - blobClientConfiguration.DownloadValidationOptions = std::move(downloadValidationOptions); - auto renamedBlobClient = Blobs::BlobClient( _detail::GetBlobUrlFromUrl(destinationDfsUrl), - m_pipeline, - std::move(blobClientConfiguration)); + m_blobContainerClient.m_pipeline, + m_blobContainerClient.m_clientConfiguration); auto renamedDirectoryClient = DataLakeDirectoryClient( std::move(destinationDfsUrl), std::move(renamedBlobClient), diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 4e5b4953d5..5abf39f303 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -343,61 +343,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Blobs::_detail::DataLakeInteroperabilityExtraOptionsKey, options.IncludeUserPrincipalName.Value()) : context); - Models::PathProperties ret; - ret.ETag = std::move(response.Value.ETag); - ret.LastModified = std::move(response.Value.LastModified); - ret.CreatedOn = std::move(response.Value.CreatedOn); - ret.Metadata = std::move(response.Value.Metadata); - ret.LeaseDuration = std::move(response.Value.LeaseDuration); - ret.LeaseState = std::move(response.Value.LeaseState); - ret.LeaseStatus = std::move(response.Value.LeaseStatus); - ret.HttpHeaders = std::move(response.Value.HttpHeaders); - ret.IsServerEncrypted = response.Value.IsServerEncrypted; - ret.EncryptionKeySha256 = std::move(response.Value.EncryptionKeySha256); - ret.EncryptionScope = std::move(response.Value.EncryptionScope); - ret.CopyId = std::move(response.Value.CopyId); - ret.CopySource = std::move(response.Value.CopySource); - ret.CopyStatus = std::move(response.Value.CopyStatus); - ret.CopyProgress = std::move(response.Value.CopyProgress); - ret.CopyCompletedOn = std::move(response.Value.CopyCompletedOn); - ret.ExpiresOn = std::move(response.Value.ExpiresOn); - ret.LastAccessedOn = std::move(response.Value.LastAccessedOn); - ret.FileSize = response.Value.BlobSize; - ret.ArchiveStatus = std::move(response.Value.ArchiveStatus); - ret.RehydratePriority = std::move(response.Value.RehydratePriority); - ret.CopyStatusDescription = std::move(response.Value.CopyStatusDescription); - ret.IsIncrementalCopy = std::move(response.Value.IsIncrementalCopy); - ret.IncrementalCopyDestinationSnapshot - = std::move(response.Value.IncrementalCopyDestinationSnapshot); - ret.VersionId = std::move(response.Value.VersionId); - ret.IsCurrentVersion = std::move(response.Value.IsCurrentVersion); - ret.IsDirectory = _detail::MetadataIndicatesIsDirectory(ret.Metadata); - auto& headers = response.RawResponse->GetHeaders(); - auto encryptionContext = headers.find(_detail::EncryptionContextHeaderName); - if (encryptionContext != headers.end()) - { - ret.EncryptionContext = encryptionContext->second; - } - auto acl = headers.find(_detail::AclHeaderName); - if (acl != headers.end()) - { - ret.Acls = Models::Acl::DeserializeAcls(acl->second); - } - auto owner = headers.find(_detail::OwnerHeaderName); - if (owner != headers.end()) - { - ret.Owner = owner->second; - } - auto group = headers.find(_detail::GroupHeaderName); - if (group != headers.end()) - { - ret.Group = group->second; - } - auto permissions = headers.find(_detail::PermissionsHeaderName); - if (permissions != headers.end()) - { - ret.Permissions = permissions->second; - } + auto ret = _detail::PathPropertiesFromBlobProperties( + std::move(response.Value), *response.RawResponse); return Azure::Response(std::move(ret), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index c6fa0aa1c5..3652ac5891 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -4,11 +4,20 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" #include "azure/storage/files/datalake/datalake_directory_client.hpp" +#include "azure/storage/files/datalake/datalake_file_client.hpp" #include "azure/storage/files/datalake/datalake_path_client.hpp" #include "azure/storage/files/datalake/datalake_service_client.hpp" #include "private/datalake_utilities.hpp" namespace Azure { namespace Storage { namespace Files { namespace DataLake { + void FileLayoutPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + auto options = m_operationOptions; + options.ContinuationToken = NextPageToken; + options.AccessConditions.IfMatch = Layout.Properties.ETag; + *this = m_fileClient->GetLayout(options, context); + } + namespace Models { Acl Acl::FromString(const std::string& aclString) diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 3e39529e2a..f6538da505 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -23,6 +23,68 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam return Win32FileTimeConverter::Win32FileTimeToDateTime(std::stoll(value)); } + Models::PathProperties PathPropertiesFromBlobProperties( + Blobs::Models::BlobProperties properties, + const Azure::Core::Http::RawResponse& response) + { + Models::PathProperties ret; + ret.ETag = std::move(properties.ETag); + ret.LastModified = std::move(properties.LastModified); + ret.CreatedOn = std::move(properties.CreatedOn); + ret.Metadata = std::move(properties.Metadata); + ret.LeaseDuration = std::move(properties.LeaseDuration); + ret.LeaseState = std::move(properties.LeaseState); + ret.LeaseStatus = std::move(properties.LeaseStatus); + ret.HttpHeaders = std::move(properties.HttpHeaders); + ret.IsServerEncrypted = properties.IsServerEncrypted; + ret.EncryptionKeySha256 = std::move(properties.EncryptionKeySha256); + ret.EncryptionScope = std::move(properties.EncryptionScope); + ret.CopyId = std::move(properties.CopyId); + ret.CopySource = std::move(properties.CopySource); + ret.CopyStatus = std::move(properties.CopyStatus); + ret.CopyProgress = std::move(properties.CopyProgress); + ret.CopyCompletedOn = std::move(properties.CopyCompletedOn); + ret.ExpiresOn = std::move(properties.ExpiresOn); + ret.LastAccessedOn = std::move(properties.LastAccessedOn); + ret.FileSize = properties.BlobSize; + ret.ArchiveStatus = std::move(properties.ArchiveStatus); + ret.RehydratePriority = std::move(properties.RehydratePriority); + ret.CopyStatusDescription = std::move(properties.CopyStatusDescription); + ret.IsIncrementalCopy = std::move(properties.IsIncrementalCopy); + ret.IncrementalCopyDestinationSnapshot + = std::move(properties.IncrementalCopyDestinationSnapshot); + ret.VersionId = std::move(properties.VersionId); + ret.IsCurrentVersion = std::move(properties.IsCurrentVersion); + ret.IsDirectory = MetadataIndicatesIsDirectory(ret.Metadata); + const auto& headers = response.GetHeaders(); + auto encryptionContext = headers.find(EncryptionContextHeaderName); + if (encryptionContext != headers.end()) + { + ret.EncryptionContext = encryptionContext->second; + } + auto acl = headers.find(AclHeaderName); + if (acl != headers.end()) + { + ret.Acls = Models::Acl::DeserializeAcls(acl->second); + } + auto owner = headers.find(OwnerHeaderName); + if (owner != headers.end()) + { + ret.Owner = owner->second; + } + auto group = headers.find(GroupHeaderName); + if (group != headers.end()) + { + ret.Group = group->second; + } + auto permissions = headers.find(PermissionsHeaderName); + if (permissions != headers.end()) + { + ret.Permissions = permissions->second; + } + return ret; + } + Azure::Core::Url GetBlobUrlFromUrl(const Azure::Core::Url& url) { std::string host = url.GetHost(); @@ -109,6 +171,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam blobOptions.ApiVersion = options.ApiVersion; blobOptions.CustomerProvidedKey = options.CustomerProvidedKey; blobOptions.EnableTenantDiscovery = options.EnableTenantDiscovery; + blobOptions.Session = options.Session; if (options.Audience.HasValue()) { blobOptions.Audience = Blobs::BlobAudience(options.Audience.Value().ToString()); diff --git a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp index b2d9116e7a..9ebbdb01dd 100644 --- a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp @@ -4,6 +4,7 @@ #pragma once #include "azure/storage/files/datalake/datalake_options.hpp" +#include "azure/storage/files/datalake/datalake_responses.hpp" #include #include @@ -31,6 +32,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam DateTime ParseFileTimeString(const std::string& value); + Models::PathProperties PathPropertiesFromBlobProperties( + Blobs::Models::BlobProperties properties, + const Azure::Core::Http::RawResponse& response); + /** * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. * diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt index e79263f17b..08a345d5b4 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt @@ -16,6 +16,7 @@ SetUpTestProxy("sdk/storage") add_executable ( azure-storage-files-datalake-test + data_locality_test.cpp datalake_directory_client_test.cpp datalake_directory_client_test.hpp datalake_file_client_test.cpp @@ -29,6 +30,7 @@ add_executable ( datalake_service_client_test.cpp datalake_service_client_test.hpp macro_guard.cpp + session_authentication_test.cpp simplified_header_test.cpp # Include shared test source code ${CMAKE_CURRENT_SOURCE_DIR}/../../../azure-storage-common/test/ut/test_base.cpp diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp new file mode 100644 index 0000000000..899b4191e3 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/data_locality_test.cpp @@ -0,0 +1,395 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "test/ut/test_base.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + namespace DataLake = Files::DataLake; + + struct LocalityRequest final + { + std::string Host; + std::string HostHeader; + std::string Range; + std::string IfMatch; + std::string Marker; + bool IsLayout; + }; + + struct LocalityState final + { + std::vector Requests; + bool NoLayout = false; + }; + + class LocalityTransportPolicy final : public Core::Http::Policies::HttpPolicy { + public: + explicit LocalityTransportPolicy(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Clone() const override + { + return std::make_unique(*this); + } + + std::unique_ptr Send( + Core::Http::Request& request, + Core::Http::Policies::NextHttpPolicy, + Core::Context const&) const override + { + const auto& query = request.GetUrl().GetQueryParameters(); + const auto& headers = request.GetHeaders(); + const auto header = [&](const char* name) { + auto value = headers.find(name); + return value == headers.end() ? std::string() : value->second; + }; + const bool isLayout = query.count("comp") && query.at("comp") == "layout"; + const auto marker = query.count("marker") ? query.at("marker") : std::string(); + m_state->Requests.push_back(LocalityRequest{ + request.GetUrl().GetHost(), + header("host"), + header("x-ms-range"), + header("if-match"), + marker, + isLayout}); + + auto response = std::make_unique( + 1, + 1, + isLayout + ? (m_state->NoLayout ? Core::Http::HttpStatusCode::NoContent + : Core::Http::HttpStatusCode::Ok) + : Core::Http::HttpStatusCode::PartialContent, + "OK"); + response->SetHeader("etag", "\"layout-etag\""); + response->SetHeader("last-modified", "Thu, 23 Aug 2001 07:00:00 GMT"); + response->SetHeader("x-ms-creation-time", "Thu, 22 Aug 2002 07:00:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetHeader("x-ms-server-encrypted", "true"); + response->SetHeader("x-ms-blob-content-length", "12"); + response->SetHeader("x-ms-blob-content-type", "application/octet-stream"); + response->SetHeader("x-ms-meta-layout", "metadata"); + response->SetHeader("x-ms-owner", "owner"); + response->SetHeader("x-ms-group", "group"); + response->SetHeader("x-ms-permissions", "rw-r-----"); + response->SetHeader("x-ms-acl", "user::rw-,group::r--,other::---"); + response->SetHeader("x-ms-encryption-context", "context"); + if (isLayout) + { + if (!m_state->NoLayout) + { + const std::string xml = marker.empty() + ? "" + "" + "page2" + : "" + "" + ""; + response->SetBody(std::vector(xml.begin(), xml.end())); + response->SetHeader("content-type", "application/xml"); + } + return response; + } + + const auto range = header("x-ms-range"); + const auto separator = range.find('-'); + const auto offset = std::stoll(range.substr(6, separator - 6)); + const auto end = std::stoll(range.substr(separator + 1)); + static const std::string data = "abcdefghijkl"; + response->SetHeader("content-length", std::to_string(end - offset + 1)); + response->SetHeader( + "content-range", "bytes " + std::to_string(offset) + "-" + std::to_string(end) + "/12"); + response->SetHeader("x-ms-download-hint", "layout"); + response->SetBodyStream(std::make_unique( + reinterpret_cast(data.data() + offset), + static_cast(end - offset + 1))); + return response; + } + + private: + std::shared_ptr m_state; + }; + + DataLake::DataLakeFileClient CreateClient( + const std::shared_ptr& state, + int origin) + { + DataLake::DataLakeClientOptions options; + options.PerRetryPolicies.emplace_back(std::make_unique(state)); + const std::string url = "https://account.dfs.core.windows.net"; + switch (origin) + { + case 0: + return DataLake::DataLakeFileClient(url + "/filesystem/file", options); + case 1: + return DataLake::DataLakeServiceClient(url, options) + .GetFileSystemClient("filesystem") + .GetFileClient("file"); + case 2: + return DataLake::DataLakeFileSystemClient(url + "/filesystem", options) + .GetFileClient("file"); + default: + return DataLake::DataLakeDirectoryClient(url + "/filesystem/directory", options) + .GetFileClient("file"); + } + } + } // namespace + + class DataLakeLocalityRecordedTest : public StorageTest { + }; + + TEST_F(DataLakeLocalityRecordedTest, GetLayoutAndDownloadHint_PLAYBACKONLY_) + { + if (!m_testContext.IsPlaybackMode()) + { + GTEST_SKIP() << "Reuses the Blob multi-endpoint recording."; + } + m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) + + "/../../../azure-storage-blobs/test/ut/recordings"; + m_testContext.RenameTest( + "DataLocalityRecordedTest", "GetLayoutMultipleEndpoints_PLAYBACKONLY_"); + auto options = InitStorageClientOptions(); + auto client = DataLake::DataLakeFileClient( + "https://" + StandardStorageAccountName() + + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", + GetTestCredential(), + options); + auto page = client.GetLayout(); + EXPECT_EQ(page.Layout.Properties.FileSize, 64 * 1024 * 1024); + EXPECT_EQ(page.Layout.Properties.HttpHeaders.ContentType, "application/octet-stream"); + ASSERT_TRUE(page.Layout.Properties.IsServerEncrypted.HasValue()); + EXPECT_TRUE(page.Layout.Properties.IsServerEncrypted.Value()); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + EXPECT_FALSE(page.Layout.Properties.IsDirectory); + std::set endpoints; + std::vector ranges; + for (; page.HasPage(); page.MoveToNextPage()) + { + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + EXPECT_GT(range.Range.Length.Value(), 0); + EXPECT_FALSE(range.Endpoint.empty()); + ranges.push_back(range); + endpoints.insert(range.Endpoint); + } + } + EXPECT_GE(endpoints.size(), 2U); + ASSERT_GT(ranges.size(), 1U); + DataLake::DownloadFileOptions downloadOptions; + downloadOptions.Range = Core::Http::HttpRange(); + downloadOptions.Range.Value().Offset = ranges[1].Range.Offset - 2; + downloadOptions.Range.Value().Length = 5; + auto response = client.Download(downloadOptions); + EXPECT_EQ(response.Value.Body->ReadToEnd().size(), 5U); + EXPECT_EQ(response.Value.FileSize, 64 * 1024 * 1024); + ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + + TEST_F(DataLakeLocalityRecordedTest, GetLayoutRangeReturnsFullFileSize_PLAYBACKONLY_) + { + if (!m_testContext.IsPlaybackMode()) + { + GTEST_SKIP() << "Reuses the Blob ranged layout recording."; + } + m_testContext.RecordingPath = std::string(AZURE_TEST_RECORDING_DIR) + + "/../../../azure-storage-blobs/test/ut/recordings"; + m_testContext.RenameTest( + "DataLocalityRecordedTest", "GetLayoutRangeReturnsFullBlobSize_PLAYBACKONLY_"); + auto clientOptions = InitStorageClientOptions(); + auto client = DataLake::DataLakeFileClient( + "https://" + StandardStorageAccountName() + + ".dfs.core.windows.net/data-locality-test/data-locality-multi-endpoint.bin", + GetTestCredential(), + clientOptions); + constexpr int64_t fileSize = 64 * 1024 * 1024; + for (bool bounded : {true, false}) + { + SCOPED_TRACE(bounded); + DataLake::GetFileLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = bounded ? 8 * 1024 * 1024 - 2 : fileSize - 16; + if (bounded) + { + options.Range.Value().Length = 5; + } + const auto offset = options.Range.Value().Offset; + bool coversOffset = false; + size_t rangeCount = 0; + for (auto page = client.GetLayout(options); page.HasPage(); page.MoveToNextPage()) + { + EXPECT_EQ(page.Layout.Properties.FileSize, fileSize); + EXPECT_GT(page.Layout.Properties.FileSize, options.Range.Value().Length.ValueOr(16)); + EXPECT_TRUE(page.Layout.Properties.ETag.HasValue()); + for (const auto& range : page.Layout.Ranges) + { + ASSERT_TRUE(range.Range.Length.HasValue()); + ASSERT_GT(range.Range.Length.Value(), 0); + ASSERT_GE(range.Range.Offset, 0); + ASSERT_LE(range.Range.Offset, fileSize); + ASSERT_LE(range.Range.Length.Value(), fileSize - range.Range.Offset); + EXPECT_FALSE(range.Endpoint.empty()); + coversOffset = coversOffset + || (range.Range.Offset <= offset + && offset - range.Range.Offset < range.Range.Length.Value()); + ++rangeCount; + } + } + EXPECT_GT(rangeCount, 0U); + EXPECT_TRUE(coversOffset); + } + } + + TEST(DataLakeLocalityTest, GetLayoutMapsPropertiesAndPages) + { + auto state = std::make_shared(); + auto client = CreateClient(state, 0); + DataLake::GetFileLayoutOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 2; + options.Range.Value().Length = 10; + options.AccessConditions.IfMatch = ETag("\"layout-etag\""); + auto page = client.GetLayout(options); + EXPECT_EQ(page.CurrentPageToken, ""); + EXPECT_EQ(page.NextPageToken.Value(), "page2"); + ASSERT_EQ(page.Layout.Ranges.size(), 1U); + EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality0.test:443"); + EXPECT_EQ(page.Layout.Properties.FileSize, 12); + EXPECT_EQ(page.Layout.Properties.Metadata.at("layout"), "metadata"); + EXPECT_EQ(page.Layout.Properties.Owner.Value(), "owner"); + EXPECT_EQ(page.Layout.Properties.Group.Value(), "group"); + EXPECT_EQ(page.Layout.Properties.Permissions.Value(), "rw-r-----"); + EXPECT_EQ(page.Layout.Properties.EncryptionContext.Value(), "context"); + ASSERT_TRUE(page.Layout.Properties.Acls.HasValue()); + EXPECT_EQ(page.Layout.Properties.Acls.Value().size(), 3U); + page.MoveToNextPage(); + EXPECT_EQ(page.CurrentPageToken, "page2"); + ASSERT_EQ(page.Layout.Ranges.size(), 1U); + EXPECT_EQ(page.Layout.Ranges[0].Range.Offset, 6); + EXPECT_EQ(page.Layout.Ranges[0].Range.Length.Value(), 6); + EXPECT_EQ(page.Layout.Ranges[0].Endpoint, "locality1.test:443"); + page.MoveToNextPage(); + EXPECT_FALSE(page.HasPage()); + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Range, "bytes=2-11"); + EXPECT_EQ(state->Requests[1].IfMatch, "\"layout-etag\""); + EXPECT_EQ(state->Requests[1].Marker, "page2"); + } + + TEST(DataLakeLocalityTest, GetLayoutNoContent) + { + auto state = std::make_shared(); + state->NoLayout = true; + auto page = CreateClient(state, 0).GetLayout(); + EXPECT_TRUE(page.Layout.Ranges.empty()); + EXPECT_EQ(page.Layout.Properties.FileSize, 12); + EXPECT_EQ(page.RawResponse->GetStatusCode(), Core::Http::HttpStatusCode::NoContent); + page.MoveToNextPage(); + EXPECT_FALSE(page.HasPage()); + } + + TEST(DataLakeLocalityTest, RoutesOneShotDownloadForAllClientOrigins) + { + for (int origin = 0; origin != 4; ++origin) + { + SCOPED_TRACE(origin); + auto state = std::make_shared(); + auto client = CreateClient(state, origin); + DataLake::DownloadFileOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 6; + options.Range.Value().Length = 1; + options.LayoutEndpoint = "locality1.test:443"; + auto response = client.Download(options); + EXPECT_EQ(response.Value.Body->ReadToEnd(), std::vector{'g'}); + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_EQ(state->Requests[0].Host, "locality1.test"); + EXPECT_EQ(state->Requests[0].HostHeader, "account.blob.core.windows.net"); + ASSERT_TRUE(response.Value.Details.DownloadHint.HasValue()); + EXPECT_EQ(response.Value.Details.DownloadHint.Value(), Blobs::Models::DownloadHint::Layout); + } + } + + TEST(DataLakeLocalityTest, RoutesManagedDownloadsForAllClientOrigins) + { + using Routing = DataLake::LayoutAwareRouting; + for (int origin = 0; origin != 4; ++origin) + { + for (const auto& routing : + {Azure::Nullable(), + Azure::Nullable(Routing::Auto), + Azure::Nullable(Routing::Enabled), + Azure::Nullable(Routing::Disabled)}) + { + for (bool toFile : {false, true}) + { + SCOPED_TRACE(origin); + SCOPED_TRACE(toFile); + SCOPED_TRACE(routing.HasValue() ? static_cast(routing.Value()) : -1); + auto state = std::make_shared(); + auto client = CreateClient(state, origin); + DataLake::DownloadFileToOptions options; + options.Range = Core::Http::HttpRange(); + options.Range.Value().Offset = 5; + options.Range.Value().Length = 7; + EXPECT_EQ(options.LayoutAwareRouting, DataLake::LayoutAwareRouting::Disabled); + if (routing.HasValue()) + { + options.LayoutAwareRouting = routing.Value(); + } + options.TransferOptions.InitialChunkSize = 1; + options.TransferOptions.ChunkSize = 6; + options.TransferOptions.Concurrency = 1; + std::vector content(7); + if (toFile) + { + const auto fileName = Core::Uuid::CreateUuid().ToString() + ".tmp"; + auto response = client.DownloadTo(fileName, options); + std::ifstream file(fileName, std::ios::binary); + content.assign(std::istreambuf_iterator(file), std::istreambuf_iterator()); + file.close(); + std::remove(fileName.c_str()); + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + } + else + { + auto response = client.DownloadTo(content.data(), content.size(), options); + EXPECT_TRUE(response.Value.Details.DownloadHint.HasValue()); + } + EXPECT_EQ(content, (std::vector{'f', 'g', 'h', 'i', 'j', 'k', 'l'})); + if (options.LayoutAwareRouting == DataLake::LayoutAwareRouting::Disabled) + { + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests.back().Host, "account.blob.core.windows.net"); + EXPECT_FALSE(state->Requests.back().IsLayout); + } + else + { + ASSERT_EQ(state->Requests.size(), 4U); + EXPECT_TRUE(state->Requests[1].IsLayout); + EXPECT_TRUE(state->Requests[2].IsLayout); + EXPECT_EQ(state->Requests.back().Host, "locality1.test"); + EXPECT_EQ(state->Requests.back().HostHeader, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests.back().IfMatch, "\"layout-etag\""); + } + } + } + } + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index 5e03711982..c49615ed65 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -26,6 +26,18 @@ namespace Azure { namespace Storage { namespace Blobs { namespace Models { namespace Azure { namespace Storage { namespace Test { + TEST(DataLakeFileClientOptionsTest, DefaultTransferSizes) + { + Files::DataLake::DownloadFileToOptions downloadOptions; + EXPECT_EQ( + downloadOptions.TransferOptions.InitialChunkSize, + downloadOptions.TransferOptions.ChunkSize); + EXPECT_EQ(downloadOptions.TransferOptions.InitialChunkSize, 4_MB); + + Files::DataLake::UploadFileFromOptions uploadOptions; + EXPECT_EQ(uploadOptions.TransferOptions.SingleUploadThreshold, 4_MB); + } + void DataLakeFileClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); @@ -820,7 +832,9 @@ namespace Azure { namespace Storage { namespace Test { { auto fileClient = *m_fileClient; const auto blobContent = RandomBuffer(static_cast(8_MB)); - fileClient.UploadFrom(blobContent.data(), blobContent.size()); + Files::DataLake::UploadFileFromOptions uploadOptions; + uploadOptions.TransferOptions.SingleUploadThreshold = blobContent.size(); + fileClient.UploadFrom(blobContent.data(), blobContent.size(), uploadOptions); auto testDownloadToBuffer = [&](int concurrency, int64_t downloadSize, diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp new file mode 100644 index 0000000000..8dc10f5840 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/session_authentication_test.cpp @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "datalake_file_client_test.hpp" + +#include + +#include +#include +#include +#include + +#include + +namespace Azure { namespace Storage { namespace Test { + + namespace { + struct SessionRequest final + { + Azure::Core::Http::HttpMethod Method; + std::string Host; + std::string Authorization; + std::string Component; + }; + + struct SessionTestState final + { + std::vector Requests; + std::list> ResponseBodies; + }; + + std::unique_ptr CreateSessionRequestCountingPolicy( + std::shared_ptr> sessionRequestCount) + { + return std::make_unique( + [sessionRequestCount + = std::move(sessionRequestCount)](const Azure::Core::Http::Request& request) { + const auto query = request.GetUrl().GetQueryParameters(); + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post + && query.count("comp") != 0 && query.at("comp") == "session") + { + ++*sessionRequestCount; + } + }); + } + + class SessionTestCredential final : public Azure::Core::Credentials::TokenCredential { + public: + SessionTestCredential() : TokenCredential("SessionTestCredential") {} + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext&, + const Azure::Core::Context&) const override + { + return {"bearer-token", Azure::DateTime::clock::now() + std::chrono::hours(1)}; + } + }; + + class CountingTokenCredential final : public Azure::Core::Credentials::TokenCredential { + public: + explicit CountingTokenCredential( + std::shared_ptr credential) + : TokenCredential("CountingTokenCredential"), m_credential(std::move(credential)) + { + } + + Azure::Core::Credentials::AccessToken GetToken( + const Azure::Core::Credentials::TokenRequestContext& tokenRequestContext, + const Azure::Core::Context& context) const override + { + ++m_getTokenCount; + return m_credential->GetToken(tokenRequestContext, context); + } + + int GetTokenCount() const { return m_getTokenCount.load(); } + + private: + std::shared_ptr m_credential; + mutable std::atomic m_getTokenCount{0}; + }; + + class SessionTestTransport final : public Azure::Core::Http::HttpTransport { + public: + explicit SessionTestTransport(std::shared_ptr state) + : m_state(std::move(state)) + { + } + + std::unique_ptr Send( + Azure::Core::Http::Request& request, + Azure::Core::Context const&) override + { + std::lock_guard lock(m_mutex); + const auto authorization = request.GetHeader("Authorization"); + const auto query = request.GetUrl().GetQueryParameters(); + const auto component = query.find("comp"); + m_state->Requests.push_back( + {request.GetMethod(), + request.GetUrl().GetHost(), + authorization.HasValue() ? authorization.Value() : std::string(), + component == query.end() ? std::string() : component->second}); + + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Post && component != query.end() + && component->second == "session") + { + const auto key = Azure::Core::Convert::Base64Encode( + std::vector{'s', 'e', 's', 's', 'i', 'o', 'n'}); + const std::string body = "id" + "2099-01-01T00:00:00Z" + "HMAC" + "session-token" + + key + ""; + return CreateResponse( + Azure::Core::Http::HttpStatusCode::Created, + "Created", + std::vector(body.begin(), body.end())); + } + + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Put) + { + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Created, "Created"); + response->SetHeader("ETag", "\"etag\""); + response->SetHeader("Last-Modified", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-request-server-encrypted", "true"); + return response; + } + + auto response = CreateResponse(Azure::Core::Http::HttpStatusCode::Ok, "OK"); + response->SetHeader("Content-Length", "0"); + response->SetHeader("x-ms-creation-time", "Mon, 01 Jan 2024 00:00:00 GMT"); + response->SetHeader("x-ms-server-encrypted", "true"); + return response; + } + + private: + std::unique_ptr CreateResponse( + Azure::Core::Http::HttpStatusCode statusCode, + const std::string& reasonPhrase, + std::vector body = {}) + { + m_state->ResponseBodies.emplace_back(std::move(body)); + auto response + = std::make_unique(1, 1, statusCode, reasonPhrase); + response->SetBodyStream( + std::make_unique(m_state->ResponseBodies.back())); + return response; + } + + std::shared_ptr m_state; + std::mutex m_mutex; + }; + + Files::DataLake::DataLakeClientOptions CreateSessionOptions( + const std::shared_ptr& state) + { + Files::DataLake::DataLakeClientOptions options; + options.Transport.Transport = std::make_shared(state); + options.Session.Mode = Blobs::SessionMode::Enabled; + options.Session.AccountName = "account"; + return options; + } + } // namespace + + TEST(DataLakeSessionAuthenticationTest, DownloadUsesBlobSessionPipeline) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileClient client( + "https://account.dfs.core.windows.net/filesystem/file", + std::make_shared(), + CreateSessionOptions(state)); + + EXPECT_NO_THROW(client.Download()); + + ASSERT_EQ(state->Requests.size(), 2U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Post); + EXPECT_EQ(state->Requests[0].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[0].Component, "session"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + EXPECT_EQ(state->Requests[1].Method, Azure::Core::Http::HttpMethod::Get); + EXPECT_EQ(state->Requests[1].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Authorization.find("Session "), 0U); + } + + TEST(DataLakeSessionAuthenticationTest, DfsOperationsUseBearer) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileClient client( + "https://account.dfs.core.windows.net/filesystem/file", + std::make_shared(), + CreateSessionOptions(state)); + + EXPECT_NO_THROW(client.Create()); + + ASSERT_EQ(state->Requests.size(), 1U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Put); + EXPECT_EQ(state->Requests[0].Host, "account.dfs.core.windows.net"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + } + + TEST(DataLakeSessionAuthenticationTest, RenamedClientRetainsBlobSessionPipeline) + { + auto state = std::make_shared(); + Files::DataLake::DataLakeFileSystemClient client( + "https://account.dfs.core.windows.net/filesystem", + std::make_shared(), + CreateSessionOptions(state)); + + auto renamedClient = client.RenameFile("source", "destination").Value; + EXPECT_NO_THROW(renamedClient.Download()); + + ASSERT_EQ(state->Requests.size(), 3U); + EXPECT_EQ(state->Requests[0].Method, Azure::Core::Http::HttpMethod::Put); + EXPECT_EQ(state->Requests[0].Host, "account.dfs.core.windows.net"); + EXPECT_EQ(state->Requests[0].Authorization.find("Bearer "), 0U); + EXPECT_EQ(state->Requests[1].Method, Azure::Core::Http::HttpMethod::Post); + EXPECT_EQ(state->Requests[1].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[1].Component, "session"); + EXPECT_EQ(state->Requests[2].Method, Azure::Core::Http::HttpMethod::Get); + EXPECT_EQ(state->Requests[2].Host, "account.blob.core.windows.net"); + EXPECT_EQ(state->Requests[2].Authorization.find("Session "), 0U); + } + + class DataLakeSessionAuthenticationRecordedTest : public DataLakeFileClientTest { + }; + + TEST_F(DataLakeSessionAuthenticationRecordedTest, DownloadUsesSessionAuthentication_PLAYBACKONLY_) + { + auto options = InitStorageClientOptions(); + options.Session.Mode = Blobs::SessionMode::Enabled; + auto sessionRequestCount = std::make_shared>(0); + options.PerRetryPolicies.emplace_back(CreateSessionRequestCountingPolicy(sessionRequestCount)); + auto credential = std::make_shared(GetTestCredential()); + Files::DataLake::DataLakeFileClient client(m_fileClient->GetUrl(), credential, options); + + EXPECT_NO_THROW(client.Download().Value.Body->ReadToEnd()); + EXPECT_NO_THROW(client.Download().Value.Body->ReadToEnd()); + EXPECT_EQ(sessionRequestCount->load(), 1); + EXPECT_EQ(credential->GetTokenCount(), 1); + EXPECT_NO_THROW(client.GetProperties()); + EXPECT_EQ(sessionRequestCount->load(), 1); + EXPECT_EQ(credential->GetTokenCount(), 2); + } + +}}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-shares/CHANGELOG.md b/sdk/storage/azure-storage-files-shares/CHANGELOG.md index ac3dc98246..c4d594b350 100644 --- a/sdk/storage/azure-storage-files-shares/CHANGELOG.md +++ b/sdk/storage/azure-storage-files-shares/CHANGELOG.md @@ -4,6 +4,13 @@ ### Features Added +- Bumped up API version to `2027-03-07`. +- Added the share creation time to `ShareClient::GetProperties` and + `ShareServiceClient::ListShares` responses. +- Added support for enabling and configuring Files Change Feed when creating or updating a share. +- Added support for addressing files and directories by file ID, retrieving their properties, and + listing a file's hard links. + ### Breaking Changes ### Bugs Fixed diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp index 96bc601993..0be9e9c959 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/rest_client.hpp @@ -32,7 +32,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2026-10-06"; + constexpr static const char* ApiVersion = "2027-03-07"; } // namespace _detail namespace Models { /** @@ -330,6 +330,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The date and time the share was last modified. */ DateTime LastModified; + /** + * The date and time the share was created. + */ + Nullable CreatedOn; /** * The ETag contains a value which represents the version of the share, in quotes. */ @@ -637,6 +641,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * the last modified time of the share. */ DateTime LastModified; + /** + * Returns the date and time the share was created. + */ + Nullable CreatedOn; /** * Returns the current share quota in GB. */ @@ -726,6 +734,18 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * Returns the current share next allowed provisioned bandwidth downgrade time. */ Nullable NextAllowedProvisionedBandwidthDowngradeTime; + /** + * Specifies whether change feed is enabled on the share. + */ + Nullable IsChangeFeedEnabled; + /** + * The number of days that change feed records are retained. + */ + Nullable ChangeFeedRetentionInDays; + /** + * The name of the blob container where change feed records are stored. + */ + Nullable ChangeFeedBlobContainerName; }; /** * @brief Specifies the option include to delete the base share and all of its snapshots. @@ -1200,7 +1220,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { AZ_STORAGE_FILES_SHARES_DLLEXPORT const static FilePropertySemantics Restore; }; /** - * @brief NFS only. Type of the file or directory. + * @brief Type of the file. */ class NfsFileType final : public Core::_internal::ExtendableEnumeration { public: @@ -1307,6 +1327,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * NFS only. Type of the file or directory. */ Nullable NfsFileType; + /** + * The name of the directory. + */ + Nullable FileName; }; } // namespace _detail /** @@ -1978,6 +2002,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * NFS only. Type of the file or directory. */ Nullable NfsFileType; + /** + * The name of the file. + */ + Nullable FileName; }; } // namespace _detail /** @@ -2499,6 +2527,34 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ Models::NfsFileType NfsFileType; }; + /** + * @brief A hard link to a file. + */ + struct HardLink final + { + /** + * The name of the hard link. + */ + StringEncoded FileName; + /** + * The fileId of the parent directory of the hard link. + */ + std::string ParentId; + }; + /** + * @brief Response type for #Azure::Storage::Files::Shares::FileClient::GetFileLinks. + */ + struct GetFileLinksResult final + { + /** + * Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetProperties. + */ + FileProperties Properties; + /** + * Array of HardLink. + */ + std::vector HardLinks; + }; } // namespace _detail } // namespace Models namespace _detail { @@ -2562,6 +2618,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { Nullable FileRequestIntent; Nullable ShareProvisionedIops; Nullable ShareProvisionedBandwidthMibps; + Nullable EnableChangeFeed; + Nullable ChangeFeedRetentionInDays; }; static Response Create( Core::Http::_internal::HttpPipeline& pipeline, @@ -2693,6 +2751,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { Nullable FileRequestIntent; Nullable ShareProvisionedIops; Nullable ShareProvisionedBandwidthMibps; + Nullable EnableChangeFeed; + Nullable ChangeFeedRetentionInDays; }; static Response SetProperties( Core::Http::_internal::HttpPipeline& pipeline, @@ -3232,6 +3292,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Core::Url& url, const CreateFileHardLinkOptions& options, const Core::Context& context); + struct GetFileFileLinksOptions final + { + Nullable LeaseId; + Nullable AllowTrailingDot; + Nullable FileRequestIntent; + }; + static Response GetFileLinks( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetFileFileLinksOptions& options, + const Core::Context& context); }; } // namespace _detail }}}} // namespace Azure::Storage::Files::Shares \ No newline at end of file diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp index 37a6d0ae32..abb599d13c 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_client.hpp @@ -91,6 +91,20 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ ShareDirectoryClient GetRootDirectoryClient() const; + /** + * @brief Creates a file client that addresses a file by its file ID. + * @param fileId The file ID of the file. + * @return A client that addresses the file by its file ID. + */ + ShareFileClient GetFileClientByFileId(const std::string& fileId) const; + + /** + * @brief Creates a directory client that addresses a directory by its file ID. + * @param fileId The file ID of the directory. + * @return A client that addresses the directory by its file ID. + */ + ShareDirectoryClient GetDirectoryClientByFileId(const std::string& fileId) const; + /** * @brief Creates the file share. * @param options Optional parameters to create this file share. diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp index 73e0d67863..02409362f4 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_constants.hpp @@ -14,6 +14,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { namespace _detail { constexpr static const char* ShareSnapshotQueryParameter = "sharesnapshot"; + constexpr static const char* FileIdQueryParameter = "fileid"; // Error codes: constexpr static const char* ParentNotFound = "ParentNotFound"; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp index 4e22eed048..d85e027c58 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_directory_client.hpp @@ -78,6 +78,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ std::string GetUrl() const { return m_shareDirectoryUrl.GetAbsoluteUrl(); } + /** + * @brief Gets the file ID when this client addresses a directory by ID. + * + * @return The file ID, or an empty string when this client addresses a directory by path. + */ + std::string GetFileId() const; + /** * @brief Create a ShareDirectoryClient that's a sub directory of the current * ShareDirectoryClient @@ -274,17 +281,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { private: Azure::Core::Url m_shareDirectoryUrl; + Azure::Nullable m_fileId; std::shared_ptr m_pipeline; _detail::ShareClientConfiguration m_clientConfiguration; explicit ShareDirectoryClient( Azure::Core::Url shareDirectoryUrl, std::shared_ptr pipeline, - _detail::ShareClientConfiguration clientConfiguration) - : m_shareDirectoryUrl(std::move(shareDirectoryUrl)), m_pipeline(std::move(pipeline)), - m_clientConfiguration(std::move(clientConfiguration)) - { - } + _detail::ShareClientConfiguration clientConfiguration); friend class ShareClient; }; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp index 3c0b9a0607..f6ad203f6f 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_file_client.hpp @@ -77,6 +77,13 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { */ std::string GetUrl() const { return m_shareFileUrl.GetAbsoluteUrl(); } + /** + * @brief Gets the file ID when this client addresses a file by ID. + * + * @return The file ID, or an empty string when this client addresses a file by path. + */ + std::string GetFileId() const; + /** * @brief Initializes a new instance of the ShareFileClient class with an identical URL * source but the specified share snapshot timestamp. @@ -446,19 +453,26 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateHardLinkOptions& options = CreateHardLinkOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Returns the hard links and properties of a file addressed by file ID. + * @param options Optional parameters for getting the file links. + * @param context Context for cancelling the operation. + * @return The file properties and hard links. + */ + Azure::Response GetFileLinks( + const GetFileLinksOptions& options = GetFileLinksOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: Azure::Core::Url m_shareFileUrl; + Azure::Nullable m_fileId; std::shared_ptr m_pipeline; _detail::ShareClientConfiguration m_clientConfiguration; explicit ShareFileClient( Azure::Core::Url shareFileUrl, std::shared_ptr pipeline, - _detail::ShareClientConfiguration clientConfiguration) - : m_shareFileUrl(std::move(shareFileUrl)), m_pipeline(std::move(pipeline)), - m_clientConfiguration(std::move(clientConfiguration)) - { - } + _detail::ShareClientConfiguration clientConfiguration); friend class ShareClient; friend class ShareDirectoryClient; diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp index f1138235f6..972eab2bda 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_options.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -272,6 +273,9 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { }; namespace _detail { + Azure::Nullable GetFileIdFromUrl(const Azure::Core::Url& url); + void AssertPathAddressed(const Azure::Nullable& fileId, const char* operationName); + struct ShareClientConfiguration { /** @@ -491,6 +495,18 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * throughput is 60 MiB/sec and maximum is 5,125 MiB/sec. */ Nullable ProvisionedMaxBandwidthMibps; + + /** + * Optional. Version 2026-06-06 and newer. Specifies whether change feed is enabled on the + * share. + */ + Nullable EnableChangeFeed; + + /** + * Optional. Version 2026-06-06 and newer. Specifies the number of days that change feed + * records are retained. Valid values are between 1 and 365. The default is 7 days. + */ + Nullable ChangeFeedRetentionInDays; }; /** @@ -581,6 +597,19 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * bandwidth is 60 MiB/sec and max is 5,120 MiB/sec. */ Nullable ProvisionedMaxBandwidthMibps; + + /** + * Optional. Version 2026-06-06 and newer. Specifies whether change feed is enabled on the + * share. + */ + Nullable EnableChangeFeed; + + /** + * Optional. Version 2026-06-06 and newer. Specifies the number of days that change feed + * records are retained. Valid values are between 1 and 365. Once change feed is enabled, this + * value can be updated independently. + */ + Nullable ChangeFeedRetentionInDays; }; /** @@ -1155,6 +1184,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { LeaseAccessConditions AccessConditions; }; + /** + * @brief Optional parameters for #Azure::Storage::Files::Shares::ShareFileClient::GetFileLinks. + */ + struct GetFileLinksOptions final + { + /** + * The operation will only succeed if the access condition is met. + */ + LeaseAccessConditions AccessConditions; + }; + /** * @brief Optional parameters for #Azure::Storage::Files::Shares::ShareFileClient::SetProperties. */ diff --git a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp index 5b6ff648c6..39255b6919 100644 --- a/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp +++ b/sdk/storage/azure-storage-files-shares/inc/azure/storage/files/shares/share_responses.hpp @@ -526,6 +526,40 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The NFS related properties for the file. */ FilePosixProperties PosixProperties; + /** + * The file name. This is populated when the file is addressed by file ID. + */ + Nullable FileName; + }; + + /** + * @brief A hard link to a file. + */ + struct FileLink final + { + /** + * The name of the hard link. + */ + std::string Name; + /** + * The file ID of the hard link's parent directory. + */ + std::string ParentId; + }; + + /** + * @brief Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetFileLinks. + */ + struct GetFileLinksResult final + { + /** + * The properties of the file. + */ + FileProperties Properties; + /** + * The hard links of the file. + */ + std::vector Links; }; /** @@ -737,6 +771,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { * The NFS related properties for the file. */ FilePosixProperties PosixProperties; + /** + * The directory name. This is populated when the directory is addressed by file ID. + */ + Nullable FileName; }; /** diff --git a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp index e5fee88cc6..859b8d94c8 100644 --- a/sdk/storage/azure-storage-files-shares/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/rest_client.cpp @@ -378,7 +378,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -403,7 +403,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -698,7 +698,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { _internal::UrlEncodeQueryParameter( ListSharesIncludeFlagsToString(options.Include.Value()))); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -731,6 +731,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { kMetadata, kProperties, kLastModified, + kCreationTime, kEtag, kQuota, kProvisionedIops, @@ -772,6 +773,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { {"Metadata", XmlTagEnum::kMetadata}, {"Properties", XmlTagEnum::kProperties}, {"Last-Modified", XmlTagEnum::kLastModified}, + {"Creation-Time", XmlTagEnum::kCreationTime}, {"Etag", XmlTagEnum::kEtag}, {"Quota", XmlTagEnum::kQuota}, {"ProvisionedIops", XmlTagEnum::kProvisionedIops}, @@ -886,6 +888,14 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { vectorElement1.Details.LastModified = DateTime::Parse(node.Value, Azure::DateTime::DateFormat::Rfc1123); } + else if ( + xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kShares && xmlPath[2] == XmlTagEnum::kShare + && xmlPath[3] == XmlTagEnum::kProperties && xmlPath[4] == XmlTagEnum::kCreationTime) + { + vectorElement1.Details.CreatedOn + = DateTime::Parse(node.Value, Azure::DateTime::DateFormat::Rfc1123); + } else if ( xmlPath.size() == 5 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kShares && xmlPath[2] == XmlTagEnum::kShare @@ -1157,7 +1167,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "service"); request.GetUrl().AppendQueryParameter("comp", "userdelegationkey"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -1291,7 +1301,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-access-tier", options.AccessTier.Value().ToString()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.EnabledProtocols.HasValue() && !options.EnabledProtocols.Value().ToString().empty()) { @@ -1341,6 +1351,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-provisioned-bandwidth-mibps", std::to_string(options.ShareProvisionedBandwidthMibps.Value())); } + if (options.EnableChangeFeed.HasValue()) + { + request.SetHeader( + "x-ms-file-enable-change-feed", options.EnableChangeFeed.Value() ? "true" : "false"); + } + if (options.ChangeFeedRetentionInDays.HasValue()) + { + request.SetHeader( + "x-ms-file-change-feed-retention-in-days", + std::to_string(options.ChangeFeedRetentionInDays.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -1390,7 +1411,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -1416,6 +1437,12 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); response.LastModified = DateTime::Parse( pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + if (pRawResponse->GetHeaders().count("x-ms-share-creation-time") != 0) + { + response.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-share-creation-time"), + Azure::DateTime::DateFormat::Rfc1123); + } response.Quota = std::stoll(pRawResponse->GetHeaders().at("x-ms-share-quota")); if (pRawResponse->GetHeaders().count("x-ms-share-provisioned-iops") != 0) { @@ -1532,6 +1559,16 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-next-allowed-provisioned-bandwidth-downgrade-time"), Azure::DateTime::DateFormat::Rfc1123); } + if (pRawResponse->GetHeaders().count("x-ms-file-change-feed-retention-in-days") != 0) + { + response.ChangeFeedRetentionInDays + = std::stoi(pRawResponse->GetHeaders().at("x-ms-file-change-feed-retention-in-days")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-blob-container-for-xfiles-change-feed") != 0) + { + response.ChangeFeedBlobContainerName + = pRawResponse->GetHeaders().at("x-ms-file-blob-container-for-xfiles-change-feed"); + } return Response(std::move(response), std::move(pRawResponse)); } Response ShareClient::Delete( @@ -1547,7 +1584,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.DeleteSnapshots.HasValue() && !options.DeleteSnapshots.Value().ToString().empty()) { request.SetHeader("x-ms-delete-snapshots", options.DeleteSnapshots.Value().ToString()); @@ -1598,7 +1635,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1637,7 +1674,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1679,7 +1716,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1718,7 +1755,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1761,7 +1798,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Sharesnapshot.HasValue() && !options.Sharesnapshot.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -1799,7 +1836,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1842,7 +1879,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "filepermission"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1878,7 +1915,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-file-permission-format", options.FilePermissionFormat.Value().ToString()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -1913,7 +1950,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Quota.HasValue()) { request.SetHeader("x-ms-share-quota", std::to_string(options.Quota.Value())); @@ -1970,6 +2007,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-share-provisioned-bandwidth-mibps", std::to_string(options.ShareProvisionedBandwidthMibps.Value())); } + if (options.EnableChangeFeed.HasValue()) + { + request.SetHeader( + "x-ms-file-enable-change-feed", options.EnableChangeFeed.Value() ? "true" : "false"); + } + if (options.ChangeFeedRetentionInDays.HasValue()) + { + request.SetHeader( + "x-ms-file-change-feed-retention-in-days", + std::to_string(options.ChangeFeedRetentionInDays.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2044,7 +2092,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2075,7 +2123,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "acl"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2227,7 +2275,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader("Content-Length", std::to_string(requestBody.Length())); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "acl"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2259,7 +2307,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); request.GetUrl().AppendQueryParameter("restype", "share"); request.GetUrl().AppendQueryParameter("comp", "stats"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -2350,7 +2398,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FilePermission.HasValue() && !options.FilePermission.Value().empty()) { request.SetHeader("x-ms-file-permission", options.FilePermission.Value()); @@ -2484,7 +2532,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -2555,6 +2603,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.NfsFileType = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } return Response( std::move(response), std::move(pRawResponse)); } @@ -2571,7 +2623,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -2595,7 +2647,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "directory"); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FilePermission.HasValue() && !options.FilePermission.Value().empty()) { request.SetHeader("x-ms-file-permission", options.FilePermission.Value()); @@ -2718,7 +2770,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -2772,7 +2824,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "maxresults", std::to_string(options.MaxResults.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Include.HasValue() && !ListFilesIncludeFlagsToString(options.Include.Value()).empty()) { @@ -3165,7 +3217,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-recursive", options.Recursive.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -3378,7 +3430,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-recursive", options.Recursive.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -3416,7 +3468,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "directory"); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (!options.RenameSource.empty()) { request.SetHeader("x-ms-file-rename-source", options.RenameSource); @@ -3531,7 +3583,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); request.SetHeader("x-ms-content-length", std::to_string(options.FileContentLength)); request.SetHeader("x-ms-type", "file"); if (options.FileContentType.HasValue() && !options.FileContentType.Value().empty()) @@ -3720,7 +3772,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Range.HasValue() && !options.Range.Value().empty()) { request.SetHeader("x-ms-range", options.Range.Value()); @@ -3929,7 +3981,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4077,6 +4129,10 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { response.NfsFileType = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } return Response( std::move(response), std::move(pRawResponse)); } @@ -4092,7 +4148,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.SetHeader( "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4123,7 +4179,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "properties"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileContentLength.HasValue()) { request.SetHeader("x-ms-content-length", std::to_string(options.FileContentLength.Value())); @@ -4284,7 +4340,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-meta-" + p.first, p.second); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4328,7 +4384,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4366,7 +4422,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4407,7 +4463,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4445,7 +4501,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -4492,7 +4548,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("Content-MD5", Core::Convert::Base64Encode(options.ContentMD5.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4589,7 +4645,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "x-ms-source-if-none-match-crc64", Core::Convert::Base64Encode(options.SourceIfNoneMatchCrc64.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4665,7 +4721,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "prevsharesnapshot", _internal::UrlEncodeQueryParameter(options.Prevsharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.Range.HasValue() && !options.Range.Value().empty()) { request.SetHeader("x-ms-range", options.Range.Value()); @@ -4813,7 +4869,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Core::Context& context) { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); for (const auto& p : options.Metadata) { request.SetHeader("x-ms-meta-" + p.first, p.second); @@ -4940,7 +4996,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "copyid", _internal::UrlEncodeQueryParameter(options.CopyId)); } request.SetHeader("x-ms-copy-action", "abort"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); @@ -4987,7 +5043,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -5196,7 +5252,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { request.SetHeader("x-ms-handle-id", options.HandleId); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.AllowTrailingDot.HasValue()) { request.SetHeader( @@ -5233,7 +5289,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("comp", "rename"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (!options.RenameSource.empty()) { request.SetHeader("x-ms-file-rename-source", options.RenameSource); @@ -5347,7 +5403,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "symboliclink"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); for (const auto& p : options.Metadata) { request.SetHeader("x-ms-meta-" + p.first, p.second); @@ -5432,7 +5488,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { request.GetUrl().AppendQueryParameter( "sharesnapshot", _internal::UrlEncodeQueryParameter(options.Sharesnapshot.Value())); } - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); if (options.FileRequestIntent.HasValue() && !options.FileRequestIntent.Value().ToString().empty()) { @@ -5460,7 +5516,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { { auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); request.GetUrl().AppendQueryParameter("restype", "hardlink"); - request.SetHeader("x-ms-version", "2026-10-06"); + request.SetHeader("x-ms-version", "2027-03-07"); request.SetHeader("x-ms-type", "file"); if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) { @@ -5514,5 +5570,249 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Response( std::move(response), std::move(pRawResponse)); } + Response FileClient::GetFileLinks( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const GetFileFileLinksOptions& options, + const Core::Context& context) + { + auto request = Core::Http::Request(Core::Http::HttpMethod::Get, url); + request.GetUrl().AppendQueryParameter("comp", "hardlinks"); + request.SetHeader("x-ms-version", "2027-03-07"); + if (options.LeaseId.HasValue() && !options.LeaseId.Value().empty()) + { + request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); + } + if (options.AllowTrailingDot.HasValue()) + { + request.SetHeader( + "x-ms-allow-trailing-dot", options.AllowTrailingDot.Value() ? "true" : "false"); + } + if (options.FileRequestIntent.HasValue() + && !options.FileRequestIntent.Value().ToString().empty()) + { + request.SetHeader("x-ms-file-request-intent", options.FileRequestIntent.Value().ToString()); + } + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (httpStatusCode != Core::Http::HttpStatusCode::Ok) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::_detail::GetFileLinksResult response; + { + const auto& responseBody = pRawResponse->GetBody(); + _internal::XmlReader reader( + reinterpret_cast(responseBody.data()), responseBody.size()); + enum class XmlTagEnum + { + kUnknown, + kHardLinks, + kHardLink, + kFileName, + kParentId, + }; + const std::unordered_map XmlTagEnumMap{ + {"HardLinks", XmlTagEnum::kHardLinks}, + {"HardLink", XmlTagEnum::kHardLink}, + {"FileName", XmlTagEnum::kFileName}, + {"ParentId", XmlTagEnum::kParentId}, + }; + std::vector xmlPath; + Models::_detail::HardLink vectorElement1; + while (true) + { + auto node = reader.Read(); + if (node.Type == _internal::XmlNodeType::End) + { + break; + } + else if (node.Type == _internal::XmlNodeType::StartTag) + { + auto ite = XmlTagEnumMap.find(node.Name); + xmlPath.push_back(ite == XmlTagEnumMap.end() ? XmlTagEnum::kUnknown : ite->second); + } + else if (node.Type == _internal::XmlNodeType::Text) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kFileName) + { + vectorElement1.FileName.Content = node.Value; + } + else if ( + xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kParentId) + { + vectorElement1.ParentId = node.Value; + } + } + else if (node.Type == _internal::XmlNodeType::Attribute) + { + if (xmlPath.size() == 3 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink && xmlPath[2] == XmlTagEnum::kFileName + && node.Name == "Encoded") + { + vectorElement1.FileName.Encoded = node.Value == std::string("true"); + } + } + else if (node.Type == _internal::XmlNodeType::EndTag) + { + if (xmlPath.size() == 2 && xmlPath[0] == XmlTagEnum::kHardLinks + && xmlPath[1] == XmlTagEnum::kHardLink) + { + response.HardLinks.push_back(std::move(vectorElement1)); + vectorElement1 = Models::_detail::HardLink(); + } + xmlPath.pop_back(); + } + } + } + response.Properties.LastModified = DateTime::Parse( + pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); + for (auto i = pRawResponse->GetHeaders().lower_bound("x-ms-meta-"); + i != pRawResponse->GetHeaders().end() && i->first.substr(0, 10) == "x-ms-meta-"; + ++i) + { + response.Properties.Metadata.emplace(i->first.substr(10), i->second); + } + response.Properties.FileSize + = std::stoll(pRawResponse->GetHeaders().at("x-ms-content-length")); + if (pRawResponse->GetHeaders().count("x-ms-content-type") != 0) + { + response.Properties.HttpHeaders.ContentType + = pRawResponse->GetHeaders().at("x-ms-content-type"); + } + response.Properties.ETag = ETag(pRawResponse->GetHeaders().at("ETag")); + if (pRawResponse->GetHeaders().count("x-ms-content-md5") != 0) + { + response.Properties.HttpHeaders.ContentHash.Value + = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-md5")); + response.Properties.HttpHeaders.ContentHash.Algorithm = HashAlgorithm::Md5; + } + if (pRawResponse->GetHeaders().count("x-ms-content-encoding") != 0) + { + response.Properties.HttpHeaders.ContentEncoding + = pRawResponse->GetHeaders().at("x-ms-content-encoding"); + } + if (pRawResponse->GetHeaders().count("x-ms-cache-control") != 0) + { + response.Properties.HttpHeaders.CacheControl + = pRawResponse->GetHeaders().at("x-ms-cache-control"); + } + if (pRawResponse->GetHeaders().count("x-ms-content-disposition") != 0) + { + response.Properties.HttpHeaders.ContentDisposition + = pRawResponse->GetHeaders().at("x-ms-content-disposition"); + } + if (pRawResponse->GetHeaders().count("x-ms-content-language") != 0) + { + response.Properties.HttpHeaders.ContentLanguage + = pRawResponse->GetHeaders().at("x-ms-content-language"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-completion-time") != 0) + { + response.Properties.CopyCompletedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-copy-completion-time"), + Azure::DateTime::DateFormat::Rfc1123); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status-description") != 0) + { + response.Properties.CopyStatusDescription + = pRawResponse->GetHeaders().at("x-ms-copy-status-description"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-id") != 0) + { + response.Properties.CopyId = pRawResponse->GetHeaders().at("x-ms-copy-id"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-progress") != 0) + { + response.Properties.CopyProgress = pRawResponse->GetHeaders().at("x-ms-copy-progress"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-source") != 0) + { + response.Properties.CopySource = pRawResponse->GetHeaders().at("x-ms-copy-source"); + } + if (pRawResponse->GetHeaders().count("x-ms-copy-status") != 0) + { + response.Properties.CopyStatus + = Models::CopyStatus(pRawResponse->GetHeaders().at("x-ms-copy-status")); + } + response.Properties.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); + if (pRawResponse->GetHeaders().count("x-ms-file-attributes") != 0) + { + response.Properties.SmbProperties.Attributes + = Models::FileAttributes(pRawResponse->GetHeaders().at("x-ms-file-attributes")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-creation-time") != 0) + { + response.Properties.SmbProperties.CreatedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-creation-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-last-write-time") != 0) + { + response.Properties.SmbProperties.LastWrittenOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-last-write-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-change-time") != 0) + { + response.Properties.SmbProperties.ChangedOn = DateTime::Parse( + pRawResponse->GetHeaders().at("x-ms-file-change-time"), + Azure::DateTime::DateFormat::Rfc3339); + } + if (pRawResponse->GetHeaders().count("x-ms-file-permission-key") != 0) + { + response.Properties.SmbProperties.PermissionKey + = pRawResponse->GetHeaders().at("x-ms-file-permission-key"); + } + response.Properties.SmbProperties.FileId = pRawResponse->GetHeaders().at("x-ms-file-id"); + response.Properties.SmbProperties.ParentFileId + = pRawResponse->GetHeaders().at("x-ms-file-parent-id"); + if (pRawResponse->GetHeaders().count("x-ms-lease-duration") != 0) + { + response.Properties.LeaseDuration + = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0) + { + response.Properties.LeaseState + = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0) + { + response.Properties.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + } + if (pRawResponse->GetHeaders().count("x-ms-mode") != 0) + { + response.Properties.FileMode = pRawResponse->GetHeaders().at("x-ms-mode"); + } + if (pRawResponse->GetHeaders().count("x-ms-owner") != 0) + { + response.Properties.Owner = pRawResponse->GetHeaders().at("x-ms-owner"); + } + if (pRawResponse->GetHeaders().count("x-ms-group") != 0) + { + response.Properties.Group = pRawResponse->GetHeaders().at("x-ms-group"); + } + if (pRawResponse->GetHeaders().count("x-ms-link-count") != 0) + { + response.Properties.LinkCount + = std::stoll(pRawResponse->GetHeaders().at("x-ms-link-count")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-file-type") != 0) + { + response.Properties.NfsFileType + = Models::NfsFileType(pRawResponse->GetHeaders().at("x-ms-file-file-type")); + } + if (pRawResponse->GetHeaders().count("x-ms-file-name") != 0) + { + response.Properties.FileName = pRawResponse->GetHeaders().at("x-ms-file-name"); + } + return Response( + std::move(response), std::move(pRawResponse)); + } } // namespace _detail }}}} // namespace Azure::Storage::Files::Shares \ No newline at end of file diff --git a/sdk/storage/azure-storage-files-shares/src/share_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_client.cpp index 00ae769c8e..d1f7c2c738 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_client.cpp @@ -3,12 +3,14 @@ #include "azure/storage/files/shares/share_client.hpp" +#include "azure/storage/files/shares/share_constants.hpp" #include "azure/storage/files/shares/share_directory_client.hpp" #include "azure/storage/files/shares/share_file_client.hpp" #include "private/package_version.hpp" #include #include +#include #include #include #include @@ -116,6 +118,30 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return ShareDirectoryClient(m_shareUrl, m_pipeline, m_clientConfiguration); } + ShareFileClient ShareClient::GetFileClientByFileId(const std::string& fileId) const + { + if (fileId.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + Core::Url fileUrl(m_shareUrl); + fileUrl.AppendQueryParameter( + _detail::FileIdQueryParameter, _internal::UrlEncodeQueryParameter(fileId)); + return ShareFileClient(std::move(fileUrl), m_pipeline, m_clientConfiguration); + } + + ShareDirectoryClient ShareClient::GetDirectoryClientByFileId(const std::string& fileId) const + { + if (fileId.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + Core::Url directoryUrl(m_shareUrl); + directoryUrl.AppendQueryParameter( + _detail::FileIdQueryParameter, _internal::UrlEncodeQueryParameter(fileId)); + return ShareDirectoryClient(std::move(directoryUrl), m_pipeline, m_clientConfiguration); + } + ShareClient ShareClient::WithSnapshot(const std::string& snapshot) const { ShareClient newClient(*this); @@ -150,6 +176,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { protocolLayerOptions.PaidBurstingMaxBandwidthMibps = options.PaidBurstingMaxBandwidthMibps; protocolLayerOptions.ShareProvisionedIops = options.ProvisionedMaxIops; protocolLayerOptions.ShareProvisionedBandwidthMibps = options.ProvisionedMaxBandwidthMibps; + protocolLayerOptions.EnableChangeFeed = options.EnableChangeFeed; + protocolLayerOptions.ChangeFeedRetentionInDays = options.ChangeFeedRetentionInDays; auto result = _detail::ShareClient::Create(*m_pipeline, m_shareUrl, protocolLayerOptions, context); Models::CreateShareResult ret; @@ -244,8 +272,17 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; auto protocolLayerOptions = _detail::ShareClient::GetSharePropertiesOptions(); protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; - return _detail::ShareClient::GetProperties( + auto response = _detail::ShareClient::GetProperties( *m_pipeline, m_shareUrl, protocolLayerOptions, context); + const auto& headers = response.RawResponse->GetHeaders(); + auto changeFeedEnabled = headers.find("x-ms-file-enable-change-feed"); + if (changeFeedEnabled != headers.end()) + { + response.Value.IsChangeFeedEnabled + = Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + changeFeedEnabled->second, "true"); + } + return response; } Azure::Response ShareClient::SetProperties( @@ -264,6 +301,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { protocolLayerOptions.PaidBurstingMaxBandwidthMibps = options.PaidBurstingMaxBandwidthMibps; protocolLayerOptions.ShareProvisionedIops = options.ProvisionedMaxIops; protocolLayerOptions.ShareProvisionedBandwidthMibps = options.ProvisionedMaxBandwidthMibps; + protocolLayerOptions.EnableChangeFeed = options.EnableChangeFeed; + protocolLayerOptions.ChangeFeedRetentionInDays = options.ChangeFeedRetentionInDays; return _detail::ShareClient::SetProperties( *m_pipeline, m_shareUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp index f659ede92b..8c40536eab 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_directory_client.cpp @@ -3,6 +3,7 @@ #include "azure/storage/files/shares/share_directory_client.hpp" +#include "azure/storage/files/shares/share_constants.hpp" #include "azure/storage/files/shares/share_file_client.hpp" #include "private/package_version.hpp" @@ -17,6 +18,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { + std::string ShareDirectoryClient::GetFileId() const { return m_fileId.ValueOr(std::string()); } + ShareDirectoryClient ShareDirectoryClient::CreateFromConnectionString( const std::string& connectionString, const std::string& shareName, @@ -39,12 +42,23 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { } } + ShareDirectoryClient::ShareDirectoryClient( + Core::Url shareDirectoryUrl, + std::shared_ptr pipeline, + _detail::ShareClientConfiguration clientConfiguration) + : m_shareDirectoryUrl(std::move(shareDirectoryUrl)), + m_fileId(_detail::GetFileIdFromUrl(m_shareDirectoryUrl)), m_pipeline(std::move(pipeline)), + m_clientConfiguration(std::move(clientConfiguration)) + { + } + ShareDirectoryClient::ShareDirectoryClient( const std::string& shareDirectoryUrl, std::shared_ptr credential, const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -68,6 +82,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -99,6 +114,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareDirectoryUrl(shareDirectoryUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareDirectoryUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -118,6 +134,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ShareDirectoryClient ShareDirectoryClient::GetSubdirectoryClient( const std::string& subdirectoryName) const { + _detail::AssertPathAddressed(m_fileId, "GetSubdirectoryClient"); auto builder = m_shareDirectoryUrl; builder.AppendPath(_internal::UrlEncodePath(subdirectoryName)); return ShareDirectoryClient(builder, m_pipeline, m_clientConfiguration); @@ -125,6 +142,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ShareFileClient ShareDirectoryClient::GetFileClient(const std::string& fileName) const { + _detail::AssertPathAddressed(m_fileId, "GetFileClient"); auto builder = m_shareDirectoryUrl; builder.AppendPath(_internal::UrlEncodePath(fileName)); return ShareFileClient(builder, m_pipeline, m_clientConfiguration); @@ -150,6 +168,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Create"); auto protocolLayerOptions = _detail::DirectoryClient::CreateDirectoryOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -217,6 +236,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateIfNotExists"); try { return Create(options, context); @@ -239,6 +259,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const RenameFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "RenameFile"); auto sourceFileUrl = m_shareDirectoryUrl; sourceFileUrl.AppendPath(_internal::UrlEncodePath(fileName)); @@ -300,6 +321,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const RenameDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "RenameSubdirectory"); auto sourceDirectoryUrl = m_shareDirectoryUrl; sourceDirectoryUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName)); @@ -359,6 +381,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Delete"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::DeleteDirectoryOptions(); protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; @@ -375,6 +398,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteDirectoryOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DeleteIfExists"); try { return Delete(options, context); @@ -417,6 +441,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ret.PosixProperties.Owner = std::move(response.Value.Owner); ret.PosixProperties.Group = std::move(response.Value.Group); ret.PosixProperties.NfsFileType = std::move(response.Value.NfsFileType); + ret.FileName = std::move(response.Value.FileName); return Azure::Response( std::move(ret), std::move(response.RawResponse)); } @@ -426,6 +451,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetDirectoryPropertiesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetProperties"); auto protocolLayerOptions = _detail::DirectoryClient::SetDirectoryPropertiesOptions(); protocolLayerOptions.FileAttributes = smbProperties.Attributes.ToString(); if (smbProperties.CreatedOn.HasValue()) @@ -483,6 +509,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetDirectoryMetadataOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetMetadata"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::SetDirectoryMetadataOptions(); protocolLayerOptions.Metadata @@ -497,6 +524,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListFilesAndDirectoriesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListFilesAndDirectories"); auto protocolLayerOptions = _detail::DirectoryClient::ListDirectoryFilesAndDirectoriesSegmentOptions(); protocolLayerOptions.Prefix = options.Prefix; @@ -572,6 +600,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListDirectoryHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListHandles"); auto protocolLayerOptions = _detail::DirectoryClient::ListDirectoryHandlesOptions(); protocolLayerOptions.Marker = options.ContinuationToken; protocolLayerOptions.MaxResults = options.PageSizeHint; @@ -631,6 +660,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseDirectoryHandleOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseHandle"); (void)options; auto protocolLayerOptions = _detail::DirectoryClient::ForceDirectoryCloseHandlesOptions(); protocolLayerOptions.HandleId = handleId; @@ -647,6 +677,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseAllDirectoryHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseAllHandles"); auto protocolLayerOptions = _detail::DirectoryClient::ForceDirectoryCloseHandlesOptions(); protocolLayerOptions.HandleId = FileAllHandles; protocolLayerOptions.Marker = options.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp index 05efc80168..bec68a5cf3 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_file_client.cpp @@ -23,10 +23,14 @@ #include #include +#include + namespace Azure { namespace Storage { namespace Files { namespace Shares { constexpr static const int32_t DefaultListAllRangesPageSizeHint = 10000; + std::string ShareFileClient::GetFileId() const { return m_fileId.ValueOr(std::string()); } + ShareFileClient ShareFileClient::CreateFromConnectionString( const std::string& connectionString, const std::string& shareName, @@ -49,12 +53,23 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { } } + ShareFileClient::ShareFileClient( + Core::Url shareFileUrl, + std::shared_ptr pipeline, + _detail::ShareClientConfiguration clientConfiguration) + : m_shareFileUrl(std::move(shareFileUrl)), + m_fileId(_detail::GetFileIdFromUrl(m_shareFileUrl)), m_pipeline(std::move(pipeline)), + m_clientConfiguration(std::move(clientConfiguration)) + { + } + ShareFileClient::ShareFileClient( const std::string& shareFileUrl, std::shared_ptr credential, const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -78,6 +93,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -109,6 +125,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareClientOptions& options) : m_shareFileUrl(shareFileUrl) { + m_fileId = _detail::GetFileIdFromUrl(m_shareFileUrl); m_clientConfiguration.AllowTrailingDot = options.AllowTrailingDot; m_clientConfiguration.AllowSourceTrailingDot = options.AllowSourceTrailingDot; m_clientConfiguration.ShareTokenIntent = options.ShareTokenIntent; @@ -145,6 +162,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Create"); auto protocolLayerOptions = _detail::FileClient::CreateFileOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -283,6 +301,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Delete"); auto protocolLayerOptions = _detail::FileClient::DeleteFileOptions(); protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; @@ -299,6 +318,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DeleteFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DeleteIfExists"); try { return Delete(options, context); @@ -320,6 +340,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "Download"); bool isStructuredMessage = false; auto protocolLayerOptions = _detail::FileClient::DownloadFileOptions(); if (options.Range.HasValue()) @@ -495,6 +516,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const StartFileCopyOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "StartCopy"); auto protocolLayerOptions = _detail::FileClient::StartFileCopyOptions(); protocolLayerOptions.Metadata = std::map(options.Metadata.begin(), options.Metadata.end()); @@ -661,6 +683,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const AbortFileCopyOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "AbortCopy"); auto protocolLayerOptions = _detail::FileClient::AbortFileCopyOptions(); protocolLayerOptions.CopyId = std::move(copyId); protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; @@ -706,6 +729,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { ret.PosixProperties.Group = std::move(response.Value.Group); ret.PosixProperties.NfsFileType = std::move(response.Value.NfsFileType); ret.PosixProperties.LinkCount = std::move(response.Value.LinkCount); + ret.FileName = std::move(response.Value.FileName); return Azure::Response(std::move(ret), std::move(response.RawResponse)); } @@ -715,6 +739,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetFilePropertiesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetProperties"); auto protocolLayerOptions = _detail::FileClient::SetFileHttpHeadersOptions(); protocolLayerOptions.FileAttributes = smbProperties.Attributes.ToString(); if (smbProperties.CreatedOn.HasValue()) @@ -798,6 +823,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const SetFileMetadataOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "SetMetadata"); auto protocolLayerOptions = _detail::FileClient::SetFileMetadataOptions(); protocolLayerOptions.Metadata = std::map(metadata.begin(), metadata.end()); @@ -814,6 +840,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileRangeOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadRange"); auto protocolLayerOptions = _detail::FileClient::UploadFileRangeOptions(); protocolLayerOptions.FileRangeWrite = "update"; protocolLayerOptions.Range = std::string("bytes=") + std::to_string(offset) + std::string("-") @@ -863,6 +890,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ClearFileRangeOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ClearRange"); auto protocolLayerOptions = _detail::FileClient::UploadFileRangeOptions(); protocolLayerOptions.FileRangeWrite = "clear"; protocolLayerOptions.Range = std::string("bytes=") + std::to_string(offset) + std::string("-") @@ -890,6 +918,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetRangeList"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -934,6 +963,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetRangeListDiff"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -979,6 +1009,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetAllRangeList"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -1028,6 +1059,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetFileRangeListOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetAllRangeListDiff"); auto protocolLayerOptions = _detail::FileClient::GetFileRangeListOptions(); if (options.Range.HasValue()) { @@ -1079,6 +1111,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ListFileHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ListHandles"); auto protocolLayerOptions = _detail::FileClient::ListFileHandlesOptions(); protocolLayerOptions.Marker = options.ContinuationToken; protocolLayerOptions.MaxResults = options.PageSizeHint; @@ -1137,6 +1170,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseFileHandleOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseHandle"); (void)options; auto protocolLayerOptions = _detail::FileClient::ForceFileCloseHandlesOptions(); protocolLayerOptions.HandleId = handleId; @@ -1152,6 +1186,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const ForceCloseAllFileHandlesOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "ForceCloseAllHandles"); auto protocolLayerOptions = _detail::FileClient::ForceFileCloseHandlesOptions(); protocolLayerOptions.HandleId = FileAllHandles; protocolLayerOptions.Marker = options.ContinuationToken; @@ -1179,6 +1214,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileToOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DownloadTo"); // Just start downloading using an initial chunk. If it's a small file, we'll get the whole // thing in one shot. If it's a large file, we'll get its full size in Content-Range and can // keep downloading it in chunks. @@ -1226,14 +1262,6 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { "Buffer is not big enough, file range size is " + std::to_string(fileRangeSize) + "."); } - int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( - buffer, static_cast(firstChunkLength), context); - if (bytesRead != firstChunkLength) - { - throw Azure::Core::RequestFailedException("Error when reading body stream."); - } - firstChunk.Value.BodyStream.reset(); - auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadFileToResult ret; ret.FileSize = response.Value.FileSize; @@ -1242,9 +1270,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadFileOptions chunkOptions; @@ -1269,19 +1296,51 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); + lastChunkResponse = std::move(chunk); } }; int64_t remainingOffset = firstChunkOffset + firstChunkLength; int64_t remainingSize = fileRangeSize - firstChunkLength; - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + int64_t bytesRead = firstChunk.Value.BodyStream->ReadToCount( + buffer, static_cast(firstChunkLength), context); + if (bytesRead != firstChunkLength) + { + throw Azure::Core::RequestFailedException("Error when reading body stream."); + } + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = fileRangeSize; return ret; @@ -1292,6 +1351,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const DownloadFileToOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "DownloadTo"); // Just start downloading using an initial chunk. If it's a small file, we'll get the whole // thing in one shot. If it's a large file, we'll get its full size in Content-Range and can // keep downloading it in chunks. @@ -1353,8 +1413,6 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { }; _internal::FileWriter fileWriter(fileName); - bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); - firstChunk.Value.BodyStream.reset(); auto returnTypeConverter = [](Azure::Response& response) { Models::DownloadFileToResult ret; @@ -1364,9 +1422,8 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); }; - auto ret = returnTypeConverter(firstChunk); - // Keep downloading the remaining in parallel + Azure::Nullable> lastChunkResponse; auto downloadChunkFunc = [&](int64_t offset, int64_t length, int64_t chunkId, int64_t numChunks) { DownloadFileOptions chunkOptions; @@ -1389,19 +1446,46 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { if (chunkId == numChunks - 1) { - ret = returnTypeConverter(chunk); + lastChunkResponse = std::move(chunk); } }; int64_t remainingOffset = firstChunkOffset + firstChunkLength; int64_t remainingSize = fileRangeSize - firstChunkLength; - _internal::ConcurrentTransfer( - remainingOffset, - remainingSize, - options.TransferOptions.ChunkSize, - options.TransferOptions.Concurrency, - downloadChunkFunc); + std::future remainingDownload; + if (remainingSize > 0 && options.TransferOptions.Concurrency > 1) + { + const int remainingConcurrency = options.TransferOptions.Concurrency - 1; + remainingDownload = std::async(std::launch::async, [&, remainingConcurrency]() { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + remainingConcurrency, + downloadChunkFunc); + }); + } + + bodyStreamToFile(*(firstChunk.Value.BodyStream), fileWriter, 0, firstChunkLength, context); + firstChunk.Value.BodyStream.reset(); + + if (remainingDownload.valid()) + { + remainingDownload.get(); + } + else if (remainingSize > 0) + { + _internal::ConcurrentTransfer( + remainingOffset, + remainingSize, + options.TransferOptions.ChunkSize, + options.TransferOptions.Concurrency, + downloadChunkFunc); + } + + auto ret = lastChunkResponse.HasValue() ? returnTypeConverter(lastChunkResponse.Value()) + : returnTypeConverter(firstChunk); ret.Value.ContentRange.Offset = firstChunkOffset; ret.Value.ContentRange.Length = fileRangeSize; return ret; @@ -1413,6 +1497,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileFromOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadFrom"); _detail::FileClient::CreateFileOptions protocolLayerOptions; protocolLayerOptions.FileContentLength = bufferSize; protocolLayerOptions.FileAttributes = options.SmbProperties.Attributes.ToString(); @@ -1526,6 +1611,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileFromOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadFrom"); _internal::FileReader fileReader(fileName); _detail::FileClient::CreateFileOptions protocolLayerOptions; @@ -1643,6 +1729,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const UploadFileRangeFromUriOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "UploadRangeFromUri"); AZURE_ASSERT_MSG(sourceRange.Length.HasValue(), "Source length cannot be null."); int64_t rangeLength = sourceRange.Length.Value(); @@ -1696,6 +1783,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateSymbolicLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateSymbolicLink"); _detail::FileClient::CreateFileSymbolicLinkOptions protocolLayerOptions; protocolLayerOptions.LinkText = linkText; if (options.CreatedOn.HasValue()) @@ -1736,6 +1824,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const GetSymbolicLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "GetSymbolicLink"); (void)options; _detail::FileClient::GetFileSymbolicLinkOptions protocolLayerOptions; protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; @@ -1755,6 +1844,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { const CreateHardLinkOptions& options, const Azure::Core::Context& context) const { + _detail::AssertPathAddressed(m_fileId, "CreateHardLink"); _detail::FileClient::CreateFileHardLinkOptions protocolLayerOptions; protocolLayerOptions.TargetFile = targetFile; protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; @@ -1775,4 +1865,62 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { return Azure::Response( std::move(ret), std::move(response.RawResponse)); } + + Azure::Response ShareFileClient::GetFileLinks( + const GetFileLinksOptions& options, + const Azure::Core::Context& context) const + { + if (!m_fileId.HasValue()) + { + throw std::logic_error("GetFileLinks requires a file ID-addressed client."); + } + + _detail::FileClient::GetFileFileLinksOptions protocolLayerOptions; + protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.AllowTrailingDot = m_clientConfiguration.AllowTrailingDot; + protocolLayerOptions.FileRequestIntent = m_clientConfiguration.ShareTokenIntent; + auto response = _detail::FileClient::GetFileLinks( + *m_pipeline, m_shareFileUrl, protocolLayerOptions, context); + + Models::GetFileLinksResult result; + auto& properties = result.Properties; + properties.CopyCompletedOn = std::move(response.Value.Properties.CopyCompletedOn); + properties.CopyId = std::move(response.Value.Properties.CopyId); + properties.CopyProgress = std::move(response.Value.Properties.CopyProgress); + properties.CopySource = std::move(response.Value.Properties.CopySource); + properties.CopyStatus = std::move(response.Value.Properties.CopyStatus); + properties.CopyStatusDescription = std::move(response.Value.Properties.CopyStatusDescription); + properties.ETag = std::move(response.Value.Properties.ETag); + properties.FileSize = response.Value.Properties.FileSize; + properties.LastModified = std::move(response.Value.Properties.LastModified); + properties.SmbProperties = std::move(response.Value.Properties.SmbProperties); + properties.HttpHeaders = std::move(response.Value.Properties.HttpHeaders); + properties.Metadata = std::move(response.Value.Properties.Metadata); + properties.IsServerEncrypted = response.Value.Properties.IsServerEncrypted; + properties.LeaseDuration = std::move(response.Value.Properties.LeaseDuration); + properties.LeaseState = std::move(response.Value.Properties.LeaseState); + properties.LeaseStatus = std::move(response.Value.Properties.LeaseStatus); + properties.FileName = std::move(response.Value.Properties.FileName); + properties.PosixProperties.Owner = std::move(response.Value.Properties.Owner); + properties.PosixProperties.Group = std::move(response.Value.Properties.Group); + properties.PosixProperties.LinkCount = std::move(response.Value.Properties.LinkCount); + properties.PosixProperties.NfsFileType = std::move(response.Value.Properties.NfsFileType); + if (response.Value.Properties.FileMode.HasValue()) + { + properties.PosixProperties.FileMode + = Models::NfsFileMode::ParseOctalFileMode(response.Value.Properties.FileMode.Value()); + } + + result.Links.reserve(response.Value.HardLinks.size()); + for (auto& hardLink : response.Value.HardLinks) + { + Models::FileLink link; + link.Name = hardLink.FileName.Encoded ? Core::Url::Decode(hardLink.FileName.Content) + : std::move(hardLink.FileName.Content); + link.ParentId = std::move(hardLink.ParentId); + result.Links.emplace_back(std::move(link)); + } + return Azure::Response( + std::move(result), std::move(response.RawResponse)); + } }}}} // namespace Azure::Storage::Files::Shares diff --git a/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp b/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp index 025051fcfa..93b4921ba0 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_lease_client.cpp @@ -22,6 +22,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "AcquireLease"); _detail::FileClient::AcquireFileLeaseOptions protocolLayerOptions; protocolLayerOptions.ProposedLeaseId = GetLeaseId(); protocolLayerOptions.Duration = static_cast(duration.count()); @@ -79,6 +80,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "RenewLease"); AZURE_ASSERT_MSG(false, "File lease doesn't support renew"); AZURE_NOT_IMPLEMENTED(); } @@ -116,6 +118,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "ReleaseLease"); _detail::FileClient::ReleaseFileLeaseOptions protocolLayerOptions; protocolLayerOptions.LeaseId = GetLeaseId(); protocolLayerOptions.AllowTrailingDot @@ -170,6 +173,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "ChangeLease"); _detail::FileClient::ChangeFileLeaseOptions protocolLayerOptions; protocolLayerOptions.LeaseId = GetLeaseId(); protocolLayerOptions.ProposedLeaseId = proposedLeaseId; @@ -237,6 +241,7 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { (void)options; if (m_fileClient.HasValue()) { + _detail::AssertPathAddressed(m_fileClient.Value().m_fileId, "BreakLease"); _detail::FileClient::BreakFileLeaseOptions protocolLayerOptions; protocolLayerOptions.AllowTrailingDot = m_fileClient.Value().m_clientConfiguration.AllowTrailingDot; diff --git a/sdk/storage/azure-storage-files-shares/src/share_options.cpp b/sdk/storage/azure-storage-files-shares/src/share_options.cpp index 345d5ba23c..b99c34746d 100644 --- a/sdk/storage/azure-storage-files-shares/src/share_options.cpp +++ b/sdk/storage/azure-storage-files-shares/src/share_options.cpp @@ -3,10 +3,44 @@ #include "azure/storage/files/shares/share_options.hpp" +#include "azure/storage/files/shares/share_constants.hpp" + +#include + +#include + namespace Azure { namespace Storage { namespace Files { namespace Shares { const ShareAudience ShareAudience::DefaultAudience(_internal::StorageDefaultAudience); + namespace _detail { + Azure::Nullable GetFileIdFromUrl(const Azure::Core::Url& url) + { + for (const auto& queryParameter : url.GetQueryParameters()) + { + if (Core::_internal::StringExtensions::LocaleInvariantCaseInsensitiveEqual( + queryParameter.first, FileIdQueryParameter)) + { + if (queryParameter.second.empty()) + { + throw std::invalid_argument("File ID cannot be empty."); + } + return queryParameter.second; + } + } + return Azure::Nullable(); + } + + void AssertPathAddressed(const Azure::Nullable& fileId, const char* operationName) + { + if (fileId.HasValue()) + { + throw std::logic_error( + std::string(operationName) + " is not supported for a file ID-addressed client."); + } + } + } // namespace _detail + namespace Models { namespace { RolePermissions ParseOctal(char c) diff --git a/sdk/storage/azure-storage-files-shares/swagger/README.md b/sdk/storage/azure-storage-files-shares/swagger/README.md index 15867f8c86..c6f72363b1 100644 --- a/sdk/storage/azure-storage-files-shares/swagger/README.md +++ b/sdk/storage/azure-storage-files-shares/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-files-shares namespace: Azure::Storage::Files::Shares output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Microsoft.FileStorage/stable/2026-12-06/file.json +input-file: https://raw.githubusercontent.com/Jinming-Hu/azure-rest-api-specs/refs/heads/stg105-file-codegen/specification/storage/data-plane/Microsoft.FileStorage/stable/2027-03-07/file.json ``` ## ModelFour Options @@ -62,6 +62,8 @@ directive: where: $["x-ms-paths"] transform: > delete $["/{shareName}?restype=share&comp=undelete"].put; + delete $["/{shareName}?restype=directory"].get; + delete $["/{shareName}"].head; ``` ### API Version @@ -79,12 +81,12 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2026-10-06"] + "enum": ["2027-03-07"] }; - from: swagger-document where: $.parameters transform: > - $.ApiVersionParameter.enum = ["2026-10-06"]; + $.ApiVersionParameter.enum = ["2027-03-07"]; ``` ### Rename Operations @@ -482,6 +484,20 @@ directive: $.ShareItemInternal.properties["Details"] = {"$ref": "#/definitions/ShareItemDetails", "x-ms-xml": {"name": "Properties"}}; $.ShareItemInternal["x-ms-client-name"] = "ShareItem"; $.ShareItemDetails.properties["ProvisionedBandwidthMiBps"]["x-ms-client-name"] = "ProvisionedBandwidthMBps"; + $.ShareItemDetails.properties["CreationTime"]["x-ms-client-name"] = "CreatedOn"; + $.ShareItemDetails.properties["CreationTime"]["x-ms-xml"] = {"name": "Creation-Time"}; + $.ShareItemDetails.properties["CreationTime"]["x-nullable"] = true; + $.ShareItemDetails.properties["CreationTime"].description = "The date and time the share was created."; + const creationTime = $.ShareItemDetails.properties["CreationTime"]; + delete $.ShareItemDetails.properties["CreationTime"]; + const properties = {}; + for (const name in $.ShareItemDetails.properties) { + properties[name] = $.ShareItemDetails.properties[name]; + if (name === "Last-Modified") { + properties["CreationTime"] = creationTime; + } + } + $.ShareItemDetails.properties = properties; delete $.ShareItemDetails.properties["EnableSmbDirectoryLease"]; delete $.ShareItemInternal.properties["Properties"]; delete $.ShareItemInternal.required; @@ -538,8 +554,65 @@ directive: $ = $.filter(p => !p["$ref"] || !p["$ref"].endsWith("#/parameters/EnableSmbDirectoryLease")); ``` +### Share Change Feed + +```yaml +directive: + - from: swagger-document + where: $.parameters + transform: > + $.EnableChangeFeed = { + "name": "x-ms-file-enable-change-feed", + "in": "header", + "type": "boolean", + "required": false, + "x-ms-client-name": "EnableChangeFeed" + }; + $.ChangeFeedRetentionInDays = { + "name": "x-ms-file-change-feed-retention-in-days", + "in": "header", + "type": "integer", + "format": "int32", + "required": false, + "x-ms-client-name": "ChangeFeedRetentionInDays" + }; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + for (const path of ["/{shareName}?restype=share", "/{shareName}?restype=share&comp=properties"]) { + $[path].put.parameters.push( + {"$ref": "#/parameters/EnableChangeFeed"}, + {"$ref": "#/parameters/ChangeFeedRetentionInDays"}); + } + - from: swagger-document + where: $["x-ms-paths"]["/{shareName}?restype=share"].get.responses["200"].headers + transform: > + $["x-ms-file-enable-change-feed"] = { + "type": "boolean", + "x-ms-client-name": "IsChangeFeedEnabled", + "x-nullable": true, + "description": "Specifies whether change feed is enabled on the share." + }; + $["x-ms-file-change-feed-retention-in-days"] = { + "type": "integer", + "format": "int32", + "x-ms-client-name": "ChangeFeedRetentionInDays", + "x-nullable": true, + "description": "The number of days that change feed records are retained." + }; + $["x-ms-file-blob-container-for-xfiles-change-feed"] = { + "type": "string", + "x-ms-client-name": "ChangeFeedBlobContainerName", + "x-nullable": true, + "description": "The name of the blob container where change feed records are stored." + }; +``` + ### GetShareProperties +The change-feed enablement header is parsed case-insensitively by the handwritten +`ShareClient::GetProperties` implementation, while the public field remains `Nullable`. + ```yaml directive: - from: swagger-document @@ -571,6 +644,19 @@ directive: $["x-ms-share-max-burst-credits-for-iops"]["x-nullable"] = true; $["x-ms-share-next-allowed-provisioned-iops-downgrade-time"]["x-nullable"] = true; $["x-ms-share-next-allowed-provisioned-bandwidth-downgrade-time"]["x-nullable"] = true; + $["x-ms-share-creation-time"]["x-ms-client-name"] = "CreatedOn"; + $["x-ms-share-creation-time"]["x-nullable"] = true; + $["x-ms-share-creation-time"].description = "Returns the date and time the share was created."; + const creationTime = $["x-ms-share-creation-time"]; + delete $["x-ms-share-creation-time"]; + const headers = {}; + for (const name in $) { + headers[name] = $[name]; + if (name === "Last-Modified") { + headers["x-ms-share-creation-time"] = creationTime; + } + } + $ = headers; - from: swagger-document where: $["x-ms-paths"]["/{shareName}?restype=share"].get.responses["200"] transform: > @@ -583,6 +669,29 @@ directive: "AccessTier": {"$ref": "#/definitions/AccessTier", "x-nullable": true, "x-ms-xml": {"name": ""}} } }; + $.schema.required = []; + for (const headerName in $.headers) { + const header = $.headers[headerName]; + const clientName = header["x-ms-client-name"] + || headerName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + const propertyName = clientName[0].toUpperCase() + clientName.slice(1); + if (propertyName !== "AccessTier") { + const property = JSON.parse(JSON.stringify(header)); + delete property["x-ms-client-name"]; + property["x-ms-xml"] = {"name": ""}; + $.schema.properties[propertyName] = property; + } + if (!$.schema.properties[propertyName]["x-nullable"]) { + $.schema.required.push(propertyName); + } + } + $.schema.properties.Metadata = { + "$ref": "#/definitions/Metadata", + "description": $.schema.properties.Metadata.description, + "x-ms-xml": {"name": ""} + }; + $.schema.properties.ProvisionedIops.description = "Returns the current share provisioned IOPS."; + delete $.headers["x-ms-file-enable-change-feed"]; ``` ### GetShareAccessPolicy @@ -807,6 +916,12 @@ directive: $.headers["x-ms-owner"]["x-nullable"] = true; $.headers["x-ms-group"]["x-nullable"] = true; $.headers["x-ms-file-file-type"]["x-nullable"] = true; + $.headers["x-ms-file-name"]["x-ms-client-name"] = "FileName"; + $.headers["x-ms-file-name"]["x-nullable"] = true; + $.headers["x-ms-file-name"].description = "The name of the directory."; + const fileName = $.headers["x-ms-file-name"]; + delete $.headers["x-ms-file-name"]; + $.headers["x-ms-file-name"] = fileName; $.schema = { "type": "object", "x-ms-client-name": "DirectoryProperties", @@ -926,6 +1041,12 @@ directive: $.headers["x-ms-group"]["x-nullable"] = true; $.headers["x-ms-file-file-type"]["x-nullable"] = true; $.headers["x-ms-link-count"]["x-nullable"] = true; + $.headers["x-ms-file-name"]["x-ms-client-name"] = "FileName"; + $.headers["x-ms-file-name"]["x-nullable"] = true; + $.headers["x-ms-file-name"].description = "The name of the file."; + const fileName = $.headers["x-ms-file-name"]; + delete $.headers["x-ms-file-name"]; + $.headers["x-ms-file-name"] = fileName; delete $.headers["x-ms-type"]; $.schema = { "type": "object", @@ -1245,6 +1366,91 @@ directive: }; ``` +### GetFileLinks + +```yaml +directive: + - from: swagger-document + where: $.definitions + transform: > + $.HardLink["x-namespace"] = "_detail"; + const propertiesResponse = $doc["x-ms-paths"]["/{shareName}/{directory}/{fileName}"].head.responses["200"]; + const propertiesSchema = JSON.parse(JSON.stringify(propertiesResponse.schema)); + propertiesSchema.required = ["SmbProperties", "HttpHeaders"]; + for (const headerName in propertiesResponse.headers) { + const header = propertiesResponse.headers[headerName]; + const propertyPath = header["x-ms-client-path"] + || header["x-ms-client-name"] + || headerName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + if (propertyPath.includes(".")) { + continue; + } + const propertyName = propertyPath[0].toUpperCase() + propertyPath.slice(1); + const property = JSON.parse(JSON.stringify(header)); + delete property["x-ms-client-name"]; + delete property["x-ms-client-path"]; + property["x-ms-xml"] = {"name": ""}; + propertiesSchema.properties[propertyName] = property; + if (!property["x-nullable"]) { + propertiesSchema.required.push(propertyName); + } + } + propertiesSchema.properties.Metadata = { + "$ref": "#/definitions/Metadata", + "description": propertiesSchema.properties.Metadata.description, + "x-ms-xml": {"name": ""} + }; + propertiesSchema.description = "Response type for #Azure::Storage::Files::Shares::ShareFileClient::GetProperties."; + $.FileProperties = propertiesSchema; + - from: swagger-document + where: $["x-ms-paths"] + transform: > + const propertiesResponse = $["/{shareName}/{directory}/{fileName}"].head.responses["200"]; + propertiesResponse.schema = {"$ref": "#/definitions/FileProperties"}; + + const operation = $["/{shareName}?comp=hardlinks"].get; + operation.operationId = "File_GetFileLinks"; + operation.parameters = operation.parameters.filter(p => + p.name !== "fileid" && p["$ref"] !== "#/parameters/ShareSnapshot"); + operation.responses["200"].schema = { + "type": "object", + "x-ms-client-name": "GetFileLinksResult", + "x-ms-sealed": false, + "x-namespace": "_detail", + "xml": {"name": "HardLinks"}, + "required": ["Properties", "HardLinks"], + "properties": { + "Properties": { + "$ref": "#/definitions/FileProperties", + "x-ms-xml": {"name": ""} + }, + "HardLinks": { + "type": "array", + "x-ms-xml": {"name": "."}, + "items": {"$ref": "#/definitions/HardLink"} + } + } + }; + + const response = operation.responses["200"]; + const headers = {}; + for (const sourceHeaderName in propertiesResponse.headers) { + let headerName = sourceHeaderName; + if (["Content-Length", "Content-Type", "Content-MD5", "Content-Encoding", + "Content-Language", "Cache-Control", "Content-Disposition"].includes(headerName)) { + headerName = "x-ms-" + headerName.toLowerCase(); + } + const header = JSON.parse(JSON.stringify(propertiesResponse.headers[sourceHeaderName])); + const propertyPath = header["x-ms-client-path"] + || header["x-ms-client-name"] + || sourceHeaderName.replace(/-(.)/g, (_, c) => c.toUpperCase()); + delete header["x-ms-client-name"]; + header["x-ms-client-path"] = "Properties." + propertyPath[0].toUpperCase() + propertyPath.slice(1); + headers[headerName] = header; + } + response.headers = headers; +``` + ### Description ```yaml diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp index 5a364a4ad4..f20d43abe1 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_client_test.cpp @@ -28,6 +28,133 @@ namespace Azure { namespace Storage { namespace Files { namespace Shares { names namespace Azure { namespace Storage { namespace Test { + TEST(FileIdAddressedClientTest, ClientStateAndValidation) + { + Files::Shares::ShareClient shareClient( + "https://account.file.core.windows.net/share?sharesnapshot=snapshot&sig=signature"); + + EXPECT_THROW(shareClient.GetFileClientByFileId(""), std::invalid_argument); + EXPECT_THROW(shareClient.GetDirectoryClientByFileId(""), std::invalid_argument); + EXPECT_THROW( + Files::Shares::ShareFileClient("https://account.file.core.windows.net/share?fileid="), + std::invalid_argument); + EXPECT_THROW( + Files::Shares::ShareDirectoryClient("https://account.file.core.windows.net/share?fileid="), + std::invalid_argument); + + auto fileClient = shareClient.GetFileClientByFileId("12345"); + EXPECT_EQ(fileClient.GetFileId(), "12345"); + auto fileUrl = Core::Url(fileClient.GetUrl()); + EXPECT_EQ(fileUrl.GetQueryParameters().at("sharesnapshot"), "snapshot"); + EXPECT_EQ(fileUrl.GetQueryParameters().at("sig"), "signature"); + EXPECT_EQ(fileUrl.GetQueryParameters().at("fileid"), "12345"); + auto snapshotFileClient = fileClient.WithShareSnapshot("new-snapshot"); + EXPECT_EQ(snapshotFileClient.GetFileId(), "12345"); + EXPECT_EQ( + Core::Url(snapshotFileClient.GetUrl()).GetQueryParameters().at("sharesnapshot"), + "new-snapshot"); + + auto directoryClient = shareClient.GetDirectoryClientByFileId("67890"); + EXPECT_EQ(directoryClient.GetFileId(), "67890"); + auto snapshotDirectoryClient = directoryClient.WithShareSnapshot("new-snapshot"); + EXPECT_EQ(snapshotDirectoryClient.GetFileId(), "67890"); + + Files::Shares::ShareFileClient uppercaseFileIdClient( + "https://account.file.core.windows.net/share?FILEID=12345"); + EXPECT_EQ(uppercaseFileIdClient.GetFileId(), "12345"); + + Files::Shares::ShareDirectoryClient uppercaseDirectoryFileIdClient( + "https://account.file.core.windows.net/share?FILEID=67890"); + EXPECT_EQ(uppercaseDirectoryFileIdClient.GetFileId(), "67890"); + + Files::Shares::ShareFileClient pathClient("https://account.file.core.windows.net/share/path"); + EXPECT_TRUE(pathClient.GetFileId().empty()); + EXPECT_THROW(pathClient.GetFileLinks(), std::logic_error); + } + + TEST(FileIdAddressedClientTest, FileUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto fileClient = shareClient.GetFileClientByFileId("12345"); + std::vector content(1); + Core::IO::MemoryBodyStream contentStream(content); + Core::Http::HttpRange sourceRange; + + EXPECT_THROW(fileClient.Create(1), std::logic_error); + EXPECT_THROW(fileClient.Delete(), std::logic_error); + EXPECT_THROW(fileClient.DeleteIfExists(), std::logic_error); + EXPECT_THROW(fileClient.Download(), std::logic_error); + EXPECT_THROW(fileClient.DownloadTo(content.data(), content.size()), std::logic_error); + EXPECT_THROW(fileClient.DownloadTo("file-that-must-not-be-created"), std::logic_error); + EXPECT_THROW(fileClient.UploadFrom(content.data(), content.size()), std::logic_error); + EXPECT_THROW(fileClient.UploadFrom("file-that-must-not-be-opened"), std::logic_error); + EXPECT_THROW( + fileClient.StartCopy("https://account.file.core.windows.net/share/source"), + std::logic_error); + EXPECT_THROW(fileClient.AbortCopy("copy-id"), std::logic_error); + EXPECT_THROW( + fileClient.SetProperties( + Files::Shares::Models::FileHttpHeaders(), Files::Shares::Models::FileSmbProperties()), + std::logic_error); + EXPECT_THROW(fileClient.SetMetadata(Storage::Metadata()), std::logic_error); + EXPECT_THROW(fileClient.UploadRange(0, contentStream), std::logic_error); + EXPECT_THROW(fileClient.ClearRange(0, 1), std::logic_error); + EXPECT_THROW(fileClient.GetRangeList(), std::logic_error); + EXPECT_THROW(fileClient.GetRangeListDiff("snapshot"), std::logic_error); + EXPECT_THROW(fileClient.GetAllRangeList(), std::logic_error); + EXPECT_THROW(fileClient.GetAllRangeListDiff("snapshot"), std::logic_error); + EXPECT_THROW(fileClient.ListHandles(), std::logic_error); + EXPECT_THROW(fileClient.ForceCloseHandle("handle-id"), std::logic_error); + EXPECT_THROW(fileClient.ForceCloseAllHandles(), std::logic_error); + EXPECT_THROW( + fileClient.UploadRangeFromUri( + 0, "https://account.file.core.windows.net/share/source", sourceRange), + std::logic_error); + EXPECT_THROW(fileClient.CreateSymbolicLink("target"), std::logic_error); + EXPECT_THROW(fileClient.GetSymbolicLink(), std::logic_error); + EXPECT_THROW(fileClient.CreateHardLink("target"), std::logic_error); + } + + TEST(FileIdAddressedClientTest, DirectoryUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto directoryClient = shareClient.GetDirectoryClientByFileId("67890"); + + EXPECT_THROW(directoryClient.GetSubdirectoryClient("child"), std::logic_error); + EXPECT_THROW(directoryClient.GetFileClient("file"), std::logic_error); + EXPECT_THROW(directoryClient.Create(), std::logic_error); + EXPECT_THROW(directoryClient.CreateIfNotExists(), std::logic_error); + EXPECT_THROW(directoryClient.RenameFile("file", "destination"), std::logic_error); + EXPECT_THROW(directoryClient.RenameSubdirectory("directory", "destination"), std::logic_error); + EXPECT_THROW(directoryClient.Delete(), std::logic_error); + EXPECT_THROW(directoryClient.DeleteIfExists(), std::logic_error); + EXPECT_THROW( + directoryClient.SetProperties(Files::Shares::Models::FileSmbProperties()), + std::logic_error); + EXPECT_THROW(directoryClient.SetMetadata(Storage::Metadata()), std::logic_error); + EXPECT_THROW(directoryClient.ListFilesAndDirectories(), std::logic_error); + EXPECT_THROW(directoryClient.ListHandles(), std::logic_error); + EXPECT_THROW(directoryClient.ForceCloseHandle("handle-id"), std::logic_error); + EXPECT_THROW(directoryClient.ForceCloseAllHandles(), std::logic_error); + } + + TEST(FileIdAddressedClientTest, LeaseUnsupportedOperations) + { + Files::Shares::ShareClient shareClient("https://account.file.core.windows.net/share"); + auto fileClient = shareClient.GetFileClientByFileId("12345"); + Files::Shares::ShareLeaseClient leaseClient( + fileClient, Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()); + EXPECT_THROW( + leaseClient.Acquire(Files::Shares::ShareLeaseClient::InfiniteLeaseDuration), + std::logic_error); + EXPECT_THROW(leaseClient.Renew(), std::logic_error); + EXPECT_THROW(leaseClient.Release(), std::logic_error); + EXPECT_THROW( + leaseClient.Change(Files::Shares::ShareLeaseClient::CreateUniqueLeaseId()), + std::logic_error); + EXPECT_THROW(leaseClient.Break(), std::logic_error); + } + void FileShareClientTest::SetUp() { FileShareServiceClientTest::SetUp(); @@ -205,7 +332,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareClientTest, ShareProperties) + TEST_F(FileShareClientTest, ShareProperties_PLAYBACKONLY_) { const int32_t quota32GB = 32; const int32_t quota64GB = 64; @@ -218,6 +345,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(m_shareClient->SetProperties(options)); auto result = m_shareClient->GetProperties(); EXPECT_EQ(quota32GB, result.Value.Quota); + ASSERT_TRUE(result.Value.CreatedOn.HasValue()); + EXPECT_TRUE(IsValidTime(result.Value.CreatedOn.Value())); options.ShareQuotaInGiB = quota64GB; EXPECT_NO_THROW(m_shareClient->SetProperties(options)); result = m_shareClient->GetProperties(); @@ -827,6 +956,54 @@ namespace Azure { namespace Storage { namespace Test { && !properties.EnableSnapshotVirtualDirectoryAccess.Value()); } + TEST_F(FileShareClientTest, ChangeFeed_PLAYBACKONLY_) + { + auto shareClient = GetShareClientForTest(LowercaseRandomString()); + + Files::Shares::CreateShareOptions createOptions; + createOptions.EnableChangeFeed = true; + createOptions.ChangeFeedRetentionInDays = 3; + shareClient.Create(createOptions); + + auto properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(3, properties.ChangeFeedRetentionInDays.Value()); + ASSERT_TRUE(properties.ChangeFeedBlobContainerName.HasValue()); + EXPECT_EQ(0U, properties.ChangeFeedBlobContainerName.Value().find("$fileschangefeed-")); + + Files::Shares::SetSharePropertiesOptions setPropertiesOptions; + setPropertiesOptions.EnableChangeFeed = true; + setPropertiesOptions.ChangeFeedRetentionInDays = 5; + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(5, properties.ChangeFeedRetentionInDays.Value()); + EXPECT_TRUE(properties.ChangeFeedBlobContainerName.HasValue()); + + setPropertiesOptions.EnableChangeFeed.Reset(); + setPropertiesOptions.ChangeFeedRetentionInDays = 30; + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_TRUE(properties.IsChangeFeedEnabled.Value()); + ASSERT_TRUE(properties.ChangeFeedRetentionInDays.HasValue()); + EXPECT_EQ(30, properties.ChangeFeedRetentionInDays.Value()); + + setPropertiesOptions.EnableChangeFeed = false; + setPropertiesOptions.ChangeFeedRetentionInDays.Reset(); + shareClient.SetProperties(setPropertiesOptions); + + properties = shareClient.GetProperties().Value; + ASSERT_TRUE(properties.IsChangeFeedEnabled.HasValue()); + EXPECT_FALSE(properties.IsChangeFeedEnabled.Value()); + } + TEST_F(FileShareClientTest, FilePermissionFormat) { auto sddlPermission diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp index 3539fb6f45..3fb8112d50 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_file_client_test.cpp @@ -30,6 +30,34 @@ namespace Azure { namespace Storage { namespace Test { m_fileClient->Create(1024); } + TEST_F(FileShareFileClientTest, FileIdAddressing_PLAYBACKONLY_) + { + auto pathProperties = m_fileClient->GetProperties().Value; + ASSERT_FALSE(pathProperties.SmbProperties.FileId.empty()); + ASSERT_FALSE(pathProperties.SmbProperties.ParentFileId.empty()); + + auto fileIdClient = m_shareClient->GetFileClientByFileId(pathProperties.SmbProperties.FileId); + auto fileIdProperties = fileIdClient.GetProperties().Value; + ASSERT_TRUE(fileIdProperties.FileName.HasValue()); + EXPECT_EQ(fileIdProperties.FileName.Value(), m_fileName); + EXPECT_EQ(fileIdProperties.SmbProperties.FileId, pathProperties.SmbProperties.FileId); + + auto links = fileIdClient.GetFileLinks().Value; + ASSERT_EQ(links.Links.size(), 1); + EXPECT_EQ(links.Links[0].Name, m_fileName); + EXPECT_EQ(links.Links[0].ParentId, pathProperties.SmbProperties.ParentFileId); + + auto directoryPathProperties = m_fileShareDirectoryClient->GetProperties().Value; + ASSERT_FALSE(directoryPathProperties.SmbProperties.FileId.empty()); + auto directoryIdClient + = m_shareClient->GetDirectoryClientByFileId(directoryPathProperties.SmbProperties.FileId); + auto directoryIdProperties = directoryIdClient.GetProperties().Value; + ASSERT_TRUE(directoryIdProperties.FileName.HasValue()); + EXPECT_EQ(directoryIdProperties.FileName.Value(), m_directoryName); + EXPECT_EQ( + directoryIdProperties.SmbProperties.FileId, directoryPathProperties.SmbProperties.FileId); + } + TEST_F(FileShareFileClientTest, CreateDeleteFiles) { { @@ -2428,16 +2456,9 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareFileClientTest, PremiumPosixProperties) + TEST_F(NfsFileShareClientTest, PremiumPosixProperties) { - auto shareServiceClient = *m_premiumShareServiceClient; - - auto shareName = LowercaseRandomString(); - auto shareClient = GetPremiumShareClientForTest(shareName); - Files::Shares::CreateShareOptions shareOptions; - shareOptions.EnabledProtocols = Files::Shares::Models::ShareProtocols::Nfs; - EXPECT_NO_THROW(shareClient.Create(shareOptions)); - auto otherProperties = m_fileClient->GetProperties().Value; + auto& shareClient = *m_shareClient; auto fileName = LowercaseRandomString(); auto fileClient = shareClient.GetRootDirectoryClient().GetFileClient(fileName); @@ -2554,8 +2575,8 @@ namespace Azure { namespace Storage { namespace Test { auto symbolicLinkClient = shareClient.GetRootDirectoryClient().GetFileClient(LowercaseRandomString()); Files::Shares::CreateSymbolicLinkOptions createSymbolicLinkOptions; - createSymbolicLinkOptions.CreatedOn = otherProperties.SmbProperties.CreatedOn; - createSymbolicLinkOptions.LastWrittenOn = otherProperties.SmbProperties.LastWrittenOn; + createSymbolicLinkOptions.CreatedOn = properties.SmbProperties.CreatedOn; + createSymbolicLinkOptions.LastWrittenOn = properties.SmbProperties.LastWrittenOn; createSymbolicLinkOptions.Metadata = RandomMetadata(); createSymbolicLinkOptions.Group = "123"; @@ -2668,6 +2689,7 @@ namespace Azure { namespace Storage { namespace Test { // From file fileClient = shareClient.GetRootDirectoryClient().GetFileClient(LowercaseRandomString()); EXPECT_NO_THROW(fileClient.UploadFrom(tempFilename, uploadOptions)); + DeleteFile(tempFilename); properties = fileClient.GetProperties().Value; EXPECT_TRUE(properties.PosixProperties.FileMode.HasValue()); EXPECT_EQ(properties.PosixProperties.FileMode.Value().ToOctalFileMode(), octalMode); @@ -2684,15 +2706,9 @@ namespace Azure { namespace Storage { namespace Test { Files::Shares::Models::NfsFileType::Regular); } - TEST_F(FileShareFileClientTest, PremiumPosixPropertiesForCopy) + TEST_F(NfsFileShareClientTest, PremiumPosixPropertiesForCopy) { - auto shareServiceClient = *m_premiumShareServiceClient; - - auto shareName = LowercaseRandomString(); - auto shareClient = GetPremiumShareClientForTest(shareName); - Files::Shares::CreateShareOptions shareOptions; - shareOptions.EnabledProtocols = Files::Shares::Models::ShareProtocols::Nfs; - EXPECT_NO_THROW(shareClient.Create(shareOptions)); + auto& shareClient = *m_shareClient; auto sourceName = LowercaseRandomString(); auto sourceClient = shareClient.GetRootDirectoryClient().GetFileClient(sourceName); diff --git a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp index f2ff8c2a8b..cb91a62dde 100644 --- a/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-shares/test/ut/share_service_client_test.cpp @@ -111,7 +111,7 @@ namespace Azure { namespace Storage { namespace Test { } } - TEST_F(FileShareServiceClientTest, ListShares) + TEST_F(FileShareServiceClientTest, ListShares_PLAYBACKONLY_) { std::string prefix1 = LowercaseRandomString(); std::string prefix2 = LowercaseRandomString(); @@ -136,6 +136,12 @@ namespace Azure { namespace Storage { namespace Test { for (const auto& share : page.Shares) { result.insert(share.Name); + if (shareSet1.find(share.Name) != shareSet1.end() + || shareSet2.find(share.Name) != shareSet2.end()) + { + ASSERT_TRUE(share.Details.CreatedOn.HasValue()); + EXPECT_TRUE(IsValidTime(share.Details.CreatedOn.Value())); + } } } for (const auto& name : shareSet1)