From 2bbc348bf3944407847eed58980dc9c57eaf3530 Mon Sep 17 00:00:00 2001 From: gumbarros Date: Thu, 9 Jul 2026 21:00:08 -0300 Subject: [PATCH 1/2] Migrate docs to DocFX --- FluentStorage.sln | 19 +++ README.md | 52 +++--- docs/.gitignore | 2 + docs/FluentStorage.Docs.csproj | 7 + docs/articles/AWS-S3-Storage.md | 72 +++++++++ docs/articles/AWS-SQS.md | 25 +++ docs/articles/Azure-Blob-Storage.md | 161 +++++++++++++++++++ docs/articles/Azure-Data-Lake.md | 114 +++++++++++++ docs/articles/Azure-Files-Storage.md | 64 ++++++++ docs/articles/Azure-Key-Vault.md | 1 + docs/articles/Azure-Queue-Storage.md | 14 ++ docs/articles/Azure-Service-Bus.md | 47 ++++++ docs/articles/Backblaze-B2-Storage.md | 38 +++++ docs/articles/Blob-Storage.md | 59 +++++++ docs/articles/Cloudflare-R2-Storage.md | 45 ++++++ docs/articles/Custom-Sinks.md | 78 +++++++++ docs/articles/Data-Compression.md | 11 ++ docs/articles/Data-Encryption.md | 46 ++++++ docs/articles/Data-Transformation.md | 30 ++++ docs/articles/DigitalOcean-Spaces-Storage.md | 41 +++++ docs/articles/FTP-Storage.md | 25 +++ docs/articles/Getting-Started.md | 57 +++++++ docs/articles/Google-Cloud-Storage.md | 62 +++++++ docs/articles/Hetzner-Storage.md | 38 +++++ docs/articles/Message-Storage.md | 85 ++++++++++ docs/articles/Migration-Guide.md | 52 ++++++ docs/articles/MinIO-Storage.md | 46 ++++++ docs/articles/SFTP-Storage.md | 24 +++ docs/articles/Standard-Storage.md | 87 ++++++++++ docs/articles/Vultr-Storage.md | 40 +++++ docs/articles/Wasabi-Storage.md | 40 +++++ docs/articles/index.md | 1 + docs/articles/toc.yml | 59 +++++++ docs/docfx.json | 75 +++++++++ docs/images/logo.png | Bin 0 -> 1834 bytes docs/index.md | 11 ++ docs/templates/public/main.css | 4 + docs/toc.yml | 4 + 38 files changed, 1610 insertions(+), 26 deletions(-) create mode 100644 docs/.gitignore create mode 100644 docs/FluentStorage.Docs.csproj create mode 100644 docs/articles/AWS-S3-Storage.md create mode 100644 docs/articles/AWS-SQS.md create mode 100644 docs/articles/Azure-Blob-Storage.md create mode 100644 docs/articles/Azure-Data-Lake.md create mode 100644 docs/articles/Azure-Files-Storage.md create mode 100644 docs/articles/Azure-Key-Vault.md create mode 100644 docs/articles/Azure-Queue-Storage.md create mode 100644 docs/articles/Azure-Service-Bus.md create mode 100644 docs/articles/Backblaze-B2-Storage.md create mode 100644 docs/articles/Blob-Storage.md create mode 100644 docs/articles/Cloudflare-R2-Storage.md create mode 100644 docs/articles/Custom-Sinks.md create mode 100644 docs/articles/Data-Compression.md create mode 100644 docs/articles/Data-Encryption.md create mode 100644 docs/articles/Data-Transformation.md create mode 100644 docs/articles/DigitalOcean-Spaces-Storage.md create mode 100644 docs/articles/FTP-Storage.md create mode 100644 docs/articles/Getting-Started.md create mode 100644 docs/articles/Google-Cloud-Storage.md create mode 100644 docs/articles/Hetzner-Storage.md create mode 100644 docs/articles/Message-Storage.md create mode 100644 docs/articles/Migration-Guide.md create mode 100644 docs/articles/MinIO-Storage.md create mode 100644 docs/articles/SFTP-Storage.md create mode 100644 docs/articles/Standard-Storage.md create mode 100644 docs/articles/Vultr-Storage.md create mode 100644 docs/articles/Wasabi-Storage.md create mode 100644 docs/articles/index.md create mode 100644 docs/articles/toc.yml create mode 100644 docs/docfx.json create mode 100644 docs/images/logo.png create mode 100644 docs/index.md create mode 100644 docs/templates/public/main.css create mode 100644 docs/toc.yml diff --git a/FluentStorage.sln b/FluentStorage.sln index 461cec04..9b2cb466 100644 --- a/FluentStorage.sln +++ b/FluentStorage.sln @@ -31,6 +31,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentStorage.Tests.FTP", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentStorage.Azure", "FluentStorage.Azure\FluentStorage.Azure.csproj", "{D5FC3BA6-3C86-4DE5-A3E7-BA5F3AA1623C}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{FF16C733-2521-4D9B-9B32-7834EFD89BA8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentStorage.Docs", "docs\FluentStorage.Docs.csproj", "{765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -209,6 +213,18 @@ Global {D5FC3BA6-3C86-4DE5-A3E7-BA5F3AA1623C}.Release|x64.Build.0 = Release|Any CPU {D5FC3BA6-3C86-4DE5-A3E7-BA5F3AA1623C}.Release|x86.ActiveCfg = Release|Any CPU {D5FC3BA6-3C86-4DE5-A3E7-BA5F3AA1623C}.Release|x86.Build.0 = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|x64.ActiveCfg = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|x64.Build.0 = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|x86.ActiveCfg = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Debug|x86.Build.0 = Debug|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|Any CPU.Build.0 = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|x64.ActiveCfg = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|x64.Build.0 = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|x86.ActiveCfg = Release|Any CPU + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -216,4 +232,7 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {5D54105E-9F32-4017-85A4-4DB893648110} EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {765ADC12-5BCC-4BEE-8B6A-EB05800D7DAA} = {FF16C733-2521-4D9B-9B32-7834EFD89BA8} + EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 72566753..e5516bc9 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ FluentStorage is released under the permissive MIT License, so it can be used in ## Features -* Unified API to interface with all major cloud providers for [Blobs](https://github.com/robinrodricks/FluentStorage/wiki/Blob-Storage) and [Messaging](https://github.com/robinrodricks/FluentStorage/wiki/Message-Storage). +* Unified API to interface with all major cloud providers for [Blobs](docs/articles/Blob-Storage.md) and [Messaging](docs/articles/Message-Storage.md). * Provides a generic interface regardless on which storage provider you are using. @@ -33,9 +33,9 @@ FluentStorage is released under the permissive MIT License, so it can be used in * [Supports providers using individual Nuget packages](#packages), with hassle-free configuration and zero learning path. -* Implements [in-memory and on-disk versions](https://github.com/robinrodricks/FluentStorage/wiki/Standard-Storage) of all the abstractions, therefore you can develop fast on a local machine or use vendor-free serverless implementations for parts of your application. +* Implements [in-memory and on-disk versions](docs/articles/Standard-Storage.md) of all the abstractions, therefore you can develop fast on a local machine or use vendor-free serverless implementations for parts of your application. -* Implements [data transformation sinks](https://github.com/robinrodricks/FluentStorage/wiki/Data-Transformation) for encryption and compression. +* Implements [data transformation sinks](docs/articles/Data-Transformation.md) for encryption and compression. * Provides asynchronous API for all methods. @@ -49,18 +49,18 @@ FluentStorage supports the following cloud storage providers: | | Documentation Link | |---------------| --------------------------------------------------------------------------- | -| | [AWS S3](https://github.com/robinrodricks/FluentStorage/wiki/AWS-S3-Storage#connect-to-aws-s3) | -| | [Azure Blobs](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Blob-Storage) | -| | [Azure Files](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Files-Storage) | -| | [Azure DataLake](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Data-Lake) | -| | [GCP](https://github.com/robinrodricks/FluentStorage/wiki/Google-Cloud-Storage) | -| | [MinIO](https://github.com/robinrodricks/FluentStorage/wiki/MinIO-Storage) | -| | [Cloudflare R2](https://github.com/robinrodricks/FluentStorage/wiki/Cloudflare-R2-Storage) | -| |[DigitalOcean Spaces](https://github.com/robinrodricks/FluentStorage/wiki/DigitalOcean-Spaces-Storage) | -| | [Wasabi](https://github.com/robinrodricks/FluentStorage/wiki/Wasabi-Storage) | -| | [Backblaze B2](https://github.com/robinrodricks/FluentStorage/wiki/Backblaze-B2-Storage) | -| | [Hetzner](https://github.com/robinrodricks/FluentStorage/wiki/Hetzner-Storage) | -| | [Vultr](https://github.com/robinrodricks/FluentStorage/wiki/Vultr-Storage) | +| | [AWS S3](docs/articles/AWS-S3-Storage.md#connect-to-aws-s3) | +| | [Azure Blobs](docs/articles/Azure-Blob-Storage.md) | +| | [Azure Files](docs/articles/Azure-Files-Storage.md) | +| | [Azure DataLake](docs/articles/Azure-Data-Lake.md) | +| | [GCP](docs/articles/Google-Cloud-Storage.md) | +| | [MinIO](docs/articles/MinIO-Storage.md) | +| | [Cloudflare R2](docs/articles/Cloudflare-R2-Storage.md) | +| |[DigitalOcean Spaces](docs/articles/DigitalOcean-Spaces-Storage.md) | +| | [Wasabi](docs/articles/Wasabi-Storage.md) | +| | [Backblaze B2](docs/articles/Backblaze-B2-Storage.md) | +| | [Hetzner](docs/articles/Hetzner-Storage.md) | +| | [Vultr](docs/articles/Vultr-Storage.md) | To add support for a new provider, search for `[ADD STORAGE PROVIDER]` across all code files and make the required changes. @@ -73,17 +73,17 @@ Stable binaries are released on NuGet, and contain everything you need to use Cl | | Package | Latest Version | Downloads | Documentation | |---------------|--------------- |----------- |----------- |----------- | -| | **[FluentStorage](https://www.nuget.org/packages/FluentStorage)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.svg)](https://www.nuget.org/packages/FluentStorage) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.svg)](https://www.nuget.org/packages/FluentStorage) | [Standard](https://github.com/robinrodricks/FluentStorage/wiki/Standard-Storage) | -| | **[FluentStorage.AWS](https://www.nuget.org/packages/FluentStorage.AWS)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS) | [S3](https://github.com/robinrodricks/FluentStorage/wiki/AWS-S3-Storage), [SQS](https://github.com/robinrodricks/FluentStorage/wiki/AWS-SQS) | -| | **[FluentStorage.GCP](https://www.nuget.org/packages/FluentStorage.GCP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.GCP.svg)](https://www.nuget.org/packages/FluentStorage.GCP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.GCP.svg)](https://www.nuget.org/packages/FluentStorage.GCP) | [GCP](https://github.com/robinrodricks/FluentStorage/wiki/Google-Cloud-Storage) | -| | **[FluentStorage.FTP](https://www.nuget.org/packages/FluentStorage.FTP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.FTP.svg)](https://www.nuget.org/packages/FluentStorage.FTP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.FTP.svg)](https://www.nuget.org/packages/FluentStorage.FTP) | [FTP](https://github.com/robinrodricks/FluentStorage/wiki/FTP-Storage) | -| | **[FluentStorage.SFTP](https://www.nuget.org/packages/FluentStorage.SFTP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.SFTP.svg)](https://www.nuget.org/packages/FluentStorage.SFTP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.SFTP.svg)](https://www.nuget.org/packages/FluentStorage.SFTP) | [SFTP](https://github.com/robinrodricks/FluentStorage/wiki/SFTP-Storage) | +| | **[FluentStorage](https://www.nuget.org/packages/FluentStorage)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.svg)](https://www.nuget.org/packages/FluentStorage) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.svg)](https://www.nuget.org/packages/FluentStorage) | [Standard](docs/articles/Standard-Storage.md) | +| | **[FluentStorage.AWS](https://www.nuget.org/packages/FluentStorage.AWS)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS) | [S3](docs/articles/AWS-S3-Storage.md), [SQS](docs/articles/AWS-SQS.md) | +| | **[FluentStorage.GCP](https://www.nuget.org/packages/FluentStorage.GCP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.GCP.svg)](https://www.nuget.org/packages/FluentStorage.GCP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.GCP.svg)](https://www.nuget.org/packages/FluentStorage.GCP) | [GCP](docs/articles/Google-Cloud-Storage.md) | +| | **[FluentStorage.FTP](https://www.nuget.org/packages/FluentStorage.FTP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.FTP.svg)](https://www.nuget.org/packages/FluentStorage.FTP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.FTP.svg)](https://www.nuget.org/packages/FluentStorage.FTP) | [FTP](docs/articles/FTP-Storage.md) | +| | **[FluentStorage.SFTP](https://www.nuget.org/packages/FluentStorage.SFTP)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.SFTP.svg)](https://www.nuget.org/packages/FluentStorage.SFTP) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.SFTP.svg)](https://www.nuget.org/packages/FluentStorage.SFTP) | [SFTP](docs/articles/SFTP-Storage.md) | | | **[FluentStorage.Azure](https://www.nuget.org/packages/FluentStorage.Azure)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.svg)](https://www.nuget.org/packages/FluentStorage.Azure) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.svg)](https://www.nuget.org/packages/FluentStorage.Azure) | --- | -| | **[FluentStorage.Azure.Blobs](https://www.nuget.org/packages/FluentStorage.Azure.Blobs)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs) | [Blob](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Blob-Storage) | -| | **[FluentStorage.Azure.Files](https://www.nuget.org/packages/FluentStorage.Azure.Files)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Files.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Files) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Files.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Files) | [File](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Files-Storage) | -| | **[FluentStorage.Azure.ServiceBus](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.ServiceBus.svg)](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.ServiceBus.svg)](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus) | [ServiceBus](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Service-Bus) | -| | **[FluentStorage.Azure.KeyVault](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.KeyVault.svg)](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.KeyVault.svg)](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) | [KeyVault](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Key-Vault) | -| | **[FluentStorage.Azure.Queues](https://www.nuget.org/packages/FluentStorage.Azure.Queues)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Queues.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Queues) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Queues.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Queues) | [Queue](https://github.com/robinrodricks/FluentStorage/wiki/Azure-Queue-Storage) | +| | **[FluentStorage.Azure.Blobs](https://www.nuget.org/packages/FluentStorage.Azure.Blobs)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs) | [Blob](docs/articles/Azure-Blob-Storage.md) | +| | **[FluentStorage.Azure.Files](https://www.nuget.org/packages/FluentStorage.Azure.Files)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Files.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Files) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Files.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Files) | [File](docs/articles/Azure-Files-Storage.md) | +| | **[FluentStorage.Azure.ServiceBus](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.ServiceBus.svg)](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.ServiceBus.svg)](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus) | [ServiceBus](docs/articles/Azure-Service-Bus.md) | +| | **[FluentStorage.Azure.KeyVault](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.KeyVault.svg)](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.KeyVault.svg)](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) | [KeyVault](docs/articles/Azure-Key-Vault.md) | +| | **[FluentStorage.Azure.Queues](https://www.nuget.org/packages/FluentStorage.Azure.Queues)** | [![Version](https://img.shields.io/nuget/vpre/FluentStorage.Azure.Queues.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Queues) | [![Downloads](https://img.shields.io/nuget/dt/FluentStorage.Azure.Queues.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Queues) | [Queue](docs/articles/Azure-Queue-Storage.md) | @@ -149,7 +149,7 @@ You can use a single, consistent API to interact with multiple cloud providers, ## Documentation -Check the [Wiki](https://github.com/robinrodricks/FluentStorage/wiki). +Check the [Wiki](docs/index.md). diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..d5bcab17 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +_site/ +api/ diff --git a/docs/FluentStorage.Docs.csproj b/docs/FluentStorage.Docs.csproj new file mode 100644 index 00000000..ec989392 --- /dev/null +++ b/docs/FluentStorage.Docs.csproj @@ -0,0 +1,7 @@ + + + netstandard2.0 + false + + + diff --git a/docs/articles/AWS-S3-Storage.md b/docs/articles/AWS-S3-Storage.md new file mode 100644 index 00000000..0732614e --- /dev/null +++ b/docs/articles/AWS-S3-Storage.md @@ -0,0 +1,72 @@ + In order to use **AWS S3** or **S3-compatible** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to AWS S3 + +You can use the `AwsS3` methods to connect to [AWS S3](https://aws.amazon.com/s3/) storage. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AwsS3( + accessKeyId, secretAccessKey, sessionToken, bucketName); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("aws.s3://keyId=...;key=...;bucket=...;region=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **region** is the AWS account region (such as `us-east-1`). +- **serviceUrl** is the optional URL of the storage provider (for example for DigitalOcean). + +If **keyId** and **key** are omitted, the AWS SDK's default approach to credential resolution will be used. For example: if running in Lambda, it will assume the Lambda execution role; if there are credentials configured in ~/.aws, it will use those; etc. See [AWS SDK Documentation](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html) for more details. + +## AWS CLI Profile Support + +If you already have credentials in the local `~/.aws/credentials` generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. Credentials file example: + + +```ini +[default] +aws_access_key_id = ASIAV44FYRYLSBYIPG67 +aws_secret_access_key = zhuh8nSUAR0pR5jbi4vhUym95JsC7ay3iiE2v9Jq +aws_session_token = FQoGZXIvY... + +[anotherProfile] +aws_access_key_id = ASIAXEKXYP2H7MRN45FD +aws_secret_access_key = 8i6jnOOurRtTiK4CWN+uUD115SYS8++ZuQjO/LtW +aws_session_token = FQoGZXI... + +... +``` + +To connect using say `anotherProfile` credentials, construct the instance by: + +```csharp +IBlobStorage bs = StorageFactory.Blobs.AwsS3( + "anotherProfile", + "bucketName", + "region"); +``` + +[AwsCliCredentials](xref:FluentStorage.AWS.AwsCliCredentials) class contains `credentials` file parrsing logic and some utility methods you mind find useful, for instance enumerating available profiles and so on. + +## Special features for S3 + +- Remote directory recursion occurs on S3 cloud rather than locally (`ListFolderAsync` API). +- `ListOptions` has a `PageSize` parameter which can be used to control the internal paging size used for remote recursion. + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/AWS-SQS.md b/docs/articles/AWS-SQS.md new file mode 100644 index 00000000..9842f82e --- /dev/null +++ b/docs/articles/AWS-SQS.md @@ -0,0 +1,25 @@ + In order to use [AWS Simple Queue Service](https://aws.amazon.com/sqs/) you need to reference +[![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) first. The provider wraps around the standard AWS SDK. + +To construct a publisher use the following: + +```csharp +IMessagePublisher queuePublisher = StorageFactory.Messages.AmazonSQSMessagePublisher( + accessKeyId, + secretAccessKey, + serviceUrl, + queueName, + regionEndpoint); + +IMessagePublisher topicPublisher = StorageFactory.Messages.AmazonSQSMessageReceiver( + accessKeyId, + secretAccessKey, + serviceUrl, + queueName, + regionEndpoint); +``` + +- **accessKeyId** and **secretAccessKey*) are credentials to access the queue. +- **serviceUrl** indicates the service URL, for instance `https://sqs.us-east-1.amazonaws.com` +- **queueName** is the name of the queue +- **retionEndpoint** is optional and defaults to `USEast1` diff --git a/docs/articles/Azure-Blob-Storage.md b/docs/articles/Azure-Blob-Storage.md new file mode 100644 index 00000000..cf240eaa --- /dev/null +++ b/docs/articles/Azure-Blob-Storage.md @@ -0,0 +1,161 @@ +In order to use Microsoft Azure blob or file storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs/). + +## Connect to Azure Blob Storage + + +To create Azure Blob Storage with Shared Key: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithSharedKey(accountName, accountKey); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithSharedKey(accountName, accountKey, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithSharedKey(accountName, accountKey, serviceUri, cloudEnvironment); +``` + +To create Azure Blob Storage with Azure AD (optionally with AD Authority endpoint): + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, applicationId, applicationSecret); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, applicationId, applicationSecret, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, applicationId, applicationSecret, activeDirectoryAuthEndpoint); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithAzureAd(accountName, tenantId, applicationId, applicationSecret, activeDirectoryAuthEndpoint, cloudEnvironment); +``` + +To create Azure Blob Storage with Token Credentials: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithTokenCredential(accountName, tokenCredential); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithTokenCredential(accountName, tokenCredential, cloudEnvironment); +``` + +To create Azure Blob Storage with Managed Identity: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName, clientId); +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithMsi(accountName, clientId, cloudEnvironment); +``` + +To create an instance of Azure Blob Storage that wraps around the native SDK `CloudBlobClient` (use the native option with caution) + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorage(client); +``` + +To create an instance of Azure Blob Storage that uses the local development storage emulator: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithLocalEmulator(); +``` + +## Connection Strings + +To use connection strings, first register the module when your program starts by calling `StorageFactory.Modules.UseAzureBlobStorage();` then use the following: + +```csharp +//using account name and key +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("azure.blob://account=account_name;key=secret_value"); + +//local development emulator +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("azure.blob://development=true"); +``` + +This storage is working with `block blobs` only. We are planning to add `append blobs` support but that requires some architectural changes and as always you're welcome to help. + +This package treats the first part of the path as **container name**. This allows you to have access to all the containers at once. For instance, path `root/file.txt` creates file `file.txt` in the root of container called `root`. `root/folder1/file.txt` creates file `file.txt` in folder `folder1` under container `root` and so on. You can check if the folder returned is a container by referring to `isContainer` custom property (`blob.Properties["IsContainer"] == "True"`). + + +## Native Operations + +You can access some native, blob storage specific operations by casting (unsafe) `IBlobStorage` to `IAzureBlobStorage`. + +#### SAS Tokens + +You can obtain a SAS (Shared Access Signature) tokens to the following objects: + +##### Storage Account + +Getting SAS token for an account involves granting limited access to entire account. To grant it, for instance, for one hour from now, create a policy first: + +```csharp +var policy = new AccountSasPolicy(DateTimeOffset.?, TimeSpan.FromHours(1)); +``` + +By default the policy is configured to give only `List` and `Read` permissions, meaning that users will be able to list containers and blobs, and also read them. You can customise policy permissions by modifying the `Permissions` flag property, for instance to also have `Write` permission you could explicitly assign it: + +```csharp +policy.Permissions = + AccountSasPermission.List | + AccountSasPermission.Read | + AccountSasPermission.Write; +``` + +Then get the policy signature: + +```csharp +string sasUrl = await _native.GetStorageSasAsync(policy, true); +``` + +The second boolean parameter indicates whether to return full URL to the storage with SAS policy or only the policy itself. Setting it to `true` is useful if you want to use this URL in say `Azure Storage Explorer` to attach that account directly. Also, in order to connect to blob storage with SAS, you need the full URL: + +To connect to an account using a policy, use the following factory method: + +```csharp +IBlobStorage sasInstance = StorageFactory.Blobs.AzureBlobStorageWithSas(sasUrl); +``` + +##### Container + +You can get container's *Shared Access Signature* in the same way as account's one, by calling to + +```csharp +string sasUrl = await _native.GetContainerSasAsync(containerName, policy, true); +``` + +This returns SAS URL that can be used in Azure Storage Explorer, or you can use it to connect in this library itself: + +```csharp +IBlobStorage sasInstance = StorageFactory.Blobs.AzureBlobStorageFromSas(sasUrl); +``` + +Note that the method's signature is identical to account's one, actually it's the same method. FluentStorage takes care of figuring out whether SAS URL is for a container or for a storage account automatically. However, in case of a container SAS, the root folder in `IBlobStorage` instance is the container itself. + +##### Blob + +In order to get a signature for a specific *blob*, you can use use `GetBlobSasAsync` method. Calling it without any parameters for a blob, returns a *read-only URL valid for 1 hour from now*: + +```csharp +string publicUrl = await _native.GetBlobSasAsync(path); +``` + +You can then redistribute this URL amongst other users so they can download the content. + +To customise the policy, pass additional parameters. For instance, to grant *read/write access for 12 hours* you can write the following code: + +```csharp +var policy = new BlobSasPolicy(TimeSpan.FromHours(12)) +{ + Permissions = BlobSasPermission.Read | BlobSasPermission.Write +}; + +string publicUrl = await _native.GetBlobSasAsync(path, policy); +``` + + +#### Blob Lease + +There is a helper utility method to acquire a block blob lease or a container lease, which is useful for virtual transactions support. For instance: + + +```csharp +using(AzureStorageLease lease = await _blobs.AcquireLeaseAsync(id, timeSpan)) +{ + // your code +} +``` + +Where the first parameter is blob id or container name, and the second is lease duration. The `BlobLease` returned implements `IDisposable` pattern so that on exit the lease is returned. Note that if blob doesn't exist, current implementation will create a zero-size file and then acquire a least, just for your convenience. The blob is not deleted automatically though. + +`AcquireLeaseAsync` also has an option to wait for the lease to be returned (third optional argument) which when set to true causes this library to try to acquire a lease every second until it's released, and re-lease it. + +It also exposes `RenewLeaseAsync()` method to renew the lease explicitly. diff --git a/docs/articles/Azure-Data-Lake.md b/docs/articles/Azure-Data-Lake.md new file mode 100644 index 00000000..ea4872af --- /dev/null +++ b/docs/articles/Azure-Data-Lake.md @@ -0,0 +1,114 @@ +## Azure Data Lake Gen 1 [deprecated] + + +In order to use Azure DataLake Gen 1, reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.DataLake.svg)](https://www.nuget.org/packages/FluentStorage.Azure.DataLake/) package first. + +To create using a factory method, use the following signature: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureDataLakeGen1StoreByClientSecret( + string accountName, + string tenantId, + string principalId, + string principalSecret, + int listBatchSize = 5000) +``` + +The last parameter *listBatchSize* indicates how to query storage for list operations - by default a batch of 5k items will be used. Note that the larger the batch size, the more data you will receive in the request. This speeds up list operations, however may result in HTTP time-out the slower your internet connection is. This feature is not available in the standard .NET SDK and was implemented from scratch. + +You can also use connection strings: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("azure.datalake.gen1://account=...;tenantId=...;principalId=...;principalSecret=...;listBatchSize=..."); +``` + +the last parameter *listBatchSize* is optional and defaults to `5000`. + +## Azure Data Lake Gen 2 + + +In order to use Azure DataLake Gen 2, reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs/) package first. + +Gen 2 is the new generation of the storage API, and you should always prefer it to Gen 1 accounts when you can. Both Gen 1 and Gen 2 providers are located in the same NuGet package. + +Gen 2 provider is 100% compatible with hierarchical namespaces. When you use blob path, the first part of the path is filesystem name, i.e. `storage.WriteTextAsync("filesystem/folder/subfolder/.../file.extension`. Apparently you cannot create files in the root folder, they always need to be prefixed with filesystem name. + +If filesystem doesn't exist, we will try to create it for you, if the account provided has enough permissions to do so. + +#### Authentication + +You can authenticate in the ways described below. To use connection strings, don't forget to call `StorageFactory.Modules.UseAzureDataLake()` somewhere when your program starts. + +##### Using **Shared Key Authentication** + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureDataLakeGen2StoreBySharedAccessKey( + accountName, + sharedKey); +``` + +or + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString( + "azure.datalake.gen2://account=...;key=..."); +``` + +##### Using **Service Principal** + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureDataLakeGen2StoreByClientSecret( + accountName, + tenantId, + principalId, + principalSecret); +``` + +or + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString( + "azure.datalake.gen2://account=...;tenantId=...;principalId=...;principalSecret=..."); +``` + +##### Using **Managed Service Identity** + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureDataLakeGen2StoreByManagedIdentity( + accountName); +``` + +or + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString( + "azure.datalake.gen2://account=...;msi"); +``` + +#### Permissions Management + +ADLS Gen 2 [supports](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-access-control) RBAC and [POSIX](https://www.usenix.org/legacy/publications/library/proceedings/usenix03/tech/freenix03/full_papers/gruenbacher/gruenbacher_html/main.html) like permissions on both file and folder level. FluentStorage fully supports permissions management on those and exposes simplified easy-to-use API to drive them. + +Because permission management is ADLS Gen 2 specific feature, you cannot use `IBlobStorage` interface, however you can cast it to `IAzureDataLakeGen2BlobStorage` which in turn implements `IBlobStorage` as well. + +In order to get permissions for an object located on a specific path, you can call the API: + +```csharp +IBlobStorage genericStorage = StorageFactory.Blobs.AzureDataLakeGen2StoreByClientSecret(name, key); +IAzureDataLakeGen2BlobStorage gen2Storage = (IAzureDataLakeGen2BlobStorage)genericStorage; + +//get permissions +AccessControl access = await _storage.GetAccessControlAsync(path); +``` + +`AccessControl` is a self explanatory structure that contains information about owning user, owning group, their permissions, and any custom ACL entries assigned to this object. + +In order to set permissions, you need to call `SetAccessControlAsync` passing back modified `AccessControl` structure. Let's say I'd like to add *write* access to a user with ID `6b157067-78b0-4478-ba7b-ade5c66f1a9a` (Active Directory Object ID). I'd write code like this (using the structure we've just got back from `GetAccessControlAsync`): + +```csharp +// add user to custom ACL +access.Acl.Add(new AclEntry(ObjectType.User, userId, false, true, false)); + +//update the ACL on Gen 2 storage +await _storage.SetAccessControlAsync(path, access); +``` \ No newline at end of file diff --git a/docs/articles/Azure-Files-Storage.md b/docs/articles/Azure-Files-Storage.md new file mode 100644 index 00000000..9f989f4d --- /dev/null +++ b/docs/articles/Azure-Files-Storage.md @@ -0,0 +1,64 @@ +In order to use Microsoft Azure blob or file storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.Blobs.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Blobs/). + +## Connect to Azure Files + + +To create file share by storage name and key: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFiles(accountName, accountKey); +IBlobStorage storage = StorageFactory.Blobs.AzureFiles(accountName, accountKey, serviceUri); +``` + +To create Azure Files with Shared Key: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithSharedKey(accountName, accountKey); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithSharedKey(accountName, accountKey, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithSharedKey(accountName, accountKey, serviceUri, cloudEnvironment); +``` + +To create Azure Files with Azure AD (optionally with AD Authority endpoint): + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithAzureAd(accountName, tenantId, applicationId, applicationSecret); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithAzureAd(accountName, tenantId, applicationId, applicationSecret, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithAzureAd(accountName, tenantId, applicationId, applicationSecret, activeDirectoryAuthEndpoint); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithAzureAd(accountName, tenantId, applicationId, applicationSecret, activeDirectoryAuthEndpoint, cloudEnvironment); +``` + +To create Azure Files with Token Credentials: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithTokenCredential(accountName, tokenCredential); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithTokenCredential(accountName, tokenCredential, cloudEnvironment); +``` + +To create Azure Files with Managed Identity: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithMsi(accountName); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithMsi(accountName, cloudEnvironment); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithMsi(accountName, clientId); +IBlobStorage storage = StorageFactory.Blobs.AzureFilesWithMsi(accountName, clientId, cloudEnvironment); +``` + +## Connection Strings + +To create with connection string, first register the module when your program starts by calling `StorageFactory.Modules.UseAzureBlobStorage();` then use the following: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("azure.file://account=account_name;key=secret_value"); +``` + +To create an instance of Azure Files that wraps around the native SDK `ShareServiceClient` (use the native option with caution) + +```csharp +IBlobStorage storage = StorageFactory.Blobs.AzureFiles(client); +``` + +## Native Operations + +You can access some native, blob storage specific operations by casting (unsafe) `IBlobStorage` to `IAzureBlobStorage`. + +See [this article](Azure-Blob-Storage.md#native-operations) for more info. \ No newline at end of file diff --git a/docs/articles/Azure-Key-Vault.md b/docs/articles/Azure-Key-Vault.md new file mode 100644 index 00000000..9962e43c --- /dev/null +++ b/docs/articles/Azure-Key-Vault.md @@ -0,0 +1 @@ + In order to use Key Vault reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.KeyVault.svg)](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) package first. diff --git a/docs/articles/Azure-Queue-Storage.md b/docs/articles/Azure-Queue-Storage.md new file mode 100644 index 00000000..956f053c --- /dev/null +++ b/docs/articles/Azure-Queue-Storage.md @@ -0,0 +1,14 @@ + In order to use Azure Queue Storage reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.Queues.svg)](https://www.nuget.org/packages/FluentStorage.Azure.Queues) package first. + + +```csharp +IMessagePublisher publisher = StorageFactory.Messages.AzureStorageQueuePublisher(); + +IMessageReceiver receiver = StorageFactory.Messages.AzureStorageQueueReceiver(); +``` + +```csharp +IMessagePublisher publisher = StorageFactory.Messages.PublisherFromConnectionString("azure.queue://account=...;key=...;queue=..."); + +IMessageReceiver receiver = StorageFactory.Messages.ReceiverFromConnectionString("azure.queue://account=..;key=...;queue=..."); +``` diff --git a/docs/articles/Azure-Service-Bus.md b/docs/articles/Azure-Service-Bus.md new file mode 100644 index 00000000..c5c8dd6e --- /dev/null +++ b/docs/articles/Azure-Service-Bus.md @@ -0,0 +1,47 @@ + + In order to use Microsoft Azure Service Bus you need to reference +[![NuGet](https://img.shields.io/nuget/v/FluentStorage.Azure.ServiceBus.svg)](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus/) first. The provider wraps around the new Microsoft Service Bus SDK package (`Azure.Messaging.ServiceBus`). + +This provider supports both topics and queues, for publishing and receiving. + +### IMessageReceiver + +This new package introduces the following methods for the `IMessageReceiver` interface: + +- **AzureServiceBusTopicReceiver()** - Creates Azure Service Bus Receiver for topic and subscriptions +- **AzureServiceBusQueueReceiver()** - Creates Azure Service Bus Receiver for queues + +Where `ServiceBusClientOptions` and `ServiceBusProcessorOptions` are optional parameters to fine-tune the options for the `ServiceBusClient` and `ServiceBusProcessor`: + +- [ServiceBusClientOptions docs](https://learn.microsoft.com/en-us/dotnet/api/azure.messaging.servicebus.servicebusclientoptions?view=azure-dotnet) +- [ServiceBusProcessorOptions docs](https://learn.microsoft.com/en-us/dotnet/api/azure.messaging.servicebus.servicebusprocessoroptions?view=azure-dotnet) + +### IMessenger +The interface used to send messages is `IMessenger` but is a bit limited because it doesn't have the notions of queues, topics and subscriptions. + +So, if you want to use the `IMessenger` interface, when passing a channel you have to use this format: +- _`q/queue name`_ for working with queues +- _`t/topic name`_ for working with topics +- _`t/topic name/subscription name`_ for working with subscriptions (as subscriptions are embedded inside topics) + +All the public methods of `IMessenger` that take a channel (or channel collections) in input need to follow this naming convention. + +### IAzureServiceBusMessenger + +The new interface `IAzureServiceBusMessenger` extends `IMessenger`. So, if you cast your `AzureServiceBusMessenger` messenger to `IAzureServiceBusMessenger` you have access to these utility methods that target directly queues, messages, and subscriptions: + +- **SendToQueueAsync()** - Sends a collection of messages to the specified queue asynchronously. +- **SendToQueueAsync()** - Sends a message to the specified queue asynchronously. +- **SendToTopicAsync()** - Sends a collection of messages to the specified topic asynchronously. +- **SendToTopicAsync()** - Sends a single message to the specified topic asynchronously. +- **SendToSubscriptionAsync()** - Sends a collection of messages to the specified topic subscription asynchronously. +- **SendToSubscriptionAsync()** - Sends a single message to the specified topic subscription asynchronously. +- **CreateQueueAsync()** - Create a new Queue in Azure ServiceBus +- **CreateTopicAsync()** - Create a new Topic in Azure ServiceBus +- **CreateSubScriptionAsync()** - Create a new Subscription in Azure ServiceBus +- **DeleteQueueAsync()** - Deletes the specified queue asynchronously. +- **DeleteSubScriptionAsyn()** - Deletes the specified topic subscription asynchronously. +- **DeleteTopicAsync()** - Deletes the specified topic asynchronously. +- **CountQueueAsync()** - Counts the number of messages in the specified queue asynchronously. +- **CountSubScriptionAsync()** - Counts the number of messages in the specified topic subscription asynchronously. +- **CountTopicAsync()** - Counts the number of messages in the specified topic asynchronously \ No newline at end of file diff --git a/docs/articles/Backblaze-B2-Storage.md b/docs/articles/Backblaze-B2-Storage.md new file mode 100644 index 00000000..83dbf2de --- /dev/null +++ b/docs/articles/Backblaze-B2-Storage.md @@ -0,0 +1,38 @@ + In order to use **Backblaze B2** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to Backblaze B2 + +You can use the `BackblazeB2` method to connect to [Backblaze B2](https://www.backblaze.com/cloud-storage/) storage servers (S3 compatible). + +```csharp +IBlobStorage storage = StorageFactory.Blobs.BackblazeB2( + accessKeyId, secretAccessKey, bucketName, region); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("backblaze.b2://keyId=...;key=...;bucket=...;region=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **region** is the Bucket region (e.g. `us-west-004`). + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/Blob-Storage.md b/docs/articles/Blob-Storage.md new file mode 100644 index 00000000..3ff75956 --- /dev/null +++ b/docs/articles/Blob-Storage.md @@ -0,0 +1,59 @@ +## Providers + +- [AWS S3 Storage](AWS-S3-Storage.md) +- [Cloudflare R2 Storage](Cloudflare-R2-Storage.md) +- [MinIO Storage](MinIO-Storage.md) +- [DigitalOcean Spaces Storage](DigitalOcean-Spaces-Storage.md) +- [Wasabi Storage](Wasabi-Storage.md) +- [Backblaze B2 Storage](Backblaze-B2-Storage.md) +- [Hetzner Storage](Hetzner-Storage.md) +- [Vultr Storage](Vultr-Storage.md) +- [Google Cloud Storage](Google-Cloud-Storage.md) +- [Azure Blob Storage](Azure-Blob-Storage.md) +- [Azure File Storage](Azure-Files-Storage.md) +- [Azure Data Lake](Azure-Data-Lake.md) +- [Azure Key Vault](Azure-Key-Vault.md) +- [In-Memory, Local Disk & ZIP](Standard-Storage.md) +- [FTP & FTPS Storage](FTP-Storage.md) +- [SFTP Storage](SFTP-Storage.md) + +## API + +All of the core methods are defined in the `IBlobStorage` interface. This is the interface that's enough for a new storage provider to implement in order to add a new storage provider. + +However, some providers support more than just basic operations, for instance Azure Blob Storage supports blob leasing, shared access signatures etc., therefore it actually implements `IAzureBlobStorage` interface that in turn implements `IBlobStorage` interface, and extends the functionality further. Same goes for AWS S3 and others. + +## Getting Started + +Blob Storage stores files. A file has only two properties - `ID` and raw data. If you build an analogy with disk filesystem, file ID is a file name. + +Blob Storage is really simple abstraction - you read or write file data by it's ID, nothing else. + +The entry point to a blog storage is [IBucket](xref:FluentStorage.Storage.IBucket) interface. This interface is small but contains all possible methods to work with blobs, such as uploading and downloading data, listing storage contents, deleting files etc. The interface is kept small so that new storage providers can be added easily, without implementing a plethora of interface methods. + +In addition to this interface, there are plency of extension methods which enrich the functionality, therefore you will see more methods than this interface actually declares. They add a lot of useful and functionality rich methods to work with storage. For instance, `IBlobStorage` upload functionality only works with streams, however extension methods allows you to upload text, stream, file or even a class as a blob. Extension methods are also provider agnostic, therefore all the rich functionality just works and doesn't have to be reimplemented in underlying data provider. + +All the storage implementations can be created either directly or using factory methods available in the `FluentStorage.StorageFactory.Blobs` class. More methods appear in that class as you reference a **NuGet** package containing specific implementations, however there are a few built-in implementations available out of the box as well. After referencing an appropriate package from NuGet you can call to a storage factory to create a respective storage implementation. + +You can also use connection strings to create blob storage instances. Connection strings are often useful if you want to completely abstract yourself from the underlying implementation. Please read the appropriate implementation details for connection string details. For instance, to create an instance of Azure Blob Storage provider you could write: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("azure.blobs://...parameters..."); +``` + +In this example we create a blob storage implementation which happens to be Microsoft Azure blob storage. The project is referencing an appropriate [nuget package](https://www.nuget.org/packages/FluentStorage.Microsoft.Azure.Storage.Blobs/). As blob storage methods promote streaming we create a `MemoryStream` over a string for simplicity sake. In your case the actual stream can come from a variety of sources. + +```csharp +public async Task BlobStorage_sample2() +{ + IBlobStorage storage = StorageFactory.Blobs.AzureBlobStorageWithSharedKey( + "storage name", + "storage key"); + + //upload it + await storage.WriteTextAsync("mycontainer/someid", "test content"); + + //read back + string content = await storage.ReadTextAsync("mycontainer/someid"); +} +``` diff --git a/docs/articles/Cloudflare-R2-Storage.md b/docs/articles/Cloudflare-R2-Storage.md new file mode 100644 index 00000000..a5133424 --- /dev/null +++ b/docs/articles/Cloudflare-R2-Storage.md @@ -0,0 +1,45 @@ + In order to use **Cloudflare R2** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to Cloudflare R2 + +You can use the `CloudflareR2` method to connect to [Cloudflare R2](https://www.cloudflare.com/products/r2/) storage servers (S3 compatible). + +For `cloudflareAccountId` you need to enter your Cloudflare Account ID that is used to build the bucket URL like `"https://{cloudflareAccountId}.r2.cloudflarestorage.com"`. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.CloudflareR2( + accessKeyId, secretAccessKey, bucketName, cloudflareAccountId); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("cloudflare.r2://keyId=...;key=...;bucket=...;accountId=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **accountId** is your Cloudflare Account ID. + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Special features for R2 + +- In file upload requests, `DisablePayloadSigning` is set to true because R2 does not support "Streaming Signature V4". +- In file upload requests, `PutObjectAsync` SDK API used instead of `TransferUtility` + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/Custom-Sinks.md b/docs/articles/Custom-Sinks.md new file mode 100644 index 00000000..7ed99589 --- /dev/null +++ b/docs/articles/Custom-Sinks.md @@ -0,0 +1,78 @@ +Part of the [Data Transformation](Data-Transformation.md) suite of functions. + +## Build Your Own Sink + +Implementing your own transformation sink is a matter of implementing a new class derived from [`ITransformSink`](xref:FluentStorage.Storage.Sinks.ITransformSink) interface, which only has two methods: + +```csharp + public interface ITransformSink + { + Stream OpenReadStream(string fullPath, Stream parentStream); + + Stream OpenWriteStream(string fullPath, Stream parentStream); + } +``` + +The first one is called when FluentStorage opens a blob for reading, so that you can replace original stream passed in `parentStream` with your own. The second one does the reverse. For instance, have a look at the implementation of `Gzip` sink, as it's the easiest one: + +```csharp +public class GZipSink : ITransformSink +{ + private readonly CompressionLevel _compressionLevel; + + public GZipSink(CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + _compressionLevel = compressionLevel; + } + + public Stream OpenReadStream(string fullPath, Stream parentStream) + { + if(parentStream == null) + return null; + + return new GZipStream(parentStream, CompressionMode.Decompress, false); + } + + public Stream OpenWriteStream(string fullPath, Stream parentStream) + { + return new GZipStream(parentStream, _compressionLevel, false); + } +} +``` + +This sink simply takes incoming stream and wraps it around in the standard built-in `GZipStream` from `System.IO.Compression` namespace. + +## Passing Your Sink To Storage + +In order to use the sink, you can simply call `.WithSinks` extension method and pass the sink you want to use. For instance, to enable GZipSink do the following: + +```csharp +IBlobStorage storage = StorageFactory.Blobs + .XXX() + .WithSinks(new GZipSink()); +``` + +You can also create an extension method if you use this often: + +```csharp +public static IBlobStorage WithGzipCompression( + this IBlobStorage blobStorage, CompressionLevel compressionLevel = CompressionLevel.Optimal) +{ + return blobStorage.WithSinks(new GZipSink(compressionLevel)); +} +``` + +## Chaining Sinks + +`.WithSinks` extension method in fact accept an array of sinks, which means that sinks can be chained together. This is useful when you need to do multiple transformations at the same time. For instance, if I would like to both compress, and encrypt data in the target storage, I could initialise my storage in the following way: + +```csharp +IBlobStorage encryptedAndCompressed = + StorageFactory.Blobs + .InMemory() + .WithSinks( + new GZipSink(), + new SymmetricEncryptionSink("To6X5XVaNNMKFfxssJS6biREGpOVZjEIC6T7cc1rJF0="))) +``` + +Note that declaration order matters here - when writing, the data is **compressed first**, and **encrypted second**. diff --git a/docs/articles/Data-Compression.md b/docs/articles/Data-Compression.md new file mode 100644 index 00000000..c347e606 --- /dev/null +++ b/docs/articles/Data-Compression.md @@ -0,0 +1,11 @@ +Part of the [Data Transformation](Data-Transformation.md) suite of functions. + +## Gzip Compression + +To create the sink, call extension method `WithGzipCompression` and optionally pass a *compression level* which defaults to `Optimal`: + +```csharp +IBlobStorage storage = StorageFactory.Blobs + .XXX() + .WithGzipCompression(CompressionLevel compressionLevel = CompressionLevel.Optimal) +``` diff --git a/docs/articles/Data-Encryption.md b/docs/articles/Data-Encryption.md new file mode 100644 index 00000000..5aa6fcce --- /dev/null +++ b/docs/articles/Data-Encryption.md @@ -0,0 +1,46 @@ +Part of the [Data Transformation](Data-Transformation.md) suite of functions. + +## AES Symmetric Encryption + +This sink implements [symmetric encryption](https://www.venafi.com/blog/what-symmetric-encryption) for upload/download data. I.e. uploaded data is encrypted with a key, and decrypted after download. + +It uses [AES](https://learn.microsoft.com/en-us/dotnet/api/system.security.cryptography.aes?view=net-7.0) encryption with default settings. You control which Key and IV are used. + +To add: + +```csharp +IBlobStorage storage = StorageFactory.Blobs + .XXX() + .WithAesSymmetricEncryption(string encryptionKey, string encryptionSecret) +``` + +## Rijndael Symmetric Encryption + +_**Note: Rijndael is obsolete in .NET 6 and beyond!**_ + +This sink implements [symmetric encryption](https://www.venafi.com/blog/what-symmetric-encryption) for upload/download data. I.e. uploaded data is encrypted with a key, and decrypted after download. + +It uses [Rijndael](https://web.archive.org/web/20070711123800/http://csrc.nist.gov/CryptoToolkit/aes/rijndael/Rijndael-ammended.pdf) encryption with default settings, which is a superset of **AES** encryption algorithm (read about [differences](https://stackoverflow.com/a/748645/80858)). You control which Key and IV are used. + +To add: + +```csharp +IBlobStorage storage = StorageFactory.Blobs + .XXX() + .WithSymmetricEncryption(string encryptionKey, string encryptionSecret) +``` + +The encryption key is a baase64 encoded binary key. To generate it, you can use the following snippet: + +```csharp +void Main() +{ + var cs = new RijndaelManaged(); + cs.GenerateKey(); + string keyBase64 = Convert.ToBase64String(cs.Key); + + Console.WriteLine("new encryption key:" + keyBase64); +} +``` + +Note that it's your own responsibility to store the key securely, make sure it's not put in plaintext anywhere it can be stoken from! \ No newline at end of file diff --git a/docs/articles/Data-Transformation.md b/docs/articles/Data-Transformation.md new file mode 100644 index 00000000..218f7012 --- /dev/null +++ b/docs/articles/Data-Transformation.md @@ -0,0 +1,30 @@ +## Sinks + - [GZip Compression](Data-Compression.md#gzip-compression) + - [AES Encryption](Data-Encryption.md#aes-symmetric-encryption) + - [Rijndael Encryption](Data-Encryption.md#rijndael-symmetric-encryption) + - [Custom Sinks](Custom-Sinks.md) + +## Architecture + +![](https://raw.githubusercontent.com/robinrodricks/FluentStorage/develop/.github/sinks-memory.svg) + +## Getting Started + +Transform sinks is another awesome feature of FluentStorage that works across all the storage providers. Transform sinks allow you to **transform data stream** for both upload and download to somehow transform the underlying stream of data. Examples of transform sinks would be *gzipping data* transparently, *encrypting it*, and so on. + +Let's say you would like to *gzip* all of the files that you upload/download to a storage. You can do that in the following way: + +```csharp +IBlobStorage myGzippedStorage = StorageFactory.Blobs + .AzureBlobStorageWithSharedKey("name", "key") + .WithGzipCompression(); +``` + +Then use the storage as you would before - all the data is compressed as you write it (with any `WriteXXX` method) and decompressed as you read it (with any `ReadXXX` method). + + +## Implementation + +Due to the nature of the transforms, they can change both the underlying data, and stream size, therefore there is an issue with storage providers, as they need to know beforehand the size of the blob you are uploading. The matter becomes more complicated when some implementations need to calculate other statistics of the data before uploading i.e. hash, CRC and so on. Therefore the only *reliable* way to stream transformed data is to actually perform all of the transofrms, and then upload it. In this implementation, FluentStorage uses in-memory transforms to achieve this, however does it extremely efficiently by using [Microsoft.IO.RecyclableMemoryStream](https://github.com/Microsoft/Microsoft.IO.RecyclableMemoryStream) package that performs memory pooling and reclaiming for you so that you don't need to worry about software slowdows. You can read more about this technique [here](http://www.philosophicalgeek.com/2015/02/06/announcing-microsoft-io-recycablememorystream/). + +This also means that today a transform sink can upload a stream only as large as the amount of RAM available on your machine. I am, however, thinking of ways to go further than that, and there are some beta implementations available that might see the light soon. diff --git a/docs/articles/DigitalOcean-Spaces-Storage.md b/docs/articles/DigitalOcean-Spaces-Storage.md new file mode 100644 index 00000000..eb239f79 --- /dev/null +++ b/docs/articles/DigitalOcean-Spaces-Storage.md @@ -0,0 +1,41 @@ + In order to use **DigitalOcean Spaces** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to DigitalOcean Spaces + +You can use the `DigitalOceanSpaces` method to connect to [DigitalOcean Spaces](https://www.digitalocean.com/products/spaces) storage (S3 compatible). + +For `digitalOceanRegion`, you need to enter the DigitalOcean Region endpoint, like `nyc3`. This is internally combined into the following string which is used as the Service URL: `"https://{digitalOceanRegion}.digitaloceanspaces.com"`. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.DigitalOceanSpaces( + accessKeyId, secretAccessKey, bucketName, digitalOceanRegion); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("do.spaces://keyId=...;key=...;bucket=...;region=...;sessionToken=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **region** is the Digital Ocean region code. +- **sessionToken** is the optional token for temporary AWS credentials. + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/FTP-Storage.md b/docs/articles/FTP-Storage.md new file mode 100644 index 00000000..9bbf4ed0 --- /dev/null +++ b/docs/articles/FTP-Storage.md @@ -0,0 +1,25 @@ + To use this, you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.FTP.svg)](https://www.nuget.org/packages/FluentStorage.FTP/) package first, which wraps [FluentFTP](https://github.com/robinrodricks/FluentFTP). + +You can use this package to connect to FTP and FTPS servers. + +The provider respects folder structure of the remote FTP share. + +## Connect to FTP Server + +You can instantiate it either by using a simple helper method accepting the most basic parameters, like hostname, username and password, however for custom scenarios you can always construct your own instance of `FtpClient` from the FluentFTP library and pass it to FluentStorage to manage: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.Ftp("myhost.com", new NetworkCredential("username", "password")); + +// specify a custom ftp port (12345) as an example, real world scenarios may need extra customisations +var client = new FtpClient("myhost.com", 12345, new NetworkCredential("username", "password")); +IBlobStorage storage = StorageFactory.Blobs.FtpFromFluentFtpClient(client); +``` + +## Connection Strings +To create from connection string, first register the module when your program starts by calling `StorageFactory.Modules.UseFtpStorage();` then use the following connections tring: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("ftp://host=hostname;user=username;password=password"); +``` + diff --git a/docs/articles/Getting-Started.md b/docs/articles/Getting-Started.md new file mode 100644 index 00000000..95948de4 --- /dev/null +++ b/docs/articles/Getting-Started.md @@ -0,0 +1,57 @@ +## Nugets + +To get started, add the main [FluentStorage](https://www.nuget.org/packages/FluentStorage) Nuget package into your .NET application. + +Then add the [packages you need](https://github.com/robinrodricks/FluentStorage#packages) as per the cloud storage providers you want to use. + +## Connect to blob storage + +To construct storage classes, you need to use the factory methods. + +| Package | Factory method | Purpose | +| ----------------------------------------------------------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Blobs.FromConnectionString` | Creates a blob storage instance from a connection string | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Blobs.DirectoryFiles` | Creates a storage for a specific disk directory | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Blobs.InMemory` | Creates a storage which stores everyting in memory | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Blobs.Virtual` | Creates a virtual storage where you can mount other storage providers to a specific virtual directory | +| [Package](https://nuget.org/packages/FluentStorage.AWS) | `StorageFactory.Blobs.AwsS3` | Creates an AWS S3 storage bucket or custom S3-compatible storage server. | +| [Package](https://nuget.org/packages/FluentStorage.AWS) | `StorageFactory.Blobs.DigitalOceanSpaces` | Creates a [DigitalOcean Spaces](https://www.digitalocean.com/products/spaces) storage (S3 compatible). | +| [Package](https://nuget.org/packages/FluentStorage.AWS) | `StorageFactory.Blobs.MinIO` | Creates a [MinIO](https://min.io/) storage server (S3 compatible). | +| [Package](https://nuget.org/packages/FluentStorage.AWS) | `StorageFactory.Blobs.Wasabi` | Creates a [Wasabi](https://wasabi.com/) storage (S3 compatible). | +| [Package](https://nuget.org/packages/FluentStorage.GCP) | `StorageFactory.Blobs.GoogleCloudStorageFromEnvironmentVariable` | Creates a Google Cloud Storage storage with credentials in environment variables | +| [Package](https://nuget.org/packages/FluentStorage.GCP) | `StorageFactory.Blobs.GoogleCloudStorageFromJsonFile` | Creates a Google Cloud Storage storage with credentials in an external JSON | +| [Package](https://nuget.org/packages/FluentStorage.GCP) | `StorageFactory.Blobs.GoogleCloudStorageFromJson` | Creates a Google Cloud Storage storage with credentials in a passed JSON string | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithLocalEmulator` | Creates Azure Blob Storage to connect to a local emulator | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithSharedKey` | Creates Azure Blob Storage with Shared key authentication | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithAzureAd` | Creates Azure Blob Storage with Azure ActiveDirectory (AAD) authentication | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithTokenCredential` | Creates Azure Blob Storage with token credentials | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithSas` | Creates Azure Blob Storage with SAS identity | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureBlobStorageWithMsi` | Creates Azure Blob Storage with Managed Identity | +| [Package](https://nuget.org/packages/FluentStorage.Azure.DataLake) | `StorageFactory.Blobs.AzureDataLakeGen1StoreByClientSecret` | Creates Azure Data Lake Gen 1 Store client | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureDataLakeStorageWithMsi` | Creates Azure Data Lake Gen 2 Storage with Managed Identity | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureDataLakeStorageWithSharedKey` | Creates Azure Data Lake Gen 2 Storage with Shared key authentication | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Blobs) | `StorageFactory.Blobs.AzureDataLakeStorageWithAzureAd` | Creates Azure Data Lake Gen 2 Storage with Azure ActiveDirectory (AAD) authentication | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Files) | `StorageFactory.Blobs.AzureFiles` | Creates Azure Files storage | +| [Package](https://nuget.org/packages/FluentStorage.Azure.KeyVault) | `StorageFactory.Blobs.AzureKeyVault` | Creates Azure Key Vault secrets storage | +| [Package](https://nuget.org/packages/FluentStorage.Azure.KeyVault) | `StorageFactory.Blobs.AzureKeyVaultWithMsi` | Creates Azure Key Vault secrets with Managed Identity | +| [Package](https://nuget.org/packages/FluentStorage.Azure.ServiceFabric) | `StorageFactory.Blobs.AzureServiceFabricReliableStorage` | Creates Azure Service Fabric storage | +| [Package](https://nuget.org/packages/FluentStorage.Databricks) | `StorageFactory.Blobs.Databricks` | Creates Azure Databricks DBFS storage | +| [Package](https://nuget.org/packages/FluentStorage.FTP) | `StorageFactory.Blobs.Ftp` | Creates an interface to FTP/FTPS servers | +| [Package](https://nuget.org/packages/FluentStorage.FTP) | `StorageFactory.Blobs.FtpFromFluentFtpClient` | Creates an interface to FTP/FTPS servers with the given client instance | +| [Package](https://nuget.org/packages/FluentStorage.SFTP) | `StorageFactory.Blobs.Sftp` | Creates an interface to SFTP servers (FTP over SSH) | + + +## Connect to message queues + +To construct storage classes, you need to use the factory methods. + +| Package | Factory method | Purpose | +| -------------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Messages.MessengerFromConnectionString` | Creates a message publisher from connection string | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Messages.InMemory` | Creates a message publisher which holds messages in memory | +| [Package](https://nuget.org/packages/FluentStorage) | `StorageFactory.Messages.Disk` | Creates a message publisher that uses local disk directory as a backing store. | +| [Package](https://nuget.org/packages/FluentStorage.AWS) | `StorageFactory.Messages.AwsSQS` | Creates Amazon Simple Queue Service publisher. | +| [Package](https://nuget.org/packages/FluentStorage.Azure.EventHub) | `StorageFactory.Messages.AzureEventHub` | Create Azure Event Hub messenger by full connection string. | +| [Package](https://nuget.org/packages/FluentStorage.Azure.Queues) | `StorageFactory.Messages.AzureStorageQueue` | Creates a message publisher to Azure Storage Queues. | +| [Package](https://nuget.org/packages/FluentStorage.Azure.ServiceBus) | `StorageFactory.Messages.AzureServiceBus` | Creates a message publisher ho Azure Service Bus Queue. | +| [Package](https://nuget.org/packages/FluentStorage.Azure.ServiceBus) | `StorageFactory.Messages.AzureServiceBusTopicReceiver` | Creates a message reciever from Azure Service Bus Queue. | diff --git a/docs/articles/Google-Cloud-Storage.md b/docs/articles/Google-Cloud-Storage.md new file mode 100644 index 00000000..c658c826 --- /dev/null +++ b/docs/articles/Google-Cloud-Storage.md @@ -0,0 +1,62 @@ + In order to use [Google Cloud Storage](https://cloud.google.com/storage/) reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.GCP.svg)](https://www.nuget.org/packages/FluentStorage.GCP) package first. + +You definitely want to use FluentStorage for working with Google Storage, as it solves quite a few issues which are hard to manage with the raw SDK: + +- Listing of files and folders +- Recursive and non-recursive listing +- Upload and download operations are continuing to be optimised + + +## Connect using Service Account IAM + +You can configure Service Account IAM permissions for accessing GCS buckets, and then simply omit the `cred` in the connection string, and then FluentStorage will fallback to connecting using the Workload Identity setup. + +You can then use [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) (ADC), which is the recommended authentication approach for workloads running on Google Cloud and even for local development via `gcloud auth application-default login`. + + +## Connect using environment variables + +If you have credentials stored in an environment variable. (As described [here](https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication)) + +```csharp +IBlobStorage storage = StorageFactory.Blobs.GoogleCloudStorageFromEnvironmentVariable(bucketName); +``` + + +## Connect using credentials file + +If you have credentials stored in an external JSON file. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.GoogleCloudStorageFromJsonFile(bucketName, filePath); +``` + + +## Connect using credentials + +If you need the credentials passed as a JSON string. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.GoogleCloudStorageFromJson( + bucketName, credentialsJsonString, isBase64EncodedString = false); +``` + +The last parameter says whether the string is base64 encoded or not, which is handy if credentials are stored in some sort of config file. + + + +## Connection Strings + +First, don't forget to initialise the module: + +```csharp +StorageFactory.Modules.UseGoogleCloudStorage(); +``` + +Then, use the string: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("google.storage://bucket=...;cred=..."); +``` + +Where **cred** is a *BASE64* encoded credential string. diff --git a/docs/articles/Hetzner-Storage.md b/docs/articles/Hetzner-Storage.md new file mode 100644 index 00000000..163c0dc5 --- /dev/null +++ b/docs/articles/Hetzner-Storage.md @@ -0,0 +1,38 @@ + In order to use **Hetzner** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to Hetzner + +You can use the `Hetzner` method to connect to [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/) storage servers (S3 compatible). + +```csharp +IBlobStorage storage = StorageFactory.Blobs.Hetzner( + accessKeyId, secretAccessKey, bucketName, region); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("hetzner://keyId=...;key=...;bucket=...;region=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **region** is the Storage region (e.g. `fsn1`, `nbg1`, `hel1`)< + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/Message-Storage.md b/docs/articles/Message-Storage.md new file mode 100644 index 00000000..6a0bdc08 --- /dev/null +++ b/docs/articles/Message-Storage.md @@ -0,0 +1,85 @@ +## Providers + +- [AWS SQS](AWS-SQS.md) +- [Azure Queue Storage](Azure-Queue-Storage.md) +- [Azure Service Bus](Azure-Service-Bus.md) +- [In-Memory & Local Disk](Standard-Storage.md#standard-message-storage) + +## Getting Started + +Messaging is intended for message passing between one or more systems in disconnected fashion. You can send a message somewhere and current or remote system picks it up for processing later when required. This paradigm somehow fits into [CQRS](https://martinfowler.com/bliki/CQRS.html) and [Message Passing](https://www.defit.org/message-passing/) architectural ideas. + +To name a few examples, [Apache Kafka](http://kafka.apache.org/), [RabbitMQ](https://www.rabbitmq.com/), [Azure Service Bus](https://azure.microsoft.com/en-gb/services/service-bus/) are all falling into this category - essentially they are designed to pass messages. Some systems are more advanced to others of course, but most often it doesn't really matter. + +FluentStorage supports many messaging providers out of the box, including **Azure Service Bus Topics and Queues**, **Azure Event Hub** and others. + +There are two abstractions available - **message publisher** and **message receiver**. As the name stands, one is publishing messages, and another is receiving them on another end. + +### Publishing Messages + +To publish messages you will usually construct an instance of `IMessagePublisher` with an appropriate implementation. All the available implementations can be created using factory methods in the `FluentStorage.StorageFactory.Messages` class. More methods appear in that class as you reference an assembly containing specific implementations. + +### Receiving Messages + +Similarly, to receive messages you can use factory methods to create receivers which all implement `IMessageReceiver` interface. + +The primary method of this interface + +```csharp +Task StartMessagePumpAsync( + Func, Task> onMessageAsync, + int maxBatchSize = 1, + CancellationToken cancellationToken = default); +``` + +starts a message pump that listens for incoming queue messages and calls `Func, Task>` as a call back to pass those messages to your code. + +`maxBatchSize` is a number specifying how many messages you are ready to handle at once in your callback. Choose this number carefully as specifying number too low will result in slower message processing, whereas number too large will increase RAM requirements for your software. + +`cancellationToken` is used to signal the message pump to stop. Not passing any parameter there will result in never stopping message pump. See example below in Use Cases for a pattern on how to use this parameter. + +You can find the list of supported messaging implementations [above](#providers). + +### Handling Large Messages + +FluentStorage provides built-in capability to handle large message content by allowing you to offload message content over a certain threshold to an external blob storage. It works in the following way: + +1. Check that message content is larger than `threshold value`. +2. If not, do the usual processing. +3. If it is, upload message content as a blob to external storage, clear message content and add a custom header `x-sn-large` that points to the blob containing message content. + +When receiving messages, it will check that `x-sn-large` header is present, and if so, will download blob, set it's content as message content, and return the message to the receiver. + +Blob is deleted from the blob storage only when message is confirmed by the receiver. + +Large message handling works **on any supported queue implementation** because it's implemented in the core library itself, outside of specific queue implementation. To enable it, call `.HandleLargeContent` on both publisher and receiver: + +```csharp +IBlobStorage offloadStorage = ...; // your blob storage for offloading content + +IMessagePublisher publisher = StorageFactory.Messages + .XXXPublisher(...) + .HandleLargeContent(offloadStorage, thresholdValue); + +IMessageReceiver receiver = StorageFactory.Messages + .XXXReceiver(...) + .HandleLargeContent(offloadStorage); +``` + +### Serialising/deserialising `QueueMessage` + +`QueueMessage` class itself is not a serialisable entity when we talk about JSON or built-in .NET binary serialisation due to the fact it is a functionally rich structure. However, you might want to transfer the whole `QueueMessage` across the wire sometimes. For these purposes you can use built-in binary methods: + +```csharp +var qm = new QueueMessage("id", "content"); +qm.DequeueCount = 4; +qm.Properties.Add("key", "value"); + +byte[] wireData = qm.ToByteArray(); + +//transfer the bytes + +QueueMessage receivedMessage = QueueMessage.FromByteArray(wireData); +``` + +These methods make sure that *all* of the message data is preserved, and also are backward compatible between any changes to this class. diff --git a/docs/articles/Migration-Guide.md b/docs/articles/Migration-Guide.md new file mode 100644 index 00000000..babf64f0 --- /dev/null +++ b/docs/articles/Migration-Guide.md @@ -0,0 +1,52 @@ +## Migrating from FluentStorage v7 to v8+ + +### Removed packages + +| | Package | Reason | +|-----|---------|--------| +| | **[FluentStorage.Databricks](https://www.nuget.org/packages/FluentStorage.Databricks)** | We will no longer maintain this package because DBFS is not a mainstream storage backend for a storage abstraction library. | +| | **[FluentStorage.Azure.EventHub](https://www.nuget.org/packages/FluentStorage.Azure.EventHub)** | Due to low community usage, we will no longer maintain this library. | +| | **[FluentStorage.Azure.DataLake](https://www.nuget.org/packages/FluentStorage.Azure.DataLake)** | We are no longer maintaining this package as it only caters to DataLake Gen 1, which has been superseded by DataLake Gen2. Gen1 is considered a legacy service and is no longer the direction Microsoft recommends for new development. | +| | **[FluentStorage.Azure.ServiceFabric](https://www.nuget.org/packages/FluentStorage.Azure.ServiceFabric)** | We will no longer maintain this package because ServiceFabric is not a first-class object storage or messaging service, which is outside our current scope, and it also has extremely low community usage. | + +## Migrating from Storage.NET + +### Packaging changes + +Change your **NuGet packages** and your **imports** using this mapping: + +| Old name | New name | +| ------------------------------------------------- | --------------------------------- | +| Storage.Net | [FluentStorage](https://www.nuget.org/packages/FluentStorage) | +| Storage.Net.Amazon.Aws | [FluentStorage.AWS](https://www.nuget.org/packages/FluentStorage.AWS) | +| Storage.Net.Gcp.CloudStorage | [FluentStorage.GCP](https://www.nuget.org/packages/FluentStorage.GCP) | +| Storage.Net.Databricks | No longer supported | +| Storage.Net.Ftp | [FluentStorage.FTP](https://www.nuget.org/packages/FluentStorage.FTP) | +| Storage.Net.Microsoft.Azure.Storage.Blobs | [FluentStorage.Azure.Blobs](https://www.nuget.org/packages/FluentStorage.Azure.Blobs) | +| Storage.Net.Microsoft.Azure.Storage.Files | [FluentStorage.Azure.Files](https://www.nuget.org/packages/FluentStorage.Azure.Files) | +| Storage.Net.Microsoft.Azure.EventHub | No longer supported | +| Storage.Net.Microsoft.Azure.ServiceBus | [FluentStorage.Azure.ServiceBus](https://www.nuget.org/packages/FluentStorage.Azure.ServiceBus) | +| Storage.Net.Microsoft.Azure.KeyVault | [FluentStorage.Azure.KeyVault](https://www.nuget.org/packages/FluentStorage.Azure.KeyVault) | +| Storage.Net.Microsoft.Azure.ServiceFabric | No longer supported | +| Storage.Net.Microsoft.Azure.Queues | [FluentStorage.Azure.Queues](https://www.nuget.org/packages/FluentStorage.Azure.Queues) | +| Storage.Net.Microsoft.Azure.DataLake.Storage.Gen1 | No longer supported | + +### Encryption changes + +We now accept the IV and Key in the constructors of the `*EncryptionSink` classes. + +Since most of the time we will be using Dependency Injection (DI) to instantiate our objects, each time the `SymmetricEncryptionSink`, the IV is set + +``` + public AesSymmetricEncryptionSink(string key) { + _cryptoAlgorithm = Aes.Create(); + _cryptoAlgorithm.Key = Convert.FromBase64String(key); + _cryptoAlgorithm.GenerateIV(); + } +``` + +This means that if a blob is stored, then it can only be unencrypted using the same instance of the Sink or with an instance of the Sink with the same IV (secret key) + +If you are storing to AWS/Azure storage, then if you try and retrieve the blob at another time, you can't read the blob even though the original key might be the same, the secret generated at the time of encryption is no longer the secret that the sink is trying to use the decrypt. + +Storage.Net has got this wrong and has used a pattern like this, which creates a transient class, however we [fixed it](https://github.com/robinrodricks/FluentStorage/pull/36) to accept the IV and Key in the constructors of the sinks. \ No newline at end of file diff --git a/docs/articles/MinIO-Storage.md b/docs/articles/MinIO-Storage.md new file mode 100644 index 00000000..ed4d3caf --- /dev/null +++ b/docs/articles/MinIO-Storage.md @@ -0,0 +1,46 @@ + In order to use **MinIO** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to MinIO + +You can use the `MinIO` method to connect to [MinIO](https://min.io/) storage servers (S3 compatible). + +For `minioServerUrl`, you need to enter the absolute server URL of your MinIO endpoint. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.MinIO( + accessKeyId, secretAccessKey, bucketName, awsRegion, minioServerUrl); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("minio.s3://keyId=...;key=...;bucket=...;region=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **region** is the AWS account region (such as `us-east-1`). +- **serviceUrl** is the URL of the MinIO storage provider. + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Special features for MinIO + +- Remote directory recursion occurs on MinIO server rather than locally (`ListFolderAsync` API). +- `ListOptions` has a `PageSize` parameter which can be used to control the internal paging size used for remote recursion. + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/SFTP-Storage.md b/docs/articles/SFTP-Storage.md new file mode 100644 index 00000000..eda95ddb --- /dev/null +++ b/docs/articles/SFTP-Storage.md @@ -0,0 +1,24 @@ + To use this, you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.SFTP.svg)](https://www.nuget.org/packages/FluentStorage.SFTP/) package first, which wraps [SSH.NET](https://github.com/sshnet/SSH.NET/). + +The provider respects folder structure of the remote SFTP share. + +## Connect to SFTP Server +You can instantiate it either by using a simple helper method accepting the most basic parameters, like hostname, port, username and password, however for custom scenarios you can always construct your own instance of `SftpClient` from the SSH.NET library and pass it to FluentStorage to manage: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.Sftp("myhost.com", "username", "password"); + +// With a custom port and a private key file +IBlobStorage storage = StorageFactory.Blobs.Sftp("myhost.com", 2222, "username", new PrivateKeyFile("filename")); + +// With both password and public-key authentication as an example, real world scenarios may need extra customisations +var connectionInfo = new ConnectionInfo("sftp.foo.com", "guest", new PasswordAuthenticationMethod("guest", "pwd"), new PrivateKeyAuthenticationMethod("rsa.key")); +IBlobStorage storage = StorageFactory.Blobs.Sftp(connectionInfo); +``` + +## Connection Strings +To create from connection string, first register the module when your program starts by calling `StorageFactory.Modules.UseSftpStorage();` then use the following connections tring: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("sftp://host=hostname;user=username;password=password"); +``` \ No newline at end of file diff --git a/docs/articles/Standard-Storage.md b/docs/articles/Standard-Storage.md new file mode 100644 index 00000000..12f63b35 --- /dev/null +++ b/docs/articles/Standard-Storage.md @@ -0,0 +1,87 @@ + Standard storage are the providers included in the base FluentStorage nuget [![NuGet](https://img.shields.io/nuget/v/FluentStorage.svg)](https://www.nuget.org/packages/FluentStorage/). + +## Standard Blob Storage + +### In-Memory + +In-memory provider stores blobs in process memory as array of bytes. Although it's available, we strongly discourage using it in production due to high memory fragmentation. It's sole purpose is for local testing, mocking etc. + +The provider is built into FluentStorage main package. + +To construct, use: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.InMemory(); +``` + +which constructs a new instance of in-memory storage. Further calls to this factory create a new unique instance. + +To create from connection string: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("inmemory://"); +``` + +### Local Disk + +Local disk providers maps a local folder to `IBlobStorage` instance, so that you can both use local disk and replicate directory structure as blob storage interface. + +The provider is built into FluentStorage main package. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.DirectoryFiles(directory); +``` + +where `directory` is an instance of `System.IO.DirectoryInfo` that points to that directory. The directory does not have to exist on local disk, however it will be created on any write operation. + +To create from connection string: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("disk://path=path_to_directory"); +``` + +### Zip File + +Zip file provider maps to a single zip archive. All the operations that include path are created as subfolders in zip archive. Archive itself doesn't need to exist, however any write operation will create a new archive and put any data you write to it. + +The provider is built into FluentStorage main package as zip API are a part of .NET Standard nowadays. It is thread safe by default. + +```csharp +IBlobStorage storage = StorageFactory.Blobs.ZipFile(pathToZipFile); +``` + +To create from connection string: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("zip://path=path_to_file"); +``` + +## Standard Message Storage + +### In-Memory + +In-memory provider creates messaging queue directly in memory, and is useful for mocking message publisher and receiver. It's not intended to be used in production. + +The provider is built into FluentStorage main package. + +To construct, use: + +```csharp +services.AddSingleton(_ => + FluentStorage.StorageFactory.Messages.InMemory(queueName) +); +``` + +### Local Disk + +Local disk messaging is backed by a local folder on disk. Every message publish call creates a new file in that folder with `.snm` extension (**S**torage **N**et **M**essage) which is a binary representation of the message. + +Message receiver polls this folder every second to check for new files, get the oldest ones and transforms into `QueueMessage`. + +The provider is built into FluentStorage main package. + +To construct from a connection string, use: + +```csharp +IMessenger messenger = StorageFactory.Messages.MessengerFromConnectionString("disk://path=the_path"); +``` diff --git a/docs/articles/Vultr-Storage.md b/docs/articles/Vultr-Storage.md new file mode 100644 index 00000000..15b36001 --- /dev/null +++ b/docs/articles/Vultr-Storage.md @@ -0,0 +1,40 @@ + In order to use **Vultr** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to Vultr + +You can use the `Vultr` method to connect to [Vultr Object Storage](https://www.vultr.com/products/object-storage/) storage servers (S3 compatible). + +```csharp +IBlobStorage storage = StorageFactory.Blobs.Vultr( + accessKeyId, secretAccessKey, bucketName, hostName); +``` + +Where **hostName** is the Vultr Storage endpoint hostname (e.g. `sgp1.vultrobjects.com`). + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("vultr://keyId=...;key=...;bucket=...;hostName=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **hostName** is the Vultr Storage endpoint hostname (e.g. `sgp1.vultrobjects.com`). + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/Wasabi-Storage.md b/docs/articles/Wasabi-Storage.md new file mode 100644 index 00000000..17d70d37 --- /dev/null +++ b/docs/articles/Wasabi-Storage.md @@ -0,0 +1,40 @@ + In order to use **Wasabi** storage you need to reference [![NuGet](https://img.shields.io/nuget/v/FluentStorage.AWS.svg)](https://www.nuget.org/packages/FluentStorage.AWS/) package first. The provider wraps around the standard AWS SDK which is updated regularly, but adds a lot of untrivial workarounds that makes your life painless. + +## Connect to Wasabi + +You can use the `Wasabi` method to connect to [Wasabi](https://wasabi.com/) storage servers (S3 compatible). + +To obtain the `wasabiServiceUrl`, refer to [this Wasabi article](https://knowledgebase.wasabi.com/hc/en-us/articles/360015106031-What-are-the-service-URLs-for-Wasabi-s-different-storage-regions), and take the values in the `Service URL` column (add `https://` as a prefix). + +```csharp +IBlobStorage storage = StorageFactory.Blobs.Wasabi( + accessKeyId, secretAccessKey, bucketName, wasabiServiceUrl); +``` + +## Connection Strings + +To create with a connection string, first reference the module: + +```csharp +StorageFactory.Modules.UseAwsStorage(); +``` + +Then construct using the following format: + +```csharp +IBlobStorage storage = StorageFactory.Blobs.FromConnectionString("wasabi://keyId=...;key=...;bucket=...;serviceUrl=..."); +``` + +where: +- **keyId** is (optional) access key ID. +- **key** is (optional) secret access key. +- **bucket** is bucket name. +- **serviceUrl** is the values in the `Service URL` column (see above section). + +## AWS CLI Profile Support + +If you already have credentials in the local credentials file generated by AWS CLI, you can also use them to connect to a bucket with FluentStorage. [Look here for more details.](AWS-S3-Storage.md#aws-cli-profile-support) + +## Native Operations + +Native operations are exposed via [IS3Storage](xref:FluentStorage.AWS.Blobs.IS3Storage) interface. diff --git a/docs/articles/index.md b/docs/articles/index.md new file mode 100644 index 00000000..52f84950 --- /dev/null +++ b/docs/articles/index.md @@ -0,0 +1 @@ +[!include[Index](../index.md)] \ No newline at end of file diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml new file mode 100644 index 00000000..e0746a69 --- /dev/null +++ b/docs/articles/toc.yml @@ -0,0 +1,59 @@ +- name: Index + href: index.md +- name: Getting Started + href: Getting-Started.md +- name: Migration Guide + href: Migration-Guide.md +- name: Standard Storage + href: Standard-Storage.md +- name: Blob Storage + href: Blob-Storage.md + items: + - name: AWS S3 Storage + href: AWS-S3-Storage.md + - name: Cloudflare R2 Storage + href: Cloudflare-R2-Storage.md + - name: MinIO Storage + href: MinIO-Storage.md + - name: DigitalOcean Spaces + href: DigitalOcean-Spaces-Storage.md + - name: Wasabi Storage + href: Wasabi-Storage.md + - name: Backblaze B2 Storage + href: Backblaze-B2-Storage.md + - name: Hetzner Storage + href: Hetzner-Storage.md + - name: Vultr Storage + href: Vultr-Storage.md + - name: GCP Storage + href: Google-Cloud-Storage.md + - name: Azure Blob Storage + href: Azure-Blob-Storage.md + - name: Azure File Storage + href: Azure-Files-Storage.md + - name: Azure Data Lake + href: Azure-Data-Lake.md + - name: Azure Key Vault + href: Azure-Key-Vault.md + - name: FTP Storage + href: FTP-Storage.md + - name: SFTP Storage + href: SFTP-Storage.md +- name: Message Storage + href: Message-Storage.md + items: + - name: AWS SQS + href: AWS-SQS.md + - name: Azure Queue Storage + href: Azure-Queue-Storage.md + - name: Azure Service Bus + href: Azure-Service-Bus.md +- name: Data Transformation + href: Data-Transformation.md + items: + - name: Data Compression + href: Data-Compression.md + - name: Data Encryption + href: Data-Encryption.md + - name: Custom Sinks + href: Custom-Sinks.md diff --git a/docs/docfx.json b/docs/docfx.json new file mode 100644 index 00000000..680a1731 --- /dev/null +++ b/docs/docfx.json @@ -0,0 +1,75 @@ +{ + "metadata": [ + { + "src": [ + { + "src": "..", + "files": [ + "FluentStorage/FluentStorage.csproj", + "FluentStorage.AWS/FluentStorage.AWS.csproj", + "FluentStorage.Azure/FluentStorage.Azure.csproj", + "FluentStorage.Azure.Blobs/FluentStorage.Azure.Blobs.csproj", + "FluentStorage.Azure.Files/FluentStorage.Azure.Files.csproj", + "FluentStorage.Azure.KeyVault/FluentStorage.Azure.KeyVault.csproj", + "FluentStorage.Azure.Queues/FluentStorage.Azure.Queues.csproj", + "FluentStorage.Azure.ServiceBus/FluentStorage.Azure.ServiceBus.csproj", + "FluentStorage.FTP/FluentStorage.FTP.csproj", + "FluentStorage.GCP/FluentStorage.GCP.csproj", + "FluentStorage.SFTP/FluentStorage.SFTP.csproj" + ] + } + ], + "dest": "api", + "properties": { + "TargetFramework": "netstandard2.0" + } + } + ], + "build": { + "content": [ + { + "files": [ + "toc.yml", + "index.md", + "articles/**.md", + "articles/toc.yml", + "api/**.yml", + "api/index.md" + ] + } + ], + "resource": [ + { + "files": [ + "images/**" + ] + } + ], + "dest": "_site", + "globalMetadata": { + "_appName":"FluentStorage", + "_disableContribution": false, + "_enableSearch": true, + "_gitContribute": { + "branch": "develop", + "apiSpecFolder":"api", + "repo": "https://github.com/robinrodricks/FluentStorage" + }, + "_gitUrlPattern": "github", + "_disableSideFilter":true, + "_appLogoPath": "images/logo.png" + }, + "globalMetadataFiles": [], + "fileMetadataFiles": [], + "template": [ + "default", + "modern", + "templates" + ], + "postProcessors": [ "ExtractSearchIndex" ], + "noLangKeyword": false, + "keepFileLink": true, + "cleanupCacheHistory": false, + "disableGitFeatures": false + } +} diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..c2b1eac871dff3ec4a5df6ceee5a46c9d6ae8ba8 GIT binary patch literal 1834 zcmV+_2i5qAP)Vc3$m9mhPArvG$_c zilQir5}j)t9ZYcaHr4C5QoXRUVhPg+%qW!zQ~+W~BLWpbdW!%58D9h{fW#1}xc`&5 zB2WP&#=vwdei}^#DuDF#s(&0u1S$a0@QXkNkdO%uUvK{_UJWOXxR0CQc$?$pa8tm(c(G*bFCl|Eujq!38kE`ws@D z2)(XS3u_T@cz_dzHI@2Ui6E!|u5kPY>iQ*6TLeJ`V6Ys9TuB8K|GykVuU7ak2HbFqj8eU|ED(D>Mdl zwukfYU*hB6fGHB269F35%?Ska0797(%y19qzo#}N5>tX1?pwUyoCPr6ff=m6_4g=J z7{d*LZVi@$eSG@M=DJ`4SPDArGoZ6qxcIT^ck3F2*d}o^-oc{(Eq?jO_QKpez!ad9 zSGf2swILDlIt>gdfFj^pE{uW2aBsT{EMgcF0|HL=asIQ<8o>mR*tL#2n4i>fftj3B ztb@Ie-ofp2-vXupo$NKsT>I{R6KtUjaZ2_zF z=7m>!Q!<+aW*VkA!w?e$x*lg{K$Ku_6QL@C3ct(gatDiEj{E&X9vqvGcVLpf??5qt zDO|73-Q{nBK;;e=gIK}dCX#Id4R-kjY8Gt9z*_*+>pT>Hfh=6JjJT<{J8?IvB_$H4UG=#<-<7Zg6c1zrKv7LJ~j#-X7o!U7%g62;$-`01A5$)CJn(l+Vy<8UbQ}s(JU`t0sGfPSXex z129zsnbEp*N?}Aw5V(m`m%&F$eY_Gu>oV)xK(;(Z6HCIP;N<}#&IA~0!iWbk^#C>K ztwmtn7%{=oTe2U^9vp+^iKKG?z*7JTTEcrXw8s-14s1S11N0!MeO(gLgTP0uY7guZ zMoS^XtE7GV0e=A$A&hXK3$)AiTtdYIF9AG$>Sa{A26R~`VZ@WHvV-E>Em`yIwt&af z${I@80&Y_qP=%KO42*;i7VE5`CX5{L^#I9T>m4dbqe`hWXaVEYHdNs(0HH~l;T4Yl zxNDn~0hR<4qkY~QX8}ZN3AaYe%E zGPNO+arOXAE#aZ*0%nlf2+8ES1&HH2T>P2ZhANx|K+T!EpEc?M;vn)C5bzT~5z2_% zp;a&{5|1Fz6v3=elA&8b1Psh=Y8y7;BmnK+vF0|l0h@3VK+B^eSdoR30DM})n+U;( z4Fa7~!2w#rDdOlFaQ$7`DM0o%uKGp(Ytg;n*tCTa=0^q8$C6L%7)0`2r1sK6+ z<~d_60KS%Rx#`lJ5#s@rG*}D3fS8g%GP`GU$6*WL>9lPLJ~Pi5-2xhp!xq5P5-zoF zK6hL^fF1;8gp~joJ8m@xkO(UQaJ7U>uA9#t4FVgEr3c{Y29i2Hr3qMi08=p7Aa;C8 z6A-ZlkOqOtuo3`QOL$v@z+{N<07^Qn1i&?NRf|E>Ve0`{@&H;4nhxCp8cuBiaJ7V! zb^|4)wg9vxJWpy1fU6~3i$T*;TL4@w;aUuu77YR$j*S2c4g(}l&u9X*1;F)upf;8o z10+PZfQDlc1oCtQX)$P8)B`k}S_0r|377gTy(Up>O4tnZE}>P1esL`YQxru}6h%qH Ye};Sd!g7e}=Kufz07*qoM6N<$f`!~D^Z)<= literal 0 HcmV?d00001 diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..3f2e9090 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,11 @@ +# Documentation + +- [Migration Guide](articles/Migration-Guide.md) - how to migrate from Storage.NET and previous versions + +- [Blob Storage](articles/Blob-Storage.md) - store unstructured binary files of any size. (AWS S3, Azure Blob Storage, GCP Storage, local folder..) + +- [Message Storage](articles/Message-Storage.md) - send and receive messages between systems. (AWS SQS, MSMQ, Azure Service Bus..) + +- [Standard Storage](articles/Standard-Storage.md) - In-Memory, Disk & ZIP storage included in the base package. + +- [Data Transformation](articles/Data-Transformation.md) - encryption and compression data transformation sinks. diff --git a/docs/templates/public/main.css b/docs/templates/public/main.css new file mode 100644 index 00000000..816171c1 --- /dev/null +++ b/docs/templates/public/main.css @@ -0,0 +1,4 @@ +#logo { + max-height: 32px; + width: auto; +} \ No newline at end of file diff --git a/docs/toc.yml b/docs/toc.yml new file mode 100644 index 00000000..090bd75d --- /dev/null +++ b/docs/toc.yml @@ -0,0 +1,4 @@ +- name: Articles + href: articles/ +- name: API + href: api/ From b125e954f3bc81094fd3da84afc4069e3839f6dc Mon Sep 17 00:00:00 2001 From: gumbarros Date: Fri, 10 Jul 2026 10:13:35 -0300 Subject: [PATCH 2/2] Rename articles to wiki --- docs/docfx.json | 4 ++-- docs/toc.yml | 4 ++-- docs/{articles => wiki}/AWS-S3-Storage.md | 0 docs/{articles => wiki}/AWS-SQS.md | 0 docs/{articles => wiki}/Azure-Blob-Storage.md | 0 docs/{articles => wiki}/Azure-Data-Lake.md | 0 docs/{articles => wiki}/Azure-Files-Storage.md | 0 docs/{articles => wiki}/Azure-Key-Vault.md | 0 docs/{articles => wiki}/Azure-Queue-Storage.md | 0 docs/{articles => wiki}/Azure-Service-Bus.md | 0 docs/{articles => wiki}/Backblaze-B2-Storage.md | 0 docs/{articles => wiki}/Blob-Storage.md | 0 docs/{articles => wiki}/Cloudflare-R2-Storage.md | 0 docs/{articles => wiki}/Custom-Sinks.md | 0 docs/{articles => wiki}/Data-Compression.md | 0 docs/{articles => wiki}/Data-Encryption.md | 0 docs/{articles => wiki}/Data-Transformation.md | 0 docs/{articles => wiki}/DigitalOcean-Spaces-Storage.md | 0 docs/{articles => wiki}/FTP-Storage.md | 0 docs/{articles => wiki}/Getting-Started.md | 0 docs/{articles => wiki}/Google-Cloud-Storage.md | 0 docs/{articles => wiki}/Hetzner-Storage.md | 0 docs/{articles => wiki}/Message-Storage.md | 0 docs/{articles => wiki}/Migration-Guide.md | 0 docs/{articles => wiki}/MinIO-Storage.md | 0 docs/{articles => wiki}/SFTP-Storage.md | 0 docs/{articles => wiki}/Standard-Storage.md | 0 docs/{articles => wiki}/Vultr-Storage.md | 0 docs/{articles => wiki}/Wasabi-Storage.md | 0 docs/{articles => wiki}/index.md | 0 docs/{articles => wiki}/toc.yml | 0 31 files changed, 4 insertions(+), 4 deletions(-) rename docs/{articles => wiki}/AWS-S3-Storage.md (100%) rename docs/{articles => wiki}/AWS-SQS.md (100%) rename docs/{articles => wiki}/Azure-Blob-Storage.md (100%) rename docs/{articles => wiki}/Azure-Data-Lake.md (100%) rename docs/{articles => wiki}/Azure-Files-Storage.md (100%) rename docs/{articles => wiki}/Azure-Key-Vault.md (100%) rename docs/{articles => wiki}/Azure-Queue-Storage.md (100%) rename docs/{articles => wiki}/Azure-Service-Bus.md (100%) rename docs/{articles => wiki}/Backblaze-B2-Storage.md (100%) rename docs/{articles => wiki}/Blob-Storage.md (100%) rename docs/{articles => wiki}/Cloudflare-R2-Storage.md (100%) rename docs/{articles => wiki}/Custom-Sinks.md (100%) rename docs/{articles => wiki}/Data-Compression.md (100%) rename docs/{articles => wiki}/Data-Encryption.md (100%) rename docs/{articles => wiki}/Data-Transformation.md (100%) rename docs/{articles => wiki}/DigitalOcean-Spaces-Storage.md (100%) rename docs/{articles => wiki}/FTP-Storage.md (100%) rename docs/{articles => wiki}/Getting-Started.md (100%) rename docs/{articles => wiki}/Google-Cloud-Storage.md (100%) rename docs/{articles => wiki}/Hetzner-Storage.md (100%) rename docs/{articles => wiki}/Message-Storage.md (100%) rename docs/{articles => wiki}/Migration-Guide.md (100%) rename docs/{articles => wiki}/MinIO-Storage.md (100%) rename docs/{articles => wiki}/SFTP-Storage.md (100%) rename docs/{articles => wiki}/Standard-Storage.md (100%) rename docs/{articles => wiki}/Vultr-Storage.md (100%) rename docs/{articles => wiki}/Wasabi-Storage.md (100%) rename docs/{articles => wiki}/index.md (100%) rename docs/{articles => wiki}/toc.yml (100%) diff --git a/docs/docfx.json b/docs/docfx.json index 680a1731..926e8eb1 100644 --- a/docs/docfx.json +++ b/docs/docfx.json @@ -31,8 +31,8 @@ "files": [ "toc.yml", "index.md", - "articles/**.md", - "articles/toc.yml", + "wiki/**.md", + "wiki/toc.yml", "api/**.yml", "api/index.md" ] diff --git a/docs/toc.yml b/docs/toc.yml index 090bd75d..bc13ebde 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -1,4 +1,4 @@ -- name: Articles - href: articles/ +- name: Wiki + href: wiki/ - name: API href: api/ diff --git a/docs/articles/AWS-S3-Storage.md b/docs/wiki/AWS-S3-Storage.md similarity index 100% rename from docs/articles/AWS-S3-Storage.md rename to docs/wiki/AWS-S3-Storage.md diff --git a/docs/articles/AWS-SQS.md b/docs/wiki/AWS-SQS.md similarity index 100% rename from docs/articles/AWS-SQS.md rename to docs/wiki/AWS-SQS.md diff --git a/docs/articles/Azure-Blob-Storage.md b/docs/wiki/Azure-Blob-Storage.md similarity index 100% rename from docs/articles/Azure-Blob-Storage.md rename to docs/wiki/Azure-Blob-Storage.md diff --git a/docs/articles/Azure-Data-Lake.md b/docs/wiki/Azure-Data-Lake.md similarity index 100% rename from docs/articles/Azure-Data-Lake.md rename to docs/wiki/Azure-Data-Lake.md diff --git a/docs/articles/Azure-Files-Storage.md b/docs/wiki/Azure-Files-Storage.md similarity index 100% rename from docs/articles/Azure-Files-Storage.md rename to docs/wiki/Azure-Files-Storage.md diff --git a/docs/articles/Azure-Key-Vault.md b/docs/wiki/Azure-Key-Vault.md similarity index 100% rename from docs/articles/Azure-Key-Vault.md rename to docs/wiki/Azure-Key-Vault.md diff --git a/docs/articles/Azure-Queue-Storage.md b/docs/wiki/Azure-Queue-Storage.md similarity index 100% rename from docs/articles/Azure-Queue-Storage.md rename to docs/wiki/Azure-Queue-Storage.md diff --git a/docs/articles/Azure-Service-Bus.md b/docs/wiki/Azure-Service-Bus.md similarity index 100% rename from docs/articles/Azure-Service-Bus.md rename to docs/wiki/Azure-Service-Bus.md diff --git a/docs/articles/Backblaze-B2-Storage.md b/docs/wiki/Backblaze-B2-Storage.md similarity index 100% rename from docs/articles/Backblaze-B2-Storage.md rename to docs/wiki/Backblaze-B2-Storage.md diff --git a/docs/articles/Blob-Storage.md b/docs/wiki/Blob-Storage.md similarity index 100% rename from docs/articles/Blob-Storage.md rename to docs/wiki/Blob-Storage.md diff --git a/docs/articles/Cloudflare-R2-Storage.md b/docs/wiki/Cloudflare-R2-Storage.md similarity index 100% rename from docs/articles/Cloudflare-R2-Storage.md rename to docs/wiki/Cloudflare-R2-Storage.md diff --git a/docs/articles/Custom-Sinks.md b/docs/wiki/Custom-Sinks.md similarity index 100% rename from docs/articles/Custom-Sinks.md rename to docs/wiki/Custom-Sinks.md diff --git a/docs/articles/Data-Compression.md b/docs/wiki/Data-Compression.md similarity index 100% rename from docs/articles/Data-Compression.md rename to docs/wiki/Data-Compression.md diff --git a/docs/articles/Data-Encryption.md b/docs/wiki/Data-Encryption.md similarity index 100% rename from docs/articles/Data-Encryption.md rename to docs/wiki/Data-Encryption.md diff --git a/docs/articles/Data-Transformation.md b/docs/wiki/Data-Transformation.md similarity index 100% rename from docs/articles/Data-Transformation.md rename to docs/wiki/Data-Transformation.md diff --git a/docs/articles/DigitalOcean-Spaces-Storage.md b/docs/wiki/DigitalOcean-Spaces-Storage.md similarity index 100% rename from docs/articles/DigitalOcean-Spaces-Storage.md rename to docs/wiki/DigitalOcean-Spaces-Storage.md diff --git a/docs/articles/FTP-Storage.md b/docs/wiki/FTP-Storage.md similarity index 100% rename from docs/articles/FTP-Storage.md rename to docs/wiki/FTP-Storage.md diff --git a/docs/articles/Getting-Started.md b/docs/wiki/Getting-Started.md similarity index 100% rename from docs/articles/Getting-Started.md rename to docs/wiki/Getting-Started.md diff --git a/docs/articles/Google-Cloud-Storage.md b/docs/wiki/Google-Cloud-Storage.md similarity index 100% rename from docs/articles/Google-Cloud-Storage.md rename to docs/wiki/Google-Cloud-Storage.md diff --git a/docs/articles/Hetzner-Storage.md b/docs/wiki/Hetzner-Storage.md similarity index 100% rename from docs/articles/Hetzner-Storage.md rename to docs/wiki/Hetzner-Storage.md diff --git a/docs/articles/Message-Storage.md b/docs/wiki/Message-Storage.md similarity index 100% rename from docs/articles/Message-Storage.md rename to docs/wiki/Message-Storage.md diff --git a/docs/articles/Migration-Guide.md b/docs/wiki/Migration-Guide.md similarity index 100% rename from docs/articles/Migration-Guide.md rename to docs/wiki/Migration-Guide.md diff --git a/docs/articles/MinIO-Storage.md b/docs/wiki/MinIO-Storage.md similarity index 100% rename from docs/articles/MinIO-Storage.md rename to docs/wiki/MinIO-Storage.md diff --git a/docs/articles/SFTP-Storage.md b/docs/wiki/SFTP-Storage.md similarity index 100% rename from docs/articles/SFTP-Storage.md rename to docs/wiki/SFTP-Storage.md diff --git a/docs/articles/Standard-Storage.md b/docs/wiki/Standard-Storage.md similarity index 100% rename from docs/articles/Standard-Storage.md rename to docs/wiki/Standard-Storage.md diff --git a/docs/articles/Vultr-Storage.md b/docs/wiki/Vultr-Storage.md similarity index 100% rename from docs/articles/Vultr-Storage.md rename to docs/wiki/Vultr-Storage.md diff --git a/docs/articles/Wasabi-Storage.md b/docs/wiki/Wasabi-Storage.md similarity index 100% rename from docs/articles/Wasabi-Storage.md rename to docs/wiki/Wasabi-Storage.md diff --git a/docs/articles/index.md b/docs/wiki/index.md similarity index 100% rename from docs/articles/index.md rename to docs/wiki/index.md diff --git a/docs/articles/toc.yml b/docs/wiki/toc.yml similarity index 100% rename from docs/articles/toc.yml rename to docs/wiki/toc.yml