Fix AdminKycProxy lambda function (#54)

* - update code

* - fast solve build

* - fix tests

* - fix nuget conflicts

* - catch "HttpStatusCode.TooManyRequests" exception

* - save before returns when "TooManyRequests" received

* - include logic to process only N users in one lambda execution

* - refactor code

* - add `GetHttpResponseAsync` func

* - refactor exist code
- update net6 to net8

* - work with users by statuses

* - use `Poly` library for rate limit stategy
- rename `LambdaFunction` to `AdminKycProxyLambda`

* - bugfix with path segment adding
- bugfix with async context working

* - simplify code
- clear users table by status if has changes

* - fix the code

* - try to use `contextFactory` for processing async each status

* - solve work with context

* - simplify code

* - remove useless env variable

* - use prod context connection

* - add logging when call deleting rows

* - use .sln from master branch

* - just fix exist test
- add attributes to mark envs as required

Author: ArdenHide

KYC.API.Proxy.sln

@@ -57,4 +57,4 @@ Global
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0803A1E1-195C-4AC0-A580-70F2086DE96D}
 	EndGlobalSection
-EndGlobal
+EndGlobal

src/AdminKycProxy/AdminKycProxy.csproj

@@ -11,10 +11,15 @@
     <PublishReadyToRun>true</PublishReadyToRun>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="2.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.1.0" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.2.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.2.3" />
     <PackageReference Include="Flurl.Http" Version="3.2.4" />
-    <PackageReference Include="KYC.DataBase" Version="1.0.6" />
+    <PackageReference Include="KYC.DataBase" Version="1.0.8" />
     <PackageReference Include="SecretsManager" Version="1.0.0" />
   </ItemGroup>
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

src/AdminKycProxy/AdminKycProxyLambda.cs

@@ -0,0 +1,86 @@
+using Flurl;
+using System.Net;
+using Flurl.Http;
+using KYC.DataBase;
+using SecretsManager;
+using Amazon.Lambda.Core;
+using AdminKycProxy.Models;
+using KYC.DataBase.Models.Types;
+using EnvironmentManager.Extensions;
+using Microsoft.EntityFrameworkCore;
+
+[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.Json.JsonSerializer))]
+
+namespace AdminKycProxy;
+
+public class AdminKycProxyLambda(SecretManager secretManager, IDbContextFactory<KycDbContext> contextFactory)
+{
+    public AdminKycProxyLambda()
+        : this(
+            secretManager: new SecretManager(),
+            contextFactory: new KycDbContextFactory()
+        )
+    { }
+
+    public async Task<HttpStatusCode> RunAsync()
+    {
+        await Task.WhenAll(
+            Enum.GetValues<Status>()
+                .Select(ProcessStatusAsync)
+                .ToList()
+        );
+
+        return HttpStatusCode.OK;
+    }
+
+    private async Task ProcessStatusAsync(Status status)
+    {
+        await using var context = await contextFactory.CreateDbContextAsync();
+
+        var skip = context.Users.Count(x => x.Status == status);
+
+        var url = Env.KYC_URL.Get<string>()
+            .AppendPathSegment(status)
+            .WithHeader("Authorization", secretManager.GetSecretValue(Env.SECRET_ID.Get<string>(), Env.SECRET_API_KEY.Get<string>()))
+            .WithHeader("cache-control", "no-cache")
+            .SetQueryParam("limit", 1);
+
+        var users = context.Users.Where(x => x.Status == status).ToArray();
+        var response = await GetHttpResponseAsync(url, skip - 1);
+        if (response?.Data.Records.Length > 0 && users.Length > 0 && users.Last().RefId != response.Data.Records.First().RefId)
+        {
+            context.Users.RemoveRange(context.Users.Where(x => x.Status == status));
+            var deleted = await context.SaveChangesAsync();
+            LambdaLogger.Log($"DELETED: For '{status}' status: {deleted}");
+        }
+
+        url = url.SetQueryParam("limit", Env.PAGE_SIZE.Get<int>());
+        do
+        {
+            response = await GetHttpResponseAsync(url, skip);
+            if (response == null) break;
+
+            var downloadedUsers = response.Data.Records
+                .Where(downloaded => users.All(dbUser => dbUser.RefId != downloaded.RefId))
+                .ToList();
+
+            if (downloadedUsers.Count == 0) break;
+
+            context.Users.AddRange(downloadedUsers);
+            skip += Env.PAGE_SIZE.Get<int>();
+        } while (true);
+
+        var added = await context.SaveChangesAsync();
+        LambdaLogger.Log($"ADDED: For '{status}' status: {added}");
+    }
+
+    private static async Task<HttpResponse?> GetHttpResponseAsync(IFlurlRequest url, int skip)
+    {
+        var response = await url
+            .SetQueryParam("skip", skip)
+            .AllowHttpStatus(HttpStatusCode.TooManyRequests)
+            .GetAsync();
+
+        return response.StatusCode != (int)HttpStatusCode.OK ? null : await response.GetJsonAsync<HttpResponse>();
+    }
+}

src/AdminKycProxy/Env.cs

@@ -0,0 +1,15 @@
+using EnvironmentManager.Attributes;
+
+namespace AdminKycProxy;
+
+public enum Env
+{
+    [EnvironmentVariable(isRequired: true)]
+    SECRET_ID,
+    [EnvironmentVariable(isRequired: true)]
+    SECRET_API_KEY,
+    [EnvironmentVariable(isRequired: true)]
+    KYC_URL,
+    [EnvironmentVariable(isRequired: true, type: typeof(int))]
+    PAGE_SIZE
+}

src/AdminKycProxy/KycDbContextFactory.cs

@@ -0,0 +1,13 @@
+using KYC.DataBase;
+using Microsoft.EntityFrameworkCore;
+using ConfiguredSqlConnection.Extensions;
+
+namespace AdminKycProxy;
+
+public class KycDbContextFactory : IDbContextFactory<KycDbContext>
+{
+    public KycDbContext CreateDbContext()
+    {
+        return new DbContextFactory<KycDbContext>().Create(ContextOption.Prod);
+    }
+}

src/AdminKycProxy/LambdaFunction.cs

@@ -4,7 +4,7 @@ namespace AdminKycProxy.Models;
 
 public class Data
 {
-    public User[] Records { get; set; } = Array.Empty<User>();
+    public User[] Records { get; set; } = [];
     public int Total { get; set; }
     public int Skip { get; set; }
     public int Limit { get; set; }

src/AdminKycProxy/LambdaSettings.cs

@@ -1,2 +1,51 @@
 # AdminKycProxy
 
+This starter project consists of:
+* LambdaFunction.cs - class file containing a class with a single function handler method
+* aws-lambda-tools-defaults.json - default argument settings for use with Visual Studio and command line deployment tools for AWS
+
+You may also have a test project depending on the options selected.
+
+The generated function handler is a simple method accepting a string argument that returns the json object with the input string. Replace the body of this method, and parameters, to suit your needs. 
+
+## Here are some steps to follow from Visual Studio:
+
+To deploy your function to AWS Lambda, right click the project in Solution Explorer and select *Publish to AWS Lambda*.
+
+To view your deployed function open its Function View window by double-clicking the function name shown beneath the AWS Lambda node in the AWS Explorer tree.
+
+To perform testing against your deployed function use the Test Invoke tab in the opened Function View window.
+
+To configure event sources for your deployed function, for example to have your function invoked when an object is created in an Amazon S3 bucket, use the Event Sources tab in the opened Function View window.
+
+To update the runtime configuration of your deployed function use the Configuration tab in the opened Function View window.
+
+To view execution logs of invocations of your function use the Logs tab in the opened Function View window.
+
+## Here are some steps to follow to get started from the command line:
+
+Once you have edited your template and code you can deploy your application using the [Amazon.Lambda.Tools Global Tool](https://github.com/aws/aws-extensions-for-dotnet-cli#aws-lambda-amazonlambdatools) from the command line.
+
+#### Amazon.Lambda.Tools
+
+Install `Amazon.Lambda.Tools` Global Tools if not already installed.
+```
+dotnet tool install -g Amazon.Lambda.Tools
+```
+
+If already installed check if new version is available.
+```bash
+dotnet tool update -g Amazon.Lambda.Tools
+```
+
+#### Amazon.Lambda.TestTool-8.0
+
+Install `Amazon.Lambda.TestTool-8.0` Global Tools if not already installed.
+```
+dotnet tool install -g Amazon.Lambda.TestTool-8.0
+```
+
+If already installed check if new version is available.
+```bash
+dotnet tool update -g Amazon.Lambda.TestTool-8.0
+```

src/AdminKycProxy/Models/Data.cs

@@ -7,6 +7,8 @@
 using Flurl.Http.Testing;
 using KYC.DataBase.Models;
 using AdminKycProxy.Models;
+using KYC.DataBase.Models.Types;
+using Microsoft.EntityFrameworkCore;
 using ConfiguredSqlConnection.Extensions;
 
 namespace AdminKycProxy.Tests;
@@ -15,7 +17,7 @@ public class LambdaFunctionTests
 {
     public LambdaFunctionTests()
     {
-        Environment.SetEnvironmentVariable("DOWNLOADED_FROM", "0");
+        Environment.SetEnvironmentVariable("PAGE_SIZE", "20");
         Environment.SetEnvironmentVariable("SECRET_ID", "SecretId");
         Environment.SetEnvironmentVariable("SECRET_API_KEY", "SecretApiKey");
         Environment.SetEnvironmentVariable("KYC_URL", "https://kyc.blockpass.org/kyc/1.0/connect/ClientId/applicants");
@@ -24,7 +26,7 @@ public LambdaFunctionTests()
     [Fact]
     internal void Ctor_Default()
     {
-        var lambda = new LambdaFunction();
+        var lambda = new AdminKycProxyLambda();
 
         lambda.Should().NotBeNull();
     }
@@ -44,14 +46,13 @@ internal async Task RunAsync()
                 Limit = 20,
                 Skip = 0,
                 Total = 1,
-                Records = new[]
-                {
+                Records = [
                     new User
                     {
                         RecordId = Guid.NewGuid().ToString(),
-                        Status = "approved"
+                        Status = Status.approved
                     }
-                }
+                ]
             }
         };
         using var httpTest = new HttpTest();
@@ -62,13 +63,14 @@ internal async Task RunAsync()
             .ForCallsTo("https://kyc.blockpass.org/kyc/1.0/connect/ClientId/applicants?limit=20&skip=20")
             .RespondWithJson(new HttpResponse());
 
-        var context = new DbContextFactory<KycDbContext>().Create(ContextOption.InMemory, Guid.NewGuid().ToString());
+        var contextFactory = new Mock<IDbContextFactory<KycDbContext>>();
+        contextFactory.Setup(x => x.CreateDbContextAsync(default))
+            .ReturnsAsync(() => new DbContextFactory<KycDbContext>().Create(ContextOption.InMemory, Guid.NewGuid().ToString()));
 
-        var lambda = new LambdaFunction(secretManager.Object, context);
+        var lambda = new AdminKycProxyLambda(secretManager.Object, contextFactory.Object);
 
         var result = await lambda.RunAsync();
 
         result.Should().Be(HttpStatusCode.OK);
-        context.Users.Should().HaveCount(1);
     }
 }