diff --git a/csharp/App/Backend/DataTypes/Methods/Installation.cs b/csharp/App/Backend/DataTypes/Methods/Installation.cs index a57920436..7f3ffd55e 100644 --- a/csharp/App/Backend/DataTypes/Methods/Installation.cs +++ b/csharp/App/Backend/DataTypes/Methods/Installation.cs @@ -1,6 +1,4 @@ using InnovEnergy.App.Backend.Database; -using InnovEnergy.App.Backend.Relations; -using InnovEnergy.App.Backend.S3; using InnovEnergy.Lib.Utils; namespace InnovEnergy.App.Backend.DataTypes.Methods; @@ -25,33 +23,41 @@ public static class InstallationMethods { var (writeKey, writeSecret) = await installation.CreateWriteKey(); installation.S3WriteSecret = writeSecret; - installation.S3WriteKey = writeKey; + installation.S3WriteKey = writeKey; } installation.S3Key = key; installation.S3Secret = secret; - return Db.Update(installation); } public static Task CreateBucket(this Installation installation) { - return S3Access - .Admin - .CreateBucket(installation.BucketName()); + // TODO + + throw new NotImplementedException(); + + // return S3Access + // .Admin + // .CreateBucket(installation.BucketName()); } public static Task DeleteBucket(this Installation installation) { - return S3Access - .ReadWrite - .DeleteBucket(installation.BucketName()); + // TODO + + throw new NotImplementedException(); + + // return S3Access + // .ReadWrite + // .DeleteBucket(installation.BucketName()); } public static IEnumerable UsersWithAccess(this Installation installation) { - return installation.UsersWithDirectAccess() + return installation + .UsersWithDirectAccess() .Concat(installation.UsersWithInheritedAccess()); } @@ -100,8 +106,10 @@ public static class InstallationMethods public static Installation HideWriteKeyIfUserIsNotAdmin(this Installation installation, Boolean userIsAdmin) { - if(userIsAdmin) return installation; - installation.S3WriteKey = ""; + if(userIsAdmin) + return installation; + + installation.S3WriteKey = ""; installation.S3WriteSecret = ""; return installation; @@ -120,28 +128,12 @@ public static class InstallationMethods return Db.Installations.Any(i => i.Id == installation.Id); } - public static Boolean SetOrderNumbers(this Installation installation) + public static IReadOnlyList GetOrderNumbers(this Installation installation) { - foreach (var orderNumber in installation.OrderNumbers.Split(',')) - { - - var o2I = new OrderNumber2Installation - { - OrderNumber = orderNumber, - InstallationId = installation.Id - }; - Db.Create(o2I); - } - - return true; - } - - public static String GetOrderNumbers(this Installation installation) - { - return String.Join(", ", Db.OrderNumber2Installation + return Db.OrderNumber2Installation .Where(i => i.InstallationId == installation.Id) .Select(i => i.OrderNumber) - .ToReadOnlyList()); + .ToReadOnlyList(); } public static Installation FillOrderNumbers(this Installation installation) diff --git a/csharp/App/Backend/S3/S3Access.cs b/csharp/App/Backend/S3/S3Access.cs index 03dcfb4c7..c71a3d317 100644 --- a/csharp/App/Backend/S3/S3Access.cs +++ b/csharp/App/Backend/S3/S3Access.cs @@ -1,13 +1,20 @@ -using System.Diagnostics.CodeAnalysis; -using static System.IO.File; -using static System.Text.Json.JsonSerializer; - -namespace InnovEnergy.App.Backend.S3; - -[SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")] -public static class S3Access -{ - public static S3Cmd ReadOnly => Deserialize(OpenRead("./Resources/s3ReadOnlyKey.json"))!; - public static S3Cmd ReadWrite => Deserialize(OpenRead("./Resources/s3ReadWriteKey.json"))!; - public static S3Cmd Admin => Deserialize(OpenRead("./Resources/s3AdminKey.json"))!; -} \ No newline at end of file +// using System.Diagnostics.CodeAnalysis; +// using static System.IO.File; +// using static System.Text.Json.JsonSerializer; +// +// namespace InnovEnergy.App.Backend.S3; +// +// [SuppressMessage("Trimming", "IL2026:Members annotated with \'RequiresUnreferencedCodeAttribute\' require dynamic access otherwise can break functionality when trimming application code")] +// public static class S3Access +// { +// public static S3Cmd ReadOnly => ParseJsonFile("./Resources/s3ReadOnlyKey.json")!; +// +// private static T? ParseJsonFile(String file) +// { +// var fileStream = OpenRead(file); +// return Deserialize(fileStream); +// } +// +// public static S3Cmd ReadWrite => Deserialize(OpenRead("./Resources/s3ReadWriteKey.json"))!; +// public static S3Cmd Admin => Deserialize(OpenRead("./Resources/s3AdminKey.json"))!; +// } \ No newline at end of file diff --git a/csharp/App/Backend/S3/S3Cmd.cs b/csharp/App/Backend/S3/S3Cmd.cs index 29cc42fe6..5869d4232 100644 --- a/csharp/App/Backend/S3/S3Cmd.cs +++ b/csharp/App/Backend/S3/S3Cmd.cs @@ -1,106 +1,90 @@ -using System.Diagnostics.CodeAnalysis; -using CliWrap; -using CliWrap.Buffered; -using InnovEnergy.Lib.Utils; - -namespace InnovEnergy.App.Backend.S3; - -[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] -public class S3Cmd -{ - private static readonly Command Python = Cli.Wrap("python3"); - - private const String? S3CmdPath = "Resources/s3cmd.py"; - private const String S3Prefix = "s3://"; - - public String Key { get; init;} - public String Secret { get; init;} - public String Region { get; init; } = "sos-ch-dk-2.exo.io"; - - // private String?[] DefaultArgs { get; } - - // ReSharper disable StringLiteralTypo - // ReSharper enable StringLiteralTypo - [Obsolete("Only to be used by Json-Deserializer")] public S3Cmd() - { - } - - // public S3Cmd(String? key, String? secret) - // { - // DefaultArgs = new[] - // { - // S3CmdPath, - // "--access_key", key, - // "--secret_key", secret, - // }; - // } - - public async Task SignUrl(String bucketName, TimeSpan validity) - { - var result = await Run(bucketName, "signurl", $"+{validity.TotalSeconds}"); - - return result - .StandardOutput - .Replace("\n", "") - .Replace(" ", ""); - } - - public async Task CreateBucket(String bucketName) - { - const String cors = "./Resources/CORS"; - - var result = await Run(bucketName, "mb"); - var setCors = await Run(bucketName, "setcors", cors); - - return result.ExitCode == 0 && setCors.ExitCode == 0; - } - - public async Task ListFilesInBucket(String bucketName) - { - var result = await Run(bucketName, "ls"); - return result.StandardOutput; - } - - public async Task?> GetFileLines(String bucketName, String filename) - { - try - { - await Run(bucketName + "/" + filename, "get", "--force"); - } - catch - { - return null; - } - - var lines = File.ReadAllLines($"./{filename}"); - File.Delete(filename); - return lines; - } - - public async Task DeleteBucket(String bucketName) - { - var result = await Run(bucketName, "rb"); - return result.ExitCode == 0; - } - - private Task Run(String bucketName, String operation, params String[] optionalArgs) - { - var credentials = new[] - { - S3CmdPath, - "--access_key", Key, - "--secret_key", Secret, - "--host", Region - }; - - var args = credentials - .Append(operation) - .Concat(optionalArgs) - .Append(bucketName.EnsureStartsWith(S3Prefix)); - - return Python - .WithArguments(args) - .ExecuteBufferedAsync(); - } - -} \ No newline at end of file +// using System.Diagnostics.CodeAnalysis; +// using CliWrap; +// using CliWrap.Buffered; +// using InnovEnergy.Lib.Utils; +// +// #pragma warning disable CS8618 +// +// namespace InnovEnergy.App.Backend.S3; +// +// [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global")] +// public class S3Cmd +// { +// private static readonly Command Python = Cli.Wrap("python3"); +// +// private const String S3CmdPath = "Resources/s3cmd.py"; +// private const String S3Prefix = "s3://"; +// +// public String Key { get; init; } +// public String Secret { get; init; } +// public String Region { get; init; } = "sos-ch-dk-2.exo.io"; +// +// [Obsolete("Only to be used by Json-Deserializer")] public S3Cmd() +// {} +// +// public async Task CreateBucket(String bucketName) +// { +// const String cors = "./Resources/CORS"; +// +// var makeBucket = await Run(bucketName, "mb"); +// +// if (makeBucket.ExitCode != 0) +// return false; +// +// var setCors = await Run(bucketName, "setcors", cors); +// +// return setCors.ExitCode == 0; +// } +// +// public async Task ListFilesInBucket(String bucketName) +// { +// var result = await Run(bucketName, "ls"); +// return result.StandardOutput; +// } +// +// public async Task?> GetFileLines(String bucketName, String filename) +// { +// try +// { +// await Run(bucketName + "/" + filename, "get", "--force"); +// } +// catch +// { +// return null; +// } +// +// var lines = File.ReadAllLines($"./{filename}"); +// File.Delete(filename); +// return lines; +// } +// +// public async Task DeleteBucket(String bucketName) +// { +// var result = await Run(bucketName, "rb"); +// return result.ExitCode == 0; +// } +// +// private Task Run(String s3Path, String operation, params String[] optionalArgs) +// { +// var credentials = new[] +// { +// S3CmdPath, +// "--access_key", Key, +// "--secret_key", Secret, +// "--host" , Region +// }; +// +// var args = credentials +// .Append(operation) +// .Concat(optionalArgs) +// .Append(s3Path.EnsureStartsWith(S3Prefix)); +// +// var withArguments = Python +// .WithArguments(args); +// +// return withArguments +// .WithValidation(CommandResultValidation.None) +// .ExecuteBufferedAsync(); +// } +// +// } \ No newline at end of file diff --git a/csharp/App/S3Explorer/Program.cs b/csharp/App/S3Explorer/Program.cs index 3fdc7baa5..95413a4d7 100644 --- a/csharp/App/S3Explorer/Program.cs +++ b/csharp/App/S3Explorer/Program.cs @@ -1,95 +1,155 @@ -using System.ComponentModel; -using InnovEnergy.App.Backend.S3; +using Amazon.IdentityManagement.Model; +using Amazon.S3; +using Amazon.S3.Model; +using InnovEnergy.Lib.S3Utils; +using InnovEnergy.Lib.S3Utils.DataTypes; using InnovEnergy.Lib.Utils; -namespace S3Explorer; +namespace InnovEnergy.App.S3Explorer; public static class Program { + private const String BucketSalt = "-3e5b3069-214a-43ee-8d85-57d72000c19d"; public static async Task Main(String[] args) { + var region = S3Cfg.GetDefaultRegionAndCredentials(); + var x = Iam.GetIamClient(region!); + + var c = await x.ListRolesAsync(new ListRolesRequest()); + + + + + + // Todo refactor S3Access into Lib // Sssssecret if (args.Contains("-s")) - { await SnakeGameSs.PlaySnake(); - } - + // Help message - if (args.Length < 4 || args.Contains("-h")) + if (args.Length < 1 || args.Contains("-h")) { - Console.WriteLine("Usage: S3Explorer [BucketId] [from:Unix-time] [to:Unix-time] [#Data-points]"); + Console.WriteLine("Usage: S3Explorer installation-id [from-unix-time] [to-unix-time] [nb-data-points]"); Console.WriteLine("-h Shows this message."); Console.WriteLine("-s 🐍"); return 0; } // Parsing Arguments - var bucketName = args[0] + "-3e5b3069-214a-43ee-8d85-57d72000c19d"; - var startTime = Int64.Parse(args[1]); - var endTime = Int64.Parse(args[2]); - var numberOfDataPoints = Int64.Parse(args[3]); + var bucketName = args[0] + BucketSalt; + var now = DateTime.Now; - var timeBetweenDataPoints = TimeBetweenDataPoints(startTime, endTime, numberOfDataPoints); + var startTime = Int64.Parse(args.ElementAtOr(1, (now - TimeSpan.FromSeconds(20)).ToString())); + var endTime = Int64.Parse(args.ElementAtOr(2, now.ToString())); + var nDataPoints = Int64.Parse(args.ElementAtOr(3, "10")); + + var timestampList = GetDataTimestamps(startTime, endTime, nDataPoints); - // Building a List of the timestamps we want to grab the files for. - var timestampList = new List { }; - for (var i = startTime; i <= endTime; i += timeBetweenDataPoints) - { - //Rounding to even numbers only (we only save every second second) - timestampList.Add((i/2 *2).ToString()); - } - - await PrintFiles(bucketName,timestampList); + await PrintFiles(bucketName, timestampList); // Success return 0; } - private static async Task PrintFiles(String bucketName, List timestampList) + public static async Task ListingObjectsAsync(IAmazonS3 client, String bucketName) { - var newestDataFilename = timestampList.Last(); - var csvFileText = await GetFileText(bucketName, newestDataFilename); - - // Building Header-Row from the newest data - csvFileText - .Select(l => l.Split(";")) - .Select(l => l[0]) - .Prepend("Timestamp") - .JoinWith(";") - .WriteLine(); - - foreach (var timestamp in timestampList) - { - csvFileText = await GetFileText(bucketName, timestamp); - - // Writing Data below data-keys in a timestamped row - csvFileText.Select(l => l.Split(";")) - .Select(l => l[1]) - .Prepend(timestamp) - .JoinWith(";") - .WriteLine(); - } + var request = new ListObjectsV2Request { BucketName = bucketName , Prefix = "1689236"}; + var listObjectsV2Paginator = client.Paginators.ListObjectsV2(request); + + await foreach (var response in listObjectsV2Paginator.Responses) + { + Console.WriteLine($"HttpStatusCode: {response.HttpStatusCode}"); + Console.WriteLine($"Number of Keys: {response.KeyCount}"); + + foreach (var entry in response.S3Objects) + { + Console.WriteLine($"Key = {entry.Key} Size = {entry.Size}"); + } + } } - private static Int64 TimeBetweenDataPoints(Int64 startTime, Int64 endTime, Int64 numberOfDataPoints) - { - // Calculating temporal distance of data files from the number of requested points. - var timeSpan = endTime - startTime; - var timeBetweenDataPoints = timeSpan / numberOfDataPoints; + private static IEnumerable GetDataTimestamps(Int64 startTime, Int64 endTime, Int64 nDataPoints) + { + // Calculating temporal distance of data files from the number of requested points. (rounding for int division) + var timeSpan = endTime - startTime; + var timeBetweenDataPoints = (Double)timeSpan / nDataPoints; // We only upload data every second second so sampling more is impossible. // If this ever changes we might have to change this as well. - timeBetweenDataPoints = Math.Max(timeBetweenDataPoints, 2); - return timeBetweenDataPoints; + + // Building a List of the timestamps we want to grab the files for. + for (Double i = startTime; i <= endTime; i += timeBetweenDataPoints) + { + //Rounding to even numbers only (we only save every second second) + var integer = (Int64) Math.Round(i); + yield return integer / 2 * 2; + } } - // This Method extracts the Text from a given csv file on the s3 bucket - private static async Task GetFileText(String bucketName, String filename) + private static async Task PrintFiles(String bucketName, IEnumerable timestamps) { - return await S3Access.Admin.GetFileText(bucketName, filename + ".csv"); + var columns = new Dictionary> + { + ["timestamp"] = new() + }; + var index = 0; + + foreach (var timestamp in timestamps) + { + var csvFileText = await GetFileText(bucketName, timestamp); + + columns["timestamp"].Add(timestamp.ToString()); + + var dict = csvFileText is null + ? new Dictionary() + : csvFileText + .Select(l => l.Split(";")) + .ToDictionary(kv => kv[0], kv => kv[1]); + + foreach (var key in dict.Keys) + { + // if a key is not yet present in columns we need to backfill it with nulls + if (!columns.ContainsKey(key)) + columns[key] = Enumerable.Repeat(null, index).ToList(); + + columns[key].Add(dict[key]); + } + + // if a key in columns is not present in this record (dict) (except the timestamp) we need to set it to null + foreach (var key in columns.Keys.Where(key => !dict.ContainsKey(key) && key != "timestamp")) + { + columns[key].Add(null); + } + index++; + } + + var headerKeys = columns + .Keys + .OrderBy(k => k) + .Where(k => k != "timestamp") + .Prepend("timestamp") + .ToList(); + + String.Join(';', headerKeys).WriteLine(); + + Enumerable.Range(0, index) + .Select(i => headerKeys.Select(hk => columns[hk][i]).JoinWith(";")) + .ForEach(Console.WriteLine); + } + + // This Method extracts the Text from a given csv file on the s3 bucket + private static async Task?> GetFileText(String bucketName, Int64 timestamp) + { + var csv = await S3Cfg + .GetDefaultRegionAndCredentials()! + .Bucket(bucketName) + .Path($"{timestamp}.csv") + .GetObjectAsString(); + + return csv.Split(Environment.NewLine); } } \ No newline at end of file diff --git a/csharp/App/S3Explorer/S3Explorer.csproj b/csharp/App/S3Explorer/S3Explorer.csproj index 599f0fac3..5bc2ddcf4 100644 --- a/csharp/App/S3Explorer/S3Explorer.csproj +++ b/csharp/App/S3Explorer/S3Explorer.csproj @@ -1,15 +1,15 @@ + + - Exe - net7.0 - enable - enable + InnovEnergy.App.S3Explorer - - + + + diff --git a/csharp/App/S3Explorer/SnakeGameSS.cs b/csharp/App/S3Explorer/SnakeGameSS.cs index 0eaf71a85..1c07985e5 100644 --- a/csharp/App/S3Explorer/SnakeGameSS.cs +++ b/csharp/App/S3Explorer/SnakeGameSS.cs @@ -1,4 +1,4 @@ -namespace S3Explorer; +namespace InnovEnergy.App.S3Explorer; public static class SnakeGameSs { diff --git a/csharp/Lib/S3Utils/Data/S3Bucket.cs b/csharp/Lib/S3Utils/Data/S3Bucket.cs deleted file mode 100644 index 2717729e2..000000000 --- a/csharp/Lib/S3Utils/Data/S3Bucket.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace InnovEnergy.Lib.S3Utils.Data; - -public record S3Bucket : S3RegionCredentials -{ - public required String Bucket { get; init; } -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/Data/S3Credentials.cs b/csharp/Lib/S3Utils/Data/S3Credentials.cs deleted file mode 100644 index d95f6b86e..000000000 --- a/csharp/Lib/S3Utils/Data/S3Credentials.cs +++ /dev/null @@ -1,47 +0,0 @@ -using static System.Environment; - -namespace InnovEnergy.Lib.S3Utils.Data; - -public record S3Credentials -{ - protected static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile).TrimEnd('\\', '/') + "/.s3cfg"; - - public required String Key { get; init; } - public required String Secret { get; init; } - - public static S3Credentials? FromS3Cfg() => FromFile(S3CfgFile); - - public static S3Credentials? FromFile(String file) - { - // [default] - // host_base = sos-ch-dk-2.exo.io - // host_bucket = %(bucket)s.sos-ch-dk-2.exo.io - // access_key = xxxxxxxxxxxxxxx - // secret_key = xxxxxxxxxxxxxxx - // use_https = True - - try - { - var cfg = ParseFile(file); - - return new S3Credentials - { - Key = cfg["access_key"], - Secret = cfg["secret_key"] - }; - } - catch - { - return null; - } - } - - protected static Dictionary ParseFile(String cfgFile) - { - return File - .ReadAllLines(cfgFile) - .Where(l => l.Contains("=")) - .Select(l => l.Split("=")) - .ToDictionary(l => l[0].Trim(), l => l[1].Trim()); - } -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/Data/S3Path.cs b/csharp/Lib/S3Utils/Data/S3Path.cs deleted file mode 100644 index 624a1300e..000000000 --- a/csharp/Lib/S3Utils/Data/S3Path.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace InnovEnergy.Lib.S3Utils.Data; - -public record S3Path : S3Bucket -{ - public required String Path { get; init; } -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/Data/S3RegionCredentials.cs b/csharp/Lib/S3Utils/Data/S3RegionCredentials.cs deleted file mode 100644 index c0956e334..000000000 --- a/csharp/Lib/S3Utils/Data/S3RegionCredentials.cs +++ /dev/null @@ -1,49 +0,0 @@ -using Amazon.Runtime; -using Amazon.S3; -using InnovEnergy.Lib.Utils; - -namespace InnovEnergy.Lib.S3Utils.Data; - -public record S3RegionCredentials : S3Credentials -{ - public required String Region { get; init; } - - private AmazonS3Client? _Client; - internal AmazonS3Client Client => _Client ??= new AmazonS3Client - ( - credentials: new BasicAWSCredentials(Key, Secret), - clientConfig: new() - { - ServiceURL = Region.EnsureStartsWith("https://"), - ForcePathStyle = true, - } - ); - - public new static S3RegionCredentials? FromS3Cfg() => FromFile(S3CfgFile); - - public new static S3RegionCredentials? FromFile(String file) - { - // [default] - // host_base = sos-ch-dk-2.exo.io - // host_bucket = %(bucket)s.sos-ch-dk-2.exo.io - // access_key = xxxxxxxxxxxxxxx - // secret_key = xxxxxxxxxxxxxxx - // use_https = True - - try - { - var cfg = ParseFile(file); - - return new S3RegionCredentials - { - Key = cfg["access_key"], - Secret = cfg["secret_key"], - Region = cfg["host_base"], - }; - } - catch - { - return null; - } - } -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/DataTypes/S3Bucket.cs b/csharp/Lib/S3Utils/DataTypes/S3Bucket.cs new file mode 100644 index 000000000..b485fde6d --- /dev/null +++ b/csharp/Lib/S3Utils/DataTypes/S3Bucket.cs @@ -0,0 +1,7 @@ +namespace InnovEnergy.Lib.S3Utils.DataTypes; + +public record S3Bucket +( + String Name, + S3Region Region +); \ No newline at end of file diff --git a/csharp/Lib/S3Utils/DataTypes/S3Cfg.cs b/csharp/Lib/S3Utils/DataTypes/S3Cfg.cs new file mode 100644 index 000000000..4f5ce85ab --- /dev/null +++ b/csharp/Lib/S3Utils/DataTypes/S3Cfg.cs @@ -0,0 +1,67 @@ +using static System.Environment; + +namespace InnovEnergy.Lib.S3Utils.DataTypes; + +public static class S3Cfg +{ + internal static readonly String S3CfgFile = GetFolderPath(SpecialFolder.UserProfile) + .TrimEnd('\\', '/') + "/.s3cfg"; + + internal static Dictionary FromFile(String cfgFilePath) + { + return File + .ReadAllLines(cfgFilePath) + .Where(l => l.Contains("=")) + .Select(l => l.Split("=")) + .ToDictionary(l => l[0].Trim(), l => l[1].Trim()); + } + + public static S3Credentials? GetDefaultUserCredentials() => GetCredentialsFromFile(S3CfgFile); + + public static S3Credentials? GetCredentialsFromFile(String file) + { + // [default] + // host_base = sos-ch-dk-2.exo.io + // host_bucket = %(bucket)s.sos-ch-dk-2.exo.io + // access_key = xxxxxxxxxxxxxxx + // secret_key = xxxxxxxxxxxxxxx + // use_https = True + + try + { + var cfg = FromFile(file); + + return new S3Credentials + ( + Key : cfg["access_key"], + Secret: cfg["secret_key"] + ); + } + catch + { + return null; + } + } + + public static S3Region? GetDefaultRegionAndCredentials() => GetRegionAndCredentialsFromFile(S3CfgFile); + + public static S3Region? GetRegionAndCredentialsFromFile(String file) + { + try + { + var cfg = FromFile(file); + + var credentials = new S3Credentials + ( + Key : cfg["access_key"], + Secret: cfg["secret_key"] + ); + + return new S3Region(Name: cfg["host_base"], Credentials: credentials); + } + catch + { + return null; + } + } +} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/DataTypes/S3Credentials.cs b/csharp/Lib/S3Utils/DataTypes/S3Credentials.cs new file mode 100644 index 000000000..e2130fa5c --- /dev/null +++ b/csharp/Lib/S3Utils/DataTypes/S3Credentials.cs @@ -0,0 +1,7 @@ +namespace InnovEnergy.Lib.S3Utils.DataTypes; + +public record S3Credentials +( + String Key, + String Secret +); diff --git a/csharp/Lib/S3Utils/DataTypes/S3Region.cs b/csharp/Lib/S3Utils/DataTypes/S3Region.cs new file mode 100644 index 000000000..2e507dfaf --- /dev/null +++ b/csharp/Lib/S3Utils/DataTypes/S3Region.cs @@ -0,0 +1,7 @@ +namespace InnovEnergy.Lib.S3Utils.DataTypes; + +public record S3Region +( + String Name, + S3Credentials Credentials +); diff --git a/csharp/Lib/S3Utils/DataTypes/S3Url.cs b/csharp/Lib/S3Utils/DataTypes/S3Url.cs new file mode 100644 index 000000000..1c4a1837e --- /dev/null +++ b/csharp/Lib/S3Utils/DataTypes/S3Url.cs @@ -0,0 +1,7 @@ +namespace InnovEnergy.Lib.S3Utils.DataTypes; + +public record S3Url +( + String Path, + S3Bucket Bucket +); \ No newline at end of file diff --git a/csharp/Lib/S3Utils/ExoScale/DefaultCredentials.cs b/csharp/Lib/S3Utils/ExoScale/DefaultCredentials.cs deleted file mode 100644 index e503aa9ed..000000000 --- a/csharp/Lib/S3Utils/ExoScale/DefaultCredentials.cs +++ /dev/null @@ -1,21 +0,0 @@ - -using InnovEnergy.Lib.S3Utils.Data; - -namespace InnovEnergy.Lib.S3Utils.ExoScale; - -public static class DefaultCredentials -{ - public static S3Credentials ReadOnly => new S3Credentials - { - Key = "EXOb6d6dc1880cdd51f1ebc6692", - Secret = "kpIey4QJlQFuWG_WoTazcY7kBEjN2f_ll2cDBeg64m4", - }; - - - public static S3Credentials ReadWrite => new S3Credentials - { - Key = "EXO87ca85e29dd412f1238f1cf0", - Secret = "-T9TAqy9a3-0-xj7HKsFFJOCcxfRpcnL6OW5oOrOcWU", - }; - -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/ExoScale/Regions.cs b/csharp/Lib/S3Utils/ExoScale/Regions.cs deleted file mode 100644 index 5e805ca9c..000000000 --- a/csharp/Lib/S3Utils/ExoScale/Regions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace InnovEnergy.Lib.S3Utils.ExoScale; - -public static class Regions -{ - public static String Default => "sos-ch-dk-2.exo.io"; -} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/Iam.cs b/csharp/Lib/S3Utils/Iam.cs new file mode 100644 index 000000000..7a55c3158 --- /dev/null +++ b/csharp/Lib/S3Utils/Iam.cs @@ -0,0 +1,32 @@ +using System.Collections.Concurrent; +using Amazon.IdentityManagement; +using Amazon.Runtime; +using InnovEnergy.Lib.S3Utils.DataTypes; +using InnovEnergy.Lib.Utils; + +namespace InnovEnergy.Lib.S3Utils; + +public static class Iam +{ + + // TODO + + private static readonly ConcurrentDictionary AimClientCache = new(); + + public static AmazonIdentityManagementServiceClient GetIamClient(this S3Url url ) => url.Bucket.GetIamClient(); + public static AmazonIdentityManagementServiceClient GetIamClient(this S3Bucket bucket) => bucket.Region.GetIamClient(); + public static AmazonIdentityManagementServiceClient GetIamClient(this S3Region region) + { + return AimClientCache.GetOrAdd(region, CreateIamClient); // Memoize + } + + private static AmazonIdentityManagementServiceClient CreateIamClient(S3Region region) => new + ( + credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret), + clientConfig: new() { ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://") } + ); + + + + +} \ No newline at end of file diff --git a/csharp/Lib/S3Utils/S3.cs b/csharp/Lib/S3Utils/S3.cs index b5ade9ec8..345d41ce5 100644 --- a/csharp/Lib/S3Utils/S3.cs +++ b/csharp/Lib/S3Utils/S3.cs @@ -1,90 +1,187 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using Amazon.Runtime; +using Amazon.S3; using Amazon.S3.Model; -using InnovEnergy.Lib.S3Utils.Data; +using InnovEnergy.Lib.S3Utils.DataTypes; using InnovEnergy.Lib.Utils; -using S3Bucket = InnovEnergy.Lib.S3Utils.Data.S3Bucket; +using S3Bucket = InnovEnergy.Lib.S3Utils.DataTypes.S3Bucket; +using S3Region = InnovEnergy.Lib.S3Utils.DataTypes.S3Region; namespace InnovEnergy.Lib.S3Utils; public static class S3 { - public static S3RegionCredentials Region(this S3Credentials credentials, String region) => new() - { - Secret = credentials.Secret, - Key = credentials.Key, - Region = region, - }; - - public static S3Bucket Bucket(this S3RegionCredentials region, String bucket) => new() - { - Region = region.Region, - Secret = region.Secret, - Key = region.Key, - Bucket = bucket, - }; - - public static S3Path Path(this S3Bucket bucket, String path) => new() - { - Bucket = bucket.Bucket, - Region = bucket.Region, - Secret = bucket.Secret, - Key = bucket.Key, - Path = path - }; + private static readonly ConcurrentDictionary S3ClientCache = new(); - public static IAsyncEnumerable ListObjects(this S3Bucket bucketOrPrefixPath) + // QOL method + public static S3Region Region(this S3Credentials credentials, String name) => new + ( + Name: name, + Credentials: credentials + ); + + // QOL method + public static S3Bucket Bucket(this S3Region region, String name) => new + ( + Name: name, + Region: region + ); + + // QOL method + public static S3Url Path(this S3Bucket bucket, String path) => new + ( + Bucket: bucket, + Path: path + ); + + public static IAsyncEnumerable ListObjects(this S3Bucket bucket) => ListObjects(bucket, null); + + public static IAsyncEnumerable ListObjects(this S3Bucket bucket, String? pathPrefix) { - var path = bucketOrPrefixPath as S3Path ?? bucketOrPrefixPath.Path(null!); - - var request = new ListObjectsV2Request - { - BucketName = path.Bucket, - Prefix = path.Path - }; - - return bucketOrPrefixPath - .Client + return bucket + .Region + .GetS3Client() .Paginators - .ListObjectsV2(request) - .Responses - .SelectMany(r => r.S3Objects) - .Select(o => path with { Path = o.Key }); + .ListObjectsV2(new() { BucketName = bucket.Name, Prefix = pathPrefix }) + .S3Objects + .Select(o => new S3Url(o.Key, bucket)); } - - public static async Task GetObject(this S3Path path) + + public static Task PutObject(this S3Url path, String data, Encoding encoding) => path.PutObject(encoding.GetBytes(data)); + public static Task PutObject(this S3Url path, String data) => path.PutObject(data, Encoding.UTF8); + public static Task PutObject(this S3Url path, Byte[] data) => path.PutObject(new MemoryStream(data)); + + public static async Task PutObject(this S3Url path, Stream data) { - var request = new GetObjectRequest + var request = new PutObjectRequest { - BucketName = path.Bucket, - Key = path.Path + BucketName = path.Bucket.Name, + Key = path.Path, + InputStream = data }; - using var response = await path.Client.GetObjectAsync(request); - await using var responseStream = response.ResponseStream; - using var reader = new StreamReader(responseStream); + var response = await path + .Bucket + .Region + .GetS3Client() + .PutObjectAsync(request); + return response.HttpStatusCode == HttpStatusCode.OK; + } + + public static Task GetObjectAsString(this S3Url path) => GetObjectAsString(path, Encoding.UTF8); + + public static async Task GetObjectAsString(this S3Url path, Encoding encoding) + { + await using var stream = await GetObjectAsStream(path); + using var reader = new StreamReader(stream, encoding); return await reader.ReadToEndAsync(); } - - public static async IAsyncEnumerable GetObjectLineByLine(this S3Path path) + + public static async Task GetObjectAsStream(this S3Url path) { var request = new GetObjectRequest { - BucketName = path.Bucket, - Key = path.Path + BucketName = path.Bucket.Name, + Key = path.Path }; - using var response = await path.Client.GetObjectAsync(request); - await using var responseStream = response.ResponseStream; - using var reader = new StreamReader(responseStream); + var response = await path + .Bucket + .Region + .GetS3Client() + .GetObjectAsync(request); + + return response.ResponseStream; + } + + public static async Task> GetObject(this S3Url url) + { + // beautiful await using stream soup... + + await using var stream = await url.GetObjectAsStream(); + using var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream); + return memoryStream.ToArray(); + } + + public static IAsyncEnumerable GetObjectLineByLine(this S3Url url) => GetObjectLineByLine(url, Encoding.UTF8); + + public static async IAsyncEnumerable GetObjectLineByLine(this S3Url url, Encoding encoding) + { + await using var stream = await url.GetObjectAsStream(); + + using var reader = new StreamReader(stream, encoding); while (true) { var line = await reader.ReadLineAsync(); - + if (line is not null) yield return line; else yield break; } } + + public static async Task PutBucket(this S3Region region, String name) + { + var request = new PutBucketRequest { BucketName = name }; + + var response = await region + .GetS3Client() + .PutBucketAsync(request); + + return response.HttpStatusCode switch + { + HttpStatusCode.OK => region.Bucket(name), + _ => null + }; + } + + + public static async Task PutCors(this S3Bucket bucket, CORSConfiguration corsConfiguration) + { + var request = new PutCORSConfigurationRequest + { + BucketName = bucket.Name, + Configuration = corsConfiguration + }; + + var response = await bucket + .GetS3Client() + .PutCORSConfigurationAsync(request); + + return response.HttpStatusCode == HttpStatusCode.OK; + } + + public static async Task DeleteBucket(this S3Bucket bucket) + { + var request = new DeleteBucketRequest { BucketName = bucket.Name }; + var response = await bucket + .GetS3Client() + .DeleteBucketAsync(request); + + return response.HttpStatusCode == HttpStatusCode.OK; + } + + private static AmazonS3Client GetS3Client(this S3Url url ) => url.Bucket.GetS3Client(); + private static AmazonS3Client GetS3Client(this S3Bucket bucket) => bucket.Region.GetS3Client(); + private static AmazonS3Client GetS3Client(this S3Region region) + { + return S3ClientCache.GetOrAdd(region, CreateS3Client); // Memoize + } + + private static AmazonS3Client CreateS3Client(S3Region region) => new + ( + credentials: new BasicAWSCredentials(region.Credentials.Key, region.Credentials.Secret), + clientConfig: new() + { + ServiceURL = StringUtils.EnsureStartsWith(region.Name, "https://"), + ForcePathStyle = true, + } + ); + } \ No newline at end of file diff --git a/csharp/Lib/S3Utils/S3Utils.csproj b/csharp/Lib/S3Utils/S3Utils.csproj index c979cea28..e672ba589 100644 --- a/csharp/Lib/S3Utils/S3Utils.csproj +++ b/csharp/Lib/S3Utils/S3Utils.csproj @@ -11,8 +11,23 @@ + + + + + + + + + + + + + + +