From 8eb5ee4dddacdedc725b522af8e40e008b0b7d60 Mon Sep 17 00:00:00 2001 From: Joren-vanGoethem Date: Wed, 20 Dec 2023 16:40:17 +0100 Subject: [PATCH] implement env variables --- .github/workflows/docker-portainer.yml | 10 ++++---- .github/workflows/dotnet.yml | 22 +++++++++--------- Importer/Importer.csproj | 6 ----- Importer/Importers/Importer.cs | 29 ++++++++++++++---------- Importer/Importers/ParkingLotImporter.cs | 9 ++++---- Importer/Importers/TreeImporter.cs | 8 ++----- Importer/Mappers.cs | 10 ++++---- Importer/Program.cs | 18 ++++----------- Importer/appsettings.json | 7 ------ docker-compose.yml | 2 ++ 10 files changed, 50 insertions(+), 71 deletions(-) delete mode 100644 Importer/appsettings.json diff --git a/.github/workflows/docker-portainer.yml b/.github/workflows/docker-portainer.yml index fadd0f2..6783128 100644 --- a/.github/workflows/docker-portainer.yml +++ b/.github/workflows/docker-portainer.yml @@ -53,9 +53,9 @@ jobs: steps: - name: Call Webhooks for Portainer redeployment run: | - IFS=',' read -ra WEBHOOKS <<< "${WEBHOOK_URLS}" - for webhook in "${WEBHOOKS[@]}"; do - curl -X POST "$webhook" - done + IFS=',' read -ra WEBHOOKS <<< "${WEBHOOK_URLS}" + for webhook in "${WEBHOOKS[@]}"; do + curl -X POST "$webhook" + done env: - WEBHOOK_URLS: ${{ secrets.WEBHOOK_URL }} + WEBHOOK_URLS: ${{ secrets.WEBHOOK_URL }} diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml index bf986cd..ca2aec9 100644 --- a/.github/workflows/dotnet.yml +++ b/.github/workflows/dotnet.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Setup .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 6.0.x - - name: Restore dependencies - run: dotnet restore - - name: Build - run: dotnet build --no-restore - - name: Test - run: dotnet test --no-build --verbosity normal + - uses: actions/checkout@v3 + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + - name: Restore dependencies + run: dotnet restore + - name: Build + run: dotnet build --no-restore + - name: Test + run: dotnet test --no-build --verbosity normal diff --git a/Importer/Importer.csproj b/Importer/Importer.csproj index a5b683c..c7adb6c 100644 --- a/Importer/Importer.csproj +++ b/Importer/Importer.csproj @@ -24,12 +24,6 @@ - - - Always - - - diff --git a/Importer/Importers/Importer.cs b/Importer/Importers/Importer.cs index 9ad67ce..616cfd5 100644 --- a/Importer/Importers/Importer.cs +++ b/Importer/Importers/Importer.cs @@ -10,7 +10,6 @@ using FrostApi.ResponseModels.ObservedProperty; using FrostApi.ResponseModels.Sensor; using FrostApi.ResponseModels.Thing; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace Importer.Importers; @@ -18,21 +17,22 @@ namespace Importer.Importers; public abstract class Importer { private readonly string _dataStreamName; + private readonly FrostApi.FrostApi _frostApi; protected readonly HttpClient Client; protected readonly string DataType; - private readonly FrostApi.FrostApi _frostApi; protected readonly ILogger Logger; - protected Importer(ILogger logger, IConfiguration config, string dataType, string dataStreamName) + protected Importer(ILogger logger, string dataType, string dataStreamName) { - _frostApi = new FrostApi.FrostApi(config["FrostBaseUrl"] ?? throw new ArgumentNullException("FrostBaseUrl")); - Username = config["Authentication:Username"] ?? throw new ArgumentNullException("Username"); - Password = config["Authentication:Password"] ?? throw new ArgumentNullException("Password"); + _frostApi = new FrostApi.FrostApi(Environment.GetEnvironmentVariable("FROST_SERVER_BASE_URL") ?? + throw new Exception("FROST_SERVER_BASE_URL not set")); + Username = Environment.GetEnvironmentVariable("DKSR_HISTORIC_USERNAME") ?? throw new Exception("DKSR_HISTORIC_USERNAME not set"); + Password = Environment.GetEnvironmentVariable("DKSR_HISTORIC_PASSWORD") ?? throw new Exception("DKSR_HISTORIC_PASSWORD not set"); Client = SetupHttpClient(); Logger = logger; DataType = dataType; _dataStreamName = dataStreamName; - + Logger.LogInformation($"Starting {DataType} Sensor Data Collection"); } @@ -68,6 +68,7 @@ protected Task CreateNewThing(Thing thing) Logger.LogError($"{DataType} {thing.Name} with Id {thing.Properties["Id"]} could not be created"); return Task.CompletedTask; } + protected async Task Update(Thing thing) { Logger.LogDebug($"{DataType} {thing.Name} with Id {thing.Id} found in Frost, updating..."); @@ -90,7 +91,7 @@ protected async Task Update(Thing thing) } } - + private async Task CreateLocationIfNotExists(Thing thing) { var locations = await _frostApi.Locations.GetLocationsForThing(thing.Id); @@ -106,6 +107,7 @@ private async Task CreateLocationIfNotExists(Thing thing) } } } + private async Task CreateNewLocation(Thing thing) { var location = new ThingLocation @@ -174,6 +176,7 @@ await CreateNewSensor(new Sensor return sensorResponse; } + private async Task CreateNewSensor(Sensor sensor) { var response = await _frostApi.Sensors.PostSensor(sensor); @@ -210,6 +213,7 @@ await CreateNewObservedProperty(new ObservedProperty return healthStateObservedPropertyResponse; } + private async Task CreateNewObservedProperty(ObservedProperty observedProperty) { var response = await _frostApi.ObservedProperties.PostObservedProperty(observedProperty); @@ -221,7 +225,7 @@ private async Task CreateNewObservedProperty(ObservedProperty observedProperty) $"ObservedProperty {observedProperty.Name} with Id {observedProperty.Id} could not be created"); } - + private async Task AddObservation(Thing thing, DataStream dataStream) { var observations = await _frostApi.Observations.GetObservationsForDataStream(dataStream.Id); @@ -248,8 +252,8 @@ private async Task AddObservation(Thing thing, DataStream dataStream) Logger.LogError(response.Content.ReadAsStringAsync().Result); } } - - + + private async Task GetOrCreateDataStream(Thing thing) { var dataStreams = await GetFrostDataStreamData(thing.Id); @@ -267,6 +271,7 @@ private async Task GetOrCreateDataStream(Thing thing) return dataStreams; } + private async Task CreateNewDataStream(Thing thing) { var observedPropertyResponse = await GetOrCreateObservedProperty(); @@ -305,7 +310,7 @@ protected Task GetFrostThingData(int id) { return _frostApi.Things.GetAllThings($"?$filter=description eq '{DataType}' &$filter= properties/Id eq '{id}'"); } - + protected Task GetFrostThingData(string id) { return _frostApi.Things.GetAllThings($"?$filter=description eq '{DataType}' &$filter= properties/Id eq '{id}'"); diff --git a/Importer/Importers/ParkingLotImporter.cs b/Importer/Importers/ParkingLotImporter.cs index 2a1addf..40d1c1b 100644 --- a/Importer/Importers/ParkingLotImporter.cs +++ b/Importer/Importers/ParkingLotImporter.cs @@ -1,7 +1,6 @@ using DKSRDomain; using FrostApi.Models.Thing; using Importer.Constants; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace Importer.Importers; @@ -10,11 +9,11 @@ public class ParkingLotImporter : Importer { private Timer _importerTimer; - public ParkingLotImporter(ILogger logger, IConfiguration config) : base(logger, config, "ParkingLot", "Occupancy") + public ParkingLotImporter(ILogger logger) : base(logger, "ParkingLot", "Occupancy") { - _importerTimer = new Timer(Import, null, 0, 60 * 1000); // every minute + _importerTimer = new Timer(Import, null, 0, 60 * 1000 * 2); // every 2 minutes } - + protected override async void Import(object? _) // object? required for running in a timer { try @@ -32,7 +31,7 @@ public ParkingLotImporter(ILogger logger, IConfiguration config) : base(logger, await CreateNewThing(thing); frostThing = await GetFrostThingData(dksrParkingLot.Sid); } - + if (frostThing.Value.Count < 1) throw new Exception($"Creating new thing with id {dksrParkingLot.Sid} seems to have failed..."); diff --git a/Importer/Importers/TreeImporter.cs b/Importer/Importers/TreeImporter.cs index 18e72df..4c26aa4 100644 --- a/Importer/Importers/TreeImporter.cs +++ b/Importer/Importers/TreeImporter.cs @@ -1,7 +1,6 @@ using DKSRDomain; using FrostApi.Models.Thing; using Importer.Constants; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace Importer.Importers; @@ -10,9 +9,9 @@ public class TreeImporter : Importer { private Timer _importerTimer; - public TreeImporter(ILogger logger, IConfiguration config) : base(logger, config, "Tree", "HealthState") + public TreeImporter(ILogger logger) : base(logger, "Tree", "HealthState") { - _importerTimer = new Timer(Import, null, 0, 60 * 1000); // every minute + _importerTimer = new Timer(Import, null, 0, 60 * 1000 * 60); // every hour } protected override async void Import(object? _) @@ -22,7 +21,6 @@ protected override async void Import(object? _) Logger.LogInformation($"Updating {DataType} Data..."); var data = await GetDksrData(); foreach (var dksrTree in data.SensorData) - { try { Thing thing; @@ -45,8 +43,6 @@ protected override async void Import(object? _) { Logger.LogError(e.ToString()); } - } - } catch (Exception e) { diff --git a/Importer/Mappers.cs b/Importer/Mappers.cs index 4b3d645..53f2b3e 100644 --- a/Importer/Mappers.cs +++ b/Importer/Mappers.cs @@ -11,22 +11,22 @@ public static class Mappers public static Thing MapDksrResponse(TreeSenseSensorData treeData, string dataType) { var properties = new Dictionary { { "Id", treeData.Sid } }; - + var observation = new Observation { Result = treeData.HealthState, PhenomenonTime = treeData.Timestamp }; - + return Thing.Create(dataType, treeData.Sid, properties, treeData.Lat, treeData.Lng, observation); } - + public static Thing MapDksrResponse(ParkingLotSensorData parkingLot, string dataType) { var properties = new Dictionary { - { "Id", parkingLot.Sid}, { "ParkingSpaceId", parkingLot.ParkingSpaceId.ToString() }, + { "Id", parkingLot.Sid }, { "ParkingSpaceId", parkingLot.ParkingSpaceId.ToString() }, { "ParkingLotId", parkingLot.ParkingLotId.ToString() } }; var observation = new Observation { Result = parkingLot.Occupied, PhenomenonTime = parkingLot.Timestamp }; - + return Thing.Create(dataType, parkingLot.Sid, properties, parkingLot.Lat, parkingLot.Lon, observation); } diff --git a/Importer/Program.cs b/Importer/Program.cs index 9b20606..7f8d3cb 100644 --- a/Importer/Program.cs +++ b/Importer/Program.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using Importer.Importers; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -8,28 +7,19 @@ namespace Importer; internal class Program { - private static ILogger _logger; - private static IConfiguration _config; + private static ILogger? _logger; - private static async Task Main(string[] args) + private static void Main(string[] args) { - var builder = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", false); - - _config = builder.Build(); - var services = new ServiceCollection(); ConfigureServices(services); IServiceProvider serviceProvider = services.BuildServiceProvider(); _logger = serviceProvider.GetRequiredService>(); - var treeImporter = new TreeImporter(_logger, _config); - var parkingLotImporter = new ParkingLotImporter(_logger, _config); + var treeImporter = new TreeImporter(_logger); + var parkingLotImporter = new ParkingLotImporter(_logger); Process.GetCurrentProcess().WaitForExit(); - - return 1; } private static void ConfigureServices(IServiceCollection services) diff --git a/Importer/appsettings.json b/Importer/appsettings.json deleted file mode 100644 index 0e15449..0000000 --- a/Importer/appsettings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "Authentication": { - "Username": "mainzhistoric", - "Password": "v&:CXc§j9f}p5TuB" - }, - "FrostBaseUrl": "https://frost.elie.de/FROST-Server/v1.1" -} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8065ea1..c8afe07 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,4 +4,6 @@ build: context: . dockerfile: Importer/Dockerfile + env_file: + - stack.env restart: unless-stopped