basic architecture change: consul + hangfire

MalwareMultiScan.ScannerWorker/MalwareMultiScan.ScannerWorker.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk.Worker">
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Hangfire" Version="1.7.17" />
+      <PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
+      <PackageReference Include="HangFire.Redis.StackExchange" Version="1.8.4" />
+    </ItemGroup>
+</Project>

MalwareMultiScan.ScannerWorker/Program.cs

@@ -0,0 +1,50 @@
+using System.Threading.Tasks;
+using Hangfire;
+using MalwareMultiScan.Backends.Backends.Interfaces;
+using MalwareMultiScan.Backends.Extensions;
+using MalwareMultiScan.Backends.Services.Implementations;
+using MalwareMultiScan.Backends.Services.Interfaces;
+using MalwareMultiScan.ScannerWorker.Services;
+using MalwareMultiScan.Shared.Extensions;
+using MalwareMultiScan.Shared.Services.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace MalwareMultiScan.ScannerWorker
+{
+    public static class Program
+    {
+        public static async Task Main(string[] args)
+        {
+            await Host.CreateDefaultBuilder(args)
+                .ConfigureLogging((context, builder) =>
+                {
+                    builder.AddConsole();
+                    builder.AddConfiguration(context.Configuration);
+                })
+                .ConfigureAppConfiguration(builder =>
+                {
+                    builder.AddJsonFile("appsettings.json");
+                    builder.AddEnvironmentVariables();
+                })
+                .ConfigureServices((context, services) =>
+                {
+                    services.AddConsul(context.Configuration);
+                    services.AddScanBackend(context.Configuration);
+
+                    services.AddHangfire(context.Configuration, 
+                        context.Configuration.GetValue<string>("BACKEND_ID"));
+
+                    services.AddSingleton<IProcessRunner, ProcessRunner>();
+                    services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
+
+                    services.AddHostedService<ConsulHostedService>();
+                })
+                .UseConsoleLifetime()
+                .Build()
+                .RunAsync();
+        }
+    }
+}

MalwareMultiScan.ScannerWorker/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "MalwareMultiScan.ScannerWorker": {
+      "commandName": "Project"
+    }
+  }
+}

MalwareMultiScan.ScannerWorker/Services/ConsulHostedService.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using Consul;
+using MalwareMultiScan.Backends.Backends.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+namespace MalwareMultiScan.ScannerWorker.Services
+{
+    public class ConsulHostedService : BackgroundService
+    {
+        private static readonly string ServiceId = Dns.GetHostName();
+        private static readonly string CheckId = $"ttl-{ServiceId}";
+
+        private readonly IConsulClient _consulClient;
+        private readonly AgentServiceRegistration _registration;
+
+        public ConsulHostedService(
+            IConsulClient consulClient,
+            IConfiguration configuration,
+            IScanBackend scanBackend)
+        {
+            _consulClient = consulClient;
+
+            _registration = new AgentServiceRegistration
+            {
+                ID = ServiceId,
+                Name = configuration.GetValue<string>("CONSUL_SERVICE_NAME"),
+
+                Meta = new Dictionary<string, string>
+                {
+                    {"BackendId", scanBackend.Id}
+                }
+            };
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                await _consulClient.Agent.PassTTL(
+                    CheckId, string.Empty, stoppingToken);
+
+                await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
+            }
+        }
+
+        public override async Task StartAsync(CancellationToken cancellationToken)
+        {
+            await _consulClient.Agent.ServiceDeregister(
+                _registration.ID, cancellationToken);
+
+            await _consulClient.Agent.ServiceRegister(
+                _registration, cancellationToken);
+
+            await _consulClient.Agent.CheckRegister(new AgentCheckRegistration
+            {
+                ID = CheckId,
+                Name = "TTL",
+                ServiceID = ServiceId,
+                TTL = TimeSpan.FromSeconds(15),
+                DeregisterCriticalServiceAfter = TimeSpan.FromSeconds(60)
+            }, cancellationToken);
+
+            await base.StartAsync(cancellationToken);
+        }
+
+        public override async Task StopAsync(CancellationToken cancellationToken)
+        {
+            await _consulClient.Agent.ServiceDeregister(
+                _registration.ID, cancellationToken);
+
+            await _consulClient.Agent.CheckDeregister(
+                CheckId, cancellationToken);
+
+            await base.StopAsync(cancellationToken);
+        }
+    }
+}

MalwareMultiScan.ScannerWorker/Services/ScanBackgroundJob.cs

@@ -0,0 +1,98 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Hangfire;
+using Hangfire.States;
+using MalwareMultiScan.Backends.Backends.Interfaces;
+using MalwareMultiScan.Shared.Enums;
+using MalwareMultiScan.Shared.Message;
+using MalwareMultiScan.Shared.Services.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace MalwareMultiScan.ScannerWorker.Services
+{
+    /// <inheritdoc />
+    public class ScanBackgroundJob : IScanBackgroundJob
+    {
+        private readonly IScanBackend _backend;
+        private readonly IConfiguration _configuration;
+        private readonly IBackgroundJobClient _jobClient;
+        private readonly ILogger<ScanBackgroundJob> _logger;
+
+        /// <summary>
+        /// Initialize scan background job.
+        /// </summary>
+        /// <param name="configuration">Configuration.</param>
+        /// <param name="logger">Logger.</param>
+        /// <param name="backend">Scan backend.</param>
+        /// <param name="jobClient"></param>
+        public ScanBackgroundJob(
+            IScanBackend backend,
+            IConfiguration configuration,
+            ILogger<ScanBackgroundJob> logger,
+            IBackgroundJobClient jobClient)
+        {
+            _backend = backend;
+            _configuration = configuration;
+            _logger = logger;
+            _jobClient = jobClient;
+        }
+
+        /// <inheritdoc />
+        public async Task Process(ScanQueueMessage message)
+        {
+            _logger.LogInformation(
+                $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
+
+            var cancellationTokenSource = new CancellationTokenSource(
+                TimeSpan.FromSeconds(_configuration.GetValue<int>("MAX_SCANNING_TIME")));
+
+            var cancellationToken = cancellationTokenSource.Token;
+
+            var result = new ScanResultMessage();
+
+            var stopwatch = new Stopwatch();
+
+            stopwatch.Start();
+
+            try
+            {
+                result.Threats = await _backend.ScanAsync(message.Uri, cancellationToken);
+
+                result.Status = ScanResultStatus.Succeeded;
+
+                _logger.LogInformation(
+                    $"Backend {_backend.Id} completed a scan of {message.Id} " +
+                    $"with result '{string.Join(", ", result.Threats)}'");
+            }
+            catch (Exception exception)
+            {
+                result.Status = ScanResultStatus.Failed;
+
+                _logger.LogError(
+                    exception, "Scanning failed with exception");
+            }
+            finally
+            {
+                stopwatch.Stop();
+            }
+
+            result.Duration = stopwatch.ElapsedMilliseconds / 1000;
+
+            try
+            {
+                _logger.LogInformation(
+                    $"Sending scan results with status {result.Status}");
+
+                _jobClient.Create<IScanResultJob>(
+                    x => x.Report(message.Id, _backend.Id, result), new EnqueuedState());
+            }
+            catch (Exception exception)
+            {
+                _logger.LogError(exception, "Failed to send scan results");
+            }
+        }
+    }
+}

MalwareMultiScan.ScannerWorker/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "REDIS_ADDRESS": "localhost:6379",
+  
+  "CONSUL_ADDRESS": "http://localhost:8500",
+  "CONSUL_SERVICE_NAME": "scanner",
+  
+  "BACKEND_ID": "dummy",
+  "MAX_SCANNING_TIME": 60
+}