Update README.md

Bump Microsoft.NET.Test.Sdk from 16.8.0 to 16.8.3

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/Clamav.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-clamav" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-clamav" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/Comodo.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-comodo" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-comodo" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/DrWeb.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-drweb" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-drweb" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/KES.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-kes" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-kes" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/McAfee.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-mcafee" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-mcafee" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/Sophos.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-sophos" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-sophos" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml

@@ -1,16 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dockerfiles/WindowsDefender.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-worker-windows-defender" />
-        <option name="buildCliOptions" value="" />
-        <option name="command" value="" />
-        <option name="containerName" value="malware-multi-scan-worker-windows-defender" />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml

@@ -1,7 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Dummy API" type="CompoundRunConfigurationType">
-    <toRun name="MalwareMultiScan.Scanner" type="DotNetProject" />
-    <toRun name="MalwareMultiScan.Api" type="DotNetProject" />
-    <method v="2" />
-  </configuration>
-</component>

.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml

@@ -1,18 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="MalwareMultiScan.Scanner/Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
-    <deployment type="dockerfile">
-      <settings>
-        <option name="imageTag" value="mindcollapse/malware-multi-scan-scanner" />
-        <option name="buildCliOptions" value="" />
-        <option name="buildOnly" value="true" />
-        <option name="command" value="" />
-        <option name="containerName" value="" />
-        <option name="contextFolderPath" value="." />
-        <option name="entrypoint" value="" />
-        <option name="commandLineOptions" value="" />
-        <option name="sourceFilePath" value="MalwareMultiScan.Scanner/Dockerfile" />
-      </settings>
-    </deployment>
-    <method v="2" />
-  </configuration>
-</component>

MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs

@@ -8,9 +8,23 @@ namespace MalwareMultiScan.Api.Attributes
     /// </summary>
     public class IsHttpUrlAttribute : ValidationAttribute
     {
+        private readonly bool _failOnNull;
+
+        /// <summary>
+        /// Initialize http URL validation attribute.
+        /// </summary>
+        /// <param name="failOnNull">True if URL is mandatory, false otherwise.</param>
+        public IsHttpUrlAttribute(bool failOnNull = true)
+        {
+            _failOnNull = failOnNull;
+        }
+
         /// <inheritdoc />
         protected override ValidationResult IsValid(object value, ValidationContext validationContext)
         {
+            if (!_failOnNull && string.IsNullOrEmpty(value.ToString()))
+                return ValidationResult.Success;
+
             if (!Uri.TryCreate((string) value, UriKind.Absolute, out var uri)
                 || !uri.IsAbsoluteUri
                 || uri.Scheme.ToLowerInvariant() != "http"

MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs

@@ -15,7 +15,10 @@ namespace MalwareMultiScan.Api.Attributes
         {
             var maxSize = validationContext
                 .GetRequiredService<IConfiguration>()
-                .GetValue<long>("MaxFileSize");
+                .GetValue<long>("FILE_SIZE_LIMIT");
+
+            if (maxSize == 0)
+                return ValidationResult.Success;
 
             var formFile = (IFormFile) value;
 

MalwareMultiScan.Api/Controllers/QueueController.cs

@@ -1,7 +1,8 @@
+using System;
 using System.ComponentModel.DataAnnotations;
 using System.Threading.Tasks;
 using MalwareMultiScan.Api.Attributes;
-using MalwareMultiScan.Api.Data.Models;
+using MalwareMultiScan.Api.Data;
 using MalwareMultiScan.Api.Services.Interfaces;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -31,13 +32,16 @@ namespace MalwareMultiScan.Api.Controllers
         /// Queue file for scanning.
         /// </summary>
         /// <param name="file">File from form data.</param>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         [HttpPost("file")]
         [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
         [ProducesResponseType(StatusCodes.Status400BadRequest)]
         public async Task<IActionResult> ScanFile(
-            [Required] [MaxFileSize] IFormFile file)
+            [Required] [MaxFileSize] IFormFile file,
+            [FromForm] [IsHttpUrl(false)] string callbackUrl = null)
         {
-            var result = await _scanResultService.CreateScanResult();
+            var result = await _scanResultService.CreateScanResult(
+                string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
 
             string storedFileId;
 
@@ -56,13 +60,16 @@ namespace MalwareMultiScan.Api.Controllers
         /// Queue URL for scanning.
         /// </summary>
         /// <param name="url">URL from form data.</param>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         [HttpPost("url")]
         [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
         [ProducesResponseType(StatusCodes.Status400BadRequest)]
         public async Task<IActionResult> ScanUrl(
-            [FromForm] [Required] [IsHttpUrl] string url)
+            [FromForm] [Required] [IsHttpUrl] string url,
+            [FromForm] [IsHttpUrl(false)] string callbackUrl = null)
         {
-            var result = await _scanResultService.CreateScanResult();
+            var result = await _scanResultService.CreateScanResult(
+                string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
 
             await _scanResultService.QueueUrlScan(result, url);
 

MalwareMultiScan.Api/Controllers/ScanResultsController.cs

@@ -1,5 +1,5 @@
 using System.Threading.Tasks;
-using MalwareMultiScan.Api.Data.Models;
+using MalwareMultiScan.Api.Data;
 using MalwareMultiScan.Api.Services.Interfaces;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -15,7 +15,7 @@ namespace MalwareMultiScan.Api.Controllers
     public class ScanResultsController : Controller
     {
         private readonly IScanResultService _scanResultService;
-        
+
         /// <summary>
         /// Initialize scan results controller.
         /// </summary>

MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs

@@ -1,18 +0,0 @@
-namespace MalwareMultiScan.Api.Data.Configuration
-{
-    /// <summary>
-    /// Scan backend.
-    /// </summary>
-    public class ScanBackend
-    {
-        /// <summary>
-        /// Backend id.
-        /// </summary>
-        public string Id { get; set; }
-        
-        /// <summary>
-        /// Backend state.
-        /// </summary>
-        public bool Enabled { get; set; }
-    }
-}

MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs

@@ -1,28 +0,0 @@
-namespace MalwareMultiScan.Api.Data.Models
-{
-    /// <summary>
-    /// Scan result entry.
-    /// </summary>
-    public class ScanResultEntry
-    {
-        /// <summary>
-        /// Completion status.
-        /// </summary>
-        public bool Completed { get; set; }
-        
-        /// <summary>
-        /// Indicates that scanning completed without error.
-        /// </summary>
-        public bool? Succeeded { get; set; }
-        
-        /// <summary>
-        /// Scanning duration in seconds.
-        /// </summary>
-        public long Duration { get; set; }
-        
-        /// <summary>
-        /// Detected names of threats.
-        /// </summary>
-        public string[] Threats { get; set; }
-    }
-}

MalwareMultiScan.Api/Data/ScanResult.cs

@@ -1,8 +1,10 @@
+using System;
 using System.Collections.Generic;
+using MalwareMultiScan.Shared.Message;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 
-namespace MalwareMultiScan.Api.Data.Models
+namespace MalwareMultiScan.Api.Data
 {
     /// <summary>
     /// Scan result.
@@ -17,9 +19,14 @@ namespace MalwareMultiScan.Api.Data.Models
         public string Id { get; set; }
 
         /// <summary>
-        /// Result entries where key is backend id and value is <see cref="ScanResultEntry"/>.
+        /// Optional callback URL.
         /// </summary>
-        public Dictionary<string, ScanResultEntry> Results { get; set; } =
+        public Uri CallbackUrl { get; set; }
-            new Dictionary<string, ScanResultEntry>();
+
+        /// <summary>
+        /// Result entries where key is backend id and value is <see cref="ScanResultMessage" />.
+        /// </summary>
+        public Dictionary<string, ScanResultMessage> Results { get; set; } =
+            new Dictionary<string, ScanResultMessage>();
     }
 }

MalwareMultiScan.Api/Dockerfile

@@ -4,6 +4,7 @@ WORKDIR /src
 
 COPY MalwareMultiScan.Api /src/MalwareMultiScan.Api
 COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
+COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
 
 RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Api/MalwareMultiScan.Api.csproj
 

MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs

@@ -14,8 +14,8 @@ namespace MalwareMultiScan.Api.Extensions
     {
         internal static void AddMongoDb(this IServiceCollection services, IConfiguration configuration)
         {
-            var client = new MongoClient(configuration.GetConnectionString("Mongo"));
+            var client = new MongoClient(configuration.GetValue<string>("MONGO_ADDRESS"));
-            var db = client.GetDatabase(configuration.GetValue<string>("DatabaseName"));
+            var db = client.GetDatabase(configuration.GetValue<string>("MONGO_DATABASE"));
 
             services.AddSingleton(client);
             services.AddSingleton(db);

MalwareMultiScan.Api/MalwareMultiScan.Api.csproj

@@ -4,24 +4,27 @@
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <Company>Volodymyr Smirnov</Company>
         <Product>MalwareMultiScan Api</Product>
-        <AssemblyVersion>1.0.1</AssemblyVersion>
+        <AssemblyVersion>1.5.0</AssemblyVersion>
-        <FileVersion>1.0.1</FileVersion>
+        <FileVersion>1.5.0</FileVersion>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
     <ItemGroup>
-      <None Update="backends.yaml">
+      <PackageReference Include="Hangfire.AspNetCore" Version="1.7.18" />
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
-      </None>
+      <PackageReference Include="MongoDB.Driver" Version="2.11.5" />
-    </ItemGroup>
+      <PackageReference Include="MongoDB.Driver.GridFS" Version="2.11.5" />
-
-    <ItemGroup>
-      <PackageReference Include="MongoDB.Driver" Version="2.11.3" />
-      <PackageReference Include="MongoDB.Driver.GridFS" Version="2.11.3" />
       <PackageReference Include="YamlDotNet" Version="8.1.2" />
     </ItemGroup>
 
     <ItemGroup>
       <ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
     </ItemGroup>
+
+    <ItemGroup>
+      <Content Update="Properties\launchSettings.json">
+        <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+        <CopyToPublishDirectory>Never</CopyToPublishDirectory>
+      </Content>
+    </ItemGroup>
 </Project>

MalwareMultiScan.Api/Program.cs

@@ -1,4 +1,5 @@
-using EasyNetQ.LightInject;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
 
@@ -7,15 +8,19 @@ namespace MalwareMultiScan.Api
     [ExcludeFromCodeCoverage]
     internal static class Program
     {
-        public static void Main(string[] args)
+        public static async Task Main(string[] args)
         {
-            CreateHostBuilder(args).Build().Run();
+            await Host.CreateDefaultBuilder(args)
-        }
+                .ConfigureWebHostDefaults(builder =>
+                {
+                    builder.ConfigureKestrel(
+                        options => options.Limits.MaxRequestBodySize = long.MaxValue);
 
-        private static IHostBuilder CreateHostBuilder(string[] args)
+                    builder.UseStartup<Startup>();
-        {
+                })
-            return Host.CreateDefaultBuilder(args)
+                .UseConsoleLifetime()
-                .ConfigureWebHostDefaults(webBuilder => webBuilder.UseStartup<Startup>());
+                .Build()
+                .RunAsync();
         }
     }
 }

MalwareMultiScan.Api/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "MalwareMultiScan.Api": {
+      "commandName": "Project"
+    }
+  }
+}

MalwareMultiScan.Api/Services/Implementations/ReceiverHostedService.cs

@@ -1,69 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Api.Services.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-
-namespace MalwareMultiScan.Api.Services.Implementations
-{
-    /// <inheritdoc />
-    public class ReceiverHostedService : IReceiverHostedService
-    {
-        private readonly IBus _bus;
-        private readonly IConfiguration _configuration;
-        private readonly ILogger<ReceiverHostedService> _logger;
-        private readonly IScanResultService _scanResultService;
-
-        /// <summary>
-        /// Initialize receiver hosted service.
-        /// </summary>
-        /// <param name="bus">EasyNetQ bus.</param>
-        /// <param name="configuration">Configuration.</param>
-        /// <param name="scanResultService">Scan result service.</param>
-        /// <param name="logger">Logger.</param>
-        public ReceiverHostedService(IBus bus, IConfiguration configuration, IScanResultService scanResultService,
-            ILogger<ReceiverHostedService> logger)
-        {
-            _bus = bus;
-            _configuration = configuration;
-            _scanResultService = scanResultService;
-            _logger = logger;
-        }
-
-
-        /// <inheritdoc />
-        public Task StartAsync(CancellationToken cancellationToken)
-        {
-            _bus.Receive<ScanResultMessage>(_configuration.GetValue<string>("ResultsSubscriptionId"), async message =>
-            {
-                message.Threats ??= new string[] { };
-
-                _logger.LogInformation(
-                    $"Received a result from {message.Backend} for {message.Id} " +
-                    $"with threats {string.Join(",", message.Threats)}");
-
-                await _scanResultService.UpdateScanResultForBackend(
-                    message.Id, message.Backend, message.Duration, true,
-                    message.Succeeded, message.Threats);
-            });
-
-            _logger.LogInformation(
-                "Started hosted service for receiving scan results");
-
-            return Task.CompletedTask;
-        }
-
-        /// <inheritdoc />
-        public Task StopAsync(CancellationToken cancellationToken)
-        {
-            _bus?.Dispose();
-
-            _logger.LogInformation(
-                "Stopped hosted service for receiving scan results");
-
-            return Task.CompletedTask;
-        }
-    }
-}

MalwareMultiScan.Api/Services/Implementations/ScanBackendService.cs

@@ -1,64 +0,0 @@
-using System;
-using System.IO;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Api.Data.Configuration;
-using MalwareMultiScan.Api.Data.Models;
-using MalwareMultiScan.Api.Services.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using YamlDotNet.Serialization;
-using YamlDotNet.Serialization.NamingConventions;
-
-namespace MalwareMultiScan.Api.Services.Implementations
-{
-    /// <inheritdoc />
-    public class ScanBackendService : IScanBackendService
-    {
-        private readonly IBus _bus;
-        private readonly ILogger<ScanBackendService> _logger;
-
-        /// <summary>
-        /// Initialise scan backend service.
-        /// </summary>
-        /// <param name="configuration">Configuration.</param>
-        /// <param name="bus">EasyNetQ bus.</param>
-        /// <param name="logger">Logger.</param>
-        /// <exception cref="FileNotFoundException">Missing backends.yaml configuration.</exception>
-        public ScanBackendService(IConfiguration configuration, IBus bus, ILogger<ScanBackendService> logger)
-        {
-            _bus = bus;
-            _logger = logger;
-
-            var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
-
-            if (!File.Exists(configurationPath))
-                throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
-
-            var configurationContent = File.ReadAllText(configurationPath);
-
-            var deserializer = new DeserializerBuilder()
-                .WithNamingConvention(CamelCaseNamingConvention.Instance)
-                .Build();
-
-            List = deserializer.Deserialize<ScanBackend[]>(configurationContent);
-        }
-
-        /// <inheritdoc />
-        public ScanBackend[] List { get; }
-
-        /// <inheritdoc />
-        public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl)
-        {
-            _logger.LogInformation(
-                $"Queueing scan for {result.Id} on {backend.Id} at {fileUrl}");
-
-            await _bus.SendAsync(backend.Id, new ScanRequestMessage
-            {
-                Id = result.Id,
-                Uri = new Uri(fileUrl)
-            });
-        }
-    }
-}

MalwareMultiScan.Api/Services/Interfaces/IReceiverHostedService.cs

@@ -1,11 +0,0 @@
-using Microsoft.Extensions.Hosting;
-
-namespace MalwareMultiScan.Api.Services.Interfaces
-{
-    /// <summary>
-    /// Receiver hosted service.
-    /// </summary>
-    public interface IReceiverHostedService : IHostedService
-    {
-    }
-}

MalwareMultiScan.Api/Services/Interfaces/IScanBackendService.cs

@@ -1,25 +0,0 @@
-using System.Threading.Tasks;
-using MalwareMultiScan.Api.Data.Configuration;
-using MalwareMultiScan.Api.Data.Models;
-
-namespace MalwareMultiScan.Api.Services.Interfaces
-{
-    /// <summary>
-    /// Scan backend service.
-    /// </summary>
-    public interface IScanBackendService
-    {
-        /// <summary>
-        /// Get list of parsed backends.
-        /// </summary>
-        ScanBackend[] List { get; }
-
-        /// <summary>
-        /// Queue URL for scan.
-        /// </summary>
-        /// <param name="result">Result entry.</param>
-        /// <param name="backend">Backend entry.</param>
-        /// <param name="fileUrl">Remote URL.</param>
-        Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl);
-    }
-}

MalwareMultiScan.Api/Services/Interfaces/IScanResultService.cs

@@ -1,6 +1,8 @@
+using System;
 using System.IO;
 using System.Threading.Tasks;
-using MalwareMultiScan.Api.Data.Models;
+using MalwareMultiScan.Api.Data;
+using MalwareMultiScan.Shared.Message;
 
 namespace MalwareMultiScan.Api.Services.Interfaces
 {
@@ -12,8 +14,9 @@ namespace MalwareMultiScan.Api.Services.Interfaces
         /// <summary>
         /// Create scan result.
         /// </summary>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         /// <returns>Scan result entry with id.</returns>
-        Task<ScanResult> CreateScanResult();
+        Task<ScanResult> CreateScanResult(Uri callbackUrl);
 
         /// <summary>
         /// Get scan result.
@@ -27,12 +30,8 @@ namespace MalwareMultiScan.Api.Services.Interfaces
         /// </summary>
         /// <param name="resultId">Result id.</param>
         /// <param name="backendId">Backend id.</param>
-        /// <param name="duration">Duration.</param>
+        /// <param name="result">Scan result.</param>
-        /// <param name="completed">Completion status.</param>
+        Task UpdateScanResultForBackend(string resultId, string backendId, ScanResultMessage result = null);
-        /// <param name="succeeded">Indicates that scanning completed without error.</param>
-        /// <param name="threats">Detected names of threats.</param>
-        Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
-            bool completed = false, bool succeeded = false, string[] threats = null);
 
         /// <summary>
         /// Queue URL for scanning.

MalwareMultiScan.Api/Services/ScanResultJob.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Hangfire;
+using Hangfire.States;
+using MalwareMultiScan.Api.Services.Interfaces;
+using MalwareMultiScan.Shared.Message;
+using MalwareMultiScan.Shared.Services.Interfaces;
+using Microsoft.Extensions.Logging;
+
+namespace MalwareMultiScan.Api.Services
+{
+    /// <summary>
+    /// Job for storing scan results.
+    /// </summary>
+    public class ScanResultJob : IScanResultJob
+    {
+        private readonly IBackgroundJobClient _backgroundJobClient;
+        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly ILogger<ScanResultJob> _logger;
+        private readonly IScanResultService _scanResultService;
+
+        /// <summary>
+        /// Initialize scan result storing job.
+        /// </summary>
+        /// <param name="logger">Logger.</param>
+        /// <param name="httpClientFactory">HTTP client factory.</param>
+        /// <param name="scanResultService">Scan result service.</param>
+        /// <param name="backgroundJobClient">Background job client.</param>
+        public ScanResultJob(
+            ILogger<ScanResultJob> logger,
+            IHttpClientFactory httpClientFactory,
+            IScanResultService scanResultService,
+            IBackgroundJobClient backgroundJobClient)
+        {
+            _logger = logger;
+            _httpClientFactory = httpClientFactory;
+            _scanResultService = scanResultService;
+            _backgroundJobClient = backgroundJobClient;
+        }
+
+        /// <inheritdoc />
+        public async Task Report(string resultId, string backendId, ScanResultMessage result)
+        {
+            _logger.LogInformation(
+                $"Received a result from {backendId} for {result} with status {result.Status} " +
+                $"and threats {string.Join(",", result.Threats)}");
+
+            await _scanResultService.UpdateScanResultForBackend(resultId, backendId, result);
+
+            var scanResult = await _scanResultService.GetScanResult(resultId);
+
+            if (scanResult?.CallbackUrl == null)
+                return;
+
+            _backgroundJobClient.Create<IScanResultJob>(
+                x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result),
+                new EnqueuedState("default"));
+        }
+
+        /// <inheritdoc />
+        public async Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result)
+        {
+            var cancellationTokenSource = new CancellationTokenSource(
+                TimeSpan.FromSeconds(5));
+
+            using var httpClient = _httpClientFactory.CreateClient();
+
+            try
+            {
+                var builder = new UriBuilder(uri)
+                {
+                    Query = $"?id={resultId}" +
+                            $"&backend={backendId}"
+                };
+
+                var response = await httpClient.PostAsync(builder.Uri,
+                    new StringContent(JsonSerializer.Serialize(result), Encoding.UTF8, "application/json"),
+                    cancellationTokenSource.Token);
+
+                response.EnsureSuccessStatusCode();
+
+                _logger.LogInformation($"Sent POST to callback URL {uri}");
+            }
+            catch (Exception exception)
+            {
+                _logger.LogError(exception, $"Failed to POST to callback URL {uri}");
+            }
+        }
+    }
+}

MalwareMultiScan.Api/Services/ScanResultService.cs

@@ -1,45 +1,57 @@
+using System;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using MalwareMultiScan.Api.Data.Models;
+using Consul;
+using Hangfire;
+using Hangfire.States;
+using MalwareMultiScan.Api.Data;
 using MalwareMultiScan.Api.Services.Interfaces;
+using MalwareMultiScan.Shared.Enums;
+using MalwareMultiScan.Shared.Message;
+using MalwareMultiScan.Shared.Services.Interfaces;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using MongoDB.Driver.GridFS;
 
-namespace MalwareMultiScan.Api.Services.Implementations
+namespace MalwareMultiScan.Api.Services
 {
     /// <inheritdoc />
     public class ScanResultService : IScanResultService
     {
         private const string CollectionName = "ScanResults";
+        private readonly IBackgroundJobClient _backgroundJobClient;
 
         private readonly IGridFSBucket _bucket;
         private readonly IMongoCollection<ScanResult> _collection;
-        private readonly IScanBackendService _scanBackendService;
+        private readonly IConsulClient _consulClient;
 
         /// <summary>
         /// Initialize scan result service.
         /// </summary>
         /// <param name="db">Mongo database.</param>
         /// <param name="bucket">GridFS bucket.</param>
-        /// <param name="scanBackendService">Scan backend service.</param>
+        /// <param name="consulClient">Consul client.</param>
-        public ScanResultService(IMongoDatabase db, IGridFSBucket bucket, IScanBackendService scanBackendService)
+        /// <param name="backgroundJobClient">Background job client.</param>
+        public ScanResultService(
+            IMongoDatabase db,
+            IGridFSBucket bucket,
+            IConsulClient consulClient,
+            IBackgroundJobClient backgroundJobClient)
         {
             _bucket = bucket;
-            _scanBackendService = scanBackendService;
+            _consulClient = consulClient;
+            _backgroundJobClient = backgroundJobClient;
 
             _collection = db.GetCollection<ScanResult>(CollectionName);
         }
-        
+
         /// <inheritdoc />
-        public async Task<ScanResult> CreateScanResult()
+        public async Task<ScanResult> CreateScanResult(Uri callbackUrl)
         {
             var scanResult = new ScanResult
             {
-                Results = _scanBackendService.List
+                CallbackUrl = callbackUrl
-                    .Where(b => b.Enabled)
-                    .ToDictionary(k => k.Id, v => new ScanResultEntry())
             };
 
             await _collection.InsertOneAsync(scanResult);
@@ -57,25 +69,43 @@ namespace MalwareMultiScan.Api.Services.Implementations
         }
 
         /// <inheritdoc />
-        public async Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
+        public async Task UpdateScanResultForBackend(string resultId, string backendId,
-            bool completed = false, bool succeeded = false, string[] threats = null)
+            ScanResultMessage result = null)
         {
+            result ??= new ScanResultMessage
+            {
+                Status = ScanResultStatus.Queued
+            };
+
             await _collection.UpdateOneAsync(
                 Builders<ScanResult>.Filter.Where(r => r.Id == resultId),
-                Builders<ScanResult>.Update.Set(r => r.Results[backendId], new ScanResultEntry
+                Builders<ScanResult>.Update.Set(r => r.Results[backendId], result));
-                {
-                    Completed = completed,
-                    Succeeded = succeeded,
-                    Duration = duration,
-                    Threats = threats ?? new string[] { }
-                }));
         }
 
         /// <inheritdoc />
         public async Task QueueUrlScan(ScanResult result, string fileUrl)
         {
-            foreach (var backend in _scanBackendService.List.Where(b => b.Enabled))
+            var message = new ScanQueueMessage
-                await _scanBackendService.QueueUrlScan(result, backend, fileUrl);
+            {
+                Id = result.Id,
+                Uri = new Uri(fileUrl)
+            };
+
+            var scanners = await _consulClient.Health.Service("scanner", null, true);
+
+            var backends = scanners.Response
+                .Select(s => s.Service.Meta.TryGetValue("BackendId", out var backendId) ? backendId : null)
+                .Where(q => q != null)
+                .Distinct()
+                .ToArray();
+
+            foreach (var backend in backends)
+            {
+                await UpdateScanResultForBackend(result.Id, backend);
+
+                _backgroundJobClient.Create<IScanBackgroundJob>(
+                    j => j.Process(message), new EnqueuedState(backend));
+            }
         }
 
         /// <inheritdoc />

MalwareMultiScan.Api/Startup.cs

@@ -1,12 +1,14 @@
 using System.Diagnostics.CodeAnalysis;
+using Hangfire;
 using MalwareMultiScan.Api.Extensions;
-using MalwareMultiScan.Api.Services.Implementations;
+using MalwareMultiScan.Api.Services;
 using MalwareMultiScan.Api.Services.Interfaces;
-using MalwareMultiScan.Backends.Extensions;
+using MalwareMultiScan.Shared.Extensions;
+using MalwareMultiScan.Shared.Services.Interfaces;
 using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 
 namespace MalwareMultiScan.Api
 {
@@ -24,27 +26,26 @@ namespace MalwareMultiScan.Api
         {
             services.AddDockerForwardedHeadersOptions();
 
-            services.Configure<KestrelServerOptions>(options =>
+            services.AddConsul(_configuration);
-            {
-                options.Limits.MaxRequestBodySize = _configuration.GetValue<int>("MaxFileSize");
-            });
-
             services.AddMongoDb(_configuration);
-            services.AddRabbitMq(_configuration);
+            services.AddHangfire(_configuration);
 
-            services.AddSingleton<IScanBackendService, ScanBackendService>();
             services.AddSingleton<IScanResultService, ScanResultService>();
+            services.AddSingleton<IScanResultJob, ScanResultJob>();
 
             services.AddControllers();
-
+            services.AddHttpClient();
-            services.AddHostedService<ReceiverHostedService>();
         }
 
-        public void Configure(IApplicationBuilder app)
+        public void Configure(IApplicationBuilder app, IHostEnvironment hostEnvironment)
         {
             app.UseRouting();
-            app.UseForwardedHeaders();
             app.UseEndpoints(endpoints => endpoints.MapControllers());
+
+            app.UseForwardedHeaders();
+
+            if (hostEnvironment.IsDevelopment())
+                app.UseHangfireDashboard();
         }
     }
 }

MalwareMultiScan.Api/appsettings.json

@@ -7,15 +7,12 @@
     }
   },
   
-  "AllowedHosts": "*",
+  "MONGO_ADDRESS": "mongodb://localhost:27017",
-  
+  "MONGO_DATABASE": "MalwareMultiScan",
-  "ConnectionStrings": {
-    "Mongo": "mongodb://localhost:27017",
-    "RabbitMQ": "host=localhost"
-  },
 
-  "DatabaseName": "MalwareMultiScan",
+  "REDIS_ADDRESS": "localhost:6379",
-  "ResultsSubscriptionId": "mms.results",
+  
-  "MaxFileSize": 52428800,
+  "CONSUL_ADDRESS": "http://localhost:8500",
-  "BackendsConfiguration": "backends.yaml"
+  
+  "FILE_SIZE_LIMIT": 52428800
 }

MalwareMultiScan.Api/backends.yaml

@@ -1,23 +0,0 @@
-- id: dummy
-  enabled: true
-  
-- id: clamav
-  enabled: true
-  
-- id: windows-defender
-  enabled: true
-  
-- id: comodo
-  enabled: false
-  
-- id: drweb
-  enabled: false
-  
-- id: kes
-  enabled: false
-  
-- id: mcafee
-  enabled: false
-  
-- id: sophos
-  enabled: false

MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs

@@ -22,17 +22,17 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
         /// Regex to extract names of threats.
         /// </summary>
         protected abstract Regex MatchRegex { get; }
-        
+
         /// <summary>
         /// Path to the backend.
         /// </summary>
         protected abstract string BackendPath { get; }
-        
+
         /// <summary>
         /// Parse StdErr instead of StdOut.
         /// </summary>
         protected virtual bool ParseStdErr { get; } = false;
-        
+
         /// <summary>
         /// Throw on non-zero exit code.
         /// </summary>
@@ -44,7 +44,7 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
         /// <param name="path">Path to the temporary file.</param>
         /// <returns>Formatted string with parameters and path.</returns>
         protected abstract string GetBackendArguments(string path);
-        
+
         /// <inheritdoc />
         public override Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
         {

MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs

@@ -4,7 +4,7 @@ using System.IO;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
-using MalwareMultiScan.Backends.Interfaces;
+using MalwareMultiScan.Backends.Backends.Interfaces;
 
 namespace MalwareMultiScan.Backends.Backends.Abstracts
 {

MalwareMultiScan.Backends/Backends/ClamavScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class ClamavScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/ComodoScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class ComodoScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/DrWebScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class DrWebScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/DummyScanBackend.cs

@@ -2,9 +2,9 @@ using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using MalwareMultiScan.Backends.Interfaces;
+using MalwareMultiScan.Backends.Backends.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class DummyScanBackend : IScanBackend

MalwareMultiScan.Backends/Backends/Interfaces/IScanBackend.cs

@@ -3,7 +3,7 @@ using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace MalwareMultiScan.Backends.Interfaces
+namespace MalwareMultiScan.Backends.Backends.Interfaces
 {
     /// <summary>
     /// Scan backend.
@@ -14,7 +14,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// Unique backend id.
         /// </summary>
         public string Id { get; }
-        
+
         /// <summary>
         /// Scan file.
         /// </summary>
@@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>List of detected threats.</returns>
         public Task<string[]> ScanAsync(string path, CancellationToken cancellationToken);
-        
+
         /// <summary>
         /// Scan URL.
         /// </summary>
@@ -30,7 +30,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>List of detected threats.</returns>
         public Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken);
-        
+
         /// <summary>
         /// Scan stream.
         /// </summary>

MalwareMultiScan.Backends/Backends/KesScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class KesScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/McAfeeScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class McAfeeScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/SophosScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class SophosScanBackend : AbstractLocalProcessScanBackend

MalwareMultiScan.Backends/Backends/WindowsDefenderScanBackend.cs

@@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
 using MalwareMultiScan.Backends.Backends.Abstracts;
 using MalwareMultiScan.Backends.Services.Interfaces;
 
-namespace MalwareMultiScan.Backends.Backends.Implementations
+namespace MalwareMultiScan.Backends.Backends
 {
     /// <inheritdoc />
     public class WindowsDefenderScanBackend : AbstractLocalProcessScanBackend
@@ -11,7 +11,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public WindowsDefenderScanBackend(IProcessRunner processRunner) : base(processRunner)
         {
         }
-        
+
         /// <inheritdoc />
         public override string Id { get; } = "windows-defender";
 
@@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         protected override Regex MatchRegex { get; } =
             new Regex(@"EngineScanCallback\(\): Threat (?<threat>[\S]+) identified",
                 RegexOptions.Compiled | RegexOptions.Multiline);
-        
+
         /// <inheritdoc />
         protected override bool ParseStdErr { get; } = true;
 

MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile

@@ -5,6 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get update && apt-get install -y clamav clamav-daemon
 RUN freshclam --quiet
 
-ENV BackendType=Clamav
+ENV BACKEND_ID=clamav
 
 ENTRYPOINT /etc/init.d/clamav-daemon start && /worker/MalwareMultiScan.Scanner

MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile

@@ -7,4 +7,4 @@ RUN wget -q https://cdn.download.comodo.com/cis/download/installs/linux/cav-linu
     
 RUN wget -q http://download.comodo.com/av/updates58/sigs/bases/bases.cav -O /opt/COMODO/scanners/bases.cav
 
-ENV BackendType=Comodo
+ENV BACKEND_ID=comodo

MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile

@@ -23,6 +23,6 @@ RUN /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && \
     drweb-ctl update && \
     kill $(cat /var/run/drweb-configd.pid)
     
-ENV BackendType=DrWeb
+ENV BACKEND_ID=drweb
     
 ENTRYPOINT /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && /worker/MalwareMultiScan.Scanner

MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile

@@ -30,6 +30,6 @@ kesl-control -B --query "FileName == \"$1\"" 2> /dev/null \n\
 exit $? \
 ' > /usr/bin/kesl-scan && chmod +x /usr/bin/kesl-scan
 
-ENV BackendType=Kes
+ENV BACKEND_ID=kes
 
 ENTRYPOINT /etc/init.d/kesl-supervisor start && /worker/MalwareMultiScan.Scanner

MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile

@@ -16,4 +16,4 @@ RUN wget -q -Nc -r -nd -l1 -A "avvepo????dat.zip" http://download.nai.com/produc
 
 WORKDIR /worker
 
-ENV BackendType=McAfee
+ENV BACKEND_ID=mcafee

MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile

@@ -9,5 +9,5 @@ RUN wget -q $SOPHOS_URL -O /tmp/SophosInstall.sh && \
     chmod +x /tmp/SophosInstall.sh && \
     /tmp/SophosInstall.sh --automatic --acceptlicence || exit 0
     
-ENV BackendType=Sophos
+ENV BACKEND_ID=sophos
 

MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile

@@ -18,4 +18,4 @@ RUN apt-get update && apt-get install -y libc6-i386
 COPY --from=backend /opt/loadlibrary/engine /opt/engine
 COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient
 
-ENV BackendType=Defender
+ENV BACKEND_ID=windows-defender

MalwareMultiScan.Backends/Enums/BackendType.cs

@@ -1,48 +0,0 @@
-namespace MalwareMultiScan.Backends.Enums
-{
-    /// <summary>
-    /// Backend type.
-    /// </summary>
-    public enum BackendType
-    {
-        /// <summary>
-        /// Dummy
-        /// </summary>
-        Dummy,
-        
-        /// <summary>
-        /// Windows Defender.
-        /// </summary>
-        Defender,
-        
-        /// <summary>
-        /// ClamAV.
-        /// </summary>
-        Clamav,
-        
-        /// <summary>
-        /// DrWeb.
-        /// </summary>
-        DrWeb,
-        
-        /// <summary>
-        /// KES.
-        /// </summary>
-        Kes,
-        
-        /// <summary>
-        /// Comodo.
-        /// </summary>
-        Comodo,
-        
-        /// <summary>
-        /// Sophos.
-        /// </summary>
-        Sophos,
-        
-        /// <summary>
-        /// McAfee.
-        /// </summary>
-        McAfee
-    }
-}

MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs

@@ -1,9 +1,7 @@
 using System;
 using System.Diagnostics.CodeAnalysis;
-using EasyNetQ;
+using MalwareMultiScan.Backends.Backends;
-using MalwareMultiScan.Backends.Backends.Implementations;
+using MalwareMultiScan.Backends.Backends.Interfaces;
-using MalwareMultiScan.Backends.Enums;
-using MalwareMultiScan.Backends.Interfaces;
 using MalwareMultiScan.Backends.Services.Implementations;
 using MalwareMultiScan.Backends.Services.Interfaces;
 using Microsoft.Extensions.Configuration;
@@ -17,55 +15,42 @@ namespace MalwareMultiScan.Backends.Extensions
     [ExcludeFromCodeCoverage]
     public static class ServiceCollectionExtensions
     {
-        /// <summary>
-        /// Add RabbitMQ service.
-        /// </summary>
-        /// <param name="services">Service collection.</param>
-        /// <param name="configuration">Configuration.</param>
-        public static void AddRabbitMq(this IServiceCollection services, IConfiguration configuration)
-        {
-            services.AddSingleton(x =>
-                RabbitHutch.CreateBus(configuration.GetConnectionString("RabbitMQ")));
-        }
-
         /// <summary>
         /// Add scanning backend.
         /// </summary>
         /// <param name="services">Service collection.</param>
         /// <param name="configuration">Configuration.</param>
         /// <exception cref="ArgumentOutOfRangeException">Unknown backend.</exception>
-        public static void AddScanningBackend(this IServiceCollection services, IConfiguration configuration)
+        public static void AddScanBackend(this IServiceCollection services, IConfiguration configuration)
         {
             services.AddSingleton<IProcessRunner, ProcessRunner>();
 
-            switch (configuration.GetValue<BackendType>("BackendType"))
+            switch (configuration.GetValue<string>("BACKEND_ID"))
             {
-                case BackendType.Dummy:
+                case "clamav":
-                    services.AddSingleton<IScanBackend, DummyScanBackend>();
-                    break;
-                case BackendType.Defender:
-                    services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
-                    break;
-                case BackendType.Clamav:
                     services.AddSingleton<IScanBackend, ClamavScanBackend>();
                     break;
-                case BackendType.DrWeb:
+                case "drweb":
                     services.AddSingleton<IScanBackend, DrWebScanBackend>();
                     break;
-                case BackendType.Kes:
+                case "kes":
                     services.AddSingleton<IScanBackend, KesScanBackend>();
                     break;
-                case BackendType.Comodo:
+                case "comodo":
                     services.AddSingleton<IScanBackend, ComodoScanBackend>();
                     break;
-                case BackendType.Sophos:
+                case "sophos":
                     services.AddSingleton<IScanBackend, SophosScanBackend>();
                     break;
-                case BackendType.McAfee:
+                case "mcafee":
                     services.AddSingleton<IScanBackend, McAfeeScanBackend>();
                     break;
+                case "windows-defender":
+                    services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
+                    break;
                 default:
-                    throw new ArgumentOutOfRangeException();
+                    services.AddSingleton<IScanBackend, DummyScanBackend>();
+                    break;
             }
         }
     }

MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj

@@ -4,17 +4,13 @@
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <Company>Volodymyr Smirnov</Company>
         <Product>MalwareMultiScan Backends</Product>
-        <AssemblyVersion>1.0.0</AssemblyVersion>
+        <AssemblyVersion>1.5.0</AssemblyVersion>
-        <FileVersion>1.0.0</FileVersion>
+        <FileVersion>1.5.0</FileVersion>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
     
     <ItemGroup>
-      <PackageReference Include="EasyNetQ" Version="5.6.0" />
+      <ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
-      <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.9" />
-      <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.9" />
-      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.9" />
-      <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.9" />
     </ItemGroup>
 
 </Project>

MalwareMultiScan.Backends/Messages/ScanResultMessage.cs

@@ -1,33 +0,0 @@
-namespace MalwareMultiScan.Backends.Messages
-{
-    /// <summary>
-    /// Scan result message.
-    /// </summary>
-    public class ScanResultMessage
-    {
-        /// <summary>
-        /// Result id.
-        /// </summary>
-        public string Id { get; set; }
-        
-        /// <summary>
-        /// Backend.
-        /// </summary>
-        public string Backend { get; set; }
-
-        /// <summary>
-        /// Status.
-        /// </summary>
-        public bool Succeeded { get; set; }
-        
-        /// <summary>
-        /// List of detected threats.
-        /// </summary>
-        public string[] Threats { get; set; }
-
-        /// <summary>
-        /// Duration.
-        /// </summary>
-        public long Duration { get; set; }
-    }
-}

MalwareMultiScan.Backends/Services/ProcessRunner.cs

@@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging;
 namespace MalwareMultiScan.Backends.Services.Implementations
 {
     /// <summary>
-    /// 
     /// </summary>
     [ExcludeFromCodeCoverage]
     public class ProcessRunner : IProcessRunner
@@ -26,7 +25,6 @@ namespace MalwareMultiScan.Backends.Services.Implementations
         }
 
         /// <summary>
-        /// 
         /// </summary>
         /// <param name="path"></param>
         /// <param name="arguments"></param>

MalwareMultiScan.Scanner/Dockerfile

@@ -4,6 +4,7 @@ WORKDIR /src
 
 COPY MalwareMultiScan.Scanner /src/MalwareMultiScan.Scanner
 COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
+COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
 
 RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
 

MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj

@@ -1,18 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk.Worker">
-
     <PropertyGroup>
         <TargetFramework>netcoreapp3.1</TargetFramework>
         <Company>Volodymyr Smirnov</Company>
         <Product>MalwareMultiScan Scanner</Product>
-        <AssemblyVersion>1.0.1</AssemblyVersion>
+        <AssemblyVersion>1.5.0</AssemblyVersion>
-        <FileVersion>1.0.1</FileVersion>
+        <FileVersion>1.5.0</FileVersion>
         <GenerateDocumentationFile>true</GenerateDocumentationFile>
     </PropertyGroup>
 
-    <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Hosting" Version="3.1.9" />
-    </ItemGroup>
-
     <ItemGroup>
       <ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
     </ItemGroup>

MalwareMultiScan.Scanner/Program.cs

@@ -1,10 +1,15 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
 using MalwareMultiScan.Backends.Extensions;
-using MalwareMultiScan.Scanner.Services.Implementations;
+using MalwareMultiScan.Backends.Services.Implementations;
+using MalwareMultiScan.Backends.Services.Interfaces;
+using MalwareMultiScan.Scanner.Services;
+using MalwareMultiScan.Shared.Extensions;
+using MalwareMultiScan.Shared.Services.Interfaces;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 
 namespace MalwareMultiScan.Scanner
 {
@@ -14,20 +19,32 @@ namespace MalwareMultiScan.Scanner
         public static async Task Main(string[] args)
         {
             await Host.CreateDefaultBuilder(args)
-                .ConfigureAppConfiguration(configure =>
+                .ConfigureLogging((context, builder) =>
                 {
-                    configure.AddJsonFile("appsettings.json");
+                    builder.AddConsole();
-                    configure.AddEnvironmentVariables();
+                    builder.AddConfiguration(context.Configuration);
+                })
+                .ConfigureAppConfiguration(builder =>
+                {
+                    builder.AddJsonFile("appsettings.json");
+                    builder.AddEnvironmentVariables();
                 })
                 .ConfigureServices((context, services) =>
                 {
-                    services.AddLogging();
+                    services.AddConsul(context.Configuration);
+                    services.AddScanBackend(context.Configuration);
 
-                    services.AddRabbitMq(context.Configuration);
+                    services.AddHangfire(context.Configuration,
-                    services.AddScanningBackend(context.Configuration);
+                        context.Configuration.GetValue<string>("BACKEND_ID"));
 
-                    services.AddHostedService<ScanHostedService>();
+                    services.AddSingleton<IProcessRunner, ProcessRunner>();
-                }).RunConsoleAsync();
+                    services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
+
+                    services.AddHostedService<ConsulHostedService>();
+                })
+                .UseConsoleLifetime()
+                .Build()
+                .RunAsync();
         }
     }
 }

MalwareMultiScan.Scanner/Properties/launchSettings.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "MalwareMultiScan.Scanner": {
+      "commandName": "Project"
+    }
+  }
+}

MalwareMultiScan.Scanner/Services/ConsulHostedService.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using Consul;
+using MalwareMultiScan.Backends.Backends.Interfaces;
+using Microsoft.Extensions.Hosting;
+
+namespace MalwareMultiScan.Scanner.Services
+{
+    /// <summary>
+    /// Consul registration hosted service.
+    /// </summary>
+    public class ConsulHostedService : BackgroundService
+    {
+        private static readonly string ServiceId = Dns.GetHostName();
+        private static readonly string CheckId = $"ttl-{ServiceId}";
+
+        private readonly IConsulClient _consulClient;
+        private readonly AgentServiceRegistration _registration;
+
+        /// <summary>
+        /// Initialize consul hosted service.
+        /// </summary>
+        /// <param name="consulClient">Consul client.</param>
+        /// <param name="scanBackend">Scan backend.</param>
+        public ConsulHostedService(
+            IConsulClient consulClient,
+            IScanBackend scanBackend)
+        {
+            _consulClient = consulClient;
+
+            _registration = new AgentServiceRegistration
+            {
+                ID = ServiceId,
+                Name = "scanner",
+
+                Meta = new Dictionary<string, string>
+                {
+                    {"BackendId", scanBackend.Id}
+                }
+            };
+        }
+
+        /// <inheritdoc />
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                await _consulClient.Agent.PassTTL(
+                    CheckId, string.Empty, stoppingToken);
+
+                await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
+            }
+        }
+
+        /// <inheritdoc />
+        public override async Task StartAsync(CancellationToken cancellationToken)
+        {
+            await _consulClient.Agent.ServiceDeregister(
+                _registration.ID, cancellationToken);
+
+            await _consulClient.Agent.ServiceRegister(
+                _registration, cancellationToken);
+
+            await _consulClient.Agent.CheckRegister(new AgentCheckRegistration
+            {
+                ID = CheckId,
+                Name = "TTL",
+                ServiceID = ServiceId,
+                TTL = TimeSpan.FromSeconds(15),
+                DeregisterCriticalServiceAfter = TimeSpan.FromSeconds(60)
+            }, cancellationToken);
+
+            await base.StartAsync(cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public override async Task StopAsync(CancellationToken cancellationToken)
+        {
+            await _consulClient.Agent.ServiceDeregister(
+                _registration.ID, cancellationToken);
+
+            await _consulClient.Agent.CheckDeregister(
+                CheckId, cancellationToken);
+
+            await base.StopAsync(cancellationToken);
+        }
+    }
+}

MalwareMultiScan.Scanner/Services/Implementations/ScanHostedService.cs

@@ -1,112 +0,0 @@
-using System;
-using System.Diagnostics;
-using System.Threading;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Backends.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using MalwareMultiScan.Scanner.Services.Interfaces;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-
-namespace MalwareMultiScan.Scanner.Services.Implementations
-{
-    /// <inheritdoc />
-    public class ScanHostedService : IScanHostedService
-    {
-        private readonly IScanBackend _backend;
-        private readonly IBus _bus;
-        private readonly IConfiguration _configuration;
-        private readonly ILogger<ScanHostedService> _logger;
-
-        /// <summary>
-        /// Initialise scan hosted service.
-        /// </summary>
-        /// <param name="logger">Logger.</param>
-        /// <param name="backend">Scan backend.</param>
-        /// <param name="bus">EasyNetQ bus.</param>
-        /// <param name="configuration">Configuration.</param>
-        public ScanHostedService(
-            ILogger<ScanHostedService> logger,
-            IScanBackend backend, IBus bus,
-            IConfiguration configuration)
-        {
-            _logger = logger;
-            _bus = bus;
-            _configuration = configuration;
-            _backend = backend;
-        }
-
-        /// <inheritdoc />
-        public Task StartAsync(CancellationToken cancellationToken)
-        {
-            _bus.Receive<ScanRequestMessage>(_backend.Id, Scan);
-
-            _logger.LogInformation(
-                $"Started scan hosting service for the backend {_backend.Id}");
-
-            return Task.CompletedTask;
-        }
-
-        /// <inheritdoc />
-        public Task StopAsync(CancellationToken cancellationToken)
-        {
-            _bus.Dispose();
-
-            _logger.LogInformation(
-                $"Stopped scan hosting service for the backend {_backend.Id}");
-
-            return Task.CompletedTask;
-        }
-
-        private async Task Scan(ScanRequestMessage message)
-        {
-            _logger.LogInformation(
-                $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
-
-            var cancellationTokenSource = new CancellationTokenSource(
-                TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
-
-            var result = new ScanResultMessage
-            {
-                Id = message.Id,
-                Backend = _backend.Id
-            };
-
-            var stopwatch = new Stopwatch();
-
-            stopwatch.Start();
-
-            try
-            {
-                result.Threats = await _backend.ScanAsync(
-                    message.Uri, cancellationTokenSource.Token);
-
-                result.Succeeded = true;
-
-                _logger.LogInformation(
-                    $"Backend {_backend.Id} completed a scan of {message.Id} " +
-                    $"with result '{string.Join(", ", result.Threats)}'");
-            }
-            catch (Exception exception)
-            {
-                result.Succeeded = false;
-
-                _logger.LogError(
-                    exception, "Scanning failed with exception");
-            }
-            finally
-            {
-                stopwatch.Stop();
-            }
-
-            result.Duration = stopwatch.ElapsedMilliseconds / 1000;
-
-            _logger.LogInformation(
-                $"Sending scan results with status {result.Succeeded}");
-
-            await _bus.SendAsync(
-                _configuration.GetValue<string>("ResultsSubscriptionId"), result);
-        }
-    }
-}

MalwareMultiScan.Scanner/Services/Interfaces/IScanHostedService.cs

@@ -1,11 +0,0 @@
-using Microsoft.Extensions.Hosting;
-
-namespace MalwareMultiScan.Scanner.Services.Interfaces
-{
-    /// <summary>
-    /// Scan hosted service.
-    /// </summary>
-    public interface IScanHostedService : IHostedService
-    {
-    }
-}

MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs

@@ -0,0 +1,102 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using Hangfire;
+using Hangfire.States;
+using MalwareMultiScan.Backends.Backends.Interfaces;
+using MalwareMultiScan.Shared.Enums;
+using MalwareMultiScan.Shared.Message;
+using MalwareMultiScan.Shared.Services.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace MalwareMultiScan.Scanner.Services
+{
+    /// <inheritdoc />
+    public class ScanBackgroundJob : IScanBackgroundJob
+    {
+        private readonly IScanBackend _backend;
+        private readonly IConfiguration _configuration;
+        private readonly IBackgroundJobClient _jobClient;
+        private readonly ILogger<ScanBackgroundJob> _logger;
+
+        /// <summary>
+        /// Initialize scan background job.
+        /// </summary>
+        /// <param name="configuration">Configuration.</param>
+        /// <param name="logger">Logger.</param>
+        /// <param name="backend">Scan backend.</param>
+        /// <param name="jobClient">Background job client.</param>
+        public ScanBackgroundJob(
+            IScanBackend backend,
+            IConfiguration configuration,
+            ILogger<ScanBackgroundJob> logger,
+            IBackgroundJobClient jobClient)
+        {
+            _backend = backend;
+            _configuration = configuration;
+            _logger = logger;
+            _jobClient = jobClient;
+        }
+
+        /// <inheritdoc />
+        public async Task Process(ScanQueueMessage message)
+        {
+            _logger.LogInformation(
+                $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
+
+            var cancellationTokenSource = new CancellationTokenSource(
+                TimeSpan.FromSeconds(_configuration.GetValue<int>("MAX_SCANNING_TIME")));
+
+            var cancellationToken = cancellationTokenSource.Token;
+
+            var result = new ScanResultMessage
+            {
+                Status = ScanResultStatus.Queued
+            };
+
+            var stopwatch = new Stopwatch();
+
+            stopwatch.Start();
+
+            try
+            {
+                result.Threats = await _backend.ScanAsync(message.Uri, cancellationToken);
+
+                result.Status = ScanResultStatus.Succeeded;
+
+                _logger.LogInformation(
+                    $"Backend {_backend.Id} completed a scan of {message.Id} " +
+                    $"with result '{string.Join(", ", result.Threats)}'");
+            }
+            catch (Exception exception)
+            {
+                result.Status = ScanResultStatus.Failed;
+
+                _logger.LogError(
+                    exception, "Scanning failed with exception");
+            }
+            finally
+            {
+                stopwatch.Stop();
+            }
+
+            result.Duration = stopwatch.ElapsedMilliseconds / 1000;
+
+            try
+            {
+                _logger.LogInformation(
+                    $"Sending scan results with status {result.Status}");
+
+                _jobClient.Create<IScanResultJob>(
+                    x => x.Report(message.Id, _backend.Id, result),
+                    new EnqueuedState("default"));
+            }
+            catch (Exception exception)
+            {
+                _logger.LogError(exception, "Failed to send scan results");
+            }
+        }
+    }
+}

MalwareMultiScan.Scanner/appsettings.json

@@ -1,18 +1,9 @@
 {
-  "Logging": {
+  "REDIS_ADDRESS": "localhost:6379",
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
-    }
-  },
-
-  "BackendType": "Dummy",
   
-  "MaxScanningTime": 60,
+  "CONSUL_ADDRESS": "http://localhost:8500",
-  "ResultsSubscriptionId": "mms.results",
   
-  "ConnectionStrings": {
+  "BACKEND_ID": "dummy",
-    "RabbitMQ": "host=localhost;prefetchcount=1"
+  "MAX_SCANNING_TIME": 60,
-  }
+  "WORKER_COUNT": 4
 }

MalwareMultiScan.Shared/Enums/ScanResultStatus.cs

@@ -0,0 +1,23 @@
+namespace MalwareMultiScan.Shared.Enums
+{
+    /// <summary>
+    /// Scan result status.
+    /// </summary>
+    public enum ScanResultStatus
+    {
+        /// <summary>
+        /// Scan is queued.
+        /// </summary>
+        Queued,
+
+        /// <summary>
+        /// Scan succeeded.
+        /// </summary>
+        Succeeded,
+
+        /// <summary>
+        /// Scan failed.
+        /// </summary>
+        Failed
+    }
+}

MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Consul;
+using Hangfire;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace MalwareMultiScan.Shared.Extensions
+{
+    /// <summary>
+    /// Extensions for IServiceCollection.
+    /// </summary>
+    [ExcludeFromCodeCoverage]
+    public static class ServiceCollectionExtensions
+    {
+        /// <summary>
+        /// Add consul to the service collection and register the node on start.
+        /// </summary>
+        /// <param name="services">Service collection.</param>
+        /// <param name="configuration">Configuration.</param>
+        public static void AddConsul(this IServiceCollection services, IConfiguration configuration)
+        {
+            services.AddSingleton<IConsulClient>(new ConsulClient(config =>
+            {
+                config.Address = configuration.GetValue<Uri>("CONSUL_ADDRESS");
+                config.WaitTime = TimeSpan.FromSeconds(120);
+            }));
+        }
+
+        /// <summary>
+        /// Add Hangfire with Redis storage.
+        /// </summary>
+        /// <param name="services">Service collection.</param>
+        /// <param name="configuration">Configuration.</param>
+        /// <param name="queues">Queue names.</param>
+        public static void AddHangfire(this IServiceCollection services,
+            IConfiguration configuration, params string[] queues)
+        {
+            if (queues.Length == 0)
+                queues = new[] {"default"};
+
+            services.AddHangfire(options =>
+                options.UseRedisStorage(configuration.GetValue<string>("REDIS_ADDRESS")));
+
+            services.AddHangfireServer(options =>
+            {
+                options.Queues = queues;
+
+                options.ServerTimeout = TimeSpan.FromSeconds(30);
+                options.HeartbeatInterval = TimeSpan.FromSeconds(5);
+                options.ServerCheckInterval = TimeSpan.FromSeconds(15);
+
+                var workerCount = configuration.GetValue<int>("WORKER_COUNT");
+
+                if (workerCount > 0)
+                    options.WorkerCount = workerCount;
+            });
+        }
+    }
+}

MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+        <Company>Volodymyr Smirnov</Company>
+        <Product>MalwareMultiScan Shared</Product>
+        <AssemblyVersion>1.5.0</AssemblyVersion>
+        <FileVersion>1.5.0</FileVersion>
+        <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Consul" Version="1.6.1.1" />
+        <PackageReference Include="Hangfire.AspNetCore" Version="1.7.18" />
+        <PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
+        <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.10" />
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.1.10" />
+        <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.10" />
+    </ItemGroup>
+</Project>

MalwareMultiScan.Shared/Message/ScanQueueMessage.cs

@@ -1,19 +1,19 @@
 using System;
 
-namespace MalwareMultiScan.Backends.Messages
+namespace MalwareMultiScan.Shared.Message
 {
     /// <summary>
-    /// Scan request message.
+    /// Scan queue message.
     /// </summary>
-    public class ScanRequestMessage
+    public class ScanQueueMessage
     {
         /// <summary>
-        /// Result id.
+        /// Scan result id.
         /// </summary>
         public string Id { get; set; }
 
         /// <summary>
-        /// Remote URL.
+        /// Scan file URL.
         /// </summary>
         public Uri Uri { get; set; }
     }

MalwareMultiScan.Shared/Message/ScanResultMessage.cs

@@ -0,0 +1,25 @@
+using MalwareMultiScan.Shared.Enums;
+
+namespace MalwareMultiScan.Shared.Message
+{
+    /// <summary>
+    /// Scan result message.
+    /// </summary>
+    public class ScanResultMessage
+    {
+        /// <summary>
+        /// Scan status.
+        /// </summary>
+        public ScanResultStatus Status { get; set; }
+
+        /// <summary>
+        /// Scan duration in seconds.
+        /// </summary>
+        public long Duration { get; set; }
+
+        /// <summary>
+        /// Detected threats.
+        /// </summary>
+        public string[] Threats { get; set; }
+    }
+}

MalwareMultiScan.Shared/Services/Interfaces/IScanBackgroundJob.cs

@@ -0,0 +1,19 @@
+using System.Threading.Tasks;
+using Hangfire;
+using MalwareMultiScan.Shared.Message;
+
+namespace MalwareMultiScan.Shared.Services.Interfaces
+{
+    /// <summary>
+    /// Scan background job.
+    /// </summary>
+    [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
+    public interface IScanBackgroundJob
+    {
+        /// <summary>
+        /// Start scan.
+        /// </summary>
+        /// <param name="message">Scan queue message.</param>
+        Task Process(ScanQueueMessage message);
+    }
+}

MalwareMultiScan.Shared/Services/Interfaces/IScanResultJob.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Threading.Tasks;
+using Hangfire;
+using MalwareMultiScan.Shared.Message;
+
+namespace MalwareMultiScan.Shared.Services.Interfaces
+{
+    /// <summary>
+    /// Scan background result job.
+    /// </summary>
+    public interface IScanResultJob
+    {
+        /// <summary>
+        /// Report job status.
+        /// </summary>
+        /// <param name="resultId">Result id.</param>
+        /// <param name="backendId">Backend id.</param>
+        /// <param name="result">Scan result.</param>
+        [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
+        Task Report(string resultId, string backendId, ScanResultMessage result);
+
+        /// <summary>
+        /// Notify remote URL on scan result.
+        /// </summary>
+        /// <param name="uri">Base URL.</param>
+        /// <param name="resultId">Result id.</param>
+        /// <param name="backendId">Backend id.</param>
+        /// <param name="result">Scan result.</param>
+        [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
+        Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result);
+    }
+}

MalwareMultiScan.Tests/Api/AttributesTests.cs

@@ -36,7 +36,7 @@ namespace MalwareMultiScan.Tests.Api
                 new MemoryConfigurationProvider(new MemoryConfigurationSource())
             })
             {
-                ["MaxFileSize"] = "100"
+                ["FILE_SIZE_LIMIT"] = "100"
             };
 
             var context = new ValidationContext(

MalwareMultiScan.Tests/Api/ControllersTests.cs

@@ -1,7 +1,7 @@
 using System.IO;
 using System.Threading.Tasks;
 using MalwareMultiScan.Api.Controllers;
-using MalwareMultiScan.Api.Data.Models;
+using MalwareMultiScan.Api.Data;
 using MalwareMultiScan.Api.Services.Interfaces;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -46,7 +46,7 @@ namespace MalwareMultiScan.Tests.Api
                 .Returns(Task.FromResult((Stream) null));
 
             _scanResultServiceMock
-                .Setup(x => x.CreateScanResult())
+                .Setup(x => x.CreateScanResult(null))
                 .Returns(Task.FromResult(new ScanResult()));
 
             _downloadController = new DownloadController(_scanResultServiceMock.Object);

MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs

@@ -1,77 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Api.Services.Implementations;
-using MalwareMultiScan.Api.Services.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Configuration.Memory;
-using Microsoft.Extensions.Logging;
-using Moq;
-using NUnit.Framework;
-
-namespace MalwareMultiScan.Tests.Api
-{
-    public class ReceiverHostedServiceTests
-    {
-        private Mock<IBus> _busMock;
-        private IReceiverHostedService _receiverHostedService;
-        private Mock<IScanResultService> _scanResultServiceMock;
-
-        [SetUp]
-        public void SetUp()
-        {
-            _busMock = new Mock<IBus>();
-
-            _busMock
-                .Setup(x => x.Receive("mms.results", It.IsAny<Func<ScanResultMessage, Task>>()))
-                .Callback<string, Func<ScanResultMessage, Task>>((s, func) =>
-                {
-                    var task = func.Invoke(new ScanResultMessage
-                    {
-                        Id = "test",
-                        Backend = "dummy",
-                        Duration = 100,
-                        Succeeded = true,
-                        Threats = new[] {"Test"}
-                    });
-
-                    task.Wait();
-                });
-
-            _scanResultServiceMock = new Mock<IScanResultService>();
-
-            var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
-            {
-                new MemoryConfigurationProvider(new MemoryConfigurationSource())
-            })
-            {
-                ["ResultsSubscriptionId"] = "mms.results"
-            };
-
-            _receiverHostedService = new ReceiverHostedService(
-                _busMock.Object,
-                configuration,
-                _scanResultServiceMock.Object,
-                Mock.Of<ILogger<ReceiverHostedService>>());
-        }
-
-        [Test]
-        public async Task TestBusReceiveScanResultMessage()
-        {
-            await _receiverHostedService.StartAsync(default);
-
-            _scanResultServiceMock.Verify(x => x.UpdateScanResultForBackend(
-                "test", "dummy", 100, true, true, new[] {"Test"}), Times.Once);
-        }
-
-        [Test]
-        public async Task TestBusIsDisposedOnStop()
-        {
-            await _receiverHostedService.StopAsync(default);
-
-            _busMock.Verify(x => x.Dispose(), Times.Once);
-        }
-    }
-}

MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs

@@ -1,63 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Api.Data.Configuration;
-using MalwareMultiScan.Api.Data.Models;
-using MalwareMultiScan.Api.Services.Implementations;
-using MalwareMultiScan.Api.Services.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Configuration.Memory;
-using Microsoft.Extensions.Logging;
-using Moq;
-using NUnit.Framework;
-
-namespace MalwareMultiScan.Tests.Api
-{
-    public class ScanBackendServiceTests
-    {
-        private Mock<IBus> _busMock;
-        private IScanBackendService _scanBackendService;
-
-        [SetUp]
-        public void SetUp()
-        {
-            var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
-            {
-                new MemoryConfigurationProvider(new MemoryConfigurationSource())
-            })
-            {
-                ["BackendsConfiguration"] = "backends.yaml"
-            };
-
-            _busMock = new Mock<IBus>();
-
-            _scanBackendService = new ScanBackendService(
-                configuration,
-                _busMock.Object,
-                Mock.Of<ILogger<ScanBackendService>>());
-        }
-
-        [Test]
-        public void TestBackendsAreNotEmpty()
-        {
-            Assert.IsNotEmpty(_scanBackendService.List);
-        }
-
-        [Test]
-        public async Task TestQueueUrlScan()
-        {
-            await _scanBackendService.QueueUrlScan(
-                new ScanResult {Id = "test"},
-                new ScanBackend {Id = "dummy"},
-                "http://test.com"
-            );
-
-            _busMock.Verify(x => x.SendAsync(
-                "dummy", It.Is<ScanRequestMessage>(r =>
-                    r.Id == "test" &&
-                    r.Uri == new Uri("http://test.com"))), Times.Once);
-        }
-    }
-}

MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs

@@ -1,9 +1,11 @@
 using System.IO;
 using System.Threading.Tasks;
-using MalwareMultiScan.Api.Data.Configuration;
+using Consul;
-using MalwareMultiScan.Api.Data.Models;
+using Hangfire;
-using MalwareMultiScan.Api.Services.Implementations;
+using MalwareMultiScan.Api.Services;
 using MalwareMultiScan.Api.Services.Interfaces;
+using MalwareMultiScan.Shared.Enums;
+using MalwareMultiScan.Shared.Message;
 using Mongo2Go;
 using MongoDB.Bson;
 using MongoDB.Driver;
@@ -17,7 +19,6 @@ namespace MalwareMultiScan.Tests.Api
     {
         private MongoDbRunner _mongoDbRunner;
         private IScanResultService _resultService;
-        private Mock<IScanBackendService> _scanBackendService;
 
         [SetUp]
         public void SetUp()
@@ -28,19 +29,10 @@ namespace MalwareMultiScan.Tests.Api
             var database = connection.GetDatabase("Test");
             var gridFsBucket = new GridFSBucket(database);
 
-            _scanBackendService = new Mock<IScanBackendService>();
-
-            _scanBackendService
-                .SetupGet(x => x.List)
-                .Returns(() => new[]
-                {
-                    new ScanBackend {Id = "dummy", Enabled = true},
-                    new ScanBackend {Id = "clamav", Enabled = true},
-                    new ScanBackend {Id = "disabled", Enabled = false}
-                });
-
             _resultService = new ScanResultService(
-                database, gridFsBucket, _scanBackendService.Object);
+                database, gridFsBucket,
+                Mock.Of<IConsulClient>(),
+                Mock.Of<IBackgroundJobClient>());
         }
 
         [TearDown]
@@ -82,13 +74,17 @@ namespace MalwareMultiScan.Tests.Api
         [Test]
         public async Task TestScanResultCreateGet()
         {
-            var result = await _resultService.CreateScanResult();
+            var result = await _resultService.CreateScanResult(null);
 
             Assert.NotNull(result.Id);
-            Assert.That(result.Results, Contains.Key("dummy"));
 
             await _resultService.UpdateScanResultForBackend(
-                result.Id, "dummy", 100, true, true, new[] {"Test"});
+                result.Id, "dummy", new ScanResultMessage
+                {
+                    Duration = 100,
+                    Status = ScanResultStatus.Succeeded,
+                    Threats = new[] {"Test"}
+                });
 
             result = await _resultService.GetScanResult(result.Id);
 
@@ -97,25 +93,10 @@ namespace MalwareMultiScan.Tests.Api
 
             var dummyResult = result.Results["dummy"];
 
-            Assert.IsTrue(dummyResult.Completed);
+            Assert.IsTrue(dummyResult.Status == ScanResultStatus.Succeeded);
-            Assert.IsTrue(dummyResult.Succeeded);
             Assert.AreEqual(dummyResult.Duration, 100);
             Assert.IsNotEmpty(dummyResult.Threats);
             Assert.That(dummyResult.Threats, Is.EquivalentTo(new[] {"Test"}));
         }
-
-        [Test]
-        public async Task TestQueueUrlScan()
-        {
-            var result = await _resultService.CreateScanResult();
-
-            await _resultService.QueueUrlScan(
-                result, "http://url.com");
-
-            _scanBackendService.Verify(x => x.QueueUrlScan(
-                It.IsAny<ScanResult>(),
-                It.IsAny<ScanBackend>(),
-                "http://url.com"), Times.Exactly(2));
-        }
     }
 }

MalwareMultiScan.Tests/Backends/BackendsTests.cs

@@ -1,7 +1,7 @@
 using System.Threading;
 using System.Threading.Tasks;
-using MalwareMultiScan.Backends.Backends.Implementations;
+using MalwareMultiScan.Backends.Backends;
-using MalwareMultiScan.Backends.Interfaces;
+using MalwareMultiScan.Backends.Backends.Interfaces;
 using MalwareMultiScan.Backends.Services.Interfaces;
 using Moq;
 using NUnit.Framework;

MalwareMultiScan.Tests/MalwareMultiScan.Tests.csproj

@@ -8,10 +8,10 @@
 
     <ItemGroup>
         <PackageReference Include="Mongo2Go" Version="2.2.14" />
-        <PackageReference Include="Moq" Version="4.14.7" />
+        <PackageReference Include="Moq" Version="4.15.2" />
         <PackageReference Include="nunit" Version="3.12.0" />
         <PackageReference Include="NUnit3TestAdapter" Version="3.17.0" />
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
     </ItemGroup>
 
     <ItemGroup>

MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs

@@ -1,86 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using EasyNetQ;
-using MalwareMultiScan.Backends.Interfaces;
-using MalwareMultiScan.Backends.Messages;
-using MalwareMultiScan.Scanner.Services.Implementations;
-using MalwareMultiScan.Scanner.Services.Interfaces;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Configuration.Memory;
-using Microsoft.Extensions.Logging;
-using Moq;
-using NUnit.Framework;
-
-namespace MalwareMultiScan.Tests.Scanner
-{
-    public class ScanHostedServiceTests
-    {
-        private Mock<IBus> _busMock;
-        private Mock<IScanBackend> _scanBackendMock;
-        private IScanHostedService _scanHostedService;
-
-        [SetUp]
-        public void SetUp()
-        {
-            var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
-            {
-                new MemoryConfigurationProvider(new MemoryConfigurationSource())
-            })
-            {
-                ["ResultsSubscriptionId"] = "mms.results"
-            };
-
-            _busMock = new Mock<IBus>();
-
-            _busMock
-                .Setup(x => x.Receive("dummy", It.IsAny<Func<ScanRequestMessage, Task>>()))
-                .Callback<string, Func<ScanRequestMessage, Task>>((s, func) =>
-                {
-                    var task = func.Invoke(new ScanRequestMessage
-                    {
-                        Id = "test",
-                        Uri = new Uri("http://test.com")
-                    });
-
-                    task.Wait();
-                });
-
-            _scanBackendMock = new Mock<IScanBackend>();
-
-            _scanBackendMock
-                .Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
-                .Returns(Task.FromResult(new[] {"Test"}));
-
-            _scanBackendMock
-                .SetupGet(x => x.Id)
-                .Returns("dummy");
-
-            _scanHostedService = new ScanHostedService(
-                Mock.Of<ILogger<ScanHostedService>>(),
-                _scanBackendMock.Object,
-                _busMock.Object,
-                configuration);
-        }
-
-        [Test]
-        public async Task TestBusReceiveScanResultMessage()
-        {
-            await _scanHostedService.StartAsync(default);
-
-            _busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(
-                m => m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test")
-            )));
-        }
-
-        [Test]
-        public async Task TestBusIsDisposedOnStop()
-        {
-            await _scanHostedService.StopAsync(default);
-
-            _busMock.Verify(x => x.Dispose(), Times.Once);
-        }
-    }
-}

MalwareMultiScan.Ui/models/scan-result-entry-flattened.ts

@@ -1,19 +1,31 @@
 import ScanResultEntry from '~/models/scan-result-entry';
+import {ScanResultStatus} from '~/models/scan-result-status';
 
 export default class ScanResultEntryFlattened implements ScanResultEntry {
   readonly id: string;
-  readonly completed: boolean;
+  readonly status: ScanResultStatus;
-  readonly succeeded: boolean | null;
   readonly duration: number;
   readonly threats: string[];
 
-  constructor(id: string, completed: boolean, succeeded: boolean | null, duration: number, threats: string[]) {
+  constructor(id: string, status: ScanResultStatus, duration: number, threats: string[]) {
     this.id = id;
-    this.completed = completed;
+    this.status = status;
-    this.succeeded = succeeded;
     this.duration = duration;
     this.threats = threats;
   }
 
+  get completed()
+  {
+    return this.status != ScanResultStatus.Queued
+  }
 
+  get succeeded()
+  {
+    return this.status === ScanResultStatus.Succeeded;
+  }
+
+  get failed()
+  {
+    return this.status === ScanResultStatus.Failed;
+  }
 }

MalwareMultiScan.Ui/models/scan-result-entry.ts

@@ -1,6 +1,7 @@
+import { ScanResultStatus } from '~/models/scan-result-status';
+
 export default interface ScanResultEntry {
-  readonly completed: boolean,
+  readonly status: ScanResultStatus,
-  readonly succeeded: boolean | null,
   readonly duration: number,
   readonly threats: string[]
 }

MalwareMultiScan.Ui/models/scan-result-status.ts

@@ -0,0 +1,6 @@
+export enum ScanResultStatus
+{
+    Queued,
+    Succeeded,
+    Failed
+}

MalwareMultiScan.Ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "malware-multi-scan-ui",
-  "version": "1.0.2",
+  "version": "1.5.0",
   "private": true,
   "scripts": {
     "dev": "nuxt-ts",
@@ -10,17 +10,17 @@
   },
   "dependencies": {
     "@nuxt/typescript-runtime": "^2.0.0",
-    "@nuxtjs/axios": "^5.12.2",
+    "@nuxtjs/axios": "^5.12.3",
     "@nuxtjs/proxy": "^2.0.1",
     "bootstrap": "^4.5.2",
-    "bootstrap-vue": "^2.17.3",
+    "bootstrap-vue": "^2.20.1",
-    "core-js": "^3.6.5",
+    "core-js": "^3.8.1",
-    "nuxt": "^2.14.6"
+    "nuxt": "^2.14.10"
   },
   "devDependencies": {
-    "@nuxt/types": "^2.14.6",
+    "@nuxt/types": "^2.14.10",
     "@nuxt/typescript-build": "^2.0.3",
-    "node-sass": "^4.14.1",
+    "node-sass": "^5.0.0",
-    "sass-loader": "^10.0.4"
+    "sass-loader": "^10.1.0"
   }
 }

MalwareMultiScan.Ui/pages/_id/index.vue

@@ -1,9 +1,9 @@
 <template>
-  <b-table :fields="fields" :items="flattenedData" class="m-0" striped>
+  <b-table :fields="fields" :items="flattenedData" :responsive="true" class="m-0" striped>
     <template #cell(completed)="data">
       <div class="h5 m-0">
-        <b-icon-check-circle-fill v-if="data.item.succeeded === true" variant="success"/>
+        <b-icon-check-circle-fill v-if="data.item.succeeded" variant="success"/>
-        <b-icon-exclamation-circle-fill v-if="data.item.succeeded === false" variant="danger"/>
+        <b-icon-exclamation-circle-fill v-if="data.item.completed && !data.item.succeeded" variant="danger"/>
 
         <b-icon-bug-fill v-if="!data.item.completed" animation="spin-pulse"
                          class="rounded-circle bg-primary text-white" scale="0.7"/>
@@ -17,7 +17,7 @@
 
     <template #cell(threats)="data">
       <div class="text-success" v-if="data.item.succeeded && !data.item.threats.length">No threats have been detected</div>
-      <div class="text-danger" v-if="data.item.succeeded === false">Scan failed to complete due to the error or timeout</div>
+      <div class="text-danger" v-if="data.item.completed && !data.item.succeeded">Scan failed to complete due to the error or timeout</div>
       <div v-if="!data.item.completed">Scan is in progress</div>
 
       <ul v-if="data.item.completed && data.item.threats.length" class="list-inline m-0">
@@ -57,8 +57,7 @@ export default Vue.extend({
     flattenedData(): ScanResultEntryFlattened[] {
       return Object
           .entries((this.data as ScanResult).results)
-          .map(([k, v]) => new ScanResultEntryFlattened(
+          .map(([k, v]) => new ScanResultEntryFlattened(k, v.status, v.duration, v.threats))
-              k, v.completed, v.succeeded, v.duration, v.threats))
     }
   },
 

MalwareMultiScan.Ui/pages/index.vue

@@ -9,6 +9,9 @@
           </b-input-group-append>
         </b-input-group>
 
+        <b-input id="callback-url" v-model="callbackUrl" class="mt-3"
+                 placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
+
         <b-progress v-if="uploading" :value="progress" animated class="mt-3"/>
       </b-tab>
 
@@ -19,6 +22,9 @@
             <b-button :disabled="!isUrl(url)" variant="primary" @click="scanUrl">Scan</b-button>
           </b-input-group-append>
         </b-input-group>
+
+        <b-input id="callback-url" v-model="callbackUrl" class="mt-3"
+                 placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
       </b-tab>
     </b-tabs>
   </b-card>
@@ -37,6 +43,7 @@ export default Vue.extend({
 
       file: null as File | null,
       url: '' as string,
+      callbackUrl: '' as string
     }
   },
 
@@ -53,6 +60,9 @@ export default Vue.extend({
 
       data.append('file', this.file, this.file.name);
 
+      if (this.isUrl(this.callbackUrl))
+        data.append('callbackUrl', this.callbackUrl);
+
       try {
         this.uploading = true;
 
@@ -80,6 +90,9 @@ export default Vue.extend({
 
       data.append('url', this.url);
 
+      if (this.isUrl(this.callbackUrl))
+        data.append('callbackUrl', this.callbackUrl);
+
       const result = await this.$axios.$post<ScanResult>('queue/url', data);
 
       await this.$router.push({name: 'id', params: {id: result.id}});

MalwareMultiScan.sln

@@ -4,17 +4,21 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{8A16A3C4-2AE3-4F63-8280-635FF7878080}"
-EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{A248B5B7-7CBB-4242-98BD-51A9E915E485}"
 ProjectSection(SolutionItems) = preProject
 	.dockerignore = .dockerignore
 	.gitignore = .gitignore
 	docker-compose.yaml = docker-compose.yaml
+	README.md = README.md
+	LICENSE = LICENSE
 EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Tests", "MalwareMultiScan.Tests\MalwareMultiScan.Tests.csproj", "{9896162D-8FC7-4911-933F-A78C94128923}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", "MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj", "{534E3C92-FD6D-401C-99D4-792DB11B57AE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -29,13 +33,17 @@ Global
 		{7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.Build.0 = Release|Any CPU
 		{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.Build.0 = Release|Any CPU
+		{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

README.md

@@ -2,10 +2,12 @@
 
 ![Tests](https://github.com/mindcollapse/MalwareMultiScan/workflows/Tests/badge.svg?event=push) ![API](https://github.com/mindcollapse/MalwareMultiScan/workflows/API/badge.svg) ![UI](https://github.com/mindcollapse/MalwareMultiScan/workflows/UI/badge.svg)  ![Scanners](https://github.com/mindcollapse/MalwareMultiScan/workflows/Scanners/badge.svg?event=push)
 
-Self-hosted [VirusTotal](https://www.virustotal.com/) wannabe API for scanning URLs and files by multiple antivirus solutions.
+Self-hosted [VirusTotal](https://www.virustotal.com/) / [OPSWAT MetaDefender](https://metadefender.opswat.com/) wannabe API for scanning URLs and files by multiple antivirus solutions.
 
 ![MalwareMultiScan UI](.github/img/malware-multi-scan-ui.gif)
 
+**IMPORTANT**: version 1.5 introduces breaking changes in containers configuration and docker-compose.yaml layout. Please see [releases](https://github.com/mindcollapse/MalwareMultiScan/releases) page and changelog of [docker-compose.yaml](https://github.com/mindcollapse/MalwareMultiScan/commits/master/docker-compose.yaml) and [README.md](https://github.com/mindcollapse/MalwareMultiScan/commits/master/README.md) for the additional details. 
+
 ## Introduction
 
 I faced a need to scan user-uploaded files in one of my work projects in an automated mode to ensure they don't contain any malware. Using VirusTotal was not an option because of a) legal restrictions and data residency limitations b) scanning by hash-sums would not be sufficient because the majority of files are generated / modified by users. 
@@ -18,6 +20,12 @@ In the end, it's nothing but the set of Docker containers running the agent. Tha
 
 **IMPORTANT**: MalwareMultiScan is not intended as a publicly-facing API / UI. It has (intentionally) no authorization, authentication, rate-limiting, or logging. Therefore, it should be used only as an internal / private API or behind the restrictive API gateway.
 
+Whole solution can be started with `docker-compose up` executed in a root folder of repository.
+
+It can be also deployed to the Docker Swarm cluster by using the command `docker stack deploy malware-multi-scan --compose-file docker-compose.yaml`.
+
+After the start the Demo Web UI will become available under http://localhost:8888.
+
 See [components](#components) chapter below and the [docker-compose.yaml](docker-compose.yaml) file.
 
 ### Configuration
@@ -26,27 +34,27 @@ Configuration of API and Scanners is performed by passing the environment variab
 
 #### MalwareMultiScan.Api
 
-* `ConnectionStrings__Mongo=mongodb://localhost:27017` - MongoDB connection string.
+* `MONGO_ADDRESS=mongodb://localhost:27017` - MongoDB connection string.
 
-* `DatabaseName=MalwareMultiScan` - MongoDB collection name.
+* `MONGO_DATABASE=MalwareMultiScan` - MongoDB collection name.
 
-* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
+* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
 
-* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
+* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
 
-* `MaxFileSize=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified.
+* `FILE_SIZE_LIMIT=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified. Set to 0 to disable the validation.
-
-* `BackendsConfiguration=backends.yaml` - Path to the [backends.yaml](MalwareMultiScan.Api/backends.yaml) file.
 
 #### MalwareMultiScan.Scanner
 
-* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
+* `BACKEND_ID=dummy` - Id of a backend.
 
-* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
+* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
 
-* `BackendType=Dummy` - A type of scanner backend used by the running instance. Should correspond to the [BackendType](MalwareMultiScan.Backends/Enums/BackendType.cs) enum.
+* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
 
-* `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
+* `MAX_SCANNING_TIME=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
+
+* `WORKER_COUNT=4` - Number of workers for parallel scanning.
 
 #### MalwareMultiScan.Ui
 
@@ -54,11 +62,25 @@ Configuration of API and Scanners is performed by passing the environment variab
 
 ### API Endpoints
 
-* POST `/api/queue/url` with a `url` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
+* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
 
- * POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
+* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
 
-* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error.
+* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `404 Not Found` error.
+
+#### Callback URL
+
+Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResultMessage](MalwareMultiScan.Shared/Message/ScanResultMessage.cs) in a body on every update from scan backends. Query string will contain `id` parameter that corresponds to the id of the scan result and `backend` parameter with the id of backend which completed the scan.
+
+I.e. when you define `callbackUrl=http://localhost:1234/scan-results`, the POST request will be made to `http://localhost:1234/scan-results?id=123&backend=dummy` with a body
+
+```json
+{
+  "Status": 1,
+  "Duration": 5,
+  "Threats": ["Malware.Dummy.Result"]
+}
+```
 
 ## Supported Scan Engines
 
@@ -77,21 +99,41 @@ More scan backends can be added in the future. Some of the popular ones do not h
 
 ## Components
 
+### Workflow
+
+1. On startup all [Scanners](MalwareMultiScan.Scanner) register themselves in [Consul](https://www.consul.io/) with a service name equal to `scanner` and the `BackendId` metadata field equal to the value of `BACKEND_ID` environment variable. They also register a TTL check and listen for [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
+
+2. Third-party client triggers `/api/queue/url` or `/api/queue/file` of the [MalwareMultiScan.Api](MalwareMultiScan.Api).
+
+3. [MalwareMultiScan.Api](MalwareMultiScan.Api) sends a query to [Consul](https://www.consul.io/) and receives the list of alive scan backends with the service name `scanner`.
+
+4. [MalwareMultiScan.Api](MalwareMultiScan.Api) schedules a [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
+
+5. [Scanners](MalwareMultiScan.Scanner) picks up a job from queue, starts the scan and sends result back to the `default` queue of [Hangfire](https://www.hangfire.io/).
+
+6. [MalwareMultiScan.Api](MalwareMultiScan.Api) picks a job from the default` queue of [Hangfire](https://www.hangfire.io/) and updates the state of the scan.
+
+7. If callback URL was specified during the step #2, [MalwareMultiScan.Api](MalwareMultiScan.Api) triggers a HTTP POST request to the specified URL. See [Callback URL](#callback-url) for details.
+
 ### Prerequisites
 
-* **MongoDB** of version 3.x. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
+* **MongoDB** of version 3.x or above. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
 
-* **RabbitMQ** of version 3.x. Used for IPC and scan tasks queueing. The communication is happening through the [EasyNetQ](https://github.com/EasyNetQ/EasyNetQ) library.
+* **Redis** of version 5.x or above. Used for tasks queueing. The communication is happening through the [Hangfire](https://www.hangfire.io/) library.
+
+* **Consul** of version 1.8.x or above. Used for service registration of scan backends. 
 
 * **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments.
 
-* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities. A simultaneous scan by a single node is not supported, yet load-balancing is possible by the built-in orchestrators' round-robin routing.
+* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities.
 
 ### Parts
 
-* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile). Configuration of available backends is performed via the [backends.yaml](MalwareMultiScan.Api/backends.yaml) location passed via the `BackendsConfiguration` environment variable. 
+* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile).
 
-* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Shared components between API and Worker. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
+* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Scan backends logic. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
+
+* [MalwareMultiScan.Shared](MalwareMultiScan.Shared) - Shared components.
 
 * [MalwareMultiScan.Scanner](MalwareMultiScan.Scanner) - .NET Core Worker service subscribes to messages corresponding to the backend id, then fires up scanning command-line utility, and parses the output. See [Dockerfile](MalwareMultiScan.Scanner/Dockerfile). The image of MalwareMultiScan.Scanner acts as a base image for the rest of the scan backends. Check Dockerfiles from the [table above](#supported-scan-engines) for details.
 

docker-compose.yaml

@@ -1,31 +1,30 @@
-version: "3.8"
+version: "3.3"
 
 services:
-  rabbitmq:
+  redis:
-    image: rabbitmq:3
+    image: redis:6
     restart: on-failure
-    expose:
-      - "5672"
     volumes:
-      - rabbitmq_etc/:/etc/rabbitmq/
+    - redis:/data
-      - rabbitmq_data:/var/lib/rabbitmq/
+   
-      - rabbitmq_logs/:/var/log/rabbitmq/
-    
   mongodb:
     image: mongo:4
     restart: on-failure
-    expose:
-      - "27019"
     volumes:
       - mongodb:/data
+      
+  # See https://github.com/hashicorp/consul/blob/master/demo/docker-compose-cluster/docker-compose.yml
+  # for a production-ready config.
+  consul:
+    image: consul:1.8
+    restart: on-failure
+    command: consul agent -dev -log-level=info -client=0.0.0.0
         
   ui:
     image: mindcollapse/malware-multi-scan-ui
     restart: on-failure
     ports:
       - "8888:8888" 
-    expose:
-      - "8888"
     depends_on:
       - api
     environment:
@@ -39,17 +38,14 @@ services:
   api:
     image: mindcollapse/malware-multi-scan-api
     restart: on-failure
-    expose:
-      - "5000"
     depends_on:
-      - rabbitmq
+      - consul
+      - redis
       - mongodb
     environment:
-      - "ConnectionStrings__RabbitMQ=host=rabbitmq;timeout=120"
+      - "REDIS_ADDRESS=redis:6379"
-      - "ConnectionStrings__Mongo=mongodb://mongodb:27017?connectTimeoutMS=120000"
+      - "CONSUL_ADDRESS=http://consul:8500"
-      - "BackendsConfiguration=/etc/backends.yaml"
+      - "MONGO_ADDRESS=mongodb://mongodb:27017?connectTimeoutMS=120000"
-    volumes:
-      - "./MalwareMultiScan.Api/backends.yaml:/etc/backends.yaml:ro"
     build:
       context: .
       dockerfile: MalwareMultiScan.Api/Dockerfile
@@ -58,9 +54,11 @@ services:
     image: mindcollapse/malware-multi-scan-scanner
     restart: on-failure
     depends_on:
-      - rabbitmq
+      - consul
+      - redis
     environment:
-      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+      - "REDIS_ADDRESS=redis:6379"
+      - "CONSUL_ADDRESS=http://consul:8500"
     build:
       context: .
       dockerfile: MalwareMultiScan.Scanner/Dockerfile
@@ -71,7 +69,8 @@ services:
     depends_on:
       - dummy-scanner
     environment:
-      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+      - "REDIS_ADDRESS=redis:6379"
+      - "CONSUL_ADDRESS=http://consul:8500"
     build:
       context: MalwareMultiScan.Backends/Dockerfiles
       dockerfile: Clamav.Dockerfile
@@ -82,7 +81,8 @@ services:
     depends_on:
       - dummy-scanner
     environment:
-      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+      - "REDIS_ADDRESS=redis:6379"
+      - "CONSUL_ADDRESS=http://consul:8500"
     build:
       context: MalwareMultiScan.Backends/Dockerfiles
       dockerfile: WindowsDefender.Dockerfile
@@ -95,7 +95,8 @@ services:
 #    depends_on:
 #      - dummy-scanner
 #    environment:
-#      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+#      - "REDIS_ADDRESS=redis:6379"
+#      - "CONSUL_ADDRESS=http://consul:8500"
 #    build:
 #      context: MalwareMultiScan.Backends/Dockerfiles
 #      dockerfile: Comodo.Dockerfile
@@ -106,7 +107,8 @@ services:
 #    depends_on:
 #      - dummy-scanner
 #    environment:
-#      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+#      - "REDIS_ADDRESS=redis:6379"
+#      - "CONSUL_ADDRESS=http://consul:8500"
 #    build:
 #      context: MalwareMultiScan.Backends/Dockerfiles
 #      dockerfile: DrWeb.Dockerfile
@@ -117,7 +119,8 @@ services:
 #    depends_on:
 #      - dummy-scanner
 #    environment:
-#      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+#      - "REDIS_ADDRESS=redis:6379"
+#      - "CONSUL_ADDRESS=http://consul:8500"
 #    build:
 #      context: MalwareMultiScan.Backends/Dockerfiles
 #      dockerfile: KES.Dockerfile
@@ -128,7 +131,8 @@ services:
 #    depends_on:
 #      - dummy-scanner
 #    environment:
-#      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+#      - "REDIS_ADDRESS=redis:6379"
+#      - "CONSUL_ADDRESS=http://consul:8500"
 #    build:
 #      context: MalwareMultiScan.Backends/Dockerfiles
 #      dockerfile: McAfee.Dockerfile
@@ -139,13 +143,12 @@ services:
 #    depends_on:
 #      - dummy-scanner
 #    environment:
-#      - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
+#      - "REDIS_ADDRESS=redis:6379"
+#      - "CONSUL_ADDRESS=http://consul:8500"
 #    build:
 #      context: MalwareMultiScan.Backends/Dockerfiles
 #      dockerfile: Sophos.Dockerfile
 
 volumes:
   mongodb:
-  rabbitmq_etc:
+  redis:
-  rabbitmq_data:
-  rabbitmq_logs: