Merge pull request #20 from mindcollapse/v1.1

Multiple improvements for version 1.1

MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs

@@ -8,9 +8,23 @@ namespace MalwareMultiScan.Api.Attributes
     /// </summary>
     public class IsHttpUrlAttribute : ValidationAttribute
     {
+        private readonly bool _failOnNull;
+
+        /// <summary>
+        /// Initialize http URL validation attribute.
+        /// </summary>
+        /// <param name="failOnNull">True if URL is mandatory, false otherwise.</param>
+        public IsHttpUrlAttribute(bool failOnNull = true)
+        {
+            _failOnNull = failOnNull;
+        }
+
         /// <inheritdoc />
         protected override ValidationResult IsValid(object value, ValidationContext validationContext)
         {
+            if (!_failOnNull && string.IsNullOrEmpty(value.ToString()))
+                return ValidationResult.Success;
+
             if (!Uri.TryCreate((string) value, UriKind.Absolute, out var uri)
                 || !uri.IsAbsoluteUri
                 || uri.Scheme.ToLowerInvariant() != "http"

MalwareMultiScan.Api/Controllers/QueueController.cs

@@ -1,3 +1,4 @@
+using System;
 using System.ComponentModel.DataAnnotations;
 using System.Threading.Tasks;
 using MalwareMultiScan.Api.Attributes;
@@ -31,13 +32,16 @@ namespace MalwareMultiScan.Api.Controllers
         /// Queue file for scanning.
         /// </summary>
         /// <param name="file">File from form data.</param>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         [HttpPost("file")]
         [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
         [ProducesResponseType(StatusCodes.Status400BadRequest)]
         public async Task<IActionResult> ScanFile(
-            [Required] [MaxFileSize] IFormFile file)
+            [Required] [MaxFileSize] IFormFile file,
+            [FromForm] [IsHttpUrl(false)] string callbackUrl = null)
         {
-            var result = await _scanResultService.CreateScanResult();
+            var result = await _scanResultService.CreateScanResult(
+                string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
 
             string storedFileId;
 
@@ -56,13 +60,16 @@ namespace MalwareMultiScan.Api.Controllers
         /// Queue URL for scanning.
         /// </summary>
         /// <param name="url">URL from form data.</param>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         [HttpPost("url")]
         [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
         [ProducesResponseType(StatusCodes.Status400BadRequest)]
         public async Task<IActionResult> ScanUrl(
-            [FromForm] [Required] [IsHttpUrl] string url)
+            [FromForm] [Required] [IsHttpUrl] string url,
+            [FromForm] [IsHttpUrl(false)] string callbackUrl = null)
         {
-            var result = await _scanResultService.CreateScanResult();
+            var result = await _scanResultService.CreateScanResult(
+                string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
 
             await _scanResultService.QueueUrlScan(result, url);
 

MalwareMultiScan.Api/Controllers/ScanResultsController.cs

@@ -15,7 +15,7 @@ namespace MalwareMultiScan.Api.Controllers
     public class ScanResultsController : Controller
     {
         private readonly IScanResultService _scanResultService;
-        
+
         /// <summary>
         /// Initialize scan results controller.
         /// </summary>

MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs

@@ -9,7 +9,7 @@ namespace MalwareMultiScan.Api.Data.Configuration
         /// Backend id.
         /// </summary>
         public string Id { get; set; }
-        
+
         /// <summary>
         /// Backend state.
         /// </summary>

MalwareMultiScan.Api/Data/Models/ScanResult.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
@@ -17,7 +18,12 @@ namespace MalwareMultiScan.Api.Data.Models
         public string Id { get; set; }
 
         /// <summary>
-        /// Result entries where key is backend id and value is <see cref="ScanResultEntry"/>.
+        /// Optional callback URL.
+        /// </summary>
+        public Uri CallbackUrl { get; set; }
+
+        /// <summary>
+        /// Result entries where key is backend id and value is <see cref="ScanResultEntry" />.
         /// </summary>
         public Dictionary<string, ScanResultEntry> Results { get; set; } =
             new Dictionary<string, ScanResultEntry>();

MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs

@@ -9,17 +9,17 @@ namespace MalwareMultiScan.Api.Data.Models
         /// Completion status.
         /// </summary>
         public bool Completed { get; set; }
-        
+
         /// <summary>
         /// Indicates that scanning completed without error.
         /// </summary>
         public bool? Succeeded { get; set; }
-        
+
         /// <summary>
         /// Scanning duration in seconds.
         /// </summary>
         public long Duration { get; set; }
-        
+
         /// <summary>
         /// Detected names of threats.
         /// </summary>

MalwareMultiScan.Api/Services/Implementations/ReceiverHostedService.cs

@@ -1,3 +1,6 @@
+using System;
+using System.Net.Http;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using EasyNetQ;
@@ -5,6 +8,7 @@ using MalwareMultiScan.Api.Services.Interfaces;
 using MalwareMultiScan.Backends.Messages;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 
 namespace MalwareMultiScan.Api.Services.Implementations
 {
@@ -13,6 +17,7 @@ namespace MalwareMultiScan.Api.Services.Implementations
     {
         private readonly IBus _bus;
         private readonly IConfiguration _configuration;
+        private readonly IHttpClientFactory _httpClientFactory;
         private readonly ILogger<ReceiverHostedService> _logger;
         private readonly IScanResultService _scanResultService;
 
@@ -23,31 +28,23 @@ namespace MalwareMultiScan.Api.Services.Implementations
         /// <param name="configuration">Configuration.</param>
         /// <param name="scanResultService">Scan result service.</param>
         /// <param name="logger">Logger.</param>
+        /// <param name="httpClientFactory">HTTP client factory.</param>
         public ReceiverHostedService(IBus bus, IConfiguration configuration, IScanResultService scanResultService,
-            ILogger<ReceiverHostedService> logger)
+            ILogger<ReceiverHostedService> logger, IHttpClientFactory httpClientFactory)
         {
             _bus = bus;
             _configuration = configuration;
             _scanResultService = scanResultService;
             _logger = logger;
+            _httpClientFactory = httpClientFactory;
         }
 
 
         /// <inheritdoc />
         public Task StartAsync(CancellationToken cancellationToken)
         {
-            _bus.Receive<ScanResultMessage>(_configuration.GetValue<string>("ResultsSubscriptionId"), async message =>
-            {
-                message.Threats ??= new string[] { };
-
-                _logger.LogInformation(
-                    $"Received a result from {message.Backend} for {message.Id} " +
-                    $"with threats {string.Join(",", message.Threats)}");
-
-                await _scanResultService.UpdateScanResultForBackend(
-                    message.Id, message.Backend, message.Duration, true,
-                    message.Succeeded, message.Threats);
-            });
+            _bus.Receive<ScanResultMessage>(
+                _configuration.GetValue<string>("ResultsSubscriptionId"), StoreScanResult);
 
             _logger.LogInformation(
                 "Started hosted service for receiving scan results");
@@ -65,5 +62,42 @@ namespace MalwareMultiScan.Api.Services.Implementations
 
             return Task.CompletedTask;
         }
+
+        private async Task StoreScanResult(ScanResultMessage message)
+        {
+            message.Threats ??= new string[] { };
+
+            _logger.LogInformation(
+                $"Received a result from {message.Backend} for {message.Id} " +
+                $"with threats {string.Join(",", message.Threats)}");
+
+            await _scanResultService.UpdateScanResultForBackend(
+                message.Id, message.Backend, message.Duration, true,
+                message.Succeeded, message.Threats);
+
+            var result = await _scanResultService.GetScanResult(message.Id);
+
+            if (result?.CallbackUrl == null)
+                return;
+
+            var cancellationTokenSource = new CancellationTokenSource(
+                TimeSpan.FromSeconds(3));
+
+            using var httpClient = _httpClientFactory.CreateClient();
+
+            try
+            {
+                var response = await httpClient.PostAsync(
+                    result.CallbackUrl,
+                    new StringContent(JsonConvert.SerializeObject(result), Encoding.UTF8, "application/json"),
+                    cancellationTokenSource.Token);
+
+                response.EnsureSuccessStatusCode();
+            }
+            catch (Exception exception)
+            {
+                _logger.LogError(exception, $"Failed to POST to callback URL {result.CallbackUrl}");
+            }
+        }
     }
 }

MalwareMultiScan.Api/Services/Implementations/ScanResultService.cs

@@ -1,3 +1,4 @@
+using System;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -31,12 +32,14 @@ namespace MalwareMultiScan.Api.Services.Implementations
 
             _collection = db.GetCollection<ScanResult>(CollectionName);
         }
-        
+
         /// <inheritdoc />
-        public async Task<ScanResult> CreateScanResult()
+        public async Task<ScanResult> CreateScanResult(Uri callbackUrl)
         {
             var scanResult = new ScanResult
             {
+                CallbackUrl = callbackUrl,
+
                 Results = _scanBackendService.List
                     .Where(b => b.Enabled)
                     .ToDictionary(k => k.Id, v => new ScanResultEntry())

MalwareMultiScan.Api/Services/Interfaces/IScanResultService.cs

@@ -1,3 +1,4 @@
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using MalwareMultiScan.Api.Data.Models;
@@ -12,8 +13,9 @@ namespace MalwareMultiScan.Api.Services.Interfaces
         /// <summary>
         /// Create scan result.
         /// </summary>
+        /// <param name="callbackUrl">Optional callback URL.</param>
         /// <returns>Scan result entry with id.</returns>
-        Task<ScanResult> CreateScanResult();
+        Task<ScanResult> CreateScanResult(Uri callbackUrl);
 
         /// <summary>
         /// Get scan result.

MalwareMultiScan.Api/Startup.cs

@@ -38,6 +38,8 @@ namespace MalwareMultiScan.Api
             services.AddControllers();
 
             services.AddHostedService<ReceiverHostedService>();
+
+            services.AddHttpClient();
         }
 
         public void Configure(IApplicationBuilder app)

MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs

@@ -22,17 +22,17 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
         /// Regex to extract names of threats.
         /// </summary>
         protected abstract Regex MatchRegex { get; }
-        
+
         /// <summary>
         /// Path to the backend.
         /// </summary>
         protected abstract string BackendPath { get; }
-        
+
         /// <summary>
         /// Parse StdErr instead of StdOut.
         /// </summary>
         protected virtual bool ParseStdErr { get; } = false;
-        
+
         /// <summary>
         /// Throw on non-zero exit code.
         /// </summary>
@@ -44,7 +44,7 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
         /// <param name="path">Path to the temporary file.</param>
         /// <returns>Formatted string with parameters and path.</returns>
         protected abstract string GetBackendArguments(string path);
-        
+
         /// <inheritdoc />
         public override Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
         {

MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs

@@ -11,7 +11,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public WindowsDefenderScanBackend(IProcessRunner processRunner) : base(processRunner)
         {
         }
-        
+
         /// <inheritdoc />
         public override string Id { get; } = "windows-defender";
 
@@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         protected override Regex MatchRegex { get; } =
             new Regex(@"EngineScanCallback\(\): Threat (?<threat>[\S]+) identified",
                 RegexOptions.Compiled | RegexOptions.Multiline);
-        
+
         /// <inheritdoc />
         protected override bool ParseStdErr { get; } = true;
 

MalwareMultiScan.Backends/Enums/BackendType.cs

@@ -9,37 +9,37 @@ namespace MalwareMultiScan.Backends.Enums
         /// Dummy
         /// </summary>
         Dummy,
-        
+
         /// <summary>
         /// Windows Defender.
         /// </summary>
         Defender,
-        
+
         /// <summary>
         /// ClamAV.
         /// </summary>
         Clamav,
-        
+
         /// <summary>
         /// DrWeb.
         /// </summary>
         DrWeb,
-        
+
         /// <summary>
         /// KES.
         /// </summary>
         Kes,
-        
+
         /// <summary>
         /// Comodo.
         /// </summary>
         Comodo,
-        
+
         /// <summary>
         /// Sophos.
         /// </summary>
         Sophos,
-        
+
         /// <summary>
         /// McAfee.
         /// </summary>

MalwareMultiScan.Backends/Interfaces/IScanBackend.cs

@@ -14,7 +14,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// Unique backend id.
         /// </summary>
         public string Id { get; }
-        
+
         /// <summary>
         /// Scan file.
         /// </summary>
@@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>List of detected threats.</returns>
         public Task<string[]> ScanAsync(string path, CancellationToken cancellationToken);
-        
+
         /// <summary>
         /// Scan URL.
         /// </summary>
@@ -30,7 +30,7 @@ namespace MalwareMultiScan.Backends.Interfaces
         /// <param name="cancellationToken">Cancellation token.</param>
         /// <returns>List of detected threats.</returns>
         public Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken);
-        
+
         /// <summary>
         /// Scan stream.
         /// </summary>

MalwareMultiScan.Backends/Messages/ScanResultMessage.cs

@@ -9,7 +9,7 @@ namespace MalwareMultiScan.Backends.Messages
         /// Result id.
         /// </summary>
         public string Id { get; set; }
-        
+
         /// <summary>
         /// Backend.
         /// </summary>
@@ -19,7 +19,7 @@ namespace MalwareMultiScan.Backends.Messages
         /// Status.
         /// </summary>
         public bool Succeeded { get; set; }
-        
+
         /// <summary>
         /// List of detected threats.
         /// </summary>

MalwareMultiScan.Backends/Services/Implementations/ProcessRunner.cs

@@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging;
 namespace MalwareMultiScan.Backends.Services.Implementations
 {
     /// <summary>
-    /// 
     /// </summary>
     [ExcludeFromCodeCoverage]
     public class ProcessRunner : IProcessRunner
@@ -26,7 +25,6 @@ namespace MalwareMultiScan.Backends.Services.Implementations
         }
 
         /// <summary>
-        /// 
         /// </summary>
         /// <param name="path"></param>
         /// <param name="arguments"></param>

MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj

@@ -10,6 +10,8 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
+        <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="3.1.9" />
     </ItemGroup>
 

MalwareMultiScan.Scanner/Program.cs

@@ -1,7 +1,10 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
+using Hangfire;
+using Hangfire.MemoryStorage;
 using MalwareMultiScan.Backends.Extensions;
 using MalwareMultiScan.Scanner.Services.Implementations;
+using MalwareMultiScan.Scanner.Services.Interfaces;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -26,7 +29,16 @@ namespace MalwareMultiScan.Scanner
                     services.AddRabbitMq(context.Configuration);
                     services.AddScanningBackend(context.Configuration);
 
+                    services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
                     services.AddHostedService<ScanHostedService>();
+
+                    services.AddHangfire(
+                        configuration => configuration.UseMemoryStorage());
+
+                    services.AddHangfireServer(options =>
+                    {
+                        options.WorkerCount = context.Configuration.GetValue<int>("WorkerCount");
+                    });
                 }).RunConsoleAsync();
         }
     }

MalwareMultiScan.Scanner/Services/Implementations/ScanBackgroundJob.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using EasyNetQ;
+using MalwareMultiScan.Backends.Interfaces;
+using MalwareMultiScan.Backends.Messages;
+using MalwareMultiScan.Scanner.Services.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+
+namespace MalwareMultiScan.Scanner.Services.Implementations
+{
+    /// <inheritdoc />
+    public class ScanBackgroundJob : IScanBackgroundJob
+    {
+        private readonly IScanBackend _backend;
+        private readonly IBus _bus;
+        private readonly IConfiguration _configuration;
+        private readonly ILogger<ScanBackgroundJob> _logger;
+
+        /// <summary>
+        /// Initialize scan background job.
+        /// </summary>
+        /// <param name="configuration">Configuration.</param>
+        /// <param name="bus">Bus.</param>
+        /// <param name="logger">Logger.</param>
+        /// <param name="backend">Scan backend.</param>
+        public ScanBackgroundJob(IConfiguration configuration, IBus bus,
+            ILogger<ScanBackgroundJob> logger, IScanBackend backend)
+        {
+            _configuration = configuration;
+            _bus = bus;
+            _logger = logger;
+            _backend = backend;
+        }
+
+        /// <inheritdoc />
+        public async Task Process(ScanRequestMessage message)
+        {
+            _logger.LogInformation(
+                $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
+
+            var cancellationTokenSource = new CancellationTokenSource(
+                TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
+
+            var result = new ScanResultMessage
+            {
+                Id = message.Id,
+                Backend = _backend.Id
+            };
+
+            var stopwatch = new Stopwatch();
+
+            stopwatch.Start();
+
+            try
+            {
+                result.Threats = await _backend.ScanAsync(
+                    message.Uri, cancellationTokenSource.Token);
+
+                result.Succeeded = true;
+
+                _logger.LogInformation(
+                    $"Backend {_backend.Id} completed a scan of {message.Id} " +
+                    $"with result '{string.Join(", ", result.Threats)}'");
+            }
+            catch (Exception exception)
+            {
+                result.Succeeded = false;
+
+                _logger.LogError(
+                    exception, "Scanning failed with exception");
+            }
+            finally
+            {
+                stopwatch.Stop();
+            }
+
+            result.Duration = stopwatch.ElapsedMilliseconds / 1000;
+
+            _logger.LogInformation(
+                $"Sending scan results with status {result.Succeeded}");
+
+            await _bus.SendAsync(
+                _configuration.GetValue<string>("ResultsSubscriptionId"), result);
+        }
+    }
+}

MalwareMultiScan.Scanner/Services/Implementations/ScanHostedService.cs

@@ -1,12 +1,10 @@
-using System;
-using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using EasyNetQ;
+using Hangfire;
 using MalwareMultiScan.Backends.Interfaces;
 using MalwareMultiScan.Backends.Messages;
 using MalwareMultiScan.Scanner.Services.Interfaces;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 
 namespace MalwareMultiScan.Scanner.Services.Implementations
@@ -16,7 +14,6 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
     {
         private readonly IScanBackend _backend;
         private readonly IBus _bus;
-        private readonly IConfiguration _configuration;
         private readonly ILogger<ScanHostedService> _logger;
 
         /// <summary>
@@ -25,22 +22,20 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
         /// <param name="logger">Logger.</param>
         /// <param name="backend">Scan backend.</param>
         /// <param name="bus">EasyNetQ bus.</param>
-        /// <param name="configuration">Configuration.</param>
         public ScanHostedService(
             ILogger<ScanHostedService> logger,
-            IScanBackend backend, IBus bus,
-            IConfiguration configuration)
+            IScanBackend backend, IBus bus)
         {
             _logger = logger;
             _bus = bus;
-            _configuration = configuration;
             _backend = backend;
         }
 
         /// <inheritdoc />
         public Task StartAsync(CancellationToken cancellationToken)
         {
-            _bus.Receive<ScanRequestMessage>(_backend.Id, Scan);
+            _bus.Receive<ScanRequestMessage>(_backend.Id, message =>
+                BackgroundJob.Enqueue<IScanBackgroundJob>(j => j.Process(message)));
 
             _logger.LogInformation(
                 $"Started scan hosting service for the backend {_backend.Id}");
@@ -58,55 +53,5 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
 
             return Task.CompletedTask;
         }
-
-        private async Task Scan(ScanRequestMessage message)
-        {
-            _logger.LogInformation(
-                $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
-
-            var cancellationTokenSource = new CancellationTokenSource(
-                TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
-
-            var result = new ScanResultMessage
-            {
-                Id = message.Id,
-                Backend = _backend.Id
-            };
-
-            var stopwatch = new Stopwatch();
-
-            stopwatch.Start();
-
-            try
-            {
-                result.Threats = await _backend.ScanAsync(
-                    message.Uri, cancellationTokenSource.Token);
-
-                result.Succeeded = true;
-
-                _logger.LogInformation(
-                    $"Backend {_backend.Id} completed a scan of {message.Id} " +
-                    $"with result '{string.Join(", ", result.Threats)}'");
-            }
-            catch (Exception exception)
-            {
-                result.Succeeded = false;
-
-                _logger.LogError(
-                    exception, "Scanning failed with exception");
-            }
-            finally
-            {
-                stopwatch.Stop();
-            }
-
-            result.Duration = stopwatch.ElapsedMilliseconds / 1000;
-
-            _logger.LogInformation(
-                $"Sending scan results with status {result.Succeeded}");
-
-            await _bus.SendAsync(
-                _configuration.GetValue<string>("ResultsSubscriptionId"), result);
-        }
     }
 }

MalwareMultiScan.Scanner/Services/Interfaces/IScanBackgroundJob.cs

@@ -0,0 +1,17 @@
+using System.Threading.Tasks;
+using MalwareMultiScan.Backends.Messages;
+
+namespace MalwareMultiScan.Scanner.Services.Interfaces
+{
+    /// <summary>
+    /// Background scanning job.
+    /// </summary>
+    public interface IScanBackgroundJob
+    {
+        /// <summary>
+        /// Process scan request in the background worker.
+        /// </summary>
+        /// <param name="requestMessage">Request message.</param>
+        Task Process(ScanRequestMessage requestMessage);
+    }
+}

MalwareMultiScan.Scanner/appsettings.json

@@ -11,6 +11,7 @@
   
   "MaxScanningTime": 60,
   "ResultsSubscriptionId": "mms.results",
+  "WorkerCount": 4,
   
   "ConnectionStrings": {
     "RabbitMQ": "host=localhost;prefetchcount=1"

MalwareMultiScan.Tests/Api/ControllersTests.cs

@@ -46,7 +46,7 @@ namespace MalwareMultiScan.Tests.Api
                 .Returns(Task.FromResult((Stream) null));
 
             _scanResultServiceMock
-                .Setup(x => x.CreateScanResult())
+                .Setup(x => x.CreateScanResult(null))
                 .Returns(Task.FromResult(new ScanResult()));
 
             _downloadController = new DownloadController(_scanResultServiceMock.Object);

MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Http;
 using System.Threading.Tasks;
 using EasyNetQ;
 using MalwareMultiScan.Api.Services.Implementations;
@@ -50,11 +51,14 @@ namespace MalwareMultiScan.Tests.Api
                 ["ResultsSubscriptionId"] = "mms.results"
             };
 
+            var httpClientFactoryMock = new Mock<IHttpClientFactory>();
+
             _receiverHostedService = new ReceiverHostedService(
                 _busMock.Object,
                 configuration,
                 _scanResultServiceMock.Object,
-                Mock.Of<ILogger<ReceiverHostedService>>());
+                Mock.Of<ILogger<ReceiverHostedService>>(),
+                httpClientFactoryMock.Object);
         }
 
         [Test]

MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs

@@ -82,7 +82,7 @@ namespace MalwareMultiScan.Tests.Api
         [Test]
         public async Task TestScanResultCreateGet()
         {
-            var result = await _resultService.CreateScanResult();
+            var result = await _resultService.CreateScanResult(null);
 
             Assert.NotNull(result.Id);
             Assert.That(result.Results, Contains.Key("dummy"));
@@ -107,7 +107,7 @@ namespace MalwareMultiScan.Tests.Api
         [Test]
         public async Task TestQueueUrlScan()
         {
-            var result = await _resultService.CreateScanResult();
+            var result = await _resultService.CreateScanResult(null);
 
             await _resultService.QueueUrlScan(
                 result, "http://url.com");

MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using EasyNetQ;
+using MalwareMultiScan.Backends.Interfaces;
+using MalwareMultiScan.Backends.Messages;
+using MalwareMultiScan.Scanner.Services.Implementations;
+using MalwareMultiScan.Scanner.Services.Interfaces;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Configuration.Memory;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NUnit.Framework;
+
+namespace MalwareMultiScan.Tests.Scanner
+{
+    public class ScanBackgroundJobTests
+    {
+        private Mock<IBus> _busMock;
+        private Mock<IScanBackend> _scanBackendMock;
+        private IScanBackgroundJob _scanBackgroundJob;
+
+        [SetUp]
+        public void SetUp()
+        {
+            var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
+            {
+                new MemoryConfigurationProvider(new MemoryConfigurationSource())
+            })
+            {
+                ["ResultsSubscriptionId"] = "mms.results"
+            };
+
+            _busMock = new Mock<IBus>();
+
+            _scanBackendMock = new Mock<IScanBackend>();
+
+            _scanBackendMock
+                .SetupGet(x => x.Id)
+                .Returns("dummy");
+
+            _scanBackendMock
+                .Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
+                .Returns(Task.FromResult(new[] {"Test"}));
+
+            _scanBackgroundJob = new ScanBackgroundJob(configuration, _busMock.Object,
+                Mock.Of<ILogger<ScanBackgroundJob>>(), _scanBackendMock.Object);
+        }
+
+        [Test]
+        public async Task TestMessageProcessing()
+        {
+            await _scanBackgroundJob.Process(new ScanRequestMessage
+            {
+                Id = "test",
+                Uri = new Uri("http://test.com")
+            });
+
+            _scanBackendMock.Verify(
+                x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()));
+
+            _busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(m =>
+                m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test"))));
+        }
+    }
+}

MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs

@@ -1,15 +1,10 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
 using System.Threading.Tasks;
 using EasyNetQ;
 using MalwareMultiScan.Backends.Interfaces;
 using MalwareMultiScan.Backends.Messages;
 using MalwareMultiScan.Scanner.Services.Implementations;
 using MalwareMultiScan.Scanner.Services.Interfaces;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Configuration.Memory;
 using Microsoft.Extensions.Logging;
 using Moq;
 using NUnit.Framework;
@@ -25,35 +20,10 @@ namespace MalwareMultiScan.Tests.Scanner
         [SetUp]
         public void SetUp()
         {
-            var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
-            {
-                new MemoryConfigurationProvider(new MemoryConfigurationSource())
-            })
-            {
-                ["ResultsSubscriptionId"] = "mms.results"
-            };
-
             _busMock = new Mock<IBus>();
 
-            _busMock
-                .Setup(x => x.Receive("dummy", It.IsAny<Func<ScanRequestMessage, Task>>()))
-                .Callback<string, Func<ScanRequestMessage, Task>>((s, func) =>
-                {
-                    var task = func.Invoke(new ScanRequestMessage
-                    {
-                        Id = "test",
-                        Uri = new Uri("http://test.com")
-                    });
-
-                    task.Wait();
-                });
-
             _scanBackendMock = new Mock<IScanBackend>();
 
-            _scanBackendMock
-                .Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
-                .Returns(Task.FromResult(new[] {"Test"}));
-
             _scanBackendMock
                 .SetupGet(x => x.Id)
                 .Returns("dummy");
@@ -61,8 +31,7 @@ namespace MalwareMultiScan.Tests.Scanner
             _scanHostedService = new ScanHostedService(
                 Mock.Of<ILogger<ScanHostedService>>(),
                 _scanBackendMock.Object,
-                _busMock.Object,
-                configuration);
+                _busMock.Object);
         }
 
         [Test]
@@ -70,9 +39,8 @@ namespace MalwareMultiScan.Tests.Scanner
         {
             await _scanHostedService.StartAsync(default);
 
-            _busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(
-                m => m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test")
-            )));
+            _busMock.Verify(
+                x => x.Receive("dummy", It.IsAny<Action<ScanRequestMessage>>()));
         }
 
         [Test]

MalwareMultiScan.Ui/pages/_id/index.vue

@@ -1,5 +1,5 @@
 <template>
-  <b-table :fields="fields" :items="flattenedData" class="m-0" striped>
+  <b-table :fields="fields" :items="flattenedData" responsive class="m-0" striped>
     <template #cell(completed)="data">
       <div class="h5 m-0">
         <b-icon-check-circle-fill v-if="data.item.succeeded === true" variant="success"/>

MalwareMultiScan.Ui/pages/index.vue

@@ -9,6 +9,9 @@
           </b-input-group-append>
         </b-input-group>
 
+        <b-input id="callback-url" v-model="callbackUrl" class="mt-3"
+                 placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
+
         <b-progress v-if="uploading" :value="progress" animated class="mt-3"/>
       </b-tab>
 
@@ -19,6 +22,9 @@
             <b-button :disabled="!isUrl(url)" variant="primary" @click="scanUrl">Scan</b-button>
           </b-input-group-append>
         </b-input-group>
+
+        <b-input id="callback-url" v-model="callbackUrl" class="mt-3"
+                 placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
       </b-tab>
     </b-tabs>
   </b-card>
@@ -37,6 +43,7 @@ export default Vue.extend({
 
       file: null as File | null,
       url: '' as string,
+      callbackUrl: '' as string
     }
   },
 
@@ -53,6 +60,9 @@ export default Vue.extend({
 
       data.append('file', this.file, this.file.name);
 
+      if (this.isUrl(this.callbackUrl))
+        data.append('callbackUrl', this.callbackUrl);
+
       try {
         this.uploading = true;
 
@@ -80,6 +90,9 @@ export default Vue.extend({
 
       data.append('url', this.url);
 
+      if (this.isUrl(this.callbackUrl))
+        data.append('callbackUrl', this.callbackUrl);
+
       const result = await this.$axios.$post<ScanResult>('queue/url', data);
 
       await this.$router.push({name: 'id', params: {id: result.id}});

MalwareMultiScan.sln

@@ -11,6 +11,8 @@ ProjectSection(SolutionItems) = preProject
 	.dockerignore = .dockerignore
 	.gitignore = .gitignore
 	docker-compose.yaml = docker-compose.yaml
+	README.md = README.md
+	LICENSE = LICENSE
 EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Tests", "MalwareMultiScan.Tests\MalwareMultiScan.Tests.csproj", "{9896162D-8FC7-4911-933F-A78C94128923}"

README.md

@@ -6,6 +6,10 @@ Self-hosted [VirusTotal](https://www.virustotal.com/) wannabe API for scanning U
 
 ![MalwareMultiScan UI](.github/img/malware-multi-scan-ui.gif)
 
+**[Check out the demo UI](http://199.247.24.56:8888/) with ClamAV, Dummy, and Windows Defender scan backends**. 
+
+The demo is running on a cheap Vultr node, so it might get slow or unavailable occasionally. 
+
 ## Introduction
 
 I faced a need to scan user-uploaded files in one of my work projects in an automated mode to ensure they don't contain any malware. Using VirusTotal was not an option because of a) legal restrictions and data residency limitations b) scanning by hash-sums would not be sufficient because the majority of files are generated / modified by users. 
@@ -18,6 +22,12 @@ In the end, it's nothing but the set of Docker containers running the agent. Tha
 
 **IMPORTANT**: MalwareMultiScan is not intended as a publicly-facing API / UI. It has (intentionally) no authorization, authentication, rate-limiting, or logging. Therefore, it should be used only as an internal / private API or behind the restrictive API gateway.
 
+Whole solution can be started with `docker-compose up` executed in a root folder of repository.
+
+It can be also deployed to the Docker Swarm cluster by using the command `docker stack deploy malware-multi-scan --compose-file docker-compose.yaml`.
+
+After the start the Demo Web UI will become available under http://localhost:8888.
+
 See [components](#components) chapter below and the [docker-compose.yaml](docker-compose.yaml) file.
 
 ### Configuration
@@ -48,18 +58,22 @@ Configuration of API and Scanners is performed by passing the environment variab
 
 * `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
 
+* `WorkerCount=4` - Number of workers for parallel scanning.
+
 #### MalwareMultiScan.Ui
 
 * `API_URL=http://localhost:5000` - Absolute URL incl. port number for the running instance of MalwareMultiScan.Api.  
 
 ### API Endpoints
 
-* POST `/api/queue/url` with a `url` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
+* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
 
- * POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
+* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
 
 * GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error.
 
+Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) in a body on every update from scan backends.
+
 ## Supported Scan Engines
 
 | Name | Dockerfile | Enabled | Comments |
@@ -85,7 +99,7 @@ More scan backends can be added in the future. Some of the popular ones do not h
 
 * **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments.
 
-* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities. A simultaneous scan by a single node is not supported, yet load-balancing is possible by the built-in orchestrators' round-robin routing.
+* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities.
 
 ### Parts
 
@@ -99,4 +113,4 @@ More scan backends can be added in the future. Some of the popular ones do not h
  
  ## Plans
  
- See [issues](https://github.com/mindcollapse/MalwareMultiScan/issues) for the list of planned features, bug-fixes, and improvements.
+ See [issues](https://github.com/mindcollapse/MalwareMultiScan/issues) for the list of planned features, bug-fixes, and improvements.