From 3f710f97f67d8353aefc4dc27a3b7a33f8c52f6d Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Mon, 2 Nov 2020 11:00:24 +0200 Subject: [PATCH] finished refactoring and README.md rewrite --- .../Dockerfiles_Clamav_Dockerfile.xml | 16 ---- .../Dockerfiles_Comodo_Dockerfile.xml | 16 ---- .../Dockerfiles_DrWeb_Dockerfile.xml | 16 ---- .../Dockerfiles_KES_Dockerfile.xml | 16 ---- .../Dockerfiles_McAfee_Dockerfile.xml | 16 ---- .../Dockerfiles_Sophos_Dockerfile.xml | 16 ---- ...Dockerfiles_WindowsDefender_Dockerfile.xml | 16 ---- .../.idea/runConfigurations/Dummy_API.xml | 7 -- .../MalwareMultiScan_Scanner_Dockerfile.xml | 18 ----- MalwareMultiScan.Api/Dockerfile | 1 + .../MalwareMultiScan.Api.csproj | 17 ++-- MalwareMultiScan.Api/Program.cs | 2 + .../Properties/launchSettings.json | 8 ++ .../Services/ScanResultJob.cs | 18 ++++- MalwareMultiScan.Api/Startup.cs | 1 + MalwareMultiScan.Api/backends.yaml | 23 ------ .../Dockerfiles/Clamav.Dockerfile | 2 +- .../Dockerfiles/Comodo.Dockerfile | 2 +- .../Dockerfiles/DrWeb.Dockerfile | 2 +- .../Dockerfiles/KES.Dockerfile | 2 +- .../Dockerfiles/McAfee.Dockerfile | 2 +- .../Dockerfiles/Sophos.Dockerfile | 2 +- .../Dockerfiles/WindowsDefender.Dockerfile | 2 +- MalwareMultiScan.Scanner/Dockerfile | 17 ++++ MalwareMultiScan.Scanner/Program.cs | 2 + .../Properties/launchSettings.json | 2 +- .../Services/ConsulHostedService.cs | 12 ++- .../Services/ScanBackgroundJob.cs | 7 +- MalwareMultiScan.Scanner/appsettings.json | 1 - .../Extensions/ServiceCollectionExtensions.cs | 2 + MalwareMultiScan.Tests/Api/AttributesTests.cs | 2 +- .../Api/ReceiverHostedServiceTests.cs | 78 ------------------- .../Api/ScanBackendServiceTests.cs | 59 -------------- .../Api/ScanResultServiceTests.cs | 49 ++++-------- .../Backends/BackendsTests.cs | 1 + .../Scanner/ScanBackgroundJobTests.cs | 62 --------------- .../Scanner/ScanHostedServiceTests.cs | 49 ------------ MalwareMultiScan.Ui/package.json | 2 +- README.md | 72 ++++++++++++----- docker-compose.yaml | 66 +++++++++------- 40 files changed, 184 insertions(+), 520 deletions(-) delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml delete mode 100644 .idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml create mode 100644 MalwareMultiScan.Api/Properties/launchSettings.json delete mode 100644 MalwareMultiScan.Api/backends.yaml create mode 100644 MalwareMultiScan.Scanner/Dockerfile delete mode 100644 MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs delete mode 100644 MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs delete mode 100644 MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs delete mode 100644 MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml deleted file mode 100644 index 7ea3be1..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml deleted file mode 100644 index cd33557..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml deleted file mode 100644 index 2df7f80..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml deleted file mode 100644 index d634c34..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml deleted file mode 100644 index 6ad531e..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml deleted file mode 100644 index 72d2c60..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml deleted file mode 100644 index 175e6e3..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml deleted file mode 100644 index 0275fa7..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml deleted file mode 100644 index f04733b..0000000 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/MalwareMultiScan.Api/Dockerfile b/MalwareMultiScan.Api/Dockerfile index 8db137f..e89945a 100644 --- a/MalwareMultiScan.Api/Dockerfile +++ b/MalwareMultiScan.Api/Dockerfile @@ -4,6 +4,7 @@ WORKDIR /src COPY MalwareMultiScan.Api /src/MalwareMultiScan.Api COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends +COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Api/MalwareMultiScan.Api.csproj diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index b4cd3be..5300da5 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -4,17 +4,11 @@ netcoreapp3.1 Volodymyr Smirnov MalwareMultiScan Api - 1.0.1 - 1.0.1 + 1.5.0 + 1.5.0 true - - - Always - - - @@ -26,4 +20,11 @@ + + + + Never + Never + + diff --git a/MalwareMultiScan.Api/Program.cs b/MalwareMultiScan.Api/Program.cs index bc8874c..ef1fcd1 100644 --- a/MalwareMultiScan.Api/Program.cs +++ b/MalwareMultiScan.Api/Program.cs @@ -1,9 +1,11 @@ +using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; namespace MalwareMultiScan.Api { + [ExcludeFromCodeCoverage] internal static class Program { public static async Task Main(string[] args) diff --git a/MalwareMultiScan.Api/Properties/launchSettings.json b/MalwareMultiScan.Api/Properties/launchSettings.json new file mode 100644 index 0000000..11e87fd --- /dev/null +++ b/MalwareMultiScan.Api/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "MalwareMultiScan.Api": { + "commandName": "Project" + } + } +} diff --git a/MalwareMultiScan.Api/Services/ScanResultJob.cs b/MalwareMultiScan.Api/Services/ScanResultJob.cs index 540bd7b..f4677b9 100644 --- a/MalwareMultiScan.Api/Services/ScanResultJob.cs +++ b/MalwareMultiScan.Api/Services/ScanResultJob.cs @@ -6,7 +6,6 @@ using System.Threading; using System.Threading.Tasks; using Hangfire; using Hangfire.States; -using MalwareMultiScan.Api.Data; using MalwareMultiScan.Api.Services.Interfaces; using MalwareMultiScan.Shared.Message; using MalwareMultiScan.Shared.Services.Interfaces; @@ -14,6 +13,9 @@ using Microsoft.Extensions.Logging; namespace MalwareMultiScan.Api.Services { + /// + /// Job for storing scan results. + /// public class ScanResultJob : IScanResultJob { private readonly IBackgroundJobClient _backgroundJobClient; @@ -21,6 +23,13 @@ namespace MalwareMultiScan.Api.Services private readonly ILogger _logger; private readonly IScanResultService _scanResultService; + /// + /// Initialize scan result storing job. + /// + /// Logger. + /// HTTP client factory. + /// Scan result service. + /// Background job client. public ScanResultJob( ILogger logger, IHttpClientFactory httpClientFactory, @@ -48,14 +57,15 @@ namespace MalwareMultiScan.Api.Services return; _backgroundJobClient.Create( - x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result), new EnqueuedState()); + x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result), + new EnqueuedState("default")); } /// public async Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result) { var cancellationTokenSource = new CancellationTokenSource( - TimeSpan.FromSeconds(3)); + TimeSpan.FromSeconds(5)); using var httpClient = _httpClientFactory.CreateClient(); @@ -72,6 +82,8 @@ namespace MalwareMultiScan.Api.Services cancellationTokenSource.Token); response.EnsureSuccessStatusCode(); + + _logger.LogInformation($"Sent POST to callback URL {uri}"); } catch (Exception exception) { diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index bcf01e9..c40d624 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Hosting; namespace MalwareMultiScan.Api { + [ExcludeFromCodeCoverage] internal class Startup { private readonly IConfiguration _configuration; diff --git a/MalwareMultiScan.Api/backends.yaml b/MalwareMultiScan.Api/backends.yaml deleted file mode 100644 index 4b3609d..0000000 --- a/MalwareMultiScan.Api/backends.yaml +++ /dev/null @@ -1,23 +0,0 @@ -- id: dummy - enabled: true - -- id: clamav - enabled: true - -- id: windows-defender - enabled: true - -- id: comodo - enabled: false - -- id: drweb - enabled: false - -- id: kes - enabled: false - -- id: mcafee - enabled: false - -- id: sophos - enabled: false \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile index 8374dd8..20ab146 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile @@ -5,6 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y clamav clamav-daemon RUN freshclam --quiet -ENV BackendType=Clamav +ENV BACKEND_ID=clamav ENTRYPOINT /etc/init.d/clamav-daemon start && /worker/MalwareMultiScan.Scanner \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile index 8e10c16..e09411b 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile @@ -7,4 +7,4 @@ RUN wget -q https://cdn.download.comodo.com/cis/download/installs/linux/cav-linu RUN wget -q http://download.comodo.com/av/updates58/sigs/bases/bases.cav -O /opt/COMODO/scanners/bases.cav -ENV BackendType=Comodo \ No newline at end of file +ENV BACKEND_ID=comodo \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile index 53040a1..f44a52c 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile @@ -23,6 +23,6 @@ RUN /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && \ drweb-ctl update && \ kill $(cat /var/run/drweb-configd.pid) -ENV BackendType=DrWeb +ENV BACKEND_ID=drweb ENTRYPOINT /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && /worker/MalwareMultiScan.Scanner \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile index 51d2de5..3591c7f 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile @@ -30,6 +30,6 @@ kesl-control -B --query "FileName == \"$1\"" 2> /dev/null \n\ exit $? \ ' > /usr/bin/kesl-scan && chmod +x /usr/bin/kesl-scan -ENV BackendType=Kes +ENV BACKEND_ID=kes ENTRYPOINT /etc/init.d/kesl-supervisor start && /worker/MalwareMultiScan.Scanner \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile index 8dadd87..523c52e 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile @@ -16,4 +16,4 @@ RUN wget -q -Nc -r -nd -l1 -A "avvepo????dat.zip" http://download.nai.com/produc WORKDIR /worker -ENV BackendType=McAfee \ No newline at end of file +ENV BACKEND_ID=mcafee \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile index 814309d..64403b0 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile @@ -9,5 +9,5 @@ RUN wget -q $SOPHOS_URL -O /tmp/SophosInstall.sh && \ chmod +x /tmp/SophosInstall.sh && \ /tmp/SophosInstall.sh --automatic --acceptlicence || exit 0 -ENV BackendType=Sophos +ENV BACKEND_ID=sophos diff --git a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile index b97083f..9600589 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile @@ -18,4 +18,4 @@ RUN apt-get update && apt-get install -y libc6-i386 COPY --from=backend /opt/loadlibrary/engine /opt/engine COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient -ENV BackendType=Defender \ No newline at end of file +ENV BACKEND_ID=windows-defender \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/Dockerfile b/MalwareMultiScan.Scanner/Dockerfile new file mode 100644 index 0000000..f5c0ce9 --- /dev/null +++ b/MalwareMultiScan.Scanner/Dockerfile @@ -0,0 +1,17 @@ +FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS builder + +WORKDIR /src + +COPY MalwareMultiScan.Scanner /src/MalwareMultiScan.Scanner +COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends +COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared + +RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj + +FROM mcr.microsoft.com/dotnet/core/runtime:3.1 + +WORKDIR /worker + +COPY --from=builder /src/publish /worker + +ENTRYPOINT ["/worker/MalwareMultiScan.Scanner"] \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/Program.cs b/MalwareMultiScan.Scanner/Program.cs index 1339b20..e612b93 100644 --- a/MalwareMultiScan.Scanner/Program.cs +++ b/MalwareMultiScan.Scanner/Program.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; using MalwareMultiScan.Backends.Extensions; using MalwareMultiScan.Backends.Services.Implementations; @@ -12,6 +13,7 @@ using Microsoft.Extensions.Logging; namespace MalwareMultiScan.Scanner { + [ExcludeFromCodeCoverage] internal static class Program { public static async Task Main(string[] args) diff --git a/MalwareMultiScan.Scanner/Properties/launchSettings.json b/MalwareMultiScan.Scanner/Properties/launchSettings.json index bd14be3..02c6827 100644 --- a/MalwareMultiScan.Scanner/Properties/launchSettings.json +++ b/MalwareMultiScan.Scanner/Properties/launchSettings.json @@ -1,7 +1,7 @@ { "$schema": "http://json.schemastore.org/launchsettings.json", "profiles": { - "MalwareMultiScan.ScannerWorker": { + "MalwareMultiScan.Scanner": { "commandName": "Project" } } diff --git a/MalwareMultiScan.Scanner/Services/ConsulHostedService.cs b/MalwareMultiScan.Scanner/Services/ConsulHostedService.cs index 1e10ead..4674c3b 100644 --- a/MalwareMultiScan.Scanner/Services/ConsulHostedService.cs +++ b/MalwareMultiScan.Scanner/Services/ConsulHostedService.cs @@ -5,11 +5,13 @@ using System.Threading; using System.Threading.Tasks; using Consul; using MalwareMultiScan.Backends.Backends.Interfaces; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; namespace MalwareMultiScan.Scanner.Services { + /// + /// Consul registration hosted service. + /// public class ConsulHostedService : BackgroundService { private static readonly string ServiceId = Dns.GetHostName(); @@ -18,9 +20,13 @@ namespace MalwareMultiScan.Scanner.Services private readonly IConsulClient _consulClient; private readonly AgentServiceRegistration _registration; + /// + /// Initialize consul hosted service. + /// + /// Consul client. + /// Scan backend. public ConsulHostedService( IConsulClient consulClient, - IConfiguration configuration, IScanBackend scanBackend) { _consulClient = consulClient; @@ -28,7 +34,7 @@ namespace MalwareMultiScan.Scanner.Services _registration = new AgentServiceRegistration { ID = ServiceId, - Name = configuration.GetValue("CONSUL_SERVICE_NAME"), + Name = "scanner", Meta = new Dictionary { diff --git a/MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs b/MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs index 9f04b7a..0fbf5da 100644 --- a/MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs +++ b/MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs @@ -18,8 +18,8 @@ namespace MalwareMultiScan.Scanner.Services { private readonly IScanBackend _backend; private readonly IConfiguration _configuration; - private readonly IBackgroundJobClient _jobClient; private readonly ILogger _logger; + private readonly IBackgroundJobClient _jobClient; /// /// Initialize scan background job. @@ -27,7 +27,7 @@ namespace MalwareMultiScan.Scanner.Services /// Configuration. /// Logger. /// Scan backend. - /// + /// Background job client. public ScanBackgroundJob( IScanBackend backend, IConfiguration configuration, @@ -90,7 +90,8 @@ namespace MalwareMultiScan.Scanner.Services $"Sending scan results with status {result.Status}"); _jobClient.Create( - x => x.Report(message.Id, _backend.Id, result), new EnqueuedState()); + x => x.Report(message.Id, _backend.Id, result), + new EnqueuedState("default")); } catch (Exception exception) { diff --git a/MalwareMultiScan.Scanner/appsettings.json b/MalwareMultiScan.Scanner/appsettings.json index 1083ca6..3edade7 100644 --- a/MalwareMultiScan.Scanner/appsettings.json +++ b/MalwareMultiScan.Scanner/appsettings.json @@ -2,7 +2,6 @@ "REDIS_ADDRESS": "localhost:6379", "CONSUL_ADDRESS": "http://localhost:8500", - "CONSUL_SERVICE_NAME": "scanner", "BACKEND_ID": "dummy", "MAX_SCANNING_TIME": 60, diff --git a/MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs b/MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs index 730ade3..bcd67ac 100644 --- a/MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs +++ b/MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.CodeAnalysis; using Consul; using Hangfire; using Microsoft.Extensions.Configuration; @@ -9,6 +10,7 @@ namespace MalwareMultiScan.Shared.Extensions /// /// Extensions for IServiceCollection. /// + [ExcludeFromCodeCoverage] public static class ServiceCollectionExtensions { /// diff --git a/MalwareMultiScan.Tests/Api/AttributesTests.cs b/MalwareMultiScan.Tests/Api/AttributesTests.cs index 86b2d4a..8938c6f 100644 --- a/MalwareMultiScan.Tests/Api/AttributesTests.cs +++ b/MalwareMultiScan.Tests/Api/AttributesTests.cs @@ -36,7 +36,7 @@ namespace MalwareMultiScan.Tests.Api new MemoryConfigurationProvider(new MemoryConfigurationSource()) }) { - ["MaxFileSize"] = "100" + ["FILE_SIZE_LIMIT"] = "100" }; var context = new ValidationContext( diff --git a/MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs b/MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs deleted file mode 100644 index d248a68..0000000 --- a/MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Threading.Tasks; -using MalwareMultiScan.Api.Services.Interfaces; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Memory; -using Microsoft.Extensions.Logging; -using Moq; -using NUnit.Framework; - -namespace MalwareMultiScan.Tests.Api -{ - public class ReceiverHostedServiceTests - { - private Mock _busMock; - private IReceiverHostedService _receiverHostedService; - private Mock _scanResultServiceMock; - - [SetUp] - public void SetUp() - { - _busMock = new Mock(); - - _busMock - .Setup(x => x.Receive("mms.results", It.IsAny>())) - .Callback>((s, func) => - { - var task = func.Invoke(new ScanResultMessage - { - Id = "test", - Backend = "dummy", - Duration = 100, - Succeeded = true, - Threats = new[] {"Test"} - }); - - task.Wait(); - }); - - _scanResultServiceMock = new Mock(); - - var configuration = new ConfigurationRoot(new List - { - new MemoryConfigurationProvider(new MemoryConfigurationSource()) - }) - { - ["ResultsSubscriptionId"] = "mms.results" - }; - - var httpClientFactoryMock = new Mock(); - - _receiverHostedService = new ReceiverHostedService( - _busMock.Object, - configuration, - _scanResultServiceMock.Object, - Mock.Of>(), - httpClientFactoryMock.Object); - } - - [Test] - public async Task TestBusReceiveScanResultMessage() - { - await _receiverHostedService.StartAsync(default); - - _scanResultServiceMock.Verify(x => x.UpdateScanResultForBackend( - "test", "dummy", 100, true, true, new[] {"Test"}), Times.Once); - } - - [Test] - public async Task TestBusIsDisposedOnStop() - { - await _receiverHostedService.StopAsync(default); - - _busMock.Verify(x => x.Dispose(), Times.Once); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs b/MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs deleted file mode 100644 index 2632efb..0000000 --- a/MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using MalwareMultiScan.Api.Data; -using MalwareMultiScan.Api.Data.Configuration; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Memory; -using Microsoft.Extensions.Logging; -using Moq; -using NUnit.Framework; - -namespace MalwareMultiScan.Tests.Api -{ - public class ScanBackendServiceTests - { - private Mock _busMock; - private IScanBackendService _scanBackendService; - - [SetUp] - public void SetUp() - { - var configuration = new ConfigurationRoot(new List - { - new MemoryConfigurationProvider(new MemoryConfigurationSource()) - }) - { - ["BackendsConfiguration"] = "backends.yaml" - }; - - _busMock = new Mock(); - - _scanBackendService = new ScanBackendService( - configuration, - _busMock.Object, - Mock.Of>()); - } - - [Test] - public void TestBackendsAreNotEmpty() - { - Assert.IsNotEmpty(_scanBackendService.List); - } - - [Test] - public async Task TestQueueUrlScan() - { - await _scanBackendService.QueueUrlScan( - new ScanResult {Id = "test"}, - new ScanBackend {Id = "dummy"}, - "http://test.com" - ); - - _busMock.Verify(x => x.SendAsync( - "dummy", It.Is(r => - r.Id == "test" && - r.Uri == new Uri("http://test.com"))), Times.Once); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs b/MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs index 903f07c..9332c5b 100644 --- a/MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs +++ b/MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs @@ -1,9 +1,11 @@ using System.IO; using System.Threading.Tasks; -using MalwareMultiScan.Api.Data; -using MalwareMultiScan.Api.Data.Configuration; +using Consul; +using Hangfire; using MalwareMultiScan.Api.Services; using MalwareMultiScan.Api.Services.Interfaces; +using MalwareMultiScan.Shared.Enums; +using MalwareMultiScan.Shared.Message; using Mongo2Go; using MongoDB.Bson; using MongoDB.Driver; @@ -17,7 +19,6 @@ namespace MalwareMultiScan.Tests.Api { private MongoDbRunner _mongoDbRunner; private IScanResultService _resultService; - private Mock _scanBackendService; [SetUp] public void SetUp() @@ -27,20 +28,11 @@ namespace MalwareMultiScan.Tests.Api var connection = new MongoClient(_mongoDbRunner.ConnectionString); var database = connection.GetDatabase("Test"); var gridFsBucket = new GridFSBucket(database); - - _scanBackendService = new Mock(); - - _scanBackendService - .SetupGet(x => x.List) - .Returns(() => new[] - { - new ScanBackend {Id = "dummy", Enabled = true}, - new ScanBackend {Id = "clamav", Enabled = true}, - new ScanBackend {Id = "disabled", Enabled = false} - }); - + _resultService = new ScanResultService( - database, gridFsBucket, _scanBackendService.Object); + database, gridFsBucket, + Mock.Of(), + Mock.Of()); } [TearDown] @@ -85,10 +77,14 @@ namespace MalwareMultiScan.Tests.Api var result = await _resultService.CreateScanResult(null); Assert.NotNull(result.Id); - Assert.That(result.Results, Contains.Key("dummy")); await _resultService.UpdateScanResultForBackend( - result.Id, "dummy", 100, true, true, new[] {"Test"}); + result.Id, "dummy", new ScanResultMessage + { + Duration = 100, + Status = ScanResultStatus.Succeeded, + Threats = new []{"Test"} + }); result = await _resultService.GetScanResult(result.Id); @@ -97,25 +93,10 @@ namespace MalwareMultiScan.Tests.Api var dummyResult = result.Results["dummy"]; - Assert.IsTrue(dummyResult.Completed); - Assert.IsTrue(dummyResult.Succeeded); + Assert.IsTrue(dummyResult.Status == ScanResultStatus.Succeeded); Assert.AreEqual(dummyResult.Duration, 100); Assert.IsNotEmpty(dummyResult.Threats); Assert.That(dummyResult.Threats, Is.EquivalentTo(new[] {"Test"})); } - - [Test] - public async Task TestQueueUrlScan() - { - var result = await _resultService.CreateScanResult(null); - - await _resultService.QueueUrlScan( - result, "http://url.com"); - - _scanBackendService.Verify(x => x.QueueUrlScan( - It.IsAny(), - It.IsAny(), - "http://url.com"), Times.Exactly(2)); - } } } \ No newline at end of file diff --git a/MalwareMultiScan.Tests/Backends/BackendsTests.cs b/MalwareMultiScan.Tests/Backends/BackendsTests.cs index aa22a03..e8bf939 100644 --- a/MalwareMultiScan.Tests/Backends/BackendsTests.cs +++ b/MalwareMultiScan.Tests/Backends/BackendsTests.cs @@ -1,6 +1,7 @@ using System.Threading; using System.Threading.Tasks; using MalwareMultiScan.Backends.Backends; +using MalwareMultiScan.Backends.Backends.Interfaces; using MalwareMultiScan.Backends.Services.Interfaces; using Moq; using NUnit.Framework; diff --git a/MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs b/MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs deleted file mode 100644 index 63f5749..0000000 --- a/MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Configuration.Memory; -using Microsoft.Extensions.Logging; -using Moq; -using NUnit.Framework; - -namespace MalwareMultiScan.Tests.Scanner -{ - public class ScanBackgroundJobTests - { - private Mock _busMock; - private Mock _scanBackendMock; - private IScanBackgroundJob _scanBackgroundJob; - - [SetUp] - public void SetUp() - { - var configuration = new ConfigurationRoot(new List - { - new MemoryConfigurationProvider(new MemoryConfigurationSource()) - }) - { - ["ResultsSubscriptionId"] = "mms.results" - }; - - _busMock = new Mock(); - - _scanBackendMock = new Mock(); - - _scanBackendMock - .SetupGet(x => x.Id) - .Returns("dummy"); - - _scanBackendMock - .Setup(x => x.ScanAsync(It.IsAny(), It.IsAny())) - .Returns(Task.FromResult(new[] {"Test"})); - - _scanBackgroundJob = new ScanBackgroundJob(configuration, _busMock.Object, - Mock.Of>(), _scanBackendMock.Object); - } - - [Test] - public async Task TestMessageProcessing() - { - await _scanBackgroundJob.Process(new ScanRequestMessage - { - Id = "test", - Uri = new Uri("http://test.com") - }); - - _scanBackendMock.Verify( - x => x.ScanAsync(It.IsAny(), It.IsAny())); - - _busMock.Verify(x => x.SendAsync("mms.results", It.Is(m => - m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test")))); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs b/MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs deleted file mode 100644 index fbb125c..0000000 --- a/MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Moq; -using NUnit.Framework; - -namespace MalwareMultiScan.Tests.Scanner -{ - public class ScanHostedServiceTests - { - private Mock _busMock; - private Mock _scanBackendMock; - private IScanHostedService _scanHostedService; - - [SetUp] - public void SetUp() - { - _busMock = new Mock(); - - _scanBackendMock = new Mock(); - - _scanBackendMock - .SetupGet(x => x.Id) - .Returns("dummy"); - - _scanHostedService = new ScanHostedService( - Mock.Of>(), - _scanBackendMock.Object, - _busMock.Object); - } - - [Test] - public async Task TestBusReceiveScanResultMessage() - { - await _scanHostedService.StartAsync(default); - - _busMock.Verify( - x => x.Receive("dummy", It.IsAny>())); - } - - [Test] - public async Task TestBusIsDisposedOnStop() - { - await _scanHostedService.StopAsync(default); - - _busMock.Verify(x => x.Dispose(), Times.Once); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Ui/package.json b/MalwareMultiScan.Ui/package.json index 494cc15..5831792 100644 --- a/MalwareMultiScan.Ui/package.json +++ b/MalwareMultiScan.Ui/package.json @@ -1,6 +1,6 @@ { "name": "malware-multi-scan-ui", - "version": "1.0.2", + "version": "1.5.0", "private": true, "scripts": { "dev": "nuxt-ts", diff --git a/README.md b/README.md index 31091ce..ee113f8 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Self-hosted [VirusTotal](https://www.virustotal.com/) wannabe API for scanning U The demo is running on a cheap Vultr node, so it might get slow or unavailable occasionally. +**IMPORTANT**: version 1.5 introduces breaking changes in containers configuration and docker-compose.yaml layout. Please see [releases](https://github.com/mindcollapse/MalwareMultiScan/releases) page and changelog of [docker-compose.yaml](https://github.com/mindcollapse/MalwareMultiScan/commits/master/docker-compose.yaml) and [README.md](https://github.com/mindcollapse/MalwareMultiScan/commits/master/README.md) for the additional details. + ## Introduction I faced a need to scan user-uploaded files in one of my work projects in an automated mode to ensure they don't contain any malware. Using VirusTotal was not an option because of a) legal restrictions and data residency limitations b) scanning by hash-sums would not be sufficient because the majority of files are generated / modified by users. @@ -36,29 +38,27 @@ Configuration of API and Scanners is performed by passing the environment variab #### MalwareMultiScan.Api -* `ConnectionStrings__Mongo=mongodb://localhost:27017` - MongoDB connection string. +* `MONGO_ADDRESS=mongodb://localhost:27017` - MongoDB connection string. -* `DatabaseName=MalwareMultiScan` - MongoDB collection name. +* `MONGO_DATABASE=MalwareMultiScan` - MongoDB collection name. -* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details. +* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue. -* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners. +* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration. -* `MaxFileSize=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified. - -* `BackendsConfiguration=backends.yaml` - Path to the [backends.yaml](MalwareMultiScan.Api/backends.yaml) file. +* `FILE_SIZE_LIMIT=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified. Set to 0 to disable the validation. #### MalwareMultiScan.Scanner -* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details. +* `BACKEND_ID=dummy` - Id of a backend. -* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners. +* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue. -* `BackendType=Dummy` - A type of scanner backend used by the running instance. Should correspond to the [BackendType](MalwareMultiScan.Backends/Enums/BackendType.cs) enum. +* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration. -* `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file. +* `MAX_SCANNING_TIME=60` - Scan time limit. It is used not just for actual scanning but also for getting the file. -* `WorkerCount=4` - Number of workers for parallel scanning. +* `WORKER_COUNT=4` - Number of workers for parallel scanning. #### MalwareMultiScan.Ui @@ -66,13 +66,25 @@ Configuration of API and Scanners is performed by passing the environment variab ### API Endpoints -* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error. +* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error. -* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error. +* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error. -* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error. +* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `404 Not Found` error. -Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) in a body on every update from scan backends. +#### Callback URL + +Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResultMessage](MalwareMultiScan.Shared/Message/ScanResultMessage.cs) in a body on every update from scan backends. Query string will contain `id` parameter that corresponds to the id of the scan result and `backend` parameter with the id of backend which completed the scan. + +I.e. when you define `callbackUrl=http://localhost:1234/scan-results`, the POST request will be made to `http://localhost:1234/scan-results?id=123&backend=dummy` with a body + +```json +{ + "Status":1, + "Duration":5, + "Threats":["Malware.Dummy.Result"] +} +``` ## Supported Scan Engines @@ -91,11 +103,29 @@ More scan backends can be added in the future. Some of the popular ones do not h ## Components +### Workflow + +1. On startup all [Scanners](MalwareMultiScan.Scanner) register themselves in [Consul](https://www.consul.io/) with a service name equal to `scanner` and the `BackendId` metadata field equal to the value of `BACKEND_ID` environment variable. They also register a TTL check and listen for [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field. + +2. Third-party client triggers `/api/queue/url` or `/api/queue/file` of the [MalwareMultiScan.Api](MalwareMultiScan.Api). + +3. [MalwareMultiScan.Api](MalwareMultiScan.Api) sends a query to [Consul](https://www.consul.io/) and receives the list of alive scan backends with the service name `scanner`. + +4. [MalwareMultiScan.Api](MalwareMultiScan.Api) schedules a [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field. + +5. [Scanners](MalwareMultiScan.Scanner) picks up a job from queue, starts the scan and sends result back to the `default` queue of [Hangfire](https://www.hangfire.io/). + +6. [MalwareMultiScan.Api](MalwareMultiScan.Api) picks a job from the default` queue of [Hangfire](https://www.hangfire.io/) and updates the state of the scan. + +7. If callback URL was specified during the step #2, [MalwareMultiScan.Api](MalwareMultiScan.Api) triggers a HTTP POST request to the specified URL. See [Callback URL](#callback-url) for details. + ### Prerequisites -* **MongoDB** of version 3.x. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp). +* **MongoDB** of version 3.x or above. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp). -* **RabbitMQ** of version 3.x. Used for IPC and scan tasks queueing. The communication is happening through the [EasyNetQ](https://github.com/EasyNetQ/EasyNetQ) library. +* **Redis** of version 5.x or above. Used for tasks queueing. The communication is happening through the [Hangfire](https://www.hangfire.io/) library. + +* **Consul** of version 1.8.x or above. Used for service registration of scan backends. * **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments. @@ -103,9 +133,11 @@ More scan backends can be added in the future. Some of the popular ones do not h ### Parts -* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile). Configuration of available backends is performed via the [backends.yaml](MalwareMultiScan.Api/backends.yaml) location passed via the `BackendsConfiguration` environment variable. +* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile). -* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Shared components between API and Worker. Includes Dockerfiles and implementation classes for third-party vendor scan backends. +* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Scan backends logic. Includes Dockerfiles and implementation classes for third-party vendor scan backends. + +* [MalwareMultiScan.Shared](MalwareMultiScan.Shared) - Shared components. * [MalwareMultiScan.Scanner](MalwareMultiScan.Scanner) - .NET Core Worker service subscribes to messages corresponding to the backend id, then fires up scanning command-line utility, and parses the output. See [Dockerfile](MalwareMultiScan.Scanner/Dockerfile). The image of MalwareMultiScan.Scanner acts as a base image for the rest of the scan backends. Check Dockerfiles from the [table above](#supported-scan-engines) for details. diff --git a/docker-compose.yaml b/docker-compose.yaml index 1373f0b..fbfe938 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,23 +1,24 @@ version: "3.8" services: - rabbitmq: - image: rabbitmq:3 + redis: + image: redis:6 restart: on-failure - expose: - - "5672" volumes: - - rabbitmq_etc/:/etc/rabbitmq/ - - rabbitmq_data:/var/lib/rabbitmq/ - - rabbitmq_logs/:/var/log/rabbitmq/ - + - redis:/data + mongodb: image: mongo:4 restart: on-failure - expose: - - "27019" volumes: - mongodb:/data + + consul: + image: consul:1.8 + restart: on-failure + volumes: + - consul:/consul/data + command: consul agent -server -bootstrap -client 0.0.0.0 ui: image: mindcollapse/malware-multi-scan-ui @@ -39,17 +40,14 @@ services: api: image: mindcollapse/malware-multi-scan-api restart: on-failure - expose: - - "5000" depends_on: - - rabbitmq + - consul + - redis - mongodb environment: - - "ConnectionStrings__RabbitMQ=host=rabbitmq;timeout=120" - - "ConnectionStrings__Mongo=mongodb://mongodb:27017?connectTimeoutMS=120000" - - "BackendsConfiguration=/etc/backends.yaml" - volumes: - - "./MalwareMultiScan.Api/backends.yaml:/etc/backends.yaml:ro" + - "REDIS_ADDRESS=redis:6379" + - "CONSUL_ADDRESS=http://consul:8500" + - "MONGO_ADDRESS=mongodb://mongodb:27017?connectTimeoutMS=120000" build: context: . dockerfile: MalwareMultiScan.Api/Dockerfile @@ -58,9 +56,11 @@ services: image: mindcollapse/malware-multi-scan-scanner restart: on-failure depends_on: - - rabbitmq + - consul + - redis environment: - - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" + - "REDIS_ADDRESS=redis:6379" + - "CONSUL_ADDRESS=http://consul:8500" build: context: . dockerfile: MalwareMultiScan.Scanner/Dockerfile @@ -71,7 +71,8 @@ services: depends_on: - dummy-scanner environment: - - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" + - "REDIS_ADDRESS=redis:6379" + - "CONSUL_ADDRESS=http://consul:8500" build: context: MalwareMultiScan.Backends/Dockerfiles dockerfile: Clamav.Dockerfile @@ -82,7 +83,8 @@ services: depends_on: - dummy-scanner environment: - - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" + - "REDIS_ADDRESS=redis:6379" + - "CONSUL_ADDRESS=http://consul:8500" build: context: MalwareMultiScan.Backends/Dockerfiles dockerfile: WindowsDefender.Dockerfile @@ -95,7 +97,8 @@ services: # depends_on: # - dummy-scanner # environment: -# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" +# - "REDIS_ADDRESS=redis:6379" +# - "CONSUL_ADDRESS=http://consul:8500" # build: # context: MalwareMultiScan.Backends/Dockerfiles # dockerfile: Comodo.Dockerfile @@ -106,7 +109,8 @@ services: # depends_on: # - dummy-scanner # environment: -# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" +# - "REDIS_ADDRESS=redis:6379" +# - "CONSUL_ADDRESS=http://consul:8500" # build: # context: MalwareMultiScan.Backends/Dockerfiles # dockerfile: DrWeb.Dockerfile @@ -117,7 +121,8 @@ services: # depends_on: # - dummy-scanner # environment: -# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" +# - "REDIS_ADDRESS=redis:6379" +# - "CONSUL_ADDRESS=http://consul:8500" # build: # context: MalwareMultiScan.Backends/Dockerfiles # dockerfile: KES.Dockerfile @@ -128,7 +133,8 @@ services: # depends_on: # - dummy-scanner # environment: -# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" +# - "REDIS_ADDRESS=redis:6379" +# - "CONSUL_ADDRESS=http://consul:8500" # build: # context: MalwareMultiScan.Backends/Dockerfiles # dockerfile: McAfee.Dockerfile @@ -139,13 +145,13 @@ services: # depends_on: # - dummy-scanner # environment: -# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120" +# - "REDIS_ADDRESS=redis:6379" +# - "CONSUL_ADDRESS=http://consul:8500" # build: # context: MalwareMultiScan.Backends/Dockerfiles # dockerfile: Sophos.Dockerfile volumes: mongodb: - rabbitmq_etc: - rabbitmq_data: - rabbitmq_logs: \ No newline at end of file + redis: + consul: \ No newline at end of file