Mirrors/MalwareMultiScan
1
0
Fork 0
mirror of https://github.com/volodymyrsmirnov/MalwareMultiScan.git synced 2025-08-23 21:12:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

finished refactoring and README.md rewrite

Browse Source
This commit is contained in:
Volodymyr Smirnov 2020-11-02 11:00:24 +02:00
parent 1e666d2ed2
commit 3f710f97f6
40 changed files with 184 additions and 520 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Clamav.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-clamav" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-clamav" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Comodo.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-comodo" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-comodo" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/DrWeb.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-drweb" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-drweb" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/KES.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-kes" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-kes" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/McAfee.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-mcafee" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-mcafee" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Sophos.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-sophos" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-sophos" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/WindowsDefender.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-windows-defender" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-windows-defender" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

7
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml generated
View File

@ -1,7 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dummy API" type="CompoundRunConfigurationType">
<toRun name="MalwareMultiScan.ScannerWorker" type="LaunchSettings" />
<toRun name="MalwareMultiScan.Api" type="DotNetProject" />
<method v="2" />
</configuration>
</component>

18
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml generated
View File

@ -1,18 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="MalwareMultiScan.Scanner/Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-scanner" />
<option name="buildCliOptions" value="" />
<option name="buildOnly" value="true" />
<option name="command" value="" />
<option name="containerName" value="" />
<option name="contextFolderPath" value="." />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Scanner/Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

1
MalwareMultiScan.Api/Dockerfile
View File

@ -4,6 +4,7 @@ WORKDIR /src
COPY MalwareMultiScan.Api /src/MalwareMultiScan.Api
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Api/MalwareMultiScan.Api.csproj

17
MalwareMultiScan.Api/MalwareMultiScan.Api.csproj
View File

@ -4,17 +4,11 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Api</Product>
<AssemblyVersion>1.0.1</AssemblyVersion>
<FileVersion>1.0.1</FileVersion>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<None Update="backends.yaml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
@ -26,4 +20,11 @@
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
</ItemGroup>
<ItemGroup>
<Content Update="Properties\launchSettings.json">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</Content>
</ItemGroup>
</Project>

2
MalwareMultiScan.Api/Program.cs
View File

@ -1,9 +1,11 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api
{
[ExcludeFromCodeCoverage]
internal static class Program
{
public static async Task Main(string[] args)

8
MalwareMultiScan.Api/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"MalwareMultiScan.Api": {
"commandName": "Project"
}
}
}

18
MalwareMultiScan.Api/Services/ScanResultJob.cs
View File

@ -6,7 +6,6 @@ using System.Threading;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
@ -14,6 +13,9 @@ using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Api.Services
{
/// <summary>
/// Job for storing scan results.
/// </summary>
public class ScanResultJob : IScanResultJob
{
private readonly IBackgroundJobClient _backgroundJobClient;
@ -21,6 +23,13 @@ namespace MalwareMultiScan.Api.Services
private readonly ILogger<ScanResultJob> _logger;
private readonly IScanResultService _scanResultService;
/// <summary>
/// Initialize scan result storing job.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="httpClientFactory">HTTP client factory.</param>
/// <param name="scanResultService">Scan result service.</param>
/// <param name="backgroundJobClient">Background job client.</param>
public ScanResultJob(
ILogger<ScanResultJob> logger,
IHttpClientFactory httpClientFactory,
@ -48,14 +57,15 @@ namespace MalwareMultiScan.Api.Services
return;
_backgroundJobClient.Create<IScanResultJob>(
x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result), new EnqueuedState());
x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result),
new EnqueuedState("default"));
}
/// <inheritdoc />
public async Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(3));
TimeSpan.FromSeconds(5));
using var httpClient = _httpClientFactory.CreateClient();
@ -72,6 +82,8 @@ namespace MalwareMultiScan.Api.Services
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
_logger.LogInformation($"Sent POST to callback URL {uri}");
}
catch (Exception exception)
{

1
MalwareMultiScan.Api/Startup.cs
View File

@ -12,6 +12,7 @@ using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api
{
[ExcludeFromCodeCoverage]
internal class Startup
{
private readonly IConfiguration _configuration;

23
MalwareMultiScan.Api/backends.yaml
View File

@ -1,23 +0,0 @@
- id: dummy
enabled: true
- id: clamav
enabled: true
- id: windows-defender
enabled: true
- id: comodo
enabled: false
- id: drweb
enabled: false
- id: kes
enabled: false
- id: mcafee
enabled: false
- id: sophos
enabled: false

2
MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile
View File

@ -5,6 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y clamav clamav-daemon
RUN freshclam --quiet
ENV BackendType=Clamav
ENV BACKEND_ID=clamav
ENTRYPOINT /etc/init.d/clamav-daemon start && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile
View File

@ -7,4 +7,4 @@ RUN wget -q https://cdn.download.comodo.com/cis/download/installs/linux/cav-linu
RUN wget -q http://download.comodo.com/av/updates58/sigs/bases/bases.cav -O /opt/COMODO/scanners/bases.cav
ENV BackendType=Comodo
ENV BACKEND_ID=comodo

2
MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile
View File

@ -23,6 +23,6 @@ RUN /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && \
drweb-ctl update && \
kill $(cat /var/run/drweb-configd.pid)
ENV BackendType=DrWeb
ENV BACKEND_ID=drweb
ENTRYPOINT /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile
View File

@ -30,6 +30,6 @@ kesl-control -B --query "FileName == \"$1\"" 2> /dev/null \n\
exit $? \
' > /usr/bin/kesl-scan && chmod +x /usr/bin/kesl-scan
ENV BackendType=Kes
ENV BACKEND_ID=kes
ENTRYPOINT /etc/init.d/kesl-supervisor start && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile
View File

@ -16,4 +16,4 @@ RUN wget -q -Nc -r -nd -l1 -A "avvepo????dat.zip" http://download.nai.com/produc
WORKDIR /worker
ENV BackendType=McAfee
ENV BACKEND_ID=mcafee

2
MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile
View File

@ -9,5 +9,5 @@ RUN wget -q $SOPHOS_URL -O /tmp/SophosInstall.sh && \
chmod +x /tmp/SophosInstall.sh && \
/tmp/SophosInstall.sh --automatic --acceptlicence || exit 0
ENV BackendType=Sophos
ENV BACKEND_ID=sophos

2
MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile
View File

@ -18,4 +18,4 @@ RUN apt-get update && apt-get install -y libc6-i386
COPY --from=backend /opt/loadlibrary/engine /opt/engine
COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient
ENV BackendType=Defender
ENV BACKEND_ID=windows-defender

17
MalwareMultiScan.Scanner/Dockerfile Normal file
View File

@ -0,0 +1,17 @@
FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS builder
WORKDIR /src
COPY MalwareMultiScan.Scanner /src/MalwareMultiScan.Scanner
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
FROM mcr.microsoft.com/dotnet/core/runtime:3.1
WORKDIR /worker
COPY --from=builder /src/publish /worker
ENTRYPOINT ["/worker/MalwareMultiScan.Scanner"]

2
MalwareMultiScan.Scanner/Program.cs
View File

@ -1,3 +1,4 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Backends.Services.Implementations;
@ -12,6 +13,7 @@ using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner
{
[ExcludeFromCodeCoverage]
internal static class Program
{
public static async Task Main(string[] args)

2
MalwareMultiScan.Scanner/Properties/launchSettings.json
View File

@ -1,7 +1,7 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"MalwareMultiScan.ScannerWorker": {
"MalwareMultiScan.Scanner": {
"commandName": "Project"
}
}

12
MalwareMultiScan.Scanner/Services/ConsulHostedService.cs
View File

@ -5,11 +5,13 @@ using System.Threading;
using System.Threading.Tasks;
using Consul;
using MalwareMultiScan.Backends.Backends.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Scanner.Services
{
/// <summary>
/// Consul registration hosted service.
/// </summary>
public class ConsulHostedService : BackgroundService
{
private static readonly string ServiceId = Dns.GetHostName();
@ -18,9 +20,13 @@ namespace MalwareMultiScan.Scanner.Services
private readonly IConsulClient _consulClient;
private readonly AgentServiceRegistration _registration;
/// <summary>
/// Initialize consul hosted service.
/// </summary>
/// <param name="consulClient">Consul client.</param>
/// <param name="scanBackend">Scan backend.</param>
public ConsulHostedService(
IConsulClient consulClient,
IConfiguration configuration,
IScanBackend scanBackend)
{
_consulClient = consulClient;
@ -28,7 +34,7 @@ namespace MalwareMultiScan.Scanner.Services
_registration = new AgentServiceRegistration
{
ID = ServiceId,
Name = configuration.GetValue<string>("CONSUL_SERVICE_NAME"),
Name = "scanner",
Meta = new Dictionary<string, string>
{

7
MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs
View File

@ -18,8 +18,8 @@ namespace MalwareMultiScan.Scanner.Services
{
private readonly IScanBackend _backend;
private readonly IConfiguration _configuration;
private readonly IBackgroundJobClient _jobClient;
private readonly ILogger<ScanBackgroundJob> _logger;
private readonly IBackgroundJobClient _jobClient;
/// <summary>
/// Initialize scan background job.
@ -27,7 +27,7 @@ namespace MalwareMultiScan.Scanner.Services
/// <param name="configuration">Configuration.</param>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
/// <param name="jobClient"></param>
/// <param name="jobClient">Background job client.</param>
public ScanBackgroundJob(
IScanBackend backend,
IConfiguration configuration,
@ -90,7 +90,8 @@ namespace MalwareMultiScan.Scanner.Services
$"Sending scan results with status {result.Status}");
_jobClient.Create<IScanResultJob>(
x => x.Report(message.Id, _backend.Id, result), new EnqueuedState());
x => x.Report(message.Id, _backend.Id, result),
new EnqueuedState("default"));
}
catch (Exception exception)
{

1
MalwareMultiScan.Scanner/appsettings.json
View File

@ -2,7 +2,6 @@
"REDIS_ADDRESS": "localhost:6379",
"CONSUL_ADDRESS": "http://localhost:8500",
"CONSUL_SERVICE_NAME": "scanner",
"BACKEND_ID": "dummy",
"MAX_SCANNING_TIME": 60,

2
MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Diagnostics.CodeAnalysis;
using Consul;
using Hangfire;
using Microsoft.Extensions.Configuration;
@ -9,6 +10,7 @@ namespace MalwareMultiScan.Shared.Extensions
/// <summary>
/// Extensions for IServiceCollection.
/// </summary>
[ExcludeFromCodeCoverage]
public static class ServiceCollectionExtensions
{
/// <summary>

2
MalwareMultiScan.Tests/Api/AttributesTests.cs
View File

@ -36,7 +36,7 @@ namespace MalwareMultiScan.Tests.Api
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["MaxFileSize"] = "100"
["FILE_SIZE_LIMIT"] = "100"
};
var context = new ValidationContext(

78
MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs
View File

@ -1,78 +0,0 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Api
{
public class ReceiverHostedServiceTests
{
private Mock<IBus> _busMock;
private IReceiverHostedService _receiverHostedService;
private Mock<IScanResultService> _scanResultServiceMock;
[SetUp]
public void SetUp()
{
_busMock = new Mock<IBus>();
_busMock
.Setup(x => x.Receive("mms.results", It.IsAny<Func<ScanResultMessage, Task>>()))
.Callback<string, Func<ScanResultMessage, Task>>((s, func) =>
{
var task = func.Invoke(new ScanResultMessage
{
Id = "test",
Backend = "dummy",
Duration = 100,
Succeeded = true,
Threats = new[] {"Test"}
});
task.Wait();
});
_scanResultServiceMock = new Mock<IScanResultService>();
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
var httpClientFactoryMock = new Mock<IHttpClientFactory>();
_receiverHostedService = new ReceiverHostedService(
_busMock.Object,
configuration,
_scanResultServiceMock.Object,
Mock.Of<ILogger<ReceiverHostedService>>(),
httpClientFactoryMock.Object);
}
[Test]
public async Task TestBusReceiveScanResultMessage()
{
await _receiverHostedService.StartAsync(default);
_scanResultServiceMock.Verify(x => x.UpdateScanResultForBackend(
"test", "dummy", 100, true, true, new[] {"Test"}), Times.Once);
}
[Test]
public async Task TestBusIsDisposedOnStop()
{
await _receiverHostedService.StopAsync(default);
_busMock.Verify(x => x.Dispose(), Times.Once);
}
}
}

59
MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs
View File

@ -1,59 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Data.Configuration;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Api
{
public class ScanBackendServiceTests
{
private Mock<IBus> _busMock;
private IScanBackendService _scanBackendService;
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["BackendsConfiguration"] = "backends.yaml"
};
_busMock = new Mock<IBus>();
_scanBackendService = new ScanBackendService(
configuration,
_busMock.Object,
Mock.Of<ILogger<ScanBackendService>>());
}
[Test]
public void TestBackendsAreNotEmpty()
{
Assert.IsNotEmpty(_scanBackendService.List);
}
[Test]
public async Task TestQueueUrlScan()
{
await _scanBackendService.QueueUrlScan(
new ScanResult {Id = "test"},
new ScanBackend {Id = "dummy"},
"http://test.com"
);
_busMock.Verify(x => x.SendAsync(
"dummy", It.Is<ScanRequestMessage>(r =>
r.Id == "test" &&
r.Uri == new Uri("http://test.com"))), Times.Once);
}
}
}

49
MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs
View File

@ -1,9 +1,11 @@
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Data.Configuration;
using Consul;
using Hangfire;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Enums;
using MalwareMultiScan.Shared.Message;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
@ -17,7 +19,6 @@ namespace MalwareMultiScan.Tests.Api
{
private MongoDbRunner _mongoDbRunner;
private IScanResultService _resultService;
private Mock<IScanBackendService> _scanBackendService;
[SetUp]
public void SetUp()
@ -27,20 +28,11 @@ namespace MalwareMultiScan.Tests.Api
var connection = new MongoClient(_mongoDbRunner.ConnectionString);
var database = connection.GetDatabase("Test");
var gridFsBucket = new GridFSBucket(database);
_scanBackendService = new Mock<IScanBackendService>();
_scanBackendService
.SetupGet(x => x.List)
.Returns(() => new[]
{
new ScanBackend {Id = "dummy", Enabled = true},
new ScanBackend {Id = "clamav", Enabled = true},
new ScanBackend {Id = "disabled", Enabled = false}
});
_resultService = new ScanResultService(
database, gridFsBucket, _scanBackendService.Object);
database, gridFsBucket,
Mock.Of<IConsulClient>(),
Mock.Of<IBackgroundJobClient>());
}
[TearDown]
@ -85,10 +77,14 @@ namespace MalwareMultiScan.Tests.Api
var result = await _resultService.CreateScanResult(null);
Assert.NotNull(result.Id);
Assert.That(result.Results, Contains.Key("dummy"));
await _resultService.UpdateScanResultForBackend(
result.Id, "dummy", 100, true, true, new[] {"Test"});
result.Id, "dummy", new ScanResultMessage
{
Duration = 100,
Status = ScanResultStatus.Succeeded,
Threats = new []{"Test"}
});
result = await _resultService.GetScanResult(result.Id);
@ -97,25 +93,10 @@ namespace MalwareMultiScan.Tests.Api
var dummyResult = result.Results["dummy"];
Assert.IsTrue(dummyResult.Completed);
Assert.IsTrue(dummyResult.Succeeded);
Assert.IsTrue(dummyResult.Status == ScanResultStatus.Succeeded);
Assert.AreEqual(dummyResult.Duration, 100);
Assert.IsNotEmpty(dummyResult.Threats);
Assert.That(dummyResult.Threats, Is.EquivalentTo(new[] {"Test"}));
}
[Test]
public async Task TestQueueUrlScan()
{
var result = await _resultService.CreateScanResult(null);
await _resultService.QueueUrlScan(
result, "http://url.com");
_scanBackendService.Verify(x => x.QueueUrlScan(
It.IsAny<ScanResult>(),
It.IsAny<ScanBackend>(),
"http://url.com"), Times.Exactly(2));
}
}
}

1
MalwareMultiScan.Tests/Backends/BackendsTests.cs
View File

@ -1,6 +1,7 @@
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Backends;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Backends.Services.Interfaces;
using Moq;
using NUnit.Framework;

62
MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs
View File

@ -1,62 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Scanner
{
public class ScanBackgroundJobTests
{
private Mock<IBus> _busMock;
private Mock<IScanBackend> _scanBackendMock;
private IScanBackgroundJob _scanBackgroundJob;
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
_busMock = new Mock<IBus>();
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
_scanBackendMock
.Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new[] {"Test"}));
_scanBackgroundJob = new ScanBackgroundJob(configuration, _busMock.Object,
Mock.Of<ILogger<ScanBackgroundJob>>(), _scanBackendMock.Object);
}
[Test]
public async Task TestMessageProcessing()
{
await _scanBackgroundJob.Process(new ScanRequestMessage
{
Id = "test",
Uri = new Uri("http://test.com")
});
_scanBackendMock.Verify(
x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()));
_busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(m =>
m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test"))));
}
}
}

49
MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs
View File

@ -1,49 +0,0 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Scanner
{
public class ScanHostedServiceTests
{
private Mock<IBus> _busMock;
private Mock<IScanBackend> _scanBackendMock;
private IScanHostedService _scanHostedService;
[SetUp]
public void SetUp()
{
_busMock = new Mock<IBus>();
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
_scanHostedService = new ScanHostedService(
Mock.Of<ILogger<ScanHostedService>>(),
_scanBackendMock.Object,
_busMock.Object);
}
[Test]
public async Task TestBusReceiveScanResultMessage()
{
await _scanHostedService.StartAsync(default);
_busMock.Verify(
x => x.Receive("dummy", It.IsAny<Action<ScanRequestMessage>>()));
}
[Test]
public async Task TestBusIsDisposedOnStop()
{
await _scanHostedService.StopAsync(default);
_busMock.Verify(x => x.Dispose(), Times.Once);
}
}
}

2
MalwareMultiScan.Ui/package.json
View File

@ -1,6 +1,6 @@
{
"name": "malware-multi-scan-ui",
"version": "1.0.2",
"version": "1.5.0",
"private": true,
"scripts": {
"dev": "nuxt-ts",

72
README.md
View File

@ -10,6 +10,8 @@ Self-hosted [VirusTotal](https://www.virustotal.com/) wannabe API for scanning U
The demo is running on a cheap Vultr node, so it might get slow or unavailable occasionally.
**IMPORTANT**: version 1.5 introduces breaking changes in containers configuration and docker-compose.yaml layout. Please see [releases](https://github.com/mindcollapse/MalwareMultiScan/releases) page and changelog of [docker-compose.yaml](https://github.com/mindcollapse/MalwareMultiScan/commits/master/docker-compose.yaml) and [README.md](https://github.com/mindcollapse/MalwareMultiScan/commits/master/README.md) for the additional details.
## Introduction
I faced a need to scan user-uploaded files in one of my work projects in an automated mode to ensure they don't contain any malware. Using VirusTotal was not an option because of a) legal restrictions and data residency limitations b) scanning by hash-sums would not be sufficient because the majority of files are generated / modified by users.
@ -36,29 +38,27 @@ Configuration of API and Scanners is performed by passing the environment variab
#### MalwareMultiScan.Api
* `ConnectionStrings__Mongo=mongodb://localhost:27017` - MongoDB connection string.
* `MONGO_ADDRESS=mongodb://localhost:27017` - MongoDB connection string.
* `DatabaseName=MalwareMultiScan` - MongoDB collection name.
* `MONGO_DATABASE=MalwareMultiScan` - MongoDB collection name.
* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
* `MaxFileSize=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified.
* `BackendsConfiguration=backends.yaml` - Path to the [backends.yaml](MalwareMultiScan.Api/backends.yaml) file.
* `FILE_SIZE_LIMIT=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified. Set to 0 to disable the validation.
#### MalwareMultiScan.Scanner
* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
* `BACKEND_ID=dummy` - Id of a backend.
* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
* `BackendType=Dummy` - A type of scanner backend used by the running instance. Should correspond to the [BackendType](MalwareMultiScan.Backends/Enums/BackendType.cs) enum.
* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
* `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
* `MAX_SCANNING_TIME=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
* `WorkerCount=4` - Number of workers for parallel scanning.
* `WORKER_COUNT=4` - Number of workers for parallel scanning.
#### MalwareMultiScan.Ui
@ -66,13 +66,25 @@ Configuration of API and Scanners is performed by passing the environment variab
### API Endpoints
* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error.
* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `404 Not Found` error.
Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) in a body on every update from scan backends.
#### Callback URL
Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResultMessage](MalwareMultiScan.Shared/Message/ScanResultMessage.cs) in a body on every update from scan backends. Query string will contain `id` parameter that corresponds to the id of the scan result and `backend` parameter with the id of backend which completed the scan.
I.e. when you define `callbackUrl=http://localhost:1234/scan-results`, the POST request will be made to `http://localhost:1234/scan-results?id=123&backend=dummy` with a body
```json
{
"Status":1,
"Duration":5,
"Threats":["Malware.Dummy.Result"]
}
```
## Supported Scan Engines
@ -91,11 +103,29 @@ More scan backends can be added in the future. Some of the popular ones do not h
## Components
### Workflow
1. On startup all [Scanners](MalwareMultiScan.Scanner) register themselves in [Consul](https://www.consul.io/) with a service name equal to `scanner` and the `BackendId` metadata field equal to the value of `BACKEND_ID` environment variable. They also register a TTL check and listen for [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
2. Third-party client triggers `/api/queue/url` or `/api/queue/file` of the [MalwareMultiScan.Api](MalwareMultiScan.Api).
3. [MalwareMultiScan.Api](MalwareMultiScan.Api) sends a query to [Consul](https://www.consul.io/) and receives the list of alive scan backends with the service name `scanner`.
4. [MalwareMultiScan.Api](MalwareMultiScan.Api) schedules a [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
5. [Scanners](MalwareMultiScan.Scanner) picks up a job from queue, starts the scan and sends result back to the `default` queue of [Hangfire](https://www.hangfire.io/).
6. [MalwareMultiScan.Api](MalwareMultiScan.Api) picks a job from the default` queue of [Hangfire](https://www.hangfire.io/) and updates the state of the scan.
7. If callback URL was specified during the step #2, [MalwareMultiScan.Api](MalwareMultiScan.Api) triggers a HTTP POST request to the specified URL. See [Callback URL](#callback-url) for details.
### Prerequisites
* **MongoDB** of version 3.x. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
* **MongoDB** of version 3.x or above. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
* **RabbitMQ** of version 3.x. Used for IPC and scan tasks queueing. The communication is happening through the [EasyNetQ](https://github.com/EasyNetQ/EasyNetQ) library.
* **Redis** of version 5.x or above. Used for tasks queueing. The communication is happening through the [Hangfire](https://www.hangfire.io/) library.
* **Consul** of version 1.8.x or above. Used for service registration of scan backends.
* **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments.
@ -103,9 +133,11 @@ More scan backends can be added in the future. Some of the popular ones do not h
### Parts
* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile). Configuration of available backends is performed via the [backends.yaml](MalwareMultiScan.Api/backends.yaml) location passed via the `BackendsConfiguration` environment variable.
* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile).
* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Shared components between API and Worker. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Scan backends logic. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
* [MalwareMultiScan.Shared](MalwareMultiScan.Shared) - Shared components.
* [MalwareMultiScan.Scanner](MalwareMultiScan.Scanner) - .NET Core Worker service subscribes to messages corresponding to the backend id, then fires up scanning command-line utility, and parses the output. See [Dockerfile](MalwareMultiScan.Scanner/Dockerfile). The image of MalwareMultiScan.Scanner acts as a base image for the rest of the scan backends. Check Dockerfiles from the [table above](#supported-scan-engines) for details.

66
docker-compose.yaml
View File

@ -1,23 +1,24 @@
version: "3.8"
services:
rabbitmq:
image: rabbitmq:3
redis:
image: redis:6
restart: on-failure
expose:
- "5672"
volumes:
- rabbitmq_etc/:/etc/rabbitmq/
- rabbitmq_data:/var/lib/rabbitmq/
- rabbitmq_logs/:/var/log/rabbitmq/
- redis:/data
mongodb:
image: mongo:4
restart: on-failure
expose:
- "27019"
volumes:
- mongodb:/data
consul:
image: consul:1.8
restart: on-failure
volumes:
- consul:/consul/data
command: consul agent -server -bootstrap -client 0.0.0.0
ui:
image: mindcollapse/malware-multi-scan-ui
@ -39,17 +40,14 @@ services:
api:
image: mindcollapse/malware-multi-scan-api
restart: on-failure
expose:
- "5000"
depends_on:
- rabbitmq
- consul
- redis
- mongodb
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;timeout=120"
- "ConnectionStrings__Mongo=mongodb://mongodb:27017?connectTimeoutMS=120000"
- "BackendsConfiguration=/etc/backends.yaml"
volumes:
- "./MalwareMultiScan.Api/backends.yaml:/etc/backends.yaml:ro"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
- "MONGO_ADDRESS=mongodb://mongodb:27017?connectTimeoutMS=120000"
build:
context: .
dockerfile: MalwareMultiScan.Api/Dockerfile
@ -58,9 +56,11 @@ services:
image: mindcollapse/malware-multi-scan-scanner
restart: on-failure
depends_on:
- rabbitmq
- consul
- redis
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: .
dockerfile: MalwareMultiScan.Scanner/Dockerfile
@ -71,7 +71,8 @@ services:
depends_on:
- dummy-scanner
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: MalwareMultiScan.Backends/Dockerfiles
dockerfile: Clamav.Dockerfile
@ -82,7 +83,8 @@ services:
depends_on:
- dummy-scanner
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: MalwareMultiScan.Backends/Dockerfiles
dockerfile: WindowsDefender.Dockerfile
@ -95,7 +97,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: Comodo.Dockerfile
@ -106,7 +109,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: DrWeb.Dockerfile
@ -117,7 +121,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: KES.Dockerfile
@ -128,7 +133,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: McAfee.Dockerfile
@ -139,13 +145,13 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: Sophos.Dockerfile
volumes:
mongodb:
rabbitmq_etc:
rabbitmq_data:
rabbitmq_logs:
redis:
consul: