Mirrors/MalwareMultiScan
1
0
Fork 0
mirror of https://github.com/volodymyrsmirnov/MalwareMultiScan.git synced 2025-08-23 21:12:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23 from mindcollapse/consul

Browse Source 
Version 1.5
This commit is contained in:
Volodymyr Smirnov 2020-11-02 11:29:04 +02:00 committed by GitHub
commit b4c5c1a507
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
85 changed files with 755 additions and 1159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Clamav.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-clamav" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-clamav" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Comodo_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Comodo.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-comodo" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-comodo" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_DrWeb_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/DrWeb.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-drweb" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-drweb" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_KES_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/KES.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-kes" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-kes" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_McAfee_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/McAfee.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-mcafee" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-mcafee" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Sophos_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/Sophos.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-sophos" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-sophos" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

16
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_WindowsDefender_Dockerfile.xml generated
View File

@ -1,16 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/WindowsDefender.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-windows-defender" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-windows-defender" />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

7
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml generated
View File

@ -1,7 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dummy API" type="CompoundRunConfigurationType">
<toRun name="MalwareMultiScan.Scanner" type="DotNetProject" />
<toRun name="MalwareMultiScan.Api" type="DotNetProject" />
<method v="2" />
</configuration>
</component>

18
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml generated
View File

@ -1,18 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="MalwareMultiScan.Scanner/Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-scanner" />
<option name="buildCliOptions" value="" />
<option name="buildOnly" value="true" />
<option name="command" value="" />
<option name="containerName" value="" />
<option name="contextFolderPath" value="." />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Scanner/Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

5
MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs
View File

@ -15,7 +15,10 @@ namespace MalwareMultiScan.Api.Attributes
{
var maxSize = validationContext
.GetRequiredService<IConfiguration>()
.GetValue<long>("MaxFileSize");
.GetValue<long>("FILE_SIZE_LIMIT");
if (maxSize == 0)
return ValidationResult.Success;
var formFile = (IFormFile) value;

2
MalwareMultiScan.Api/Controllers/QueueController.cs
View File

@ -2,7 +2,7 @@ using System;
using System.ComponentModel.DataAnnotations;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Attributes;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

2
MalwareMultiScan.Api/Controllers/ScanResultsController.cs
View File

@ -1,5 +1,5 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

18
MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs
View File

@ -1,18 +0,0 @@
namespace MalwareMultiScan.Api.Data.Configuration
{
/// <summary>
/// Scan backend.
/// </summary>
public class ScanBackend
{
/// <summary>
/// Backend id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend state.
/// </summary>
public bool Enabled { get; set; }
}
}

28
MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs
View File

@ -1,28 +0,0 @@
namespace MalwareMultiScan.Api.Data.Models
{
/// <summary>
/// Scan result entry.
/// </summary>
public class ScanResultEntry
{
/// <summary>
/// Completion status.
/// </summary>
public bool Completed { get; set; }
/// <summary>
/// Indicates that scanning completed without error.
/// </summary>
public bool? Succeeded { get; set; }
/// <summary>
/// Scanning duration in seconds.
/// </summary>
public long Duration { get; set; }
/// <summary>
/// Detected names of threats.
/// </summary>
public string[] Threats { get; set; }
}
}

9
MalwareMultiScan.Api/Data/Models/ScanResult.cs → MalwareMultiScan.Api/Data/ScanResult.cs
View File

@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using MalwareMultiScan.Shared.Message;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace MalwareMultiScan.Api.Data.Models
namespace MalwareMultiScan.Api.Data
{
/// <summary>
/// Scan result.
@ -23,9 +24,9 @@ namespace MalwareMultiScan.Api.Data.Models
public Uri CallbackUrl { get; set; }
/// <summary>
/// Result entries where key is backend id and value is <see cref="ScanResultEntry" />.
/// Result entries where key is backend id and value is <see cref="ScanResultMessage" />.
/// </summary>
public Dictionary<string, ScanResultEntry> Results { get; set; } =
new Dictionary<string, ScanResultEntry>();
public Dictionary<string, ScanResultMessage> Results { get; set; } =
new Dictionary<string, ScanResultMessage>();
}
}

1
MalwareMultiScan.Api/Dockerfile
View File

@ -4,6 +4,7 @@ WORKDIR /src
COPY MalwareMultiScan.Api /src/MalwareMultiScan.Api
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Api/MalwareMultiScan.Api.csproj

4
MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs
View File

@ -14,8 +14,8 @@ namespace MalwareMultiScan.Api.Extensions
{
internal static void AddMongoDb(this IServiceCollection services, IConfiguration configuration)
{
var client = new MongoClient(configuration.GetConnectionString("Mongo"));
var db = client.GetDatabase(configuration.GetValue<string>("DatabaseName"));
var client = new MongoClient(configuration.GetValue<string>("MONGO_ADDRESS"));
var db = client.GetDatabase(configuration.GetValue<string>("MONGO_DATABASE"));
services.AddSingleton(client);
services.AddSingleton(db);

19
MalwareMultiScan.Api/MalwareMultiScan.Api.csproj
View File

@ -4,18 +4,14 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Api</Product>
<AssemblyVersion>1.0.1</AssemblyVersion>
<FileVersion>1.0.1</FileVersion>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<None Update="backends.yaml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
<PackageReference Include="MongoDB.Driver" Version="2.11.3" />
<PackageReference Include="MongoDB.Driver.GridFS" Version="2.11.3" />
<PackageReference Include="YamlDotNet" Version="8.1.2" />
@ -24,4 +20,11 @@
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
</ItemGroup>
<ItemGroup>
<Content Update="Properties\launchSettings.json">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</Content>
</ItemGroup>
</Project>

21
MalwareMultiScan.Api/Program.cs
View File

@ -1,4 +1,5 @@
using EasyNetQ.LightInject;
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
@ -7,15 +8,19 @@ namespace MalwareMultiScan.Api
[ExcludeFromCodeCoverage]
internal static class Program
{
public static void Main(string[] args)
public static async Task Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
await Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(builder =>
{
builder.ConfigureKestrel(
options => options.Limits.MaxRequestBodySize = long.MaxValue);
private static IHostBuilder CreateHostBuilder(string[] args)
{
return Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(webBuilder => webBuilder.UseStartup<Startup>());
builder.UseStartup<Startup>();
})
.UseConsoleLifetime()
.Build()
.RunAsync();
}
}
}

8
MalwareMultiScan.Api/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"MalwareMultiScan.Api": {
"commandName": "Project"
}
}
}

103
MalwareMultiScan.Api/Services/Implementations/ReceiverHostedService.cs
View File

@ -1,103 +0,0 @@
using System;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace MalwareMultiScan.Api.Services.Implementations
{
/// <inheritdoc />
public class ReceiverHostedService : IReceiverHostedService
{
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ReceiverHostedService> _logger;
private readonly IScanResultService _scanResultService;
/// <summary>
/// Initialize receiver hosted service.
/// </summary>
/// <param name="bus">EasyNetQ bus.</param>
/// <param name="configuration">Configuration.</param>
/// <param name="scanResultService">Scan result service.</param>
/// <param name="logger">Logger.</param>
/// <param name="httpClientFactory">HTTP client factory.</param>
public ReceiverHostedService(IBus bus, IConfiguration configuration, IScanResultService scanResultService,
ILogger<ReceiverHostedService> logger, IHttpClientFactory httpClientFactory)
{
_bus = bus;
_configuration = configuration;
_scanResultService = scanResultService;
_logger = logger;
_httpClientFactory = httpClientFactory;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanResultMessage>(
_configuration.GetValue<string>("ResultsSubscriptionId"), StoreScanResult);
_logger.LogInformation(
"Started hosted service for receiving scan results");
return Task.CompletedTask;
}
/// <inheritdoc />
public Task StopAsync(CancellationToken cancellationToken)
{
_bus?.Dispose();
_logger.LogInformation(
"Stopped hosted service for receiving scan results");
return Task.CompletedTask;
}
private async Task StoreScanResult(ScanResultMessage message)
{
message.Threats ??= new string[] { };
_logger.LogInformation(
$"Received a result from {message.Backend} for {message.Id} " +
$"with threats {string.Join(",", message.Threats)}");
await _scanResultService.UpdateScanResultForBackend(
message.Id, message.Backend, message.Duration, true,
message.Succeeded, message.Threats);
var result = await _scanResultService.GetScanResult(message.Id);
if (result?.CallbackUrl == null)
return;
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(3));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var response = await httpClient.PostAsync(
result.CallbackUrl,
new StringContent(JsonConvert.SerializeObject(result), Encoding.UTF8, "application/json"),
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed to POST to callback URL {result.CallbackUrl}");
}
}
}
}

64
MalwareMultiScan.Api/Services/Implementations/ScanBackendService.cs
View File

@ -1,64 +0,0 @@
using System;
using System.IO;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace MalwareMultiScan.Api.Services.Implementations
{
/// <inheritdoc />
public class ScanBackendService : IScanBackendService
{
private readonly IBus _bus;
private readonly ILogger<ScanBackendService> _logger;
/// <summary>
/// Initialise scan backend service.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="bus">EasyNetQ bus.</param>
/// <param name="logger">Logger.</param>
/// <exception cref="FileNotFoundException">Missing backends.yaml configuration.</exception>
public ScanBackendService(IConfiguration configuration, IBus bus, ILogger<ScanBackendService> logger)
{
_bus = bus;
_logger = logger;
var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
if (!File.Exists(configurationPath))
throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
var configurationContent = File.ReadAllText(configurationPath);
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
List = deserializer.Deserialize<ScanBackend[]>(configurationContent);
}
/// <inheritdoc />
public ScanBackend[] List { get; }
/// <inheritdoc />
public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl)
{
_logger.LogInformation(
$"Queueing scan for {result.Id} on {backend.Id} at {fileUrl}");
await _bus.SendAsync(backend.Id, new ScanRequestMessage
{
Id = result.Id,
Uri = new Uri(fileUrl)
});
}
}
}

11
MalwareMultiScan.Api/Services/Interfaces/IReceiverHostedService.cs
View File

@ -1,11 +0,0 @@
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api.Services.Interfaces
{
/// <summary>
/// Receiver hosted service.
/// </summary>
public interface IReceiverHostedService : IHostedService
{
}
}

25
MalwareMultiScan.Api/Services/Interfaces/IScanBackendService.cs
View File

@ -1,25 +0,0 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
namespace MalwareMultiScan.Api.Services.Interfaces
{
/// <summary>
/// Scan backend service.
/// </summary>
public interface IScanBackendService
{
/// <summary>
/// Get list of parsed backends.
/// </summary>
ScanBackend[] List { get; }
/// <summary>
/// Queue URL for scan.
/// </summary>
/// <param name="result">Result entry.</param>
/// <param name="backend">Backend entry.</param>
/// <param name="fileUrl">Remote URL.</param>
Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl);
}
}

11
MalwareMultiScan.Api/Services/Interfaces/IScanResultService.cs
View File

@ -1,7 +1,8 @@
using System;
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Api.Services.Interfaces
{
@ -29,12 +30,8 @@ namespace MalwareMultiScan.Api.Services.Interfaces
/// </summary>
/// <param name="resultId">Result id.</param>
/// <param name="backendId">Backend id.</param>
/// <param name="duration">Duration.</param>
/// <param name="completed">Completion status.</param>
/// <param name="succeeded">Indicates that scanning completed without error.</param>
/// <param name="threats">Detected names of threats.</param>
Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
bool completed = false, bool succeeded = false, string[] threats = null);
/// <param name="result">Scan result.</param>
Task UpdateScanResultForBackend(string resultId, string backendId, ScanResultMessage result = null);
/// <summary>
/// Queue URL for scanning.

94
MalwareMultiScan.Api/Services/ScanResultJob.cs Normal file
View File

@ -0,0 +1,94 @@
using System;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Api.Services
{
/// <summary>
/// Job for storing scan results.
/// </summary>
public class ScanResultJob : IScanResultJob
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ScanResultJob> _logger;
private readonly IScanResultService _scanResultService;
/// <summary>
/// Initialize scan result storing job.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="httpClientFactory">HTTP client factory.</param>
/// <param name="scanResultService">Scan result service.</param>
/// <param name="backgroundJobClient">Background job client.</param>
public ScanResultJob(
ILogger<ScanResultJob> logger,
IHttpClientFactory httpClientFactory,
IScanResultService scanResultService,
IBackgroundJobClient backgroundJobClient)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
_scanResultService = scanResultService;
_backgroundJobClient = backgroundJobClient;
}
/// <inheritdoc />
public async Task Report(string resultId, string backendId, ScanResultMessage result)
{
_logger.LogInformation(
$"Received a result from {backendId} for {result} with status {result.Status} " +
$"and threats {string.Join(",", result.Threats)}");
await _scanResultService.UpdateScanResultForBackend(resultId, backendId, result);
var scanResult = await _scanResultService.GetScanResult(resultId);
if (scanResult?.CallbackUrl == null)
return;
_backgroundJobClient.Create<IScanResultJob>(
x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result),
new EnqueuedState("default"));
}
/// <inheritdoc />
public async Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(5));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var builder = new UriBuilder(uri)
{
Query = $"?id={resultId}" +
$"&backend={backendId}"
};
var response = await httpClient.PostAsync(builder.Uri,
new StringContent(JsonSerializer.Serialize(result), Encoding.UTF8, "application/json"),
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
_logger.LogInformation($"Sent POST to callback URL {uri}");
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed to POST to callback URL {uri}");
}
}
}
}

71
MalwareMultiScan.Api/Services/Implementations/ScanResultService.cs → MalwareMultiScan.Api/Services/ScanResultService.cs
View File

@ -2,33 +2,46 @@ using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using Consul;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Enums;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace MalwareMultiScan.Api.Services.Implementations
namespace MalwareMultiScan.Api.Services
{
/// <inheritdoc />
public class ScanResultService : IScanResultService
{
private const string CollectionName = "ScanResults";
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IGridFSBucket _bucket;
private readonly IMongoCollection<ScanResult> _collection;
private readonly IScanBackendService _scanBackendService;
private readonly IConsulClient _consulClient;
/// <summary>
/// Initialize scan result service.
/// </summary>
/// <param name="db">Mongo database.</param>
/// <param name="bucket">GridFS bucket.</param>
/// <param name="scanBackendService">Scan backend service.</param>
public ScanResultService(IMongoDatabase db, IGridFSBucket bucket, IScanBackendService scanBackendService)
/// <param name="consulClient">Consul client.</param>
/// <param name="backgroundJobClient">Background job client.</param>
public ScanResultService(
IMongoDatabase db,
IGridFSBucket bucket,
IConsulClient consulClient,
IBackgroundJobClient backgroundJobClient)
{
_bucket = bucket;
_scanBackendService = scanBackendService;
_consulClient = consulClient;
_backgroundJobClient = backgroundJobClient;
_collection = db.GetCollection<ScanResult>(CollectionName);
}
@ -38,11 +51,7 @@ namespace MalwareMultiScan.Api.Services.Implementations
{
var scanResult = new ScanResult
{
CallbackUrl = callbackUrl,
Results = _scanBackendService.List
.Where(b => b.Enabled)
.ToDictionary(k => k.Id, v => new ScanResultEntry())
CallbackUrl = callbackUrl
};
await _collection.InsertOneAsync(scanResult);
@ -60,25 +69,43 @@ namespace MalwareMultiScan.Api.Services.Implementations
}
/// <inheritdoc />
public async Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
bool completed = false, bool succeeded = false, string[] threats = null)
public async Task UpdateScanResultForBackend(string resultId, string backendId,
ScanResultMessage result = null)
{
result ??= new ScanResultMessage
{
Status = ScanResultStatus.Queued
};
await _collection.UpdateOneAsync(
Builders<ScanResult>.Filter.Where(r => r.Id == resultId),
Builders<ScanResult>.Update.Set(r => r.Results[backendId], new ScanResultEntry
{
Completed = completed,
Succeeded = succeeded,
Duration = duration,
Threats = threats ?? new string[] { }
}));
Builders<ScanResult>.Update.Set(r => r.Results[backendId], result));
}
/// <inheritdoc />
public async Task QueueUrlScan(ScanResult result, string fileUrl)
{
foreach (var backend in _scanBackendService.List.Where(b => b.Enabled))
await _scanBackendService.QueueUrlScan(result, backend, fileUrl);
var message = new ScanQueueMessage
{
Id = result.Id,
Uri = new Uri(fileUrl)
};
var scanners = await _consulClient.Health.Service("scanner", null, true);
var backends = scanners.Response
.Select(s => s.Service.Meta.TryGetValue("BackendId", out var backendId) ? backendId : null)
.Where(q => q != null)
.Distinct()
.ToArray();
foreach (var backend in backends)
{
await UpdateScanResultForBackend(result.Id, backend);
_backgroundJobClient.Create<IScanBackgroundJob>(
j => j.Process(message), new EnqueuedState(backend));
}
}
/// <inheritdoc />

33
MalwareMultiScan.Api/Startup.cs
View File

@ -1,12 +1,14 @@
using System.Diagnostics.CodeAnalysis;
using Hangfire;
using MalwareMultiScan.Api.Extensions;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Shared.Extensions;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api
{
@ -24,29 +26,26 @@ namespace MalwareMultiScan.Api
{
services.AddDockerForwardedHeadersOptions();
services.Configure<KestrelServerOptions>(options =>
{
options.Limits.MaxRequestBodySize = _configuration.GetValue<int>("MaxFileSize");
});
services.AddConsul(_configuration);
services.AddMongoDb(_configuration);
services.AddRabbitMq(_configuration);
services.AddSingleton<IScanBackendService, ScanBackendService>();
services.AddHangfire(_configuration);
services.AddSingleton<IScanResultService, ScanResultService>();
services.AddSingleton<IScanResultJob, ScanResultJob>();
services.AddControllers();
services.AddHostedService<ReceiverHostedService>();
services.AddHttpClient();
}
public void Configure(IApplicationBuilder app)
public void Configure(IApplicationBuilder app, IHostEnvironment hostEnvironment)
{
app.UseRouting();
app.UseForwardedHeaders();
app.UseEndpoints(endpoints => endpoints.MapControllers());
app.UseForwardedHeaders();
if (hostEnvironment.IsDevelopment())
app.UseHangfireDashboard();
}
}
}

17
MalwareMultiScan.Api/appsettings.json
View File

@ -7,15 +7,12 @@
}
},
"AllowedHosts": "*",
"ConnectionStrings": {
"Mongo": "mongodb://localhost:27017",
"RabbitMQ": "host=localhost"
},
"MONGO_ADDRESS": "mongodb://localhost:27017",
"MONGO_DATABASE": "MalwareMultiScan",
"DatabaseName": "MalwareMultiScan",
"ResultsSubscriptionId": "mms.results",
"MaxFileSize": 52428800,
"BackendsConfiguration": "backends.yaml"
"REDIS_ADDRESS": "localhost:6379",
"CONSUL_ADDRESS": "http://localhost:8500",
"FILE_SIZE_LIMIT": 52428800
}

23
MalwareMultiScan.Api/backends.yaml
View File

@ -1,23 +0,0 @@
- id: dummy
enabled: true
- id: clamav
enabled: true
- id: windows-defender
enabled: true
- id: comodo
enabled: false
- id: drweb
enabled: false
- id: kes
enabled: false
- id: mcafee
enabled: false
- id: sophos
enabled: false

2
MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs
View File

@ -4,7 +4,7 @@ using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Abstracts
{

2
MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs → MalwareMultiScan.Backends/Backends/ClamavScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class ClamavScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs → MalwareMultiScan.Backends/Backends/ComodoScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class ComodoScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs → MalwareMultiScan.Backends/Backends/DrWebScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class DrWebScanBackend : AbstractLocalProcessScanBackend

4
MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs → MalwareMultiScan.Backends/Backends/DummyScanBackend.cs
View File

@ -2,9 +2,9 @@ using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class DummyScanBackend : IScanBackend

2
MalwareMultiScan.Backends/Interfaces/IScanBackend.cs → MalwareMultiScan.Backends/Backends/Interfaces/IScanBackend.cs
View File

@ -3,7 +3,7 @@ using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace MalwareMultiScan.Backends.Interfaces
namespace MalwareMultiScan.Backends.Backends.Interfaces
{
/// <summary>
/// Scan backend.

2
MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs → MalwareMultiScan.Backends/Backends/KesScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class KesScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/McAfeeScanBackend.cs → MalwareMultiScan.Backends/Backends/McAfeeScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class McAfeeScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs → MalwareMultiScan.Backends/Backends/SophosScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class SophosScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs → MalwareMultiScan.Backends/Backends/WindowsDefenderScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class WindowsDefenderScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile
View File

@ -5,6 +5,6 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y clamav clamav-daemon
RUN freshclam --quiet
ENV BackendType=Clamav
ENV BACKEND_ID=clamav
ENTRYPOINT /etc/init.d/clamav-daemon start && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile
View File

@ -7,4 +7,4 @@ RUN wget -q https://cdn.download.comodo.com/cis/download/installs/linux/cav-linu
RUN wget -q http://download.comodo.com/av/updates58/sigs/bases/bases.cav -O /opt/COMODO/scanners/bases.cav
ENV BackendType=Comodo
ENV BACKEND_ID=comodo

2
MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile
View File

@ -23,6 +23,6 @@ RUN /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && \
drweb-ctl update && \
kill $(cat /var/run/drweb-configd.pid)
ENV BackendType=DrWeb
ENV BACKEND_ID=drweb
ENTRYPOINT /opt/drweb.com/bin/drweb-configd -d -p /var/run/drweb-configd.pid && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile
View File

@ -30,6 +30,6 @@ kesl-control -B --query "FileName == \"$1\"" 2> /dev/null \n\
exit $? \
' > /usr/bin/kesl-scan && chmod +x /usr/bin/kesl-scan
ENV BackendType=Kes
ENV BACKEND_ID=kes
ENTRYPOINT /etc/init.d/kesl-supervisor start && /worker/MalwareMultiScan.Scanner

2
MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile
View File

@ -16,4 +16,4 @@ RUN wget -q -Nc -r -nd -l1 -A "avvepo????dat.zip" http://download.nai.com/produc
WORKDIR /worker
ENV BackendType=McAfee
ENV BACKEND_ID=mcafee

2
MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile
View File

@ -9,5 +9,5 @@ RUN wget -q $SOPHOS_URL -O /tmp/SophosInstall.sh && \
chmod +x /tmp/SophosInstall.sh && \
/tmp/SophosInstall.sh --automatic --acceptlicence || exit 0
ENV BackendType=Sophos
ENV BACKEND_ID=sophos

2
MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile
View File

@ -18,4 +18,4 @@ RUN apt-get update && apt-get install -y libc6-i386
COPY --from=backend /opt/loadlibrary/engine /opt/engine
COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient
ENV BackendType=Defender
ENV BACKEND_ID=windows-defender

48
MalwareMultiScan.Backends/Enums/BackendType.cs
View File

@ -1,48 +0,0 @@
namespace MalwareMultiScan.Backends.Enums
{
/// <summary>
/// Backend type.
/// </summary>
public enum BackendType
{
/// <summary>
/// Dummy
/// </summary>
Dummy,
/// <summary>
/// Windows Defender.
/// </summary>
Defender,
/// <summary>
/// ClamAV.
/// </summary>
Clamav,
/// <summary>
/// DrWeb.
/// </summary>
DrWeb,
/// <summary>
/// KES.
/// </summary>
Kes,
/// <summary>
/// Comodo.
/// </summary>
Comodo,
/// <summary>
/// Sophos.
/// </summary>
Sophos,
/// <summary>
/// McAfee.
/// </summary>
McAfee
}
}

45
MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs
View File

@ -1,9 +1,7 @@
using System;
using System.Diagnostics.CodeAnalysis;
using EasyNetQ;
using MalwareMultiScan.Backends.Backends.Implementations;
using MalwareMultiScan.Backends.Enums;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Backends.Services.Implementations;
using MalwareMultiScan.Backends.Services.Interfaces;
using Microsoft.Extensions.Configuration;
@ -17,55 +15,42 @@ namespace MalwareMultiScan.Backends.Extensions
[ExcludeFromCodeCoverage]
public static class ServiceCollectionExtensions
{
/// <summary>
/// Add RabbitMQ service.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
public static void AddRabbitMq(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton(x =>
RabbitHutch.CreateBus(configuration.GetConnectionString("RabbitMQ")));
}
/// <summary>
/// Add scanning backend.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
/// <exception cref="ArgumentOutOfRangeException">Unknown backend.</exception>
public static void AddScanningBackend(this IServiceCollection services, IConfiguration configuration)
public static void AddScanBackend(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton<IProcessRunner, ProcessRunner>();
switch (configuration.GetValue<BackendType>("BackendType"))
switch (configuration.GetValue<string>("BACKEND_ID"))
{
case BackendType.Dummy:
services.AddSingleton<IScanBackend, DummyScanBackend>();
break;
case BackendType.Defender:
services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
break;
case BackendType.Clamav:
case "clamav":
services.AddSingleton<IScanBackend, ClamavScanBackend>();
break;
case BackendType.DrWeb:
case "drweb":
services.AddSingleton<IScanBackend, DrWebScanBackend>();
break;
case BackendType.Kes:
case "kes":
services.AddSingleton<IScanBackend, KesScanBackend>();
break;
case BackendType.Comodo:
case "comodo":
services.AddSingleton<IScanBackend, ComodoScanBackend>();
break;
case BackendType.Sophos:
case "sophos":
services.AddSingleton<IScanBackend, SophosScanBackend>();
break;
case BackendType.McAfee:
case "mcafee":
services.AddSingleton<IScanBackend, McAfeeScanBackend>();
break;
case "windows-defender":
services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
break;
default:
throw new ArgumentOutOfRangeException();
services.AddSingleton<IScanBackend, DummyScanBackend>();
break;
}
}
}

10
MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj
View File

@ -4,17 +4,13 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Backends</Product>
<AssemblyVersion>1.0.0</AssemblyVersion>
<FileVersion>1.0.0</FileVersion>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="EasyNetQ" Version="5.6.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.9" />
<ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
</ItemGroup>
</Project>

33
MalwareMultiScan.Backends/Messages/ScanResultMessage.cs
View File

@ -1,33 +0,0 @@
namespace MalwareMultiScan.Backends.Messages
{
/// <summary>
/// Scan result message.
/// </summary>
public class ScanResultMessage
{
/// <summary>
/// Result id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend.
/// </summary>
public string Backend { get; set; }
/// <summary>
/// Status.
/// </summary>
public bool Succeeded { get; set; }
/// <summary>
/// List of detected threats.
/// </summary>
public string[] Threats { get; set; }
/// <summary>
/// Duration.
/// </summary>
public long Duration { get; set; }
}
}

0
MalwareMultiScan.Backends/Services/Implementations/ProcessRunner.cs → MalwareMultiScan.Backends/Services/ProcessRunner.cs
View File

1
MalwareMultiScan.Scanner/Dockerfile
View File

@ -4,6 +4,7 @@ WORKDIR /src
COPY MalwareMultiScan.Scanner /src/MalwareMultiScan.Scanner
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj

11
MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
View File

@ -1,20 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Scanner</Product>
<AssemblyVersion>1.0.1</AssemblyVersion>
<FileVersion>1.0.1</FileVersion>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="3.1.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
</ItemGroup>

43
MalwareMultiScan.Scanner/Program.cs
View File

@ -1,13 +1,15 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.MemoryStorage;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using MalwareMultiScan.Backends.Services.Implementations;
using MalwareMultiScan.Backends.Services.Interfaces;
using MalwareMultiScan.Scanner.Services;
using MalwareMultiScan.Shared.Extensions;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner
{
@ -17,29 +19,32 @@ namespace MalwareMultiScan.Scanner
public static async Task Main(string[] args)
{
await Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(configure =>
.ConfigureLogging((context, builder) =>
{
configure.AddJsonFile("appsettings.json");
configure.AddEnvironmentVariables();
builder.AddConsole();
builder.AddConfiguration(context.Configuration);
})
.ConfigureAppConfiguration(builder =>
{
builder.AddJsonFile("appsettings.json");
builder.AddEnvironmentVariables();
})
.ConfigureServices((context, services) =>
{
services.AddLogging();
services.AddConsul(context.Configuration);
services.AddScanBackend(context.Configuration);
services.AddRabbitMq(context.Configuration);
services.AddScanningBackend(context.Configuration);
services.AddHangfire(context.Configuration,
context.Configuration.GetValue<string>("BACKEND_ID"));
services.AddSingleton<IProcessRunner, ProcessRunner>();
services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
services.AddHostedService<ScanHostedService>();
services.AddHangfire(
configuration => configuration.UseMemoryStorage());
services.AddHangfireServer(options =>
{
options.WorkerCount = context.Configuration.GetValue<int>("WorkerCount");
});
}).RunConsoleAsync();
services.AddHostedService<ConsulHostedService>();
})
.UseConsoleLifetime()
.Build()
.RunAsync();
}
}
}

8
MalwareMultiScan.Scanner/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"MalwareMultiScan.Scanner": {
"commandName": "Project"
}
}
}

91
MalwareMultiScan.Scanner/Services/ConsulHostedService.cs Normal file
View File

@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Consul;
using MalwareMultiScan.Backends.Backends.Interfaces;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Scanner.Services
{
/// <summary>
/// Consul registration hosted service.
/// </summary>
public class ConsulHostedService : BackgroundService
{
private static readonly string ServiceId = Dns.GetHostName();
private static readonly string CheckId = $"ttl-{ServiceId}";
private readonly IConsulClient _consulClient;
private readonly AgentServiceRegistration _registration;
/// <summary>
/// Initialize consul hosted service.
/// </summary>
/// <param name="consulClient">Consul client.</param>
/// <param name="scanBackend">Scan backend.</param>
public ConsulHostedService(
IConsulClient consulClient,
IScanBackend scanBackend)
{
_consulClient = consulClient;
_registration = new AgentServiceRegistration
{
ID = ServiceId,
Name = "scanner",
Meta = new Dictionary<string, string>
{
{"BackendId", scanBackend.Id}
}
};
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
await _consulClient.Agent.PassTTL(
CheckId, string.Empty, stoppingToken);
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
}
}
/// <inheritdoc />
public override async Task StartAsync(CancellationToken cancellationToken)
{
await _consulClient.Agent.ServiceDeregister(
_registration.ID, cancellationToken);
await _consulClient.Agent.ServiceRegister(
_registration, cancellationToken);
await _consulClient.Agent.CheckRegister(new AgentCheckRegistration
{
ID = CheckId,
Name = "TTL",
ServiceID = ServiceId,
TTL = TimeSpan.FromSeconds(15),
DeregisterCriticalServiceAfter = TimeSpan.FromSeconds(60)
}, cancellationToken);
await base.StartAsync(cancellationToken);
}
/// <inheritdoc />
public override async Task StopAsync(CancellationToken cancellationToken)
{
await _consulClient.Agent.ServiceDeregister(
_registration.ID, cancellationToken);
await _consulClient.Agent.CheckDeregister(
CheckId, cancellationToken);
await base.StopAsync(cancellationToken);
}
}
}

57
MalwareMultiScan.Scanner/Services/Implementations/ScanHostedService.cs
View File

@ -1,57 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using Hangfire;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
{
/// <inheritdoc />
public class ScanHostedService : IScanHostedService
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly ILogger<ScanHostedService> _logger;
/// <summary>
/// Initialise scan hosted service.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
/// <param name="bus">EasyNetQ bus.</param>
public ScanHostedService(
ILogger<ScanHostedService> logger,
IScanBackend backend, IBus bus)
{
_logger = logger;
_bus = bus;
_backend = backend;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanRequestMessage>(_backend.Id, message =>
BackgroundJob.Enqueue<IScanBackgroundJob>(j => j.Process(message)));
_logger.LogInformation(
$"Started scan hosting service for the backend {_backend.Id}");
return Task.CompletedTask;
}
/// <inheritdoc />
public Task StopAsync(CancellationToken cancellationToken)
{
_bus.Dispose();
_logger.LogInformation(
$"Stopped scan hosting service for the backend {_backend.Id}");
return Task.CompletedTask;
}
}
}

17
MalwareMultiScan.Scanner/Services/Interfaces/IScanBackgroundJob.cs
View File

@ -1,17 +0,0 @@
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Messages;
namespace MalwareMultiScan.Scanner.Services.Interfaces
{
/// <summary>
/// Background scanning job.
/// </summary>
public interface IScanBackgroundJob
{
/// <summary>
/// Process scan request in the background worker.
/// </summary>
/// <param name="requestMessage">Request message.</param>
Task Process(ScanRequestMessage requestMessage);
}
}

11
MalwareMultiScan.Scanner/Services/Interfaces/IScanHostedService.cs
View File

@ -1,11 +0,0 @@
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Scanner.Services.Interfaces
{
/// <summary>
/// Scan hosted service.
/// </summary>
public interface IScanHostedService : IHostedService
{
}
}

61
MalwareMultiScan.Scanner/Services/Implementations/ScanBackgroundJob.cs → MalwareMultiScan.Scanner/Services/ScanBackgroundJob.cs
View File

@ -2,52 +2,58 @@ using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Shared.Enums;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
namespace MalwareMultiScan.Scanner.Services
{
/// <inheritdoc />
public class ScanBackgroundJob : IScanBackgroundJob
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly ILogger<ScanBackgroundJob> _logger;
private readonly IBackgroundJobClient _jobClient;
/// <summary>
/// Initialize scan background job.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="bus">Bus.</param>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
public ScanBackgroundJob(IConfiguration configuration, IBus bus,
ILogger<ScanBackgroundJob> logger, IScanBackend backend)
/// <param name="jobClient">Background job client.</param>
public ScanBackgroundJob(
IScanBackend backend,
IConfiguration configuration,
ILogger<ScanBackgroundJob> logger,
IBackgroundJobClient jobClient)
{
_configuration = configuration;
_bus = bus;
_logger = logger;
_backend = backend;
_configuration = configuration;
_logger = logger;
_jobClient = jobClient;
}
/// <inheritdoc />
public async Task Process(ScanRequestMessage message)
public async Task Process(ScanQueueMessage message)
{
_logger.LogInformation(
$"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
TimeSpan.FromSeconds(_configuration.GetValue<int>("MAX_SCANNING_TIME")));
var cancellationToken = cancellationTokenSource.Token;
var result = new ScanResultMessage
{
Id = message.Id,
Backend = _backend.Id
Status = ScanResultStatus.Queued
};
var stopwatch = new Stopwatch();
@ -56,10 +62,9 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
try
{
result.Threats = await _backend.ScanAsync(
message.Uri, cancellationTokenSource.Token);
result.Threats = await _backend.ScanAsync(message.Uri, cancellationToken);
result.Succeeded = true;
result.Status = ScanResultStatus.Succeeded;
_logger.LogInformation(
$"Backend {_backend.Id} completed a scan of {message.Id} " +
@ -67,7 +72,7 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
}
catch (Exception exception)
{
result.Succeeded = false;
result.Status = ScanResultStatus.Failed;
_logger.LogError(
exception, "Scanning failed with exception");
@ -79,11 +84,19 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
result.Duration = stopwatch.ElapsedMilliseconds / 1000;
_logger.LogInformation(
$"Sending scan results with status {result.Succeeded}");
try
{
_logger.LogInformation(
$"Sending scan results with status {result.Status}");
await _bus.SendAsync(
_configuration.GetValue<string>("ResultsSubscriptionId"), result);
_jobClient.Create<IScanResultJob>(
x => x.Report(message.Id, _backend.Id, result),
new EnqueuedState("default"));
}
catch (Exception exception)
{
_logger.LogError(exception, "Failed to send scan results");
}
}
}
}

20
MalwareMultiScan.Scanner/appsettings.json
View File

@ -1,19 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"BackendType": "Dummy",
"REDIS_ADDRESS": "localhost:6379",
"MaxScanningTime": 60,
"ResultsSubscriptionId": "mms.results",
"WorkerCount": 4,
"CONSUL_ADDRESS": "http://localhost:8500",
"ConnectionStrings": {
"RabbitMQ": "host=localhost;prefetchcount=1"
}
"BACKEND_ID": "dummy",
"MAX_SCANNING_TIME": 60,
"WORKER_COUNT": 4
}

23
MalwareMultiScan.Shared/Enums/ScanResultStatus.cs Normal file
View File

@ -0,0 +1,23 @@
namespace MalwareMultiScan.Shared.Enums
{
/// <summary>
/// Scan result status.
/// </summary>
public enum ScanResultStatus
{
/// <summary>
/// Scan is queued.
/// </summary>
Queued,
/// <summary>
/// Scan succeeded.
/// </summary>
Succeeded,
/// <summary>
/// Scan failed.
/// </summary>
Failed
}
}

60
MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs Normal file
View File

@ -0,0 +1,60 @@
using System;
using System.Diagnostics.CodeAnalysis;
using Consul;
using Hangfire;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
namespace MalwareMultiScan.Shared.Extensions
{
/// <summary>
/// Extensions for IServiceCollection.
/// </summary>
[ExcludeFromCodeCoverage]
public static class ServiceCollectionExtensions
{
/// <summary>
/// Add consul to the service collection and register the node on start.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
public static void AddConsul(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton<IConsulClient>(new ConsulClient(config =>
{
config.Address = configuration.GetValue<Uri>("CONSUL_ADDRESS");
config.WaitTime = TimeSpan.FromSeconds(120);
}));
}
/// <summary>
/// Add Hangfire with Redis storage.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
/// <param name="queues">Queue names.</param>
public static void AddHangfire(this IServiceCollection services,
IConfiguration configuration, params string[] queues)
{
if (queues.Length == 0)
queues = new[] {"default"};
services.AddHangfire(options =>
options.UseRedisStorage(configuration.GetValue<string>("REDIS_ADDRESS")));
services.AddHangfireServer(options =>
{
options.Queues = queues;
options.ServerTimeout = TimeSpan.FromSeconds(30);
options.HeartbeatInterval = TimeSpan.FromSeconds(5);
options.ServerCheckInterval = TimeSpan.FromSeconds(15);
var workerCount = configuration.GetValue<int>("WORKER_COUNT");
if (workerCount > 0)
options.WorkerCount = workerCount;
});
}
}
}

19
MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj Normal file
View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Shared</Product>
<AssemblyVersion>1.5.0</AssemblyVersion>
<FileVersion>1.5.0</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Consul" Version="1.6.1.1" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.9" />
</ItemGroup>
</Project>

12
MalwareMultiScan.Backends/Messages/ScanRequestMessage.cs → MalwareMultiScan.Shared/Message/ScanQueueMessage.cs
View File

@ -1,19 +1,19 @@
using System;
namespace MalwareMultiScan.Backends.Messages
namespace MalwareMultiScan.Shared.Message
{
/// <summary>
/// Scan request message.
/// Scan queue message.
/// </summary>
public class ScanRequestMessage
public class ScanQueueMessage
{
/// <summary>
/// Result id.
/// Scan result id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Remote URL.
/// Scan file URL.
/// </summary>
public Uri Uri { get; set; }
}

25
MalwareMultiScan.Shared/Message/ScanResultMessage.cs Normal file
View File

@ -0,0 +1,25 @@
using MalwareMultiScan.Shared.Enums;
namespace MalwareMultiScan.Shared.Message
{
/// <summary>
/// Scan result message.
/// </summary>
public class ScanResultMessage
{
/// <summary>
/// Scan status.
/// </summary>
public ScanResultStatus Status { get; set; }
/// <summary>
/// Scan duration in seconds.
/// </summary>
public long Duration { get; set; }
/// <summary>
/// Detected threats.
/// </summary>
public string[] Threats { get; set; }
}
}

19
MalwareMultiScan.Shared/Services/Interfaces/IScanBackgroundJob.cs Normal file
View File

@ -0,0 +1,19 @@
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Shared.Services.Interfaces
{
/// <summary>
/// Scan background job.
/// </summary>
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public interface IScanBackgroundJob
{
/// <summary>
/// Start scan.
/// </summary>
/// <param name="message">Scan queue message.</param>
Task Process(ScanQueueMessage message);
}
}

32
MalwareMultiScan.Shared/Services/Interfaces/IScanResultJob.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Shared.Services.Interfaces
{
/// <summary>
/// Scan background result job.
/// </summary>
public interface IScanResultJob
{
/// <summary>
/// Report job status.
/// </summary>
/// <param name="resultId">Result id.</param>
/// <param name="backendId">Backend id.</param>
/// <param name="result">Scan result.</param>
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task Report(string resultId, string backendId, ScanResultMessage result);
/// <summary>
/// Notify remote URL on scan result.
/// </summary>
/// <param name="uri">Base URL.</param>
/// <param name="resultId">Result id.</param>
/// <param name="backendId">Backend id.</param>
/// <param name="result">Scan result.</param>
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result);
}
}

2
MalwareMultiScan.Tests/Api/AttributesTests.cs
View File

@ -36,7 +36,7 @@ namespace MalwareMultiScan.Tests.Api
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["MaxFileSize"] = "100"
["FILE_SIZE_LIMIT"] = "100"
};
var context = new ValidationContext(

2
MalwareMultiScan.Tests/Api/ControllersTests.cs
View File

@ -1,7 +1,7 @@
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Controllers;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

81
MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs
View File

@ -1,81 +0,0 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Api
{
public class ReceiverHostedServiceTests
{
private Mock<IBus> _busMock;
private IReceiverHostedService _receiverHostedService;
private Mock<IScanResultService> _scanResultServiceMock;
[SetUp]
public void SetUp()
{
_busMock = new Mock<IBus>();
_busMock
.Setup(x => x.Receive("mms.results", It.IsAny<Func<ScanResultMessage, Task>>()))
.Callback<string, Func<ScanResultMessage, Task>>((s, func) =>
{
var task = func.Invoke(new ScanResultMessage
{
Id = "test",
Backend = "dummy",
Duration = 100,
Succeeded = true,
Threats = new[] {"Test"}
});
task.Wait();
});
_scanResultServiceMock = new Mock<IScanResultService>();
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
var httpClientFactoryMock = new Mock<IHttpClientFactory>();
_receiverHostedService = new ReceiverHostedService(
_busMock.Object,
configuration,
_scanResultServiceMock.Object,
Mock.Of<ILogger<ReceiverHostedService>>(),
httpClientFactoryMock.Object);
}
[Test]
public async Task TestBusReceiveScanResultMessage()
{
await _receiverHostedService.StartAsync(default);
_scanResultServiceMock.Verify(x => x.UpdateScanResultForBackend(
"test", "dummy", 100, true, true, new[] {"Test"}), Times.Once);
}
[Test]
public async Task TestBusIsDisposedOnStop()
{
await _receiverHostedService.StopAsync(default);
_busMock.Verify(x => x.Dispose(), Times.Once);
}
}
}

63
MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs
View File

@ -1,63 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Api
{
public class ScanBackendServiceTests
{
private Mock<IBus> _busMock;
private IScanBackendService _scanBackendService;
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["BackendsConfiguration"] = "backends.yaml"
};
_busMock = new Mock<IBus>();
_scanBackendService = new ScanBackendService(
configuration,
_busMock.Object,
Mock.Of<ILogger<ScanBackendService>>());
}
[Test]
public void TestBackendsAreNotEmpty()
{
Assert.IsNotEmpty(_scanBackendService.List);
}
[Test]
public async Task TestQueueUrlScan()
{
await _scanBackendService.QueueUrlScan(
new ScanResult {Id = "test"},
new ScanBackend {Id = "dummy"},
"http://test.com"
);
_busMock.Verify(x => x.SendAsync(
"dummy", It.Is<ScanRequestMessage>(r =>
r.Id == "test" &&
r.Uri == new Uri("http://test.com"))), Times.Once);
}
}
}

51
MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs
View File

@ -1,9 +1,11 @@
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Implementations;
using Consul;
using Hangfire;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Enums;
using MalwareMultiScan.Shared.Message;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
@ -17,7 +19,6 @@ namespace MalwareMultiScan.Tests.Api
{
private MongoDbRunner _mongoDbRunner;
private IScanResultService _resultService;
private Mock<IScanBackendService> _scanBackendService;
[SetUp]
public void SetUp()
@ -27,20 +28,11 @@ namespace MalwareMultiScan.Tests.Api
var connection = new MongoClient(_mongoDbRunner.ConnectionString);
var database = connection.GetDatabase("Test");
var gridFsBucket = new GridFSBucket(database);
_scanBackendService = new Mock<IScanBackendService>();
_scanBackendService
.SetupGet(x => x.List)
.Returns(() => new[]
{
new ScanBackend {Id = "dummy", Enabled = true},
new ScanBackend {Id = "clamav", Enabled = true},
new ScanBackend {Id = "disabled", Enabled = false}
});
_resultService = new ScanResultService(
database, gridFsBucket, _scanBackendService.Object);
database, gridFsBucket,
Mock.Of<IConsulClient>(),
Mock.Of<IBackgroundJobClient>());
}
[TearDown]
@ -85,10 +77,14 @@ namespace MalwareMultiScan.Tests.Api
var result = await _resultService.CreateScanResult(null);
Assert.NotNull(result.Id);
Assert.That(result.Results, Contains.Key("dummy"));
await _resultService.UpdateScanResultForBackend(
result.Id, "dummy", 100, true, true, new[] {"Test"});
result.Id, "dummy", new ScanResultMessage
{
Duration = 100,
Status = ScanResultStatus.Succeeded,
Threats = new []{"Test"}
});
result = await _resultService.GetScanResult(result.Id);
@ -97,25 +93,10 @@ namespace MalwareMultiScan.Tests.Api
var dummyResult = result.Results["dummy"];
Assert.IsTrue(dummyResult.Completed);
Assert.IsTrue(dummyResult.Succeeded);
Assert.IsTrue(dummyResult.Status == ScanResultStatus.Succeeded);
Assert.AreEqual(dummyResult.Duration, 100);
Assert.IsNotEmpty(dummyResult.Threats);
Assert.That(dummyResult.Threats, Is.EquivalentTo(new[] {"Test"}));
}
[Test]
public async Task TestQueueUrlScan()
{
var result = await _resultService.CreateScanResult(null);
await _resultService.QueueUrlScan(
result, "http://url.com");
_scanBackendService.Verify(x => x.QueueUrlScan(
It.IsAny<ScanResult>(),
It.IsAny<ScanBackend>(),
"http://url.com"), Times.Exactly(2));
}
}
}

4
MalwareMultiScan.Tests/Backends/BackendsTests.cs
View File

@ -1,7 +1,7 @@
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Backends.Implementations;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Backends.Services.Interfaces;
using Moq;
using NUnit.Framework;

68
MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs
View File

@ -1,68 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Scanner
{
public class ScanBackgroundJobTests
{
private Mock<IBus> _busMock;
private Mock<IScanBackend> _scanBackendMock;
private IScanBackgroundJob _scanBackgroundJob;
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
_busMock = new Mock<IBus>();
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
_scanBackendMock
.Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new[] {"Test"}));
_scanBackgroundJob = new ScanBackgroundJob(configuration, _busMock.Object,
Mock.Of<ILogger<ScanBackgroundJob>>(), _scanBackendMock.Object);
}
[Test]
public async Task TestMessageProcessing()
{
await _scanBackgroundJob.Process(new ScanRequestMessage
{
Id = "test",
Uri = new Uri("http://test.com")
});
_scanBackendMock.Verify(
x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()));
_busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(m =>
m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test"))));
}
}
}

54
MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs
View File

@ -1,54 +0,0 @@
using System;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Scanner
{
public class ScanHostedServiceTests
{
private Mock<IBus> _busMock;
private Mock<IScanBackend> _scanBackendMock;
private IScanHostedService _scanHostedService;
[SetUp]
public void SetUp()
{
_busMock = new Mock<IBus>();
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
_scanHostedService = new ScanHostedService(
Mock.Of<ILogger<ScanHostedService>>(),
_scanBackendMock.Object,
_busMock.Object);
}
[Test]
public async Task TestBusReceiveScanResultMessage()
{
await _scanHostedService.StartAsync(default);
_busMock.Verify(
x => x.Receive("dummy", It.IsAny<Action<ScanRequestMessage>>()));
}
[Test]
public async Task TestBusIsDisposedOnStop()
{
await _scanHostedService.StopAsync(default);
_busMock.Verify(x => x.Dispose(), Times.Once);
}
}
}

22
MalwareMultiScan.Ui/models/scan-result-entry-flattened.ts
View File

@ -1,19 +1,31 @@
import ScanResultEntry from '~/models/scan-result-entry';
import {ScanResultStatus} from '~/models/scan-result-status';
export default class ScanResultEntryFlattened implements ScanResultEntry {
readonly id: string;
readonly completed: boolean;
readonly succeeded: boolean | null;
readonly status: ScanResultStatus;
readonly duration: number;
readonly threats: string[];
constructor(id: string, completed: boolean, succeeded: boolean | null, duration: number, threats: string[]) {
constructor(id: string, status: ScanResultStatus, duration: number, threats: string[]) {
this.id = id;
this.completed = completed;
this.succeeded = succeeded;
this.status = status;
this.duration = duration;
this.threats = threats;
}
get completed()
{
return this.status != ScanResultStatus.Queued
}
get succeeded()
{
return this.status === ScanResultStatus.Succeeded;
}
get failed()
{
return this.status === ScanResultStatus.Failed;
}
}

5
MalwareMultiScan.Ui/models/scan-result-entry.ts
View File

@ -1,6 +1,7 @@
import { ScanResultStatus } from '~/models/scan-result-status';
export default interface ScanResultEntry {
readonly completed: boolean,
readonly succeeded: boolean | null,
readonly status: ScanResultStatus,
readonly duration: number,
readonly threats: string[]
}

6
MalwareMultiScan.Ui/models/scan-result-status.ts Normal file
View File

@ -0,0 +1,6 @@
export enum ScanResultStatus
{
Queued,
Succeeded,
Failed
}

2
MalwareMultiScan.Ui/package.json
View File

@ -1,6 +1,6 @@
{
"name": "malware-multi-scan-ui",
"version": "1.0.2",
"version": "1.5.0",
"private": true,
"scripts": {
"dev": "nuxt-ts",

11
MalwareMultiScan.Ui/pages/_id/index.vue
View File

@ -1,9 +1,9 @@
<template>
<b-table :fields="fields" :items="flattenedData" responsive class="m-0" striped>
<b-table :fields="fields" :items="flattenedData" :responsive="true" class="m-0" striped>
<template #cell(completed)="data">
<div class="h5 m-0">
<b-icon-check-circle-fill v-if="data.item.succeeded === true" variant="success"/>
<b-icon-exclamation-circle-fill v-if="data.item.succeeded === false" variant="danger"/>
<b-icon-check-circle-fill v-if="data.item.succeeded" variant="success"/>
<b-icon-exclamation-circle-fill v-if="data.item.completed && !data.item.succeeded" variant="danger"/>
<b-icon-bug-fill v-if="!data.item.completed" animation="spin-pulse"
class="rounded-circle bg-primary text-white" scale="0.7"/>
@ -17,7 +17,7 @@
<template #cell(threats)="data">
<div class="text-success" v-if="data.item.succeeded && !data.item.threats.length">No threats have been detected</div>
<div class="text-danger" v-if="data.item.succeeded === false">Scan failed to complete due to the error or timeout</div>
<div class="text-danger" v-if="data.item.completed && !data.item.succeeded">Scan failed to complete due to the error or timeout</div>
<div v-if="!data.item.completed">Scan is in progress</div>
<ul v-if="data.item.completed && data.item.threats.length" class="list-inline m-0">
@ -57,8 +57,7 @@ export default Vue.extend({
flattenedData(): ScanResultEntryFlattened[] {
return Object
.entries((this.data as ScanResult).results)
.map(([k, v]) => new ScanResultEntryFlattened(
k, v.completed, v.succeeded, v.duration, v.threats))
.map(([k, v]) => new ScanResultEntryFlattened(k, v.status, v.duration, v.threats))
}
},

18
MalwareMultiScan.sln
View File

@ -4,8 +4,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{8A16A3C4-2AE3-4F63-8280-635FF7878080}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{A248B5B7-7CBB-4242-98BD-51A9E915E485}"
ProjectSection(SolutionItems) = preProject
.dockerignore = .dockerignore
@ -17,6 +15,10 @@ EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Tests", "MalwareMultiScan.Tests\MalwareMultiScan.Tests.csproj", "{9896162D-8FC7-4911-933F-A78C94128923}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", "MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj", "{534E3C92-FD6D-401C-99D4-792DB11B57AE}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -31,13 +33,17 @@ Global
{7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.Build.0 = Release|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.Build.0 = Release|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.Build.0 = Release|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.Build.0 = Release|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

72
README.md
View File

@ -10,6 +10,8 @@ Self-hosted [VirusTotal](https://www.virustotal.com/) wannabe API for scanning U
The demo is running on a cheap Vultr node, so it might get slow or unavailable occasionally.
**IMPORTANT**: version 1.5 introduces breaking changes in containers configuration and docker-compose.yaml layout. Please see [releases](https://github.com/mindcollapse/MalwareMultiScan/releases) page and changelog of [docker-compose.yaml](https://github.com/mindcollapse/MalwareMultiScan/commits/master/docker-compose.yaml) and [README.md](https://github.com/mindcollapse/MalwareMultiScan/commits/master/README.md) for the additional details.
## Introduction
I faced a need to scan user-uploaded files in one of my work projects in an automated mode to ensure they don't contain any malware. Using VirusTotal was not an option because of a) legal restrictions and data residency limitations b) scanning by hash-sums would not be sufficient because the majority of files are generated / modified by users.
@ -36,29 +38,27 @@ Configuration of API and Scanners is performed by passing the environment variab
#### MalwareMultiScan.Api
* `ConnectionStrings__Mongo=mongodb://localhost:27017` - MongoDB connection string.
* `MONGO_ADDRESS=mongodb://localhost:27017` - MongoDB connection string.
* `DatabaseName=MalwareMultiScan` - MongoDB collection name.
* `MONGO_DATABASE=MalwareMultiScan` - MongoDB collection name.
* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
* `MaxFileSize=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified.
* `BackendsConfiguration=backends.yaml` - Path to the [backends.yaml](MalwareMultiScan.Api/backends.yaml) file.
* `FILE_SIZE_LIMIT=52428800` - Maximum size of a file that can be handled for the file scanning. The size of the URL content is not verified. Set to 0 to disable the validation.
#### MalwareMultiScan.Scanner
* `ConnectionStrings__RabbitMQ=host=localhost` - RabbitMQ connection string. See [EasyNetQ Wiki](https://github.com/EasyNetQ/EasyNetQ/wiki/Connecting-to-RabbitMQ) for details.
* `BACKEND_ID=dummy` - Id of a backend.
* `ResultsSubscriptionId=mms.results` - RabbitMQ subscription id for scan results. Should match with values defined for scanners.
* `REDIS_ADDRESS=localhost:6379` - Redis address for the distributed task queue.
* `BackendType=Dummy` - A type of scanner backend used by the running instance. Should correspond to the [BackendType](MalwareMultiScan.Backends/Enums/BackendType.cs) enum.
* `CONSUL_ADDRESS=http://localhost:8500` - Consul address for the service registration.
* `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
* `MAX_SCANNING_TIME=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
* `WorkerCount=4` - Number of workers for parallel scanning.
* `WORKER_COUNT=4` - Number of workers for parallel scanning.
#### MalwareMultiScan.Ui
@ -66,13 +66,25 @@ Configuration of API and Scanners is performed by passing the environment variab
### API Endpoints
* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `400 Bad Request` error.
* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error.
* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/ScanResult.cs) or `404 Not Found` error.
Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) in a body on every update from scan backends.
#### Callback URL
Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResultMessage](MalwareMultiScan.Shared/Message/ScanResultMessage.cs) in a body on every update from scan backends. Query string will contain `id` parameter that corresponds to the id of the scan result and `backend` parameter with the id of backend which completed the scan.
I.e. when you define `callbackUrl=http://localhost:1234/scan-results`, the POST request will be made to `http://localhost:1234/scan-results?id=123&backend=dummy` with a body
```json
{
"Status":1,
"Duration":5,
"Threats":["Malware.Dummy.Result"]
}
```
## Supported Scan Engines
@ -91,11 +103,29 @@ More scan backends can be added in the future. Some of the popular ones do not h
## Components
### Workflow
1. On startup all [Scanners](MalwareMultiScan.Scanner) register themselves in [Consul](https://www.consul.io/) with a service name equal to `scanner` and the `BackendId` metadata field equal to the value of `BACKEND_ID` environment variable. They also register a TTL check and listen for [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
2. Third-party client triggers `/api/queue/url` or `/api/queue/file` of the [MalwareMultiScan.Api](MalwareMultiScan.Api).
3. [MalwareMultiScan.Api](MalwareMultiScan.Api) sends a query to [Consul](https://www.consul.io/) and receives the list of alive scan backends with the service name `scanner`.
4. [MalwareMultiScan.Api](MalwareMultiScan.Api) schedules a [Hangfire](https://www.hangfire.io/) background job in a queue named under the `BackendId` metadata field.
5. [Scanners](MalwareMultiScan.Scanner) picks up a job from queue, starts the scan and sends result back to the `default` queue of [Hangfire](https://www.hangfire.io/).
6. [MalwareMultiScan.Api](MalwareMultiScan.Api) picks a job from the default` queue of [Hangfire](https://www.hangfire.io/) and updates the state of the scan.
7. If callback URL was specified during the step #2, [MalwareMultiScan.Api](MalwareMultiScan.Api) triggers a HTTP POST request to the specified URL. See [Callback URL](#callback-url) for details.
### Prerequisites
* **MongoDB** of version 3.x. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
* **MongoDB** of version 3.x or above. Used for storing scan results and files in GridFS. The communication is happening through the [official C#/.NET driver](https://docs.mongodb.com/drivers/csharp).
* **RabbitMQ** of version 3.x. Used for IPC and scan tasks queueing. The communication is happening through the [EasyNetQ](https://github.com/EasyNetQ/EasyNetQ) library.
* **Redis** of version 5.x or above. Used for tasks queueing. The communication is happening through the [Hangfire](https://www.hangfire.io/) library.
* **Consul** of version 1.8.x or above. Used for service registration of scan backends.
* **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments.
@ -103,9 +133,11 @@ More scan backends can be added in the future. Some of the popular ones do not h
### Parts
* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile). Configuration of available backends is performed via the [backends.yaml](MalwareMultiScan.Api/backends.yaml) location passed via the `BackendsConfiguration` environment variable.
* [MalwareMultiScan.Api](MalwareMultiScan.Api) - Simple ASP.NET Core WebApi for queueing files & urls for the scan and returning the result. Also acts as a receiver of scan results from the scanning backend nodes. See [Dockerfile](MalwareMultiScan.Api/Dockerfile).
* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Shared components between API and Worker. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
* [MalwareMultiScan.Backends](MalwareMultiScan.Backends) - Scan backends logic. Includes Dockerfiles and implementation classes for third-party vendor scan backends.
* [MalwareMultiScan.Shared](MalwareMultiScan.Shared) - Shared components.
* [MalwareMultiScan.Scanner](MalwareMultiScan.Scanner) - .NET Core Worker service subscribes to messages corresponding to the backend id, then fires up scanning command-line utility, and parses the output. See [Dockerfile](MalwareMultiScan.Scanner/Dockerfile). The image of MalwareMultiScan.Scanner acts as a base image for the rest of the scan backends. Check Dockerfiles from the [table above](#supported-scan-engines) for details.

65
docker-compose.yaml
View File

@ -1,23 +1,24 @@
version: "3.8"
services:
rabbitmq:
image: rabbitmq:3
redis:
image: redis:6
restart: on-failure
expose:
- "5672"
volumes:
- rabbitmq_etc/:/etc/rabbitmq/
- rabbitmq_data:/var/lib/rabbitmq/
- rabbitmq_logs/:/var/log/rabbitmq/
- redis:/data
mongodb:
image: mongo:4
restart: on-failure
expose:
- "27019"
volumes:
- mongodb:/data
# See https://github.com/hashicorp/consul/blob/master/demo/docker-compose-cluster/docker-compose.yml
# for a production-ready config.
consul:
image: consul:1.8
restart: on-failure
command: consul agent -dev -log-level=info -bind=0.0.0.0 -client=0.0.0.0
ui:
image: mindcollapse/malware-multi-scan-ui
@ -39,17 +40,14 @@ services:
api:
image: mindcollapse/malware-multi-scan-api
restart: on-failure
expose:
- "5000"
depends_on:
- rabbitmq
- consul
- redis
- mongodb
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;timeout=120"
- "ConnectionStrings__Mongo=mongodb://mongodb:27017?connectTimeoutMS=120000"
- "BackendsConfiguration=/etc/backends.yaml"
volumes:
- "./MalwareMultiScan.Api/backends.yaml:/etc/backends.yaml:ro"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
- "MONGO_ADDRESS=mongodb://mongodb:27017?connectTimeoutMS=120000"
build:
context: .
dockerfile: MalwareMultiScan.Api/Dockerfile
@ -58,9 +56,11 @@ services:
image: mindcollapse/malware-multi-scan-scanner
restart: on-failure
depends_on:
- rabbitmq
- consul
- redis
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: .
dockerfile: MalwareMultiScan.Scanner/Dockerfile
@ -71,7 +71,8 @@ services:
depends_on:
- dummy-scanner
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: MalwareMultiScan.Backends/Dockerfiles
dockerfile: Clamav.Dockerfile
@ -82,7 +83,8 @@ services:
depends_on:
- dummy-scanner
environment:
- "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
- "REDIS_ADDRESS=redis:6379"
- "CONSUL_ADDRESS=http://consul:8500"
build:
context: MalwareMultiScan.Backends/Dockerfiles
dockerfile: WindowsDefender.Dockerfile
@ -95,7 +97,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: Comodo.Dockerfile
@ -106,7 +109,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: DrWeb.Dockerfile
@ -117,7 +121,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: KES.Dockerfile
@ -128,7 +133,8 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: McAfee.Dockerfile
@ -139,13 +145,12 @@ services:
# depends_on:
# - dummy-scanner
# environment:
# - "ConnectionStrings__RabbitMQ=host=rabbitmq;prefetchcount=1;timeout=120"
# - "REDIS_ADDRESS=redis:6379"
# - "CONSUL_ADDRESS=http://consul:8500"
# build:
# context: MalwareMultiScan.Backends/Dockerfiles
# dockerfile: Sophos.Dockerfile
volumes:
mongodb:
rabbitmq_etc:
rabbitmq_data:
rabbitmq_logs:
redis: