Mirrors/MalwareMultiScan
1
0
Fork 0
mirror of https://github.com/volodymyrsmirnov/MalwareMultiScan.git synced 2025-08-23 13:02:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

basic architecture change: consul + hangfire

Browse Source
This commit is contained in:
Volodymyr Smirnov 2020-11-01 22:25:48 +02:00
parent 3f49c30b67
commit b918a82255
66 changed files with 571 additions and 728 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dummy_API.xml generated
View File

@ -1,6 +1,6 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dummy API" type="CompoundRunConfigurationType">
<toRun name="MalwareMultiScan.Scanner" type="DotNetProject" />
<toRun name="MalwareMultiScan.ScannerWorker" type="LaunchSettings" />
<toRun name="MalwareMultiScan.Api" type="DotNetProject" />
<method v="2" />
</configuration>

5
MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs
View File

@ -15,7 +15,10 @@ namespace MalwareMultiScan.Api.Attributes
{
var maxSize = validationContext
.GetRequiredService<IConfiguration>()
.GetValue<long>("MaxFileSize");
.GetValue<long>("FILE_SIZE_LIMIT");
if (maxSize == 0)
return ValidationResult.Success;
var formFile = (IFormFile) value;

2
MalwareMultiScan.Api/Controllers/QueueController.cs
View File

@ -2,7 +2,7 @@ using System;
using System.ComponentModel.DataAnnotations;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Attributes;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

2
MalwareMultiScan.Api/Controllers/ScanResultsController.cs
View File

@ -1,5 +1,5 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

18
MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs
View File

@ -1,18 +0,0 @@
namespace MalwareMultiScan.Api.Data.Configuration
{
/// <summary>
/// Scan backend.
/// </summary>
public class ScanBackend
{
/// <summary>
/// Backend id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend state.
/// </summary>
public bool Enabled { get; set; }
}
}

28
MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs
View File

@ -1,28 +0,0 @@
namespace MalwareMultiScan.Api.Data.Models
{
/// <summary>
/// Scan result entry.
/// </summary>
public class ScanResultEntry
{
/// <summary>
/// Completion status.
/// </summary>
public bool Completed { get; set; }
/// <summary>
/// Indicates that scanning completed without error.
/// </summary>
public bool? Succeeded { get; set; }
/// <summary>
/// Scanning duration in seconds.
/// </summary>
public long Duration { get; set; }
/// <summary>
/// Detected names of threats.
/// </summary>
public string[] Threats { get; set; }
}
}

9
MalwareMultiScan.Api/Data/Models/ScanResult.cs → MalwareMultiScan.Api/Data/ScanResult.cs
View File

@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using MalwareMultiScan.Shared.Message;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace MalwareMultiScan.Api.Data.Models
namespace MalwareMultiScan.Api.Data
{
/// <summary>
/// Scan result.
@ -23,9 +24,9 @@ namespace MalwareMultiScan.Api.Data.Models
public Uri CallbackUrl { get; set; }
/// <summary>
/// Result entries where key is backend id and value is <see cref="ScanResultEntry" />.
/// Result entries where key is backend id and value is <see cref="ScanResultMessage" />.
/// </summary>
public Dictionary<string, ScanResultEntry> Results { get; set; } =
new Dictionary<string, ScanResultEntry>();
public Dictionary<string, ScanResultMessage> Results { get; set; } =
new Dictionary<string, ScanResultMessage>();
}
}

4
MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs
View File

@ -14,8 +14,8 @@ namespace MalwareMultiScan.Api.Extensions
{
internal static void AddMongoDb(this IServiceCollection services, IConfiguration configuration)
{
var client = new MongoClient(configuration.GetConnectionString("Mongo"));
var db = client.GetDatabase(configuration.GetValue<string>("DatabaseName"));
var client = new MongoClient(configuration.GetValue<string>("MONGO_ADDRESS"));
var db = client.GetDatabase(configuration.GetValue<string>("MONGO_DATABASE"));
services.AddSingleton(client);
services.AddSingleton(db);

2
MalwareMultiScan.Api/MalwareMultiScan.Api.csproj
View File

@ -16,6 +16,8 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
<PackageReference Include="MongoDB.Driver" Version="2.11.3" />
<PackageReference Include="MongoDB.Driver.GridFS" Version="2.11.3" />
<PackageReference Include="YamlDotNet" Version="8.1.2" />

21
MalwareMultiScan.Api/Program.cs
View File

@ -1,21 +1,24 @@
using EasyNetQ.LightInject;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api
{
[ExcludeFromCodeCoverage]
internal static class Program
{
public static void Main(string[] args)
public static async Task Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
await Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(builder =>
{
builder.ConfigureKestrel(
options => options.Limits.MaxRequestBodySize = long.MaxValue);
private static IHostBuilder CreateHostBuilder(string[] args)
{
return Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(webBuilder => webBuilder.UseStartup<Startup>());
builder.UseStartup<Startup>();
})
.UseConsoleLifetime()
.Build()
.RunAsync();
}
}
}

103
MalwareMultiScan.Api/Services/Implementations/ReceiverHostedService.cs
View File

@ -1,103 +0,0 @@
using System;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace MalwareMultiScan.Api.Services.Implementations
{
/// <inheritdoc />
public class ReceiverHostedService : IReceiverHostedService
{
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ReceiverHostedService> _logger;
private readonly IScanResultService _scanResultService;
/// <summary>
/// Initialize receiver hosted service.
/// </summary>
/// <param name="bus">EasyNetQ bus.</param>
/// <param name="configuration">Configuration.</param>
/// <param name="scanResultService">Scan result service.</param>
/// <param name="logger">Logger.</param>
/// <param name="httpClientFactory">HTTP client factory.</param>
public ReceiverHostedService(IBus bus, IConfiguration configuration, IScanResultService scanResultService,
ILogger<ReceiverHostedService> logger, IHttpClientFactory httpClientFactory)
{
_bus = bus;
_configuration = configuration;
_scanResultService = scanResultService;
_logger = logger;
_httpClientFactory = httpClientFactory;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanResultMessage>(
_configuration.GetValue<string>("ResultsSubscriptionId"), StoreScanResult);
_logger.LogInformation(
"Started hosted service for receiving scan results");
return Task.CompletedTask;
}
/// <inheritdoc />
public Task StopAsync(CancellationToken cancellationToken)
{
_bus?.Dispose();
_logger.LogInformation(
"Stopped hosted service for receiving scan results");
return Task.CompletedTask;
}
private async Task StoreScanResult(ScanResultMessage message)
{
message.Threats ??= new string[] { };
_logger.LogInformation(
$"Received a result from {message.Backend} for {message.Id} " +
$"with threats {string.Join(",", message.Threats)}");
await _scanResultService.UpdateScanResultForBackend(
message.Id, message.Backend, message.Duration, true,
message.Succeeded, message.Threats);
var result = await _scanResultService.GetScanResult(message.Id);
if (result?.CallbackUrl == null)
return;
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(3));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var response = await httpClient.PostAsync(
result.CallbackUrl,
new StringContent(JsonConvert.SerializeObject(result), Encoding.UTF8, "application/json"),
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed to POST to callback URL {result.CallbackUrl}");
}
}
}
}

64
MalwareMultiScan.Api/Services/Implementations/ScanBackendService.cs
View File

@ -1,64 +0,0 @@
using System;
using System.IO;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace MalwareMultiScan.Api.Services.Implementations
{
/// <inheritdoc />
public class ScanBackendService : IScanBackendService
{
private readonly IBus _bus;
private readonly ILogger<ScanBackendService> _logger;
/// <summary>
/// Initialise scan backend service.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="bus">EasyNetQ bus.</param>
/// <param name="logger">Logger.</param>
/// <exception cref="FileNotFoundException">Missing backends.yaml configuration.</exception>
public ScanBackendService(IConfiguration configuration, IBus bus, ILogger<ScanBackendService> logger)
{
_bus = bus;
_logger = logger;
var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
if (!File.Exists(configurationPath))
throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
var configurationContent = File.ReadAllText(configurationPath);
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
List = deserializer.Deserialize<ScanBackend[]>(configurationContent);
}
/// <inheritdoc />
public ScanBackend[] List { get; }
/// <inheritdoc />
public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl)
{
_logger.LogInformation(
$"Queueing scan for {result.Id} on {backend.Id} at {fileUrl}");
await _bus.SendAsync(backend.Id, new ScanRequestMessage
{
Id = result.Id,
Uri = new Uri(fileUrl)
});
}
}
}

11
MalwareMultiScan.Api/Services/Interfaces/IReceiverHostedService.cs
View File

@ -1,11 +0,0 @@
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api.Services.Interfaces
{
/// <summary>
/// Receiver hosted service.
/// </summary>
public interface IReceiverHostedService : IHostedService
{
}
}

25
MalwareMultiScan.Api/Services/Interfaces/IScanBackendService.cs
View File

@ -1,25 +0,0 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
namespace MalwareMultiScan.Api.Services.Interfaces
{
/// <summary>
/// Scan backend service.
/// </summary>
public interface IScanBackendService
{
/// <summary>
/// Get list of parsed backends.
/// </summary>
ScanBackend[] List { get; }
/// <summary>
/// Queue URL for scan.
/// </summary>
/// <param name="result">Result entry.</param>
/// <param name="backend">Backend entry.</param>
/// <param name="fileUrl">Remote URL.</param>
Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl);
}
}

11
MalwareMultiScan.Api/Services/Interfaces/IScanResultService.cs
View File

@ -1,7 +1,8 @@
using System;
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Api.Services.Interfaces
{
@ -29,12 +30,8 @@ namespace MalwareMultiScan.Api.Services.Interfaces
/// </summary>
/// <param name="resultId">Result id.</param>
/// <param name="backendId">Backend id.</param>
/// <param name="duration">Duration.</param>
/// <param name="completed">Completion status.</param>
/// <param name="succeeded">Indicates that scanning completed without error.</param>
/// <param name="threats">Detected names of threats.</param>
Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
bool completed = false, bool succeeded = false, string[] threats = null);
/// <param name="result">Scan result.</param>
Task UpdateScanResultForBackend(string resultId, string backendId, ScanResultMessage result = null);
/// <summary>
/// Queue URL for scanning.

82
MalwareMultiScan.Api/Services/ScanResultJob.cs Normal file
View File

@ -0,0 +1,82 @@
using System;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Api.Services
{
public class ScanResultJob : IScanResultJob
{
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ScanResultJob> _logger;
private readonly IScanResultService _scanResultService;
public ScanResultJob(
ILogger<ScanResultJob> logger,
IHttpClientFactory httpClientFactory,
IScanResultService scanResultService,
IBackgroundJobClient backgroundJobClient)
{
_logger = logger;
_httpClientFactory = httpClientFactory;
_scanResultService = scanResultService;
_backgroundJobClient = backgroundJobClient;
}
/// <inheritdoc />
public async Task Report(string resultId, string backendId, ScanResultMessage result)
{
_logger.LogInformation(
$"Received a result from {backendId} for {result} with status {result.Status} " +
$"and threats {string.Join(",", result.Threats)}");
await _scanResultService.UpdateScanResultForBackend(resultId, backendId, result);
var scanResult = await _scanResultService.GetScanResult(resultId);
if (scanResult?.CallbackUrl == null)
return;
_backgroundJobClient.Create<IScanResultJob>(
x => x.Notify(scanResult.CallbackUrl, resultId, backendId, result), new EnqueuedState());
}
/// <inheritdoc />
public async Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(3));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var builder = new UriBuilder(uri)
{
Query = $"?id={resultId}" +
$"&backend={backendId}"
};
var response = await httpClient.PostAsync(builder.Uri,
new StringContent(JsonSerializer.Serialize(result), Encoding.UTF8, "application/json"),
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed to POST to callback URL {uri}");
}
}
}
}

67
MalwareMultiScan.Api/Services/Implementations/ScanResultService.cs → MalwareMultiScan.Api/Services/ScanResultService.cs
View File

@ -2,33 +2,45 @@ using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using Consul;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace MalwareMultiScan.Api.Services.Implementations
namespace MalwareMultiScan.Api.Services
{
/// <inheritdoc />
public class ScanResultService : IScanResultService
{
private const string CollectionName = "ScanResults";
private readonly IBackgroundJobClient _backgroundJobClient;
private readonly IGridFSBucket _bucket;
private readonly IMongoCollection<ScanResult> _collection;
private readonly IScanBackendService _scanBackendService;
private readonly IConsulClient _consulClient;
/// <summary>
/// Initialize scan result service.
/// </summary>
/// <param name="db">Mongo database.</param>
/// <param name="bucket">GridFS bucket.</param>
/// <param name="scanBackendService">Scan backend service.</param>
public ScanResultService(IMongoDatabase db, IGridFSBucket bucket, IScanBackendService scanBackendService)
/// <param name="consulClient">Consul client.</param>
/// <param name="backgroundJobClient">Background job client.</param>
public ScanResultService(
IMongoDatabase db,
IGridFSBucket bucket,
IConsulClient consulClient,
IBackgroundJobClient backgroundJobClient)
{
_bucket = bucket;
_scanBackendService = scanBackendService;
_consulClient = consulClient;
_backgroundJobClient = backgroundJobClient;
_collection = db.GetCollection<ScanResult>(CollectionName);
}
@ -38,11 +50,7 @@ namespace MalwareMultiScan.Api.Services.Implementations
{
var scanResult = new ScanResult
{
CallbackUrl = callbackUrl,
Results = _scanBackendService.List
.Where(b => b.Enabled)
.ToDictionary(k => k.Id, v => new ScanResultEntry())
CallbackUrl = callbackUrl
};
await _collection.InsertOneAsync(scanResult);
@ -60,25 +68,40 @@ namespace MalwareMultiScan.Api.Services.Implementations
}
/// <inheritdoc />
public async Task UpdateScanResultForBackend(string resultId, string backendId, long duration,
bool completed = false, bool succeeded = false, string[] threats = null)
public async Task UpdateScanResultForBackend(string resultId, string backendId,
ScanResultMessage result = null)
{
result ??= new ScanResultMessage();
await _collection.UpdateOneAsync(
Builders<ScanResult>.Filter.Where(r => r.Id == resultId),
Builders<ScanResult>.Update.Set(r => r.Results[backendId], new ScanResultEntry
{
Completed = completed,
Succeeded = succeeded,
Duration = duration,
Threats = threats ?? new string[] { }
}));
Builders<ScanResult>.Update.Set(r => r.Results[backendId], result));
}
/// <inheritdoc />
public async Task QueueUrlScan(ScanResult result, string fileUrl)
{
foreach (var backend in _scanBackendService.List.Where(b => b.Enabled))
await _scanBackendService.QueueUrlScan(result, backend, fileUrl);
var message = new ScanQueueMessage
{
Id = result.Id,
Uri = new Uri(fileUrl)
};
var scanners = await _consulClient.Health.Service("scanner", null, true);
var backends = scanners.Response
.Select(s => s.Service.Meta.TryGetValue("BackendId", out var backendId) ? backendId : null)
.Where(q => q != null)
.Distinct()
.ToArray();
foreach (var backend in backends)
{
await UpdateScanResultForBackend(result.Id, backend);
_backgroundJobClient.Create<IScanBackgroundJob>(
j => j.Process(message), new EnqueuedState(backend));
}
}
/// <inheritdoc />

34
MalwareMultiScan.Api/Startup.cs
View File

@ -1,16 +1,17 @@
using System.Diagnostics.CodeAnalysis;
using Hangfire;
using MalwareMultiScan.Api.Extensions;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Shared.Extensions;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Api
{
[ExcludeFromCodeCoverage]
internal class Startup
{
private readonly IConfiguration _configuration;
@ -24,29 +25,26 @@ namespace MalwareMultiScan.Api
{
services.AddDockerForwardedHeadersOptions();
services.Configure<KestrelServerOptions>(options =>
{
options.Limits.MaxRequestBodySize = _configuration.GetValue<int>("MaxFileSize");
});
services.AddConsul(_configuration);
services.AddMongoDb(_configuration);
services.AddRabbitMq(_configuration);
services.AddSingleton<IScanBackendService, ScanBackendService>();
services.AddHangfire(_configuration);
services.AddSingleton<IScanResultService, ScanResultService>();
services.AddSingleton<IScanResultJob, ScanResultJob>();
services.AddControllers();
services.AddHostedService<ReceiverHostedService>();
services.AddHttpClient();
}
public void Configure(IApplicationBuilder app)
public void Configure(IApplicationBuilder app, IHostEnvironment hostEnvironment)
{
app.UseRouting();
app.UseForwardedHeaders();
app.UseEndpoints(endpoints => endpoints.MapControllers());
app.UseForwardedHeaders();
if (hostEnvironment.IsDevelopment())
app.UseHangfireDashboard();
}
}
}

17
MalwareMultiScan.Api/appsettings.json
View File

@ -7,15 +7,12 @@
}
},
"AllowedHosts": "*",
"ConnectionStrings": {
"Mongo": "mongodb://localhost:27017",
"RabbitMQ": "host=localhost"
},
"MONGO_ADDRESS": "mongodb://localhost:27017",
"MONGO_DATABASE": "MalwareMultiScan",
"DatabaseName": "MalwareMultiScan",
"ResultsSubscriptionId": "mms.results",
"MaxFileSize": 52428800,
"BackendsConfiguration": "backends.yaml"
"REDIS_ADDRESS": "localhost:6379",
"CONSUL_ADDRESS": "http://localhost:8500",
"FILE_SIZE_LIMIT": 52428800
}

2
MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs
View File

@ -4,7 +4,7 @@ using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Abstracts
{

2
MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs → MalwareMultiScan.Backends/Backends/ClamavScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class ClamavScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs → MalwareMultiScan.Backends/Backends/ComodoScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class ComodoScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs → MalwareMultiScan.Backends/Backends/DrWebScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class DrWebScanBackend : AbstractLocalProcessScanBackend

4
MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs → MalwareMultiScan.Backends/Backends/DummyScanBackend.cs
View File

@ -2,9 +2,9 @@ using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class DummyScanBackend : IScanBackend

2
MalwareMultiScan.Backends/Interfaces/IScanBackend.cs → MalwareMultiScan.Backends/Backends/Interfaces/IScanBackend.cs
View File

@ -3,7 +3,7 @@ using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace MalwareMultiScan.Backends.Interfaces
namespace MalwareMultiScan.Backends.Backends.Interfaces
{
/// <summary>
/// Scan backend.

2
MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs → MalwareMultiScan.Backends/Backends/KesScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class KesScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/McAfeeScanBackend.cs → MalwareMultiScan.Backends/Backends/McAfeeScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class McAfeeScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs → MalwareMultiScan.Backends/Backends/SophosScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class SophosScanBackend : AbstractLocalProcessScanBackend

2
MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs → MalwareMultiScan.Backends/Backends/WindowsDefenderScanBackend.cs
View File

@ -2,7 +2,7 @@ using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using MalwareMultiScan.Backends.Services.Interfaces;
namespace MalwareMultiScan.Backends.Backends.Implementations
namespace MalwareMultiScan.Backends.Backends
{
/// <inheritdoc />
public class WindowsDefenderScanBackend : AbstractLocalProcessScanBackend

48
MalwareMultiScan.Backends/Enums/BackendType.cs
View File

@ -1,48 +0,0 @@
namespace MalwareMultiScan.Backends.Enums
{
/// <summary>
/// Backend type.
/// </summary>
public enum BackendType
{
/// <summary>
/// Dummy
/// </summary>
Dummy,
/// <summary>
/// Windows Defender.
/// </summary>
Defender,
/// <summary>
/// ClamAV.
/// </summary>
Clamav,
/// <summary>
/// DrWeb.
/// </summary>
DrWeb,
/// <summary>
/// KES.
/// </summary>
Kes,
/// <summary>
/// Comodo.
/// </summary>
Comodo,
/// <summary>
/// Sophos.
/// </summary>
Sophos,
/// <summary>
/// McAfee.
/// </summary>
McAfee
}
}

45
MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs
View File

@ -1,9 +1,7 @@
using System;
using System.Diagnostics.CodeAnalysis;
using EasyNetQ;
using MalwareMultiScan.Backends.Backends.Implementations;
using MalwareMultiScan.Backends.Enums;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Backends.Services.Implementations;
using MalwareMultiScan.Backends.Services.Interfaces;
using Microsoft.Extensions.Configuration;
@ -17,55 +15,42 @@ namespace MalwareMultiScan.Backends.Extensions
[ExcludeFromCodeCoverage]
public static class ServiceCollectionExtensions
{
/// <summary>
/// Add RabbitMQ service.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
public static void AddRabbitMq(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton(x =>
RabbitHutch.CreateBus(configuration.GetConnectionString("RabbitMQ")));
}
/// <summary>
/// Add scanning backend.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
/// <exception cref="ArgumentOutOfRangeException">Unknown backend.</exception>
public static void AddScanningBackend(this IServiceCollection services, IConfiguration configuration)
public static void AddScanBackend(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton<IProcessRunner, ProcessRunner>();
switch (configuration.GetValue<BackendType>("BackendType"))
switch (configuration.GetValue<string>("BACKEND_ID"))
{
case BackendType.Dummy:
services.AddSingleton<IScanBackend, DummyScanBackend>();
break;
case BackendType.Defender:
services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
break;
case BackendType.Clamav:
case "clamav":
services.AddSingleton<IScanBackend, ClamavScanBackend>();
break;
case BackendType.DrWeb:
case "drweb":
services.AddSingleton<IScanBackend, DrWebScanBackend>();
break;
case BackendType.Kes:
case "kes":
services.AddSingleton<IScanBackend, KesScanBackend>();
break;
case BackendType.Comodo:
case "comodo":
services.AddSingleton<IScanBackend, ComodoScanBackend>();
break;
case BackendType.Sophos:
case "sophos":
services.AddSingleton<IScanBackend, SophosScanBackend>();
break;
case BackendType.McAfee:
case "mcafee":
services.AddSingleton<IScanBackend, McAfeeScanBackend>();
break;
case "windows-defender":
services.AddSingleton<IScanBackend, WindowsDefenderScanBackend>();
break;
default:
throw new ArgumentOutOfRangeException();
services.AddSingleton<IScanBackend, DummyScanBackend>();
break;
}
}
}

6
MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj
View File

@ -10,11 +10,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="EasyNetQ" Version="5.6.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.9" />
<ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
</ItemGroup>
</Project>

20
MalwareMultiScan.Backends/Messages/ScanRequestMessage.cs
View File

@ -1,20 +0,0 @@
using System;
namespace MalwareMultiScan.Backends.Messages
{
/// <summary>
/// Scan request message.
/// </summary>
public class ScanRequestMessage
{
/// <summary>
/// Result id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Remote URL.
/// </summary>
public Uri Uri { get; set; }
}
}

33
MalwareMultiScan.Backends/Messages/ScanResultMessage.cs
View File

@ -1,33 +0,0 @@
namespace MalwareMultiScan.Backends.Messages
{
/// <summary>
/// Scan result message.
/// </summary>
public class ScanResultMessage
{
/// <summary>
/// Result id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend.
/// </summary>
public string Backend { get; set; }
/// <summary>
/// Status.
/// </summary>
public bool Succeeded { get; set; }
/// <summary>
/// List of detected threats.
/// </summary>
public string[] Threats { get; set; }
/// <summary>
/// Duration.
/// </summary>
public long Duration { get; set; }
}
}

0
MalwareMultiScan.Backends/Services/Implementations/ProcessRunner.cs → MalwareMultiScan.Backends/Services/ProcessRunner.cs
View File

16
MalwareMultiScan.Scanner/Dockerfile
View File

@ -1,16 +0,0 @@
FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS builder
WORKDIR /src
COPY MalwareMultiScan.Scanner /src/MalwareMultiScan.Scanner
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
FROM mcr.microsoft.com/dotnet/core/runtime:3.1
WORKDIR /worker
COPY --from=builder /src/publish /worker
ENTRYPOINT ["/worker/MalwareMultiScan.Scanner"]

21
MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
View File

@ -1,21 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<Company>Volodymyr Smirnov</Company>
<Product>MalwareMultiScan Scanner</Product>
<AssemblyVersion>1.0.1</AssemblyVersion>
<FileVersion>1.0.1</FileVersion>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="3.1.9" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
</ItemGroup>
</Project>

45
MalwareMultiScan.Scanner/Program.cs
View File

@ -1,45 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.MemoryStorage;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Scanner
{
[ExcludeFromCodeCoverage]
internal static class Program
{
public static async Task Main(string[] args)
{
await Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(configure =>
{
configure.AddJsonFile("appsettings.json");
configure.AddEnvironmentVariables();
})
.ConfigureServices((context, services) =>
{
services.AddLogging();
services.AddRabbitMq(context.Configuration);
services.AddScanningBackend(context.Configuration);
services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
services.AddHostedService<ScanHostedService>();
services.AddHangfire(
configuration => configuration.UseMemoryStorage());
services.AddHangfireServer(options =>
{
options.WorkerCount = context.Configuration.GetValue<int>("WorkerCount");
});
}).RunConsoleAsync();
}
}
}

57
MalwareMultiScan.Scanner/Services/Implementations/ScanHostedService.cs
View File

@ -1,57 +0,0 @@
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using Hangfire;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
{
/// <inheritdoc />
public class ScanHostedService : IScanHostedService
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly ILogger<ScanHostedService> _logger;
/// <summary>
/// Initialise scan hosted service.
/// </summary>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
/// <param name="bus">EasyNetQ bus.</param>
public ScanHostedService(
ILogger<ScanHostedService> logger,
IScanBackend backend, IBus bus)
{
_logger = logger;
_bus = bus;
_backend = backend;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanRequestMessage>(_backend.Id, message =>
BackgroundJob.Enqueue<IScanBackgroundJob>(j => j.Process(message)));
_logger.LogInformation(
$"Started scan hosting service for the backend {_backend.Id}");
return Task.CompletedTask;
}
/// <inheritdoc />
public Task StopAsync(CancellationToken cancellationToken)
{
_bus.Dispose();
_logger.LogInformation(
$"Stopped scan hosting service for the backend {_backend.Id}");
return Task.CompletedTask;
}
}
}

17
MalwareMultiScan.Scanner/Services/Interfaces/IScanBackgroundJob.cs
View File

@ -1,17 +0,0 @@
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Messages;
namespace MalwareMultiScan.Scanner.Services.Interfaces
{
/// <summary>
/// Background scanning job.
/// </summary>
public interface IScanBackgroundJob
{
/// <summary>
/// Process scan request in the background worker.
/// </summary>
/// <param name="requestMessage">Request message.</param>
Task Process(ScanRequestMessage requestMessage);
}
}

11
MalwareMultiScan.Scanner/Services/Interfaces/IScanHostedService.cs
View File

@ -1,11 +0,0 @@
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Scanner.Services.Interfaces
{
/// <summary>
/// Scan hosted service.
/// </summary>
public interface IScanHostedService : IHostedService
{
}
}

19
MalwareMultiScan.Scanner/appsettings.json
View File

@ -1,19 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"BackendType": "Dummy",
"MaxScanningTime": 60,
"ResultsSubscriptionId": "mms.results",
"WorkerCount": 4,
"ConnectionStrings": {
"RabbitMQ": "host=localhost;prefetchcount=1"
}
}

15
MalwareMultiScan.ScannerWorker/MalwareMultiScan.ScannerWorker.csproj Normal file
View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Hangfire" Version="1.7.17" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="HangFire.Redis.StackExchange" Version="1.8.4" />
</ItemGroup>
</Project>

50
MalwareMultiScan.ScannerWorker/Program.cs Normal file
View File

@ -0,0 +1,50 @@
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Backends.Services.Implementations;
using MalwareMultiScan.Backends.Services.Interfaces;
using MalwareMultiScan.ScannerWorker.Services;
using MalwareMultiScan.Shared.Extensions;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.ScannerWorker
{
public static class Program
{
public static async Task Main(string[] args)
{
await Host.CreateDefaultBuilder(args)
.ConfigureLogging((context, builder) =>
{
builder.AddConsole();
builder.AddConfiguration(context.Configuration);
})
.ConfigureAppConfiguration(builder =>
{
builder.AddJsonFile("appsettings.json");
builder.AddEnvironmentVariables();
})
.ConfigureServices((context, services) =>
{
services.AddConsul(context.Configuration);
services.AddScanBackend(context.Configuration);
services.AddHangfire(context.Configuration,
context.Configuration.GetValue<string>("BACKEND_ID"));
services.AddSingleton<IProcessRunner, ProcessRunner>();
services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
services.AddHostedService<ConsulHostedService>();
})
.UseConsoleLifetime()
.Build()
.RunAsync();
}
}
}

8
MalwareMultiScan.ScannerWorker/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"MalwareMultiScan.ScannerWorker": {
"commandName": "Project"
}
}
}

82
MalwareMultiScan.ScannerWorker/Services/ConsulHostedService.cs Normal file
View File

@ -0,0 +1,82 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Consul;
using MalwareMultiScan.Backends.Backends.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.ScannerWorker.Services
{
public class ConsulHostedService : BackgroundService
{
private static readonly string ServiceId = Dns.GetHostName();
private static readonly string CheckId = $"ttl-{ServiceId}";
private readonly IConsulClient _consulClient;
private readonly AgentServiceRegistration _registration;
public ConsulHostedService(
IConsulClient consulClient,
IConfiguration configuration,
IScanBackend scanBackend)
{
_consulClient = consulClient;
_registration = new AgentServiceRegistration
{
ID = ServiceId,
Name = configuration.GetValue<string>("CONSUL_SERVICE_NAME"),
Meta = new Dictionary<string, string>
{
{"BackendId", scanBackend.Id}
}
};
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
await _consulClient.Agent.PassTTL(
CheckId, string.Empty, stoppingToken);
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
}
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
await _consulClient.Agent.ServiceDeregister(
_registration.ID, cancellationToken);
await _consulClient.Agent.ServiceRegister(
_registration, cancellationToken);
await _consulClient.Agent.CheckRegister(new AgentCheckRegistration
{
ID = CheckId,
Name = "TTL",
ServiceID = ServiceId,
TTL = TimeSpan.FromSeconds(15),
DeregisterCriticalServiceAfter = TimeSpan.FromSeconds(60)
}, cancellationToken);
await base.StartAsync(cancellationToken);
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
await _consulClient.Agent.ServiceDeregister(
_registration.ID, cancellationToken);
await _consulClient.Agent.CheckDeregister(
CheckId, cancellationToken);
await base.StopAsync(cancellationToken);
}
}
}

63
MalwareMultiScan.Scanner/Services/Implementations/ScanBackgroundJob.cs → MalwareMultiScan.ScannerWorker/Services/ScanBackgroundJob.cs
View File

@ -2,53 +2,56 @@ using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Hangfire;
using Hangfire.States;
using MalwareMultiScan.Backends.Backends.Interfaces;
using MalwareMultiScan.Shared.Enums;
using MalwareMultiScan.Shared.Message;
using MalwareMultiScan.Shared.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
namespace MalwareMultiScan.ScannerWorker.Services
{
/// <inheritdoc />
public class ScanBackgroundJob : IScanBackgroundJob
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly IBackgroundJobClient _jobClient;
private readonly ILogger<ScanBackgroundJob> _logger;
/// <summary>
/// Initialize scan background job.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="bus">Bus.</param>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
public ScanBackgroundJob(IConfiguration configuration, IBus bus,
ILogger<ScanBackgroundJob> logger, IScanBackend backend)
/// <param name="jobClient"></param>
public ScanBackgroundJob(
IScanBackend backend,
IConfiguration configuration,
ILogger<ScanBackgroundJob> logger,
IBackgroundJobClient jobClient)
{
_configuration = configuration;
_bus = bus;
_logger = logger;
_backend = backend;
_configuration = configuration;
_logger = logger;
_jobClient = jobClient;
}
/// <inheritdoc />
public async Task Process(ScanRequestMessage message)
public async Task Process(ScanQueueMessage message)
{
_logger.LogInformation(
$"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
TimeSpan.FromSeconds(_configuration.GetValue<int>("MAX_SCANNING_TIME")));
var result = new ScanResultMessage
{
Id = message.Id,
Backend = _backend.Id
};
var cancellationToken = cancellationTokenSource.Token;
var result = new ScanResultMessage();
var stopwatch = new Stopwatch();
@ -56,10 +59,9 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
try
{
result.Threats = await _backend.ScanAsync(
message.Uri, cancellationTokenSource.Token);
result.Threats = await _backend.ScanAsync(message.Uri, cancellationToken);
result.Succeeded = true;
result.Status = ScanResultStatus.Succeeded;
_logger.LogInformation(
$"Backend {_backend.Id} completed a scan of {message.Id} " +
@ -67,7 +69,7 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
}
catch (Exception exception)
{
result.Succeeded = false;
result.Status = ScanResultStatus.Failed;
_logger.LogError(
exception, "Scanning failed with exception");
@ -79,11 +81,18 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
result.Duration = stopwatch.ElapsedMilliseconds / 1000;
_logger.LogInformation(
$"Sending scan results with status {result.Succeeded}");
try
{
_logger.LogInformation(
$"Sending scan results with status {result.Status}");
await _bus.SendAsync(
_configuration.GetValue<string>("ResultsSubscriptionId"), result);
_jobClient.Create<IScanResultJob>(
x => x.Report(message.Id, _backend.Id, result), new EnqueuedState());
}
catch (Exception exception)
{
_logger.LogError(exception, "Failed to send scan results");
}
}
}
}

9
MalwareMultiScan.ScannerWorker/appsettings.json Normal file
View File

@ -0,0 +1,9 @@
{
"REDIS_ADDRESS": "localhost:6379",
"CONSUL_ADDRESS": "http://localhost:8500",
"CONSUL_SERVICE_NAME": "scanner",
"BACKEND_ID": "dummy",
"MAX_SCANNING_TIME": 60
}

9
MalwareMultiScan.Shared/Enums/ScanResultStatus.cs Normal file
View File

@ -0,0 +1,9 @@
namespace MalwareMultiScan.Shared.Enums
{
public enum ScanResultStatus
{
Queued,
Succeeded,
Failed
}
}

40
MalwareMultiScan.Shared/Extensions/ServiceCollectionExtensions.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using Consul;
using Hangfire;
using Hangfire.Redis;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
namespace MalwareMultiScan.Shared.Extensions
{
/// <summary>
/// Extensions for IServiceCollection.
/// </summary>
public static class ServiceCollectionExtensions
{
/// <summary>
/// Add consul to the service collection and register the node on start.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration.</param>
public static void AddConsul(this IServiceCollection services, IConfiguration configuration)
{
services.AddSingleton<IConsulClient>(new ConsulClient(config =>
{
config.Address = configuration.GetValue<Uri>("CONSUL_ADDRESS");
}));
}
public static void AddHangfire(this IServiceCollection services,
IConfiguration configuration, params string[] queues)
{
if (queues.Length == 0)
queues = new[] {"default"};
services.AddHangfire(options =>
options.UseRedisStorage(configuration.GetValue<string>("REDIS_ADDRESS")));
services.AddHangfireServer(options => options.Queues = queues);
}
}
}

20
MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj Normal file
View File

@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Consul" Version="1.6.1.1" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.Redis.StackExchange" Version="1.8.4" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="3.1.9" />
<PackageReference Include="MongoDB.Bson" Version="2.11.3" />
</ItemGroup>
<ItemGroup>
<Folder Include="Services" />
</ItemGroup>
</Project>

10
MalwareMultiScan.Shared/Message/ScanQueueMessage.cs Normal file
View File

@ -0,0 +1,10 @@
using System;
namespace MalwareMultiScan.Shared.Message
{
public class ScanQueueMessage
{
public string Id { get; set; }
public Uri Uri { get; set; }
}
}

13
MalwareMultiScan.Shared/Message/ScanResultMessage.cs Normal file
View File

@ -0,0 +1,13 @@
using MalwareMultiScan.Shared.Enums;
namespace MalwareMultiScan.Shared.Message
{
public class ScanResultMessage
{
public ScanResultStatus Status { get; set; }
public long Duration { get; set; }
public string[] Threats { get; set; }
}
}

12
MalwareMultiScan.Shared/Services/Interfaces/IScanBackgroundJob.cs Normal file
View File

@ -0,0 +1,12 @@
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Shared.Services.Interfaces
{
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public interface IScanBackgroundJob
{
Task Process(ScanQueueMessage message);
}
}

16
MalwareMultiScan.Shared/Services/Interfaces/IScanResultJob.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Shared.Message;
namespace MalwareMultiScan.Shared.Services.Interfaces
{
public interface IScanResultJob
{
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task Report(string resultId, string backendId, ScanResultMessage result);
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task Notify(Uri uri, string resultId, string backendId, ScanResultMessage result);
}
}

2
MalwareMultiScan.Tests/Api/ControllersTests.cs
View File

@ -1,7 +1,7 @@
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Controllers;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Services.Interfaces;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;

3
MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs
View File

@ -2,10 +2,7 @@ using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;

6
MalwareMultiScan.Tests/Api/ScanBackendServiceTests.cs
View File

@ -1,12 +1,8 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;

4
MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs
View File

@ -1,8 +1,8 @@
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using MalwareMultiScan.Api.Services.Implementations;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Api.Services.Interfaces;
using Mongo2Go;
using MongoDB.Bson;

3
MalwareMultiScan.Tests/Backends/BackendsTests.cs
View File

@ -1,7 +1,6 @@
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Backends.Implementations;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Backends;
using MalwareMultiScan.Backends.Services.Interfaces;
using Moq;
using NUnit.Framework;

6
MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs
View File

@ -1,13 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;

5
MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs
View File

@ -1,10 +1,5 @@
using System;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;

22
MalwareMultiScan.Ui/models/scan-result-entry-flattened.ts
View File

@ -1,19 +1,31 @@
import ScanResultEntry from '~/models/scan-result-entry';
import {ScanResultStatus} from '~/models/scan-result-status';
export default class ScanResultEntryFlattened implements ScanResultEntry {
readonly id: string;
readonly completed: boolean;
readonly succeeded: boolean | null;
readonly status: ScanResultStatus;
readonly duration: number;
readonly threats: string[];
constructor(id: string, completed: boolean, succeeded: boolean | null, duration: number, threats: string[]) {
constructor(id: string, status: ScanResultStatus, duration: number, threats: string[]) {
this.id = id;
this.completed = completed;
this.succeeded = succeeded;
this.status = status;
this.duration = duration;
this.threats = threats;
}
get completed()
{
return this.status != ScanResultStatus.Queued
}
get succeeded()
{
return this.status === ScanResultStatus.Succeeded;
}
get failed()
{
return this.status === ScanResultStatus.Failed;
}
}

5
MalwareMultiScan.Ui/models/scan-result-entry.ts
View File

@ -1,6 +1,7 @@
import { ScanResultStatus } from '~/models/scan-result-status';
export default interface ScanResultEntry {
readonly completed: boolean,
readonly succeeded: boolean | null,
readonly status: ScanResultStatus,
readonly duration: number,
readonly threats: string[]
}

11
MalwareMultiScan.Ui/pages/_id/index.vue
View File

@ -1,9 +1,9 @@
<template>
<b-table :fields="fields" :items="flattenedData" responsive class="m-0" striped>
<b-table :fields="fields" :items="flattenedData" :responsive="true" class="m-0" striped>
<template #cell(completed)="data">
<div class="h5 m-0">
<b-icon-check-circle-fill v-if="data.item.succeeded === true" variant="success"/>
<b-icon-exclamation-circle-fill v-if="data.item.succeeded === false" variant="danger"/>
<b-icon-check-circle-fill v-if="data.item.succeeded" variant="success"/>
<b-icon-exclamation-circle-fill v-if="data.item.completed && !data.item.succeeded" variant="danger"/>
<b-icon-bug-fill v-if="!data.item.completed" animation="spin-pulse"
class="rounded-circle bg-primary text-white" scale="0.7"/>
@ -17,7 +17,7 @@
<template #cell(threats)="data">
<div class="text-success" v-if="data.item.succeeded && !data.item.threats.length">No threats have been detected</div>
<div class="text-danger" v-if="data.item.succeeded === false">Scan failed to complete due to the error or timeout</div>
<div class="text-danger" v-if="data.item.completed && !data.item.succeeded">Scan failed to complete due to the error or timeout</div>
<div v-if="!data.item.completed">Scan is in progress</div>
<ul v-if="data.item.completed && data.item.threats.length" class="list-inline m-0">
@ -57,8 +57,7 @@ export default Vue.extend({
flattenedData(): ScanResultEntryFlattened[] {
return Object
.entries((this.data as ScanResult).results)
.map(([k, v]) => new ScanResultEntryFlattened(
k, v.completed, v.succeeded, v.duration, v.threats))
.map(([k, v]) => new ScanResultEntryFlattened(k, v.status, v.duration, v.threats))
}
},

18
MalwareMultiScan.sln
View File

@ -4,8 +4,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends",
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{8A16A3C4-2AE3-4F63-8280-635FF7878080}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{A248B5B7-7CBB-4242-98BD-51A9E915E485}"
ProjectSection(SolutionItems) = preProject
.dockerignore = .dockerignore
@ -17,6 +15,10 @@ EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Tests", "MalwareMultiScan.Tests\MalwareMultiScan.Tests.csproj", "{9896162D-8FC7-4911-933F-A78C94128923}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.ScannerWorker", "MalwareMultiScan.ScannerWorker\MalwareMultiScan.ScannerWorker.csproj", "{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", "MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj", "{534E3C92-FD6D-401C-99D4-792DB11B57AE}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -31,13 +33,17 @@ Global
{7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.Build.0 = Release|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.Build.0 = Release|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9896162D-8FC7-4911-933F-A78C94128923}.Release|Any CPU.Build.0 = Release|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D36ED4DD-4EEA-4609-8AED-B2FD496E4C90}.Release|Any CPU.Build.0 = Release|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{534E3C92-FD6D-401C-99D4-792DB11B57AE}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal