Mirrors/MalwareMultiScan
1
0
Fork 0
mirror of https://github.com/volodymyrsmirnov/MalwareMultiScan.git synced 2025-08-23 21:12:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

version 1.1L

Browse Source 
* UI responsive table fix
* Background scanning for backends
* Callback URL parameter to notify on scan results
This commit is contained in:
Volodymyr Smirnov 2020-10-30 11:20:08 +02:00
parent 3ff8d1d05f
commit ee071811e8
30 changed files with 345 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs
View File

@ -8,9 +8,23 @@ namespace MalwareMultiScan.Api.Attributes
/// </summary>
public class IsHttpUrlAttribute : ValidationAttribute
{
private readonly bool _failOnNull;
/// <summary>
/// Initialize http URL validation attribute.
/// </summary>
/// <param name="failOnNull">True if URL is mandatory, false otherwise.</param>
public IsHttpUrlAttribute(bool failOnNull = true)
{
_failOnNull = failOnNull;
}
/// <inheritdoc />
protected override ValidationResult IsValid(object value, ValidationContext validationContext)
{
if (!_failOnNull && string.IsNullOrEmpty(value.ToString()))
return ValidationResult.Success;
if (!Uri.TryCreate((string) value, UriKind.Absolute, out var uri)
|| !uri.IsAbsoluteUri
|| uri.Scheme.ToLowerInvariant() != "http"

15
MalwareMultiScan.Api/Controllers/QueueController.cs
View File

@ -1,3 +1,4 @@
using System;
using System.ComponentModel.DataAnnotations;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Attributes;
@ -31,13 +32,16 @@ namespace MalwareMultiScan.Api.Controllers
/// Queue file for scanning.
/// </summary>
/// <param name="file">File from form data.</param>
/// <param name="callbackUrl">Optional callback URL.</param>
[HttpPost("file")]
[ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]
public async Task<IActionResult> ScanFile(
[Required] [MaxFileSize] IFormFile file)
[Required] [MaxFileSize] IFormFile file,
[FromForm] [IsHttpUrl(false)] string callbackUrl = null)
{
var result = await _scanResultService.CreateScanResult();
var result = await _scanResultService.CreateScanResult(
string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
string storedFileId;
@ -56,13 +60,16 @@ namespace MalwareMultiScan.Api.Controllers
/// Queue URL for scanning.
/// </summary>
/// <param name="url">URL from form data.</param>
/// <param name="callbackUrl">Optional callback URL.</param>
[HttpPost("url")]
[ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]
public async Task<IActionResult> ScanUrl(
[FromForm] [Required] [IsHttpUrl] string url)
[FromForm] [Required] [IsHttpUrl] string url,
[FromForm] [IsHttpUrl(false)] string callbackUrl = null)
{
var result = await _scanResultService.CreateScanResult();
var result = await _scanResultService.CreateScanResult(
string.IsNullOrEmpty(callbackUrl) ? null : new Uri(callbackUrl));
await _scanResultService.QueueUrlScan(result, url);

2
MalwareMultiScan.Api/Controllers/ScanResultsController.cs
View File

@ -15,7 +15,7 @@ namespace MalwareMultiScan.Api.Controllers
public class ScanResultsController : Controller
{
private readonly IScanResultService _scanResultService;
/// <summary>
/// Initialize scan results controller.
/// </summary>

2
MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs
View File

@ -9,7 +9,7 @@ namespace MalwareMultiScan.Api.Data.Configuration
/// Backend id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend state.
/// </summary>

8
MalwareMultiScan.Api/Data/Models/ScanResult.cs
View File

@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
@ -17,7 +18,12 @@ namespace MalwareMultiScan.Api.Data.Models
public string Id { get; set; }
/// <summary>
/// Result entries where key is backend id and value is <see cref="ScanResultEntry"/>.
/// Optional callback URL.
/// </summary>
public Uri CallbackUrl { get; set; }
/// <summary>
/// Result entries where key is backend id and value is <see cref="ScanResultEntry" />.
/// </summary>
public Dictionary<string, ScanResultEntry> Results { get; set; } =
new Dictionary<string, ScanResultEntry>();

6
MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs
View File

@ -9,17 +9,17 @@ namespace MalwareMultiScan.Api.Data.Models
/// Completion status.
/// </summary>
public bool Completed { get; set; }
/// <summary>
/// Indicates that scanning completed without error.
/// </summary>
public bool? Succeeded { get; set; }
/// <summary>
/// Scanning duration in seconds.
/// </summary>
public long Duration { get; set; }
/// <summary>
/// Detected names of threats.
/// </summary>

60
MalwareMultiScan.Api/Services/Implementations/ReceiverHostedService.cs
View File

@ -1,3 +1,6 @@
using System;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
@ -5,6 +8,7 @@ using MalwareMultiScan.Api.Services.Interfaces;
using MalwareMultiScan.Backends.Messages;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace MalwareMultiScan.Api.Services.Implementations
{
@ -13,6 +17,7 @@ namespace MalwareMultiScan.Api.Services.Implementations
{
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ReceiverHostedService> _logger;
private readonly IScanResultService _scanResultService;
@ -23,31 +28,23 @@ namespace MalwareMultiScan.Api.Services.Implementations
/// <param name="configuration">Configuration.</param>
/// <param name="scanResultService">Scan result service.</param>
/// <param name="logger">Logger.</param>
/// <param name="httpClientFactory">HTTP client factory.</param>
public ReceiverHostedService(IBus bus, IConfiguration configuration, IScanResultService scanResultService,
ILogger<ReceiverHostedService> logger)
ILogger<ReceiverHostedService> logger, IHttpClientFactory httpClientFactory)
{
_bus = bus;
_configuration = configuration;
_scanResultService = scanResultService;
_logger = logger;
_httpClientFactory = httpClientFactory;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanResultMessage>(_configuration.GetValue<string>("ResultsSubscriptionId"), async message =>
{
message.Threats ??= new string[] { };
_logger.LogInformation(
$"Received a result from {message.Backend} for {message.Id} " +
$"with threats {string.Join(",", message.Threats)}");
await _scanResultService.UpdateScanResultForBackend(
message.Id, message.Backend, message.Duration, true,
message.Succeeded, message.Threats);
});
_bus.Receive<ScanResultMessage>(
_configuration.GetValue<string>("ResultsSubscriptionId"), StoreScanResult);
_logger.LogInformation(
"Started hosted service for receiving scan results");
@ -65,5 +62,42 @@ namespace MalwareMultiScan.Api.Services.Implementations
return Task.CompletedTask;
}
private async Task StoreScanResult(ScanResultMessage message)
{
message.Threats ??= new string[] { };
_logger.LogInformation(
$"Received a result from {message.Backend} for {message.Id} " +
$"with threats {string.Join(",", message.Threats)}");
await _scanResultService.UpdateScanResultForBackend(
message.Id, message.Backend, message.Duration, true,
message.Succeeded, message.Threats);
var result = await _scanResultService.GetScanResult(message.Id);
if (result?.CallbackUrl == null)
return;
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(3));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var response = await httpClient.PostAsync(
result.CallbackUrl,
new StringContent(JsonConvert.SerializeObject(result), Encoding.UTF8, "application/json"),
cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(exception, $"Failed to POST to callback URL {result.CallbackUrl}");
}
}
}
}

7
MalwareMultiScan.Api/Services/Implementations/ScanResultService.cs
View File

@ -1,3 +1,4 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@ -31,12 +32,14 @@ namespace MalwareMultiScan.Api.Services.Implementations
_collection = db.GetCollection<ScanResult>(CollectionName);
}
/// <inheritdoc />
public async Task<ScanResult> CreateScanResult()
public async Task<ScanResult> CreateScanResult(Uri callbackUrl)
{
var scanResult = new ScanResult
{
CallbackUrl = callbackUrl,
Results = _scanBackendService.List
.Where(b => b.Enabled)
.ToDictionary(k => k.Id, v => new ScanResultEntry())

4
MalwareMultiScan.Api/Services/Interfaces/IScanResultService.cs
View File

@ -1,3 +1,4 @@
using System;
using System.IO;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
@ -12,8 +13,9 @@ namespace MalwareMultiScan.Api.Services.Interfaces
/// <summary>
/// Create scan result.
/// </summary>
/// <param name="callbackUrl">Optional callback URL.</param>
/// <returns>Scan result entry with id.</returns>
Task<ScanResult> CreateScanResult();
Task<ScanResult> CreateScanResult(Uri callbackUrl);
/// <summary>
/// Get scan result.

2
MalwareMultiScan.Api/Startup.cs
View File

@ -38,6 +38,8 @@ namespace MalwareMultiScan.Api
services.AddControllers();
services.AddHostedService<ReceiverHostedService>();
services.AddHttpClient();
}
public void Configure(IApplicationBuilder app)

8
MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs
View File

@ -22,17 +22,17 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
/// Regex to extract names of threats.
/// </summary>
protected abstract Regex MatchRegex { get; }
/// <summary>
/// Path to the backend.
/// </summary>
protected abstract string BackendPath { get; }
/// <summary>
/// Parse StdErr instead of StdOut.
/// </summary>
protected virtual bool ParseStdErr { get; } = false;
/// <summary>
/// Throw on non-zero exit code.
/// </summary>
@ -44,7 +44,7 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
/// <param name="path">Path to the temporary file.</param>
/// <returns>Formatted string with parameters and path.</returns>
protected abstract string GetBackendArguments(string path);
/// <inheritdoc />
public override Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
{

4
MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs
View File

@ -11,7 +11,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public WindowsDefenderScanBackend(IProcessRunner processRunner) : base(processRunner)
{
}
/// <inheritdoc />
public override string Id { get; } = "windows-defender";
@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
protected override Regex MatchRegex { get; } =
new Regex(@"EngineScanCallback\(\): Threat (?<threat>[\S]+) identified",
RegexOptions.Compiled | RegexOptions.Multiline);
/// <inheritdoc />
protected override bool ParseStdErr { get; } = true;

14
MalwareMultiScan.Backends/Enums/BackendType.cs
View File

@ -9,37 +9,37 @@ namespace MalwareMultiScan.Backends.Enums
/// Dummy
/// </summary>
Dummy,
/// <summary>
/// Windows Defender.
/// </summary>
Defender,
/// <summary>
/// ClamAV.
/// </summary>
Clamav,
/// <summary>
/// DrWeb.
/// </summary>
DrWeb,
/// <summary>
/// KES.
/// </summary>
Kes,
/// <summary>
/// Comodo.
/// </summary>
Comodo,
/// <summary>
/// Sophos.
/// </summary>
Sophos,
/// <summary>
/// McAfee.
/// </summary>

6
MalwareMultiScan.Backends/Interfaces/IScanBackend.cs
View File

@ -14,7 +14,7 @@ namespace MalwareMultiScan.Backends.Interfaces
/// Unique backend id.
/// </summary>
public string Id { get; }
/// <summary>
/// Scan file.
/// </summary>
@ -22,7 +22,7 @@ namespace MalwareMultiScan.Backends.Interfaces
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of detected threats.</returns>
public Task<string[]> ScanAsync(string path, CancellationToken cancellationToken);
/// <summary>
/// Scan URL.
/// </summary>
@ -30,7 +30,7 @@ namespace MalwareMultiScan.Backends.Interfaces
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of detected threats.</returns>
public Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken);
/// <summary>
/// Scan stream.
/// </summary>

4
MalwareMultiScan.Backends/Messages/ScanResultMessage.cs
View File

@ -9,7 +9,7 @@ namespace MalwareMultiScan.Backends.Messages
/// Result id.
/// </summary>
public string Id { get; set; }
/// <summary>
/// Backend.
/// </summary>
@ -19,7 +19,7 @@ namespace MalwareMultiScan.Backends.Messages
/// Status.
/// </summary>
public bool Succeeded { get; set; }
/// <summary>
/// List of detected threats.
/// </summary>

2
MalwareMultiScan.Backends/Services/Implementations/ProcessRunner.cs
View File

@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Backends.Services.Implementations
{
/// <summary>
///
/// </summary>
[ExcludeFromCodeCoverage]
public class ProcessRunner : IProcessRunner
@ -26,7 +25,6 @@ namespace MalwareMultiScan.Backends.Services.Implementations
}
/// <summary>
///
/// </summary>
/// <param name="path"></param>
/// <param name="arguments"></param>

2
MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj
View File

@ -10,6 +10,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.17" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="3.1.9" />
</ItemGroup>

12
MalwareMultiScan.Scanner/Program.cs
View File

@ -1,7 +1,10 @@
using System.Diagnostics.CodeAnalysis;
using System.Threading.Tasks;
using Hangfire;
using Hangfire.MemoryStorage;
using MalwareMultiScan.Backends.Extensions;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@ -26,7 +29,16 @@ namespace MalwareMultiScan.Scanner
services.AddRabbitMq(context.Configuration);
services.AddScanningBackend(context.Configuration);
services.AddSingleton<IScanBackgroundJob, ScanBackgroundJob>();
services.AddHostedService<ScanHostedService>();
services.AddHangfire(
configuration => configuration.UseMemoryStorage());
services.AddHangfireServer(options =>
{
options.WorkerCount = context.Configuration.GetValue<int>("WorkerCount");
});
}).RunConsoleAsync();
}
}

89
MalwareMultiScan.Scanner/Services/Implementations/ScanBackgroundJob.cs Normal file
View File

@ -0,0 +1,89 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
{
/// <inheritdoc />
public class ScanBackgroundJob : IScanBackgroundJob
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly ILogger<ScanBackgroundJob> _logger;
/// <summary>
/// Initialize scan background job.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="bus">Bus.</param>
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
public ScanBackgroundJob(IConfiguration configuration, IBus bus,
ILogger<ScanBackgroundJob> logger, IScanBackend backend)
{
_configuration = configuration;
_bus = bus;
_logger = logger;
_backend = backend;
}
/// <inheritdoc />
public async Task Process(ScanRequestMessage message)
{
_logger.LogInformation(
$"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
var result = new ScanResultMessage
{
Id = message.Id,
Backend = _backend.Id
};
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
result.Threats = await _backend.ScanAsync(
message.Uri, cancellationTokenSource.Token);
result.Succeeded = true;
_logger.LogInformation(
$"Backend {_backend.Id} completed a scan of {message.Id} " +
$"with result '{string.Join(", ", result.Threats)}'");
}
catch (Exception exception)
{
result.Succeeded = false;
_logger.LogError(
exception, "Scanning failed with exception");
}
finally
{
stopwatch.Stop();
}
result.Duration = stopwatch.ElapsedMilliseconds / 1000;
_logger.LogInformation(
$"Sending scan results with status {result.Succeeded}");
await _bus.SendAsync(
_configuration.GetValue<string>("ResultsSubscriptionId"), result);
}
}
}

63
MalwareMultiScan.Scanner/Services/Implementations/ScanHostedService.cs
View File

@ -1,12 +1,10 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using Hangfire;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Scanner.Services.Implementations
@ -16,7 +14,6 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
{
private readonly IScanBackend _backend;
private readonly IBus _bus;
private readonly IConfiguration _configuration;
private readonly ILogger<ScanHostedService> _logger;
/// <summary>
@ -25,22 +22,20 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
/// <param name="logger">Logger.</param>
/// <param name="backend">Scan backend.</param>
/// <param name="bus">EasyNetQ bus.</param>
/// <param name="configuration">Configuration.</param>
public ScanHostedService(
ILogger<ScanHostedService> logger,
IScanBackend backend, IBus bus,
IConfiguration configuration)
IScanBackend backend, IBus bus)
{
_logger = logger;
_bus = bus;
_configuration = configuration;
_backend = backend;
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken)
{
_bus.Receive<ScanRequestMessage>(_backend.Id, Scan);
_bus.Receive<ScanRequestMessage>(_backend.Id, message =>
BackgroundJob.Enqueue<IScanBackgroundJob>(j => j.Process(message)));
_logger.LogInformation(
$"Started scan hosting service for the backend {_backend.Id}");
@ -58,55 +53,5 @@ namespace MalwareMultiScan.Scanner.Services.Implementations
return Task.CompletedTask;
}
private async Task Scan(ScanRequestMessage message)
{
_logger.LogInformation(
$"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}");
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(_configuration.GetValue<int>("MaxScanningTime")));
var result = new ScanResultMessage
{
Id = message.Id,
Backend = _backend.Id
};
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
result.Threats = await _backend.ScanAsync(
message.Uri, cancellationTokenSource.Token);
result.Succeeded = true;
_logger.LogInformation(
$"Backend {_backend.Id} completed a scan of {message.Id} " +
$"with result '{string.Join(", ", result.Threats)}'");
}
catch (Exception exception)
{
result.Succeeded = false;
_logger.LogError(
exception, "Scanning failed with exception");
}
finally
{
stopwatch.Stop();
}
result.Duration = stopwatch.ElapsedMilliseconds / 1000;
_logger.LogInformation(
$"Sending scan results with status {result.Succeeded}");
await _bus.SendAsync(
_configuration.GetValue<string>("ResultsSubscriptionId"), result);
}
}
}

17
MalwareMultiScan.Scanner/Services/Interfaces/IScanBackgroundJob.cs Normal file
View File

@ -0,0 +1,17 @@
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Messages;
namespace MalwareMultiScan.Scanner.Services.Interfaces
{
/// <summary>
/// Background scanning job.
/// </summary>
public interface IScanBackgroundJob
{
/// <summary>
/// Process scan request in the background worker.
/// </summary>
/// <param name="requestMessage">Request message.</param>
Task Process(ScanRequestMessage requestMessage);
}
}

1
MalwareMultiScan.Scanner/appsettings.json
View File

@ -11,6 +11,7 @@
"MaxScanningTime": 60,
"ResultsSubscriptionId": "mms.results",
"WorkerCount": 4,
"ConnectionStrings": {
"RabbitMQ": "host=localhost;prefetchcount=1"

2
MalwareMultiScan.Tests/Api/ControllersTests.cs
View File

@ -46,7 +46,7 @@ namespace MalwareMultiScan.Tests.Api
.Returns(Task.FromResult((Stream) null));
_scanResultServiceMock
.Setup(x => x.CreateScanResult())
.Setup(x => x.CreateScanResult(null))
.Returns(Task.FromResult(new ScanResult()));
_downloadController = new DownloadController(_scanResultServiceMock.Object);

6
MalwareMultiScan.Tests/Api/ReceiverHostedServiceTests.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Api.Services.Implementations;
@ -50,11 +51,14 @@ namespace MalwareMultiScan.Tests.Api
["ResultsSubscriptionId"] = "mms.results"
};
var httpClientFactoryMock = new Mock<IHttpClientFactory>();
_receiverHostedService = new ReceiverHostedService(
_busMock.Object,
configuration,
_scanResultServiceMock.Object,
Mock.Of<ILogger<ReceiverHostedService>>());
Mock.Of<ILogger<ReceiverHostedService>>(),
httpClientFactoryMock.Object);
}
[Test]

4
MalwareMultiScan.Tests/Api/ScanResultServiceTests.cs
View File

@ -82,7 +82,7 @@ namespace MalwareMultiScan.Tests.Api
[Test]
public async Task TestScanResultCreateGet()
{
var result = await _resultService.CreateScanResult();
var result = await _resultService.CreateScanResult(null);
Assert.NotNull(result.Id);
Assert.That(result.Results, Contains.Key("dummy"));
@ -107,7 +107,7 @@ namespace MalwareMultiScan.Tests.Api
[Test]
public async Task TestQueueUrlScan()
{
var result = await _resultService.CreateScanResult();
var result = await _resultService.CreateScanResult(null);
await _resultService.QueueUrlScan(
result, "http://url.com");

68
MalwareMultiScan.Tests/Scanner/ScanBackgroundJobTests.cs Normal file
View File

@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace MalwareMultiScan.Tests.Scanner
{
public class ScanBackgroundJobTests
{
private Mock<IBus> _busMock;
private Mock<IScanBackend> _scanBackendMock;
private IScanBackgroundJob _scanBackgroundJob;
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
_busMock = new Mock<IBus>();
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
_scanBackendMock
.Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new[] {"Test"}));
_scanBackgroundJob = new ScanBackgroundJob(configuration, _busMock.Object,
Mock.Of<ILogger<ScanBackgroundJob>>(), _scanBackendMock.Object);
}
[Test]
public async Task TestMessageProcessing()
{
await _scanBackgroundJob.Process(new ScanRequestMessage
{
Id = "test",
Uri = new Uri("http://test.com")
});
_scanBackendMock.Verify(
x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()));
_busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(m =>
m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test"))));
}
}
}

38
MalwareMultiScan.Tests/Scanner/ScanHostedServiceTests.cs
View File

@ -1,15 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EasyNetQ;
using MalwareMultiScan.Backends.Interfaces;
using MalwareMultiScan.Backends.Messages;
using MalwareMultiScan.Scanner.Services.Implementations;
using MalwareMultiScan.Scanner.Services.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration.Memory;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
@ -25,35 +20,10 @@ namespace MalwareMultiScan.Tests.Scanner
[SetUp]
public void SetUp()
{
var configuration = new ConfigurationRoot(new List<IConfigurationProvider>
{
new MemoryConfigurationProvider(new MemoryConfigurationSource())
})
{
["ResultsSubscriptionId"] = "mms.results"
};
_busMock = new Mock<IBus>();
_busMock
.Setup(x => x.Receive("dummy", It.IsAny<Func<ScanRequestMessage, Task>>()))
.Callback<string, Func<ScanRequestMessage, Task>>((s, func) =>
{
var task = func.Invoke(new ScanRequestMessage
{
Id = "test",
Uri = new Uri("http://test.com")
});
task.Wait();
});
_scanBackendMock = new Mock<IScanBackend>();
_scanBackendMock
.Setup(x => x.ScanAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()))
.Returns(Task.FromResult(new[] {"Test"}));
_scanBackendMock
.SetupGet(x => x.Id)
.Returns("dummy");
@ -61,8 +31,7 @@ namespace MalwareMultiScan.Tests.Scanner
_scanHostedService = new ScanHostedService(
Mock.Of<ILogger<ScanHostedService>>(),
_scanBackendMock.Object,
_busMock.Object,
configuration);
_busMock.Object);
}
[Test]
@ -70,9 +39,8 @@ namespace MalwareMultiScan.Tests.Scanner
{
await _scanHostedService.StartAsync(default);
_busMock.Verify(x => x.SendAsync("mms.results", It.Is<ScanResultMessage>(
m => m.Succeeded && m.Backend == "dummy" && m.Id == "test" && m.Threats.Contains("Test")
)));
_busMock.Verify(
x => x.Receive("dummy", It.IsAny<Action<ScanRequestMessage>>()));
}
[Test]

13
MalwareMultiScan.Ui/pages/index.vue
View File

@ -9,6 +9,9 @@
</b-input-group-append>
</b-input-group>
<b-input id="callback-url" v-model="callbackUrl" class="mt-3"
placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
<b-progress v-if="uploading" :value="progress" animated class="mt-3"/>
</b-tab>
@ -19,6 +22,9 @@
<b-button :disabled="!isUrl(url)" variant="primary" @click="scanUrl">Scan</b-button>
</b-input-group-append>
</b-input-group>
<b-input id="callback-url" v-model="callbackUrl" class="mt-3"
placeholder="Optional callback URL. I.e. https://pipedream.com/" type="url"/>
</b-tab>
</b-tabs>
</b-card>
@ -37,6 +43,7 @@ export default Vue.extend({
file: null as File | null,
url: '' as string,
callbackUrl: '' as string
}
},
@ -53,6 +60,9 @@ export default Vue.extend({
data.append('file', this.file, this.file.name);
if (this.isUrl(this.callbackUrl))
data.append('callbackUrl', this.callbackUrl);
try {
this.uploading = true;
@ -80,6 +90,9 @@ export default Vue.extend({
data.append('url', this.url);
if (this.isUrl(this.callbackUrl))
data.append('callbackUrl', this.callbackUrl);
const result = await this.$axios.$post<ScanResult>('queue/url', data);
await this.$router.push({name: 'id', params: {id: result.id}});

2
MalwareMultiScan.sln
View File

@ -11,6 +11,8 @@ ProjectSection(SolutionItems) = preProject
.dockerignore = .dockerignore
.gitignore = .gitignore
docker-compose.yaml = docker-compose.yaml
README.md = README.md
LICENSE = LICENSE
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Tests", "MalwareMultiScan.Tests\MalwareMultiScan.Tests.csproj", "{9896162D-8FC7-4911-933F-A78C94128923}"

18
README.md
View File

@ -18,6 +18,12 @@ In the end, it's nothing but the set of Docker containers running the agent. Tha
**IMPORTANT**: MalwareMultiScan is not intended as a publicly-facing API / UI. It has (intentionally) no authorization, authentication, rate-limiting, or logging. Therefore, it should be used only as an internal / private API or behind the restrictive API gateway.
Whole solution can be started with `docker-compose up` executed in a root folder of repository.
It can be also deployed to the Docker Swarm cluster by using the command `docker stack deploy malware-multi-scan --compose-file docker-compose.yaml`.
After the start the Demo Web UI will become available under http://localhost:8888.
See [components](#components) chapter below and the [docker-compose.yaml](docker-compose.yaml) file.
### Configuration
@ -48,18 +54,22 @@ Configuration of API and Scanners is performed by passing the environment variab
* `MaxScanningTime=60` - Scan time limit. It is used not just for actual scanning but also for getting the file.
* `WorkerCount=4` - Number of workers for parallel scanning.
#### MalwareMultiScan.Ui
* `API_URL=http://localhost:5000` - Absolute URL incl. port number for the running instance of MalwareMultiScan.Api.
### API Endpoints
* POST `/api/queue/url` with a `url` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/url` with a `url` parameter passed via the form data.. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* POST `/api/queue/file` with a `file` parameter passed via the form data. Returns `201 Accepted` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `400 Bad Request` error.
* GET `/api/results/{result-id}` where `{result-id}` corresponds to the id value of a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs). Returns `200 OK` response with a [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) or `404 Not Found` error.
Both `/api/queue/url` and `/api/queue/file` also accept an optional `callbackUrl` parameter with the http(s) URL in it. This URL will be requested by the POST method with JSON serialized [ScanResult](MalwareMultiScan.Api/Data/Models/ScanResult.cs) in a body on every update from scan backends.
## Supported Scan Engines
| Name | Dockerfile | Enabled | Comments |
@ -85,7 +95,7 @@ More scan backends can be added in the future. Some of the popular ones do not h
* **Docker** and **docker-compose** running under Windows (in Linux containers mode), Linux, or OSX. Docker Compose is needed only for test / local deployments.
* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities. A simultaneous scan by a single node is not supported, yet load-balancing is possible by the built-in orchestrators' round-robin routing.
* **Optional**: DockerSwarm / Kubernetes cluster for scaling up the scanning capacities.
### Parts
@ -99,4 +109,4 @@ More scan backends can be added in the future. Some of the popular ones do not h
## Plans
See [issues](https://github.com/mindcollapse/MalwareMultiScan/issues) for the list of planned features, bug-fixes, and improvements.
See [issues](https://github.com/mindcollapse/MalwareMultiScan/issues) for the list of planned features, bug-fixes, and improvements.