diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml new file mode 100644 index 0000000..aca45d3 --- /dev/null +++ b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/Dockerfiles_Clamav_Dockerfile.xml @@ -0,0 +1,24 @@ + + + + + + + + + + \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs index ea44aac..11e2f98 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs @@ -12,7 +12,12 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts public abstract class AbstractLocalProcessScanBackend : AbstractScanBackend { private readonly ILogger _logger; - + + protected AbstractLocalProcessScanBackend(ILogger logger) + { + _logger = logger; + } + protected virtual Regex MatchRegex { get; } protected virtual string BackendPath { get; } protected virtual bool ParseStdErr { get; } @@ -22,11 +27,6 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts throw new NotImplementedException(); } - protected AbstractLocalProcessScanBackend(ILogger logger) - { - _logger = logger; - } - public override async Task ScanAsync(string path, CancellationToken cancellationToken) { var process = new Process @@ -38,12 +38,12 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts WorkingDirectory = Path.GetDirectoryName(BackendPath) ?? Directory.GetCurrentDirectory() } }; - + _logger.LogInformation( $"Starting process {process.StartInfo.FileName} " + $"with arguments {process.StartInfo.Arguments} " + $"in working directory {process.StartInfo.WorkingDirectory}"); - + process.Start(); cancellationToken.Register(() => @@ -51,14 +51,14 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts if (!process.HasExited) process.Kill(true); }); - + process.WaitForExit(); _logger.LogInformation($"Process has exited with code {process.ExitCode}"); var standardOutput = await process.StandardOutput.ReadToEndAsync(); var standardError = await process.StandardError.ReadToEndAsync(); - + _logger.LogDebug($"Process standard output: {standardOutput}"); _logger.LogDebug($"Process standard error: {standardError}"); diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs index 166d89c..e1aaee8 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs @@ -11,9 +11,9 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts public abstract class AbstractScanBackend : IScanBackend { public virtual string Name => throw new NotImplementedException(); - + public virtual DateTime DatabaseLastUpdate => throw new NotImplementedException(); - + public virtual Task ScanAsync(string path, CancellationToken cancellationToken) { throw new NotImplementedException(); @@ -23,14 +23,14 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts { using var httpClient = new HttpClient(); await using var uriStream = await httpClient.GetStreamAsync(uri); - + return await ScanAsync(uriStream, cancellationToken); } public async Task ScanAsync(IFormFile file, CancellationToken cancellationToken) { await using var fileStream = file.OpenReadStream(); - + return await ScanAsync(fileStream, cancellationToken); } @@ -41,7 +41,9 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts try { await using (var tempFileStream = File.OpenWrite(tempFile)) + { await stream.CopyToAsync(tempFileStream, cancellationToken); + } return await ScanAsync(tempFile, cancellationToken); } diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs new file mode 100644 index 0000000..c2eefbe --- /dev/null +++ b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs @@ -0,0 +1,32 @@ +using System; +using System.IO; +using System.Text.RegularExpressions; +using MalwareMultiScan.Backends.Backends.Abstracts; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Backends.Backends.Implementations +{ + public class ClamavScanBackend : AbstractLocalProcessScanBackend + { + public ClamavScanBackend(ILogger logger) : base(logger) + { + } + + public override string Name { get; } = "Clamav"; + + public override DateTime DatabaseLastUpdate => + File.GetLastWriteTime("/var/lib/clamav/daily.cvd"); + + protected override string BackendPath { get; } = "/usr/bin/clamscan"; + + protected override Regex MatchRegex { get; } = + new Regex(@"(\S+): (?[\S]+) FOUND", RegexOptions.Compiled | RegexOptions.Multiline); + + protected override bool ParseStdErr { get; } = false; + + protected override string GetBackendArguments(string path) + { + return $"--no-summary {path}"; + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs index 4401d2e..faf1457 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; @@ -9,22 +8,26 @@ namespace MalwareMultiScan.Backends.Backends.Implementations { public class WindowsDefenderScanBackend : AbstractLocalProcessScanBackend { + public WindowsDefenderScanBackend(ILogger logger) : base(logger) + { + } + public override string Name { get; } = "Windows Defender"; public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/opt/engine/mpavbase.vdm"); protected override string BackendPath { get; } = "/opt/mpclient"; - protected override Regex MatchRegex { get; } = - new Regex(@"EngineScanCallback\(\)\: Threat (?[\S]+) identified", + + protected override Regex MatchRegex { get; } = + new Regex(@"EngineScanCallback\(\): Threat (?[\S]+) identified", RegexOptions.Compiled | RegexOptions.Multiline); - + protected override bool ParseStdErr { get; } = true; - protected override string GetBackendArguments(string path) => path; - - public WindowsDefenderScanBackend(ILogger logger) : base(logger) + protected override string GetBackendArguments(string path) { + return path; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile index 372960e..310618e 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile @@ -1,8 +1,10 @@ -FROM mindcollapse/malware-multi-scan-base:latest +FROM mindcollapse/malware-multi-scan-worker:latest ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y clamav RUN freshclam --quiet -ENV MULTI_SCAN_BACKEND_BIN=/usr/bin/clamscan \ No newline at end of file +ENV MULTI_SCAN_BACKEND_BIN=/usr/bin/clamscan + +ENV BackendType=Clamav \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile index 3da0522..6444031 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile @@ -18,5 +18,4 @@ RUN apt-get update && apt-get install -y libc6-i386 COPY --from=backend /opt/loadlibrary/engine /opt/engine COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient -ENV BackendType=Defender -ENV ScanTimeout=300 \ No newline at end of file +ENV BackendType=Defender \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs b/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs index 651cf9d..ba6f232 100644 --- a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs +++ b/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs @@ -8,10 +8,10 @@ namespace MalwareMultiScan.Shared.Attributes protected override ValidationResult IsValid(object value, ValidationContext validationContext) { var uri = (Uri) value; - + if (uri == null || uri.Scheme != "http" && uri.Scheme != "https") return new ValidationResult("Only http(s) URLs are supported"); - + return ValidationResult.Success; } } diff --git a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs b/MalwareMultiScan.Shared/Data/Enums/BackendType.cs index c2e3f4f..ff69881 100644 --- a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs +++ b/MalwareMultiScan.Shared/Data/Enums/BackendType.cs @@ -2,6 +2,7 @@ namespace MalwareMultiScan.Shared.Data.Enums { public enum BackendType { - Defender + Defender, + Clamav } } \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs b/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs index febdd54..0d33c63 100644 --- a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs +++ b/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs @@ -6,13 +6,13 @@ namespace MalwareMultiScan.Shared.Data.Responses public class ResultResponse { public string Backend { get; set; } - + public bool Success { get; set; } public DateTime? DatabaseLastUpdate { get; set; } public bool Detected => Threats?.Any() == true; - + public string[] Threats { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs b/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs index ef75d2e..c36601b 100644 --- a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs +++ b/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs @@ -9,9 +9,9 @@ namespace MalwareMultiScan.Shared.Interfaces public interface IScanBackend { public string Name { get; } - + public DateTime DatabaseLastUpdate { get; } - + public Task ScanAsync(string path, CancellationToken cancellationToken); public Task ScanAsync(Uri uri, CancellationToken cancellationToken); public Task ScanAsync(IFormFile file, CancellationToken cancellationToken); diff --git a/MalwareMultiScan.Worker/Controllers/ScanController.cs b/MalwareMultiScan.Worker/Controllers/ScanController.cs index 4b9bb19..c41dadd 100644 --- a/MalwareMultiScan.Worker/Controllers/ScanController.cs +++ b/MalwareMultiScan.Worker/Controllers/ScanController.cs @@ -21,8 +21,10 @@ namespace MalwareMultiScan.Worker.Controllers var temporaryFile = Path.GetTempFileName(); await using (var temporaryFileSteam = System.IO.File.OpenWrite(temporaryFile)) + { await request.InputFile.CopyToAsync(temporaryFileSteam); - + } + BackgroundJob.Enqueue( x => x.ScanFile(temporaryFile, request.CallbackUrl)); @@ -33,7 +35,7 @@ namespace MalwareMultiScan.Worker.Controllers [ProducesResponseType(StatusCodes.Status202Accepted)] [ProducesResponseType(StatusCodes.Status400BadRequest)] [Route("/scan/url")] - public IActionResult ScanUrl(UrlRequest request) + public IActionResult ScanUrl([FromForm] UrlRequest request) { BackgroundJob.Enqueue( x => x.ScanUrl(request.InputUrl, request.CallbackUrl)); diff --git a/MalwareMultiScan.Worker/Dockerfile b/MalwareMultiScan.Worker/Dockerfile index e097379..1791d67 100644 --- a/MalwareMultiScan.Worker/Dockerfile +++ b/MalwareMultiScan.Worker/Dockerfile @@ -6,8 +6,7 @@ COPY MalwareMultiScan.Worker /src/MalwareMultiScan.Worker COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends -RUN dotnet publish -c Release -r linux-x64 /p:PublishSingleFile=true -o ./publish \ - MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj +RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 diff --git a/MalwareMultiScan.Worker/Jobs/ScanJob.cs b/MalwareMultiScan.Worker/Jobs/ScanJob.cs index 12d8a07..fbc4a83 100644 --- a/MalwareMultiScan.Worker/Jobs/ScanJob.cs +++ b/MalwareMultiScan.Worker/Jobs/ScanJob.cs @@ -1,8 +1,11 @@ using System; using System.IO; +using System.Net.Http; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Hangfire; using MalwareMultiScan.Backends.Backends.Implementations; using MalwareMultiScan.Shared.Data.Enums; using MalwareMultiScan.Shared.Data.Responses; @@ -14,37 +17,49 @@ namespace MalwareMultiScan.Worker.Jobs { public class ScanJob { - private readonly ILogger _logger; private readonly IScanBackend _backend; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; private readonly int _scanTimeout; - - public ScanJob(IConfiguration configuration, ILogger logger) + + public ScanJob(IConfiguration configuration, ILogger logger, + IHttpClientFactory httpClientFactory) { _logger = logger; + _httpClientFactory = httpClientFactory; _scanTimeout = configuration.GetValue("ScanTimeout"); - + _backend = configuration.GetValue("BackendType") switch { BackendType.Defender => new WindowsDefenderScanBackend(logger), + BackendType.Clamav => new ClamavScanBackend(logger), _ => throw new NotImplementedException() }; } - public async Task PostResult(ResultResponse response, Uri callbackUrl, CancellationToken cancellationToken) + private async Task PostResult(ResultResponse response, Uri callbackUrl, CancellationToken cancellationToken) { var serializedResponse = JsonSerializer.Serialize( response, typeof(ResultResponse), new JsonSerializerOptions { WriteIndented = true }); - - _logger.LogInformation(serializedResponse); + + _logger.LogInformation( + $"Sending following payload to {callbackUrl}: {serializedResponse}"); + + using var httpClient = _httpClientFactory.CreateClient(); + + var callbackResponse = await httpClient.PostAsync(callbackUrl, + new StringContent(serializedResponse, Encoding.UTF8, "application/json"), cancellationToken); + + _logger.LogInformation($"Callback URL {callbackUrl} returned a status {callbackResponse.StatusCode}"); } private async Task Scan(Func> scanMethod, Uri callbackUrl) { var cancellationTokenSource = new CancellationTokenSource(); - + cancellationTokenSource.CancelAfter(_scanTimeout * 1000); var cancellationToken = cancellationTokenSource.Token; @@ -53,7 +68,7 @@ namespace MalwareMultiScan.Worker.Jobs { Backend = _backend.Name }; - + try { response.Success = true; @@ -63,18 +78,20 @@ namespace MalwareMultiScan.Worker.Jobs catch (Exception exception) { response.Success = false; - + _logger.LogError(exception, "Scanning failed with exception"); } await PostResult(response, callbackUrl, cancellationToken); } + [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public async Task ScanUrl(Uri url, Uri callbackUrl) { await Scan(async t => await _backend.ScanAsync(url, t), callbackUrl); } + [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] public async Task ScanFile(string file, Uri callbackUrl) { try diff --git a/MalwareMultiScan.Worker/Startup.cs b/MalwareMultiScan.Worker/Startup.cs index c4e2f38..bd3ba6f 100644 --- a/MalwareMultiScan.Worker/Startup.cs +++ b/MalwareMultiScan.Worker/Startup.cs @@ -13,6 +13,7 @@ namespace MalwareMultiScan.Worker { services.AddLogging(); services.AddControllers(); + services.AddHttpClient(); services.AddSingleton();