From 46d696cdb12dafdaa714b104cb2a33d179f2c138 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Thu, 22 Oct 2020 22:20:21 +0300 Subject: [PATCH 1/6] initial commit for API --- .dockerignore | 1 + .../Controllers/WeatherForecastController.cs | 39 +++++++++++++++ MalwareMultiScan.Api/Dockerfile | 0 .../MalwareMultiScan.Api.csproj | 8 ++++ MalwareMultiScan.Api/Program.cs | 23 +++++++++ .../Properties/launchSettings.json | 30 ++++++++++++ MalwareMultiScan.Api/Startup.cs | 48 +++++++++++++++++++ MalwareMultiScan.Api/WeatherForecast.cs | 15 ++++++ .../appsettings.Development.json | 9 ++++ MalwareMultiScan.Api/appsettings.json | 10 ++++ MalwareMultiScan.sln | 6 +++ 11 files changed, 189 insertions(+) create mode 100644 MalwareMultiScan.Api/Controllers/WeatherForecastController.cs create mode 100644 MalwareMultiScan.Api/Dockerfile create mode 100644 MalwareMultiScan.Api/MalwareMultiScan.Api.csproj create mode 100644 MalwareMultiScan.Api/Program.cs create mode 100644 MalwareMultiScan.Api/Properties/launchSettings.json create mode 100644 MalwareMultiScan.Api/Startup.cs create mode 100644 MalwareMultiScan.Api/WeatherForecast.cs create mode 100644 MalwareMultiScan.Api/appsettings.Development.json create mode 100644 MalwareMultiScan.Api/appsettings.json diff --git a/.dockerignore b/.dockerignore index 217331b..c377dad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ .git +*.Dockerfile Dockerfile bin obj \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs b/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs new file mode 100644 index 0000000..8bf3c3c --- /dev/null +++ b/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Api.Controllers +{ + [ApiController] + [Route("[controller]")] + public class WeatherForecastController : ControllerBase + { + private static readonly string[] Summaries = new[] + { + "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" + }; + + private readonly ILogger _logger; + + public WeatherForecastController(ILogger logger) + { + _logger = logger; + } + + [HttpGet] + public IEnumerable Get() + { + var rng = new Random(); + return Enumerable.Range(1, 5).Select(index => new WeatherForecast + { + Date = DateTime.Now.AddDays(index), + TemperatureC = rng.Next(-20, 55), + Summary = Summaries[rng.Next(Summaries.Length)] + }) + .ToArray(); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Dockerfile b/MalwareMultiScan.Api/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj new file mode 100644 index 0000000..68ded6d --- /dev/null +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -0,0 +1,8 @@ + + + + netcoreapp3.1 + + + + diff --git a/MalwareMultiScan.Api/Program.cs b/MalwareMultiScan.Api/Program.cs new file mode 100644 index 0000000..c1e647c --- /dev/null +++ b/MalwareMultiScan.Api/Program.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Api +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Properties/launchSettings.json b/MalwareMultiScan.Api/Properties/launchSettings.json new file mode 100644 index 0000000..e2cb472 --- /dev/null +++ b/MalwareMultiScan.Api/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:45236", + "sslPort": 44367 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "MalwareMultiScan.Api": { + "commandName": "Project", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs new file mode 100644 index 0000000..a61cb51 --- /dev/null +++ b/MalwareMultiScan.Api/Startup.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.HttpsPolicy; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Api +{ + public class Startup + { + public Startup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) + { + services.AddControllers(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseHttpsRedirection(); + + app.UseRouting(); + + app.UseAuthorization(); + + app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/WeatherForecast.cs b/MalwareMultiScan.Api/WeatherForecast.cs new file mode 100644 index 0000000..19f8687 --- /dev/null +++ b/MalwareMultiScan.Api/WeatherForecast.cs @@ -0,0 +1,15 @@ +using System; + +namespace MalwareMultiScan.Api +{ + public class WeatherForecast + { + public DateTime Date { get; set; } + + public int TemperatureC { get; set; } + + public int TemperatureF => 32 + (int) (TemperatureC / 0.5556); + + public string Summary { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/appsettings.Development.json b/MalwareMultiScan.Api/appsettings.Development.json new file mode 100644 index 0000000..8983e0f --- /dev/null +++ b/MalwareMultiScan.Api/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/MalwareMultiScan.Api/appsettings.json b/MalwareMultiScan.Api/appsettings.json new file mode 100644 index 0000000..d9d9a9b --- /dev/null +++ b/MalwareMultiScan.Api/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/MalwareMultiScan.sln b/MalwareMultiScan.sln index 4d3aab6..4bdf84b 100644 --- a/MalwareMultiScan.sln +++ b/MalwareMultiScan.sln @@ -6,6 +6,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends", "MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj", "{382B49AC-0FFA-44FC-875D-9D4692DDC05D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -24,5 +26,9 @@ Global {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Debug|Any CPU.Build.0 = Debug|Any CPU {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Release|Any CPU.ActiveCfg = Release|Any CPU {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Release|Any CPU.Build.0 = Release|Any CPU + {7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal From 96695664e0ffc0cb8967fcfcbe8c3a980edac6b6 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Fri, 23 Oct 2020 11:11:19 +0300 Subject: [PATCH 2/6] basic structure and initial scan request service --- .../Controllers/WeatherForecastController.cs | 39 -------- .../Configuration/BackendConfiguration.cs | 9 ++ .../Data/Models/ScanRequest.cs | 19 ++++ .../Data/Models/ScanRequestEntry.cs | 9 ++ .../MalwareMultiScan.Api.csproj | 20 +++++ MalwareMultiScan.Api/Program.cs | 14 ++- .../Properties/launchSettings.json | 30 ------- .../Services/BackendConfigurationReader.cs | 29 ++++++ .../Services/ScanRequestService.cs | 90 +++++++++++++++++++ MalwareMultiScan.Api/Startup.cs | 42 ++++----- MalwareMultiScan.Api/WeatherForecast.cs | 15 ---- MalwareMultiScan.Api/appsettings.json | 10 ++- MalwareMultiScan.Api/backends.yaml | 7 ++ .../AbstractLocalProcessScanBackend.cs | 4 +- .../Implementations/ClamavScanBackend.cs | 6 +- .../Implementations/ComodoScanBackend.cs | 8 +- .../Implementations/DrWebScanBackend.cs | 6 +- .../Implementations/KesScanBackend.cs | 6 +- .../Implementations/McAffeeScanBackend.cs | 6 +- .../Implementations/SophosScanBackend.cs | 4 +- .../MalwareMultiScan.Worker.csproj | 4 + .../Properties/launchSettings.json | 30 ------- MalwareMultiScan.Worker/Startup.cs | 3 +- 23 files changed, 242 insertions(+), 168 deletions(-) delete mode 100644 MalwareMultiScan.Api/Controllers/WeatherForecastController.cs create mode 100644 MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs create mode 100644 MalwareMultiScan.Api/Data/Models/ScanRequest.cs create mode 100644 MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs delete mode 100644 MalwareMultiScan.Api/Properties/launchSettings.json create mode 100644 MalwareMultiScan.Api/Services/BackendConfigurationReader.cs create mode 100644 MalwareMultiScan.Api/Services/ScanRequestService.cs delete mode 100644 MalwareMultiScan.Api/WeatherForecast.cs create mode 100644 MalwareMultiScan.Api/backends.yaml delete mode 100644 MalwareMultiScan.Worker/Properties/launchSettings.json diff --git a/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs b/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs deleted file mode 100644 index 8bf3c3c..0000000 --- a/MalwareMultiScan.Api/Controllers/WeatherForecastController.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; - -namespace MalwareMultiScan.Api.Controllers -{ - [ApiController] - [Route("[controller]")] - public class WeatherForecastController : ControllerBase - { - private static readonly string[] Summaries = new[] - { - "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" - }; - - private readonly ILogger _logger; - - public WeatherForecastController(ILogger logger) - { - _logger = logger; - } - - [HttpGet] - public IEnumerable Get() - { - var rng = new Random(); - return Enumerable.Range(1, 5).Select(index => new WeatherForecast - { - Date = DateTime.Now.AddDays(index), - TemperatureC = rng.Next(-20, 55), - Summary = Summaries[rng.Next(Summaries.Length)] - }) - .ToArray(); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs b/MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs new file mode 100644 index 0000000..3524440 --- /dev/null +++ b/MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs @@ -0,0 +1,9 @@ +namespace MalwareMultiScan.Api.Data.Configuration +{ + public class BackendConfiguration + { + public string Id { get; set; } + public string Name { get; set; } + public string Endpoint { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Models/ScanRequest.cs b/MalwareMultiScan.Api/Data/Models/ScanRequest.cs new file mode 100644 index 0000000..23c9af3 --- /dev/null +++ b/MalwareMultiScan.Api/Data/Models/ScanRequest.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace MalwareMultiScan.Api.Data.Models +{ + public class ScanRequest + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } + + [BsonRepresentation(BsonType.ObjectId)] + public string FileId { get; set; } + + public Dictionary Results { get; set; } = + new Dictionary(); + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs b/MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs new file mode 100644 index 0000000..87dccd8 --- /dev/null +++ b/MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs @@ -0,0 +1,9 @@ +namespace MalwareMultiScan.Api.Data.Models +{ + public class ScanRequestEntry + { + public bool Completed { get; set; } + public bool Succeeded { get; set; } + public string[] Threats { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index 68ded6d..8e7a9c3 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -4,5 +4,25 @@ netcoreapp3.1 + + + + + + + Always + + + + + + + + + + + <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> + + diff --git a/MalwareMultiScan.Api/Program.cs b/MalwareMultiScan.Api/Program.cs index c1e647c..13d0b5b 100644 --- a/MalwareMultiScan.Api/Program.cs +++ b/MalwareMultiScan.Api/Program.cs @@ -1,23 +1,19 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; namespace MalwareMultiScan.Api { - public class Program + public static class Program { public static void Main(string[] args) { CreateHostBuilder(args).Build().Run(); } - public static IHostBuilder CreateHostBuilder(string[] args) => - Host.CreateDefaultBuilder(args) + private static IHostBuilder CreateHostBuilder(string[] args) + { + return Host.CreateDefaultBuilder(args) .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); + } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Properties/launchSettings.json b/MalwareMultiScan.Api/Properties/launchSettings.json deleted file mode 100644 index e2cb472..0000000 --- a/MalwareMultiScan.Api/Properties/launchSettings.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "http://json.schemastore.org/launchsettings.json", - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:45236", - "sslPort": 44367 - } - }, - "profiles": { - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "launchUrl": "weatherforecast", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "MalwareMultiScan.Api": { - "commandName": "Project", - "launchBrowser": true, - "launchUrl": "weatherforecast", - "applicationUrl": "https://localhost:5001;http://localhost:5000", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs b/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs new file mode 100644 index 0000000..b6a9b0a --- /dev/null +++ b/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs @@ -0,0 +1,29 @@ +using System.IO; +using MalwareMultiScan.Api.Data.Configuration; +using Microsoft.Extensions.Configuration; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MalwareMultiScan.Api.Services +{ + public class BackendConfigurationReader + { + public BackendConfigurationReader(IConfiguration configuration) + { + var configurationPath = configuration.GetValue("BackendsConfiguration"); + + if (!File.Exists(configurationPath)) + throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath); + + var configurationContent = File.ReadAllText(configurationPath); + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + Backends = deserializer.Deserialize(configurationContent); + } + + public BackendConfiguration[] Backends { get; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ScanRequestService.cs b/MalwareMultiScan.Api/Services/ScanRequestService.cs new file mode 100644 index 0000000..6813342 --- /dev/null +++ b/MalwareMultiScan.Api/Services/ScanRequestService.cs @@ -0,0 +1,90 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using MalwareMultiScan.Api.Data.Models; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; + +namespace MalwareMultiScan.Api.Services +{ + public class ScanRequestService + { + private const string CollectionName = "ScanRequests"; + + private readonly GridFSBucket _bucket; + private readonly IMongoCollection _collection; + private readonly BackendConfigurationReader _configurationReader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public ScanRequestService(IMongoDatabase db, BackendConfigurationReader configurationReader, + IHttpClientFactory httpClientFactory, ILogger logger) + { + _configurationReader = configurationReader; + _httpClientFactory = httpClientFactory; + _logger = logger; + + _bucket = new GridFSBucket(db); + _collection = db.GetCollection(CollectionName); + } + + private async Task QueueScans(string requestId, string fileName, Stream fileStream) + { + foreach (var backend in _configurationReader.Backends) + { + var cancellationTokenSource = new CancellationTokenSource( + TimeSpan.FromSeconds(5)); + + using var httpClient = _httpClientFactory.CreateClient(); + + try + { + var endpointUri = new Uri( + new Uri(backend.Endpoint), "/scan/file"); + + var formContent = new MultipartFormDataContent + { + {new StringContent("value1"), "callbackUrl"}, + {new StreamContent(fileStream), "inputFile", fileName} + }; + + var response = await httpClient.PostAsync( + endpointUri, formContent, cancellationTokenSource.Token); + + response.EnsureSuccessStatusCode(); + + // TODO: update scan request + } + catch (Exception e) + { + _logger.LogError(e, + $"Failed to queue scanning job on {backend.Id} ({backend.Endpoint})"); + } + } + } + + public async Task InitializeNewRequest(string fileName, Stream fileStream, + CancellationToken cancellationToken = default) + { + var fileId = await _bucket.UploadFromStreamAsync(fileName, fileStream, + cancellationToken: cancellationToken); + + var scanRequest = new ScanRequest + { + FileId = fileId.ToString() + }; + + await _collection.InsertOneAsync(scanRequest, new InsertOneOptions + { + BypassDocumentValidation = false + }, cancellationToken); + + await QueueScans(scanRequest.Id, fileName, fileStream); + + return scanRequest.Id; + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index a61cb51..7ed32be 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -1,47 +1,43 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; +using MalwareMultiScan.Api.Services; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.HttpsPolicy; -using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; +using MongoDB.Driver; namespace MalwareMultiScan.Api { public class Startup { + private readonly IConfiguration _configuration; + public Startup(IConfiguration configuration) { - Configuration = configuration; + _configuration = configuration; } - - public IConfiguration Configuration { get; } - - // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) { + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton( + serviceProvider => + { + var db = new MongoClient(_configuration.GetConnectionString("Mongo")); + + return db.GetDatabase( + _configuration.GetValue("DatabaseName")); + }); + services.AddControllers(); + services.AddHttpClient(); } - // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - if (env.IsDevelopment()) - { - app.UseDeveloperExceptionPage(); - } - - app.UseHttpsRedirection(); - app.UseRouting(); - app.UseAuthorization(); - app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); } } diff --git a/MalwareMultiScan.Api/WeatherForecast.cs b/MalwareMultiScan.Api/WeatherForecast.cs deleted file mode 100644 index 19f8687..0000000 --- a/MalwareMultiScan.Api/WeatherForecast.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; - -namespace MalwareMultiScan.Api -{ - public class WeatherForecast - { - public DateTime Date { get; set; } - - public int TemperatureC { get; set; } - - public int TemperatureF => 32 + (int) (TemperatureC / 0.5556); - - public string Summary { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/appsettings.json b/MalwareMultiScan.Api/appsettings.json index d9d9a9b..62ff795 100644 --- a/MalwareMultiScan.Api/appsettings.json +++ b/MalwareMultiScan.Api/appsettings.json @@ -6,5 +6,13 @@ "Microsoft.Hosting.Lifetime": "Information" } }, - "AllowedHosts": "*" + "AllowedHosts": "*", + + "ConnectionStrings": { + "Mongo": "mongodb://localhost:27017" + }, + + "DatabaseName": "MalwareMultiScan", + + "BackendsConfiguration": "backends.yaml" } diff --git a/MalwareMultiScan.Api/backends.yaml b/MalwareMultiScan.Api/backends.yaml new file mode 100644 index 0000000..068828c --- /dev/null +++ b/MalwareMultiScan.Api/backends.yaml @@ -0,0 +1,7 @@ +- id: windows-defender + name: Windows Defender + endpoint: http://localhost:9902 + +- id: clamav + name: ClamAV + endpoint: http://localhost:9901 \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs index f46606b..5e4c344 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs @@ -56,13 +56,13 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts process.WaitForExit(); _logger.LogInformation($"Process has exited with code {process.ExitCode}"); - + var standardOutput = await process.StandardOutput.ReadToEndAsync(); var standardError = await process.StandardError.ReadToEndAsync(); _logger.LogDebug($"Process standard output: {standardOutput}"); _logger.LogDebug($"Process standard error: {standardError}"); - + if (ThrowOnNonZeroExitCode && process.ExitCode != 0) throw new ApplicationException($"Process has terminated with an exit code {process.ExitCode}"); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs index 2aa6e69..2d2de61 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs @@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public ClamavScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "clamav"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/var/lib/clamav/daily.cvd"); protected override string BackendPath { get; } = "/usr/bin/clamscan"; - + protected override Regex MatchRegex { get; } = new Regex(@"(\S+): (?[\S]+) FOUND", RegexOptions.Compiled | RegexOptions.Multiline); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs index 2d40818..6ef2bde 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs @@ -11,16 +11,16 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public ComodoScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "comodo"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/opt/COMODO/scanners/bases.cav"); protected override string BackendPath { get; } = "/opt/COMODO/cmdscan"; - + protected override Regex MatchRegex { get; } = - new Regex(@".* ---> Found Virus, Malware Name is (?.*)", + new Regex(@".* ---> Found Virus, Malware Name is (?.*)", RegexOptions.Compiled | RegexOptions.Multiline); protected override string GetBackendArguments(string path) diff --git a/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs index ba8d11b..b042b52 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs @@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public DrWebScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "drweb"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/var/opt/drweb.com/version/version.ini"); protected override string BackendPath { get; } = "/usr/bin/drweb-ctl"; - + protected override Regex MatchRegex { get; } = new Regex(@".* - infected with (?[\S ]+)", RegexOptions.Compiled | RegexOptions.Multiline); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs index 89b9b3b..6127780 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs @@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public KesScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "kes"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/var/opt/kaspersky/kesl/common/updates/avbases/klsrl.dat"); protected override string BackendPath { get; } = "/bin/bash"; - + protected override Regex MatchRegex { get; } = new Regex(@"[ +]DetectName.*: (?.*)", RegexOptions.Compiled | RegexOptions.Multiline); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs index 1970951..1791d8a 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs @@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public McAfeeScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "mcafeee"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/usr/local/uvscan/avvscan.dat"); protected override string BackendPath { get; } = "/usr/local/uvscan/uvscan"; - + protected override bool ThrowOnNonZeroExitCode { get; } = false; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs index d363a77..8c9cfe9 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs @@ -11,9 +11,9 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public SophosScanBackend(ILogger logger) : base(logger) { } - + public override string Id { get; } = "sophos"; - + public override DateTime DatabaseLastUpdate => File.GetLastWriteTime("/opt/sophos-av/lib/sav/vdlsync.upd"); diff --git a/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj b/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj index 3f7bdca..ded602c 100644 --- a/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj +++ b/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj @@ -13,5 +13,9 @@ + + + <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> + diff --git a/MalwareMultiScan.Worker/Properties/launchSettings.json b/MalwareMultiScan.Worker/Properties/launchSettings.json deleted file mode 100644 index 2c9b6ab..0000000 --- a/MalwareMultiScan.Worker/Properties/launchSettings.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "http://json.schemastore.org/launchsettings.json", - "iisSettings": { - "windowsAuthentication": false, - "anonymousAuthentication": true, - "iisExpress": { - "applicationUrl": "http://localhost:53549", - "sslPort": 44380 - } - }, - "profiles": { - "IIS Express": { - "commandName": "IISExpress", - "launchBrowser": true, - "launchUrl": "weatherforecast", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "MalwareMultiScan.Worker": { - "commandName": "Project", - "launchBrowser": true, - "launchUrl": "weatherforecast", - "applicationUrl": "https://localhost:5001;http://localhost:5000", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/MalwareMultiScan.Worker/Startup.cs b/MalwareMultiScan.Worker/Startup.cs index bd3ba6f..a4e1b19 100644 --- a/MalwareMultiScan.Worker/Startup.cs +++ b/MalwareMultiScan.Worker/Startup.cs @@ -27,7 +27,8 @@ namespace MalwareMultiScan.Worker { app.UseRouting(); - app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); + app.UseEndpoints( + endpoints => endpoints.MapControllers()); app.UseHangfireServer(); } From b6c3cb4131bf915964fd27d2eb97fc074a50f8f7 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Sun, 25 Oct 2020 16:11:36 +0200 Subject: [PATCH 3/6] basic skeleton for the API and queueing --- .../Controllers/DownloadController.cs | 31 +++++ .../Controllers/QueueController.cs | 15 +++ .../Controllers/ScanBackendsController.cs | 41 +++++++ .../Controllers/ScanResultsController.cs | 31 +++++ ...BackendConfiguration.cs => ScanBackend.cs} | 2 +- .../Data/Models/ScanRequest.cs | 19 --- .../Data/Models/ScanResult.cs | 16 +++ ...ScanRequestEntry.cs => ScanResultEntry.cs} | 2 +- .../Data/Response/ScanBackendResponse.cs | 9 ++ .../MalwareMultiScan.Api.csproj | 8 +- .../Services/BackendConfigurationReader.cs | 29 ----- .../Services/ScanBackendService.cs | 111 ++++++++++++++++++ .../Services/ScanRequestService.cs | 90 -------------- .../Services/ScanResultsService.cs | 97 +++++++++++++++ MalwareMultiScan.Api/Startup.cs | 4 +- MalwareMultiScan.Api/appsettings.json | 3 +- .../Backends/Abstracts/AbstractScanBackend.cs | 2 - .../Implementations/ClamavScanBackend.cs | 5 - .../Implementations/ComodoScanBackend.cs | 3 - .../Implementations/DrWebScanBackend.cs | 5 - .../Implementations/KesScanBackend.cs | 5 - .../Implementations/McAffeeScanBackend.cs | 5 - .../Implementations/SophosScanBackend.cs | 5 - .../WindowsDefenderScanBackend.cs | 5 - .../Data/Responses/ResultResponse.cs | 9 +- .../Interfaces/IScanBackend.cs | 2 - .../Controllers/ScanController.cs | 9 ++ MalwareMultiScan.Worker/Jobs/ScanJob.cs | 1 - 28 files changed, 374 insertions(+), 190 deletions(-) create mode 100644 MalwareMultiScan.Api/Controllers/DownloadController.cs create mode 100644 MalwareMultiScan.Api/Controllers/QueueController.cs create mode 100644 MalwareMultiScan.Api/Controllers/ScanBackendsController.cs create mode 100644 MalwareMultiScan.Api/Controllers/ScanResultsController.cs rename MalwareMultiScan.Api/Data/Configuration/{BackendConfiguration.cs => ScanBackend.cs} (83%) delete mode 100644 MalwareMultiScan.Api/Data/Models/ScanRequest.cs create mode 100644 MalwareMultiScan.Api/Data/Models/ScanResult.cs rename MalwareMultiScan.Api/Data/Models/{ScanRequestEntry.cs => ScanResultEntry.cs} (84%) create mode 100644 MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs delete mode 100644 MalwareMultiScan.Api/Services/BackendConfigurationReader.cs create mode 100644 MalwareMultiScan.Api/Services/ScanBackendService.cs delete mode 100644 MalwareMultiScan.Api/Services/ScanRequestService.cs create mode 100644 MalwareMultiScan.Api/Services/ScanResultsService.cs diff --git a/MalwareMultiScan.Api/Controllers/DownloadController.cs b/MalwareMultiScan.Api/Controllers/DownloadController.cs new file mode 100644 index 0000000..8f4414b --- /dev/null +++ b/MalwareMultiScan.Api/Controllers/DownloadController.cs @@ -0,0 +1,31 @@ +using System.Threading.Tasks; +using MalwareMultiScan.Api.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace MalwareMultiScan.Api.Controllers +{ + [Route("download")] + public class DownloadController : Controller + { + private readonly ScanResultsService _scanResultsService; + + public DownloadController(ScanResultsService scanResultsService) + { + _scanResultsService = scanResultsService; + } + + [HttpGet("{id}")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + public async Task Index(string id) + { + var fileStream = await _scanResultsService.ObtainFile(id); + + if (fileStream == null) + return NotFound(); + + return File(fileStream, "application/octet-stream"); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/QueueController.cs b/MalwareMultiScan.Api/Controllers/QueueController.cs new file mode 100644 index 0000000..908751e --- /dev/null +++ b/MalwareMultiScan.Api/Controllers/QueueController.cs @@ -0,0 +1,15 @@ +using Microsoft.AspNetCore.Mvc; + +namespace MalwareMultiScan.Api.Controllers +{ + [ApiController] + [Route("queue")] + [Produces("application/json")] + public class QueueController : Controller + { + public IActionResult Index() + { + return Ok(); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs b/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs new file mode 100644 index 0000000..c65e430 --- /dev/null +++ b/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs @@ -0,0 +1,41 @@ +using System.Linq; +using System.Threading.Tasks; +using MalwareMultiScan.Api.Data.Configuration; +using MalwareMultiScan.Api.Data.Response; +using MalwareMultiScan.Api.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace MalwareMultiScan.Api.Controllers +{ + [ApiController] + [Route("backends")] + [Produces("application/json")] + public class ScanBackendsController : Controller + { + private readonly ScanBackendService _scanBackendService; + + public ScanBackendsController(ScanBackendService scanBackendService) + { + _scanBackendService = scanBackendService; + } + + private async Task GetScanBackendResponse(ScanBackend backend) + { + return new ScanBackendResponse + { + Id = backend.Id, + Name = backend.Name, + Online = await _scanBackendService.Ping(backend) + }; + } + + [HttpGet] + [ProducesResponseType(typeof(ScanBackendResponse[]), StatusCodes.Status200OK)] + public async Task Index() + { + return Ok(await Task.WhenAll( + _scanBackendService.List.Select(GetScanBackendResponse).ToArray())); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs new file mode 100644 index 0000000..2d8885a --- /dev/null +++ b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs @@ -0,0 +1,31 @@ +using System.Threading.Tasks; +using MalwareMultiScan.Api.Services; +using MalwareMultiScan.Shared.Data.Responses; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace MalwareMultiScan.Api.Controllers +{ + [ApiController] + [Route("results")] + [Produces("application/json")] + public class ScanResultsController : Controller + { + private readonly ScanResultsService _scanResultsService; + + public ScanResultsController(ScanResultsService scanResultsService) + { + _scanResultsService = scanResultsService; + } + + [HttpPost("{id}")] + [ProducesResponseType(StatusCodes.Status204NoContent)] + public async Task Index(string id, [FromBody] ResultResponse result) + { + await _scanResultsService.UpdateScanResultForBackend( + id, result.Backend, true, result.Success, result.Threats); + + return NoContent(); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs similarity index 83% rename from MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs rename to MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs index 3524440..a2cb9ad 100644 --- a/MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs +++ b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs @@ -1,6 +1,6 @@ namespace MalwareMultiScan.Api.Data.Configuration { - public class BackendConfiguration + public class ScanBackend { public string Id { get; set; } public string Name { get; set; } diff --git a/MalwareMultiScan.Api/Data/Models/ScanRequest.cs b/MalwareMultiScan.Api/Data/Models/ScanRequest.cs deleted file mode 100644 index 23c9af3..0000000 --- a/MalwareMultiScan.Api/Data/Models/ScanRequest.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Collections.Generic; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; - -namespace MalwareMultiScan.Api.Data.Models -{ - public class ScanRequest - { - [BsonId] - [BsonRepresentation(BsonType.ObjectId)] - public string Id { get; set; } - - [BsonRepresentation(BsonType.ObjectId)] - public string FileId { get; set; } - - public Dictionary Results { get; set; } = - new Dictionary(); - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Models/ScanResult.cs b/MalwareMultiScan.Api/Data/Models/ScanResult.cs new file mode 100644 index 0000000..c61369c --- /dev/null +++ b/MalwareMultiScan.Api/Data/Models/ScanResult.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace MalwareMultiScan.Api.Data.Models +{ + public class ScanResult + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } + + public Dictionary Results { get; set; } = + new Dictionary(); + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs b/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs similarity index 84% rename from MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs rename to MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs index 87dccd8..dd3b536 100644 --- a/MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs +++ b/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs @@ -1,6 +1,6 @@ namespace MalwareMultiScan.Api.Data.Models { - public class ScanRequestEntry + public class ScanResultEntry { public bool Completed { get; set; } public bool Succeeded { get; set; } diff --git a/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs b/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs new file mode 100644 index 0000000..a7afe8f --- /dev/null +++ b/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs @@ -0,0 +1,9 @@ +namespace MalwareMultiScan.Api.Data.Response +{ + public class ScanBackendResponse + { + public string Id { get; set; } + public string Name { get; set; } + public bool Online { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index 8e7a9c3..b4895c0 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -4,10 +4,6 @@ netcoreapp3.1 - - - - Always @@ -24,5 +20,9 @@ <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> + + + + diff --git a/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs b/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs deleted file mode 100644 index b6a9b0a..0000000 --- a/MalwareMultiScan.Api/Services/BackendConfigurationReader.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.IO; -using MalwareMultiScan.Api.Data.Configuration; -using Microsoft.Extensions.Configuration; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace MalwareMultiScan.Api.Services -{ - public class BackendConfigurationReader - { - public BackendConfigurationReader(IConfiguration configuration) - { - var configurationPath = configuration.GetValue("BackendsConfiguration"); - - if (!File.Exists(configurationPath)) - throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath); - - var configurationContent = File.ReadAllText(configurationPath); - - var deserializer = new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .Build(); - - Backends = deserializer.Deserialize(configurationContent); - } - - public BackendConfiguration[] Backends { get; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ScanBackendService.cs b/MalwareMultiScan.Api/Services/ScanBackendService.cs new file mode 100644 index 0000000..f1ab944 --- /dev/null +++ b/MalwareMultiScan.Api/Services/ScanBackendService.cs @@ -0,0 +1,111 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using MalwareMultiScan.Api.Data.Configuration; +using MalwareMultiScan.Shared.Data.Requests; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MalwareMultiScan.Api.Services +{ + public class ScanBackendService + { + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public ScanBackendService(IConfiguration configuration, IHttpClientFactory httpClientFactory, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _logger = logger; + var configurationPath = configuration.GetValue("BackendsConfiguration"); + + if (!File.Exists(configurationPath)) + throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath); + + var configurationContent = File.ReadAllText(configurationPath); + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + List = deserializer.Deserialize(configurationContent); + } + + public ScanBackend[] List { get; } + + public async Task Ping(ScanBackend backend) + { + var cancellationTokenSource = new CancellationTokenSource( + TimeSpan.FromSeconds(1)); + + try + { + using var httpClient = _httpClientFactory.CreateClient(); + + var pingResponse = await httpClient.GetAsync( + new Uri(new Uri(backend.Endpoint), "/ping"), cancellationTokenSource.Token); + + pingResponse.EnsureSuccessStatusCode(); + } + catch (Exception exception) + { + _logger.LogError( + exception, $"Failed to ping {backend.Id}"); + + return false; + } + + return true; + } + + private async Task QueueScan(ScanBackend backend, string uri, HttpContent content) + { + var cancellationTokenSource = new CancellationTokenSource( + TimeSpan.FromSeconds(5)); + + using var httpClient = _httpClientFactory.CreateClient(); + + try + { + var response = await httpClient.PostAsync( + new Uri(new Uri(backend.Endpoint), uri), content, cancellationTokenSource.Token); + + response.EnsureSuccessStatusCode(); + } + catch (Exception exception) + { + _logger.LogError( + exception, $"Failed to initiate scan against {backend.Id}"); + + return false; + } + + return true; + } + + public async Task QueueFileScan( + ScanBackend backend, string fileName, Stream fileStream, string callbackUrl) + { + return await QueueScan(backend, "/scan/file", new MultipartFormDataContent + { + {new StringContent(callbackUrl), nameof(FileRequest.CallbackUrl)}, + {new StreamContent(fileStream), nameof(FileRequest.InputFile), fileName} + }); + } + + public async Task QueueUrlScan( + ScanBackend backend, string url, string callbackUrl) + { + return await QueueScan(backend, "/scan/url", new MultipartFormDataContent + { + {new StringContent(callbackUrl), nameof(UrlRequest.CallbackUrl)}, + {new StringContent(url), nameof(UrlRequest.InputUrl)} + }); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ScanRequestService.cs b/MalwareMultiScan.Api/Services/ScanRequestService.cs deleted file mode 100644 index 6813342..0000000 --- a/MalwareMultiScan.Api/Services/ScanRequestService.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using MalwareMultiScan.Api.Data.Models; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using MongoDB.Driver.GridFS; - -namespace MalwareMultiScan.Api.Services -{ - public class ScanRequestService - { - private const string CollectionName = "ScanRequests"; - - private readonly GridFSBucket _bucket; - private readonly IMongoCollection _collection; - private readonly BackendConfigurationReader _configurationReader; - private readonly IHttpClientFactory _httpClientFactory; - private readonly ILogger _logger; - - public ScanRequestService(IMongoDatabase db, BackendConfigurationReader configurationReader, - IHttpClientFactory httpClientFactory, ILogger logger) - { - _configurationReader = configurationReader; - _httpClientFactory = httpClientFactory; - _logger = logger; - - _bucket = new GridFSBucket(db); - _collection = db.GetCollection(CollectionName); - } - - private async Task QueueScans(string requestId, string fileName, Stream fileStream) - { - foreach (var backend in _configurationReader.Backends) - { - var cancellationTokenSource = new CancellationTokenSource( - TimeSpan.FromSeconds(5)); - - using var httpClient = _httpClientFactory.CreateClient(); - - try - { - var endpointUri = new Uri( - new Uri(backend.Endpoint), "/scan/file"); - - var formContent = new MultipartFormDataContent - { - {new StringContent("value1"), "callbackUrl"}, - {new StreamContent(fileStream), "inputFile", fileName} - }; - - var response = await httpClient.PostAsync( - endpointUri, formContent, cancellationTokenSource.Token); - - response.EnsureSuccessStatusCode(); - - // TODO: update scan request - } - catch (Exception e) - { - _logger.LogError(e, - $"Failed to queue scanning job on {backend.Id} ({backend.Endpoint})"); - } - } - } - - public async Task InitializeNewRequest(string fileName, Stream fileStream, - CancellationToken cancellationToken = default) - { - var fileId = await _bucket.UploadFromStreamAsync(fileName, fileStream, - cancellationToken: cancellationToken); - - var scanRequest = new ScanRequest - { - FileId = fileId.ToString() - }; - - await _collection.InsertOneAsync(scanRequest, new InsertOneOptions - { - BypassDocumentValidation = false - }, cancellationToken); - - await QueueScans(scanRequest.Id, fileName, fileStream); - - return scanRequest.Id; - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ScanResultsService.cs b/MalwareMultiScan.Api/Services/ScanResultsService.cs new file mode 100644 index 0000000..83de671 --- /dev/null +++ b/MalwareMultiScan.Api/Services/ScanResultsService.cs @@ -0,0 +1,97 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using MalwareMultiScan.Api.Data.Configuration; +using MalwareMultiScan.Api.Data.Models; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; + +namespace MalwareMultiScan.Api.Services +{ + public class ScanResultsService + { + private const string CollectionName = "ScanResults"; + + private readonly IMongoCollection _collection; + private readonly GridFSBucket _bucket; + + private readonly ScanBackendService _scanBackendService; + + public ScanResultsService(IMongoDatabase db, ScanBackendService scanBackendService) + { + _scanBackendService = scanBackendService; + + _collection = db.GetCollection(CollectionName); + _bucket = new GridFSBucket(db); + } + + public async Task CreateScanResult() + { + var scanResult = new ScanResult(); + + await _collection.InsertOneAsync(scanResult); + + return scanResult; + } + + public async Task UpdateScanResultForBackend( + string resultId, string backendId, bool completed, bool succeeded, string[] threats = null) + { + var filterScanResult = Builders.Filter.Where(r => r.Id == resultId); + + var updateScanResult = Builders.Update.Set(r => r.Results[backendId], new ScanResultEntry + { + Completed = completed, + Succeeded = succeeded, + Threats = threats + }); + + await _collection.UpdateOneAsync(filterScanResult, updateScanResult); + } + + public async Task QueueFileScan(ScanResult result, string fileName, Stream fileStream, string callbackUrl) + { + foreach (var backend in _scanBackendService.List) + { + var queueResult = await _scanBackendService.QueueFileScan( + backend, fileName, fileStream, callbackUrl); + + await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult); + } + } + + public async Task QueueUrlScan(ScanResult result, string url, string callbackUrl) + { + foreach (var backend in _scanBackendService.List) + { + var queueResult = await _scanBackendService.QueueUrlScan( + backend, url, callbackUrl); + + await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult); + } + } + + public async Task StoreFile(string fileName, Stream fileStream) + { + var objectId = await _bucket.UploadFromStreamAsync(fileName, fileStream); + + return objectId.ToString(); + } + + public async Task ObtainFile(string id) + { + if (!ObjectId.TryParse(id, out var objectId)) + return null; + + return await _bucket.OpenDownloadStreamAsync(objectId, new GridFSDownloadOptions + { + Seekable = true + }); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index 7ed32be..12d3f37 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -18,8 +18,8 @@ namespace MalwareMultiScan.Api public void ConfigureServices(IServiceCollection services) { - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton( serviceProvider => diff --git a/MalwareMultiScan.Api/appsettings.json b/MalwareMultiScan.Api/appsettings.json index 62ff795..4ff2c96 100644 --- a/MalwareMultiScan.Api/appsettings.json +++ b/MalwareMultiScan.Api/appsettings.json @@ -14,5 +14,6 @@ "DatabaseName": "MalwareMultiScan", - "BackendsConfiguration": "backends.yaml" + "BackendsConfiguration": "backends.yaml", + "StoreFileUploads": true } diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs index da66eb7..49633b2 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs @@ -12,8 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts { public virtual string Id => throw new NotImplementedException(); - public virtual DateTime DatabaseLastUpdate => throw new NotImplementedException(); - public virtual Task ScanAsync(string path, CancellationToken cancellationToken) { throw new NotImplementedException(); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs index 2d2de61..0ccb4d5 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "clamav"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/var/lib/clamav/daily.cvd"); - protected override string BackendPath { get; } = "/usr/bin/clamscan"; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs index 6ef2bde..ac8b3a2 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs @@ -14,9 +14,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "comodo"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/opt/COMODO/scanners/bases.cav"); - protected override string BackendPath { get; } = "/opt/COMODO/cmdscan"; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs index b042b52..a98dcb6 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "drweb"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/var/opt/drweb.com/version/version.ini"); - protected override string BackendPath { get; } = "/usr/bin/drweb-ctl"; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs index 6127780..2448351 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "kes"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/var/opt/kaspersky/kesl/common/updates/avbases/klsrl.dat"); - protected override string BackendPath { get; } = "/bin/bash"; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs index 1791d8a..9c4d76a 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "mcafeee"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/usr/local/uvscan/avvscan.dat"); - protected override string BackendPath { get; } = "/usr/local/uvscan/uvscan"; protected override bool ThrowOnNonZeroExitCode { get; } = false; diff --git a/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs index 8c9cfe9..9b8cecd 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "sophos"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/opt/sophos-av/lib/sav/vdlsync.upd"); - protected override string BackendPath { get; } = "/opt/sophos-av/bin/savscan"; protected override bool ThrowOnNonZeroExitCode { get; } = false; diff --git a/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs index 2dd4129..8eb69b0 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; @@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations public override string Id { get; } = "windows-defender"; - public override DateTime DatabaseLastUpdate => - File.GetLastWriteTime("/opt/engine/mpavbase.vdm"); - protected override string BackendPath { get; } = "/opt/mpclient"; protected override Regex MatchRegex { get; } = diff --git a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs b/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs index 0d33c63..0f7542a 100644 --- a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs +++ b/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs @@ -1,18 +1,17 @@ using System; +using System.ComponentModel.DataAnnotations; using System.Linq; namespace MalwareMultiScan.Shared.Data.Responses { public class ResultResponse { + [Required] public string Backend { get; set; } + [Required] public bool Success { get; set; } - - public DateTime? DatabaseLastUpdate { get; set; } - - public bool Detected => Threats?.Any() == true; - + public string[] Threats { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs b/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs index e10d3c5..dcf0411 100644 --- a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs +++ b/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs @@ -10,8 +10,6 @@ namespace MalwareMultiScan.Shared.Interfaces { public string Id { get; } - public DateTime DatabaseLastUpdate { get; } - public Task ScanAsync(string path, CancellationToken cancellationToken); public Task ScanAsync(Uri uri, CancellationToken cancellationToken); public Task ScanAsync(IFormFile file, CancellationToken cancellationToken); diff --git a/MalwareMultiScan.Worker/Controllers/ScanController.cs b/MalwareMultiScan.Worker/Controllers/ScanController.cs index c41dadd..230e440 100644 --- a/MalwareMultiScan.Worker/Controllers/ScanController.cs +++ b/MalwareMultiScan.Worker/Controllers/ScanController.cs @@ -12,6 +12,15 @@ namespace MalwareMultiScan.Worker.Controllers [Produces("application/json")] public class ScanController : ControllerBase { + + [HttpGet] + [ProducesResponseType(StatusCodes.Status200OK)] + [Route("/ping")] + public IActionResult Ping() + { + return Ok("pong"); + } + [HttpPost] [ProducesResponseType(StatusCodes.Status202Accepted)] [ProducesResponseType(StatusCodes.Status400BadRequest)] diff --git a/MalwareMultiScan.Worker/Jobs/ScanJob.cs b/MalwareMultiScan.Worker/Jobs/ScanJob.cs index 9fa7c05..7dcf178 100644 --- a/MalwareMultiScan.Worker/Jobs/ScanJob.cs +++ b/MalwareMultiScan.Worker/Jobs/ScanJob.cs @@ -78,7 +78,6 @@ namespace MalwareMultiScan.Worker.Jobs { response.Success = true; response.Threats = await scanMethod(cancellationToken); - response.DatabaseLastUpdate = _backend.DatabaseLastUpdate; } catch (Exception exception) { From 29156b7f1042c392ed63a035df456760b6119ce3 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Mon, 26 Oct 2020 16:20:47 +0200 Subject: [PATCH 4/6] change the communication protocol to RMQ --- .../Controllers/DownloadController.cs | 8 +- .../Controllers/QueueController.cs | 47 +++++++++- .../Controllers/ScanBackendsController.cs | 41 --------- .../Controllers/ScanResultsController.cs | 25 +++--- .../Data/Configuration/ScanBackend.cs | 1 - .../MalwareMultiScan.Api.csproj | 2 + .../Services/ReceiverHostedService.cs | 41 +++++++++ .../Services/ScanBackendService.cs | 88 +++---------------- ...ResultsService.cs => ScanResultService.cs} | 46 +++++----- MalwareMultiScan.Api/Startup.cs | 42 ++++++++- MalwareMultiScan.Api/appsettings.json | 8 +- MalwareMultiScan.Api/backends.yaml | 9 +- .../Implementations/ClamavScanBackend.cs | 2 + .../Implementations/DummyScanBackend.cs | 39 ++++++++ .../Extensions/ServiceCollectionExtensions.cs | 32 +++++++ .../MalwareMultiScan.Scanner.csproj | 16 ++++ MalwareMultiScan.Scanner/Program.cs | 31 +++++++ .../Properties/launchSettings.json | 10 +++ MalwareMultiScan.Scanner/ScanHostedService.cs | 73 +++++++++++++++ MalwareMultiScan.Scanner/settings.json | 20 +++++ .../Attributes/UrlValidationAttribute.cs | 4 +- .../Data/Enums/BackendType.cs | 1 + .../Data/Messages/ScanRequestMessage.cs | 11 +++ .../Data/Messages/ScanResultMessage.cs | 9 ++ MalwareMultiScan.sln | 6 ++ 25 files changed, 439 insertions(+), 173 deletions(-) delete mode 100644 MalwareMultiScan.Api/Controllers/ScanBackendsController.cs create mode 100644 MalwareMultiScan.Api/Services/ReceiverHostedService.cs rename MalwareMultiScan.Api/Services/{ScanResultsService.cs => ScanResultService.cs} (65%) create mode 100644 MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs create mode 100644 MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs create mode 100644 MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj create mode 100644 MalwareMultiScan.Scanner/Program.cs create mode 100644 MalwareMultiScan.Scanner/Properties/launchSettings.json create mode 100644 MalwareMultiScan.Scanner/ScanHostedService.cs create mode 100644 MalwareMultiScan.Scanner/settings.json create mode 100644 MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs create mode 100644 MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs diff --git a/MalwareMultiScan.Api/Controllers/DownloadController.cs b/MalwareMultiScan.Api/Controllers/DownloadController.cs index 8f4414b..ead47f3 100644 --- a/MalwareMultiScan.Api/Controllers/DownloadController.cs +++ b/MalwareMultiScan.Api/Controllers/DownloadController.cs @@ -8,11 +8,11 @@ namespace MalwareMultiScan.Api.Controllers [Route("download")] public class DownloadController : Controller { - private readonly ScanResultsService _scanResultsService; + private readonly ScanResultService _scanResultService; - public DownloadController(ScanResultsService scanResultsService) + public DownloadController(ScanResultService scanResultService) { - _scanResultsService = scanResultsService; + _scanResultService = scanResultService; } [HttpGet("{id}")] @@ -20,7 +20,7 @@ namespace MalwareMultiScan.Api.Controllers [ProducesResponseType(StatusCodes.Status404NotFound)] public async Task Index(string id) { - var fileStream = await _scanResultsService.ObtainFile(id); + var fileStream = await _scanResultService.ObtainFile(id); if (fileStream == null) return NotFound(); diff --git a/MalwareMultiScan.Api/Controllers/QueueController.cs b/MalwareMultiScan.Api/Controllers/QueueController.cs index 908751e..de4e9c4 100644 --- a/MalwareMultiScan.Api/Controllers/QueueController.cs +++ b/MalwareMultiScan.Api/Controllers/QueueController.cs @@ -1,3 +1,10 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using MalwareMultiScan.Api.Services; +using MalwareMultiScan.Shared.Attributes; +using MalwareMultiScan.Shared.Data.Responses; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; namespace MalwareMultiScan.Api.Controllers @@ -7,9 +14,45 @@ namespace MalwareMultiScan.Api.Controllers [Produces("application/json")] public class QueueController : Controller { - public IActionResult Index() + private readonly ScanResultService _scanResultService; + + public QueueController( ScanResultService scanResultService) { - return Ok(); + _scanResultService = scanResultService; + } + + [HttpPost("file")] + [ProducesResponseType(typeof(ResultResponse), StatusCodes.Status201Created)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task ScanFile([FromForm] IFormFile file) + { + var result = await _scanResultService.CreateScanResult(); + + string storedFileId; + + await using (var uploadFileStream = file.OpenReadStream()) + storedFileId = await _scanResultService.StoreFile(file.Name, uploadFileStream); + + await _scanResultService.QueueUrlScan(result, Url.Action("Index", "Download", new {id = storedFileId}, + Request.Scheme, Request.Host.Value)); + + return Created(Url.Action("Index", "ScanResults", new {id = result.Id}, + Request.Scheme, Request.Host.Value), result); + } + + [HttpPost("url")] + [ProducesResponseType(typeof(ResultResponse), StatusCodes.Status201Created)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task ScanUrl([FromForm] [UrlValidation] Uri url) + { + var result = await _scanResultService.CreateScanResult(); + + var resultUrl = Url.Action("Index", "ScanResults", new {id = result.Id}, + Request.Scheme, Request.Host.Value); + + await _scanResultService.QueueUrlScan(result, url.ToString()); + + return Created(resultUrl, result); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs b/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs deleted file mode 100644 index c65e430..0000000 --- a/MalwareMultiScan.Api/Controllers/ScanBackendsController.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using MalwareMultiScan.Api.Data.Configuration; -using MalwareMultiScan.Api.Data.Response; -using MalwareMultiScan.Api.Services; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; - -namespace MalwareMultiScan.Api.Controllers -{ - [ApiController] - [Route("backends")] - [Produces("application/json")] - public class ScanBackendsController : Controller - { - private readonly ScanBackendService _scanBackendService; - - public ScanBackendsController(ScanBackendService scanBackendService) - { - _scanBackendService = scanBackendService; - } - - private async Task GetScanBackendResponse(ScanBackend backend) - { - return new ScanBackendResponse - { - Id = backend.Id, - Name = backend.Name, - Online = await _scanBackendService.Ping(backend) - }; - } - - [HttpGet] - [ProducesResponseType(typeof(ScanBackendResponse[]), StatusCodes.Status200OK)] - public async Task Index() - { - return Ok(await Task.WhenAll( - _scanBackendService.List.Select(GetScanBackendResponse).ToArray())); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs index 2d8885a..c2aca1f 100644 --- a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs +++ b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using MalwareMultiScan.Api.Data.Models; using MalwareMultiScan.Api.Services; using MalwareMultiScan.Shared.Data.Responses; using Microsoft.AspNetCore.Http; @@ -11,21 +12,23 @@ namespace MalwareMultiScan.Api.Controllers [Produces("application/json")] public class ScanResultsController : Controller { - private readonly ScanResultsService _scanResultsService; + private readonly ScanResultService _scanResultService; - public ScanResultsController(ScanResultsService scanResultsService) + public ScanResultsController(ScanResultService scanResultService) { - _scanResultsService = scanResultsService; + _scanResultService = scanResultService; } - - [HttpPost("{id}")] - [ProducesResponseType(StatusCodes.Status204NoContent)] - public async Task Index(string id, [FromBody] ResultResponse result) + + [HttpGet("{id}")] + [ProducesResponseType(typeof(ScanResult), StatusCodes.Status200OK)] + public async Task Index(string id) { - await _scanResultsService.UpdateScanResultForBackend( - id, result.Backend, true, result.Success, result.Threats); - - return NoContent(); + var scanResult = await _scanResultService.GetScanResult(id); + + if (scanResult == null) + return NotFound(); + + return Ok(scanResult); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs index a2cb9ad..a230136 100644 --- a/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs +++ b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs @@ -4,6 +4,5 @@ namespace MalwareMultiScan.Api.Data.Configuration { public string Id { get; set; } public string Name { get; set; } - public string Endpoint { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index b4895c0..bb6c35a 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -11,8 +11,10 @@ + + diff --git a/MalwareMultiScan.Api/Services/ReceiverHostedService.cs b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs new file mode 100644 index 0000000..61a2402 --- /dev/null +++ b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs @@ -0,0 +1,41 @@ +using System.Threading; +using System.Threading.Tasks; +using EasyNetQ; +using MalwareMultiScan.Shared.Data.Messages; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; + +namespace MalwareMultiScan.Api.Services +{ + public class ReceiverHostedService : IHostedService + { + private readonly IBus _bus; + private readonly IConfiguration _configuration; + private readonly ScanResultService _scanResultService; + + public ReceiverHostedService(IBus bus, IConfiguration configuration, ScanResultService scanResultService) + { + _bus = bus; + _configuration = configuration; + _scanResultService = scanResultService; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _bus.Receive(_configuration.GetValue("ResultsSubscriptionId"), async message => + { + await _scanResultService.UpdateScanResultForBackend( + message.Id, message.Backend, true, true, message.Threats); + }); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _bus.Dispose(); + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ScanBackendService.cs b/MalwareMultiScan.Api/Services/ScanBackendService.cs index f1ab944..81bc744 100644 --- a/MalwareMultiScan.Api/Services/ScanBackendService.cs +++ b/MalwareMultiScan.Api/Services/ScanBackendService.cs @@ -1,12 +1,11 @@ using System; using System.IO; -using System.Net.Http; -using System.Threading; using System.Threading.Tasks; +using EasyNetQ; using MalwareMultiScan.Api.Data.Configuration; -using MalwareMultiScan.Shared.Data.Requests; +using MalwareMultiScan.Api.Data.Models; +using MalwareMultiScan.Shared.Data.Messages; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -14,14 +13,12 @@ namespace MalwareMultiScan.Api.Services { public class ScanBackendService { - private readonly IHttpClientFactory _httpClientFactory; - private readonly ILogger _logger; + private readonly IBus _bus; - public ScanBackendService(IConfiguration configuration, IHttpClientFactory httpClientFactory, - ILogger logger) + public ScanBackendService(IConfiguration configuration, IBus bus) { - _httpClientFactory = httpClientFactory; - _logger = logger; + _bus = bus; + var configurationPath = configuration.GetValue("BackendsConfiguration"); if (!File.Exists(configurationPath)) @@ -37,74 +34,13 @@ namespace MalwareMultiScan.Api.Services } public ScanBackend[] List { get; } - - public async Task Ping(ScanBackend backend) + + public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl) { - var cancellationTokenSource = new CancellationTokenSource( - TimeSpan.FromSeconds(1)); - - try + await _bus.SendAsync(backend.Id, new ScanRequestMessage { - using var httpClient = _httpClientFactory.CreateClient(); - - var pingResponse = await httpClient.GetAsync( - new Uri(new Uri(backend.Endpoint), "/ping"), cancellationTokenSource.Token); - - pingResponse.EnsureSuccessStatusCode(); - } - catch (Exception exception) - { - _logger.LogError( - exception, $"Failed to ping {backend.Id}"); - - return false; - } - - return true; - } - - private async Task QueueScan(ScanBackend backend, string uri, HttpContent content) - { - var cancellationTokenSource = new CancellationTokenSource( - TimeSpan.FromSeconds(5)); - - using var httpClient = _httpClientFactory.CreateClient(); - - try - { - var response = await httpClient.PostAsync( - new Uri(new Uri(backend.Endpoint), uri), content, cancellationTokenSource.Token); - - response.EnsureSuccessStatusCode(); - } - catch (Exception exception) - { - _logger.LogError( - exception, $"Failed to initiate scan against {backend.Id}"); - - return false; - } - - return true; - } - - public async Task QueueFileScan( - ScanBackend backend, string fileName, Stream fileStream, string callbackUrl) - { - return await QueueScan(backend, "/scan/file", new MultipartFormDataContent - { - {new StringContent(callbackUrl), nameof(FileRequest.CallbackUrl)}, - {new StreamContent(fileStream), nameof(FileRequest.InputFile), fileName} - }); - } - - public async Task QueueUrlScan( - ScanBackend backend, string url, string callbackUrl) - { - return await QueueScan(backend, "/scan/url", new MultipartFormDataContent - { - {new StringContent(callbackUrl), nameof(UrlRequest.CallbackUrl)}, - {new StringContent(url), nameof(UrlRequest.InputUrl)} + Id = result.Id, + Uri = new Uri(fileUrl) }); } } diff --git a/MalwareMultiScan.Api/Services/ScanResultsService.cs b/MalwareMultiScan.Api/Services/ScanResultService.cs similarity index 65% rename from MalwareMultiScan.Api/Services/ScanResultsService.cs rename to MalwareMultiScan.Api/Services/ScanResultService.cs index 83de671..6e85f8c 100644 --- a/MalwareMultiScan.Api/Services/ScanResultsService.cs +++ b/MalwareMultiScan.Api/Services/ScanResultService.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -13,7 +14,7 @@ using MongoDB.Driver.GridFS; namespace MalwareMultiScan.Api.Services { - public class ScanResultsService + public class ScanResultService { private const string CollectionName = "ScanResults"; @@ -22,7 +23,7 @@ namespace MalwareMultiScan.Api.Services private readonly ScanBackendService _scanBackendService; - public ScanResultsService(IMongoDatabase db, ScanBackendService scanBackendService) + public ScanResultService(IMongoDatabase db, ScanBackendService scanBackendService) { _scanBackendService = scanBackendService; @@ -32,15 +33,27 @@ namespace MalwareMultiScan.Api.Services public async Task CreateScanResult() { - var scanResult = new ScanResult(); + var scanResult = new ScanResult + { + Results = _scanBackendService.List.ToDictionary( + k => k.Id, v => new ScanResultEntry()) + }; await _collection.InsertOneAsync(scanResult); return scanResult; } + + public async Task GetScanResult(string id) + { + var result = await _collection.FindAsync( + Builders.Filter.Where(r => r.Id == id)); + + return await result.FirstOrDefaultAsync(); + } - public async Task UpdateScanResultForBackend( - string resultId, string backendId, bool completed, bool succeeded, string[] threats = null) + public async Task UpdateScanResultForBackend(string resultId, string backendId, + bool completed = false, bool succeeded = false, string[] threats = null) { var filterScanResult = Builders.Filter.Where(r => r.Id == resultId); @@ -54,31 +67,16 @@ namespace MalwareMultiScan.Api.Services await _collection.UpdateOneAsync(filterScanResult, updateScanResult); } - public async Task QueueFileScan(ScanResult result, string fileName, Stream fileStream, string callbackUrl) + public async Task QueueUrlScan(ScanResult result, string fileUrl) { foreach (var backend in _scanBackendService.List) - { - var queueResult = await _scanBackendService.QueueFileScan( - backend, fileName, fileStream, callbackUrl); - - await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult); - } - } - - public async Task QueueUrlScan(ScanResult result, string url, string callbackUrl) - { - foreach (var backend in _scanBackendService.List) - { - var queueResult = await _scanBackendService.QueueUrlScan( - backend, url, callbackUrl); - - await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult); - } + await _scanBackendService.QueueUrlScan(result, backend, fileUrl); } public async Task StoreFile(string fileName, Stream fileStream) { - var objectId = await _bucket.UploadFromStreamAsync(fileName, fileStream); + var objectId = await _bucket.UploadFromStreamAsync( + fileName, fileStream); return objectId.ToString(); } diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index 12d3f37..3685a83 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -1,8 +1,11 @@ +using EasyNetQ; using MalwareMultiScan.Api.Services; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.OpenApi.Models; using MongoDB.Driver; namespace MalwareMultiScan.Api @@ -18,8 +21,11 @@ namespace MalwareMultiScan.Api public void ConfigureServices(IServiceCollection services) { + services.AddSingleton(x => + RabbitHutch.CreateBus(_configuration.GetConnectionString("RabbitMQ"))); + services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton( serviceProvider => @@ -32,13 +38,45 @@ namespace MalwareMultiScan.Api services.AddControllers(); services.AddHttpClient(); + + services.AddSwaggerGen(options => + { + options.SwaggerDoc("MalwareMultiScan", + new OpenApiInfo + { + Title = "MalwareMultiScan", + Version = "1.0.0" + }); + }); + + services.AddHostedService(); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.UseRouting(); + + var forwardingOptions = new ForwardedHeadersOptions + { + ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto + }; - app.UseEndpoints(endpoints => { endpoints.MapControllers(); }); + forwardingOptions.KnownNetworks.Clear(); + forwardingOptions.KnownProxies.Clear(); + + app.UseForwardedHeaders(forwardingOptions); + + app.UseEndpoints(endpoints => endpoints.MapControllers()); + + app.UseSwagger(); + + app.UseSwaggerUI(options => + { + options.DocumentTitle = "MalwareMultiScan"; + + options.SwaggerEndpoint( + $"/swagger/{options.DocumentTitle}/swagger.json", options.DocumentTitle); + }); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/appsettings.json b/MalwareMultiScan.Api/appsettings.json index 4ff2c96..1f83738 100644 --- a/MalwareMultiScan.Api/appsettings.json +++ b/MalwareMultiScan.Api/appsettings.json @@ -9,11 +9,13 @@ "AllowedHosts": "*", "ConnectionStrings": { - "Mongo": "mongodb://localhost:27017" + "Mongo": "mongodb://localhost:27017", + "RabbitMQ": "host=localhost" }, "DatabaseName": "MalwareMultiScan", - "BackendsConfiguration": "backends.yaml", - "StoreFileUploads": true + "ResultsSubscriptionId": "mms.results", + + "BackendsConfiguration": "backends.yaml" } diff --git a/MalwareMultiScan.Api/backends.yaml b/MalwareMultiScan.Api/backends.yaml index 068828c..2c35e07 100644 --- a/MalwareMultiScan.Api/backends.yaml +++ b/MalwareMultiScan.Api/backends.yaml @@ -1,7 +1,2 @@ -- id: windows-defender - name: Windows Defender - endpoint: http://localhost:9902 - -- id: clamav - name: ClamAV - endpoint: http://localhost:9901 \ No newline at end of file +- id: dummy + name: Dummy Backend \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs index 0ccb4d5..8798d8c 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs @@ -17,6 +17,8 @@ namespace MalwareMultiScan.Backends.Backends.Implementations protected override Regex MatchRegex { get; } = new Regex(@"(\S+): (?[\S]+) FOUND", RegexOptions.Compiled | RegexOptions.Multiline); + protected override bool ThrowOnNonZeroExitCode { get; } = false; + protected override string GetBackendArguments(string path) { return $"--no-summary {path}"; diff --git a/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs new file mode 100644 index 0000000..42338db --- /dev/null +++ b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using MalwareMultiScan.Shared.Interfaces; +using Microsoft.AspNetCore.Http; + +namespace MalwareMultiScan.Backends.Backends.Implementations +{ + public class DummyScanBackend : IScanBackend + { + public string Id { get; } = "dummy"; + + public Task ScanAsync(string path, CancellationToken cancellationToken) + { + return Scan(); + } + + public Task ScanAsync(Uri uri, CancellationToken cancellationToken) + { + return Scan(); + } + + public Task ScanAsync(IFormFile file, CancellationToken cancellationToken) + { + return Scan(); + } + + public Task ScanAsync(Stream stream, CancellationToken cancellationToken) + { + return Scan(); + } + + private static Task Scan() + { + return Task.FromResult(new[] {"Malware.Dummy.Result"}); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs b/MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..da1d90b --- /dev/null +++ b/MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,32 @@ +using System; +using MalwareMultiScan.Backends.Backends.Implementations; +using MalwareMultiScan.Shared.Data.Enums; +using MalwareMultiScan.Shared.Interfaces; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Scanner.Extensions +{ + public static class ServiceCollectionExtensions + { + public static void AddScanningBackend(this IServiceCollection services, BackendType type) + { + using var provider = services.BuildServiceProvider(); + + var logger = provider.GetService(); + + services.AddSingleton(type switch + { + BackendType.Dummy => new DummyScanBackend(), + BackendType.Defender => new WindowsDefenderScanBackend(logger), + BackendType.Clamav => new ClamavScanBackend(logger), + BackendType.DrWeb => new DrWebScanBackend(logger), + BackendType.Kes => new KesScanBackend(logger), + BackendType.Comodo => new ComodoScanBackend(logger), + BackendType.Sophos => new SophosScanBackend(logger), + BackendType.McAfee => new McAfeeScanBackend(logger), + _ => throw new NotImplementedException() + }); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj new file mode 100644 index 0000000..ef10cce --- /dev/null +++ b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj @@ -0,0 +1,16 @@ + + + + netcoreapp3.1 + + + + + + + + + + + + diff --git a/MalwareMultiScan.Scanner/Program.cs b/MalwareMultiScan.Scanner/Program.cs new file mode 100644 index 0000000..034996d --- /dev/null +++ b/MalwareMultiScan.Scanner/Program.cs @@ -0,0 +1,31 @@ +using System.Threading.Tasks; +using MalwareMultiScan.Scanner.Extensions; +using MalwareMultiScan.Shared.Data.Enums; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace MalwareMultiScan.Scanner +{ + public static class Program + { + public static async Task Main(string[] args) + { + await Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration(configure => + { + configure.AddJsonFile("settings.json"); + configure.AddEnvironmentVariables(); + }) + .ConfigureServices((context, services) => + { + services.AddLogging(); + + services.AddScanningBackend( + context.Configuration.GetValue("BackendType")); + + services.AddHostedService(); + }).RunConsoleAsync(); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/Properties/launchSettings.json b/MalwareMultiScan.Scanner/Properties/launchSettings.json new file mode 100644 index 0000000..f97ddfe --- /dev/null +++ b/MalwareMultiScan.Scanner/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "MalwareMultiScan.Scanner": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/MalwareMultiScan.Scanner/ScanHostedService.cs b/MalwareMultiScan.Scanner/ScanHostedService.cs new file mode 100644 index 0000000..214ab5b --- /dev/null +++ b/MalwareMultiScan.Scanner/ScanHostedService.cs @@ -0,0 +1,73 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using EasyNetQ; +using MalwareMultiScan.Shared.Data.Messages; +using MalwareMultiScan.Shared.Interfaces; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace MalwareMultiScan.Scanner +{ + public class ScanHostedService : IHostedService + { + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + private readonly IScanBackend _scanBackend; + + private IBus _bus; + + public ScanHostedService(ILogger logger, IConfiguration configuration, IScanBackend scanBackend) + { + _logger = logger; + _configuration = configuration; + _scanBackend = scanBackend; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _bus = RabbitHutch.CreateBus( + _configuration.GetConnectionString("RabbitMQ")); + + _bus.Receive( + _scanBackend.Id, Scan); + + return Task.CompletedTask; + } + + + public Task StopAsync(CancellationToken cancellationToken) + { + _bus.Dispose(); + + return Task.CompletedTask; + } + + private async Task Scan(ScanRequestMessage message) + { + _logger.LogInformation( + $"Starting scan of {message.Uri} via backend {_scanBackend.Id}"); + + var cancellationTokenSource = new CancellationTokenSource( + TimeSpan.FromSeconds(_configuration.GetValue("MaxScanningTime"))); + + try + { + await _bus.SendAsync(_configuration.GetValue("ResultsSubscriptionId"), new ScanResultMessage + { + Id = message.Id, + Backend = _scanBackend.Id, + + Threats = await _scanBackend.ScanAsync( + message.Uri, cancellationTokenSource.Token) + }); + } + catch (Exception exception) + { + _logger.LogError( + exception, "Scanning failed with exception"); + } + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/settings.json b/MalwareMultiScan.Scanner/settings.json new file mode 100644 index 0000000..18e14f3 --- /dev/null +++ b/MalwareMultiScan.Scanner/settings.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + + "BackendType": "Dummy", + + "MaxWorkers": 5, + "MaxScanningTime": 60, + + "ResultsSubscriptionId": "mms.results", + + "ConnectionStrings": { + "RabbitMQ": "host=localhost;prefetchcount=1" + } +} diff --git a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs b/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs index ba6f232..3bd58e0 100644 --- a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs +++ b/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs @@ -9,8 +9,8 @@ namespace MalwareMultiScan.Shared.Attributes { var uri = (Uri) value; - if (uri == null || uri.Scheme != "http" && uri.Scheme != "https") - return new ValidationResult("Only http(s) URLs are supported"); + if (uri == null || !uri.IsAbsoluteUri || uri.Scheme != "http" && uri.Scheme != "https") + return new ValidationResult("Only http(s) absolute URLs are supported"); return ValidationResult.Success; } diff --git a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs b/MalwareMultiScan.Shared/Data/Enums/BackendType.cs index 31e4fea..7bdad0c 100644 --- a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs +++ b/MalwareMultiScan.Shared/Data/Enums/BackendType.cs @@ -2,6 +2,7 @@ namespace MalwareMultiScan.Shared.Data.Enums { public enum BackendType { + Dummy, Defender, Clamav, DrWeb, diff --git a/MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs b/MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs new file mode 100644 index 0000000..c5b857e --- /dev/null +++ b/MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs @@ -0,0 +1,11 @@ +using System; + +namespace MalwareMultiScan.Shared.Data.Messages +{ + public class ScanRequestMessage + { + public string Id { get; set; } + + public Uri Uri { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs b/MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs new file mode 100644 index 0000000..f288000 --- /dev/null +++ b/MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs @@ -0,0 +1,9 @@ +namespace MalwareMultiScan.Shared.Data.Messages +{ + public class ScanResultMessage + { + public string Id { get; set; } + public string Backend { get; set; } + public string[] Threats { get; set; } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.sln b/MalwareMultiScan.sln index 4bdf84b..5bf071d 100644 --- a/MalwareMultiScan.sln +++ b/MalwareMultiScan.sln @@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Scanner", "MalwareMultiScan.Scanner\MalwareMultiScan.Scanner.csproj", "{8A16A3C4-2AE3-4F63-8280-635FF7878080}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,9 @@ Global {7B63B897-D390-4617-821F-F96799CBA2F4}.Debug|Any CPU.Build.0 = Debug|Any CPU {7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {7B63B897-D390-4617-821F-F96799CBA2F4}.Release|Any CPU.Build.0 = Release|Any CPU + {8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A16A3C4-2AE3-4F63-8280-635FF7878080}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A16A3C4-2AE3-4F63-8280-635FF7878080}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal From 14c9c269798c1087202874882bc7678648bf3ba5 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Mon, 26 Oct 2020 17:06:29 +0200 Subject: [PATCH 5/6] refactoring, documentation pending --- .../Attributes/HttpUrlValidationAttribute.cs | 6 +- .../Controllers/QueueController.cs | 31 ++--- .../Controllers/ScanResultsController.cs | 3 +- .../Data/Models/ScanResult.cs | 2 +- .../Data/Response/ScanBackendResponse.cs | 9 -- .../MalwareMultiScan.Api.csproj | 1 + .../Services/ReceiverHostedService.cs | 18 ++- .../Services/ScanBackendService.cs | 14 ++- .../Services/ScanResultService.cs | 24 ++-- MalwareMultiScan.Api/Startup.cs | 14 +-- .../AbstractLocalProcessScanBackend.cs | 9 +- .../Backends/Abstracts/AbstractScanBackend.cs | 10 +- .../Implementations/ComodoScanBackend.cs | 2 - .../Implementations/DummyScanBackend.cs | 2 +- .../Enums/BackendType.cs | 2 +- .../Extensions/ServiceCollectionExtensions.cs | 12 +- .../Interfaces/IScanBackend.cs | 2 +- .../MalwareMultiScan.Backends.csproj | 4 + .../Messages/ScanRequestMessage.cs | 4 +- .../Messages/ScanResultMessage.cs | 2 +- .../Dockerfile | 0 .../MalwareMultiScan.Scanner.csproj | 4 + MalwareMultiScan.Scanner/Program.cs | 7 +- .../Properties/launchSettings.json | 10 -- .../{ => Services}/ScanHostedService.cs | 40 ++++--- .../{settings.json => appsettings.json} | 4 +- .../Data/Requests/BasicRequest.cs | 13 -- .../Data/Requests/FileRequest.cs | 11 -- .../Data/Requests/UrlRequest.cs | 13 -- .../Data/Responses/ResultResponse.cs | 17 --- .../MalwareMultiScan.Shared.csproj | 12 -- .../Controllers/ScanController.cs | 55 --------- MalwareMultiScan.Worker/Jobs/ScanJob.cs | 111 ------------------ .../MalwareMultiScan.Worker.csproj | 21 ---- MalwareMultiScan.Worker/Program.cs | 19 --- MalwareMultiScan.Worker/Startup.cs | 36 ------ .../appsettings.Development.json | 9 -- MalwareMultiScan.Worker/appsettings.json | 13 -- MalwareMultiScan.sln | 12 -- 39 files changed, 121 insertions(+), 457 deletions(-) rename MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs => MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs (65%) delete mode 100644 MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs rename {MalwareMultiScan.Shared/Data => MalwareMultiScan.Backends}/Enums/BackendType.cs (78%) rename {MalwareMultiScan.Scanner => MalwareMultiScan.Backends}/Extensions/ServiceCollectionExtensions.cs (83%) rename {MalwareMultiScan.Shared => MalwareMultiScan.Backends}/Interfaces/IScanBackend.cs (92%) rename {MalwareMultiScan.Shared/Data => MalwareMultiScan.Backends}/Messages/ScanRequestMessage.cs (71%) rename {MalwareMultiScan.Shared/Data => MalwareMultiScan.Backends}/Messages/ScanResultMessage.cs (78%) rename {MalwareMultiScan.Worker => MalwareMultiScan.Scanner}/Dockerfile (100%) delete mode 100644 MalwareMultiScan.Scanner/Properties/launchSettings.json rename MalwareMultiScan.Scanner/{ => Services}/ScanHostedService.cs (58%) rename MalwareMultiScan.Scanner/{settings.json => appsettings.json} (94%) delete mode 100644 MalwareMultiScan.Shared/Data/Requests/BasicRequest.cs delete mode 100644 MalwareMultiScan.Shared/Data/Requests/FileRequest.cs delete mode 100644 MalwareMultiScan.Shared/Data/Requests/UrlRequest.cs delete mode 100644 MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs delete mode 100644 MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj delete mode 100644 MalwareMultiScan.Worker/Controllers/ScanController.cs delete mode 100644 MalwareMultiScan.Worker/Jobs/ScanJob.cs delete mode 100644 MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj delete mode 100644 MalwareMultiScan.Worker/Program.cs delete mode 100644 MalwareMultiScan.Worker/Startup.cs delete mode 100644 MalwareMultiScan.Worker/appsettings.Development.json delete mode 100644 MalwareMultiScan.Worker/appsettings.json diff --git a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs b/MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs similarity index 65% rename from MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs rename to MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs index 3bd58e0..feef5c4 100644 --- a/MalwareMultiScan.Shared/Attributes/UrlValidationAttribute.cs +++ b/MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs @@ -1,16 +1,16 @@ using System; using System.ComponentModel.DataAnnotations; -namespace MalwareMultiScan.Shared.Attributes +namespace MalwareMultiScan.Api.Attributes { - public class UrlValidationAttribute : ValidationAttribute + public class HttpUrlValidationAttribute : ValidationAttribute { protected override ValidationResult IsValid(object value, ValidationContext validationContext) { var uri = (Uri) value; if (uri == null || !uri.IsAbsoluteUri || uri.Scheme != "http" && uri.Scheme != "https") - return new ValidationResult("Only http(s) absolute URLs are supported"); + return new ValidationResult("Only absolute http(s) URLs are supported"); return ValidationResult.Success; } diff --git a/MalwareMultiScan.Api/Controllers/QueueController.cs b/MalwareMultiScan.Api/Controllers/QueueController.cs index de4e9c4..3cb44ab 100644 --- a/MalwareMultiScan.Api/Controllers/QueueController.cs +++ b/MalwareMultiScan.Api/Controllers/QueueController.cs @@ -1,9 +1,8 @@ using System; -using System.IO; using System.Threading.Tasks; +using MalwareMultiScan.Api.Attributes; +using MalwareMultiScan.Api.Data.Models; using MalwareMultiScan.Api.Services; -using MalwareMultiScan.Shared.Attributes; -using MalwareMultiScan.Shared.Data.Responses; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -16,13 +15,13 @@ namespace MalwareMultiScan.Api.Controllers { private readonly ScanResultService _scanResultService; - public QueueController( ScanResultService scanResultService) + public QueueController(ScanResultService scanResultService) { _scanResultService = scanResultService; } - + [HttpPost("file")] - [ProducesResponseType(typeof(ResultResponse), StatusCodes.Status201Created)] + [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)] [ProducesResponseType(StatusCodes.Status400BadRequest)] public async Task ScanFile([FromForm] IFormFile file) { @@ -31,27 +30,29 @@ namespace MalwareMultiScan.Api.Controllers string storedFileId; await using (var uploadFileStream = file.OpenReadStream()) + { storedFileId = await _scanResultService.StoreFile(file.Name, uploadFileStream); - - await _scanResultService.QueueUrlScan(result, Url.Action("Index", "Download", new {id = storedFileId}, + } + + await _scanResultService.QueueUrlScan(result, Url.Action("Index", "Download", new {id = storedFileId}, Request.Scheme, Request.Host.Value)); - return Created(Url.Action("Index", "ScanResults", new {id = result.Id}, + return Created(Url.Action("Index", "ScanResults", new {id = result.Id}, Request.Scheme, Request.Host.Value), result); } - + [HttpPost("url")] - [ProducesResponseType(typeof(ResultResponse), StatusCodes.Status201Created)] + [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)] [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task ScanUrl([FromForm] [UrlValidation] Uri url) + public async Task ScanUrl([FromForm] [HttpUrlValidation] Uri url) { var result = await _scanResultService.CreateScanResult(); - - var resultUrl = Url.Action("Index", "ScanResults", new {id = result.Id}, + + var resultUrl = Url.Action("Index", "ScanResults", new {id = result.Id}, Request.Scheme, Request.Host.Value); await _scanResultService.QueueUrlScan(result, url.ToString()); - + return Created(resultUrl, result); } } diff --git a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs index c2aca1f..5137920 100644 --- a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs +++ b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs @@ -1,7 +1,6 @@ using System.Threading.Tasks; using MalwareMultiScan.Api.Data.Models; using MalwareMultiScan.Api.Services; -using MalwareMultiScan.Shared.Data.Responses; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -18,7 +17,7 @@ namespace MalwareMultiScan.Api.Controllers { _scanResultService = scanResultService; } - + [HttpGet("{id}")] [ProducesResponseType(typeof(ScanResult), StatusCodes.Status200OK)] public async Task Index(string id) diff --git a/MalwareMultiScan.Api/Data/Models/ScanResult.cs b/MalwareMultiScan.Api/Data/Models/ScanResult.cs index c61369c..76315c8 100644 --- a/MalwareMultiScan.Api/Data/Models/ScanResult.cs +++ b/MalwareMultiScan.Api/Data/Models/ScanResult.cs @@ -10,7 +10,7 @@ namespace MalwareMultiScan.Api.Data.Models [BsonRepresentation(BsonType.ObjectId)] public string Id { get; set; } - public Dictionary Results { get; set; } = + public Dictionary Results { get; set; } = new Dictionary(); } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs b/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs deleted file mode 100644 index a7afe8f..0000000 --- a/MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace MalwareMultiScan.Api.Data.Response -{ - public class ScanBackendResponse - { - public string Id { get; set; } - public string Name { get; set; } - public bool Online { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index bb6c35a..472b337 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -23,6 +23,7 @@ + diff --git a/MalwareMultiScan.Api/Services/ReceiverHostedService.cs b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs index 61a2402..6996d5d 100644 --- a/MalwareMultiScan.Api/Services/ReceiverHostedService.cs +++ b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs @@ -1,9 +1,10 @@ using System.Threading; using System.Threading.Tasks; using EasyNetQ; -using MalwareMultiScan.Shared.Data.Messages; +using MalwareMultiScan.Backends.Messages; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace MalwareMultiScan.Api.Services { @@ -12,21 +13,31 @@ namespace MalwareMultiScan.Api.Services private readonly IBus _bus; private readonly IConfiguration _configuration; private readonly ScanResultService _scanResultService; + private readonly ILogger _logger; - public ReceiverHostedService(IBus bus, IConfiguration configuration, ScanResultService scanResultService) + public ReceiverHostedService(IBus bus, IConfiguration configuration, ScanResultService scanResultService, + ILogger logger) { _bus = bus; _configuration = configuration; _scanResultService = scanResultService; + _logger = logger; } public Task StartAsync(CancellationToken cancellationToken) { _bus.Receive(_configuration.GetValue("ResultsSubscriptionId"), async message => { + _logger.LogInformation( + $"Received a result from {message.Backend} for {message.Id} " + + $"with threats {string.Join(",", message.Threats)}"); + await _scanResultService.UpdateScanResultForBackend( message.Id, message.Backend, true, true, message.Threats); }); + + _logger.LogInformation( + "Started hosted service for receiving scan results"); return Task.CompletedTask; } @@ -35,6 +46,9 @@ namespace MalwareMultiScan.Api.Services { _bus.Dispose(); + _logger.LogInformation( + "Stopped hosted service for receiving scan results"); + return Task.CompletedTask; } } diff --git a/MalwareMultiScan.Api/Services/ScanBackendService.cs b/MalwareMultiScan.Api/Services/ScanBackendService.cs index 81bc744..d8926b9 100644 --- a/MalwareMultiScan.Api/Services/ScanBackendService.cs +++ b/MalwareMultiScan.Api/Services/ScanBackendService.cs @@ -4,8 +4,9 @@ using System.Threading.Tasks; using EasyNetQ; using MalwareMultiScan.Api.Data.Configuration; using MalwareMultiScan.Api.Data.Models; -using MalwareMultiScan.Shared.Data.Messages; +using MalwareMultiScan.Backends.Messages; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -14,11 +15,13 @@ namespace MalwareMultiScan.Api.Services public class ScanBackendService { private readonly IBus _bus; + private readonly ILogger _logger; - public ScanBackendService(IConfiguration configuration, IBus bus) + public ScanBackendService(IConfiguration configuration, IBus bus, ILogger logger) { _bus = bus; - + _logger = logger; + var configurationPath = configuration.GetValue("BackendsConfiguration"); if (!File.Exists(configurationPath)) @@ -34,9 +37,12 @@ namespace MalwareMultiScan.Api.Services } public ScanBackend[] List { get; } - + public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl) { + _logger.LogInformation( + $"Queueing scan for {result.Id} on {backend.Id} at {fileUrl}"); + await _bus.SendAsync(backend.Id, new ScanRequestMessage { Id = result.Id, diff --git a/MalwareMultiScan.Api/Services/ScanResultService.cs b/MalwareMultiScan.Api/Services/ScanResultService.cs index 6e85f8c..93b1926 100644 --- a/MalwareMultiScan.Api/Services/ScanResultService.cs +++ b/MalwareMultiScan.Api/Services/ScanResultService.cs @@ -1,13 +1,7 @@ -using System; using System.IO; using System.Linq; -using System.Net.Http; -using System.Threading; using System.Threading.Tasks; -using MalwareMultiScan.Api.Data.Configuration; using MalwareMultiScan.Api.Data.Models; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.GridFS; @@ -17,16 +11,16 @@ namespace MalwareMultiScan.Api.Services public class ScanResultService { private const string CollectionName = "ScanResults"; - - private readonly IMongoCollection _collection; private readonly GridFSBucket _bucket; - + + private readonly IMongoCollection _collection; + private readonly ScanBackendService _scanBackendService; public ScanResultService(IMongoDatabase db, ScanBackendService scanBackendService) { _scanBackendService = scanBackendService; - + _collection = db.GetCollection(CollectionName); _bucket = new GridFSBucket(db); } @@ -40,7 +34,7 @@ namespace MalwareMultiScan.Api.Services }; await _collection.InsertOneAsync(scanResult); - + return scanResult; } @@ -51,12 +45,12 @@ namespace MalwareMultiScan.Api.Services return await result.FirstOrDefaultAsync(); } - - public async Task UpdateScanResultForBackend(string resultId, string backendId, + + public async Task UpdateScanResultForBackend(string resultId, string backendId, bool completed = false, bool succeeded = false, string[] threats = null) { var filterScanResult = Builders.Filter.Where(r => r.Id == resultId); - + var updateScanResult = Builders.Update.Set(r => r.Results[backendId], new ScanResultEntry { Completed = completed, @@ -85,7 +79,7 @@ namespace MalwareMultiScan.Api.Services { if (!ObjectId.TryParse(id, out var objectId)) return null; - + return await _bucket.OpenDownloadStreamAsync(objectId, new GridFSDownloadOptions { Seekable = true diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index 3685a83..6fbc9ec 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -18,7 +18,7 @@ namespace MalwareMultiScan.Api { _configuration = configuration; } - + public void ConfigureServices(IServiceCollection services) { services.AddSingleton(x => @@ -26,7 +26,7 @@ namespace MalwareMultiScan.Api services.AddSingleton(); services.AddSingleton(); - + services.AddSingleton( serviceProvider => { @@ -35,7 +35,7 @@ namespace MalwareMultiScan.Api return db.GetDatabase( _configuration.GetValue("DatabaseName")); }); - + services.AddControllers(); services.AddHttpClient(); @@ -48,14 +48,14 @@ namespace MalwareMultiScan.Api Version = "1.0.0" }); }); - + services.AddHostedService(); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.UseRouting(); - + var forwardingOptions = new ForwardedHeadersOptions { ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto @@ -67,13 +67,13 @@ namespace MalwareMultiScan.Api app.UseForwardedHeaders(forwardingOptions); app.UseEndpoints(endpoints => endpoints.MapControllers()); - + app.UseSwagger(); app.UseSwaggerUI(options => { options.DocumentTitle = "MalwareMultiScan"; - + options.SwaggerEndpoint( $"/swagger/{options.DocumentTitle}/swagger.json", options.DocumentTitle); }); diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs index 5e4c344..b00dc5b 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs @@ -18,15 +18,12 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts _logger = logger; } - protected virtual Regex MatchRegex { get; } - protected virtual string BackendPath { get; } + protected abstract Regex MatchRegex { get; } + protected abstract string BackendPath { get; } protected virtual bool ParseStdErr { get; } protected virtual bool ThrowOnNonZeroExitCode { get; } = true; - protected virtual string GetBackendArguments(string path) - { - throw new NotImplementedException(); - } + protected abstract string GetBackendArguments(string path); public override async Task ScanAsync(string path, CancellationToken cancellationToken) { diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs index 49633b2..03aa90a 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs @@ -3,23 +3,21 @@ using System.IO; using System.Net.Http; using System.Threading; using System.Threading.Tasks; -using MalwareMultiScan.Shared.Interfaces; +using MalwareMultiScan.Backends.Interfaces; using Microsoft.AspNetCore.Http; namespace MalwareMultiScan.Backends.Backends.Abstracts { public abstract class AbstractScanBackend : IScanBackend { - public virtual string Id => throw new NotImplementedException(); + public abstract string Id { get; } - public virtual Task ScanAsync(string path, CancellationToken cancellationToken) - { - throw new NotImplementedException(); - } + public abstract Task ScanAsync(string path, CancellationToken cancellationToken); public async Task ScanAsync(Uri uri, CancellationToken cancellationToken) { using var httpClient = new HttpClient(); + await using var uriStream = await httpClient.GetStreamAsync(uri); return await ScanAsync(uriStream, cancellationToken); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs index ac8b3a2..e222d3e 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Text.RegularExpressions; using MalwareMultiScan.Backends.Backends.Abstracts; using Microsoft.Extensions.Logging; diff --git a/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs index 42338db..fb70597 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs @@ -2,7 +2,7 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; -using MalwareMultiScan.Shared.Interfaces; +using MalwareMultiScan.Backends.Interfaces; using Microsoft.AspNetCore.Http; namespace MalwareMultiScan.Backends.Backends.Implementations diff --git a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs b/MalwareMultiScan.Backends/Enums/BackendType.cs similarity index 78% rename from MalwareMultiScan.Shared/Data/Enums/BackendType.cs rename to MalwareMultiScan.Backends/Enums/BackendType.cs index 7bdad0c..cc935b3 100644 --- a/MalwareMultiScan.Shared/Data/Enums/BackendType.cs +++ b/MalwareMultiScan.Backends/Enums/BackendType.cs @@ -1,4 +1,4 @@ -namespace MalwareMultiScan.Shared.Data.Enums +namespace MalwareMultiScan.Backends.Enums { public enum BackendType { diff --git a/MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs b/MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs similarity index 83% rename from MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs rename to MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs index da1d90b..a7005c8 100644 --- a/MalwareMultiScan.Scanner/Extensions/ServiceCollectionExtensions.cs +++ b/MalwareMultiScan.Backends/Extensions/ServiceCollectionExtensions.cs @@ -1,20 +1,20 @@ using System; using MalwareMultiScan.Backends.Backends.Implementations; -using MalwareMultiScan.Shared.Data.Enums; -using MalwareMultiScan.Shared.Interfaces; +using MalwareMultiScan.Backends.Enums; +using MalwareMultiScan.Backends.Interfaces; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace MalwareMultiScan.Scanner.Extensions +namespace MalwareMultiScan.Backends.Extensions { public static class ServiceCollectionExtensions { public static void AddScanningBackend(this IServiceCollection services, BackendType type) { using var provider = services.BuildServiceProvider(); - - var logger = provider.GetService(); - + + var logger = provider.GetService>(); + services.AddSingleton(type switch { BackendType.Dummy => new DummyScanBackend(), diff --git a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs b/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs similarity index 92% rename from MalwareMultiScan.Shared/Interfaces/IScanBackend.cs rename to MalwareMultiScan.Backends/Interfaces/IScanBackend.cs index dcf0411..b234ce5 100644 --- a/MalwareMultiScan.Shared/Interfaces/IScanBackend.cs +++ b/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; -namespace MalwareMultiScan.Shared.Interfaces +namespace MalwareMultiScan.Backends.Interfaces { public interface IScanBackend { diff --git a/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj b/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj index fb7bbed..0ce0c90 100644 --- a/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj +++ b/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj @@ -8,4 +8,8 @@ + + + + diff --git a/MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs b/MalwareMultiScan.Backends/Messages/ScanRequestMessage.cs similarity index 71% rename from MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs rename to MalwareMultiScan.Backends/Messages/ScanRequestMessage.cs index c5b857e..2ba674c 100644 --- a/MalwareMultiScan.Shared/Data/Messages/ScanRequestMessage.cs +++ b/MalwareMultiScan.Backends/Messages/ScanRequestMessage.cs @@ -1,11 +1,11 @@ using System; -namespace MalwareMultiScan.Shared.Data.Messages +namespace MalwareMultiScan.Backends.Messages { public class ScanRequestMessage { public string Id { get; set; } - + public Uri Uri { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs b/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs similarity index 78% rename from MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs rename to MalwareMultiScan.Backends/Messages/ScanResultMessage.cs index f288000..f6627d7 100644 --- a/MalwareMultiScan.Shared/Data/Messages/ScanResultMessage.cs +++ b/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs @@ -1,4 +1,4 @@ -namespace MalwareMultiScan.Shared.Data.Messages +namespace MalwareMultiScan.Backends.Messages { public class ScanResultMessage { diff --git a/MalwareMultiScan.Worker/Dockerfile b/MalwareMultiScan.Scanner/Dockerfile similarity index 100% rename from MalwareMultiScan.Worker/Dockerfile rename to MalwareMultiScan.Scanner/Dockerfile diff --git a/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj index ef10cce..5930542 100644 --- a/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj +++ b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj @@ -13,4 +13,8 @@ + + + <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> + diff --git a/MalwareMultiScan.Scanner/Program.cs b/MalwareMultiScan.Scanner/Program.cs index 034996d..eba4512 100644 --- a/MalwareMultiScan.Scanner/Program.cs +++ b/MalwareMultiScan.Scanner/Program.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; -using MalwareMultiScan.Scanner.Extensions; -using MalwareMultiScan.Shared.Data.Enums; +using MalwareMultiScan.Backends.Enums; +using MalwareMultiScan.Backends.Extensions; +using MalwareMultiScan.Scanner.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -14,7 +15,7 @@ namespace MalwareMultiScan.Scanner await Host.CreateDefaultBuilder(args) .ConfigureAppConfiguration(configure => { - configure.AddJsonFile("settings.json"); + configure.AddJsonFile("appsettings.json"); configure.AddEnvironmentVariables(); }) .ConfigureServices((context, services) => diff --git a/MalwareMultiScan.Scanner/Properties/launchSettings.json b/MalwareMultiScan.Scanner/Properties/launchSettings.json deleted file mode 100644 index f97ddfe..0000000 --- a/MalwareMultiScan.Scanner/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "MalwareMultiScan.Scanner": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} diff --git a/MalwareMultiScan.Scanner/ScanHostedService.cs b/MalwareMultiScan.Scanner/Services/ScanHostedService.cs similarity index 58% rename from MalwareMultiScan.Scanner/ScanHostedService.cs rename to MalwareMultiScan.Scanner/Services/ScanHostedService.cs index 214ab5b..5c52d0d 100644 --- a/MalwareMultiScan.Scanner/ScanHostedService.cs +++ b/MalwareMultiScan.Scanner/Services/ScanHostedService.cs @@ -2,27 +2,27 @@ using System; using System.Threading; using System.Threading.Tasks; using EasyNetQ; -using MalwareMultiScan.Shared.Data.Messages; -using MalwareMultiScan.Shared.Interfaces; +using MalwareMultiScan.Backends.Interfaces; +using MalwareMultiScan.Backends.Messages; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -namespace MalwareMultiScan.Scanner +namespace MalwareMultiScan.Scanner.Services { - public class ScanHostedService : IHostedService + internal class ScanHostedService : IHostedService { + private readonly IScanBackend _backend; private readonly IConfiguration _configuration; private readonly ILogger _logger; - private readonly IScanBackend _scanBackend; private IBus _bus; - public ScanHostedService(ILogger logger, IConfiguration configuration, IScanBackend scanBackend) + public ScanHostedService(ILogger logger, IConfiguration configuration, IScanBackend backend) { _logger = logger; _configuration = configuration; - _scanBackend = scanBackend; + _backend = backend; } public Task StartAsync(CancellationToken cancellationToken) @@ -30,38 +30,48 @@ namespace MalwareMultiScan.Scanner _bus = RabbitHutch.CreateBus( _configuration.GetConnectionString("RabbitMQ")); - _bus.Receive( - _scanBackend.Id, Scan); + _bus.Receive(_backend.Id, Scan); + + _logger.LogInformation( + $"Started scan hosting service for the backend {_backend.Id}"); return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) { _bus.Dispose(); + _logger.LogInformation( + $"Stopped scan hosting service for the backend {_backend.Id}"); + return Task.CompletedTask; } private async Task Scan(ScanRequestMessage message) { _logger.LogInformation( - $"Starting scan of {message.Uri} via backend {_scanBackend.Id}"); + $"Starting scan of {message.Uri} via backend {_backend.Id} from {message.Id}"); var cancellationTokenSource = new CancellationTokenSource( TimeSpan.FromSeconds(_configuration.GetValue("MaxScanningTime"))); try { - await _bus.SendAsync(_configuration.GetValue("ResultsSubscriptionId"), new ScanResultMessage + var result = new ScanResultMessage { Id = message.Id, - Backend = _scanBackend.Id, + Backend = _backend.Id, - Threats = await _scanBackend.ScanAsync( + Threats = await _backend.ScanAsync( message.Uri, cancellationTokenSource.Token) - }); + }; + + _logger.LogInformation( + $"Backend {_backend.Id} completed a scan of {message.Id} " + + $"with result '{string.Join(", ", result.Threats)}'"); + + await _bus.SendAsync(_configuration.GetValue("ResultsSubscriptionId"), result); } catch (Exception exception) { diff --git a/MalwareMultiScan.Scanner/settings.json b/MalwareMultiScan.Scanner/appsettings.json similarity index 94% rename from MalwareMultiScan.Scanner/settings.json rename to MalwareMultiScan.Scanner/appsettings.json index 18e14f3..3c086e9 100644 --- a/MalwareMultiScan.Scanner/settings.json +++ b/MalwareMultiScan.Scanner/appsettings.json @@ -8,10 +8,8 @@ }, "BackendType": "Dummy", - - "MaxWorkers": 5, - "MaxScanningTime": 60, + "MaxScanningTime": 60, "ResultsSubscriptionId": "mms.results", "ConnectionStrings": { diff --git a/MalwareMultiScan.Shared/Data/Requests/BasicRequest.cs b/MalwareMultiScan.Shared/Data/Requests/BasicRequest.cs deleted file mode 100644 index 0870b99..0000000 --- a/MalwareMultiScan.Shared/Data/Requests/BasicRequest.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.ComponentModel.DataAnnotations; -using MalwareMultiScan.Shared.Attributes; - -namespace MalwareMultiScan.Shared.Data.Requests -{ - public abstract class BasicRequest - { - [Required] - [UrlValidation] - public Uri CallbackUrl { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Requests/FileRequest.cs b/MalwareMultiScan.Shared/Data/Requests/FileRequest.cs deleted file mode 100644 index 612bd18..0000000 --- a/MalwareMultiScan.Shared/Data/Requests/FileRequest.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using Microsoft.AspNetCore.Http; - -namespace MalwareMultiScan.Shared.Data.Requests -{ - public class FileRequest : BasicRequest - { - [Required] - public IFormFile InputFile { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Requests/UrlRequest.cs b/MalwareMultiScan.Shared/Data/Requests/UrlRequest.cs deleted file mode 100644 index e9a0478..0000000 --- a/MalwareMultiScan.Shared/Data/Requests/UrlRequest.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.ComponentModel.DataAnnotations; -using MalwareMultiScan.Shared.Attributes; - -namespace MalwareMultiScan.Shared.Data.Requests -{ - public class UrlRequest : BasicRequest - { - [Required] - [UrlValidation] - public Uri InputUrl { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs b/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs deleted file mode 100644 index 0f7542a..0000000 --- a/MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.ComponentModel.DataAnnotations; -using System.Linq; - -namespace MalwareMultiScan.Shared.Data.Responses -{ - public class ResultResponse - { - [Required] - public string Backend { get; set; } - - [Required] - public bool Success { get; set; } - - public string[] Threats { get; set; } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj b/MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj deleted file mode 100644 index 5d2c164..0000000 --- a/MalwareMultiScan.Shared/MalwareMultiScan.Shared.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - - netcoreapp3.1 - - - - - - - - diff --git a/MalwareMultiScan.Worker/Controllers/ScanController.cs b/MalwareMultiScan.Worker/Controllers/ScanController.cs deleted file mode 100644 index 230e440..0000000 --- a/MalwareMultiScan.Worker/Controllers/ScanController.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System.IO; -using System.Threading.Tasks; -using Hangfire; -using MalwareMultiScan.Shared.Data.Requests; -using MalwareMultiScan.Worker.Jobs; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; - -namespace MalwareMultiScan.Worker.Controllers -{ - [ApiController] - [Produces("application/json")] - public class ScanController : ControllerBase - { - - [HttpGet] - [ProducesResponseType(StatusCodes.Status200OK)] - [Route("/ping")] - public IActionResult Ping() - { - return Ok("pong"); - } - - [HttpPost] - [ProducesResponseType(StatusCodes.Status202Accepted)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [Route("/scan/file")] - public async Task ScanFile([FromForm] FileRequest request) - { - var temporaryFile = Path.GetTempFileName(); - - await using (var temporaryFileSteam = System.IO.File.OpenWrite(temporaryFile)) - { - await request.InputFile.CopyToAsync(temporaryFileSteam); - } - - BackgroundJob.Enqueue( - x => x.ScanFile(temporaryFile, request.CallbackUrl)); - - return Accepted(request.CallbackUrl); - } - - [HttpPost] - [ProducesResponseType(StatusCodes.Status202Accepted)] - [ProducesResponseType(StatusCodes.Status400BadRequest)] - [Route("/scan/url")] - public IActionResult ScanUrl([FromForm] UrlRequest request) - { - BackgroundJob.Enqueue( - x => x.ScanUrl(request.InputUrl, request.CallbackUrl)); - - return Accepted(request.CallbackUrl); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Worker/Jobs/ScanJob.cs b/MalwareMultiScan.Worker/Jobs/ScanJob.cs deleted file mode 100644 index 7dcf178..0000000 --- a/MalwareMultiScan.Worker/Jobs/ScanJob.cs +++ /dev/null @@ -1,111 +0,0 @@ -using System; -using System.IO; -using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Hangfire; -using MalwareMultiScan.Backends.Backends.Implementations; -using MalwareMultiScan.Shared.Data.Enums; -using MalwareMultiScan.Shared.Data.Responses; -using MalwareMultiScan.Shared.Interfaces; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace MalwareMultiScan.Worker.Jobs -{ - public class ScanJob - { - private readonly IScanBackend _backend; - private readonly IHttpClientFactory _httpClientFactory; - private readonly ILogger _logger; - private readonly int _scanTimeout; - - public ScanJob(IConfiguration configuration, ILogger logger, - IHttpClientFactory httpClientFactory) - { - _logger = logger; - _httpClientFactory = httpClientFactory; - _scanTimeout = configuration.GetValue("ScanTimeout"); - - _backend = configuration.GetValue("BackendType") switch - { - BackendType.Defender => new WindowsDefenderScanBackend(logger), - BackendType.Clamav => new ClamavScanBackend(logger), - BackendType.DrWeb => new DrWebScanBackend(logger), - BackendType.Kes => new KesScanBackend(logger), - BackendType.Comodo => new ComodoScanBackend(logger), - BackendType.Sophos => new SophosScanBackend(logger), - BackendType.McAfee => new McAfeeScanBackend(logger), - _ => throw new NotImplementedException() - }; - } - - private async Task PostResult(ResultResponse response, Uri callbackUrl, CancellationToken cancellationToken) - { - var serializedResponse = JsonSerializer.Serialize( - response, typeof(ResultResponse), new JsonSerializerOptions - { - WriteIndented = true - }); - - _logger.LogInformation( - $"Sending following payload to {callbackUrl}: {serializedResponse}"); - - using var httpClient = _httpClientFactory.CreateClient(); - - var callbackResponse = await httpClient.PostAsync(callbackUrl, - new StringContent(serializedResponse, Encoding.UTF8, "application/json"), cancellationToken); - - _logger.LogInformation($"Callback URL {callbackUrl} returned a status {callbackResponse.StatusCode}"); - } - - private async Task Scan(Func> scanMethod, Uri callbackUrl) - { - var cancellationTokenSource = new CancellationTokenSource(); - - cancellationTokenSource.CancelAfter(_scanTimeout * 1000); - - var cancellationToken = cancellationTokenSource.Token; - - var response = new ResultResponse - { - Backend = _backend.Id - }; - - try - { - response.Success = true; - response.Threats = await scanMethod(cancellationToken); - } - catch (Exception exception) - { - response.Success = false; - - _logger.LogError(exception, "Scanning failed with exception"); - } - - await PostResult(response, callbackUrl, cancellationToken); - } - - [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public async Task ScanUrl(Uri url, Uri callbackUrl) - { - await Scan(async t => await _backend.ScanAsync(url, t), callbackUrl); - } - - [AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)] - public async Task ScanFile(string file, Uri callbackUrl) - { - try - { - await Scan(async t => await _backend.ScanAsync(file, t), callbackUrl); - } - finally - { - File.Delete(file); - } - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj b/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj deleted file mode 100644 index ded602c..0000000 --- a/MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - netcoreapp3.1 - - - - - - - - - - - - - - <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> - - - diff --git a/MalwareMultiScan.Worker/Program.cs b/MalwareMultiScan.Worker/Program.cs deleted file mode 100644 index afcc239..0000000 --- a/MalwareMultiScan.Worker/Program.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.Hosting; - -namespace MalwareMultiScan.Worker -{ - public static class Program - { - public static void Main(string[] args) - { - CreateHostBuilder(args).Build().Run(); - } - - private static IHostBuilder CreateHostBuilder(string[] args) - { - return Host.CreateDefaultBuilder(args) - .ConfigureWebHostDefaults(builder => { builder.UseStartup(); }); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Worker/Startup.cs b/MalwareMultiScan.Worker/Startup.cs deleted file mode 100644 index a4e1b19..0000000 --- a/MalwareMultiScan.Worker/Startup.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Hangfire; -using Hangfire.MemoryStorage; -using MalwareMultiScan.Worker.Jobs; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.Extensions.DependencyInjection; - -namespace MalwareMultiScan.Worker -{ - public class Startup - { - public void ConfigureServices(IServiceCollection services) - { - services.AddLogging(); - services.AddControllers(); - services.AddHttpClient(); - - services.AddSingleton(); - - services.AddHangfire( - configuration => configuration.UseMemoryStorage()); - - services.AddHangfireServer(); - } - - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) - { - app.UseRouting(); - - app.UseEndpoints( - endpoints => endpoints.MapControllers()); - - app.UseHangfireServer(); - } - } -} \ No newline at end of file diff --git a/MalwareMultiScan.Worker/appsettings.Development.json b/MalwareMultiScan.Worker/appsettings.Development.json deleted file mode 100644 index 8983e0f..0000000 --- a/MalwareMultiScan.Worker/appsettings.Development.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information" - } - } -} diff --git a/MalwareMultiScan.Worker/appsettings.json b/MalwareMultiScan.Worker/appsettings.json deleted file mode 100644 index 1c64e84..0000000 --- a/MalwareMultiScan.Worker/appsettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Warning", - "MalwareMultiScan": "Debug" - } - }, - - "AllowedHosts": "*", - - "BackendType": "", - "ScanTimeout": 300 -} diff --git a/MalwareMultiScan.sln b/MalwareMultiScan.sln index 5bf071d..20591bc 100644 --- a/MalwareMultiScan.sln +++ b/MalwareMultiScan.sln @@ -1,9 +1,5 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Worker", "MalwareMultiScan.Worker\MalwareMultiScan.Worker.csproj", "{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", "MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj", "{9E0A0B50-741F-4A49-97A2-0B337374347F}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends", "MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj", "{382B49AC-0FFA-44FC-875D-9D4692DDC05D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Api", "MalwareMultiScan.Api\MalwareMultiScan.Api.csproj", "{7B63B897-D390-4617-821F-F96799CBA2F4}" @@ -16,14 +12,6 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Release|Any CPU.Build.0 = Release|Any CPU - {9E0A0B50-741F-4A49-97A2-0B337374347F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9E0A0B50-741F-4A49-97A2-0B337374347F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9E0A0B50-741F-4A49-97A2-0B337374347F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9E0A0B50-741F-4A49-97A2-0B337374347F}.Release|Any CPU.Build.0 = Release|Any CPU {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Debug|Any CPU.Build.0 = Debug|Any CPU {382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Release|Any CPU.ActiveCfg = Release|Any CPU From 62cdcdbb4993713660862410b39bd221b04ee619 Mon Sep 17 00:00:00 2001 From: Volodymyr Smirnov Date: Mon, 26 Oct 2020 21:24:40 +0200 Subject: [PATCH 6/6] finalize refactoring & ready for the PR --- ...> MalwareMultiScan_Scanner_Dockerfile.xml} | 4 +- ...tionAttribute.cs => IsHttpUrlAttribute.cs} | 12 ++-- .../Attributes/MaxFileSizeAttribute.cs | 28 ++++++++ .../Controllers/DownloadController.cs | 2 + .../Controllers/QueueController.cs | 20 +++--- .../Controllers/ScanResultsController.cs | 1 + .../Data/Configuration/ScanBackend.cs | 2 +- .../Data/Models/ScanResultEntry.cs | 2 +- .../Extensions/ServiceCollectionExtensions.cs | 42 +++++++++++ .../MalwareMultiScan.Api.csproj | 8 --- MalwareMultiScan.Api/Program.cs | 2 +- .../Services/ReceiverHostedService.cs | 26 +++++-- .../Services/ScanBackendService.cs | 21 +++++- .../Services/ScanResultService.cs | 71 ++++++++++++++----- MalwareMultiScan.Api/Startup.cs | 56 ++------------- .../appsettings.Development.json | 9 --- MalwareMultiScan.Api/appsettings.json | 3 +- MalwareMultiScan.Api/backends.yaml | 2 +- .../AbstractLocalProcessScanBackend.cs | 2 +- .../Backends/Abstracts/AbstractScanBackend.cs | 8 --- .../Implementations/DummyScanBackend.cs | 6 -- .../Dockerfiles/Clamav.Dockerfile | 2 +- .../Dockerfiles/Comodo.Dockerfile | 2 +- .../Dockerfiles/DrWeb.Dockerfile | 2 +- .../Dockerfiles/KES.Dockerfile | 2 +- .../Dockerfiles/McAfee.Dockerfile | 2 +- .../Dockerfiles/Sophos.Dockerfile | 2 +- .../Dockerfiles/WindowsDefender.Dockerfile | 2 +- .../Interfaces/IScanBackend.cs | 3 - .../MalwareMultiScan.Backends.csproj | 7 +- .../Messages/ScanResultMessage.cs | 2 + MalwareMultiScan.Scanner/Dockerfile | 10 +-- .../MalwareMultiScan.Scanner.csproj | 5 -- .../Services/ScanHostedService.cs | 25 ++++--- 34 files changed, 229 insertions(+), 164 deletions(-) rename .idea/.idea.MalwareMultiScan/.idea/runConfigurations/{MalwareMultiScan_Worker_Dockerfile.xml => MalwareMultiScan_Scanner_Dockerfile.xml} (72%) rename MalwareMultiScan.Api/Attributes/{HttpUrlValidationAttribute.cs => IsHttpUrlAttribute.cs} (51%) create mode 100644 MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs create mode 100644 MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs delete mode 100644 MalwareMultiScan.Api/appsettings.Development.json diff --git a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Worker_Dockerfile.xml b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml similarity index 72% rename from .idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Worker_Dockerfile.xml rename to .idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml index 5c2be23..5a37aed 100644 --- a/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Worker_Dockerfile.xml +++ b/.idea/.idea.MalwareMultiScan/.idea/runConfigurations/MalwareMultiScan_Scanner_Dockerfile.xml @@ -1,5 +1,5 @@ - + diff --git a/MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs b/MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs similarity index 51% rename from MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs rename to MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs index feef5c4..cda488e 100644 --- a/MalwareMultiScan.Api/Attributes/HttpUrlValidationAttribute.cs +++ b/MalwareMultiScan.Api/Attributes/IsHttpUrlAttribute.cs @@ -3,13 +3,17 @@ using System.ComponentModel.DataAnnotations; namespace MalwareMultiScan.Api.Attributes { - public class HttpUrlValidationAttribute : ValidationAttribute + /// + /// Validate URI to be an absolute http(s) URL. + /// + internal class IsHttpUrlAttribute : ValidationAttribute { + /// protected override ValidationResult IsValid(object value, ValidationContext validationContext) { - var uri = (Uri) value; - - if (uri == null || !uri.IsAbsoluteUri || uri.Scheme != "http" && uri.Scheme != "https") + if (!Uri.TryCreate((string)value, UriKind.Absolute, out var uri) + || !uri.IsAbsoluteUri + || uri.Scheme != "http" && uri.Scheme != "https") return new ValidationResult("Only absolute http(s) URLs are supported"); return ValidationResult.Success; diff --git a/MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs b/MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs new file mode 100644 index 0000000..56baa38 --- /dev/null +++ b/MalwareMultiScan.Api/Attributes/MaxFileSizeAttribute.cs @@ -0,0 +1,28 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace MalwareMultiScan.Api.Attributes +{ + /// + /// Validate uploaded file size for the max file size defined in settings. + /// + public class MaxFileSizeAttribute : ValidationAttribute + { + /// + protected override ValidationResult IsValid(object value, ValidationContext validationContext) + { + var maxSize = validationContext + .GetRequiredService() + .GetValue("MaxFileSize"); + + var formFile = (IFormFile) value; + + if (formFile == null || formFile.Length > maxSize) + return new ValidationResult($"File exceeds the maximum size of {maxSize} bytes"); + + return ValidationResult.Success; + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/DownloadController.cs b/MalwareMultiScan.Api/Controllers/DownloadController.cs index ead47f3..af088da 100644 --- a/MalwareMultiScan.Api/Controllers/DownloadController.cs +++ b/MalwareMultiScan.Api/Controllers/DownloadController.cs @@ -5,7 +5,9 @@ using Microsoft.AspNetCore.Mvc; namespace MalwareMultiScan.Api.Controllers { + [ApiController] [Route("download")] + [Produces("application/octet-stream")] public class DownloadController : Controller { private readonly ScanResultService _scanResultService; diff --git a/MalwareMultiScan.Api/Controllers/QueueController.cs b/MalwareMultiScan.Api/Controllers/QueueController.cs index 3cb44ab..b27e0db 100644 --- a/MalwareMultiScan.Api/Controllers/QueueController.cs +++ b/MalwareMultiScan.Api/Controllers/QueueController.cs @@ -1,4 +1,4 @@ -using System; +using System.ComponentModel.DataAnnotations; using System.Threading.Tasks; using MalwareMultiScan.Api.Attributes; using MalwareMultiScan.Api.Data.Models; @@ -23,37 +23,33 @@ namespace MalwareMultiScan.Api.Controllers [HttpPost("file")] [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)] [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task ScanFile([FromForm] IFormFile file) + public async Task ScanFile( + [Required, MaxFileSize] IFormFile file) { var result = await _scanResultService.CreateScanResult(); string storedFileId; await using (var uploadFileStream = file.OpenReadStream()) - { storedFileId = await _scanResultService.StoreFile(file.Name, uploadFileStream); - } await _scanResultService.QueueUrlScan(result, Url.Action("Index", "Download", new {id = storedFileId}, Request.Scheme, Request.Host.Value)); - return Created(Url.Action("Index", "ScanResults", new {id = result.Id}, - Request.Scheme, Request.Host.Value), result); + return CreatedAtAction("Index", "ScanResults", new {id = result.Id}, result); } [HttpPost("url")] [ProducesResponseType(typeof(ScanResult), StatusCodes.Status201Created)] [ProducesResponseType(StatusCodes.Status400BadRequest)] - public async Task ScanUrl([FromForm] [HttpUrlValidation] Uri url) + public async Task ScanUrl( + [FromForm, Required, IsHttpUrl] string url) { var result = await _scanResultService.CreateScanResult(); - var resultUrl = Url.Action("Index", "ScanResults", new {id = result.Id}, - Request.Scheme, Request.Host.Value); + await _scanResultService.QueueUrlScan(result, url); - await _scanResultService.QueueUrlScan(result, url.ToString()); - - return Created(resultUrl, result); + return CreatedAtAction("Index", "ScanResults", new {id = result.Id}, result); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs index 5137920..661a873 100644 --- a/MalwareMultiScan.Api/Controllers/ScanResultsController.cs +++ b/MalwareMultiScan.Api/Controllers/ScanResultsController.cs @@ -20,6 +20,7 @@ namespace MalwareMultiScan.Api.Controllers [HttpGet("{id}")] [ProducesResponseType(typeof(ScanResult), StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] public async Task Index(string id) { var scanResult = await _scanResultService.GetScanResult(id); diff --git a/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs index a230136..55f369c 100644 --- a/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs +++ b/MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs @@ -3,6 +3,6 @@ namespace MalwareMultiScan.Api.Data.Configuration public class ScanBackend { public string Id { get; set; } - public string Name { get; set; } + public bool Enabled { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs b/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs index dd3b536..a566c3d 100644 --- a/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs +++ b/MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs @@ -3,7 +3,7 @@ namespace MalwareMultiScan.Api.Data.Models public class ScanResultEntry { public bool Completed { get; set; } - public bool Succeeded { get; set; } + public bool? Succeeded { get; set; } public string[] Threats { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs b/MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..feb6a21 --- /dev/null +++ b/MalwareMultiScan.Api/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,42 @@ +using System.Net; +using EasyNetQ; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using MongoDB.Driver; + +namespace MalwareMultiScan.Api.Extensions +{ + internal static class ServiceCollectionExtensions + { + public static void AddMongoDb(this IServiceCollection services, IConfiguration configuration) + { + services.AddSingleton( + serviceProvider => + { + var db = new MongoClient(configuration.GetConnectionString("Mongo")); + + return db.GetDatabase( + configuration.GetValue("DatabaseName")); + }); + } + + public static void AddRabbitMq(this IServiceCollection services, IConfiguration configuration) + { + services.AddSingleton(x => + RabbitHutch.CreateBus(configuration.GetConnectionString("RabbitMQ"))); + } + + public static void AddDockerForwardedHeadersOptions(this IServiceCollection services) + { + services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("::ffff:10.0.0.0"), 104)); + options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("::ffff:192.168.0.0"), 112)); + options.KnownNetworks.Add(new IPNetwork(IPAddress.Parse("::ffff:172.16.0.0"), 108)); + }); + } + } +} \ No newline at end of file diff --git a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj index 472b337..312f84f 100644 --- a/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj +++ b/MalwareMultiScan.Api/MalwareMultiScan.Api.csproj @@ -14,18 +14,10 @@ - - - <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> - - - - - diff --git a/MalwareMultiScan.Api/Program.cs b/MalwareMultiScan.Api/Program.cs index 13d0b5b..acaf861 100644 --- a/MalwareMultiScan.Api/Program.cs +++ b/MalwareMultiScan.Api/Program.cs @@ -13,7 +13,7 @@ namespace MalwareMultiScan.Api private static IHostBuilder CreateHostBuilder(string[] args) { return Host.CreateDefaultBuilder(args) - .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup(); }); + .ConfigureWebHostDefaults(webBuilder => webBuilder.UseStartup()); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/Services/ReceiverHostedService.cs b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs index 6996d5d..9ead132 100644 --- a/MalwareMultiScan.Api/Services/ReceiverHostedService.cs +++ b/MalwareMultiScan.Api/Services/ReceiverHostedService.cs @@ -8,14 +8,24 @@ using Microsoft.Extensions.Logging; namespace MalwareMultiScan.Api.Services { + /// + /// Receiver hosted service. + /// public class ReceiverHostedService : IHostedService { private readonly IBus _bus; private readonly IConfiguration _configuration; - private readonly ScanResultService _scanResultService; private readonly ILogger _logger; + private readonly ScanResultService _scanResultService; - public ReceiverHostedService(IBus bus, IConfiguration configuration, ScanResultService scanResultService, + /// + /// Create receiver hosted service. + /// + /// Service bus. + /// Configuration. + /// Scan result service. + /// Logger. + public ReceiverHostedService(IBus bus, IConfiguration configuration, ScanResultService scanResultService, ILogger logger) { _bus = bus; @@ -24,6 +34,7 @@ namespace MalwareMultiScan.Api.Services _logger = logger; } + /// public Task StartAsync(CancellationToken cancellationToken) { _bus.Receive(_configuration.GetValue("ResultsSubscriptionId"), async message => @@ -31,21 +42,24 @@ namespace MalwareMultiScan.Api.Services _logger.LogInformation( $"Received a result from {message.Backend} for {message.Id} " + $"with threats {string.Join(",", message.Threats)}"); - + await _scanResultService.UpdateScanResultForBackend( - message.Id, message.Backend, true, true, message.Threats); + message.Id, message.Backend, true, + message.Succeeded, message.Threats ?? new string[] { }); }); - + _logger.LogInformation( "Started hosted service for receiving scan results"); return Task.CompletedTask; } + + /// public Task StopAsync(CancellationToken cancellationToken) { _bus.Dispose(); - + _logger.LogInformation( "Stopped hosted service for receiving scan results"); diff --git a/MalwareMultiScan.Api/Services/ScanBackendService.cs b/MalwareMultiScan.Api/Services/ScanBackendService.cs index d8926b9..71ca347 100644 --- a/MalwareMultiScan.Api/Services/ScanBackendService.cs +++ b/MalwareMultiScan.Api/Services/ScanBackendService.cs @@ -12,11 +12,21 @@ using YamlDotNet.Serialization.NamingConventions; namespace MalwareMultiScan.Api.Services { + /// + /// Scan backends service. + /// public class ScanBackendService { private readonly IBus _bus; private readonly ILogger _logger; + /// + /// Create scan backend service. + /// + /// Configuration. + /// Service bus. + /// Logger. + /// Missing BackendsConfiguration YAML file. public ScanBackendService(IConfiguration configuration, IBus bus, ILogger logger) { _bus = bus; @@ -36,13 +46,22 @@ namespace MalwareMultiScan.Api.Services List = deserializer.Deserialize(configurationContent); } + /// + /// List of available scan backends. + /// public ScanBackend[] List { get; } + /// + /// Queue URL scan. + /// + /// Scan result instance. + /// Scan backend. + /// File download URL. public async Task QueueUrlScan(ScanResult result, ScanBackend backend, string fileUrl) { _logger.LogInformation( $"Queueing scan for {result.Id} on {backend.Id} at {fileUrl}"); - + await _bus.SendAsync(backend.Id, new ScanRequestMessage { Id = result.Id, diff --git a/MalwareMultiScan.Api/Services/ScanResultService.cs b/MalwareMultiScan.Api/Services/ScanResultService.cs index 93b1926..231c88e 100644 --- a/MalwareMultiScan.Api/Services/ScanResultService.cs +++ b/MalwareMultiScan.Api/Services/ScanResultService.cs @@ -8,29 +8,41 @@ using MongoDB.Driver.GridFS; namespace MalwareMultiScan.Api.Services { + /// + /// Scan results service. + /// public class ScanResultService { private const string CollectionName = "ScanResults"; + private readonly GridFSBucket _bucket; - private readonly IMongoCollection _collection; - private readonly ScanBackendService _scanBackendService; + /// + /// Create scan result service. + /// + /// Mongo database. + /// Scan backend service. public ScanResultService(IMongoDatabase db, ScanBackendService scanBackendService) { _scanBackendService = scanBackendService; - _collection = db.GetCollection(CollectionName); _bucket = new GridFSBucket(db); + _collection = db.GetCollection(CollectionName); } + /// + /// Create scan result. + /// + /// Scan result. public async Task CreateScanResult() { var scanResult = new ScanResult { - Results = _scanBackendService.List.ToDictionary( - k => k.Id, v => new ScanResultEntry()) + Results = _scanBackendService.List + .Where(b => b.Enabled) + .ToDictionary(k => k.Id, v => new ScanResultEntry()) }; await _collection.InsertOneAsync(scanResult); @@ -38,6 +50,11 @@ namespace MalwareMultiScan.Api.Services return scanResult; } + /// + /// Get scan result. + /// + /// Scan result id. + /// Scan result. public async Task GetScanResult(string id) { var result = await _collection.FindAsync( @@ -46,27 +63,44 @@ namespace MalwareMultiScan.Api.Services return await result.FirstOrDefaultAsync(); } + /// + /// Update scan status for the backend. + /// + /// Result id. + /// Backend id. + /// If the scan has been completed. + /// If the scan has been succeeded. + /// List of found threats. public async Task UpdateScanResultForBackend(string resultId, string backendId, bool completed = false, bool succeeded = false, string[] threats = null) { - var filterScanResult = Builders.Filter.Where(r => r.Id == resultId); - - var updateScanResult = Builders.Update.Set(r => r.Results[backendId], new ScanResultEntry - { - Completed = completed, - Succeeded = succeeded, - Threats = threats - }); - - await _collection.UpdateOneAsync(filterScanResult, updateScanResult); + await _collection.UpdateOneAsync( + Builders.Filter.Where(r => r.Id == resultId), + Builders.Update.Set(r => r.Results[backendId], new ScanResultEntry + { + Completed = completed, + Succeeded = succeeded, + Threats = threats ?? new string[] { } + })); } + /// + /// Queue URL scan. + /// + /// Scan result instance. + /// File URL. public async Task QueueUrlScan(ScanResult result, string fileUrl) { - foreach (var backend in _scanBackendService.List) + foreach (var backend in _scanBackendService.List.Where(b => b.Enabled)) await _scanBackendService.QueueUrlScan(result, backend, fileUrl); } + /// + /// Store file. + /// + /// File name. + /// File stream. + /// Stored file id. public async Task StoreFile(string fileName, Stream fileStream) { var objectId = await _bucket.UploadFromStreamAsync( @@ -75,6 +109,11 @@ namespace MalwareMultiScan.Api.Services return objectId.ToString(); } + /// + /// Obtain stored file stream. + /// + /// File id. + /// File seekable stream. public async Task ObtainFile(string id) { if (!ObjectId.TryParse(id, out var objectId)) diff --git a/MalwareMultiScan.Api/Startup.cs b/MalwareMultiScan.Api/Startup.cs index 6fbc9ec..5039640 100644 --- a/MalwareMultiScan.Api/Startup.cs +++ b/MalwareMultiScan.Api/Startup.cs @@ -1,12 +1,8 @@ -using EasyNetQ; +using MalwareMultiScan.Api.Extensions; using MalwareMultiScan.Api.Services; using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.OpenApi.Models; -using MongoDB.Driver; namespace MalwareMultiScan.Api { @@ -21,62 +17,24 @@ namespace MalwareMultiScan.Api public void ConfigureServices(IServiceCollection services) { - services.AddSingleton(x => - RabbitHutch.CreateBus(_configuration.GetConnectionString("RabbitMQ"))); + services.AddDockerForwardedHeadersOptions(); + + services.AddMongoDb(_configuration); + services.AddRabbitMq(_configuration); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton( - serviceProvider => - { - var db = new MongoClient(_configuration.GetConnectionString("Mongo")); - - return db.GetDatabase( - _configuration.GetValue("DatabaseName")); - }); - services.AddControllers(); - services.AddHttpClient(); - - services.AddSwaggerGen(options => - { - options.SwaggerDoc("MalwareMultiScan", - new OpenApiInfo - { - Title = "MalwareMultiScan", - Version = "1.0.0" - }); - }); services.AddHostedService(); } - public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + public void Configure(IApplicationBuilder app) { app.UseRouting(); - - var forwardingOptions = new ForwardedHeadersOptions - { - ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto - }; - - forwardingOptions.KnownNetworks.Clear(); - forwardingOptions.KnownProxies.Clear(); - - app.UseForwardedHeaders(forwardingOptions); - + app.UseForwardedHeaders(); app.UseEndpoints(endpoints => endpoints.MapControllers()); - - app.UseSwagger(); - - app.UseSwaggerUI(options => - { - options.DocumentTitle = "MalwareMultiScan"; - - options.SwaggerEndpoint( - $"/swagger/{options.DocumentTitle}/swagger.json", options.DocumentTitle); - }); } } } \ No newline at end of file diff --git a/MalwareMultiScan.Api/appsettings.Development.json b/MalwareMultiScan.Api/appsettings.Development.json deleted file mode 100644 index 8983e0f..0000000 --- a/MalwareMultiScan.Api/appsettings.Development.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information" - } - } -} diff --git a/MalwareMultiScan.Api/appsettings.json b/MalwareMultiScan.Api/appsettings.json index 1f83738..c2e1ca1 100644 --- a/MalwareMultiScan.Api/appsettings.json +++ b/MalwareMultiScan.Api/appsettings.json @@ -14,8 +14,7 @@ }, "DatabaseName": "MalwareMultiScan", - "ResultsSubscriptionId": "mms.results", - + "MaxFileSize": 1048576, "BackendsConfiguration": "backends.yaml" } diff --git a/MalwareMultiScan.Api/backends.yaml b/MalwareMultiScan.Api/backends.yaml index 2c35e07..586e21a 100644 --- a/MalwareMultiScan.Api/backends.yaml +++ b/MalwareMultiScan.Api/backends.yaml @@ -1,2 +1,2 @@ - id: dummy - name: Dummy Backend \ No newline at end of file + enabled: true \ No newline at end of file diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs index b00dc5b..802750c 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs @@ -20,7 +20,7 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts protected abstract Regex MatchRegex { get; } protected abstract string BackendPath { get; } - protected virtual bool ParseStdErr { get; } + protected virtual bool ParseStdErr { get; } = false; protected virtual bool ThrowOnNonZeroExitCode { get; } = true; protected abstract string GetBackendArguments(string path); diff --git a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs index 03aa90a..6229b1f 100644 --- a/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs @@ -4,7 +4,6 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; using MalwareMultiScan.Backends.Interfaces; -using Microsoft.AspNetCore.Http; namespace MalwareMultiScan.Backends.Backends.Abstracts { @@ -23,13 +22,6 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts return await ScanAsync(uriStream, cancellationToken); } - public async Task ScanAsync(IFormFile file, CancellationToken cancellationToken) - { - await using var fileStream = file.OpenReadStream(); - - return await ScanAsync(fileStream, cancellationToken); - } - public async Task ScanAsync(Stream stream, CancellationToken cancellationToken) { var tempFile = Path.GetTempFileName(); diff --git a/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs index fb70597..b835515 100644 --- a/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs +++ b/MalwareMultiScan.Backends/Backends/Implementations/DummyScanBackend.cs @@ -3,7 +3,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; using MalwareMultiScan.Backends.Interfaces; -using Microsoft.AspNetCore.Http; namespace MalwareMultiScan.Backends.Backends.Implementations { @@ -21,11 +20,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations return Scan(); } - public Task ScanAsync(IFormFile file, CancellationToken cancellationToken) - { - return Scan(); - } - public Task ScanAsync(Stream stream, CancellationToken cancellationToken) { return Scan(); diff --git a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile index 310618e..dd34136 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Clamav.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest ENV DEBIAN_FRONTEND noninteractive diff --git a/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile index 7ebead2..8e10c16 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Comodo.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest RUN apt-get update && apt-get install wget -y diff --git a/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile index 168156f..c6073db 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/DrWeb.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest ARG DRWEB_KEY ENV DRWEB_KEY=$DRWEB_KEY diff --git a/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile index 5a7fbd6..80cff17 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/KES.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest ARG KES_KEY ENV KES_KEY=$KES_KEY diff --git a/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile index b8dd8d4..8dadd87 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/McAfee.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest RUN apt-get update && apt-get install unzip wget -y diff --git a/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile index ce57e8b..814309d 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/Sophos.Dockerfile @@ -1,4 +1,4 @@ -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest RUN apt-get update && apt-get install wget -y diff --git a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile index 6444031..b97083f 100644 --- a/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile +++ b/MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile @@ -11,7 +11,7 @@ WORKDIR /opt/loadlibrary/engine RUN curl -L "https://go.microsoft.com/fwlink/?LinkID=121721&arch=x86" --output mpan-fe.exe RUN cabextract mpan-fe.exe && rm mpan-fe.exe -FROM mindcollapse/malware-multi-scan-worker:latest +FROM mindcollapse/malware-multi-scan-scanner:latest RUN apt-get update && apt-get install -y libc6-i386 diff --git a/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs b/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs index b234ce5..ea872f1 100644 --- a/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs +++ b/MalwareMultiScan.Backends/Interfaces/IScanBackend.cs @@ -2,17 +2,14 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; namespace MalwareMultiScan.Backends.Interfaces { public interface IScanBackend { public string Id { get; } - public Task ScanAsync(string path, CancellationToken cancellationToken); public Task ScanAsync(Uri uri, CancellationToken cancellationToken); - public Task ScanAsync(IFormFile file, CancellationToken cancellationToken); public Task ScanAsync(Stream stream, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj b/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj index 0ce0c90..ae69f21 100644 --- a/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj +++ b/MalwareMultiScan.Backends/MalwareMultiScan.Backends.csproj @@ -3,13 +3,10 @@ netcoreapp3.1 - - - - - + + diff --git a/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs b/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs index f6627d7..21807b7 100644 --- a/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs +++ b/MalwareMultiScan.Backends/Messages/ScanResultMessage.cs @@ -4,6 +4,8 @@ namespace MalwareMultiScan.Backends.Messages { public string Id { get; set; } public string Backend { get; set; } + + public bool Succeeded { get; set; } public string[] Threats { get; set; } } } \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/Dockerfile b/MalwareMultiScan.Scanner/Dockerfile index 1791d67..bab5501 100644 --- a/MalwareMultiScan.Scanner/Dockerfile +++ b/MalwareMultiScan.Scanner/Dockerfile @@ -2,11 +2,10 @@ FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS builder WORKDIR /src -COPY MalwareMultiScan.Worker /src/MalwareMultiScan.Worker -COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared +COPY MalwareMultiScan.Worker /src/MalwareMultiScan.Scanner COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends -RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj +RUN dotnet publish -c Release -r linux-x64 -o ./publish MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 @@ -14,7 +13,4 @@ WORKDIR /worker COPY --from=builder /src/publish /worker -ENV ASPNETCORE_ENVIRONMENT=Production -ENV ASPNETCORE_URLS=http://+:9901 - -ENTRYPOINT ["/worker/MalwareMultiScan.Worker"] \ No newline at end of file +ENTRYPOINT ["/worker/MalwareMultiScan.Scanner"] \ No newline at end of file diff --git a/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj index 5930542..2d98265 100644 --- a/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj +++ b/MalwareMultiScan.Scanner/MalwareMultiScan.Scanner.csproj @@ -11,10 +11,5 @@ - - - - - <_ContentIncludedByDefault Remove="Properties\launchSettings.json" /> diff --git a/MalwareMultiScan.Scanner/Services/ScanHostedService.cs b/MalwareMultiScan.Scanner/Services/ScanHostedService.cs index 5c52d0d..f9dc666 100644 --- a/MalwareMultiScan.Scanner/Services/ScanHostedService.cs +++ b/MalwareMultiScan.Scanner/Services/ScanHostedService.cs @@ -56,28 +56,35 @@ namespace MalwareMultiScan.Scanner.Services var cancellationTokenSource = new CancellationTokenSource( TimeSpan.FromSeconds(_configuration.GetValue("MaxScanningTime"))); + var result = new ScanResultMessage + { + Id = message.Id, + Backend = _backend.Id + }; + try { - var result = new ScanResultMessage - { - Id = message.Id, - Backend = _backend.Id, + result.Threats = await _backend.ScanAsync( + message.Uri, cancellationTokenSource.Token); - Threats = await _backend.ScanAsync( - message.Uri, cancellationTokenSource.Token) - }; + result.Succeeded = true; _logger.LogInformation( $"Backend {_backend.Id} completed a scan of {message.Id} " + $"with result '{string.Join(", ", result.Threats)}'"); - - await _bus.SendAsync(_configuration.GetValue("ResultsSubscriptionId"), result); } catch (Exception exception) { + result.Succeeded = false; + _logger.LogError( exception, "Scanning failed with exception"); } + finally + { + await _bus.SendAsync( + _configuration.GetValue("ResultsSubscriptionId"), result); + } } } } \ No newline at end of file