basic structure and initial scan request service

MalwareMultiScan.Api/Controllers/WeatherForecastController.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Logging;
-
-namespace MalwareMultiScan.Api.Controllers
-{
-    [ApiController]
-    [Route("[controller]")]
-    public class WeatherForecastController : ControllerBase
-    {
-        private static readonly string[] Summaries = new[]
-        {
-            "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
-        };
-
-        private readonly ILogger<WeatherForecastController> _logger;
-
-        public WeatherForecastController(ILogger<WeatherForecastController> logger)
-        {
-            _logger = logger;
-        }
-
-        [HttpGet]
-        public IEnumerable<WeatherForecast> Get()
-        {
-            var rng = new Random();
-            return Enumerable.Range(1, 5).Select(index => new WeatherForecast
-                {
-                    Date = DateTime.Now.AddDays(index),
-                    TemperatureC = rng.Next(-20, 55),
-                    Summary = Summaries[rng.Next(Summaries.Length)]
-                })
-                .ToArray();
-        }
-    }
-}

MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs

@@ -0,0 +1,9 @@
+namespace MalwareMultiScan.Api.Data.Configuration
+{
+    public class BackendConfiguration
+    {
+        public string Id { get; set; }
+        public string Name { get; set; }
+        public string Endpoint { get; set; }
+    }
+}

MalwareMultiScan.Api/Data/Models/ScanRequest.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace MalwareMultiScan.Api.Data.Models
+{
+    public class ScanRequest
+    {
+        [BsonId]
+        [BsonRepresentation(BsonType.ObjectId)]
+        public string Id { get; set; } 
+        
+        [BsonRepresentation(BsonType.ObjectId)]
+        public string FileId { get; set; }
+        
+        public Dictionary<string, ScanRequestEntry> Results { get; set; } = 
+            new Dictionary<string, ScanRequestEntry>();
+    }
+}

MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs

@@ -0,0 +1,9 @@
+namespace MalwareMultiScan.Api.Data.Models
+{
+    public class ScanRequestEntry
+    {
+        public bool Completed { get; set; }
+        public bool Succeeded { get; set; }
+        public string[] Threats { get; set; }
+    }
+}

MalwareMultiScan.Api/MalwareMultiScan.Api.csproj

@@ -4,5 +4,25 @@
         <TargetFramework>netcoreapp3.1</TargetFramework>
     </PropertyGroup>
 
+    <ItemGroup>
+      <Folder Include="Controllers" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="backends.yaml">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="MongoDB.Driver" Version="2.11.3" />
+      <PackageReference Include="MongoDB.Driver.GridFS" Version="2.11.3" />
+      <PackageReference Include="YamlDotNet" Version="8.1.2" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <_ContentIncludedByDefault Remove="Properties\launchSettings.json" />
+    </ItemGroup>
+
 
 </Project>

MalwareMultiScan.Api/Program.cs

@@ -1,23 +1,19 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
 
 namespace MalwareMultiScan.Api
 {
-    public class Program
+    public static class Program
     {
         public static void Main(string[] args)
         {
             CreateHostBuilder(args).Build().Run();
         }
 
-        public static IHostBuilder CreateHostBuilder(string[] args) =>
+        private static IHostBuilder CreateHostBuilder(string[] args)
-            Host.CreateDefaultBuilder(args)
+        {
+            return Host.CreateDefaultBuilder(args)
                 .ConfigureWebHostDefaults(webBuilder => { webBuilder.UseStartup<Startup>(); });
+        }
     }
 }

MalwareMultiScan.Api/Properties/launchSettings.json

@@ -1,30 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:45236",
-      "sslPort": 44367
-    }
-  },
-  "profiles": {
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "weatherforecast",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    },
-    "MalwareMultiScan.Api": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "weatherforecast",
-      "applicationUrl": "https://localhost:5001;http://localhost:5000",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    }
-  }
-}

MalwareMultiScan.Api/Services/BackendConfigurationReader.cs

@@ -0,0 +1,29 @@
+using System.IO;
+using MalwareMultiScan.Api.Data.Configuration;
+using Microsoft.Extensions.Configuration;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace MalwareMultiScan.Api.Services
+{
+    public class BackendConfigurationReader
+    {
+        public BackendConfigurationReader(IConfiguration configuration)
+        {
+            var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
+
+            if (!File.Exists(configurationPath))
+                throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
+
+            var configurationContent = File.ReadAllText(configurationPath);
+
+            var deserializer = new DeserializerBuilder()
+                .WithNamingConvention(CamelCaseNamingConvention.Instance)
+                .Build();
+
+            Backends = deserializer.Deserialize<BackendConfiguration[]>(configurationContent);
+        }
+
+        public BackendConfiguration[] Backends { get; }
+    }
+}

MalwareMultiScan.Api/Services/ScanRequestService.cs

@@ -0,0 +1,90 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using MalwareMultiScan.Api.Data.Models;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using MongoDB.Driver.GridFS;
+
+namespace MalwareMultiScan.Api.Services
+{
+    public class ScanRequestService
+    {
+        private const string CollectionName = "ScanRequests";
+
+        private readonly GridFSBucket _bucket;
+        private readonly IMongoCollection<ScanRequest> _collection;
+        private readonly BackendConfigurationReader _configurationReader;
+        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly ILogger<ScanRequestService> _logger;
+
+        public ScanRequestService(IMongoDatabase db, BackendConfigurationReader configurationReader,
+            IHttpClientFactory httpClientFactory, ILogger<ScanRequestService> logger)
+        {
+            _configurationReader = configurationReader;
+            _httpClientFactory = httpClientFactory;
+            _logger = logger;
+
+            _bucket = new GridFSBucket(db);
+            _collection = db.GetCollection<ScanRequest>(CollectionName);
+        }
+
+        private async Task QueueScans(string requestId, string fileName, Stream fileStream)
+        {
+            foreach (var backend in _configurationReader.Backends)
+            {
+                var cancellationTokenSource = new CancellationTokenSource(
+                    TimeSpan.FromSeconds(5));
+
+                using var httpClient = _httpClientFactory.CreateClient();
+
+                try
+                {
+                    var endpointUri = new Uri(
+                        new Uri(backend.Endpoint), "/scan/file");
+
+                    var formContent = new MultipartFormDataContent
+                    {
+                        {new StringContent("value1"), "callbackUrl"},
+                        {new StreamContent(fileStream), "inputFile", fileName}
+                    };
+
+                    var response = await httpClient.PostAsync(
+                        endpointUri, formContent, cancellationTokenSource.Token);
+
+                    response.EnsureSuccessStatusCode();
+
+                    // TODO: update scan request
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e,
+                        $"Failed to queue scanning job on {backend.Id} ({backend.Endpoint})");
+                }
+            }
+        }
+
+        public async Task<string> InitializeNewRequest(string fileName, Stream fileStream,
+            CancellationToken cancellationToken = default)
+        {
+            var fileId = await _bucket.UploadFromStreamAsync(fileName, fileStream,
+                cancellationToken: cancellationToken);
+
+            var scanRequest = new ScanRequest
+            {
+                FileId = fileId.ToString()
+            };
+
+            await _collection.InsertOneAsync(scanRequest, new InsertOneOptions
+            {
+                BypassDocumentValidation = false
+            }, cancellationToken);
+
+            await QueueScans(scanRequest.Id, fileName, fileStream);
+
+            return scanRequest.Id;
+        }
+    }
+}

MalwareMultiScan.Api/Startup.cs

@@ -1,47 +1,43 @@
-using System;
+using MalwareMultiScan.Api.Services;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
+using MongoDB.Driver;
-using Microsoft.Extensions.Logging;
 
 namespace MalwareMultiScan.Api
 {
     public class Startup
     {
+        private readonly IConfiguration _configuration;
+
         public Startup(IConfiguration configuration)
         {
-            Configuration = configuration;
+            _configuration = configuration;
         }
-
+        
-        public IConfiguration Configuration { get; }
-
-        // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
+            services.AddSingleton<BackendConfigurationReader>();
+            services.AddSingleton<ScanRequestService>();
+            
+            services.AddSingleton(
+                serviceProvider =>
+                {
+                    var db = new MongoClient(_configuration.GetConnectionString("Mongo"));
+
+                    return db.GetDatabase(
+                        _configuration.GetValue<string>("DatabaseName"));
+                });
+            
             services.AddControllers();
+            services.AddHttpClient();
         }
 
-        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
         public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         {
-            if (env.IsDevelopment())
-            {
-                app.UseDeveloperExceptionPage();
-            }
-
-            app.UseHttpsRedirection();
-
             app.UseRouting();
 
-            app.UseAuthorization();
-
             app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
         }
     }

MalwareMultiScan.Api/WeatherForecast.cs

@@ -1,15 +0,0 @@
-using System;
-
-namespace MalwareMultiScan.Api
-{
-    public class WeatherForecast
-    {
-        public DateTime Date { get; set; }
-
-        public int TemperatureC { get; set; }
-
-        public int TemperatureF => 32 + (int) (TemperatureC / 0.5556);
-
-        public string Summary { get; set; }
-    }
-}

MalwareMultiScan.Api/appsettings.json

@@ -6,5 +6,13 @@
       "Microsoft.Hosting.Lifetime": "Information"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  
+  "ConnectionStrings": {
+    "Mongo": "mongodb://localhost:27017"
+  },
+
+  "DatabaseName": "MalwareMultiScan",
+  
+  "BackendsConfiguration": "backends.yaml"
 }

MalwareMultiScan.Api/backends.yaml

@@ -0,0 +1,7 @@
+- id: windows-defender
+  name: Windows Defender
+  endpoint: http://localhost:9902
+  
+- id: clamav
+  name: ClamAV
+  endpoint: http://localhost:9901

MalwareMultiScan.Backends/Backends/Abstracts/AbstractLocalProcessScanBackend.cs

@@ -56,13 +56,13 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
             process.WaitForExit();
 
             _logger.LogInformation($"Process has exited with code {process.ExitCode}");
-            
+
             var standardOutput = await process.StandardOutput.ReadToEndAsync();
             var standardError = await process.StandardError.ReadToEndAsync();
 
             _logger.LogDebug($"Process standard output: {standardOutput}");
             _logger.LogDebug($"Process standard error: {standardError}");
-            
+
             if (ThrowOnNonZeroExitCode && process.ExitCode != 0)
                 throw new ApplicationException($"Process has terminated with an exit code {process.ExitCode}");
 

MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs

@@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public ClamavScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "clamav";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/var/lib/clamav/daily.cvd");
 
         protected override string BackendPath { get; } = "/usr/bin/clamscan";
-        
+
         protected override Regex MatchRegex { get; } =
             new Regex(@"(\S+): (?<threat>[\S]+) FOUND", RegexOptions.Compiled | RegexOptions.Multiline);
 

MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs

@@ -11,16 +11,16 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public ComodoScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "comodo";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/opt/COMODO/scanners/bases.cav");
 
         protected override string BackendPath { get; } = "/opt/COMODO/cmdscan";
-        
+
         protected override Regex MatchRegex { get; } =
-            new Regex(@".* ---> Found Virus, Malware Name is (?<threat>.*)", 
+            new Regex(@".* ---> Found Virus, Malware Name is (?<threat>.*)",
                 RegexOptions.Compiled | RegexOptions.Multiline);
 
         protected override string GetBackendArguments(string path)

MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs

@@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public DrWebScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "drweb";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/var/opt/drweb.com/version/version.ini");
 
         protected override string BackendPath { get; } = "/usr/bin/drweb-ctl";
-        
+
         protected override Regex MatchRegex { get; } =
             new Regex(@".* - infected with (?<threat>[\S ]+)", RegexOptions.Compiled | RegexOptions.Multiline);
 

MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs

@@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public KesScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "kes";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/var/opt/kaspersky/kesl/common/updates/avbases/klsrl.dat");
 
         protected override string BackendPath { get; } = "/bin/bash";
-        
+
         protected override Regex MatchRegex { get; } =
             new Regex(@"[ +]DetectName.*: (?<threat>.*)", RegexOptions.Compiled | RegexOptions.Multiline);
 

MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs

@@ -11,14 +11,14 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public McAfeeScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "mcafeee";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/usr/local/uvscan/avvscan.dat");
 
         protected override string BackendPath { get; } = "/usr/local/uvscan/uvscan";
-        
+
         protected override bool ThrowOnNonZeroExitCode { get; } = false;
 
         protected override Regex MatchRegex { get; } =

MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs

@@ -11,9 +11,9 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
         public SophosScanBackend(ILogger logger) : base(logger)
         {
         }
-        
+
         public override string Id { get; } = "sophos";
-        
+
         public override DateTime DatabaseLastUpdate =>
             File.GetLastWriteTime("/opt/sophos-av/lib/sav/vdlsync.upd");
 

MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj

@@ -13,5 +13,9 @@
       <PackageReference Include="Hangfire.AspNetCore" Version="1.7.16" />
       <PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
     </ItemGroup>
+
+    <ItemGroup>
+      <_ContentIncludedByDefault Remove="Properties\launchSettings.json" />
+    </ItemGroup>
     
 </Project>

MalwareMultiScan.Worker/Properties/launchSettings.json

@@ -1,30 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "iisSettings": {
-    "windowsAuthentication": false,
-    "anonymousAuthentication": true,
-    "iisExpress": {
-      "applicationUrl": "http://localhost:53549",
-      "sslPort": 44380
-    }
-  },
-  "profiles": {
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "weatherforecast",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    },
-    "MalwareMultiScan.Worker": {
-      "commandName": "Project",
-      "launchBrowser": true,
-      "launchUrl": "weatherforecast",
-      "applicationUrl": "https://localhost:5001;http://localhost:5000",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    }
-  }
-}

MalwareMultiScan.Worker/Startup.cs

@@ -27,7 +27,8 @@ namespace MalwareMultiScan.Worker
         {
             app.UseRouting();
 
-            app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
+            app.UseEndpoints(
+                endpoints => endpoints.MapControllers());
 
             app.UseHangfireServer();
         }