Mirrors/MalwareMultiScan
1
0
Fork 0
mirror of https://github.com/volodymyrsmirnov/MalwareMultiScan.git synced 2025-08-23 21:12:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

basic skeleton for the API and queueing

Browse Source
This commit is contained in:
Volodymyr Smirnov 2020-10-25 16:11:36 +02:00
parent 96695664e0
commit b6c3cb4131
28 changed files with 374 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
MalwareMultiScan.Api/Controllers/DownloadController.cs Normal file
View File

@ -0,0 +1,31 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace MalwareMultiScan.Api.Controllers
{
[Route("download")]
public class DownloadController : Controller
{
private readonly ScanResultsService _scanResultsService;
public DownloadController(ScanResultsService scanResultsService)
{
_scanResultsService = scanResultsService;
}
[HttpGet("{id}")]
[ProducesResponseType(StatusCodes.Status200OK)]
[ProducesResponseType(StatusCodes.Status404NotFound)]
public async Task<IActionResult> Index(string id)
{
var fileStream = await _scanResultsService.ObtainFile(id);
if (fileStream == null)
return NotFound();
return File(fileStream, "application/octet-stream");
}
}
}

15
MalwareMultiScan.Api/Controllers/QueueController.cs Normal file
View File

@ -0,0 +1,15 @@
using Microsoft.AspNetCore.Mvc;
namespace MalwareMultiScan.Api.Controllers
{
[ApiController]
[Route("queue")]
[Produces("application/json")]
public class QueueController : Controller
{
public IActionResult Index()
{
return Ok();
}
}
}

41
MalwareMultiScan.Api/Controllers/ScanBackendsController.cs Normal file
View File

@ -0,0 +1,41 @@
using System.Linq;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Response;
using MalwareMultiScan.Api.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace MalwareMultiScan.Api.Controllers
{
[ApiController]
[Route("backends")]
[Produces("application/json")]
public class ScanBackendsController : Controller
{
private readonly ScanBackendService _scanBackendService;
public ScanBackendsController(ScanBackendService scanBackendService)
{
_scanBackendService = scanBackendService;
}
private async Task<ScanBackendResponse> GetScanBackendResponse(ScanBackend backend)
{
return new ScanBackendResponse
{
Id = backend.Id,
Name = backend.Name,
Online = await _scanBackendService.Ping(backend)
};
}
[HttpGet]
[ProducesResponseType(typeof(ScanBackendResponse[]), StatusCodes.Status200OK)]
public async Task<IActionResult> Index()
{
return Ok(await Task.WhenAll(
_scanBackendService.List.Select(GetScanBackendResponse).ToArray()));
}
}
}

31
MalwareMultiScan.Api/Controllers/ScanResultsController.cs Normal file
View File

@ -0,0 +1,31 @@
using System.Threading.Tasks;
using MalwareMultiScan.Api.Services;
using MalwareMultiScan.Shared.Data.Responses;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace MalwareMultiScan.Api.Controllers
{
[ApiController]
[Route("results")]
[Produces("application/json")]
public class ScanResultsController : Controller
{
private readonly ScanResultsService _scanResultsService;
public ScanResultsController(ScanResultsService scanResultsService)
{
_scanResultsService = scanResultsService;
}
[HttpPost("{id}")]
[ProducesResponseType(StatusCodes.Status204NoContent)]
public async Task<IActionResult> Index(string id, [FromBody] ResultResponse result)
{
await _scanResultsService.UpdateScanResultForBackend(
id, result.Backend, true, result.Success, result.Threats);
return NoContent();
}
}
}

2
MalwareMultiScan.Api/Data/Configuration/BackendConfiguration.cs → MalwareMultiScan.Api/Data/Configuration/ScanBackend.cs
View File

@ -1,6 +1,6 @@
namespace MalwareMultiScan.Api.Data.Configuration
{
public class BackendConfiguration
public class ScanBackend
{
public string Id { get; set; }
public string Name { get; set; }

19
MalwareMultiScan.Api/Data/Models/ScanRequest.cs
View File

@ -1,19 +0,0 @@
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace MalwareMultiScan.Api.Data.Models
{
public class ScanRequest
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string Id { get; set; }
[BsonRepresentation(BsonType.ObjectId)]
public string FileId { get; set; }
public Dictionary<string, ScanRequestEntry> Results { get; set; } =
new Dictionary<string, ScanRequestEntry>();
}
}

16
MalwareMultiScan.Api/Data/Models/ScanResult.cs Normal file
View File

@ -0,0 +1,16 @@
using System.Collections.Generic;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace MalwareMultiScan.Api.Data.Models
{
public class ScanResult
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string Id { get; set; }
public Dictionary<string, ScanResultEntry> Results { get; set; } =
new Dictionary<string, ScanResultEntry>();
}
}

2
MalwareMultiScan.Api/Data/Models/ScanRequestEntry.cs → MalwareMultiScan.Api/Data/Models/ScanResultEntry.cs
View File

@ -1,6 +1,6 @@
namespace MalwareMultiScan.Api.Data.Models
{
public class ScanRequestEntry
public class ScanResultEntry
{
public bool Completed { get; set; }
public bool Succeeded { get; set; }

9
MalwareMultiScan.Api/Data/Response/ScanBackendResponse.cs Normal file
View File

@ -0,0 +1,9 @@
namespace MalwareMultiScan.Api.Data.Response
{
public class ScanBackendResponse
{
public string Id { get; set; }
public string Name { get; set; }
public bool Online { get; set; }
}
}

8
MalwareMultiScan.Api/MalwareMultiScan.Api.csproj
View File

@ -4,10 +4,6 @@
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<Folder Include="Controllers" />
</ItemGroup>
<ItemGroup>
<None Update="backends.yaml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
@ -24,5 +20,9 @@
<_ContentIncludedByDefault Remove="Properties\launchSettings.json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
</ItemGroup>
</Project>

29
MalwareMultiScan.Api/Services/BackendConfigurationReader.cs
View File

@ -1,29 +0,0 @@
using System.IO;
using MalwareMultiScan.Api.Data.Configuration;
using Microsoft.Extensions.Configuration;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace MalwareMultiScan.Api.Services
{
public class BackendConfigurationReader
{
public BackendConfigurationReader(IConfiguration configuration)
{
var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
if (!File.Exists(configurationPath))
throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
var configurationContent = File.ReadAllText(configurationPath);
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
Backends = deserializer.Deserialize<BackendConfiguration[]>(configurationContent);
}
public BackendConfiguration[] Backends { get; }
}
}

111
MalwareMultiScan.Api/Services/ScanBackendService.cs Normal file
View File

@ -0,0 +1,111 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Shared.Data.Requests;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace MalwareMultiScan.Api.Services
{
public class ScanBackendService
{
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ScanBackendService> _logger;
public ScanBackendService(IConfiguration configuration, IHttpClientFactory httpClientFactory,
ILogger<ScanBackendService> logger)
{
_httpClientFactory = httpClientFactory;
_logger = logger;
var configurationPath = configuration.GetValue<string>("BackendsConfiguration");
if (!File.Exists(configurationPath))
throw new FileNotFoundException("Missing BackendsConfiguration YAML file", configurationPath);
var configurationContent = File.ReadAllText(configurationPath);
var deserializer = new DeserializerBuilder()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
List = deserializer.Deserialize<ScanBackend[]>(configurationContent);
}
public ScanBackend[] List { get; }
public async Task<bool> Ping(ScanBackend backend)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(1));
try
{
using var httpClient = _httpClientFactory.CreateClient();
var pingResponse = await httpClient.GetAsync(
new Uri(new Uri(backend.Endpoint), "/ping"), cancellationTokenSource.Token);
pingResponse.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(
exception, $"Failed to ping {backend.Id}");
return false;
}
return true;
}
private async Task<bool> QueueScan(ScanBackend backend, string uri, HttpContent content)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(5));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var response = await httpClient.PostAsync(
new Uri(new Uri(backend.Endpoint), uri), content, cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
}
catch (Exception exception)
{
_logger.LogError(
exception, $"Failed to initiate scan against {backend.Id}");
return false;
}
return true;
}
public async Task<bool> QueueFileScan(
ScanBackend backend, string fileName, Stream fileStream, string callbackUrl)
{
return await QueueScan(backend, "/scan/file", new MultipartFormDataContent
{
{new StringContent(callbackUrl), nameof(FileRequest.CallbackUrl)},
{new StreamContent(fileStream), nameof(FileRequest.InputFile), fileName}
});
}
public async Task<bool> QueueUrlScan(
ScanBackend backend, string url, string callbackUrl)
{
return await QueueScan(backend, "/scan/url", new MultipartFormDataContent
{
{new StringContent(callbackUrl), nameof(UrlRequest.CallbackUrl)},
{new StringContent(url), nameof(UrlRequest.InputUrl)}
});
}
}
}

90
MalwareMultiScan.Api/Services/ScanRequestService.cs
View File

@ -1,90 +0,0 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Models;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace MalwareMultiScan.Api.Services
{
public class ScanRequestService
{
private const string CollectionName = "ScanRequests";
private readonly GridFSBucket _bucket;
private readonly IMongoCollection<ScanRequest> _collection;
private readonly BackendConfigurationReader _configurationReader;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<ScanRequestService> _logger;
public ScanRequestService(IMongoDatabase db, BackendConfigurationReader configurationReader,
IHttpClientFactory httpClientFactory, ILogger<ScanRequestService> logger)
{
_configurationReader = configurationReader;
_httpClientFactory = httpClientFactory;
_logger = logger;
_bucket = new GridFSBucket(db);
_collection = db.GetCollection<ScanRequest>(CollectionName);
}
private async Task QueueScans(string requestId, string fileName, Stream fileStream)
{
foreach (var backend in _configurationReader.Backends)
{
var cancellationTokenSource = new CancellationTokenSource(
TimeSpan.FromSeconds(5));
using var httpClient = _httpClientFactory.CreateClient();
try
{
var endpointUri = new Uri(
new Uri(backend.Endpoint), "/scan/file");
var formContent = new MultipartFormDataContent
{
{new StringContent("value1"), "callbackUrl"},
{new StreamContent(fileStream), "inputFile", fileName}
};
var response = await httpClient.PostAsync(
endpointUri, formContent, cancellationTokenSource.Token);
response.EnsureSuccessStatusCode();
// TODO: update scan request
}
catch (Exception e)
{
_logger.LogError(e,
$"Failed to queue scanning job on {backend.Id} ({backend.Endpoint})");
}
}
}
public async Task<string> InitializeNewRequest(string fileName, Stream fileStream,
CancellationToken cancellationToken = default)
{
var fileId = await _bucket.UploadFromStreamAsync(fileName, fileStream,
cancellationToken: cancellationToken);
var scanRequest = new ScanRequest
{
FileId = fileId.ToString()
};
await _collection.InsertOneAsync(scanRequest, new InsertOneOptions
{
BypassDocumentValidation = false
}, cancellationToken);
await QueueScans(scanRequest.Id, fileName, fileStream);
return scanRequest.Id;
}
}
}

97
MalwareMultiScan.Api/Services/ScanResultsService.cs Normal file
View File

@ -0,0 +1,97 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Api.Data.Configuration;
using MalwareMultiScan.Api.Data.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace MalwareMultiScan.Api.Services
{
public class ScanResultsService
{
private const string CollectionName = "ScanResults";
private readonly IMongoCollection<ScanResult> _collection;
private readonly GridFSBucket _bucket;
private readonly ScanBackendService _scanBackendService;
public ScanResultsService(IMongoDatabase db, ScanBackendService scanBackendService)
{
_scanBackendService = scanBackendService;
_collection = db.GetCollection<ScanResult>(CollectionName);
_bucket = new GridFSBucket(db);
}
public async Task<ScanResult> CreateScanResult()
{
var scanResult = new ScanResult();
await _collection.InsertOneAsync(scanResult);
return scanResult;
}
public async Task UpdateScanResultForBackend(
string resultId, string backendId, bool completed, bool succeeded, string[] threats = null)
{
var filterScanResult = Builders<ScanResult>.Filter.Where(r => r.Id == resultId);
var updateScanResult = Builders<ScanResult>.Update.Set(r => r.Results[backendId], new ScanResultEntry
{
Completed = completed,
Succeeded = succeeded,
Threats = threats
});
await _collection.UpdateOneAsync(filterScanResult, updateScanResult);
}
public async Task QueueFileScan(ScanResult result, string fileName, Stream fileStream, string callbackUrl)
{
foreach (var backend in _scanBackendService.List)
{
var queueResult = await _scanBackendService.QueueFileScan(
backend, fileName, fileStream, callbackUrl);
await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult);
}
}
public async Task QueueUrlScan(ScanResult result, string url, string callbackUrl)
{
foreach (var backend in _scanBackendService.List)
{
var queueResult = await _scanBackendService.QueueUrlScan(
backend, url, callbackUrl);
await UpdateScanResultForBackend(result.Id, backend.Id, !queueResult, queueResult);
}
}
public async Task<string> StoreFile(string fileName, Stream fileStream)
{
var objectId = await _bucket.UploadFromStreamAsync(fileName, fileStream);
return objectId.ToString();
}
public async Task<Stream> ObtainFile(string id)
{
if (!ObjectId.TryParse(id, out var objectId))
return null;
return await _bucket.OpenDownloadStreamAsync(objectId, new GridFSDownloadOptions
{
Seekable = true
});
}
}
}

4
MalwareMultiScan.Api/Startup.cs
View File

@ -18,8 +18,8 @@ namespace MalwareMultiScan.Api
public void ConfigureServices(IServiceCollection services)
{
services.AddSingleton<BackendConfigurationReader>();
services.AddSingleton<ScanRequestService>();
services.AddSingleton<ScanBackendService>();
services.AddSingleton<ScanResultsService>();
services.AddSingleton(
serviceProvider =>

3
MalwareMultiScan.Api/appsettings.json
View File

@ -14,5 +14,6 @@
"DatabaseName": "MalwareMultiScan",
"BackendsConfiguration": "backends.yaml"
"BackendsConfiguration": "backends.yaml",
"StoreFileUploads": true
}

2
MalwareMultiScan.Backends/Backends/Abstracts/AbstractScanBackend.cs
View File

@ -12,8 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Abstracts
{
public virtual string Id => throw new NotImplementedException();
public virtual DateTime DatabaseLastUpdate => throw new NotImplementedException();
public virtual Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
{
throw new NotImplementedException();

5
MalwareMultiScan.Backends/Backends/Implementations/ClamavScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "clamav";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/var/lib/clamav/daily.cvd");
protected override string BackendPath { get; } = "/usr/bin/clamscan";
protected override Regex MatchRegex { get; } =

3
MalwareMultiScan.Backends/Backends/Implementations/ComodoScanBackend.cs
View File

@ -14,9 +14,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "comodo";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/opt/COMODO/scanners/bases.cav");
protected override string BackendPath { get; } = "/opt/COMODO/cmdscan";
protected override Regex MatchRegex { get; } =

5
MalwareMultiScan.Backends/Backends/Implementations/DrWebScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "drweb";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/var/opt/drweb.com/version/version.ini");
protected override string BackendPath { get; } = "/usr/bin/drweb-ctl";
protected override Regex MatchRegex { get; } =

5
MalwareMultiScan.Backends/Backends/Implementations/KesScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "kes";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/var/opt/kaspersky/kesl/common/updates/avbases/klsrl.dat");
protected override string BackendPath { get; } = "/bin/bash";
protected override Regex MatchRegex { get; } =

5
MalwareMultiScan.Backends/Backends/Implementations/McAffeeScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "mcafeee";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/usr/local/uvscan/avvscan.dat");
protected override string BackendPath { get; } = "/usr/local/uvscan/uvscan";
protected override bool ThrowOnNonZeroExitCode { get; } = false;

5
MalwareMultiScan.Backends/Backends/Implementations/SophosScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "sophos";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/opt/sophos-av/lib/sav/vdlsync.upd");
protected override string BackendPath { get; } = "/opt/sophos-av/bin/savscan";
protected override bool ThrowOnNonZeroExitCode { get; } = false;

5
MalwareMultiScan.Backends/Backends/Implementations/WindowsDefenderScanBackend.cs
View File

@ -1,5 +1,3 @@
using System;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
@ -14,9 +12,6 @@ namespace MalwareMultiScan.Backends.Backends.Implementations
public override string Id { get; } = "windows-defender";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/opt/engine/mpavbase.vdm");
protected override string BackendPath { get; } = "/opt/mpclient";
protected override Regex MatchRegex { get; } =

9
MalwareMultiScan.Shared/Data/Responses/ResultResponse.cs
View File

@ -1,18 +1,17 @@
using System;
using System.ComponentModel.DataAnnotations;
using System.Linq;
namespace MalwareMultiScan.Shared.Data.Responses
{
public class ResultResponse
{
[Required]
public string Backend { get; set; }
[Required]
public bool Success { get; set; }
public DateTime? DatabaseLastUpdate { get; set; }
public bool Detected => Threats?.Any() == true;
public string[] Threats { get; set; }
}
}

2
MalwareMultiScan.Shared/Interfaces/IScanBackend.cs
View File

@ -10,8 +10,6 @@ namespace MalwareMultiScan.Shared.Interfaces
{
public string Id { get; }
public DateTime DatabaseLastUpdate { get; }
public Task<string[]> ScanAsync(string path, CancellationToken cancellationToken);
public Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken);
public Task<string[]> ScanAsync(IFormFile file, CancellationToken cancellationToken);

9
MalwareMultiScan.Worker/Controllers/ScanController.cs
View File

@ -12,6 +12,15 @@ namespace MalwareMultiScan.Worker.Controllers
[Produces("application/json")]
public class ScanController : ControllerBase
{
[HttpGet]
[ProducesResponseType(StatusCodes.Status200OK)]
[Route("/ping")]
public IActionResult Ping()
{
return Ok("pong");
}
[HttpPost]
[ProducesResponseType(StatusCodes.Status202Accepted)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]

1
MalwareMultiScan.Worker/Jobs/ScanJob.cs
View File

@ -78,7 +78,6 @@ namespace MalwareMultiScan.Worker.Jobs
{
response.Success = true;
response.Threats = await scanMethod(cancellationToken);
response.DatabaseLastUpdate = _backend.DatabaseLastUpdate;
}
catch (Exception exception)
{