initial commit of a worker skeleton

This commit is contained in:
Volodymyr Smirnov 2020-10-20 16:20:38 +03:00
commit 7e63c77419
28 changed files with 665 additions and 0 deletions

4
.dockerignore Normal file
View File

@ -0,0 +1,4 @@
.git
Dockerfile
bin
obj

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
bin/
obj/
/packages/
riderModule.iml
/_ReSharper.Caches/

View File

@ -0,0 +1,24 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Dockerfiles/WindowsDefender.Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker-windows-defender" />
<option name="buildCliOptions" value="" />
<option name="command" value="" />
<option name="containerName" value="malware-multi-scan-worker-windows-defender" />
<option name="entrypoint" value="" />
<option name="portBindings">
<list>
<DockerPortBindingImpl>
<option name="containerPort" value="9901" />
<option name="hostPort" value="9901" />
</DockerPortBindingImpl>
</list>
</option>
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Backends/Dockerfiles/WindowsDefender.Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

View File

@ -0,0 +1,18 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="MalwareMultiScan.Worker/Dockerfile" type="docker-deploy" factoryName="dockerfile" server-name="Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="mindcollapse/malware-multi-scan-worker" />
<option name="buildCliOptions" value="" />
<option name="buildOnly" value="true" />
<option name="command" value="" />
<option name="containerName" value="" />
<option name="contextFolderPath" value="." />
<option name="entrypoint" value="" />
<option name="commandLineOptions" value="" />
<option name="sourceFilePath" value="MalwareMultiScan.Worker/Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

View File

@ -0,0 +1,72 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Backends.Backends.Abstracts
{
public abstract class AbstractLocalProcessScanBackend : AbstractScanBackend
{
private readonly ILogger _logger;
protected virtual Regex MatchRegex { get; }
protected virtual string BackendPath { get; }
protected virtual bool ParseStdErr { get; }
protected virtual string GetBackendArguments(string path)
{
throw new NotImplementedException();
}
protected AbstractLocalProcessScanBackend(ILogger logger)
{
_logger = logger;
}
public override async Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
{
var process = new Process
{
StartInfo = new ProcessStartInfo(BackendPath, GetBackendArguments(path))
{
RedirectStandardError = true,
RedirectStandardOutput = true,
WorkingDirectory = Path.GetDirectoryName(BackendPath) ?? Directory.GetCurrentDirectory()
}
};
_logger.LogInformation(
$"Starting process {process.StartInfo.FileName} " +
$"with arguments {process.StartInfo.Arguments} " +
$"in working directory {process.StartInfo.WorkingDirectory}");
process.Start();
cancellationToken.Register(() =>
{
if (!process.HasExited)
process.Kill(true);
});
process.WaitForExit();
_logger.LogInformation($"Process has exited with code {process.ExitCode}");
var standardOutput = await process.StandardOutput.ReadToEndAsync();
var standardError = await process.StandardError.ReadToEndAsync();
_logger.LogDebug($"Process standard output: {standardOutput}");
_logger.LogDebug($"Process standard error: {standardError}");
return MatchRegex
.Matches(ParseStdErr ? standardError : standardOutput)
.Where(x => x.Success)
.Select(x => x.Groups["threat"].Value)
.ToArray();
}
}
}

View File

@ -0,0 +1,54 @@
using System;
using System.IO;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Shared.Interfaces;
using Microsoft.AspNetCore.Http;
namespace MalwareMultiScan.Backends.Backends.Abstracts
{
public abstract class AbstractScanBackend : IScanBackend
{
public virtual string Name => throw new NotImplementedException();
public virtual DateTime DatabaseLastUpdate => throw new NotImplementedException();
public virtual Task<string[]> ScanAsync(string path, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public async Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken)
{
using var httpClient = new HttpClient();
await using var uriStream = await httpClient.GetStreamAsync(uri);
return await ScanAsync(uriStream, cancellationToken);
}
public async Task<string[]> ScanAsync(IFormFile file, CancellationToken cancellationToken)
{
await using var fileStream = file.OpenReadStream();
return await ScanAsync(fileStream, cancellationToken);
}
public async Task<string[]> ScanAsync(Stream stream, CancellationToken cancellationToken)
{
var tempFile = Path.GetTempFileName();
try
{
await using (var tempFileStream = File.OpenWrite(tempFile))
await stream.CopyToAsync(tempFileStream, cancellationToken);
return await ScanAsync(tempFile, cancellationToken);
}
finally
{
File.Delete(tempFile);
}
}
}
}

View File

@ -0,0 +1,30 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Text.RegularExpressions;
using MalwareMultiScan.Backends.Backends.Abstracts;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Backends.Backends.Implementations
{
public class WindowsDefenderScanBackend : AbstractLocalProcessScanBackend
{
public override string Name { get; } = "Windows Defender";
public override DateTime DatabaseLastUpdate =>
File.GetLastWriteTime("/opt/engine/mpavbase.vdm");
protected override string BackendPath { get; } = "/opt/mpclient";
protected override Regex MatchRegex { get; } =
new Regex(@"EngineScanCallback\(\)\: Threat (?<threat>[\S]+) identified",
RegexOptions.Compiled | RegexOptions.Multiline);
protected override bool ParseStdErr { get; } = true;
protected override string GetBackendArguments(string path) => path;
public WindowsDefenderScanBackend(ILogger logger) : base(logger)
{
}
}
}

View File

@ -0,0 +1,8 @@
FROM mindcollapse/malware-multi-scan-base:latest
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y clamav
RUN freshclam --quiet
ENV MULTI_SCAN_BACKEND_BIN=/usr/bin/clamscan

View File

@ -0,0 +1,22 @@
FROM ubuntu:bionic AS backend
RUN apt-get update
RUN apt-get install -y build-essential ca-certificates cabextract libc6-dev-i386 git-core curl
WORKDIR /opt/loadlibrary
RUN git clone https://github.com/taviso/loadlibrary.git /opt/loadlibrary
RUN make
WORKDIR /opt/loadlibrary/engine
RUN curl -L "https://go.microsoft.com/fwlink/?LinkID=121721&arch=x86" --output mpan-fe.exe
RUN cabextract mpan-fe.exe && rm mpan-fe.exe
FROM mindcollapse/malware-multi-scan-worker:latest
RUN apt-get update && apt-get install -y libc6-i386
COPY --from=backend /opt/loadlibrary/engine /opt/engine
COPY --from=backend /opt/loadlibrary/mpclient /opt/mpclient
ENV BackendType=Defender
ENV ScanTimeout=300

View File

@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,18 @@
using System;
using System.ComponentModel.DataAnnotations;
namespace MalwareMultiScan.Shared.Attributes
{
public class UrlValidationAttribute : ValidationAttribute
{
protected override ValidationResult IsValid(object value, ValidationContext validationContext)
{
var uri = (Uri) value;
if (uri == null || uri.Scheme != "http" && uri.Scheme != "https")
return new ValidationResult("Only http(s) URLs are supported");
return ValidationResult.Success;
}
}
}

View File

@ -0,0 +1,7 @@
namespace MalwareMultiScan.Shared.Data.Enums
{
public enum BackendType
{
Defender
}
}

View File

@ -0,0 +1,13 @@
using System;
using System.ComponentModel.DataAnnotations;
using MalwareMultiScan.Shared.Attributes;
namespace MalwareMultiScan.Shared.Data.Requests
{
public abstract class BasicRequest
{
[Required]
[UrlValidation]
public Uri CallbackUrl { get; set; }
}
}

View File

@ -0,0 +1,11 @@
using System.ComponentModel.DataAnnotations;
using Microsoft.AspNetCore.Http;
namespace MalwareMultiScan.Shared.Data.Requests
{
public class FileRequest : BasicRequest
{
[Required]
public IFormFile InputFile { get; set; }
}
}

View File

@ -0,0 +1,13 @@
using System;
using System.ComponentModel.DataAnnotations;
using MalwareMultiScan.Shared.Attributes;
namespace MalwareMultiScan.Shared.Data.Requests
{
public class UrlRequest : BasicRequest
{
[Required]
[UrlValidation]
public Uri InputUrl { get; set; }
}
}

View File

@ -0,0 +1,18 @@
using System;
using System.Linq;
namespace MalwareMultiScan.Shared.Data.Responses
{
public class ResultResponse
{
public string Backend { get; set; }
public bool Success { get; set; }
public DateTime? DatabaseLastUpdate { get; set; }
public bool Detected => Threats?.Any() == true;
public string[] Threats { get; set; }
}
}

View File

@ -0,0 +1,20 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace MalwareMultiScan.Shared.Interfaces
{
public interface IScanBackend
{
public string Name { get; }
public DateTime DatabaseLastUpdate { get; }
public Task<string[]> ScanAsync(string path, CancellationToken cancellationToken);
public Task<string[]> ScanAsync(Uri uri, CancellationToken cancellationToken);
public Task<string[]> ScanAsync(IFormFile file, CancellationToken cancellationToken);
public Task<string[]> ScanAsync(Stream stream, CancellationToken cancellationToken);
}
}

View File

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Http.Features" Version="3.1.9" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.9" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,44 @@
using System.IO;
using System.Threading.Tasks;
using Hangfire;
using MalwareMultiScan.Shared.Data.Requests;
using MalwareMultiScan.Worker.Jobs;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
namespace MalwareMultiScan.Worker.Controllers
{
[ApiController]
[Produces("application/json")]
public class ScanController : ControllerBase
{
[HttpPost]
[ProducesResponseType(StatusCodes.Status202Accepted)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]
[Route("/scan/file")]
public async Task<IActionResult> ScanFile([FromForm] FileRequest request)
{
var temporaryFile = Path.GetTempFileName();
await using (var temporaryFileSteam = System.IO.File.OpenWrite(temporaryFile))
await request.InputFile.CopyToAsync(temporaryFileSteam);
BackgroundJob.Enqueue<ScanJob>(
x => x.ScanFile(temporaryFile, request.CallbackUrl));
return Accepted(request.CallbackUrl);
}
[HttpPost]
[ProducesResponseType(StatusCodes.Status202Accepted)]
[ProducesResponseType(StatusCodes.Status400BadRequest)]
[Route("/scan/url")]
public IActionResult ScanUrl(UrlRequest request)
{
BackgroundJob.Enqueue<ScanJob>(
x => x.ScanUrl(request.InputUrl, request.CallbackUrl));
return Accepted(request.CallbackUrl);
}
}
}

View File

@ -0,0 +1,21 @@
FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS builder
WORKDIR /src
COPY MalwareMultiScan.Worker /src/MalwareMultiScan.Worker
COPY MalwareMultiScan.Shared /src/MalwareMultiScan.Shared
COPY MalwareMultiScan.Backends /src/MalwareMultiScan.Backends
RUN dotnet publish -c Release -r linux-x64 /p:PublishSingleFile=true -o ./publish \
MalwareMultiScan.Worker/MalwareMultiScan.Worker.csproj
FROM mcr.microsoft.com/dotnet/core/aspnet:3.1
WORKDIR /worker
COPY --from=builder /src/publish /worker
ENV ASPNETCORE_ENVIRONMENT=Production
ENV ASPNETCORE_URLS=http://+:9901
ENTRYPOINT ["/worker/MalwareMultiScan.Worker"]

View File

@ -0,0 +1,90 @@
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using MalwareMultiScan.Backends.Backends.Implementations;
using MalwareMultiScan.Shared.Data.Enums;
using MalwareMultiScan.Shared.Data.Responses;
using MalwareMultiScan.Shared.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace MalwareMultiScan.Worker.Jobs
{
public class ScanJob
{
private readonly ILogger<ScanJob> _logger;
private readonly IScanBackend _backend;
private readonly int _scanTimeout;
public ScanJob(IConfiguration configuration, ILogger<ScanJob> logger)
{
_logger = logger;
_scanTimeout = configuration.GetValue<int>("ScanTimeout");
_backend = configuration.GetValue<BackendType>("BackendType") switch
{
BackendType.Defender => new WindowsDefenderScanBackend(logger),
_ => throw new NotImplementedException()
};
}
public async Task PostResult(ResultResponse response, Uri callbackUrl, CancellationToken cancellationToken)
{
var serializedResponse = JsonSerializer.Serialize(
response, typeof(ResultResponse), new JsonSerializerOptions
{
WriteIndented = true
});
_logger.LogInformation(serializedResponse);
}
private async Task Scan(Func<CancellationToken, Task<string[]>> scanMethod, Uri callbackUrl)
{
var cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(_scanTimeout * 1000);
var cancellationToken = cancellationTokenSource.Token;
var response = new ResultResponse
{
Backend = _backend.Name
};
try
{
response.Success = true;
response.Threats = await scanMethod(cancellationToken);
response.DatabaseLastUpdate = _backend.DatabaseLastUpdate;
}
catch (Exception exception)
{
response.Success = false;
_logger.LogError(exception, "Scanning failed with exception");
}
await PostResult(response, callbackUrl, cancellationToken);
}
public async Task ScanUrl(Uri url, Uri callbackUrl)
{
await Scan(async t => await _backend.ScanAsync(url, t), callbackUrl);
}
public async Task ScanFile(string file, Uri callbackUrl)
{
try
{
await Scan(async t => await _backend.ScanAsync(file, t), callbackUrl);
}
finally
{
File.Delete(file);
}
}
}
}

View File

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj" />
<ProjectReference Include="..\MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.16" />
<PackageReference Include="Hangfire.MemoryStorage" Version="1.7.0" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,19 @@
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
namespace MalwareMultiScan.Worker
{
public static class Program
{
public static void Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
private static IHostBuilder CreateHostBuilder(string[] args)
{
return Host.CreateDefaultBuilder(args)
.ConfigureWebHostDefaults(builder => { builder.UseStartup<Startup>(); });
}
}
}

View File

@ -0,0 +1,30 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"iisSettings": {
"windowsAuthentication": false,
"anonymousAuthentication": true,
"iisExpress": {
"applicationUrl": "http://localhost:53549",
"sslPort": 44380
}
},
"profiles": {
"IIS Express": {
"commandName": "IISExpress",
"launchBrowser": true,
"launchUrl": "weatherforecast",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"MalwareMultiScan.Worker": {
"commandName": "Project",
"launchBrowser": true,
"launchUrl": "weatherforecast",
"applicationUrl": "https://localhost:5001;http://localhost:5000",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}

View File

@ -0,0 +1,34 @@
using Hangfire;
using Hangfire.MemoryStorage;
using MalwareMultiScan.Worker.Jobs;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.DependencyInjection;
namespace MalwareMultiScan.Worker
{
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
services.AddLogging();
services.AddControllers();
services.AddSingleton<ScanJob>();
services.AddHangfire(
configuration => configuration.UseMemoryStorage());
services.AddHangfireServer();
}
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
app.UseRouting();
app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
app.UseHangfireServer();
}
}
}

View File

@ -0,0 +1,9 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
}

View File

@ -0,0 +1,13 @@
{
"Logging": {
"LogLevel": {
"Default": "Warning",
"MalwareMultiScan": "Debug"
}
},
"AllowedHosts": "*",
"BackendType": "",
"ScanTimeout": 300
}

28
MalwareMultiScan.sln Normal file
View File

@ -0,0 +1,28 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Worker", "MalwareMultiScan.Worker\MalwareMultiScan.Worker.csproj", "{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Shared", "MalwareMultiScan.Shared\MalwareMultiScan.Shared.csproj", "{9E0A0B50-741F-4A49-97A2-0B337374347F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MalwareMultiScan.Backends", "MalwareMultiScan.Backends\MalwareMultiScan.Backends.csproj", "{382B49AC-0FFA-44FC-875D-9D4692DDC05D}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5D515E0A-B2C6-4C1D-88F6-C296F73409FA}.Release|Any CPU.Build.0 = Release|Any CPU
{9E0A0B50-741F-4A49-97A2-0B337374347F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9E0A0B50-741F-4A49-97A2-0B337374347F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9E0A0B50-741F-4A49-97A2-0B337374347F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9E0A0B50-741F-4A49-97A2-0B337374347F}.Release|Any CPU.Build.0 = Release|Any CPU
{382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{382B49AC-0FFA-44FC-875D-9D4692DDC05D}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal