diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c4667b8a..b55d0b10 100755 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -47,7 +47,7 @@ jobs: component-search-directories: services - name: Components to Nomad Jobs - uses: mfdlabs/component-nomad-parser-action@v15 + uses: mfdlabs/component-nomad-parser-action@v18 id: components-to-nomad-jobs env: NOMAD_ENVIRONMENT: ${{ github.event.inputs.nomad_environment }} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5d9f6c0d..0ff88f10 100755 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -14,21 +14,6 @@ "/p:Platform=AnyCPU", ], "problemMatcher": "$msCompile" - }, - { - "label": "build-full", - "command": "dotnet", - "type": "process", - "args": [ - "build", - "${workspaceFolder}/services/grid-bot/src/Grid.Bot.csproj", - "/property:GenerateFullPaths=true", - "/consoleloggerparameters:NoSummary", - "/p:Configuration=Debug", - "/p:Platform=AnyCPU", - "/p:LocalBuild=true" - ], - "problemMatcher": "$msCompile" } ] } \ No newline at end of file diff --git a/lib/Directory.Build.props b/lib/Directory.Build.props index d594f044..7bf200e0 100644 --- a/lib/Directory.Build.props +++ b/lib/Directory.Build.props @@ -39,8 +39,6 @@ portable true snupkg - - mfdlabs.$(MSBuildProjectName) @@ -60,13 +58,4 @@ - - - - README.md - - - - - diff --git a/lib/clients/client-settings-client/ClientSettingsClient.cs b/lib/clients/client-settings-client/ClientSettingsClient.cs index 4add09aa..88f231fd 100644 --- a/lib/clients/client-settings-client/ClientSettingsClient.cs +++ b/lib/clients/client-settings-client/ClientSettingsClient.cs @@ -28,7 +28,16 @@ public partial interface IClientSettingsClient /// API Key for request /// OK /// A server side error occurred. - System.Threading.Tasks.Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key); + ClientApplicationSettingsResponse GetApplicationSettings(string applicationName, string x_Api_Key = null); + + /// + /// Returns the complete settings dictionary for a Roblox client application. + /// + /// The name of the client application. + /// API Key for request + /// OK + /// A server side error occurred. + System.Threading.Tasks.Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key = null); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// @@ -39,6 +48,43 @@ public partial interface IClientSettingsClient /// OK /// A server side error occurred. System.Threading.Tasks.Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key, System.Threading.CancellationToken cancellationToken); + + /// + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + ClientApplicationSettingsResponse GetRccOnlyClientApplicationSettings(string applicationName, string bucketName); + + /// + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + System.Threading.Tasks.Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName); + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + System.Threading.Tasks.Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName, System.Threading.CancellationToken cancellationToken); + + /// + /// Imports the complete settings dictionary for a Roblox client application. + /// + /// API Key for request + /// OK + /// A server side error occurred. + void ImportApplicationSetting(string x_Api_Key, ImportClientApplicationSettingsRequest request); + /// /// Imports the complete settings dictionary for a Roblox client application. /// @@ -56,6 +102,14 @@ public partial interface IClientSettingsClient /// A server side error occurred. System.Threading.Tasks.Task ImportApplicationSettingAsync(string x_Api_Key, ImportClientApplicationSettingsRequest request, System.Threading.CancellationToken cancellationToken); + /// + /// Refreshes the complete settings dictionary for all Roblox client applications. + /// + /// API Key for request + /// OK + /// A server side error occurred. + void RefreshAllClientApplicationSettings(string x_Api_Key); + /// /// Refreshes the complete settings dictionary for all Roblox client applications. /// @@ -73,6 +127,15 @@ public partial interface IClientSettingsClient /// A server side error occurred. System.Threading.Tasks.Task RefreshAllClientApplicationSettingsAsync(string x_Api_Key, System.Threading.CancellationToken cancellationToken); + /// + /// Returns the setting for a Roblox client application. + /// + /// The name of the client application. + /// The name of the client application setting. + /// OK + /// A server side error occurred. + ClientApplicationSettingResponse GetClientApplicationSetting(string applicationName, string settingName); + /// /// Returns the setting for a Roblox client application. /// @@ -92,6 +155,14 @@ public partial interface IClientSettingsClient /// A server side error occurred. System.Threading.Tasks.Task GetClientApplicationSettingAsync(string applicationName, string settingName, System.Threading.CancellationToken cancellationToken); + /// + /// Sets the setting for a Roblox client application. + /// + /// API Key for request + /// OK + /// A server side error occurred. + SetClientApplicationSettingResponse SetClientApplicationSetting(string x_Api_Key, SetClientApplicationSettingRequest request); + /// /// Sets the setting for a Roblox client application. /// @@ -141,7 +212,19 @@ private Newtonsoft.Json.JsonSerializerSettings CreateSerializerSettings() /// API Key for request /// OK /// A server side error occurred. - public virtual System.Threading.Tasks.Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key) + public virtual ClientApplicationSettingsResponse GetApplicationSettings(string applicationName, string x_Api_Key = null) + { + return System.Threading.Tasks.Task.Run(async () => await GetApplicationSettingsAsync(applicationName, x_Api_Key, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + + /// + /// Returns the complete settings dictionary for a Roblox client application. + /// + /// The name of the client application. + /// API Key for request + /// OK + /// A server side error occurred. + public virtual System.Threading.Tasks.Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key = null) { return GetApplicationSettingsAsync(applicationName, x_Api_Key, System.Threading.CancellationToken.None); } @@ -232,6 +315,124 @@ public virtual async System.Threading.Tasks.Task + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + public virtual ClientApplicationSettingsResponse GetRccOnlyClientApplicationSettings(string applicationName, string bucketName) + { + return System.Threading.Tasks.Task.Run(async () => await GetRccOnlyClientApplicationSettingsAsync(applicationName, bucketName, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + + /// + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + public virtual System.Threading.Tasks.Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName) + { + return GetRccOnlyClientApplicationSettingsAsync(applicationName, bucketName, System.Threading.CancellationToken.None); + } + + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// + /// Returns the complete settings dictionary for a Roblox client application, restricted to RCC only. + /// + /// The name of the client application. + /// The name of the settings bucket. + /// OK + /// A server side error occurred. + public virtual async System.Threading.Tasks.Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName, System.Threading.CancellationToken cancellationToken) + { + if (applicationName == null) + throw new System.ArgumentNullException("applicationName"); + + var urlBuilder_ = new System.Text.StringBuilder(); + urlBuilder_.Append(_baseUrl != null ? _baseUrl.TrimEnd('/') : "").Append("/v2/settings/secured-settings/").Append(System.Uri.EscapeDataString(ConvertToString(applicationName, System.Globalization.CultureInfo.InvariantCulture))); + if (!string.IsNullOrWhiteSpace(bucketName)) + urlBuilder_.Append("/bucket/").Append(System.Uri.EscapeDataString(ConvertToString(bucketName, System.Globalization.CultureInfo.InvariantCulture))); + + var client_ = new System.Net.Http.HttpClient(); + var disposeClient_ = true; + try + { + using (var request_ = new System.Net.Http.HttpRequestMessage()) + { + request_.Method = new System.Net.Http.HttpMethod("GET"); + request_.Headers.Accept.Add(System.Net.Http.Headers.MediaTypeWithQualityHeaderValue.Parse("application/json")); + + var url_ = urlBuilder_.ToString(); + request_.RequestUri = new System.Uri(url_, System.UriKind.RelativeOrAbsolute); + + var response_ = await client_.SendAsync(request_, System.Net.Http.HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + var disposeResponse_ = true; + try + { + var headers_ = System.Linq.Enumerable.ToDictionary(response_.Headers, h_ => h_.Key, h_ => h_.Value); + if (response_.Content != null && response_.Content.Headers != null) + { + foreach (var item_ in response_.Content.Headers) + headers_[item_.Key] = item_.Value; + } + + var status_ = (int)response_.StatusCode; + if (status_ == 200) + { + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + if (objectResponse_.Object == null) + { + throw new ApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); + } + return objectResponse_.Object; + } + else + if (status_ == 400) + { + string responseText_ = (response_.Content == null) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ApiException("1: The application name is invalid.", status_, responseText_, headers_, null); + } + else + if (status_ == 401) + { + string responseText_ = (response_.Content == null) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ApiException("1: The application name is invalid.", status_, responseText_, headers_, null); + } + else + { + var responseData_ = response_.Content == null ? null : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); + throw new ApiException("The HTTP status code of the response was not expected (" + status_ + ").", status_, responseData_, headers_, null); + } + } + finally + { + if (disposeResponse_) + response_.Dispose(); + } + } + } + finally + { + if (disposeClient_) + client_.Dispose(); + } + } + + /// + /// Imports the complete settings dictionary for a Roblox client application. + /// + /// API Key for request + /// OK + /// A server side error occurred. + public virtual void ImportApplicationSetting(string x_Api_Key, ImportClientApplicationSettingsRequest request) + { + System.Threading.Tasks.Task.Run(async () => await ImportApplicationSettingAsync(x_Api_Key, request, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + /// /// Imports the complete settings dictionary for a Roblox client application. /// @@ -320,6 +521,17 @@ public virtual async System.Threading.Tasks.Task ImportApplicationSettingAsync(s } } + /// + /// Refreshes the complete settings dictionary for all Roblox client applications. + /// + /// API Key for request + /// OK + /// A server side error occurred. + public virtual void RefreshAllClientApplicationSettings(string x_Api_Key) + { + System.Threading.Tasks.Task.Run(async () => await RefreshAllClientApplicationSettingsAsync(x_Api_Key, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + /// /// Refreshes the complete settings dictionary for all Roblox client applications. /// @@ -396,6 +608,18 @@ public virtual async System.Threading.Tasks.Task RefreshAllClientApplicationSett } } + /// + /// Returns the setting for a Roblox client application. + /// + /// The name of the client application. + /// The name of the client application setting. + /// OK + /// A server side error occurred. + public virtual ClientApplicationSettingResponse GetClientApplicationSetting(string applicationName, string settingName) + { + return System.Threading.Tasks.Task.Run(async () => await GetClientApplicationSettingAsync(applicationName, settingName, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + /// /// Returns the setting for a Roblox client application. /// @@ -495,6 +719,17 @@ public virtual async System.Threading.Tasks.Task + /// Sets the setting for a Roblox client application. + /// + /// API Key for request + /// OK + /// A server side error occurred. + public virtual SetClientApplicationSettingResponse SetClientApplicationSetting(string x_Api_Key, SetClientApplicationSettingRequest request) + { + return System.Threading.Tasks.Task.Run(async () => await SetClientApplicationSettingAsync(x_Api_Key, request, System.Threading.CancellationToken.None)).GetAwaiter().GetResult(); + } + /// /// Sets the setting for a Roblox client application. /// @@ -810,4 +1045,4 @@ public ApiException(string message, int statusCode, string response, System.Coll #pragma warning restore 1591 #pragma warning restore 8073 #pragma warning restore 3016 -#pragma warning restore 8603 +#pragma warning restore 8603 \ No newline at end of file diff --git a/lib/grid/client/Factories/Lua.cs b/lib/grid/client/Factories/Lua.cs index c3dc0ec2..488897df 100644 --- a/lib/grid/client/Factories/Lua.cs +++ b/lib/grid/client/Factories/Lua.cs @@ -84,9 +84,10 @@ public static string ToString(IEnumerable result) /// The index to set. /// The value. /// Unsupported Lua argument type. - public static void SetArg(LuaValue[] args, int index, object value) + public static void SetArg(this LuaValue[] args, int index, object value) { var luaValue = new LuaValue(); + switch (value) { case int _: @@ -138,13 +139,13 @@ public static void SetArg(LuaValue[] args, int index, object value) /// /// The /// The actual value of the - public static object ConvertLua(LuaValue luaValue) + public static object ConvertLua(this LuaValue luaValue) => luaValue.type switch { LuaType.LUA_TBOOLEAN => Convert.ToBoolean(luaValue.value), LuaType.LUA_TNUMBER => Convert.ToDouble(luaValue.value), LuaType.LUA_TSTRING => luaValue.value, - LuaType.LUA_TTABLE => GetValues(luaValue.table), + LuaType.LUA_TTABLE => luaValue.table.GetValues(), _ => null, }; @@ -157,8 +158,8 @@ public static LuaValue[] NewArgs(params object[] args) { var luaValues = new LuaValue[args.Length]; - for (int i = 0; i < args.Length; i++) - SetArg(luaValues, i, args[i]); + for (int i = 0; i < args.Length; i++) + luaValues.SetArg(i, args[i]); return luaValues; } @@ -168,10 +169,12 @@ public static LuaValue[] NewArgs(params object[] args) /// /// The arguments. /// The raw values. - public static object[] GetValues(LuaValue[] args) + public static object[] GetValues(this LuaValue[] args) { var values = new object[args.Length]; - for (var i = 0; i < args.Length; i++) values[i] = ConvertLua(args[i]); + + for (var i = 0; i < args.Length; i++) values[i] = args[i].ConvertLua(); + return values; } } diff --git a/lib/grid/diagnostics/Grid.Diagnostics.csproj b/lib/grid/diagnostics/Grid.Diagnostics.csproj new file mode 100644 index 00000000..8e0d2aeb --- /dev/null +++ b/lib/grid/diagnostics/Grid.Diagnostics.csproj @@ -0,0 +1,30 @@ + + + + Helper methods for discovering diagnostics information about servers + + preview + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + \ No newline at end of file diff --git a/lib/grid/diagnostics/Implementation/ServerInfo.cs b/lib/grid/diagnostics/Implementation/ServerInfo.cs new file mode 100644 index 00000000..6b4517f8 --- /dev/null +++ b/lib/grid/diagnostics/Implementation/ServerInfo.cs @@ -0,0 +1,269 @@ +namespace Grid.Diagnostics; + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; + +using Windows.Win32; +using Windows.Win32.Foundation; +using Windows.Win32.System.SystemInformation; + +using Logging; + +/// +/// Model for server info. +/// +public class ServerInfo : IServerInfo +{ + private const float _KiloBytesInGigabyte = 1048576; + private const float _BytesInGigabyte = 1073741824; + private const int _TimerIntervalInMilliseconds = 2000; + + /// + /// The logical core count for the host machine. + /// + public int LogicalCoreCount { get; set; } + + /// + /// The physical core count for the host machine. + /// + public int PhysicalCoreCount { get; set; } + + /// + /// The physical memory in GiB. + /// + public float TotalPhysicalMemoryInGigabytes { get; set; } + + /// + /// The assembly version. + /// + public string AssemblyVersion { get; set; } + + /// + /// The kernel version. + /// + public string KernelVersion { get; set; } + + /// + /// Gets a static instance of + /// + /// The + public static ServerInfo GetInstance() + { + return new ServerInfo + { + AssemblyVersion = Assembly.GetEntryAssembly()?.GetName().Version.ToString(), + LogicalCoreCount = Environment.ProcessorCount, + PhysicalCoreCount = GetPhysicalCoreCount(), + TotalPhysicalMemoryInGigabytes = GetTotalPhysicalMemoryInGigabytes(), + KernelVersion = GetKernelVersion() + }; + } + + /// + /// Gets the Kernel version depending on + /// + /// An optional logger. + /// The Kernel Version + public static string GetKernelVersion(ILogger logger = null) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return Environment.OSVersion.Version.ToString(); + + return File.ReadAllText("/proc/version").Split(' ')[2]; + } + catch (Exception ex) + { + logger?.Error(ex); + + return null; + } + } + + /// + /// Gets the total physical memory in GB depending on + /// + /// An optional logger. + /// The total physical memory. + public static float GetTotalPhysicalMemoryInGigabytes(ILogger logger = null) => ExtractMemInfoValue("MemTotal", logger); + + /// + /// Gets the available physical memory in GB depending on + /// + /// An optional logger. + /// The available physical memory. + public static float GetAvailablePhysicalMemoryInGigabytes(ILogger logger = null) => ExtractMemInfoValue("MemAvailable", logger); + + /// + /// Gets the physical core count depending on the + /// + /// An to pass in. + /// The physical core count of the server. + public static int GetPhysicalCoreCount(ILogger logger = null) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return GetPhysicalCoreCountWindows(logger); + + return GetPhysicalCoreCountLinux(logger); + } + + private static float ExtractMemInfoValue(string metric, ILogger logger) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return ExtractMemInfoValueWindows(metric, logger); + + return ExtractMemInfoValueLinux(metric, logger); + } + + private static unsafe float ExtractMemInfoValueWindows(string metric, ILogger logger) + { + try + { + MEMORYSTATUSEX lpState = new(); + + lpState.dwLength = (uint)Marshal.SizeOf(lpState); + + if (!PInvoke.GlobalMemoryStatusEx(&lpState)) + throw new InvalidOperationException($"Error calling to Kernel32::GlobalMemoryStatusEx: {(WIN32_ERROR)Marshal.GetLastWin32Error()}"); + + var memInfo = metric switch + { + "MemTotal" => lpState.ullTotalPhys, + "MemAvailable" => lpState.ullAvailPhys, + _ => (ulong)0, + }; + + return memInfo / _BytesInGigabyte; + } + catch (Exception ex) + { + logger?.Error(ex); + + return 0; + } + } + + private static float ExtractMemInfoValueLinux(string metric, ILogger logger) + { + try + { + float.TryParse( + new string( + (File.ReadAllLines("/proc/meminfo") + .FirstOrDefault(line => line.StartsWith(metric)) ?? string.Empty) + .Where(char.IsDigit).ToArray() + ), + out var memInfo + ); + + return memInfo / _KiloBytesInGigabyte; + } + catch (Exception ex) + { + logger?.Error(ex); + + return 0; + } + } + + private static unsafe int GetPhysicalCoreCountWindows(ILogger logger) + { + uint bufferLength = 0; + PInvoke.GetLogicalProcessorInformationEx(LOGICAL_PROCESSOR_RELATIONSHIP.RelationAll, null, ref bufferLength); + + var buffer = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)Marshal.AllocHGlobal((int)bufferLength); + + static int _CountBitsSet(ulong value) + { + int count = 0; + while (value != 0) + { + count += (int)(value & 1); + value >>= 1; + } + return count; + } + + try + { + if (!PInvoke.GetLogicalProcessorInformationEx(LOGICAL_PROCESSOR_RELATIONSHIP.RelationAll, buffer, ref bufferLength)) + { + logger?.Error("Error calling to Kernel32::GetSystemTimes: {0}", (WIN32_ERROR)Marshal.GetLastWin32Error()); + + return 0; + } + + int physicalCoreCount = 0; + int logicalCoreCount = 0; + + byte* ptr = (byte*)buffer; + byte* endPtr = ptr + bufferLength; + + while (ptr < endPtr) + { + var information = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)ptr; + + if (information->Relationship == LOGICAL_PROCESSOR_RELATIONSHIP.RelationProcessorCore) + { + physicalCoreCount++; + for (int i = 0; i < information->Anonymous.Processor.GroupCount; i++) + logicalCoreCount += _CountBitsSet(information->Anonymous.Processor.GroupMask[i].Mask); + } + + ptr += information->Size; + } + + return physicalCoreCount * logicalCoreCount; + } + catch (Exception ex) + { + logger?.Error(ex); + return 0; + } + finally + { + if (buffer != null) + Marshal.FreeHGlobal((IntPtr)buffer); + } + } + + private static int GetPhysicalCoreCountLinux(ILogger logger) + { + try + { + var cpuInfo = File.ReadAllLines("/proc/cpuinfo"); + var currentLine = cpuInfo.FirstOrDefault(line => line.StartsWith("cpu cores")); + if (currentLine == null) + { + logger?.Error("Unable to find cpu cores line(s) in /proc/cpuinfo"); + return 0; + } + + var info = (from s in currentLine.Split(':') + select s.Trim()).ToArray(); + + if (info.Length != 2) + { + logger?.Error("Unable to parse 'cpu cores' line: {0}", currentLine); + return 0; + } + + var firstInfo = info[1]; + if (!int.TryParse(firstInfo, out int num)) + { + logger?.Error("Unable to parse 'cpu cores' value: {0}", firstInfo); + return 0; + } + + return (from line in cpuInfo + where line.StartsWith("physical id") + select line).Distinct().Count() * num; + } + catch (Exception ex) + { + logger?.Error(ex); + return 0; + } + } +} \ No newline at end of file diff --git a/lib/grid/process-management-docker/Docker/Interfaces/IServerInfo.cs b/lib/grid/diagnostics/Interfaces/IServerInfo.cs similarity index 95% rename from lib/grid/process-management-docker/Docker/Interfaces/IServerInfo.cs rename to lib/grid/diagnostics/Interfaces/IServerInfo.cs index 49e1c6cf..56b23d1d 100644 --- a/lib/grid/process-management-docker/Docker/Interfaces/IServerInfo.cs +++ b/lib/grid/diagnostics/Interfaces/IServerInfo.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.Diagnostics; /// /// Represents physical server info. @@ -29,4 +29,4 @@ public interface IServerInfo /// The kernel version. /// string KernelVersion { get; set; } -} +} \ No newline at end of file diff --git a/lib/grid/diagnostics/NativeMethods.txt b/lib/grid/diagnostics/NativeMethods.txt new file mode 100644 index 00000000..00992df4 --- /dev/null +++ b/lib/grid/diagnostics/NativeMethods.txt @@ -0,0 +1,5 @@ +WIN32_ERROR + +GetSystemTimes +GlobalMemoryStatusEx +GetLogicalProcessorInformationEx \ No newline at end of file diff --git a/lib/grid/job-management/Grid.JobManagement.csproj b/lib/grid/job-management/Grid.JobManagement.csproj new file mode 100644 index 00000000..b482ebc9 --- /dev/null +++ b/lib/grid/job-management/Grid.JobManagement.csproj @@ -0,0 +1,14 @@ + + + Shared library for allocating grid-server jobs! + + 1.2.5 + + + + + + + + + diff --git a/lib/grid/job-management/Implementation/JobManagerGridServer.cs b/lib/grid/job-management/Implementation/JobManagerGridServer.cs new file mode 100644 index 00000000..3e1482a0 --- /dev/null +++ b/lib/grid/job-management/Implementation/JobManagerGridServer.cs @@ -0,0 +1,218 @@ +namespace Grid.JobManagement; + +using System; +using System.Collections.Generic; + +using Logging; + +using Grid; +using Grid.Client; +using Grid.Commands; +using ProcessManagement.Core; + +using GridJob = Grid.Client.Job; +using Job = ProcessManagement.Core.Job; + +/// +/// Implementation for a Grid Server job manager. +/// +internal class JobManagerGridServer : IJobManagerGridServer +{ + /// + /// The logger. + /// + private readonly ILogger _logger; + + /// + /// The job manager. + /// + public JobManagerBase JobManager { get; private set; } + + /// + /// Construct a new instance of + /// + /// The + /// The + /// + /// - cannot be null. + /// - cannot be null. + /// + public JobManagerGridServer(ILogger logger, JobManagerBase jobManager) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + JobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + } + + /// + public void Start() => JobManager.Start(); + + /// + public void Stop() => JobManager.Stop(); + + /// + public int GetInstanceCount() => JobManager.GetInstanceCount(); + + /// + public int GetReadyInstanceCount() => JobManager.GetReadyInstanceCount(); + + /// + public int GetActiveJobsCount() => JobManager.GetActiveJobsCount(); + + /// + public string GetVersion() => JobManager.GetVersion(); + + /// + public IReadOnlyCollection GetAllRunningJobIds() => JobManager.GetAllRunningJobIds(); + + /// + public void AddOrUpdateActiveJob(IJob job, IGridServerInstance instance) => JobManager.AddOrUpdateActiveJob(job, instance); + + /// + public (bool isAvailable, JobRejectionReason? rejectionReason) IsResourceAvailable(GridServerResource resourceNeeded) => JobManager.IsResourceAvailable(resourceNeeded); + + /// + public GridServerResource GetAllocatedResource() => JobManager.GetAllocatedResource(); + + /// + public void RenewLease(IJob job, double leaseTimeInSeconds) => JobManager.RenewLease(job, leaseTimeInSeconds); + + /// + public (GridServerServiceSoap soapInterface, IGridServerInstance instance, JobRejectionReason? rejectionReason) NewJob( + IJob job, + double expirationInSeconds, + bool waitForReadyInstance = false, + bool addToActiveJobs = true + ) => JobManager.NewJob(job, expirationInSeconds, waitForReadyInstance, addToActiveJobs); + + /// + public GridServerServiceSoap GetJob(IJob job) => JobManager.GetJob(job); + + /// + public void CloseJob(IJob job, bool removeFromActiveJobs = true) => JobManager.CloseJob(job, removeFromActiveJobs); + + /// + public IReadOnlyCollection GetUnexpectedExitGameJobs() => JobManager.GetUnexpectedExitGameJobs(); + + /// + public void DispatchRequestToAllActiveJobs(Action action) => JobManager.DispatchRequestToAllActiveJobs(action); + + /// + public string GetGridServerInstanceId(string jobId) => JobManager.GetGridServerInstanceId(jobId); + + /// + public bool UpdateGridServerInstance(GridServerResourceJob job) => JobManager.UpdateGridServerInstance(job); + + /// + public virtual double RenewLease(string jobId, double expirationInSeconds) + { + var job = new Job(jobId); + _logger.Information("RenewLease starting. {0}, expirationInSeconds = {1}", job, expirationInSeconds); + + JobManager.RenewLease(job, expirationInSeconds); + + using var soap = JobManager.GetJob(job); + var newExpiration = soap.RenewLease(jobId, expirationInSeconds); + _logger.Information("RenewLease completed. {0}, expirationInSeconds = {1}, returned value = {2}", job, expirationInSeconds, newExpiration); + + return newExpiration; + } + + /// + public virtual void CloseJob(string jobId) + { + var job = new Job(jobId); + _logger.Information("CloseJob starting. {0}", job); + + try + { + using var soap = JobManager.GetJob(job); + soap.CloseJob(jobId); + } + finally + { + JobManager.CloseJob(job, false); + } + + _logger.Information("CloseJob completed. {0}", job); + } + + /// + public LuaValue[] RunBatchJob(GridJob gridJob, ScriptExecution script) + { + _logger.Information("RunBatchJob starting. Job ID = {0}", gridJob.id); + + var job = new Job(gridJob.id); + + try + { + var (soap, _, rejectionReason) = JobManager.NewJob(job, gridJob.expirationInSeconds, true); + if (rejectionReason != null) + throw new Exception($"JobManager.NewJob was rejected. Rejection Reason: {rejectionReason.Value}"); + + using (soap) + { + var data = soap.BatchJobEx(gridJob, script); + + JobManager.CloseJob(job, true); + + _logger.Information("RunBatchJob completed. Job ID = {0}, Category = {1}, ExpirationInSeconds = {2}", gridJob.id, gridJob.category, gridJob.expirationInSeconds); + + return data; + } + } + catch (Exception ex) + { + JobManager.CloseJob(job, false); + + _logger.Error( + "RunJob failed. Job ID = {0}, Category = {1}, ExpirationInSeconds = {2}, Exception = {3}", + gridJob.id, + gridJob.category, + gridJob.expirationInSeconds, + ex + ); + + throw; + } + } + + /// + public LuaValue[] RunBatchJob(GridJob gridJob, GridCommand script) + { + _logger.Information("RunBatchJob starting. Job ID = {0}", gridJob.id); + + var job = new Job(gridJob.id); + + try + { + var (soap, _, rejectionReason) = JobManager.NewJob(job, gridJob.expirationInSeconds, true); + if (rejectionReason != null) + throw new Exception($"JobManager.NewJob was rejected. Rejection Reason: {rejectionReason.Value}"); + + using (soap) + { + var data = soap.BatchJobEx(gridJob, script); + + JobManager.CloseJob(job, true); + + _logger.Information("RunBatchJob completed. Job ID = {0}, Category = {1}, ExpirationInSeconds = {2}", gridJob.id, gridJob.category, gridJob.expirationInSeconds); + + return data; + } + } + catch (Exception ex) + { + JobManager.CloseJob(job, false); + + _logger.Error( + "RunJob failed. Job ID = {0}, Category = {1}, ExpirationInSeconds = {2}, Exception = {3}", + gridJob.id, + gridJob.category, + gridJob.expirationInSeconds, + ex + ); + + throw; + } + } +} diff --git a/lib/grid/job-management/Implementation/JobManagerGridServerFactory.cs b/lib/grid/job-management/Implementation/JobManagerGridServerFactory.cs new file mode 100644 index 00000000..d4e91997 --- /dev/null +++ b/lib/grid/job-management/Implementation/JobManagerGridServerFactory.cs @@ -0,0 +1,59 @@ +namespace Grid.JobManagement; + +using System; +using System.Runtime.InteropServices; + +using Random; +using Logging; +using ClientSettings.Client; + +using PortManagement; +using ProcessManagement; +using ProcessManagement.Core; +using ProcessManagement.Docker; + +/// +/// Default implementation of +/// +public class JobManagerGridServerFactory : IJobManagerGridServerFactory +{ + /// + public IJobManagerGridServer GetJobManager(ILogger logger, IClientSettingsClient clientSettingsClient, IJobManagerSettings settings) + { + if (logger == null) throw new ArgumentNullException(nameof(logger)); + if (clientSettingsClient == null) throw new ArgumentNullException(nameof(clientSettingsClient)); + if (settings == null) throw new ArgumentNullException(nameof(settings)); + + var portAllocator = new PortAllocator(logger); + + JobManagerBase jobManager; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (settings is not IGridServerProcessSettings processSettings) + throw new ArgumentException($"{nameof(settings)} must be of type {nameof(IGridServerProcessSettings)} when on {OSPlatform.Windows}.", nameof(settings)); + + jobManager = new ProcessJobManager( + logger, + portAllocator, + processSettings, + clientSettingsClient + ); + } + else + { + if (settings is not IGridServerDockerSettings dockerSettings) + throw new ArgumentException($"{nameof(settings)} must be of type {nameof(IGridServerProcessSettings)} when on {RuntimeInformation.OSDescription}.", nameof(settings)); + + jobManager = new DockerJobManager( + logger, + portAllocator, + dockerSettings, + RandomFactory.GetDefaultRandom(), + clientSettingsClient + ); + } + + return new JobManagerGridServer(logger, jobManager); + } +} diff --git a/lib/grid/job-management/Interfaces/IJobManagerGridServer.cs b/lib/grid/job-management/Interfaces/IJobManagerGridServer.cs new file mode 100644 index 00000000..0bb1f3ca --- /dev/null +++ b/lib/grid/job-management/Interfaces/IJobManagerGridServer.cs @@ -0,0 +1,182 @@ +namespace Grid.JobManagement; + +using System; +using System.Collections.Generic; + +using Grid.Client; +using Grid.Commands; +using Grid.ProcessManagement.Core; + +using GridJob = Grid.Client.Job; + +/// +/// Interface for a job manager. +/// +/// +/// Designed to be independent +/// of both Win32 and Linux Grid Servers. +/// +public interface IJobManagerGridServer +{ + /// + /// Gets the raw job manager. + /// + JobManagerBase JobManager { get; } + + /// + /// Start the job manager. + /// + void Start(); + + /// + /// Stops the job manager. + /// + void Stop(); + + /// + /// Get the current amount of instances. + /// + /// The count of instances. + int GetInstanceCount(); + + /// + /// Get the current amount of ready instances. + /// + /// The count of ready instances. + int GetReadyInstanceCount(); + + /// + /// Get the current amount of active jobs. + /// + /// The count of active jobs. + int GetActiveJobsCount(); + + /// + /// Get a list of all running job ids. + /// + /// A list of running job ids. + IReadOnlyCollection GetAllRunningJobIds(); + + /// + /// Adds or updates an active job. + /// + /// The job. + /// The instancee. + void AddOrUpdateActiveJob(IJob job, IGridServerInstance instance); + + /// + /// Is a Grid Server resource available? + /// + /// The resource needed. + /// A tuple of if it is available and a JobRejection reason. + (bool isAvailable, JobRejectionReason? rejectionReason) IsResourceAvailable(GridServerResource resourceNeeded); + + /// + /// Get the allocated resource. + /// + /// The allocated Grid Server resource. + GridServerResource GetAllocatedResource(); + + /// + /// Renew a job lease. + /// + /// The job. + /// The new expiration in seconds. + void RenewLease(IJob job, double expirationInSeconds); + + /// + /// Create a new job on Grid Server. + /// + /// The job. + /// The job's TTL. + /// Should wait for instance to become ready? + /// Add this job to the active jobs list? + /// The SOAP interface, the instance and a job rejection reason. + /// Cannot create a new job, since job already exists + (GridServerServiceSoap soapInterface, IGridServerInstance instance, JobRejectionReason? rejectionReason) NewJob( + IJob job, + double expirationInSeconds, + bool waitForReadyInstance = false, + bool addToActiveJobs = true + ); + + /// + /// Get the SOAP interface for a job. + /// + /// The job. + /// The SOAP interface associated with the job. + /// + /// - Job not found. + /// - Job found but the instance has already exited. + /// + GridServerServiceSoap GetJob(IJob job); + + /// + /// Close a job. + /// + /// The job. + /// Attempt a GC. + void CloseJob(IJob job, bool attemptToRecycle); + + /// + /// Get the current Grid Server version. + /// + /// The Grid Server version. + string GetVersion(); + + /// + /// Gets the unexpectedly closed game jobs. + /// + /// + IReadOnlyCollection GetUnexpectedExitGameJobs(); + + /// + /// Dispatch a SOAP message to every running job. + /// + /// The SOAP message. + void DispatchRequestToAllActiveJobs(Action action); + + /// + /// Get the instance id for an Grid Server by it's job id. + /// + /// The job id. + /// The Grid Server instance id. + string GetGridServerInstanceId(string jobId); + + /// + /// Update an Grid Server instance's resources. + /// + /// The resources job. + /// If the update was successful or not. + bool UpdateGridServerInstance(GridServerResourceJob job); + + /// + /// Renew the lease of a job. + /// + /// The ID of the job. + /// The new expiration in seconds. + /// The new expiration in seconds. + double RenewLease(string jobId, double expirationInSeconds); + + /// + /// Close an Grid Server job. + /// + /// The job ID. + void CloseJob(string jobId); + + /// + /// Run a batch job. + /// + /// The SOAP job. + /// The script. + /// The result of the batch job. + LuaValue[] RunBatchJob(GridJob gridJob, ScriptExecution script); + + /// + /// Run a batch job. + /// + /// The SOAP job. + /// The script. + /// The result of the batch job. + LuaValue[] RunBatchJob(GridJob gridJob, GridCommand script); +} \ No newline at end of file diff --git a/lib/grid/job-management/Interfaces/IJobManagerGridServerFactory.cs b/lib/grid/job-management/Interfaces/IJobManagerGridServerFactory.cs new file mode 100644 index 00000000..a53a36c9 --- /dev/null +++ b/lib/grid/job-management/Interfaces/IJobManagerGridServerFactory.cs @@ -0,0 +1,36 @@ +namespace Grid.JobManagement; + +using System; +using System.Runtime.InteropServices; + +using Logging; +using ClientSettings.Client; + +using ProcessManagement; +using ProcessManagement.Core; +using ProcessManagement.Docker; + +/// +/// Factory for spitting out +/// based on operating system. +/// +public interface IJobManagerGridServerFactory +{ + /// + /// Gets an instance of based on OS. + /// + /// The + /// The + /// The that are either or + /// An instance of proxying either or + /// + /// - cannot be null. + /// - cannot be null. + /// - cannot be null. + /// + /// + /// - must be of type when on . + /// - must be of type when on . + /// + IJobManagerGridServer GetJobManager(ILogger logger, IClientSettingsClient clientSettingsClient, IJobManagerSettings settings); +} diff --git a/lib/grid/port-management/Grid.PortManagement.csproj b/lib/grid/port-management/Grid.PortManagement.csproj index cde2123b..58102428 100644 --- a/lib/grid/port-management/Grid.PortManagement.csproj +++ b/lib/grid/port-management/Grid.PortManagement.csproj @@ -1,12 +1,12 @@  Shared library for port allocation on grid-servers! - Grid + Grid.PortManagement - + diff --git a/lib/grid/port-management/Implementation/PortAllocator.cs b/lib/grid/port-management/Implementation/PortAllocator.cs index 4415294a..fcb2e2e9 100644 --- a/lib/grid/port-management/Implementation/PortAllocator.cs +++ b/lib/grid/port-management/Implementation/PortAllocator.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.PortManagement; using System; using System.Linq; diff --git a/lib/grid/port-management/Interfaces/IPortAllocator.cs b/lib/grid/port-management/Interfaces/IPortAllocator.cs index e8befab8..3241c2b1 100644 --- a/lib/grid/port-management/Interfaces/IPortAllocator.cs +++ b/lib/grid/port-management/Interfaces/IPortAllocator.cs @@ -1,21 +1,20 @@ -namespace Grid +namespace Grid.PortManagement; + +/// +/// A class to allocate ports. +/// +public interface IPortAllocator { /// - /// A class to allocate ports. + /// Find the next available port in a range. /// - public interface IPortAllocator - { - /// - /// Find the next available port in a range. - /// - /// The port. - /// Failed to find an open port. - int FindNextAvailablePort(); + /// The port. + /// Failed to find an open port. + int FindNextAvailablePort(); - /// - /// Remove a port from the memory cache. - /// - /// The port. - void RemovePortFromCacheIfExists(int port); - } + /// + /// Remove a port from the memory cache. + /// + /// The port. + void RemovePortFromCacheIfExists(int port); } diff --git a/lib/grid/process-management-core/Enums/JobRejectionReason.cs b/lib/grid/process-management-core/Enums/JobRejectionReason.cs index 1fc68e3d..b190e03c 100644 --- a/lib/grid/process-management-core/Enums/JobRejectionReason.cs +++ b/lib/grid/process-management-core/Enums/JobRejectionReason.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Reasoning for a job being rejected. @@ -10,6 +10,11 @@ public enum JobRejectionReason /// NoReadyInstance, + /// + /// The settings file for GridServer was invalid. + /// + GridServerSettingsFileInvalid, + /// /// There was not enough CPU available. /// diff --git a/lib/grid/process-management-core/ExponentialBackoff.cs b/lib/grid/process-management-core/ExponentialBackoff.cs index f00d1ed1..cc399905 100644 --- a/lib/grid/process-management-core/ExponentialBackoff.cs +++ b/lib/grid/process-management-core/ExponentialBackoff.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; diff --git a/lib/grid/process-management-core/Grid.ProcessManagement.Core.csproj b/lib/grid/process-management-core/Grid.ProcessManagement.Core.csproj index 6cb5e34f..8dff3be8 100644 --- a/lib/grid/process-management-core/Grid.ProcessManagement.Core.csproj +++ b/lib/grid/process-management-core/Grid.ProcessManagement.Core.csproj @@ -1,7 +1,7 @@  Shared library for core interaction with grid server processes! - Grid + Grid.ProcessManagement.Core @@ -19,8 +19,10 @@ + + diff --git a/lib/grid/process-management-core/Implementation/GridServerInstanceBase.cs b/lib/grid/process-management-core/Implementation/GridServerInstanceBase.cs index c8375f9d..692f4289 100644 --- a/lib/grid/process-management-core/Implementation/GridServerInstanceBase.cs +++ b/lib/grid/process-management-core/Implementation/GridServerInstanceBase.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; using System.Net; @@ -52,6 +52,12 @@ public abstract class GridServerInstanceBase : IGridServerInstance /// public string Version { get; } + /// + public string ApplicationName { get; } + + /// + public string BucketName { get; } + /// public DateTime ExpirationTime { get; set; } @@ -65,15 +71,21 @@ public abstract class GridServerInstanceBase : IGridServerInstance /// The version /// The port /// The + /// The name of the application in client settings. + /// The name of the application bucket in client settings. protected GridServerInstanceBase( ILogger logger, string version, int port, - IJobManagerSettings settings + IJobManagerSettings settings, + string applicationName, + string bucketName ) { Logger = logger; Version = version; + ApplicationName = applicationName; + BucketName = bucketName; Port = port; _Settings = settings; } @@ -84,15 +96,21 @@ IJobManagerSettings settings /// The /// The port /// The + /// The name of the application in client settings. + /// The name of the application bucket in client settings. protected GridServerInstanceBase( ILogger logger, int port, - IJobManagerSettings settings + IJobManagerSettings settings, + string applicationName, + string bucketName ) { Logger = logger; Port = port; Version = GetVersionFromGridServer(); + ApplicationName = applicationName; + BucketName = bucketName; _Settings = settings; } diff --git a/lib/grid/process-management-core/Implementation/JobManagerBase.cs b/lib/grid/process-management-core/Implementation/JobManagerBase.cs index 77c0459b..703be51b 100644 --- a/lib/grid/process-management-core/Implementation/JobManagerBase.cs +++ b/lib/grid/process-management-core/Implementation/JobManagerBase.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; using System.Linq; @@ -11,6 +11,9 @@ using Logging; using Client; +using Diagnostics; +using PortManagement; +using ClientSettings.Client; /// /// Represents the base abstract class for all job managers. @@ -21,11 +24,15 @@ public abstract class JobManagerBase private const int _DefaultIntervalToQueryForRunningJobs = 3000; private const int _DefaultGridServerJobTimeoutInMilliseconds = 300000; private const int _DefaultTimeToCheckForNewGridServerVersion = 10000; + private const int _DefaultTimeToFetchGridServerApplicationSettings = 30000; private const int _DefaultManagePopulateGridServerIntervalMilliseconds = 10000; private const int _DefaultPopulateInstanceSleepInterval = 100; private const int _DefaultExpirationForRecoveredInstances = 300000; private const int _NegativeFive = -5; + private static readonly TimeSpan _InitialGridServerApplicationSettingsFetchSleepIntervalBase = TimeSpan.FromMilliseconds(300); + private static readonly TimeSpan _InitialGridServerApplicationSettingsFetchSleepIntervalMax = TimeSpan.FromSeconds(15); + /// /// The resource allocation tracker. /// @@ -51,18 +58,31 @@ public abstract class JobManagerBase private readonly object _PopulateGridServerLock = new(); private readonly IJobManagerSettings _Settings; + private readonly IClientSettingsClient _ClientSettingsClient; + private readonly ISettingsFileWriter _SettingsFileWriter; /// /// The Grid Server version. /// protected string GridServerVersion; + /// + /// The application settings name. + /// + protected string ApplicationName; + + /// + /// The application bucket name. + /// + protected string BucketName; + /// /// The ready instances. /// protected BlockingCollection ReadyInstances = new(new ConcurrentStack()); private bool _IsRunning; + private DateTime _LastSuccessfulGridServerApplicationSettingsFetchTime; private volatile int _PopulateThreadsWorking; private volatile bool _LastInstanceCreationFailed; @@ -73,24 +93,74 @@ public abstract class JobManagerBase /// The /// The /// The + /// The /// The /// /// - cannot be null. /// - cannot be null. /// - cannot be null. + /// - cannot be null. /// protected JobManagerBase( ILogger logger, IJobManagerSettings settings, IPortAllocator portAllocator, + IClientSettingsClient clientSettingsClient, + ResourceAllocationTracker resourceAllocationTracker + ) : this( + logger, + settings, + portAllocator, + clientSettingsClient, + new SettingsFileWriter(logger), + resourceAllocationTracker + ) + { + } + + /// + /// Constructs a new instance of + /// + /// The + /// The + /// The + /// The + /// The + /// The + /// + /// - cannot be null. + /// - cannot be null. + /// - cannot be null. + /// - cannot be null. + /// - cannot be null. + /// + protected JobManagerBase( + ILogger logger, + IJobManagerSettings settings, + IPortAllocator portAllocator, + IClientSettingsClient clientSettingsClient, + ISettingsFileWriter settingsFileWriter, ResourceAllocationTracker resourceAllocationTracker ) { Logger = logger ?? throw new ArgumentNullException(nameof(logger)); _Settings = settings ?? throw new ArgumentNullException(nameof(settings)); PortAllocator = portAllocator ?? throw new ArgumentNullException(nameof(portAllocator)); - - ResourceAllocationTracker = resourceAllocationTracker ?? new ResourceAllocationTracker(); + _ClientSettingsClient = clientSettingsClient ?? throw new ArgumentNullException(nameof(clientSettingsClient)); + _SettingsFileWriter = settingsFileWriter ?? throw new ArgumentNullException(nameof(settingsFileWriter)); + + ResourceAllocationTracker = resourceAllocationTracker + ?? new ResourceAllocationTracker( + ServerInfo.GetPhysicalCoreCount(Logger), + _Settings.GridServerMaxThreads, + (long)(ServerInfo.GetAvailablePhysicalMemoryInGigabytes(Logger) * 1024), + () => _Settings.IsGridServerCpuAllocationCheckEnabled, + () => _Settings.IsGridServerThreadsAllocationCheckEnabled, + () => _Settings.IsGridServerMemoryAllocationCheckEnabled, + () => _Settings.GridServerCpuOverAllocationRatio, + () => _Settings.GridServerThreadsOverAllocationRatio, + () => _Settings.GridServerMemoryOverAllocationRatio + ); } /// @@ -127,19 +197,21 @@ public IReadOnlyCollection GetAllRunningJobIds() /// public void Start() { - Logger.Information("Starting JobManager using GridServerProcessManagement.Core"); + Logger.Information("Starting JobManager using Grid.ProcessManagement.Core"); _IsRunning = true; do { ReadGridServerLocation(true); + ReadGridServerApplicationName(); } - while (string.IsNullOrWhiteSpace(GridServerVersion)); + while (string.IsNullOrWhiteSpace(GridServerVersion) || string.IsNullOrWhiteSpace(ApplicationName)); RecoverRunningInstances(); Task.Factory.StartNew(CheckGridServerVersion, TaskCreationOptions.LongRunning); + Task.Factory.StartNew(CheckGridServerApplicationSettings, TaskCreationOptions.LongRunning); Task.Factory.StartNew(ManagePopulateReadyInstanceThreads, TaskCreationOptions.LongRunning); Task.Factory.StartNew(ClearExpiredJobs, TaskCreationOptions.LongRunning); } @@ -149,7 +221,7 @@ public void Start() /// public void Stop() { - Logger.Information("Stopping JobManager using GridServerProcessManagement.Core"); + Logger.Information("Stopping JobManager using Grid.ProcessManagement.Core"); _IsRunning = false; } @@ -298,6 +370,18 @@ public void CloseJob(IJob job, bool attemptToRecycle) /// The Grid Server version. public string GetVersion() => GridServerVersion; + /// + /// Get the current client settings application name. + /// + /// The client settings application name. + public string GetApplicationName() => ApplicationName; + + /// + /// Get the current client settings bucket name. + /// + /// The client settings bucket name. + public string GetBucketName() => BucketName; + /// /// Gets the unexpectedly closed game jobs. /// @@ -320,6 +404,12 @@ public void DispatchRequestToAllActiveJobs(Action action) } } + /// + /// Gets the last successful client settings fetch time. + /// + /// The last successful client settings fetch time. + public DateTime GetLastSuccessfulGridServerApplicationSettingsFetchTime() => _LastSuccessfulGridServerApplicationSettingsFetchTime; + /// /// Get the instance id for an Grid Server by it's job id. /// @@ -394,6 +484,7 @@ public void DispatchRequestToAllActiveJobs(Action action) protected void CheckForGridServerReady() { if (string.IsNullOrWhiteSpace(GridServerVersion)) throw new Exception("Grid Server Version not set"); + if (string.IsNullOrWhiteSpace(ApplicationName)) throw new Exception("Grid Server ApplicationName not set"); } private void DoCloseJob(IJob job, bool attemptToRecycle) @@ -420,8 +511,9 @@ private void DoCloseJob(IJob job, bool attemptToRecycle) } Logger.Information( - "DoCloseJob. Did not recycle instance - version out of date. CurrentVersion: {0}. UseCount: {1}, Instance Version: {2}, Instance ID: {3}", + "DoCloseJob. Did not recycle instance - version out of date. CurrentVersion: {0}. Current ApplicationName: {1}. UseCount: {2}, Instance Version: {3}, Instance ID: {4}", GridServerVersion, + ApplicationName, instance.UseCount, instance.Version, instance.Id @@ -512,6 +604,155 @@ private void KillOutOfDateReadyInstances() } } + private bool UpdateGridServerApplicationSettingsWithRetries(string applicationName, string bucketName, byte maxAttempts) + { + Logger.Information("UpdateGridServerApplicationSettingsWithRetries. Fetching Grid Server application settings with {0} retries.", maxAttempts); + + var didUpdate = UpdateGridServerApplicationSettings(applicationName, bucketName); + for (byte index = 1; !didUpdate && index <= maxAttempts; didUpdate = UpdateGridServerApplicationSettings(applicationName, bucketName)) + { + var sleepInterval = ExponentialBackoff.CalculateBackoff( + index++, + 10, + _InitialGridServerApplicationSettingsFetchSleepIntervalBase, + _InitialGridServerApplicationSettingsFetchSleepIntervalMax, + Jitter.Equal + ); + + Logger.Warning( + "UpdateGridServerApplicationSettingsWithRetries. Failed to update Grid Server Application Settings. ApplicationName: {0}. BucketName: {1}. Sleeping for {2} seconds", + applicationName, + bucketName, + sleepInterval.TotalSeconds + ); + + Thread.Sleep(sleepInterval); + } + + return didUpdate; + } + + private void CheckGridServerApplicationSettings() + { + while (_IsRunning) + { + Thread.Sleep(_DefaultTimeToFetchGridServerApplicationSettings); + + try + { + ReadGridServerApplicationName(); + UpdateGridServerApplicationSettings(ApplicationName, BucketName); + } + catch (Exception ex) + { + Logger.Error("Error in CheckRccApplicationSettings. Exception: {0}", ex); + } + } + } + + private void ReadGridServerApplicationName() + { + var newApplicationName = _Settings.GridServerSettingsApplicationName; + var newBucketName = _Settings.GridServerSettingsBucketName; + + if (ApplicationName != newApplicationName || BucketName != newBucketName) + { + if (ApplicationName != newApplicationName) + Logger.Information( + "ReadGridServerApplicationName. Grid Server ApplicationName changed or loaded for the first time. ApplicationName = {0}. Old Value = {1}.", + newApplicationName, + ApplicationName + ); + + if (BucketName != newBucketName) + Logger.Information( + "ReadGridServerApplicationName. Grid Server BucketName changed or loaded for the first time. BucketName = {0}. Old Value = {1}.", + newBucketName, + BucketName + ); + + if (UpdateGridServerApplicationSettingsWithRetries(newApplicationName, newBucketName, 10)) + { + Logger.Information( + "ReadGridServerApplicationName. Successfully changed Grid Server ApplicationName. ApplicationName = {0}, Old ApplicationName Value = {1}, BucketName = {2}, Old BucketName Value = {3}", + newApplicationName, + ApplicationName, + newBucketName, + BucketName + ); + + ApplicationName = newApplicationName; + BucketName = newBucketName; + + KillOutOfDateReadyInstances(); + + return; + } + + Logger.Warning( + "ReadGridServerApplicationName. Failed to change Grid Server ApplicationName. Failed ApplicationName = {0}, Current ApplicationName = {1}, Failed BucketName = {2}, Current BucketName = {3}", + newApplicationName, + ApplicationName, + newBucketName, + BucketName + ); + } + } + + /// + /// Update the Grid Server app settings. + /// + /// The application name. + /// The bucket name. + /// If the update was successful or not. + protected bool UpdateGridServerApplicationSettings(string applicationName, string bucketName) + { + try + { + return TryUpdateGridServerApplicationSettings(applicationName, bucketName); + } + catch (Exception ex) + { + Logger.Error("CheckGridServerApplicationSettings. Error in TryUpdateGridServerApplicationSettings. Exception: {0}", ex); + } + + return false; + } + + private bool TryUpdateGridServerApplicationSettings(string applicationName, string bucketName) + { + if (string.IsNullOrWhiteSpace(_Settings.GridServerApplicationSettingsFilePath)) + { + Logger.Error("TryUpdateGridServerApplicationSettings. GridServerApplicationSettingsFilePath cannot be null or blank."); + + return false; + } + + Logger.Information("TryUpdateGridServerApplicationSettings. Fetching Grid Server application settings from secured endpoint. ApplicationName: {0}. BucketName: {1}", applicationName, bucketName); + + var newGridServerApplicationSettings = _ClientSettingsClient.GetRccOnlyClientApplicationSettings(applicationName, bucketName); + if (newGridServerApplicationSettings?.ApplicationSettings == null || newGridServerApplicationSettings.ApplicationSettings.Count == 0) + { + Logger.Error("TryUpdateGridServerApplicationSettings. ClientApplicationSettingsResponse was null or empty."); + + return false; + } + + Logger.Information("TryUpdateGridServerApplicationSettings. Successfully fetched latest Grid Server application settings. Count: {0}", newGridServerApplicationSettings.ApplicationSettings.Count); + if (!_SettingsFileWriter.WriteSettingsFile(_Settings.GridServerApplicationSettingsFilePath, newGridServerApplicationSettings)) + { + Logger.Error("TryUpdateGridServerApplicationSettings. Error writing settings file."); + + return false; + } + + Logger.Information("TryUpdateGridServerApplicationSettings. Successfully wrote latest GridServer application settings to file: {0}", _Settings.GridServerApplicationSettingsFilePath); + + _LastSuccessfulGridServerApplicationSettingsFetchTime = DateTime.UtcNow; + + return true; + } + private void ManagePopulateReadyInstanceThreads() { var ctsList = new Stack(_Settings.PopulateReadyGridServerInstanceThreads); @@ -526,7 +767,7 @@ private void ManagePopulateReadyInstanceThreads() { var threadsToCreate = desiredNumberOfThreads - ctsList.Count; - Logger.Debug( + Logger.Verbose( "ManagePopulateReadyInstanceThreads. Need to create {0} threads. SettingValue = {1}, threadCount = {2}", threadsToCreate, desiredNumberOfThreads, @@ -544,7 +785,7 @@ private void ManagePopulateReadyInstanceThreads() else if (ctsList.Count > desiredNumberOfThreads) { var threadsToCancel = ctsList.Count - desiredNumberOfThreads; - Logger.Debug( + Logger.Verbose( "ManagePopulateReadyInstanceThreads. Need to cancel {0} threads. SettingValue = {1}, threadCount = {2}", threadsToCancel, desiredNumberOfThreads, @@ -555,7 +796,7 @@ private void ManagePopulateReadyInstanceThreads() ctsList.Pop().Cancel(); } else - Logger.Debug("ManagePopulateReadyInstanceThreads. SettingValue = {0}, threadCount = {1}", desiredNumberOfThreads, ctsList.Count); + Logger.Verbose("ManagePopulateReadyInstanceThreads. SettingValue = {0}, threadCount = {1}", desiredNumberOfThreads, ctsList.Count); } catch (Exception ex) { @@ -588,7 +829,7 @@ private void PopulateReadyInstances(object state) Thread.Sleep(_DefaultPopulateInstanceSleepInterval); if (!ShouldPopulateReadyInstance()) - Logger.Debug( + Logger.Verbose( "PopulateReadyInstances. No new instance created because there are enough instances. Ready instances: {0}, PopulateThreadsWorking: {1}, Active Jobs: {2}, MaxInstances: {3}, ReadyInstancesToKeepInReserve: {4}. ThreadId: {5}.", ReadyInstances.Count, _PopulateThreadsWorking, @@ -657,9 +898,18 @@ private void PopulateReadyInstances(object state) } } + private bool IsGridServerApplicationSettingsFileValid() + { + var window = DateTime.UtcNow - _Settings.GridServerApplicationSettingsValidWindow; + + return _LastSuccessfulGridServerApplicationSettingsFetchTime >= window; + } + private (IGridServerInstance, JobRejectionReason?) GetReadyInstance(string jobId, double expirationInSeconds, bool waitForReadyInstance) { var sw = Stopwatch.StartNew(); + if (!IsGridServerApplicationSettingsFileValid()) + return (default, JobRejectionReason.GridServerSettingsFileInvalid); IGridServerInstance instance; while (true) @@ -812,20 +1062,28 @@ private void RecoverRunningInstances() if (instance.Version != GridServerVersion) { Logger.Information( - "RecoverRunningInstances. Recovered a ready instance with no jobs and a version mismatch. Instance ID = {0}, Port = {1}, Version = {2}, Expected Version = {3}", + "RecoverRunningInstances. Recovered a ready instance with no jobs and a version mismatch. Instance ID = {0}, Port = {1}, Version = {2}, ApplicationName = {3}, BucketName = {4}, Expected Version = {5}, Expected ApplicationName = {6}, Expected BucketName = {7}", instance.Id, instance.Port, instance.Version, - GridServerVersion + instance.ApplicationName, + instance.BucketName, + GridServerVersion, + ApplicationName, + BucketName ); throw new Exception( string.Format( - "Version mismatch for jobless instance. Instance ID = {0}, Port = {1}, Version = {2}, Expected Version = {3}", + "Version mismatch for jobless instance. Instance ID = {0}, Port = {1}, Version = {2}, ApplicationName = {3}, BucketName = {4}, Expected Version = {5}, Expected ApplicationName = {6}, Expected BucketName = {7}", instance.Id, instance.Port, instance.Version, - GridServerVersion + instance.ApplicationName, + instance.BucketName, + GridServerVersion, + ApplicationName, + BucketName ) ); } @@ -833,10 +1091,12 @@ private void RecoverRunningInstances() ReadyInstances.Add(instance); Logger.Information( - "RecoverRunningInstances. Recovered a ready instance. Instance ID = {0}, Port = {1}, Version = {2}", + "RecoverRunningInstances. Recovered a ready instance. Instance ID = {0}, Port = {1}, Version = {2}, ApplicationName = {3}, BucketName = {4}", instance.Id, instance.Port, - instance.Version + instance.Version, + instance.ApplicationName, + instance.BucketName ); return; diff --git a/lib/grid/process-management-core/Implementation/ResourceAllocationTracker.cs b/lib/grid/process-management-core/Implementation/ResourceAllocationTracker.cs index 503e8dc8..5b80d73f 100644 --- a/lib/grid/process-management-core/Implementation/ResourceAllocationTracker.cs +++ b/lib/grid/process-management-core/Implementation/ResourceAllocationTracker.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; diff --git a/lib/grid/process-management-core/Implementation/SettingsFileWriter.cs b/lib/grid/process-management-core/Implementation/SettingsFileWriter.cs new file mode 100644 index 00000000..e9e42b28 --- /dev/null +++ b/lib/grid/process-management-core/Implementation/SettingsFileWriter.cs @@ -0,0 +1,76 @@ +namespace Grid.ProcessManagement.Core; + +using System; +using System.IO; +using System.Text; +using System.Threading; + +using Newtonsoft.Json; + +using Logging; +using ClientSettings.Client; + +/// +public class SettingsFileWriter : ISettingsFileWriter +{ + + private const int _MaxAttempts = 3; + private const int _SleepIntervalMilliseconds = 50; + + private readonly ILogger _Logger; + + /// + /// Constructs a new instance of + /// + /// The + /// cannot be null. + public SettingsFileWriter(ILogger logger) + { + _Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public bool WriteSettingsFile(string filePath, ClientApplicationSettingsResponse rccApplicationSettings) + { + var fileContents = JsonConvert.SerializeObject(rccApplicationSettings); + + for (int i = 0; i < _MaxAttempts; i++) + { + try + { + _Logger.Information("WriteSettingsFile. Attempting to write settings file to {0}. Attempt #{1}", filePath, i + 1); + _Logger.Verbose("WriteSettingsFile. FileContents: {0}", fileContents); + + TryWriteSettingsFile(filePath, fileContents, filePath + ".tmp"); + + return true; + } + catch (Exception ex) + { + _Logger.Error("WriteSettingsFile. Error: {0}", ex); + } + + Thread.Sleep(_SleepIntervalMilliseconds); + } + + return false; + } + + private void TryWriteSettingsFile(string filePath, string fileContents, string tempFilePath) + { + if (File.Exists(filePath)) + { + _Logger.Debug("TryWriteSettingsFile. {0} already exists. Attempting to write fileContents to {1}", filePath, tempFilePath); + File.WriteAllText(tempFilePath, fileContents, Encoding.ASCII); + + _Logger.Debug("TryWriteSettingsFile. Attempting to replace {0} with {1}", filePath, tempFilePath); + File.Replace(tempFilePath, filePath, null, true); + + return; + } + + _Logger.Debug("TryWriteSettingsFile. {0} does not exist. Attempting to write fileContents to {1}", filePath, filePath); + + File.WriteAllText(filePath, fileContents, Encoding.ASCII); + } +} diff --git a/lib/grid/process-management-core/Interfaces/IGridServerInstance.cs b/lib/grid/process-management-core/Interfaces/IGridServerInstance.cs index a3a4efad..d04d3e8e 100644 --- a/lib/grid/process-management-core/Interfaces/IGridServerInstance.cs +++ b/lib/grid/process-management-core/Interfaces/IGridServerInstance.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; using System.Diagnostics; @@ -56,19 +56,19 @@ public interface IGridServerInstance : IDisposable int Port { get; } /// - /// The RCC version. + /// The Grid Server version. /// string Version { get; } /// /// The application settings name. /// - //string ApplicationName { get; } + string ApplicationName { get; } /// /// The application bucket name. /// - //string BucketName { get; } + string BucketName { get; } /// /// Get the SOAP interface for this Grid Server Instance. @@ -78,7 +78,7 @@ public interface IGridServerInstance : IDisposable GridServerServiceSoap GetSoapInterface(int timeoutInMilliseconds); /// - /// Start the RCC instance. + /// Start the Grid Server instance. /// /// True if the instance was started. bool Start(); @@ -92,7 +92,7 @@ public interface IGridServerInstance : IDisposable void WaitForServiceToBecomeAvailable(bool forceTry, Stopwatch stopwatch); /// - /// Update the resource limits for this RCC instance. + /// Update the resource limits for this Grid Server instance. /// /// The new max cores. /// The new max threads. diff --git a/lib/grid/process-management-core/Interfaces/IJob.cs b/lib/grid/process-management-core/Interfaces/IJob.cs index 3db8d939..da06d6c3 100644 --- a/lib/grid/process-management-core/Interfaces/IJob.cs +++ b/lib/grid/process-management-core/Interfaces/IJob.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Represents a Grid Server Job. diff --git a/lib/grid/process-management-core/Interfaces/IJobManagerSettings.cs b/lib/grid/process-management-core/Interfaces/IJobManagerSettings.cs index a4f79fd0..c9b9b99a 100644 --- a/lib/grid/process-management-core/Interfaces/IJobManagerSettings.cs +++ b/lib/grid/process-management-core/Interfaces/IJobManagerSettings.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; using System; @@ -27,6 +27,7 @@ public interface IJobManagerSettings /// int ReadyInstancesToKeepInReserve { get; } + /// /// The maximum amount of start attempts for an Grid Server instance. /// @@ -36,4 +37,59 @@ public interface IJobManagerSettings /// The maximum amount of time to wait for the Grid Server SOAP port to become available. /// TimeSpan GridServerWaitForTcpSleepInterval { get; } + + /// + /// The Grid Server application settings name. + /// + string GridServerSettingsApplicationName { get; } + + /// + /// The Grid Server applicatiom bucket name. + /// + string GridServerSettingsBucketName { get; } + + /// + /// The Grid Server application settings file path. + /// + string GridServerApplicationSettingsFilePath { get; } + + /// + /// The valid window in which to update application settings. + /// + TimeSpan GridServerApplicationSettingsValidWindow { get; } + + /// + /// Grid Server max threads. + /// + public int GridServerMaxThreads { get; } + + /// + /// Is Grid Server CPU allocation check enabled. + /// + public bool IsGridServerCpuAllocationCheckEnabled { get; } + + /// + /// Is Grid Server threads allocation check enabled? + /// + public bool IsGridServerThreadsAllocationCheckEnabled { get; } + + /// + /// Is Grid Server memory allocation check enabled? + /// + public bool IsGridServerMemoryAllocationCheckEnabled { get; } + + /// + /// Grid Server CPU over-allocation ratio. + /// + public double GridServerCpuOverAllocationRatio { get; } + + /// + /// Grid Server threads over-allocation ratio. + /// + public double GridServerThreadsOverAllocationRatio { get; } + + /// + /// Grid Server memory over-allocation ratio. + /// + public double GridServerMemoryOverAllocationRatio { get; } } diff --git a/lib/grid/process-management-core/Interfaces/ISettingsFileWriter.cs b/lib/grid/process-management-core/Interfaces/ISettingsFileWriter.cs new file mode 100644 index 00000000..14445ae3 --- /dev/null +++ b/lib/grid/process-management-core/Interfaces/ISettingsFileWriter.cs @@ -0,0 +1,17 @@ +namespace Grid.ProcessManagement.Core; + +using ClientSettings.Client; + +/// +/// Represents a class to write application settings. +/// +public interface ISettingsFileWriter +{ + /// + /// Write the settings file. + /// + /// The path of the settings file. + /// The application settings response. + /// True if it successfully wrote. + bool WriteSettingsFile(string filePath, ClientApplicationSettingsResponse rccApplicationSettings); +} diff --git a/lib/grid/process-management-core/Interfaces/IUnmanagedGridServerInstance.cs b/lib/grid/process-management-core/Interfaces/IUnmanagedGridServerInstance.cs index 1ba0bbba..a6859538 100644 --- a/lib/grid/process-management-core/Interfaces/IUnmanagedGridServerInstance.cs +++ b/lib/grid/process-management-core/Interfaces/IUnmanagedGridServerInstance.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Represents an unmanaged Grid Server Instance. @@ -11,7 +11,7 @@ public interface IUnmanagedGridServerInstance string Id { get; } /// - /// Kill the RCC instance. + /// Kill the Grid Server instance. /// void Kill(); } diff --git a/lib/grid/process-management-core/Jitter.cs b/lib/grid/process-management-core/Jitter.cs index 1dd0c216..5470b55d 100644 --- a/lib/grid/process-management-core/Jitter.cs +++ b/lib/grid/process-management-core/Jitter.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Represents Jitter used for deleting files. diff --git a/lib/grid/process-management-core/Models/GameJob.cs b/lib/grid/process-management-core/Models/GameJob.cs index a2f8d212..089e3c1d 100644 --- a/lib/grid/process-management-core/Models/GameJob.cs +++ b/lib/grid/process-management-core/Models/GameJob.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Implementation for an Grid Server game job. diff --git a/lib/grid/process-management-core/Models/GridServerResource.cs b/lib/grid/process-management-core/Models/GridServerResource.cs index fccbf052..75a76bc5 100644 --- a/lib/grid/process-management-core/Models/GridServerResource.cs +++ b/lib/grid/process-management-core/Models/GridServerResource.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Represents the resources for an Grid Server instance. diff --git a/lib/grid/process-management-core/Models/GridServerResourceJob.cs b/lib/grid/process-management-core/Models/GridServerResourceJob.cs index e9f3fca2..0163cee6 100644 --- a/lib/grid/process-management-core/Models/GridServerResourceJob.cs +++ b/lib/grid/process-management-core/Models/GridServerResourceJob.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// /// Represents a job to discover the Grid Server resources. diff --git a/lib/grid/process-management-core/Models/Job.cs b/lib/grid/process-management-core/Models/Job.cs index 0e0f074a..09b0e3aa 100644 --- a/lib/grid/process-management-core/Models/Job.cs +++ b/lib/grid/process-management-core/Models/Job.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement.Core; /// public class Job : IJob diff --git a/lib/grid/process-management-docker/Docker/Extensions/IContainerOperationsExtension.cs b/lib/grid/process-management-docker/Docker/Extensions/IContainerOperationsExtension.cs index 113f79d5..b5545e94 100644 --- a/lib/grid/process-management-docker/Docker/Extensions/IContainerOperationsExtension.cs +++ b/lib/grid/process-management-docker/Docker/Extensions/IContainerOperationsExtension.cs @@ -1,4 +1,7 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Net; @@ -8,9 +11,6 @@ using System.Threading.Tasks; using System.Collections.Generic; -using Docker.DotNet; -using Docker.DotNet.Models; - using Newtonsoft.Json; using Version = System.Version; diff --git a/lib/grid/process-management-docker/Docker/Implementation/DockerLogger.cs b/lib/grid/process-management-docker/Docker/Implementation/DockerLogger.cs index ebac775a..5de32ff1 100644 --- a/lib/grid/process-management-docker/Docker/Implementation/DockerLogger.cs +++ b/lib/grid/process-management-docker/Docker/Implementation/DockerLogger.cs @@ -1,8 +1,8 @@ -namespace Grid; +using Docker.DotNet.Models; -using System; +namespace Grid.ProcessManagement.Docker; -using Docker.DotNet.Models; +using System; using Logging; diff --git a/lib/grid/process-management-docker/Docker/Implementation/DockerOperationBase.cs b/lib/grid/process-management-docker/Docker/Implementation/DockerOperationBase.cs index 44875b65..8bf5271a 100644 --- a/lib/grid/process-management-docker/Docker/Implementation/DockerOperationBase.cs +++ b/lib/grid/process-management-docker/Docker/Implementation/DockerOperationBase.cs @@ -1,11 +1,11 @@ -namespace Grid; +using Docker.DotNet; + +namespace Grid.ProcessManagement.Docker; using System; using System.Diagnostics; using System.Threading.Tasks; -using Docker.DotNet; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Implementation/DockerSocketHttpClient.cs b/lib/grid/process-management-docker/Docker/Implementation/DockerSocketHttpClient.cs index a31ce66b..c197b201 100644 --- a/lib/grid/process-management-docker/Docker/Implementation/DockerSocketHttpClient.cs +++ b/lib/grid/process-management-docker/Docker/Implementation/DockerSocketHttpClient.cs @@ -1,4 +1,6 @@ -namespace Grid; +using Docker.DotNet; + +namespace Grid.ProcessManagement.Docker; using System; using System.Net; @@ -6,8 +8,6 @@ using System.Net.Http; using System.Net.Sockets; -using Docker.DotNet; - using Microsoft.Net.Http.Client; /// diff --git a/lib/grid/process-management-docker/Docker/Implementation/GridServerDockerAuthority.cs b/lib/grid/process-management-docker/Docker/Implementation/GridServerDockerAuthority.cs index 83b979af..238be29f 100644 --- a/lib/grid/process-management-docker/Docker/Implementation/GridServerDockerAuthority.cs +++ b/lib/grid/process-management-docker/Docker/Implementation/GridServerDockerAuthority.cs @@ -1,15 +1,18 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Threading; using System.Threading.Tasks; -using Docker.DotNet; -using Docker.DotNet.Models; - using Newtonsoft.Json; using Logging; +using Diagnostics; + +using Core; /// /// Represents the default docker authority for Grid Server. @@ -42,6 +45,7 @@ public class GridServerDockerAuthority private readonly CreateContainerOperation _CreateContainerOperation; private readonly StartContainerOperation _StartContainerOperation; private readonly RemoveContainerOperation _RemoveContainerOperation; + private readonly StopContainerOperation _StopContainerOperation; private readonly KillContainerOperation _KillContainerOperation; private readonly UpdateContainerOperation _UpdateContainerOperation; @@ -67,10 +71,11 @@ internal GridServerDockerAuthority(ILogger logger, DockerClient dockerClient, IG _CreateContainerOperation = new CreateContainerOperation(_Logger, dockerClient); _StartContainerOperation = new StartContainerOperation(_Logger, dockerClient); _RemoveContainerOperation = new RemoveContainerOperation(_Logger, dockerClient, _GridServerSettings); + _StopContainerOperation = new StopContainerOperation(_Logger, dockerClient, _GridServerSettings); _KillContainerOperation = new KillContainerOperation(_Logger, dockerClient); _UpdateContainerOperation = new UpdateContainerOperation(_Logger, dockerClient, DockerSocketHttpClient.CreateClient(dockerClient)); - if (serverInfo != null && serverInfo.PhysicalCoreCount != 0) + if (serverInfo.PhysicalCoreCount != 0) { _PhysicalCoreToLogicalCoreRatio = serverInfo.LogicalCoreCount / serverInfo.PhysicalCoreCount; @@ -157,6 +162,44 @@ internal async Task CreateContainerAsync(CreateContainerParameters creat internal async Task StartContainerAsync(string containerName) => await _StartContainerOperation.ExecuteAsync(containerName); + /// + /// Stop a container with retries. + /// + /// The ID of the container + /// An awaitable task. + internal async Task StopContainerWithRetriesAsync(string containerId) + { + int maxAttemptsToWaitForContainerExit = _GridServerSettings.MaxAttemptsToWaitForContainerExit; + var sleepInterval = _GridServerSettings.ContainerStopSleepIntervalMilliseconds.HasValue + ? TimeSpan.FromMilliseconds(_GridServerSettings.ContainerStopSleepIntervalMilliseconds.Value) + : _TimeToSleepBetweenStopContainerAttempts; + + for (int attempt = 0; attempt < maxAttemptsToWaitForContainerExit; ++attempt) + { + if (await StopContainerAsync(containerId).ConfigureAwait(false)) + break; + + _Logger.Warning( + "StopContainerWithRetriesAsync: containerStopped FAILED for {0} after {1} attempts; sleeping for {2} between retries", + containerId, + attempt, + sleepInterval + ); + + if (attempt == maxAttemptsToWaitForContainerExit - 1) + await _KillContainerOperation.ExecuteAsync(containerId).ConfigureAwait(false); + else + await Task.Delay(sleepInterval); + } + } + + /// + /// Stop a container. + /// + /// The ID of the container + /// True if the container stopped. + internal async Task StopContainerAsync(string containerId) => await _StopContainerOperation.ExecuteAsync(containerId); + /// /// Remove a container with retries. /// diff --git a/lib/grid/process-management-docker/Docker/Interfaces/IGridServerSettings.cs b/lib/grid/process-management-docker/Docker/Interfaces/IGridServerDockerSettings.cs similarity index 66% rename from lib/grid/process-management-docker/Docker/Interfaces/IGridServerSettings.cs rename to lib/grid/process-management-docker/Docker/Interfaces/IGridServerDockerSettings.cs index 2b7d8397..07b60bf5 100644 --- a/lib/grid/process-management-docker/Docker/Interfaces/IGridServerSettings.cs +++ b/lib/grid/process-management-docker/Docker/Interfaces/IGridServerDockerSettings.cs @@ -1,18 +1,30 @@ -using System; +namespace Grid.ProcessManagement.Docker; + +using System; using System.Collections.Generic; -namespace Grid; +using Core; /// /// Represents the Grid Server settings. /// public interface IGridServerDockerSettings : IJobManagerSettings { + /// + /// Gets the base url for Grid Server + /// + string BaseUrl { get; } + /// /// The name of the container. /// string GridServerImageName { get; } + /// + /// The Grid Server container tag. + /// + string GridServerImageTag { get; } + /// /// Docker registry username. /// @@ -49,14 +61,56 @@ public interface IGridServerDockerSettings : IJobManagerSettings TimeSpan MaxDelayBeforeFetchingNewGridServerContainer { get; } /// - /// The directory where shared Grid Server logs are stored. + /// The directory where shared Grid Server Service logs are stored. /// string GridServerSharedDirectoryLogs { get; } /// - /// The directory where shared Grid Server internal scripts are stored. + /// The name of the file to cache app settings in. + /// + string GridServerApplicationSettingsFileName { get; } + + /// + /// The directory where shared app data is stored. + /// + string GridServerSharedDirectoryAppData { get; } + + /// + /// Additonal volume mappings for Grid Server containers. + /// + string[] GridServerAdditionalVolumeMappings { get; } + +#if !GRID_SERVER_FOR_WINE + /// + /// The directory where shared Grid Server cache is stored. + /// + string GridServerSharedDirectoryCache { get; } + + /// + /// The directory where shared temp files are stored. /// - string GridServerSharedDirectoryInternalScripts { get; } + string GridServerSharedDirectoryTemp { get; } + + /// + /// Is Grid Server's UDP port range enabled? + /// + bool IsGridServerUdpLimitedPortRangeEnabled { get; } + + /// + /// Starting port for Grid Server containers. + /// + int? GridServerContainerStartingPort { get; } + + /// + /// Ending port for Grid Server containers. + /// + int? GridServerContainerEndingPort { get; } + + /// + /// Is pass UDP port range to Grid Server enabled? + /// + bool IsPassUdpPortRangeToGridServerEnabled { get; } +#endif /// /// The amount of cores to reserve per Grid Server instance. @@ -68,11 +122,6 @@ public interface IGridServerDockerSettings : IJobManagerSettings /// long GridServerMaxMemoryInBytes { get; } - /// - /// The maximum amount of Grid Server threads. - /// - int GridServerMaxThreads { get; } - /// /// Envrionment variables to be passed into containers. /// @@ -83,11 +132,6 @@ public interface IGridServerDockerSettings : IJobManagerSettings /// string HttpAccessKey { get; } - /// - /// The Grid Server container tag. - /// - string GridServerImageTag { get; } - /// /// Primary DNS server for Grid Server containers. /// diff --git a/lib/grid/process-management-docker/Docker/Operations/CheckImageOperation.cs b/lib/grid/process-management-docker/Docker/Operations/CheckImageOperation.cs index 0d4c72b0..355dacd0 100644 --- a/lib/grid/process-management-docker/Docker/Operations/CheckImageOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/CheckImageOperation.cs @@ -1,11 +1,11 @@ -namespace Grid; +using Docker.DotNet; + +namespace Grid.ProcessManagement.Docker; using System; using System.Threading; using System.Threading.Tasks; -using Docker.DotNet; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Operations/CreateContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/CreateContainerOperation.cs index 64e0ae30..aff7e4a2 100644 --- a/lib/grid/process-management-docker/Docker/Operations/CreateContainerOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/CreateContainerOperation.cs @@ -1,10 +1,11 @@ -namespace Grid; - -using System.Threading.Tasks; - + using Docker.DotNet; using Docker.DotNet.Models; +namespace Grid.ProcessManagement.Docker; + +using System.Threading.Tasks; + using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Operations/CreateImageOperation.cs b/lib/grid/process-management-docker/Docker/Operations/CreateImageOperation.cs index 7f7698f8..9a845abd 100644 --- a/lib/grid/process-management-docker/Docker/Operations/CreateImageOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/CreateImageOperation.cs @@ -1,12 +1,12 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Threading; using System.Threading.Tasks; -using Docker.DotNet; -using Docker.DotNet.Models; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Operations/HasExitedOperation.cs b/lib/grid/process-management-docker/Docker/Operations/HasExitedOperation.cs index 4d80c257..6c242811 100644 --- a/lib/grid/process-management-docker/Docker/Operations/HasExitedOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/HasExitedOperation.cs @@ -1,11 +1,11 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System.Threading.Tasks; using System.Collections.Generic; -using Docker.DotNet; -using Docker.DotNet.Models; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Operations/KillContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/KillContainerOperation.cs index 1c468a10..991b2c46 100644 --- a/lib/grid/process-management-docker/Docker/Operations/KillContainerOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/KillContainerOperation.cs @@ -1,10 +1,9 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; -using System; -using System.Threading.Tasks; +namespace Grid.ProcessManagement.Docker; -using Docker.DotNet; -using Docker.DotNet.Models; +using System.Threading.Tasks; using Logging; diff --git a/lib/grid/process-management-docker/Docker/Operations/RemoveContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/RemoveContainerOperation.cs index b9b37136..c6948216 100644 --- a/lib/grid/process-management-docker/Docker/Operations/RemoveContainerOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/RemoveContainerOperation.cs @@ -1,11 +1,11 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Threading.Tasks; -using Docker.DotNet; -using Docker.DotNet.Models; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Operations/StartContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/StartContainerOperation.cs index 55ebcfcc..9c3f87fc 100644 --- a/lib/grid/process-management-docker/Docker/Operations/StartContainerOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/StartContainerOperation.cs @@ -1,9 +1,9 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; -using System.Threading.Tasks; +namespace Grid.ProcessManagement.Docker; -using Docker.DotNet; -using Docker.DotNet.Models; +using System.Threading.Tasks; using Logging; diff --git a/lib/grid/process-management-docker/Docker/Operations/StopContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/StopContainerOperation.cs new file mode 100644 index 00000000..0077a831 --- /dev/null +++ b/lib/grid/process-management-docker/Docker/Operations/StopContainerOperation.cs @@ -0,0 +1,43 @@ +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; + +using System; +using System.Threading.Tasks; + +using Logging; + +/// +/// Represents the kill container operation. +/// +internal class StopContainerOperation : DockerOperationBase +{ + private readonly IGridServerDockerSettings _GridServerSettings; + + /// + /// Construct a new instance of + /// + /// The + /// The + /// The + /// cannot be null. + public StopContainerOperation(ILogger logger, IDockerClient dockerClient, IGridServerDockerSettings gridServerSettings) + : base(logger, dockerClient, "StopContainer") + { + _GridServerSettings = gridServerSettings ?? throw new ArgumentNullException(nameof(gridServerSettings)); + } + + /// + protected async override Task<(bool, bool)> DoExecuteAsync(string containerId) + { + var parameters = new ContainerStopParameters + { + WaitBeforeKillSeconds = (uint)_GridServerSettings.ContainerStopWaitBeforeKillInSeconds + }; + + await DockerClient.Containers.StopContainerAsync(containerId, parameters).ConfigureAwait(false); + + return (true, true); + } +} diff --git a/lib/grid/process-management-docker/Docker/Operations/UpdateContainerOperation.cs b/lib/grid/process-management-docker/Docker/Operations/UpdateContainerOperation.cs index 306ae985..58049a2f 100644 --- a/lib/grid/process-management-docker/Docker/Operations/UpdateContainerOperation.cs +++ b/lib/grid/process-management-docker/Docker/Operations/UpdateContainerOperation.cs @@ -1,12 +1,13 @@ -namespace Grid; + +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Net.Http; using System.Threading.Tasks; -using Docker.DotNet; -using Docker.DotNet.Models; - using Logging; /// diff --git a/lib/grid/process-management-docker/Docker/Parameters/GridServerContainerUpdateParameters.cs b/lib/grid/process-management-docker/Docker/Parameters/GridServerContainerUpdateParameters.cs index 2baa3d4b..17d84b1f 100644 --- a/lib/grid/process-management-docker/Docker/Parameters/GridServerContainerUpdateParameters.cs +++ b/lib/grid/process-management-docker/Docker/Parameters/GridServerContainerUpdateParameters.cs @@ -1,7 +1,6 @@ -namespace Grid; - -using Docker.DotNet.Models; +using Docker.DotNet.Models; +namespace Grid.ProcessManagement.Docker; /// public class GridServerContainerUpdateParameters : ContainerUpdateParameters diff --git a/lib/grid/process-management-docker/Grid.ProcessManagement.Docker.csproj b/lib/grid/process-management-docker/Grid.ProcessManagement.Docker.csproj index 33c800ee..98337dad 100644 --- a/lib/grid/process-management-docker/Grid.ProcessManagement.Docker.csproj +++ b/lib/grid/process-management-docker/Grid.ProcessManagement.Docker.csproj @@ -1,7 +1,9 @@ Shared library for interaction with Docker based grid server processes! - Grid + + Grid.ProcessManagement.Docker + $(DefineConstants);GRID_SERVER_FOR_WINE @@ -10,6 +12,8 @@ + + diff --git a/lib/grid/process-management-docker/Implementation/GridServerDockerContainer.cs b/lib/grid/process-management-docker/Implementation/GridServerDockerContainer.cs index 4defc824..4507a447 100644 --- a/lib/grid/process-management-docker/Implementation/GridServerDockerContainer.cs +++ b/lib/grid/process-management-docker/Implementation/GridServerDockerContainer.cs @@ -1,15 +1,21 @@ -namespace Grid; +#if !GRID_SERVER_FOR_WINE + +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; +using System.Net; +using System.Runtime; using System.Diagnostics; using System.Threading.Tasks; using System.Collections.Generic; -using Docker.DotNet.Models; - using Logging; -using Commands; +using Grid; +using Core; +using Grid.Commands; /// /// Represents the Docker implementation of @@ -21,23 +27,43 @@ public sealed class GridServerDockerContainer : GridServerInstanceBase /// public const string PortLabel = "port"; + /// + /// The Grid Server version label. + /// + public const string GridServerVersionLabel = "rcc_version"; + + /// + /// The Grid Server application name label. + /// + public const string GridServerApplicationNameLabel = "rcc_application_name"; + + /// + /// The Grid Server bucket name label. + /// + public const string GridServerBucketNameLabel = "rcc_bucket_name"; + /// /// The image name label. /// public const string ImageNameLabel = "image_name"; /// - /// The Grid Server version label. + /// The Grid Server appdata directory. /// - public const string GridServerVersionLabel = "grid_server_version"; + public const string GridServerAppDataPath = "/opt/roblox/appdata/RCCService"; - private const string _GridServerLogPath = "/opt/grid/.wine/dosdevices/c:/users/root/AppData/Local/Roblox"; - private const string _GridServerInternalScriptsPath = "/opt/grid/internalscripts"; - private const string _X11SocketPath = "/tmp/.X11-unix"; + private const string _GridServerLogPath = "/var/log/RCCService"; + private const string _GridServerCachePath = "/opt/roblox/cache/RCCService"; + private const string _GridServerTempPath = "/opt/roblox/tmp/RCCService"; + private const string _GridServerCoreDumpPath = "/var/tmp"; + private const string _DefaultGridServerContainerPortRange = "49152 65535"; + + private const int _MillisecondToSecond = 1000; private readonly GridServerDockerAuthority _DockerAuthority; private readonly IGridServerDockerSettings _GridServerSettings; private readonly string _GridServerImageName; + private readonly string _ApplicationSettingsPath; private bool _Disposed; @@ -67,18 +93,22 @@ public sealed class GridServerDockerContainer : GridServerInstanceBase /// The port /// The version. /// The + /// The Grid Server application name. + /// The Grid Server bucket name. /// The - /// The Grid Server image name. + /// The Grid Server Service image name. /// must be > 0 internal GridServerDockerContainer( ILogger logger, int port, string version, IGridServerDockerSettings gridServerSettings, + string applicationName, + string bucketName, GridServerDockerAuthority dockerAuthority, string gridServerImageName ) - : base(logger, version, port, gridServerSettings) + : base(logger, version, port, gridServerSettings, applicationName, bucketName) { if (port < 1) throw new ArgumentException("Port must be > 0", PortLabel); @@ -86,9 +116,17 @@ string gridServerImageName _DockerAuthority = dockerAuthority; _GridServerImageName = gridServerImageName; - ContainerName = string.Format("grid-server-{0}-gr", Guid.NewGuid()); + if (!string.IsNullOrEmpty(gridServerSettings.GridServerApplicationSettingsFileName)) + _ApplicationSettingsPath = $"{GridServerAppDataPath}/{gridServerSettings.GridServerApplicationSettingsFileName}"; + + ContainerName = string.Format("grid-Server-{0}-gr", Guid.NewGuid()); - Logger.Information("Constructing GridServerDockerContainer, ContainerName: {0}, Port: {1}, Version: {2}", ContainerName, Port, Version); + Logger.Information( + "Constructing GridServerDockerContainer, ContainerName = {0}, Port = {1}, Version = {2}", + ContainerName, + Port, + Version + ); } /// @@ -110,12 +148,9 @@ private async Task StartAsync() if (string.IsNullOrEmpty(ContainerID)) { Logger.Error( - "Failed to Create Container", - new - { - ContainerName, - Version - } + "Failed to Create Container, ContainerName = {0}, Version = {1}", + ContainerName, + Version ); return false; @@ -127,17 +162,18 @@ private async Task StartAsync() private bool WaitForContainerStart() { var sw = Stopwatch.StartNew(); + try { WaitForServiceToBecomeAvailable(false, sw); - InitializeHA(); + InitializeHighAvailability(); return true; } catch (Exception ex) { var format = string.Format( - "Error waiting for Grid Server Service to become available. Container Name: {0}, Version: {1}. Exception: {2}", + "Error waiting for Grid Server Service Service to become available. Container Name: {0}, Version: {1}. Exception: {2}", ContainerName, Version, ex @@ -149,11 +185,10 @@ private bool WaitForContainerStart() } } - private void InitializeHA() + private void InitializeHighAvailability() { - using var soap = GetSoapInterface(10000); + using var soap = GetSoapInterface(60 * _MillisecondToSecond); -#if !PRE_JSON_EXECUTION var command = new ExecuteScriptCommand( new("highavailability", new Dictionary()) ); @@ -164,23 +199,15 @@ private void InitializeHA() }; soap.BatchJobEx(job, command); - -#else - var lua = ScriptProvider.GetScript("HighAvailability"); - - var job = new Client.Job - { - id = Guid.NewGuid().ToString(), - expirationInSeconds = 10000 - }; - - soap.BatchJobEx(job, lua); -#endif } private List GetContainerMounts() { - return new List() + var additionalMounts = new List(); + foreach (var mount in _GridServerSettings.GridServerAdditionalVolumeMappings) + additionalMounts.Add(BuildMountFromString(mount)); + + var mounts = new List() { new() { @@ -192,35 +219,67 @@ private List GetContainerMounts() new() { Type = "bind", - Source = GetSource(_GridServerSettings.GridServerSharedDirectoryInternalScripts), + Source = GetSource(_GridServerSettings.GridServerSharedDirectoryCache), + ReadOnly = false, + Target = _GridServerCachePath + }, + new() + { + Type = "bind", + Source = GetSource(_GridServerSettings.GridServerSharedDirectoryTemp), ReadOnly = false, - Target = _GridServerInternalScriptsPath + Target = _GridServerTempPath }, new() { Type = "bind", - Source = _X11SocketPath, - ReadOnly = true, - Target = _X11SocketPath + Source = GetSource(_GridServerSettings.GridServerSharedDirectoryAppData), + ReadOnly = false, + Target = GridServerAppDataPath + }, + new() + { + Type = "bind", + Source = _GridServerCoreDumpPath, + ReadOnly = false, + Target = _GridServerCoreDumpPath } }; + mounts.AddRange(additionalMounts); + + return mounts; + string GetSource(string settingsValue) => string.IsNullOrEmpty(_GridServerSettings.MountPathOverride) ? settingsValue : _GridServerSettings.MountPathOverride; + + Mount BuildMountFromString(string mountString) + { + var parts = mountString.Split(':'); + if (parts.Length != 2) + throw new Exception($"Invalid mount string: {mountString}. Expected format: source:target"); + + var isReadOnly = parts.Length >= 3 && parts[2].Equals("ro", StringComparison.OrdinalIgnoreCase); + + return new Mount + { + Type = "bind", + Source = parts[0], + Target = parts[1], + ReadOnly = isReadOnly + }; + } } private List GetEnvironmentVariables() { var environmentVariables = new List { - $"PORT={Port}", - $"SETTINGS_KEY={_GridServerSettings.GridServerSettingsKey}" + $"RCC_PORT={Port}", + $"RCC_HTTP_ACCESS_KEY={_GridServerSettings.HttpAccessKey}" }; - if (Environment.GetEnvironmentVariable("DISPLAY") != null) - environmentVariables.Add($"DISPLAY={Environment.GetEnvironmentVariable("DISPLAY")}"); - if (_GridServerSettings.GridServerMaxThreads > 0) environmentVariables.Add($"MAXIMUM_THREADS={_GridServerSettings.GridServerMaxThreads}"); @@ -228,8 +287,14 @@ private List GetEnvironmentVariables() if (maxMemory > 0) environmentVariables.Add($"MAXIMUM_MEMORY={maxMemory}"); - if (!string.IsNullOrWhiteSpace(_GridServerSettings.HttpAccessKey)) - environmentVariables.Add($"HTTP_ACCESS_KEY={_GridServerSettings.HttpAccessKey}"); + if (!string.IsNullOrWhiteSpace(_GridServerSettings.GridServerSettingsKey)) + environmentVariables.Add($"RCC_SETTINGS_KEY={_GridServerSettings.GridServerSettingsKey}"); + + if (_GridServerSettings.IsPassUdpPortRangeToGridServerEnabled && _GridServerSettings.GridServerContainerStartingPort != null) + environmentVariables.Add($"UDP_PORT_LOW={_GridServerSettings.GridServerContainerStartingPort.Value}"); + + if (_GridServerSettings.IsPassUdpPortRangeToGridServerEnabled && _GridServerSettings.GridServerContainerEndingPort != null) + environmentVariables.Add($"UDP_PORT_HIGH={_GridServerSettings.GridServerContainerEndingPort.Value}"); if (_GridServerSettings.GridServerEnvironmentVariables != null) foreach (var environmentVariable in _GridServerSettings.GridServerEnvironmentVariables) @@ -240,13 +305,60 @@ private List GetEnvironmentVariables() private CreateContainerParameters GetCreateContainerParameters() { - if (string.IsNullOrEmpty(_GridServerSettings.GridServerSettingsKey)) - throw new Exception("Unable to start a new Grid Server container, GridServerSettingsKey is set to null or is empty"); + if (string.IsNullOrEmpty(_GridServerSettings.HttpAccessKey)) + throw new Exception("Unable to start a new Grid Server container, HttpAccessKey is set to null or is empty"); + + if (string.IsNullOrEmpty(_GridServerSettings.BaseUrl)) + throw new Exception("Unable to start a new Grid Server container, BaseUrl is set to null or is empty"); + + var containerParameters = new List() + { + "--port", Port.ToString(), + "--baseUrl", _GridServerSettings.BaseUrl + }; + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerApplicationName)) + containerParameters.AddRange(new[] { "--applicationName", _GridServerSettings.GridServerApplicationName }); + + if (!string.IsNullOrEmpty(_ApplicationSettingsPath)) + { + Logger.Information("GetCreateContainerParameters. Adding --settingsFile command line option: {0}", _ApplicationSettingsPath); + + containerParameters.AddRange(new[] { "--settingsFile", _ApplicationSettingsPath }); + } + + var sysctls = new Dictionary(); + if (_GridServerSettings.IsGridServerUdpLimitedPortRangeEnabled) + if (_GridServerSettings.GridServerContainerStartingPort != null && _GridServerSettings.GridServerContainerEndingPort != null) + if (_GridServerSettings.IsPassUdpPortRangeToGridServerEnabled) + { + containerParameters.AddRange(new[] { "--udpPortLow", _GridServerSettings.GridServerContainerStartingPort.Value.ToString() }); + containerParameters.AddRange(new[] { "--udpPortHigh", _GridServerSettings.GridServerContainerEndingPort.Value.ToString() }); + + sysctls.Add("net.ipv4.ip_local_port_range", _DefaultGridServerContainerPortRange); + } + else + sysctls.Add("net.ipv4.ip_local_port_range", $"{_GridServerSettings.GridServerContainerStartingPort} {_GridServerSettings.GridServerContainerEndingPort}"); + else + { + if (_GridServerSettings.GridServerContainerStartingPort != null || _GridServerSettings.GridServerContainerEndingPort != null) + throw new Exception( + string.Format( + "ServerVIP data for server {0} is invalid. StartingPort:{1}, EndingPort:{2}", + Dns.GetHostName(), + _GridServerSettings.GridServerContainerStartingPort, + _GridServerSettings.GridServerContainerEndingPort + ) + ); + + sysctls.Add("net.ipv4.ip_local_port_range", _DefaultGridServerContainerPortRange); + } var parameters = new CreateContainerParameters { Image = $"{_GridServerImageName}:{Version}", Name = ContainerName, + Cmd = containerParameters, Env = GetEnvironmentVariables() }; @@ -254,12 +366,15 @@ private CreateContainerParameters GetCreateContainerParameters() { [PortLabel] = Port.ToString(), [GridServerVersionLabel] = Version, - [ImageNameLabel] = _GridServerImageName + [ImageNameLabel] = _GridServerImageName, + [GridServerApplicationNameLabel] = ApplicationName, + [GridServerBucketNameLabel] = BucketName }; parameters.Labels = labels; parameters.HostConfig = new HostConfig { + CapAdd = new List { "SYS_PTRACE" }, Mounts = GetContainerMounts(), Memory = _GridServerSettings.GridServerMaxMemoryInBytes, Ulimits = new List @@ -304,9 +419,11 @@ public override void Dispose() { if (_Disposed) return; - _DockerAuthority.KillContainerAsync(ContainerID).Wait(); + _DockerAuthority.StopContainerWithRetriesAsync(ContainerID).Wait(); _DockerAuthority.RemoveContainerWithRetriesAsync(ContainerID).Wait(); _Disposed = true; } } + +#endif diff --git a/lib/grid/process-management-docker/Implementation/JobManager.cs b/lib/grid/process-management-docker/Implementation/JobManager.cs index cdaea0f3..a94a8f4b 100644 --- a/lib/grid/process-management-docker/Implementation/JobManager.cs +++ b/lib/grid/process-management-docker/Implementation/JobManager.cs @@ -1,16 +1,21 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; using System; using System.Linq; using System.Threading; using System.Collections.Generic; -using Docker.DotNet; -using Docker.DotNet.Models; - using Random; using Logging; +using Core; +using Diagnostics; +using PortManagement; +using ClientSettings.Client; + /// /// Represents the Docker implementation for /// @@ -29,25 +34,27 @@ public class DockerJobManager : JobManagerBase /// /// The /// The - /// The + /// The /// The + /// The /// The /// The /// - /// - cannot be null. + /// - cannot be null. /// - cannot be null. /// public DockerJobManager( ILogger logger, IPortAllocator portAllocator, - IGridServerDockerSettings rccSettings, + IGridServerDockerSettings gridServerSettings, IRandom random, + IClientSettingsClient clientSettingsClient, IServerInfo serverInfo = null, ResourceAllocationTracker resourceAllocationTracker = null ) - : base(logger, rccSettings, portAllocator, resourceAllocationTracker) + : base(logger, gridServerSettings, portAllocator, clientSettingsClient, resourceAllocationTracker) { - _GridServerSettings = rccSettings ?? throw new ArgumentNullException(nameof(rccSettings)); + _GridServerSettings = gridServerSettings ?? throw new ArgumentNullException(nameof(gridServerSettings)); _ActiveContainerFilter = new ContainersListParameters { @@ -61,11 +68,11 @@ public DockerJobManager( _Random = random ?? throw new ArgumentNullException(nameof(random)); _DockerClient = CreateDockerClient(); - _DockerAuthority = new GridServerDockerAuthority(Logger, _DockerClient, _GridServerSettings, serverInfo); + _DockerAuthority = new GridServerDockerAuthority(Logger, _DockerClient, _GridServerSettings, serverInfo ?? ServerInfo.GetInstance()); } /// - public override int GetInstanceCount() => _DockerClient.Containers.ListContainersAsync(_ActiveContainerFilter, default).Result.Count; + public override int GetInstanceCount() => _DockerClient.Containers.ListContainersAsync(_ActiveContainerFilter).Result.Count; /// public override string GetGridServerInstanceId(string jobId) @@ -124,7 +131,7 @@ protected override bool OnGridServerVersionChange(string newGridServerVersion, b /// protected override IGridServerInstance CreateNewGridServerInstance(int port) - => new GridServerDockerContainer(Logger, port, GridServerVersion, _GridServerSettings, _DockerAuthority, _GridServerSettings.GridServerImageName); + => new GridServerDockerContainer(Logger, port, GridServerVersion, _GridServerSettings, ApplicationName, BucketName, _DockerAuthority, _GridServerSettings.GridServerImageName); /// protected override IReadOnlyCollection FindUnexpectedExitGameJobs() @@ -153,7 +160,7 @@ protected override IReadOnlyCollection GetRunningG /// protected override IGridServerInstance RecoverGridServerInstance(IUnmanagedGridServerInstance instance) { - if (!(instance is UnmanagedGridServerDockerContainer dockerContainer)) + if (instance is not UnmanagedGridServerDockerContainer dockerContainer) return null; if (!dockerContainer.Container.Labels.ContainsKey(GridServerDockerContainer.PortLabel) || @@ -164,19 +171,23 @@ protected override IGridServerInstance RecoverGridServerInstance(IUnmanagedGridS var containerName = dockerContainer.Container.Names[0].Trim('/'); var version = dockerContainer.Container.Labels[GridServerDockerContainer.GridServerVersionLabel]; var tcpPort = Convert.ToInt32(dockerContainer.Container.Labels[GridServerDockerContainer.PortLabel]); + dockerContainer.Container.Labels.TryGetValue(GridServerDockerContainer.GridServerApplicationNameLabel, out string applicationName); + dockerContainer.Container.Labels.TryGetValue(GridServerDockerContainer.GridServerBucketNameLabel, out string bucketName); - var config = _DockerClient.Containers.InspectContainerAsync(instance.Id, default).Result.HostConfig; + var config = _DockerClient.Containers.InspectContainerAsync(instance.Id).Result.HostConfig; long maximumMemoryInMegabytes = config.Memory / 1024 / 1024; long cpuperiod = config.CPUPeriod; long cpuquota = config.CPUQuota; double maximumCores = _DockerAuthority.CalculatePhysicalCores(cpuperiod, cpuquota); Logger.Information( - "Found a running GridServer container. Container ID = {0} with name {1} on TCP port: {2} with Grid Server Version: {3}, maximumCores: {4}, maximumMemoryInMegabytes: {5}", + "Found a running GridServer container. Container ID = {0} with name {1} on TCP port: {2} with Grid Server Version: {3}, applicationName: {4}, bucketName: {5}, maximumCores: {6}, maximumMemoryInMegabytes: {7}", dockerContainer.Container.ID, containerName, tcpPort, version, + applicationName, + bucketName, maximumCores, maximumMemoryInMegabytes ); @@ -186,6 +197,8 @@ protected override IGridServerInstance RecoverGridServerInstance(IUnmanagedGridS tcpPort, version, _GridServerSettings, + applicationName, + bucketName, _DockerAuthority, containerName ) @@ -206,7 +219,7 @@ private IReadOnlyCollection ListRunningContainers() { try { - return _DockerClient.Containers.ListContainersAsync(_ActiveContainerFilter, default) + return _DockerClient.Containers.ListContainersAsync(_ActiveContainerFilter) .Result .ToList() .AsReadOnly(); diff --git a/lib/grid/process-management-docker/Implementation/UnmanagedGridServerDockerContainer.cs b/lib/grid/process-management-docker/Implementation/UnmanagedGridServerDockerContainer.cs index f653eba0..0b9b5e1a 100644 --- a/lib/grid/process-management-docker/Implementation/UnmanagedGridServerDockerContainer.cs +++ b/lib/grid/process-management-docker/Implementation/UnmanagedGridServerDockerContainer.cs @@ -1,12 +1,14 @@ -namespace Grid; +using Docker.DotNet; +using Docker.DotNet.Models; -using System.Threading; +namespace Grid.ProcessManagement.Docker; -using Docker.DotNet; -using Docker.DotNet.Models; +using System.Threading; using Logging; +using Core; + /// /// Represents the implementation for Grid Server Docker containers. /// diff --git a/lib/grid/process-management-docker/Implementation/WineGridServerDockerContainer.cs b/lib/grid/process-management-docker/Implementation/WineGridServerDockerContainer.cs new file mode 100644 index 00000000..a3fe1cf8 --- /dev/null +++ b/lib/grid/process-management-docker/Implementation/WineGridServerDockerContainer.cs @@ -0,0 +1,372 @@ +#if GRID_SERVER_FOR_WINE + +using Docker.DotNet.Models; + +namespace Grid.ProcessManagement.Docker; + +using System; +using System.IO; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Collections.Generic; + +using Logging; + +using Grid; +using Core; +using Grid.Commands; + +/// +/// Represents the Wine Docker implementation of +/// +public sealed class GridServerDockerContainer : GridServerInstanceBase +{ + /// + /// The port label. + /// + public const string PortLabel = "port"; + + /// + /// The image name label. + /// + public const string ImageNameLabel = "image_name"; + + /// + /// The Grid Server application name label. + /// + public const string GridServerApplicationNameLabel = "rcc_application_name"; + + /// + /// The Grid Server bucket name label. + /// + public const string GridServerBucketNameLabel = "rcc_bucket_name"; + + /// + /// The Grid Server Service version label. + /// + public const string GridServerVersionLabel = "rcc_version"; + + /// + /// The Grid Server appdata directory. + /// + public const string GridServerClientSettingsPath = "/opt/roblox/rcc_service"; + + private const string _GridServerLogPath = "/opt/roblox/.wine/dosdevices/c:/users/root/AppData/Local/Roblox"; + private const string _X11SocketPath = "/tmp/.X11-unix"; + + private const int _MillisecondToSecond = 1000; + + private readonly GridServerDockerAuthority _DockerAuthority; + private readonly IGridServerDockerSettings _GridServerSettings; + private readonly string _GridServerImageName; + + private bool _Disposed; + + /// + public override bool HasExited => _DockerAuthority.HasContainerExited(ContainerName).Result; + + /// + public override string Id => ContainerID; + + /// + public override string Name => ContainerName; + + /// + /// The name of the container. + /// + internal string ContainerName { get; set; } + + /// + /// The ID of the container. + /// + internal string ContainerID { get; set; } + + /// + /// Construct a new instance of + /// + /// The + /// The port + /// The version. + /// The + /// The Grid Server application name. + /// The Grid Server bucket name. + /// The + /// The Grid Server Service image name. + /// must be > 0 + internal GridServerDockerContainer( + ILogger logger, + int port, + string version, + IGridServerDockerSettings gridServerSettings, + string applicationName, + string bucketName, + GridServerDockerAuthority dockerAuthority, + string gridServerImageName + ) + : base(logger, version, port, gridServerSettings, applicationName, bucketName) + { + if (port < 1) throw new ArgumentException("Port must be > 0", PortLabel); + + _GridServerSettings = gridServerSettings; + _DockerAuthority = dockerAuthority; + _GridServerImageName = gridServerImageName; + + ContainerName = string.Format("grid-server-{0}-gr", Guid.NewGuid()); + + Logger.Information( + "Constructing GridServerDockerContainer, ContainerName = {0}, Port = {1}, Version = {2}", + ContainerName, + Port, + Version + ); + } + + /// + public override bool Start() => StartAsync().Result; + + private async Task StartAsync() + { + if (!await _DockerAuthority.CheckImageAsync(_GridServerImageName, Version).ConfigureAwait(false)) + { + Logger.Information("Pulling container {0}:{1}", _GridServerImageName, Version); + + await _DockerAuthority.CreateImageAsync(_GridServerImageName, Version).ConfigureAwait(false); + } + + ContainerID = await _DockerAuthority.CreateContainerAsync(GetCreateContainerParameters()).ConfigureAwait(false); + + Logger.Information("Created a new Container successfully with ID: {0}", ContainerID); + + if (string.IsNullOrEmpty(ContainerID)) + { + Logger.Error( + "Failed to Create Container, ContainerName = {0}, Version = {1}", + ContainerName, + Version + ); + + return false; + } + + return await _DockerAuthority.StartContainerAsync(ContainerName).ConfigureAwait(false) && WaitForContainerStart(); + } + + private bool WaitForContainerStart() + { + var sw = Stopwatch.StartNew(); + try + { + WaitForServiceToBecomeAvailable(false, sw); + InitializeHighAvailability(); + + return true; + } + catch (Exception ex) + { + var format = string.Format( + "Error waiting for Grid Server Service Service to become available. Container Name: {0}, Version: {1}. Exception: {2}", + ContainerName, + Version, + ex + ); + + Logger.Error(format); + + throw new Exception(format); + } + } + + private void InitializeHighAvailability() + { + using var soap = GetSoapInterface(60 * _MillisecondToSecond); + + var command = new ExecuteScriptCommand( + new("highavailability", new Dictionary()) + ); + var job = new Grid.Client.Job + { + id = Guid.NewGuid().ToString(), + expirationInSeconds = 10000 + }; + + soap.BatchJobEx(job, command); + } + + private List GetContainerMounts() + { + var additionalMounts = new List(); + foreach (var mount in _GridServerSettings.GridServerAdditionalVolumeMappings) + additionalMounts.Add(BuildMountFromString(mount)); + + var mounts = new List() + { + new() + { + Type = "bind", + Source = GetSource(_GridServerSettings.GridServerSharedDirectoryLogs), + ReadOnly = false, + Target = _GridServerLogPath + }, + new() + { + Type = "bind", + Source = Path.Combine(_GridServerSettings.GridServerSharedDirectoryAppData, _GridServerSettings.GridServerApplicationSettingsFileName), + ReadOnly = false, + Target = $"{GridServerClientSettingsPath}/{_GridServerSettings.GridServerApplicationSettingsFileName}" + }, + new() + { + Type = "bind", + Source = _X11SocketPath, + ReadOnly = true, + Target = _X11SocketPath + } + }; + + mounts.AddRange(additionalMounts); + + return mounts; + + string GetSource(string settingsValue) => string.IsNullOrEmpty(_GridServerSettings.MountPathOverride) + ? settingsValue + : _GridServerSettings.MountPathOverride; + + Mount BuildMountFromString(string mountString) + { + var parts = mountString.Split(':'); + if (parts.Length != 2) + throw new Exception($"Invalid mount string: {mountString}. Expected format: source:target"); + + var isReadOnly = parts.Length >= 3 && parts[2].Equals("ro", StringComparison.OrdinalIgnoreCase); + + return new Mount + { + Type = "bind", + Source = parts[0], + Target = parts[1], + ReadOnly = isReadOnly + }; + } + } + + private List GetEnvironmentVariables() + { + if (string.IsNullOrEmpty(_GridServerSettings.BaseUrl)) + throw new Exception("Unable to start a new Grid Server container, BaseUrl is set to null or is empty"); + + var environmentVariables = new List + { + $"RCC_PORT={Port}", + $"RCC_HTTP_ACCESS_KEY={_GridServerSettings.HttpAccessKey}", + $"BASE_URL={_GridServerSettings.BaseUrl}" + }; + + if (Environment.GetEnvironmentVariable("DISPLAY") != null) + environmentVariables.Add($"DISPLAY={Environment.GetEnvironmentVariable("DISPLAY")}"); + + if (_GridServerSettings.GridServerMaxThreads > 0) + environmentVariables.Add($"MAXIMUM_THREADS={_GridServerSettings.GridServerMaxThreads}"); + + int maxMemory = (int)(_GridServerSettings.GridServerMaxMemoryInBytes / 1048576); + if (maxMemory > 0) + environmentVariables.Add($"MAXIMUM_MEMORY={maxMemory}"); + + if (!string.IsNullOrWhiteSpace(_GridServerSettings.GridServerSettingsKey)) + environmentVariables.Add($"RCC_SETTINGS_KEY={_GridServerSettings.GridServerSettingsKey}"); + + if (_GridServerSettings.GridServerEnvironmentVariables != null) + foreach (var environmentVariable in _GridServerSettings.GridServerEnvironmentVariables) + environmentVariables.Add($"{environmentVariable.Key}={environmentVariable.Value}"); + + return environmentVariables; + } + + private CreateContainerParameters GetCreateContainerParameters() + { + if (string.IsNullOrEmpty(_GridServerSettings.HttpAccessKey)) + throw new Exception("Unable to start a new Grid Server Service container, HttpAccessKey is set to null or is empty"); + + var containerParameters = new List(); + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerSettingsApplicationName)) + containerParameters.AddRange(new[] { "-ApplicationName", _GridServerSettings.GridServerSettingsApplicationName }); + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerApplicationSettingsFileName)) + { + Logger.Information("GetCreateContainerParameters. Adding --settingsFile command line option: {0}", _GridServerSettings.GridServerApplicationSettingsFileName); + + containerParameters.AddRange(new[] { "-SettingsFile", _GridServerSettings.GridServerApplicationSettingsFileName }); + } + + var parameters = new CreateContainerParameters + { + Image = $"{_GridServerImageName}:{Version}", + Name = ContainerName, + Cmd = containerParameters, + Env = GetEnvironmentVariables() + }; + + var labels = new Dictionary + { + [PortLabel] = Port.ToString(), + [GridServerVersionLabel] = Version, + [ImageNameLabel] = _GridServerImageName, + [GridServerApplicationNameLabel] = ApplicationName, + [GridServerBucketNameLabel] = BucketName + }; + + parameters.Labels = labels; + parameters.HostConfig = new HostConfig + { + Mounts = GetContainerMounts(), + Memory = _GridServerSettings.GridServerMaxMemoryInBytes, + Ulimits = new List + { + new() + { + Name = "core", + Hard = 9999999999, + Soft = 9999999999 + }, + new() + { + Name = "nofile", + Hard = 8192, + Soft = 4096 + } + }, + NetworkMode = "host" + }; + + if (_GridServerSettings.ReservedCoresPerGridServerInstance != null) + { + parameters.HostConfig.CPUPeriod = 100000; + parameters.HostConfig.CPUQuota = _DockerAuthority.CalculateCpuQuota(_GridServerSettings.ReservedCoresPerGridServerInstance.Value, 100000); + } + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerPrimaryDnsServer)) + { + var dnsConfiguration = new List { _GridServerSettings.GridServerPrimaryDnsServer }; + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerSecondaryDnsServer)) + dnsConfiguration.Add(_GridServerSettings.GridServerSecondaryDnsServer); + + parameters.HostConfig.DNS = dnsConfiguration; + } + + return parameters; + } + + /// + public override void Dispose() + { + if (_Disposed) return; + + _DockerAuthority.KillContainerAsync(ContainerID).Wait(); + _DockerAuthority.RemoveContainerWithRetriesAsync(ContainerID).Wait(); + + _Disposed = true; + } +} + +#endif diff --git a/lib/grid/process-management/Enums/ScriptType.cs b/lib/grid/process-management/Enums/ScriptType.cs index 58033be8..2c75deda 100644 --- a/lib/grid/process-management/Enums/ScriptType.cs +++ b/lib/grid/process-management/Enums/ScriptType.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; /// /// Represents the type of script. diff --git a/lib/grid/process-management/Extensions/ProcessExtensions.cs b/lib/grid/process-management/Extensions/ProcessExtensions.cs index 15503a27..d57c5b47 100644 --- a/lib/grid/process-management/Extensions/ProcessExtensions.cs +++ b/lib/grid/process-management/Extensions/ProcessExtensions.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.Net; @@ -6,10 +6,13 @@ using System.Diagnostics; using System.Runtime.InteropServices; -using PInvoke; +using Microsoft.Win32.SafeHandles; -using Win32Exception = System.ComponentModel.Win32Exception; +using Windows.Win32; +using Windows.Win32.Foundation; +using Windows.Win32.System.Threading; +using Win32Exception = System.ComponentModel.Win32Exception; internal static class ProcessExtensions { @@ -34,41 +37,37 @@ public static bool SafeGetHasExited(this Process process) { if (process == null) return true; - var hProcess = Kernel32.SafeObjectHandle.Null; + SafeFileHandle hProcess = null; try { - var processHandle = Kernel32.OpenProcess(Kernel32.ProcessAccess.PROCESS_QUERY_LIMITED_INFORMATION, false, process.Id); - if (processHandle == Kernel32.SafeObjectHandle.Null || processHandle.IsInvalid) + hProcess = PInvoke.OpenProcess_SafeHandle(PROCESS_ACCESS_RIGHTS.PROCESS_QUERY_LIMITED_INFORMATION, false, (uint)process.Id); + if (hProcess.IsInvalid) return true; - return Kernel32.GetExitCodeProcess(processHandle.DangerousGetHandle(), out var lpExitCode) && lpExitCode != 259; - } - catch (Exception ex) when (ex is InvalidOperationException or NotSupportedException or Win32Exception or COMException) - { - return false; // Handle either exists and we don't have access, or it doesn't exist. Assume exists. + return PInvoke.GetExitCodeProcess(hProcess, out var lpExitCode) && lpExitCode != 259; } finally { - if (hProcess != Kernel32.SafeObjectHandle.Null) + if (!hProcess.IsInvalid && !hProcess.IsClosed) hProcess.Close(); } } - public static (bool, Win32ErrorCode) ForceKill(this Process proc) + public static (bool, WIN32_ERROR) ForceKill(this Process proc) { if (proc == null || proc.SafeGetHasExited()) - return (false, Win32ErrorCode.ERROR_PROCESS_ABORTED); + return (false, WIN32_ERROR.ERROR_PROCESS_ABORTED); - var objHandle = Kernel32.OpenProcess(Kernel32.ProcessAccess.PROCESS_TERMINATE, false, proc.Id); - if (objHandle == Kernel32.SafeObjectHandle.Null) - return (false, Kernel32.GetLastError()); + var hProcess = PInvoke.OpenProcess_SafeHandle(PROCESS_ACCESS_RIGHTS.PROCESS_TERMINATE, false, (uint)proc.Id); + if (hProcess.IsInvalid) + return (false, (WIN32_ERROR)Marshal.GetLastWin32Error()); - if (!Kernel32.TerminateProcess(objHandle.DangerousGetHandle(), 0)) - return (false, Kernel32.GetLastError()); + if (!PInvoke.TerminateProcess(hProcess, 0)) + return (false, (WIN32_ERROR)Marshal.GetLastWin32Error()); - objHandle.Close(); + hProcess.Close(); - return (true, Win32ErrorCode.NERR_Success); + return (true, WIN32_ERROR.NO_ERROR); } } diff --git a/lib/grid/process-management/Grid.ProcessManagement.csproj b/lib/grid/process-management/Grid.ProcessManagement.csproj index a514cb8e..d5a8ef4e 100644 --- a/lib/grid/process-management/Grid.ProcessManagement.csproj +++ b/lib/grid/process-management/Grid.ProcessManagement.csproj @@ -1,14 +1,20 @@  Shared library for allocating grid-server processes! - Grid + + Grid.ProcessManagement + preview - - - - + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/lib/grid/process-management/Helper/ManagedIpHelper.cs b/lib/grid/process-management/Helper/ManagedIpHelper.cs index b41b3a15..b7af8b74 100644 --- a/lib/grid/process-management/Helper/ManagedIpHelper.cs +++ b/lib/grid/process-management/Helper/ManagedIpHelper.cs @@ -1,14 +1,15 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.Net; using System.Collections; -using System.Net.Sockets; using System.Collections.Generic; -using System.Net.NetworkInformation; using System.Runtime.InteropServices; -using PInvoke; +using Windows.Win32; +using Windows.Win32.Foundation; +using Windows.Win32.Networking.WinSock; +using Windows.Win32.NetworkManagement.IpHelper; #region Managed IP Helper API @@ -54,19 +55,30 @@ internal class TcpRow private readonly IPEndPoint _localEndPoint; private readonly IPEndPoint _remoteEndPoint; - private readonly TcpState _state; + private readonly MIB_TCP_STATE _state; private readonly uint _processId; #endregion #region Constructors - public TcpRow(IPHlpApi.MIB_TCPROW_OWNER_PID tcpRow) + public TcpRow(MIB_TCPROW_OWNER_PID tcpRow) { _state = tcpRow.dwState; _processId = tcpRow.dwOwningPid; - _localEndPoint = new IPEndPoint(tcpRow.LocalAddr, tcpRow.LocalPort); - _remoteEndPoint = new IPEndPoint(tcpRow.RemoteAddr, tcpRow.RemotePort); + + var localPort = PInvoke.ntohs((ushort)tcpRow.dwLocalPort); + var remotePort = PInvoke.ntohs((ushort)tcpRow.dwRemotePort); + + if (tcpRow.dwLocalPort <= 0) + _localEndPoint = new IPEndPoint(tcpRow.dwLocalAddr, 0); + else + _localEndPoint = new IPEndPoint(tcpRow.dwLocalAddr, localPort); + + if (tcpRow.dwRemotePort <= 0) + _remoteEndPoint = new IPEndPoint(tcpRow.dwRemoteAddr, 0); + else + _remoteEndPoint = new IPEndPoint(tcpRow.dwRemoteAddr, remotePort); } #endregion @@ -77,7 +89,7 @@ public TcpRow(IPHlpApi.MIB_TCPROW_OWNER_PID tcpRow) public IPEndPoint RemoteEndPoint => _remoteEndPoint; - public TcpState State => _state; + public MIB_TCP_STATE State => _state; public uint ProcessId => _processId; @@ -91,54 +103,51 @@ internal static class ManagedIpHelper { #region Public Methods - public static TcpTable GetExtendedTcpTable(bool sorted) + public unsafe static TcpTable GetExtendedTcpTable(bool sorted) { var tcpRows = new List(); - - var tcpTable = IntPtr.Zero; - int tcpTableLength = 0; - - if ( - IPHlpApi.GetExtendedTcpTable( - tcpTable, - ref tcpTableLength, - sorted, - AddressFamily.InterNetwork, - IPHlpApi.TCP_TABLE_CLASS.TCP_TABLE_OWNER_PID_ALL, - 0 - ) != Win32ErrorCode.ERROR_SUCCESS - ) + + void* pTcpTable = null; + uint pdwSize = 0; + + try { - try + + if ( + PInvoke.GetExtendedTcpTable( + null, + ref pdwSize, + sorted, + (uint)ADDRESS_FAMILY.AF_INET, + TCP_TABLE_CLASS.TCP_TABLE_OWNER_PID_LISTENER, + 0 + ) != (uint)WIN32_ERROR.NO_ERROR + ) { - tcpTable = Marshal.AllocHGlobal(tcpTableLength); - + pTcpTable = (void*)Marshal.AllocHGlobal((int)pdwSize); + if ( - IPHlpApi.GetExtendedTcpTable( - tcpTable, - ref tcpTableLength, - true, - AddressFamily.InterNetwork, - IPHlpApi.TCP_TABLE_CLASS.TCP_TABLE_OWNER_PID_ALL, + PInvoke.GetExtendedTcpTable( + pTcpTable, + ref pdwSize, + sorted, + (uint)ADDRESS_FAMILY.AF_INET, + TCP_TABLE_CLASS.TCP_TABLE_OWNER_PID_LISTENER, 0 - ) == Win32ErrorCode.ERROR_SUCCESS + ) == (uint)WIN32_ERROR.NO_ERROR ) { - var table = (IPHlpApi.MIB_TCPTABLE_OWNER_PID)Marshal.PtrToStructure(tcpTable, typeof(IPHlpApi.MIB_TCPTABLE_OWNER_PID)); - - var rowPtr = (IntPtr)((long)tcpTable + Marshal.SizeOf(table.dwNumEntries)); - for (int i = 0; i < table.dwNumEntries; ++i) - { - tcpRows.Add(new TcpRow((IPHlpApi.MIB_TCPROW_OWNER_PID)Marshal.PtrToStructure(rowPtr, typeof(IPHlpApi.MIB_TCPROW_OWNER_PID)))); - rowPtr = (IntPtr)((long)rowPtr + Marshal.SizeOf(typeof(IPHlpApi.MIB_TCPROW_OWNER_PID))); - } + var table = (MIB_TCPTABLE_OWNER_PID*)pTcpTable; + + for (int i = 0; i < table->dwNumEntries; ++i) + tcpRows.Add(new TcpRow(table->table[i])); } } - finally - { - if (tcpTable != IntPtr.Zero) - Marshal.FreeHGlobal(tcpTable); - } + } + finally + { + if (pTcpTable != null) + Marshal.FreeHGlobal((IntPtr)pTcpTable); } return new TcpTable(tcpRows); diff --git a/lib/grid/process-management/Helper/TcpHealthCheck.cs b/lib/grid/process-management/Helper/TcpHealthCheck.cs index bfde6cdb..1657f4f0 100644 --- a/lib/grid/process-management/Helper/TcpHealthCheck.cs +++ b/lib/grid/process-management/Helper/TcpHealthCheck.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System.Linq; using System.Diagnostics; diff --git a/lib/grid/process-management/Implementation/GridServerFileHelper.cs b/lib/grid/process-management/Implementation/GridServerFileHelper.cs index c16686f9..7bcc63c9 100644 --- a/lib/grid/process-management/Implementation/GridServerFileHelper.cs +++ b/lib/grid/process-management/Implementation/GridServerFileHelper.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.IO; diff --git a/lib/grid/process-management/Implementation/GridServerProcess.cs b/lib/grid/process-management/Implementation/GridServerProcess.cs index 29675e13..63c90f87 100644 --- a/lib/grid/process-management/Implementation/GridServerProcess.cs +++ b/lib/grid/process-management/Implementation/GridServerProcess.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.Diagnostics; @@ -7,12 +7,16 @@ using Logging; using Commands; +using Core; + /// /// Represents the Docker implementation of /// public sealed class GridServerProcess : GridServerInstanceBase { - private readonly IGridServerProcess _Process; + private const int _MillisecondToSecond = 1000; + + private readonly IRawGridServerProcess _Process; private readonly IGridServerProcessSettings _GridServerSettings; private readonly IGridServerFileHelper _FileHelper; @@ -39,7 +43,9 @@ public sealed class GridServerProcess : GridServerInstanceBase /// The port /// The version. /// The - /// The + /// The GridServer application name. + /// The GridServer bucket name. + /// The /// The /// must be > 0 internal GridServerProcess( @@ -47,10 +53,12 @@ internal GridServerProcess( int port, string version, IGridServerProcessSettings gridServerSettings, - IGridServerProcess gridServerProcess, + string applicationName, + string bucketName, + IRawGridServerProcess gridServerProcess, IGridServerFileHelper fileHelper = null ) - : base(logger, version, port, gridServerSettings) + : base(logger, version, port, gridServerSettings, applicationName, bucketName) { if (port < 1) throw new ArgumentException("Port must be > 0", nameof(port)); @@ -61,19 +69,42 @@ internal GridServerProcess( _FileHelper = fileHelper ?? new GridServerFileHelper(gridServerSettings); - Logger.Information("Constructing GridServerProcess", - new - { - ProcessName, - Port, - Version - } + Logger.Information( + "Constructing GridServerProcess, ProcessName = {0}, Port = {1}, Version = {2}", + ProcessName, + Port, + Version ); } /// public override bool Start() - => _Process.Start(_GridServerSettings.GridServerExecutableName, _FileHelper.GetGridServerPath(true), Port) && WaitForProcessStart(); + => _Process.Start(_GridServerSettings.GridServerExecutableName, _FileHelper.GetGridServerPath(), Port, _GridServerSettings.GridServerMaxThreads, _GridServerSettings.GridServerMaxMemoryInBytes, GetArguments()) && WaitForProcessStart(); + + private string GetArguments() + { + var arguments = new List() + { + "-Console" + }; + + if (_GridServerSettings.VerboseLoggingEnabled) + arguments.Add("-Verbose"); + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerSettingsApplicationName)) + arguments.AddRange(new[] { "-ApplicationName", _GridServerSettings.GridServerSettingsApplicationName }); + + if (!string.IsNullOrEmpty(_GridServerSettings.GridServerApplicationSettingsFileName)) + { + Logger.Information("GetArguments. Adding -SettingsFile command line option: {0}", _GridServerSettings.GridServerApplicationSettingsFileName); + + arguments.AddRange(new[] { "-SettingsFile", _GridServerSettings.GridServerApplicationSettingsFileName }); + } + + arguments.Add(Port.ToString()); + + return string.Join(" ", arguments); + } private bool WaitForProcessStart() { @@ -82,7 +113,7 @@ private bool WaitForProcessStart() try { WaitForServiceToBecomeAvailable(false, sw); - InitializeHA(); + InitializeHighAvailability(); return true; } @@ -101,9 +132,9 @@ private bool WaitForProcessStart() } } - private void InitializeHA() + private void InitializeHighAvailability() { - using var soap = GetSoapInterface(10000); + using var soap = GetSoapInterface(60 * _MillisecondToSecond); #if !PRE_JSON_EXECUTION var command = new ExecuteScriptCommand( diff --git a/lib/grid/process-management/Implementation/JobManager.cs b/lib/grid/process-management/Implementation/JobManager.cs index 10739aea..1c92f493 100644 --- a/lib/grid/process-management/Implementation/JobManager.cs +++ b/lib/grid/process-management/Implementation/JobManager.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.IO; @@ -7,6 +7,10 @@ using System.Collections.Generic; using Logging; +using ClientSettings.Client; + +using Core; +using PortManagement; /// /// Represents the Process implementation for @@ -22,6 +26,7 @@ public class ProcessJobManager : JobManagerBase /// The /// The /// The + /// The /// The /// The /// @@ -31,10 +36,11 @@ public ProcessJobManager( ILogger logger, IPortAllocator portAllocator, IGridServerProcessSettings gridServerSettings, + IClientSettingsClient clientSettingsClient, ResourceAllocationTracker resourceAllocationTracker = null, IGridServerFileHelper gridServerFileHelper = null ) - : base(logger, gridServerSettings, portAllocator, resourceAllocationTracker) + : base(logger, gridServerSettings, portAllocator, clientSettingsClient, new WindowsSettingsFileWriter(logger, gridServerFileHelper ?? new GridServerFileHelper(gridServerSettings)), resourceAllocationTracker) { _GridServerSettings = gridServerSettings ?? throw new ArgumentNullException(nameof(gridServerSettings)); @@ -81,9 +87,9 @@ select process.RawProcess.Id.ToString()).ToList() ); /// - protected override string GetLatestGridServerVersion() => ReadRccVersion(); + protected override string GetLatestGridServerVersion() => ReadGridServerVersion(); - private string ReadRccVersion() + private string ReadGridServerVersion() { var fileVersionInfo = FileVersionInfo.GetVersionInfo(_FileHelper.GetFullyQualifiedGridServerPath()); @@ -96,7 +102,7 @@ protected override bool OnGridServerVersionChange(string newGridServerVersion, b /// protected override IGridServerInstance CreateNewGridServerInstance(int port) - => new GridServerProcess(Logger, port, GridServerVersion, _GridServerSettings, new RawGridServerProcess(), _FileHelper); + => new GridServerProcess(Logger, port, GridServerVersion, _GridServerSettings, ApplicationName, BucketName, new RawGridServerProcess(), _FileHelper); /// protected override IReadOnlyCollection FindUnexpectedExitGameJobs() @@ -130,6 +136,9 @@ protected override IGridServerInstance RecoverGridServerInstance(IUnmanagedGridS if (unmanagedGridServerProcess.Process.HasExited) return null; + // This has no detection for application name or bucket name + // as there is no way to tag processes + var name = unmanagedGridServerProcess.Process.RawProcess.Id.ToString(); var port = unmanagedGridServerProcess.Process.EndPoint.Port; @@ -144,6 +153,8 @@ protected override IGridServerInstance RecoverGridServerInstance(IUnmanagedGridS port, GetVersion(), _GridServerSettings, + ApplicationName, + BucketName, unmanagedGridServerProcess.Process, _FileHelper ) @@ -157,24 +168,27 @@ protected override void OnGetJobInstanceHasExited() { } - private IReadOnlyCollection ListRunningProcesses() + private IReadOnlyCollection ListRunningProcesses() { try { var processes = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(_GridServerSettings.GridServerExecutableName)); + var newProcesses = new List(); - return processes.Where(p => p.GetProcessEndPoint(out _)).Select(p => + foreach (var process in processes) { - p.GetProcessEndPoint(out var endpoint); + if (!process.GetProcessEndPoint(out var endPoint)) continue; + + newProcesses.Add(new RawGridServerProcess(process, endPoint)); + } - return new RawGridServerProcess(p, endpoint); - }).ToArray(); + return newProcesses; } catch (Exception ex) { Logger.Error("ListRunningProcesses: {0}", ex); - return Array.Empty(); + return Array.Empty(); } } } diff --git a/lib/grid/process-management/Implementation/RawGridServerProcess.cs b/lib/grid/process-management/Implementation/RawGridServerProcess.cs index 29afd944..81416620 100644 --- a/lib/grid/process-management/Implementation/RawGridServerProcess.cs +++ b/lib/grid/process-management/Implementation/RawGridServerProcess.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.IO; @@ -7,9 +7,9 @@ using Logging; -/// +/// [DebuggerDisplay($"{{{nameof(ToString)}(), nq}}")] -internal class RawGridServerProcess : IGridServerProcess, IDisposable +internal class RawGridServerProcess : IRawGridServerProcess, IDisposable { private Process _process; private IPEndPoint _endpoint; @@ -49,25 +49,25 @@ public RawGridServerProcess(Process process, IPEndPoint endPoint) _endpoint = endPoint ?? throw new ArgumentNullException(nameof(endPoint)); } - /// + /// public Process RawProcess => _process; - /// + /// public bool HasExited => _disposed || _process.SafeGetHasExited(); - /// + /// public IPEndPoint EndPoint => _endpoint; - /// + /// public bool IsDisposed => _disposed; - /// + /// public void SetLogger(ILogger logger) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - /// + /// public void Kill() { var (didClose, errorCode) = _process.ForceKill(); @@ -91,11 +91,13 @@ public void Dispose() _disposed = true; } - /// + /// public bool Start( string executableName, string workingDirectory = null, int port = 53640, + int maxThreads = 0, + long maxMemoryInBytes = 0, string args = null ) { @@ -128,6 +130,13 @@ public bool Start( else startInfo.Arguments = $"{port} -Console -Verbose"; + if (maxThreads > 0) + startInfo.Environment.Add("MAXIMUM_THREADS", maxThreads.ToString()); + + int maxMemory = (int)(maxMemoryInBytes / 1048576); + if (maxMemory > 0) + startInfo.Environment.Add("MAXIMUM_MEMORY", maxMemory.ToString()); + _endpoint = new IPEndPoint(IPAddress.Loopback, port); _process = Process.Start(startInfo); diff --git a/lib/grid/process-management/Implementation/UnmanagedGridServerProcess.cs b/lib/grid/process-management/Implementation/UnmanagedGridServerProcess.cs index 2e4a04b9..8647a855 100644 --- a/lib/grid/process-management/Implementation/UnmanagedGridServerProcess.cs +++ b/lib/grid/process-management/Implementation/UnmanagedGridServerProcess.cs @@ -1,7 +1,8 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System.Diagnostics; +using Core; using Logging; /// @@ -17,14 +18,14 @@ public sealed class UnmanagedGridServerProcess : IUnmanagedGridServerInstance /// /// The container. /// - public IGridServerProcess Process { get; } + public IRawGridServerProcess Process { get; } /// /// Contruct a new instance of /// /// The - /// The - public UnmanagedGridServerProcess(ILogger logger, IGridServerProcess process) + /// The + public UnmanagedGridServerProcess(ILogger logger, IRawGridServerProcess process) { _Logger = logger; diff --git a/lib/grid/process-management/Implementation/WindowsSettingsFileWriter.cs b/lib/grid/process-management/Implementation/WindowsSettingsFileWriter.cs new file mode 100644 index 00000000..0eecf225 --- /dev/null +++ b/lib/grid/process-management/Implementation/WindowsSettingsFileWriter.cs @@ -0,0 +1,86 @@ +namespace Grid.ProcessManagement; + +using System; +using System.IO; +using System.Text; +using System.Threading; + +using Newtonsoft.Json; + +using Logging; +using ClientSettings.Client; + +using Core; + +/// +public class WindowsSettingsFileWriter : ISettingsFileWriter +{ + + private const int _MaxAttempts = 3; + private const int _SleepIntervalMilliseconds = 50; + + private readonly ILogger _Logger; + private readonly IGridServerFileHelper _GridServerFileHelper; + + /// + /// Constructs a new instance of + /// + /// The + /// The + /// + /// - cannot be null. + /// - cannot be null. + /// + public WindowsSettingsFileWriter(ILogger logger, IGridServerFileHelper gridServerFileHelper) + { + _Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _GridServerFileHelper = gridServerFileHelper ?? throw new ArgumentNullException(nameof(gridServerFileHelper)); + } + + /// + public bool WriteSettingsFile(string filePath, ClientApplicationSettingsResponse rccApplicationSettings) + { + var fileContents = JsonConvert.SerializeObject(rccApplicationSettings); + + filePath = Path.Combine(_GridServerFileHelper.GetGridServerPath(), Path.GetFileName(filePath)); // It has to be in the same directory for some reason. + + for (int i = 0; i < _MaxAttempts; i++) + { + try + { + _Logger.Information("WriteSettingsFile. Attempting to write settings file to {0}. Attempt #{1}", filePath, i + 1); + _Logger.Verbose("WriteSettingsFile. FileContents: {0}", fileContents); + + TryWriteSettingsFile(filePath, fileContents, filePath + ".tmp"); + + return true; + } + catch (Exception ex) + { + _Logger.Error("WriteSettingsFile. Error: {0}", ex); + } + + Thread.Sleep(_SleepIntervalMilliseconds); + } + + return false; + } + + private void TryWriteSettingsFile(string filePath, string fileContents, string tempFilePath) + { + if (File.Exists(filePath)) + { + _Logger.Debug("TryWriteSettingsFile. {0} already exists. Attempting to write fileContents to {1}", filePath, tempFilePath); + File.WriteAllText(tempFilePath, fileContents, Encoding.ASCII); + + _Logger.Debug("TryWriteSettingsFile. Attempting to replace {0} with {1}", filePath, tempFilePath); + File.Replace(tempFilePath, filePath, null, true); + + return; + } + + _Logger.Debug("TryWriteSettingsFile. {0} does not exist. Attempting to write fileContents to {1}", filePath, filePath); + + File.WriteAllText(filePath, fileContents, Encoding.ASCII); + } +} diff --git a/lib/grid/process-management/Interfaces/IGridServerFileHelper.cs b/lib/grid/process-management/Interfaces/IGridServerFileHelper.cs index 6f35b68f..47362351 100644 --- a/lib/grid/process-management/Interfaces/IGridServerFileHelper.cs +++ b/lib/grid/process-management/Interfaces/IGridServerFileHelper.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; diff --git a/lib/grid/process-management/Interfaces/IGridServerSettings.cs b/lib/grid/process-management/Interfaces/IGridServerSettings.cs index 6648f923..d69ab598 100644 --- a/lib/grid/process-management/Interfaces/IGridServerSettings.cs +++ b/lib/grid/process-management/Interfaces/IGridServerSettings.cs @@ -1,4 +1,6 @@ -namespace Grid; +namespace Grid.ProcessManagement; + +using Core; /// /// Represents the Grid Server settings. @@ -19,4 +21,19 @@ public interface IGridServerProcessSettings : IJobManagerSettings /// Gets the name of the Windows Registry Value used by process-based grid-servers. /// string GridServerRegistryValueName { get; } + + /// + /// The maximum amount of GridServer Service memory in bytes. + /// + long GridServerMaxMemoryInBytes { get; } + + /// + /// Determines if verbose logging is enabled or not. + /// + bool VerboseLoggingEnabled { get; } + + /// + /// The name of the file to cache app settings in. + /// + string GridServerApplicationSettingsFileName { get; } } diff --git a/lib/grid/process-management/Interfaces/IGridServerProcess.cs b/lib/grid/process-management/Interfaces/IRawGridServerProcess.cs similarity index 79% rename from lib/grid/process-management/Interfaces/IGridServerProcess.cs rename to lib/grid/process-management/Interfaces/IRawGridServerProcess.cs index 79aa7f4e..eef9e5a2 100644 --- a/lib/grid/process-management/Interfaces/IGridServerProcess.cs +++ b/lib/grid/process-management/Interfaces/IRawGridServerProcess.cs @@ -1,4 +1,4 @@ -namespace Grid; +namespace Grid.ProcessManagement; using System; using System.Net; @@ -9,7 +9,7 @@ /// /// Wrapper for a process owned by the arbiters. /// -public interface IGridServerProcess : IDisposable +public interface IRawGridServerProcess : IDisposable { /// /// The raw process. @@ -46,8 +46,10 @@ public interface IGridServerProcess : IDisposable /// /// Name of the executable. /// The working directory of the executable. - /// The port of the grid server. + /// The port of the rcc. + /// The max threads + /// The max memory /// The optional arguments /// The process - bool Start(string executableName, string workingDirectory = null, int port = 53640, string args = null); + bool Start(string executableName, string workingDirectory = null, int port = 53640, int maxThreads = 0, long maxMemory = 0, string args = null); } diff --git a/lib/grid/process-management/NativeMethods.txt b/lib/grid/process-management/NativeMethods.txt new file mode 100644 index 00000000..b728c726 --- /dev/null +++ b/lib/grid/process-management/NativeMethods.txt @@ -0,0 +1,12 @@ +WIN32_ERROR + +ADDRESS_FAMILY +GetExtendedTcpTable +MIB_TCPTABLE_OWNER_PID + +PROCESS_ACCESS_RIGHTS +GetExitCodeProcess +OpenProcess +TerminateProcess + +ntohs diff --git a/services/grid-bot/.component.yaml b/services/grid-bot/.component.yaml index 2f5f9e9e..dcc4b714 100755 --- a/services/grid-bot/.component.yaml +++ b/services/grid-bot/.component.yaml @@ -28,6 +28,8 @@ deployment: operator: "set_contains" value: "grid-bot" + vault_role: grid-bot-${{ env.NOMAD_ENVIRONMENT }} + containers: # Maps to the groups section in Nomad - image: mfdlabs/grid-bot resources: @@ -61,10 +63,106 @@ deployment: - '/var/run/docker.sock:/var/run/docker.sock' - '/tmp/.X11-unix:/tmp/.X11-unix' - - '/_/data/grid-bot/scripts:/_/data/grid-bot/scripts' - '/_/data/grid-bot/logs:/tmp/mfdlabs/logs' - - '/_/data/grid-bot/rcc-logs:/_/data/grid-bot/rcc-logs' + - '/_/data/grid-bot/rcc:/_/data/grid-bot/rcc' # Simply here to just to ensure it actually gets created on the host + + artifacts: + - source: "https://curl.se/ca/cacert.pem" + destination: "local/initial" + config_maps: + + - destination: local/scripts/00-generate-cacert.sh + env: false + perms: "755" + on_change: noop + data: | + #!/bin/sh + set -eu + + CACERT_INITIAL="/local/initial/cacert.pem" + CACERT_FINAL="/_/data/grid-bot/cacert.pem" + + : "$${VAULT_ADDR:?VAULT_ADDR is not set}" + : "$${VAULT_TOKEN:?VAULT_TOKEN is not set}" + + echo "[pre-script] Fetching Vault issuer chain..." + + HTTP_CODE=$(curl -s -o /tmp/vault_ca_chain_response.json -w "%%{http_code}" \ + -H "X-Vault-Token: $${VAULT_TOKEN}" \ + "${VAULT_ADDR}/v1/pki_int/cert/ca_chain") + + if [ "$HTTP_CODE" != "200" ]; then + echo "[pre-script] ERROR: Vault returned HTTP $${HTTP_CODE}" >&2 + cat /tmp/vault_ca_chain_response.json >&2 + exit 1 + fi + + if ! jq -e . /tmp/vault_ca_chain_response.json >/dev/null 2>&1; then + echo "[pre-script] ERROR: response is not valid JSON" >&2 + echo "[pre-script] Raw response follows:" >&2 + cat /tmp/vault_ca_chain_response.json >&2 + exit 1 + fi + + CHAIN=$(jq -r '.data.ca_chain' /tmp/vault_ca_chain_response.json) + + if [ -z "$CHAIN" ] || [ "$CHAIN" = "null" ]; then + echo "[pre-script] ERROR: ca_chain field missing or null" >&2 + cat /tmp/vault_ca_chain_response.json >&2 + exit 1 + fi + + echo "[pre-script] Writing combined cacert to $${CACERT_FINAL}..." + cat "$CACERT_INITIAL" > "$CACERT_FINAL" + printf '%s\n' "$CHAIN" >> "$CACERT_FINAL" + + echo "[pre-script] cacert.pem written successfully." + + - destination: local/scripts/01-generate-client-settings-token.sh + env: false + perms: "755" + on_change: noop + data: | + #!/bin/sh + set -eu + + : "$${VAULT_ADDR:?VAULT_ADDR is not set}" + : "$${VAULT_TOKEN:?VAULT_TOKEN is not set}" + + ROLE="nomad-client-settings" + + RESPONSE=$(curl -sf \ + -H "X-Vault-Token: $${VAULT_TOKEN}" \ + -X POST \ + "$${VAULT_ADDR}/v1/auth/token/create/${ROLE}") + + TOKEN=$(echo "$${RESPONSE}" | jq -r '.auth.client_token') + + if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then + echo "[pre-script] ERROR: failed to mint client-settings token" >&2 + echo "$RESPONSE" >&2 + exit 1 + fi + + umask 077 + export ClientSettingsVaultToken="$${TOKEN}" + echo "[pre-script] Client-settings token written." + + - destination: local/scripts/02-ensure-data-exists.sh + env: false + perms: "755" + on_change: noop + data: | + #!/bin/sh + set -eu + + mkdir -p /_/data/grid-bot/rcc/app-data + mkdir -p /_/data/grid-bot/rcc/rcc-logs + mkdir -p /_/data/grid-bot/rcc/scripts + + cp /all/scripts/* /_/data/grid-bot/rcc/scripts/ -r + - destination: secrets/file.env env: true on_change: restart @@ -73,7 +171,12 @@ deployment: WebServerBindAddress="http://{{ env "NOMAD_IP_http" }}:{{ env "NOMAD_PORT_http" }}" GridBotGrpcServerEndpoint="http://{{ env "NOMAD_IP_grpc" }}:{{ env "NOMAD_PORT_grpc" }}" + ClientSettingsVaultAddress="${{ env.VAULT_ADDR }}" + DISPLAY=:1 DEFAULT_LOG_LEVEL=Information + PRE_SCRIPTS_DIR="/local/scripts" + VAULT_ADDR="${{ env.VAULT_ADDR }}" - VAULT_TOKEN="${{ env.VAULT_TOKEN }}" + VAULT_TOKEN="{{ env "VAULT_TOKEN" }}" + diff --git a/services/grid-bot/Dockerfile b/services/grid-bot/Dockerfile index 0cd07563..7dc71aa1 100755 --- a/services/grid-bot/Dockerfile +++ b/services/grid-bot/Dockerfile @@ -1,9 +1,17 @@ FROM mcr.microsoft.com/dotnet/aspnet:8.0.1-jammy +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ + jq \ + curl \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /all COPY . /all COPY ./ssl/global-root-ca.crt /usr/local/share/ca-certificates/global-root-ca.crt RUN chmod 644 /usr/local/share/ca-certificates/global-root-ca.crt && update-ca-certificates -CMD ["dotnet", "/all/Grid.Bot.dll"] +RUN chmod +x /all/entrypoint.sh + +ENTRYPOINT ["/all/entrypoint.sh"] diff --git a/services/grid-bot/entrypoint.sh b/services/grid-bot/entrypoint.sh new file mode 100644 index 00000000..41201c85 --- /dev/null +++ b/services/grid-bot/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/sh + +# If PRE_SCRIPTS_DIR defined, then loop thru it and run each script (chmod +x) before starting the main script +if [ -n "$PRE_SCRIPTS_DIR" ]; then + for script in "$PRE_SCRIPTS_DIR"/*.sh; do + if [ -f "$script" ]; then + echo "Running pre-script: $script" + + dos2unix "$script" 2>/dev/null || true + chmod +x "$script" + + # Source the script to run it in the current shell context + . "$script" + fi + done +fi + +dotnet /all/Grid.Bot.dll diff --git a/services/grid-bot/grid-bot.slnx b/services/grid-bot/grid-bot.slnx index eac64188..6ebb1a79 100644 --- a/services/grid-bot/grid-bot.slnx +++ b/services/grid-bot/grid-bot.slnx @@ -10,6 +10,8 @@ + + diff --git a/services/grid-bot/lib/commands/Modules/Commands/ExecuteScript.cs b/services/grid-bot/lib/commands/Modules/Commands/ExecuteScript.cs index c89bb6b0..42e45a77 100755 --- a/services/grid-bot/lib/commands/Modules/Commands/ExecuteScript.cs +++ b/services/grid-bot/lib/commands/Modules/Commands/ExecuteScript.cs @@ -26,6 +26,8 @@ namespace Grid.Bot.Commands.Public; using Extensions; using Grid.Commands; +using Grid.ProcessManagement; +using Grid.ProcessManagement.Core; using ClientJob = Client.Job; diff --git a/services/grid-bot/lib/commands/Modules/Commands/Support.cs b/services/grid-bot/lib/commands/Modules/Commands/Support.cs index 424ed91c..07f77eea 100755 --- a/services/grid-bot/lib/commands/Modules/Commands/Support.cs +++ b/services/grid-bot/lib/commands/Modules/Commands/Support.cs @@ -7,10 +7,10 @@ namespace Grid.Bot.Commands.Public; using System.Runtime.InteropServices; using Discord; - using Discord.Commands; using Networking; +using Grid.ProcessManagement; using Extensions; diff --git a/services/grid-bot/lib/commands/Modules/Interactions/ExecuteScript.cs b/services/grid-bot/lib/commands/Modules/Interactions/ExecuteScript.cs index 8294171c..071e197c 100755 --- a/services/grid-bot/lib/commands/Modules/Interactions/ExecuteScript.cs +++ b/services/grid-bot/lib/commands/Modules/Interactions/ExecuteScript.cs @@ -26,6 +26,8 @@ namespace Grid.Bot.Interactions.Public; using Extensions; using Grid.Commands; +using Grid.ProcessManagement; +using Grid.ProcessManagement.Core; using ClientJob = Client.Job; diff --git a/services/grid-bot/lib/commands/Modules/Interactions/Support.cs b/services/grid-bot/lib/commands/Modules/Interactions/Support.cs index 9ea0e7cd..693dbe70 100755 --- a/services/grid-bot/lib/commands/Modules/Interactions/Support.cs +++ b/services/grid-bot/lib/commands/Modules/Interactions/Support.cs @@ -11,6 +11,8 @@ namespace Grid.Bot.Interactions.Public; using Networking; +using Grid.ProcessManagement; + /// /// Interaction handler for the support commands. /// diff --git a/services/grid-bot/lib/settings/Providers/GridSettings.cs b/services/grid-bot/lib/settings/Providers/GridSettings.cs index c8ed59e2..056c17ac 100755 --- a/services/grid-bot/lib/settings/Providers/GridSettings.cs +++ b/services/grid-bot/lib/settings/Providers/GridSettings.cs @@ -6,6 +6,10 @@ using Logging; +using ProcessManagement; +using ProcessManagement.Core; +using ProcessManagement.Docker; + /// /// Settings provider for all arbiter related stuff. /// @@ -109,7 +113,7 @@ public class GridSettings : BaseSettingsProvider, IGridServerDockerSettings, IGr string.Empty ); - /// + /// public TimeSpan MaxDelayBeforeFetchingNewGridServerContainer => GetOrDefault( nameof(MaxDelayBeforeFetchingNewGridServerContainer), TimeSpan.FromSeconds(10) @@ -121,12 +125,47 @@ public class GridSettings : BaseSettingsProvider, IGridServerDockerSettings, IGr () => System.IO.Path.Combine(Directory.GetCurrentDirectory(), "logs") ); - /// + /// + /// Gets the directory where shared Grid Server Service internal scripts are stored. + /// + /// + /// Originally, this was built into process-management-docker, + /// but was removed because it is technically not feasible + /// public string GridServerSharedDirectoryInternalScripts => GetOrDefault( nameof(GridServerSharedDirectoryInternalScripts), () => System.IO.Path.Combine(Directory.GetCurrentDirectory(), "internal-scripts") ); + /// + /// Gets the directory where shared Grid Server Service internal scripts are stored inside the container. + /// + public string GridServerInsideDirectoryInternalScripts => GetOrDefault( + nameof(GridServerInsideDirectoryInternalScripts), + "/opt/roblox/rcc_service/internalscripts" + ); + + /// + public string BaseUrl => GetOrDefault( + nameof(BaseUrl), + "http://www.sitetest4.robloxlabs.com" + ); + + /// + public string GridServerSharedDirectoryAppData => GetOrDefault( + nameof(GridServerSharedDirectoryAppData), + () => System.IO.Path.Combine(Directory.GetCurrentDirectory(), "app-data") + ); + + /// + public string[] GridServerAdditionalVolumeMappingsSetting => GetOrDefault( + nameof(GridServerAdditionalVolumeMappings), + Array.Empty + ); + + /// + public string[] GridServerAdditionalVolumeMappings { get; set; } + /// public int? ReservedCoresPerGridServerInstance => GetOrDefault( nameof(ReservedCoresPerGridServerInstance), @@ -139,7 +178,7 @@ null as int? 500 * 1024 * 1024 ); - /// + /// public int GridServerMaxThreads => GetOrDefault( nameof(GridServerMaxThreads), 0 @@ -152,9 +191,9 @@ null as IDictionary ); /// - public string HttpAccessKey => GetOrDefault( + public string HttpAccessKey => GetOrDefault( nameof(HttpAccessKey), - string.Empty + Guid.NewGuid().ToString() // Appeasing the original thing where all I need is the tags ); /// @@ -217,6 +256,67 @@ null as int? TimeSpan.FromSeconds(5) ); + + /// + public string GridServerSettingsApplicationName => GetOrDefault( + nameof(GridServerSettingsApplicationName), + () => "RCCService" + GridServerSettingsKey + ); + + /// + public string GridServerSettingsBucketName => GetOrDefault( + nameof(GridServerSettingsBucketName), + string.Empty + ); + + /// + public string GridServerApplicationSettingsFilePath => GetOrDefault( + nameof(GridServerApplicationSettingsFilePath), + () => System.IO.Path.Combine(GridServerSharedDirectoryAppData, GridServerApplicationSettingsFileName) + ); + + /// + public TimeSpan GridServerApplicationSettingsValidWindow => GetOrDefault( + nameof(GridServerApplicationSettingsValidWindow), + TimeSpan.FromHours(1) + ); + + /// + public bool IsGridServerCpuAllocationCheckEnabled => GetOrDefault( + nameof(IsGridServerCpuAllocationCheckEnabled), + false + ); + + /// + public bool IsGridServerThreadsAllocationCheckEnabled => GetOrDefault( + nameof(IsGridServerThreadsAllocationCheckEnabled), + false + ); + + /// + public bool IsGridServerMemoryAllocationCheckEnabled => GetOrDefault( + nameof(IsGridServerMemoryAllocationCheckEnabled), + false + ); + + /// + public double GridServerCpuOverAllocationRatio => GetOrDefault( + nameof(GridServerCpuOverAllocationRatio), + 1 + ); + + /// + public double GridServerThreadsOverAllocationRatio => GetOrDefault( + nameof(GridServerThreadsOverAllocationRatio), + 1 + ); + + /// + public double GridServerMemoryOverAllocationRatio => GetOrDefault( + nameof(GridServerMemoryOverAllocationRatio), + 1 + ); + /// public TimeSpan MaxTimeToWaitForImage => GetOrDefault( nameof(MaxTimeToWaitForImage), @@ -246,4 +346,16 @@ null as int? nameof(GridServerRegistryValueName), () => throw new InvalidOperationException($"Missing required configuration value '{nameof(GridServerRegistryValueName)}") ); + + /// + public bool VerboseLoggingEnabled => GetOrDefault( + nameof(VerboseLoggingEnabled), + false + ); + + /// + public string GridServerApplicationSettingsFileName => GetOrDefault( + nameof(GridServerApplicationSettingsFileName), + "grid-server-settings.json" + ); } diff --git a/services/grid-bot/lib/utility/Implementation/AvatarUtility.cs b/services/grid-bot/lib/utility/Implementation/AvatarUtility.cs index b0db52c7..07e30351 100755 --- a/services/grid-bot/lib/utility/Implementation/AvatarUtility.cs +++ b/services/grid-bot/lib/utility/Implementation/AvatarUtility.cs @@ -17,6 +17,7 @@ using Threading.Extensions; using Grid.Commands; +using Grid.ProcessManagement.Core; using GridJob = Grid.Client.Job; diff --git a/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs index 8a26c753..0378445c 100644 --- a/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs +++ b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactory.cs @@ -412,6 +412,34 @@ public Secrets GetSettingsForApplication(string application, bool withDependenci } } + /// + public Secrets GetBucketedSettingsForApplication(string application, string bucketName, bool withDependencies = true) + { + if (string.IsNullOrWhiteSpace(application)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(application)), nameof(application)); + + if (string.IsNullOrWhiteSpace(bucketName)) + throw new ArgumentException(string.Format("'{0}' cannot be null or whitespace!", nameof(bucketName)), nameof(bucketName)); + + var settings = GetSettingsForApplication(application, withDependencies); + + if (settings == null) + return null; + + var bucketedSettings = new Dictionary(); + + foreach (var kvp in settings) + { + if (kvp.Key.StartsWith($"{bucketName}_")) + { + var keyWithoutBucket = kvp.Key.Substring(bucketName.Length + 1); + bucketedSettings[keyWithoutBucket] = kvp.Value; + } + } + + return bucketedSettings; + } + /// public T GetSettingForApplication(string application, string setting, bool withDependencies = true) { diff --git a/services/grid-bot/lib/utility/Implementation/ClientSettingsFactoryProxy.cs b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactoryProxy.cs new file mode 100644 index 00000000..2db4ff53 --- /dev/null +++ b/services/grid-bot/lib/utility/Implementation/ClientSettingsFactoryProxy.cs @@ -0,0 +1,140 @@ +namespace Grid.Bot.Utility; + +using System; +using System.Threading; +using System.Threading.Tasks; + +using ClientSettings.Client; + + +/// +/// Proxy for GSPM. +/// +/// +/// The only method that is implemented is +/// +public class ClientSettingsFactoryProxyClient : IClientSettingsClient +{ + private readonly IClientSettingsFactory _clientSettingsFactory; + + /// + /// Initializes a new instance of the class. + /// + /// The instance. + /// Thrown if is null. + public ClientSettingsFactoryProxyClient(IClientSettingsFactory clientSettingsFactory) + { + _clientSettingsFactory = clientSettingsFactory ?? throw new ArgumentNullException(nameof(clientSettingsFactory)); + } + + /// + public ClientApplicationSettingsResponse GetRccOnlyClientApplicationSettings(string applicationName, string bucketName) + => new() + { + ApplicationSettings = !string.IsNullOrWhiteSpace(bucketName) + ? _clientSettingsFactory.GetBucketedSettingsForApplication(applicationName, bucketName) + : _clientSettingsFactory.GetSettingsForApplication(applicationName) + }; + + /// + public ClientApplicationSettingsResponse GetApplicationSettings(string applicationName, string x_Api_Key = null) + { + throw new NotImplementedException(); + } + + /// + public Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key = null) + { + throw new NotImplementedException(); + } + + /// + public Task GetApplicationSettingsAsync(string applicationName, string x_Api_Key, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /// + public ClientApplicationSettingResponse GetClientApplicationSetting(string applicationName, string settingName) + { + throw new NotImplementedException(); + } + + /// + public Task GetClientApplicationSettingAsync(string applicationName, string settingName) + { + throw new NotImplementedException(); + } + + /// + public Task GetClientApplicationSettingAsync(string applicationName, string settingName, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /// + public Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName) + { + throw new NotImplementedException(); + } + + /// + public Task GetRccOnlyClientApplicationSettingsAsync(string applicationName, string bucketName, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /// + public void ImportApplicationSetting(string x_Api_Key, ImportClientApplicationSettingsRequest request) + { + throw new NotImplementedException(); + } + + /// + public Task ImportApplicationSettingAsync(string x_Api_Key, ImportClientApplicationSettingsRequest request) + { + throw new NotImplementedException(); + } + + /// + public Task ImportApplicationSettingAsync(string x_Api_Key, ImportClientApplicationSettingsRequest request, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /// + public void RefreshAllClientApplicationSettings(string x_Api_Key) + { + throw new NotImplementedException(); + } + + /// + public Task RefreshAllClientApplicationSettingsAsync(string x_Api_Key) + { + throw new NotImplementedException(); + } + + /// + public Task RefreshAllClientApplicationSettingsAsync(string x_Api_Key, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /// + public SetClientApplicationSettingResponse SetClientApplicationSetting(string x_Api_Key, SetClientApplicationSettingRequest request) + { + throw new NotImplementedException(); + } + + /// + public Task SetClientApplicationSettingAsync(string x_Api_Key, SetClientApplicationSettingRequest request) + { + throw new NotImplementedException(); + } + + /// + public Task SetClientApplicationSettingAsync(string x_Api_Key, SetClientApplicationSettingRequest request, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/services/grid-bot/lib/utility/Implementation/JobManager.cs b/services/grid-bot/lib/utility/Implementation/JobManager.cs index 1b160ddf..cdff3e54 100755 --- a/services/grid-bot/lib/utility/Implementation/JobManager.cs +++ b/services/grid-bot/lib/utility/Implementation/JobManager.cs @@ -5,6 +5,9 @@ namespace Grid.Bot.Utility; using Client; +using JobManagement; +using ProcessManagement.Core; + #if DEBUG /// @@ -80,55 +83,41 @@ public void DispatchRequestToAllActiveJobs(Action action) /// public class JobManager : IJobManager { - private readonly DockerJobManager _dockerJobManager; - private readonly ProcessJobManager _processJobManager; + private readonly IJobManagerGridServer _jobManager; /// /// Initializes a new instance of the class. /// - /// The docker job manager. - /// The process job manager. - /// If both and are null. - public JobManager(DockerJobManager dockerJobManager, ProcessJobManager processJobManager) - { - if (dockerJobManager == null && processJobManager == null) - throw new ArgumentNullException(nameof(dockerJobManager), "Both dockerJobManager and processJobManager cannot be null."); - - _dockerJobManager = dockerJobManager; - _processJobManager = processJobManager; - } - - private JobManagerBase GetJobManager() + /// The + /// cannot be null. + public JobManager(IJobManagerGridServer jobManager) { - if (_dockerJobManager != null) - return _dockerJobManager; - - return _processJobManager; + _jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } /// - public int GetInstanceCount() => GetJobManager().GetInstanceCount(); + public int GetInstanceCount() => _jobManager.GetInstanceCount(); /// - public int GetReadyInstanceCount() => GetJobManager().GetReadyInstanceCount(); + public int GetReadyInstanceCount() => _jobManager.GetReadyInstanceCount(); /// - public int GetActiveJobsCount() => GetJobManager().GetActiveJobsCount(); + public int GetActiveJobsCount() => _jobManager.GetActiveJobsCount(); /// - public IReadOnlyCollection GetAllRunningJobIds() => GetJobManager().GetAllRunningJobIds(); + public IReadOnlyCollection GetAllRunningJobIds() => _jobManager.GetAllRunningJobIds(); /// - public void AddOrUpdateActiveJob(IJob job, IGridServerInstance instance) => GetJobManager().AddOrUpdateActiveJob(job, instance); + public void AddOrUpdateActiveJob(IJob job, IGridServerInstance instance) => _jobManager.AddOrUpdateActiveJob(job, instance); /// - public (bool isAvailable, JobRejectionReason? rejectionReason) IsResourceAvailable(GridServerResource resourceNeeded) => GetJobManager().IsResourceAvailable(resourceNeeded); + public (bool isAvailable, JobRejectionReason? rejectionReason) IsResourceAvailable(GridServerResource resourceNeeded) => _jobManager.IsResourceAvailable(resourceNeeded); /// - public GridServerResource GetAllocatedResource() => GetJobManager().GetAllocatedResource(); + public GridServerResource GetAllocatedResource() => _jobManager.GetAllocatedResource(); /// - public void RenewLease(IJob job, double leaseTimeInSeconds) => GetJobManager().RenewLease(job, leaseTimeInSeconds); + public void RenewLease(IJob job, double leaseTimeInSeconds) => _jobManager.RenewLease(job, leaseTimeInSeconds); /// public (GridServerServiceSoap soapInterface, IGridServerInstance instance, JobRejectionReason? rejectionReason) NewJob( @@ -136,26 +125,26 @@ private JobManagerBase GetJobManager() double expirationInSeconds, bool waitForReadyInstance = false, bool addToActiveJobs = true - ) => GetJobManager().NewJob(job, expirationInSeconds, waitForReadyInstance, addToActiveJobs); + ) => _jobManager.NewJob(job, expirationInSeconds, waitForReadyInstance, addToActiveJobs); /// - public GridServerServiceSoap GetJob(IJob job) => GetJobManager().GetJob(job); + public GridServerServiceSoap GetJob(IJob job) => _jobManager.GetJob(job); /// - public void CloseJob(IJob job, bool removeFromActiveJobs = true) => GetJobManager().CloseJob(job, removeFromActiveJobs); + public void CloseJob(IJob job, bool removeFromActiveJobs = true) => _jobManager.CloseJob(job, removeFromActiveJobs); /// - public string GetVersion() => GetJobManager().GetVersion(); + public string GetVersion() => _jobManager.GetVersion(); /// - public IReadOnlyCollection GetUnexpectedExitGameJobs() => GetJobManager().GetUnexpectedExitGameJobs(); + public IReadOnlyCollection GetUnexpectedExitGameJobs() => _jobManager.GetUnexpectedExitGameJobs(); /// - public void DispatchRequestToAllActiveJobs(Action action) => GetJobManager().DispatchRequestToAllActiveJobs(action); + public void DispatchRequestToAllActiveJobs(Action action) => _jobManager.DispatchRequestToAllActiveJobs(action); /// - public string GetGridServerInstanceId(string jobId) => GetJobManager().GetGridServerInstanceId(jobId); + public string GetGridServerInstanceId(string jobId) => _jobManager.GetGridServerInstanceId(jobId); /// - public bool UpdateGridServerInstance(GridServerResourceJob job) => GetJobManager().UpdateGridServerInstance(job); + public bool UpdateGridServerInstance(GridServerResourceJob job) => _jobManager.UpdateGridServerInstance(job); } diff --git a/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs b/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs index 53452f81..7cdcd045 100644 --- a/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs +++ b/services/grid-bot/lib/utility/Interfaces/IClientSettingsFactory.cs @@ -29,6 +29,19 @@ public interface IClientSettingsFactory /// is null or whitespace. Secrets GetSettingsForApplication(string application, bool withDependencies = true); + /// + /// Gets the settings for the specified application and bucket. + /// + /// The name of the application. + /// The name of the bucket. + /// if set to true [with dependencies]. + /// The settings for the specified application and bucket. + /// + /// - is null or whitespace. + /// - is null or whitespace + /// + Secrets GetBucketedSettingsForApplication(string application, string bucketName, bool withDependencies = true); + /// /// Gets the specific setting for the specified application. /// diff --git a/services/grid-bot/lib/utility/Interfaces/IJobManager.cs b/services/grid-bot/lib/utility/Interfaces/IJobManager.cs index 313af301..24223fc8 100755 --- a/services/grid-bot/lib/utility/Interfaces/IJobManager.cs +++ b/services/grid-bot/lib/utility/Interfaces/IJobManager.cs @@ -5,6 +5,8 @@ namespace Grid.Bot.Utility; using Client; +using ProcessManagement.Core; + /// /// Interface for a job manager. /// diff --git a/services/grid-bot/lib/utility/Shared.Utility.csproj b/services/grid-bot/lib/utility/Shared.Utility.csproj index 3e3ffbc9..806ef504 100755 --- a/services/grid-bot/lib/utility/Shared.Utility.csproj +++ b/services/grid-bot/lib/utility/Shared.Utility.csproj @@ -18,6 +18,7 @@ + diff --git a/services/grid-bot/lib/web/Routes/ClientSettings.cs b/services/grid-bot/lib/web/Routes/ClientSettings.cs index 75719445..884f96ad 100644 --- a/services/grid-bot/lib/web/Routes/ClientSettings.cs +++ b/services/grid-bot/lib/web/Routes/ClientSettings.cs @@ -77,5 +77,5 @@ public async Task GetApplicationSettings(HttpContext context) } await context.Response.WriteAsJsonAsync(new { applicationSettings }); - } + } } diff --git a/services/grid-bot/scripts/.luacheckrc b/services/grid-bot/scripts/.luacheckrc new file mode 100644 index 00000000..5d9c5726 --- /dev/null +++ b/services/grid-bot/scripts/.luacheckrc @@ -0,0 +1,30 @@ +-- luacheck: ignore +globals = { + -- global variables + "game", "workspace", "script", "plugin", + + -- global functions + "delay", "getfenv", "setfenv", "settings", "spawn", "tick", "time", + "typeof", "unpack", "UserSettings", "wait", "warn", "version", + + -- types + "Axes", "BrickColor", "CFrame", "Color3", "ColorSequence", "ColorSequenceKeypoint", + "Enum", "Faces", "Instance", "NumberRange", "NumberSequence", "NumberSequenceKeypoint", + "PhysicalProperties", "Random", "Ray", "Rect", "Region3", "Region3int16", "TweenInfo", + "UDim", "UDim2", "Vector2", "Vector3", "Vector3int16", "DockWidgetPluginGuiInfo", + + -- math library + "math.clamp", "math.noise", "math.sign", + + -- debug library + "debug.profilebegin", "debug.profileend", +} + +-- fix methods +ignore = {"self", "super"} + +-- prevent max line lengths +max_line_length = false +max_code_line_length = false +max_string_line_length = false +max_comment_line_length = false \ No newline at end of file diff --git a/services/grid-bot/scripts/modules/assetValidation/Hat.lua b/services/grid-bot/scripts/modules/assetValidation/Hat.lua new file mode 100644 index 00000000..d4cf7d43 --- /dev/null +++ b/services/grid-bot/scripts/modules/assetValidation/Hat.lua @@ -0,0 +1,21 @@ +local Hat = {} + +function Hat.Validate(loadedObjects) + if #loadedObjects ~= 1 then + return false, "InvalidStructure" + end + + local instance = loadedObjects[1] + + if not instance:IsA("Accoutrement") then + return false, "InvalidStructure" + end + + -- TODO: Implement proper asset validation here + local isValid = math.random() > 0.5 + local validationResult = isValid and "Success" or "TestingInvalid" + + return isValid, validationResult +end + +return Hat \ No newline at end of file diff --git a/services/grid-bot/scripts/scripts/.gitignore b/services/grid-bot/scripts/scripts/.gitignore new file mode 100644 index 00000000..c4fb3326 Binary files /dev/null and b/services/grid-bot/scripts/scripts/.gitignore differ diff --git a/services/grid-bot/scripts/scripts/TexturePack.lua b/services/grid-bot/scripts/scripts/TexturePack.lua new file mode 100644 index 00000000..eb874427 --- /dev/null +++ b/services/grid-bot/scripts/scripts/TexturePack.lua @@ -0,0 +1,14 @@ +-- TexturePack v1.0 + +local baseUrl, texturePackId = ... + +local RCCTexturePackGenerator = game:GetService("RCCTexturePackGenerator") +RCCTexturePackGenerator:AddProfilingCheckpoint("RCCTexturePackGeneratorScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local result, requestedUrls = RCCTexturePackGenerator:Process(baseUrl, texturePackId) +RCCTexturePackGenerator:AddProfilingCheckpoint("RCCTexturePackGeneratorFinished") + +return result, requestedUrls diff --git a/services/grid-bot/scripts/scripts/ValidateUgcContent.lua b/services/grid-bot/scripts/scripts/ValidateUgcContent.lua new file mode 100644 index 00000000..8b6d1db8 --- /dev/null +++ b/services/grid-bot/scripts/scripts/ValidateUgcContent.lua @@ -0,0 +1,95 @@ +local argumentsTable = ... + +-- This flag enables local debugging. Debug mode will do the following: +-- Expects gameserver.txt to contain an assetId instead of assetHash + checksum +-- Provides output information for the result +local FFlagDebugUGCServerValidation = game:DefineFastFlag("DebugUGCServerValidation", false) + +local ScriptContext = game:GetService("ScriptContext") +local HttpService = game:GetService("HttpService") +local CorePackages = game:GetService("CorePackages") +local ContentProvider = game:GetService("ContentProvider") + +pcall(function() ContentProvider:SetBaseUrl(argumentsTable.BaseUrl) end) +ScriptContext.ScriptsDisabled = true + +assert(CorePackages, "CorePackages is not loaded!") +local UGCValidation = require(CorePackages.UGCValidation) + +local function parseBaseUrlInformation(baseUrl) + -- keep a copy of the base url (https://www.roblox.com/) + -- append a trailing slash if there isn't one + if baseUrl:sub(#baseUrl) ~= "/" then + baseUrl = baseUrl .. "/" + end + + -- parse out scheme (http, https) + local _, schemeEnd = baseUrl:find("://") + + -- parse out the prefix (www, kyle, ying, etc.) + local prefixIndex, prefixEnd = baseUrl:find("%.", schemeEnd + 1) + local basePrefix = baseUrl:sub(schemeEnd + 1, prefixIndex - 1) + + -- parse out the domain (roblox.com/, sitetest1.robloxlabs.com/, etc.) + local baseDomain = baseUrl:sub(prefixEnd + 1) + + return baseUrl, basePrefix, baseDomain +end + +local function createAssetGameUrl(baseDomain) + return "https://assetgame." .. baseDomain +end + +local _, _, BASE_DOMAIN = parseBaseUrlInformation(argumentsTable.BaseUrl) +local ASSET_GAME = createAssetGameUrl(BASE_DOMAIN) + +local ASSET_TYPE_MAP = { + Hat = Enum.AssetType.Hat, + HairAccessory = Enum.AssetType.HairAccessory, + FaceAccessory = Enum.AssetType.FaceAccessory, + NeckAccessory = Enum.AssetType.NeckAccessory, + ShoulderAccessory = Enum.AssetType.ShoulderAccessory, + FrontAccessory = Enum.AssetType.FrontAccessory, + BackAccessory = Enum.AssetType.BackAccessory, + WaistAccessory = Enum.AssetType.WaistAccessory, +} + +local function validateAsset(assetInfo) + local assetUrl + if FFlagDebugUGCServerValidation then + assetUrl = ASSET_GAME .. string.format("asset/?id=%s", assetInfo.assetId) + else + assetUrl = ASSET_GAME .. string.format("asset/?marAssetHash=%s&marChecksum=%s", assetInfo.assetHash, assetInfo.checksum) + end + local objects = game:GetObjects(assetUrl) + + local assetTypeEnum = assert(ASSET_TYPE_MAP[assetInfo.assetType], "Invalid Asset Type") + + local success, reasons = UGCValidation.validate(objects, assetTypeEnum, true) + if success then + return true, "Success" + else + return false, table.concat(reasons, "\n") + end +end + +local assetsToValidate = argumentsTable.Assets +local resultData = {} + +for i, assetInfo in ipairs(assetsToValidate) do + local isValid, validationResult = validateAsset(assetInfo) + + resultData[i] = { + assetHash = assetInfo.assetHash, + isValid = isValid, + validationResult = validationResult + } +end + +local resultJson = HttpService:JSONEncode(resultData) + +if FFlagDebugUGCServerValidation then + print("DEBUG", "resultData", resultJson) +end + +return resultJson \ No newline at end of file diff --git a/services/grid-bot/scripts/scripts/highavailability.lua b/services/grid-bot/scripts/scripts/highavailability.lua new file mode 100644 index 00000000..945eead5 --- /dev/null +++ b/services/grid-bot/scripts/scripts/highavailability.lua @@ -0,0 +1,3 @@ +-- This script serves as content init for containers. + +return; diff --git a/services/grid-bot/scripts/thumbnails/AnimationManifest.lua b/services/grid-bot/scripts/thumbnails/AnimationManifest.lua new file mode 100644 index 00000000..fcb5c65e --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/AnimationManifest.lua @@ -0,0 +1,376 @@ +-- AnimationManifest.txt +-- V 1.0.3 + +-- Exports multiple frames +-- https://spocktopus.roblox.local/Internals_of_3D_Animated_Thumbnails + +local baseUrl, characterAppearanceUrl, animationUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +local BundleLoader = require(ThumbnailGenerator:GetThumbnailModule("BundleLoader")) + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local FRAME_RATE = 60 +local originalJointCFramesMap = {} +local originalPartsCFramesMap = {} + +local function getOriginalJointCFrames(object) + for _, child in pairs(object:GetChildren()) do + if child:IsA("Motor6D") then + originalJointCFramesMap[child] = child.C1 + end + if child:IsA("BasePart") then + originalPartsCFramesMap[child] = child.CFrame + end + getOriginalJointCFrames(child) + end +end + +local animationObjects = game:GetObjects(animationUrl) +local animations = {} +local rotateCharacter = false + +local function getAnimations(model) + for _, child in pairs(model:GetChildren()) do + if child:IsA("StringValue") and child.Name == "swim" then + rotateCharacter = true + elseif child:IsA("StringValue") and child.Name == "swimidle" then + -- Don't include swimidle if we are thumnailing the swim animation + if child.Parent and child.Parent:FindFirstChild("swim") then + return + end + end + + if child:IsA("Animation") then + table.insert(animations, child) + else + getAnimations(child) + end + end +end + +for _, animationModel in pairs(animationObjects) do + if animationModel:IsA("Animation") then + table.insert(animations, animationModel) + else + getAnimations(animationModel) + end +end + +local bundleId + +if #animations == 1 then + local anim = animations[1] + local bundleIdValue = anim:FindFirstChild("ThumbnailBundleId") + if bundleIdValue and bundleIdValue:IsA("NumberValue") then + bundleId = bundleIdValue.Value + end +end + +local character + +if bundleId then + -- Thumbnail with the loaded bundle + character = BundleLoader.LoadBundleCharacter(baseUrl, bundleId) + + ThumbnailGenerator:AddProfilingCheckpoint("BundleCharacterLoaded") +else + -- Use characterAppearanceUrl + local player = game:GetService("Players"):CreateLocalPlayer(0) + player.CharacterAppearance = characterAppearanceUrl + player:LoadCharacterBlocking() + character = player.Character + + ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") +end + +local humanoid = character:FindFirstChildOfClass("Humanoid") +getOriginalJointCFrames(character) + +local animator = humanoid:FindFirstChild("Animator") +if not animator then + animator = Instance.new("Animator") + animator.Parent = humanoid +end + +if rotateCharacter then + local rootPart = character:FindFirstChild("HumanoidRootPart") + if rootPart then + rootPart.CFrame = rootPart.CFrame * CFrame.Angles(math.rad(-90), 0, 0) + end +end + +local animationId = "" +local bestWeight = -1 +for _, anim in pairs(animations) do + local weight = anim:FindFirstChild("Weight") + if weight then + if weight.Value > bestWeight then + bestWeight = weight.Value + animationId = anim.AnimationId + end + else + animationId = anim.AnimationId + end +end + +local KeyframeSequenceProvider = game:GetService("KeyframeSequenceProvider") +local keyframeSequence = KeyframeSequenceProvider:GetKeyframeSequence(animationId) + +local animationObj = Instance.new("Animation") +animationObj.AnimationId = KeyframeSequenceProvider:RegisterActiveKeyframeSequence(keyframeSequence) + +for _, track in pairs(humanoid:GetPlayingAnimationTracks()) do + track:Stop(0) +end + +local track = humanoid:LoadAnimation(animationObj) +track:Play(0) + +ThumbnailGenerator:AddProfilingCheckpoint("AnimationsLoaded") + +local finished = false + +local function stepNextFrame() + local delta = 1/FRAME_RATE + + if track.TimePosition + delta > track.Length then + delta = track.Length - track.TimePosition + end + + if delta <= 0.005 or track.TimePosition >= track.Length or finished then + return false + end + + local preStepTimePosition = track.TimePosition + animator:StepAnimations(delta) + + if track.TimePosition <= preStepTimePosition then + finished = true + end + + return true +end + +local math_floor = math.floor +local math_sin = math.sin +local math_cos = math.cos +local math_min = math.min +local math_max = math.max +local function round(number) + return math_floor(number*10000 + 0.5)/10000 +end + +local function vector3ToTable(vector) + return { + x = round(vector.X), + y = round(vector.Y), + z = round(vector.Z) + } +end + +local animationData = {} + +while stepNextFrame() do + local frameAnimationData = {} + + for _, obj in pairs(character:GetChildren()) do + if obj:IsA("BasePart") or obj:IsA("Accoutrement") then + local part = obj + if obj:IsA("Accoutrement") then + part = obj:FindFirstChild("Handle") + end + if part and part.Name ~= "HumanoidRootPart" then + local posAndRotation = {} + posAndRotation["Position"] = vector3ToTable(part.Position) + local axis, angle = part.CFrame:toAxisAngle() + local halfAngle = angle/2 + posAndRotation["Rotation"] = { + x = round(math_sin(halfAngle)*axis.X), + y = round(math_sin(halfAngle)*axis.Y), + z = round(math_sin(halfAngle)*axis.Z), + w = round(math_cos(halfAngle)) + } + + frameAnimationData[obj.Name] = posAndRotation + end + end + end + table.insert(animationData, frameAnimationData) +end + +ThumbnailGenerator:AddProfilingCheckpoint("AnimationDataCollected") + +-- Take note of finishing CFrames for the animation +-- The CameraResult position will be transformed based on this +local animatedPartPositionsMap = {} + +for part, _ in pairs(originalPartsCFramesMap) do + animatedPartPositionsMap[part] = part.Position +end + +-- Restore original CFrames +for motor, origC1 in pairs(originalJointCFramesMap) do + motor.C1 = origC1 +end + +for part, cframe in pairs(originalPartsCFramesMap) do + part.Anchored = true + part.CFrame = cframe +end + +local partsArray = {} +for _, obj in pairs(character:GetChildren()) do + if obj:IsA("BasePart") or obj:IsA("Accoutrement") then + if obj.Name ~= "HumanoidRootPart" then + table.insert(partsArray, obj) + end + end +end + +local string_sub = string.sub +local string_len = string.len +local function strEndsWith(str, val) + return string_sub(str, -string_len(val)) == val +end + +local function replaceChar(pos, str, r) + return str:sub(1, pos - 1) ..r.. str:sub(pos + 1) +end + +game:GetService("Selection"):Set(partsArray) +local objsStrOutput, requestedUrls = ThumbnailGenerator:Click("SplitObjs", 0, 0, true) + +ThumbnailGenerator:AddProfilingCheckpoint("ObjFilesGenerated") + +local decodedObjsStrOutput = game:GetService("HttpService"):JSONDecode(objsStrOutput) +local partObjsResult = {} +local textures = {} +local cameraResult = nil + +local totalAABB = { + ["min"] = {}, + ["max"] = {} +} + +local function addToTotalAABB(partName, partAABB) + local part = character:FindFirstChild(partName) + if part and animatedPartPositionsMap[part] then + local currentPosition = part.Position + local animatedPosition = animatedPartPositionsMap[part] + local offset = animatedPosition - currentPosition + + local minJSON = partAABB["min"] + local currentMin = Vector3.new(minJSON.x, minJSON.y, minJSON.z) + currentMin = Vector3.new(currentMin.X, currentMin.Y + offset.Y, currentMin.Z) + partAABB["min"] = vector3ToTable(currentMin) + + local maxJSON = partAABB["min"] + local currentMax = Vector3.new(maxJSON.x, maxJSON.y, maxJSON.z) + currentMax = Vector3.new(currentMax.X, currentMax.Y + offset.Y, currentMax.Z) + partAABB["min"] = vector3ToTable(currentMax) + end + + for xyzkey, val in pairs(partAABB["min"]) do + if not totalAABB["min"][xyzkey] then + totalAABB["min"][xyzkey] = val + else + totalAABB["min"][xyzkey] = math_min(totalAABB["min"][xyzkey], val) + end + end + + for xyzkey, val in pairs(partAABB["max"]) do + if not totalAABB["max"][xyzkey] then + totalAABB["max"][xyzkey] = val + else + totalAABB["max"][xyzkey] = math_max(totalAABB["max"][xyzkey], val) + end + end +end + +local function resolveCameraResult(partName, cameraJSON) + local part = character:FindFirstChild(partName) + if part and animatedPartPositionsMap[part] then + local currentPosition = part.Position + local animatedPosition = animatedPartPositionsMap[part] + local offset = animatedPosition - currentPosition + + local positionJSON = cameraJSON["position"] + local cameraPosition = Vector3.new(positionJSON.x, positionJSON.y, positionJSON.z) + cameraPosition = cameraPosition + offset + cameraJSON["position"] = vector3ToTable(cameraPosition) + end + + if partName == "Head" then + cameraResult = cameraJSON + elseif cameraResult == nil then -- Fallback if Head doesn't exist for some reason + cameraResult = cameraJSON + end +end + +-- Process the SplitObjs output to consolidate it for the animation output. +-- The data for common textures is mapped to the same output in the textures table. +-- The Camera and AABB fields are consolidated into global fields. +for key, val in pairs(decodedObjsStrOutput) do + local decodedPartObj = game:GetService("HttpService"):JSONDecode(val) + if decodedPartObj["files"] then + local files = decodedPartObj["files"] + for fileName, fileInfo in pairs(files) do + local fileContent = fileInfo.content + if strEndsWith(fileName, ".png") then + local newFileName = fileName + while textures[newFileName] and textures[newFileName].content ~= fileContent do + local charsFromEnd = string_len("Tex.png") + local replacePos = string_len(newFileName) - charsFromEnd + local newNumber = tostring(tonumber(newFileName:sub(replacePos, replacePos)) + 1) + newFileName = replaceChar(replacePos, newFileName, newNumber) + end + textures[newFileName] = { + content = fileContent + } + files["texture"] = newFileName + files[fileName] = nil + end + end + end + + -- Calculate total aabb + if decodedPartObj["AABB"] then + addToTotalAABB(key, decodedPartObj["AABB"]) + decodedPartObj["AABB"] = nil + end + + -- Resolve overall Camera JSON. + if decodedPartObj["camera"] then + resolveCameraResult(key, decodedPartObj["camera"]) + decodedPartObj["camera"] = nil + end + partObjsResult[key] = decodedPartObj +end + +-- Special camera position and direction for rotated character +if rotateCharacter then + local rootPart = character:FindFirstChild("HumanoidRootPart") + if rootPart then + local cameraOffset = Vector3.new(6, 5, 7) * .7 + local cameraPosition = Vector3.new(rootPart.Position.X, rootPart.Position.Y, rootPart.Position.Z) - cameraOffset + local cameraDirection = (rootPart.Position - cameraPosition).unit + cameraResult["position"] = vector3ToTable(cameraPosition) + cameraResult["direction"] = vector3ToTable(cameraDirection) + end +end + +local resultData = { + Frames = animationData, + Camera = cameraResult, + AABB = totalAABB, + PartObjs = partObjsResult, + Textures = textures +} + +ThumbnailGenerator:AddProfilingCheckpoint("ResultFinalized") +return game:GetService("HttpService"):JSONEncode(resultData), requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/AnimationSilhouette.lua b/services/grid-bot/scripts/thumbnails/AnimationSilhouette.lua new file mode 100644 index 00000000..48edc7ce --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/AnimationSilhouette.lua @@ -0,0 +1,177 @@ +-- AnimationSilhouette.lua +-- Generates a Silhouette of a character doing the animation in the color requested + +local assetUrl, baseUrl, x, y, silhouetteColor = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +local BundleLoader = require(ThumbnailGenerator:GetThumbnailModule("BundleLoader")) + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local emoteAnim = game:GetObjects(assetUrl)[1] + +local bundleId = 401 -- Default is Alexandra Ninniflip +local bundleIdValue = emoteAnim:FindFirstChild("ThumbnailBundleId") +if bundleIdValue and bundleIdValue:IsA("NumberValue") then + bundleId = bundleIdValue.Value +end + +-- Default keyframe to use in thumbnail is middle keyframe +local thumbnailKeyframeNumber +local thumbnailKeyframeValue = emoteAnim:FindFirstChild("ThumbnailKeyframe") +if thumbnailKeyframeValue and thumbnailKeyframeValue:IsA("NumberValue") then + thumbnailKeyframeNumber = thumbnailKeyframeValue.Value +end + +local thumbnailZoom = 1 +local thumbnailZoomValue = emoteAnim:FindFirstChild("ThumbnailZoom") +if thumbnailZoomValue and thumbnailZoomValue:IsA("NumberValue") then + thumbnailZoom = thumbnailZoomValue.Value +end + +local fieldOfView = 20 +local fieldOfViewValue = emoteAnim:FindFirstChild("ThumbnailFieldOfView") +if fieldOfViewValue and fieldOfViewValue:IsA("NumberValue") then + fieldOfView = fieldOfViewValue.Value +end + +local verticalOffset = 0 +local verticalOffsetValue = emoteAnim:FindFirstChild("ThumbnailVerticalOffset") +if verticalOffsetValue and verticalOffsetValue:IsA("NumberValue") then + verticalOffset = verticalOffsetValue.Value +end + +local horizontalOffset = 0 +local horizontalOffsetValue = emoteAnim:FindFirstChild("ThumbnailHorizontalOffset") +if horizontalOffsetValue and horizontalOffsetValue:IsA("NumberValue") then + horizontalOffset = horizontalOffsetValue.Value +end + +local rotationDegrees = 0 +local thumbnailRotationValue = emoteAnim:FindFirstChild("ThumbnailCharacterRotation") +if thumbnailRotationValue and thumbnailRotationValue:IsA("NumberValue") then + rotationDegrees = thumbnailRotationValue.Value +end + +local bundleCharacter = BundleLoader.LoadBundleCharacter(baseUrl, bundleId) +ThumbnailGenerator:AddProfilingCheckpoint("BundleCharacterLoaded") + +local r, g, b = unpack(silhouetteColor:split("/")) +local silhouetteColor3 = Color3.fromRGB(tonumber(r), tonumber(g), tonumber(b)) + +local overrideColor3Value = emoteAnim:FindFirstChild("ThumbnailSilhouetteColor") +if overrideColor3Value and overrideColor3Value:IsA("Color3Value") then + silhouetteColor3 = overrideColor3Value.Value +end + +local KeyframeSequenceProvider = game:GetService("KeyframeSequenceProvider") + +local kfs = KeyframeSequenceProvider:GetKeyframeSequence(emoteAnim.AnimationId) +local emoteKeyframes = kfs:GetKeyframes() + +ThumbnailGenerator:AddProfilingCheckpoint("KeyframesLoaded") + +local function getJointBetween(part0, part1) + for _, obj in pairs(part1:GetChildren()) do + if obj:IsA("Motor6D") and obj.Part0 == part0 then + return obj + end + end +end + +local function applyPose(character, poseKeyframe) + local function recurApplyPoses(parentPose, poseObject) + if parentPose then + local joint = getJointBetween(character[parentPose.Name], character[poseObject.Name]) + joint.C1 = joint.C1 * poseObject.CFrame:inverse() + end + + for _, subPose in pairs(poseObject:GetSubPoses()) do + recurApplyPoses(poseObject, subPose) + end + end + + for _, poseObj in pairs(poseKeyframe:GetPoses()) do + recurApplyPoses(nil, poseObj) + end +end + +local thumbnailKeyframe +if thumbnailKeyframeNumber then + -- Check that the index provided as the keyframe number is valid + if thumbnailKeyframeNumber > 0 and thumbnailKeyframeNumber <= #emoteKeyframes then + thumbnailKeyframe = emoteKeyframes[thumbnailKeyframeNumber] + else + thumbnailKeyframe = emoteKeyframes[math.ceil(#emoteKeyframes/2)] + end +else + thumbnailKeyframe = emoteKeyframes[math.ceil(#emoteKeyframes/2)] +end + +if rotationDegrees ~= 0 then + local rootPose = thumbnailKeyframe:GetPoses()[1] + if rootPose then + local upperTorsoPose = rootPose:GetSubPoses()[1] + if upperTorsoPose then + upperTorsoPose.CFrame = upperTorsoPose.CFrame * CFrame.Angles(0, math.rad(rotationDegrees), 0) + end + end +end + +applyPose(bundleCharacter, thumbnailKeyframe) + +local function getCameraOffset(fov, extentsSize) + local xSize, ySize, zSize = extentsSize.X, extentsSize.Y, extentsSize.Z + + local maxSize = math.sqrt(xSize^2 + ySize^2 + zSize^2) + local fovMultiplier = 1 / math.tan(math.rad(fov) / 2) + + local halfSize = maxSize / 2 + return halfSize * fovMultiplier +end + +local function zoomExtents(model, lookVector, thumbnailCamera) + local modelCFrame = model:GetModelCFrame() + + local position = modelCFrame.p + position = position + Vector3.new(horizontalOffset, -verticalOffset, 0) + + local extentsSize = model:GetExtentsSize() + local cameraOffset = getCameraOffset(thumbnailCamera.FieldOfView, extentsSize) + + local zoomFactor = 1 / thumbnailZoom + cameraOffset = cameraOffset * zoomFactor + + local cameraRotation = thumbnailCamera.CFrame - thumbnailCamera.CFrame.p + thumbnailCamera.CFrame = cameraRotation + position + (lookVector * cameraOffset) +end + +local function createThumbnailCamera(model) + local modelCFrame = model:GetModelCFrame() + local lookVector = modelCFrame.lookVector + + local humanoidRootPart = model:FindFirstChild("HumanoidRootPart") + if humanoidRootPart then + lookVector = humanoidRootPart.CFrame.lookVector + end + + local thumbnailCamera = Instance.new("Camera") + thumbnailCamera.Name = "ThumbnailCamera" + thumbnailCamera.Parent = model + + thumbnailCamera.FieldOfView = fieldOfView + thumbnailCamera.CFrame = CFrame.new(modelCFrame.p + (lookVector * 5), modelCFrame.p) + thumbnailCamera.Focus = modelCFrame + + zoomExtents(model, lookVector, thumbnailCamera) +end + +createThumbnailCamera(bundleCharacter) + +local result, requestedUrls = game:GetService("ThumbnailGenerator"):ClickSilhouette(x, y, silhouetteColor3) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Avatar.lua b/services/grid-bot/scripts/thumbnails/Avatar.lua new file mode 100644 index 00000000..73c5374c --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Avatar.lua @@ -0,0 +1,31 @@ +-- Avatar v1.0.2 +-- This is the thumbnail script for R6 avatars. Straight up and down, with the right arm out if they have a gear. + +local characterAppearanceUrl, baseUrl, fileExtension, x, y = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local player = game:GetService("Players"):CreateLocalPlayer(0) +player.CharacterAppearance = characterAppearanceUrl +player:LoadCharacterBlocking() + +ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") + +-- Raise up the character's arm if they have gear. +if player.Character then + for _, child in pairs(player.Character:GetChildren()) do + if child:IsA("Tool") then + player.Character.Torso["Right Shoulder"].CurrentAngle = math.rad(90) + break + end + end +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/AvatarAnimation.lua b/services/grid-bot/scripts/thumbnails/AvatarAnimation.lua new file mode 100644 index 00000000..d5554237 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/AvatarAnimation.lua @@ -0,0 +1,129 @@ +-- AvatarAnimation V 1.0.3 +-- Creates a thumbnail from the middle Keyframe of an animation. + +-- Example arguments: +-- https://www.sitetest3.robloxlabs.com/Asset/AvatarAccoutrements.ashx?AvatarHash=80dcfa39a8e90f690d2be0e56239abbe&AssetIDs=42900214,32357663,32357631,32357619,32357584,32357558&ResolvedAvatarType=R15, +-- http://www.sitetest3.robloxlabs.com/, +-- Png, +-- 600, +-- 600, +-- http://www.sitetest3.robloxlabs.com/Asset/?hash=ca005a9e83ac95bd5068029afdc65496 + +local characterAppearanceUrl, baseUrl, fileExtension, x, y, animationUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local player = game:GetService("Players"):CreateLocalPlayer(0) +player.CharacterAppearance = characterAppearanceUrl +player:LoadCharacterBlocking() + +ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") + +local function getJointBetween(part0, part1) + for _, obj in pairs(part1:GetChildren()) do + if obj:IsA("Motor6D") and obj.Part0 == part0 then + return obj + end + end +end + +local function applyR15Pose(character, poseKeyframe) + local function recurApplyPoses(parentPose, poseObject) + if parentPose then + local joint = getJointBetween(character[parentPose.Name], character[poseObject.Name]) + joint.C1 = joint.C1 * poseObject.CFrame:inverse() + end + for _, subPose in pairs(poseObject:GetSubPoses()) do + recurApplyPoses(poseObject, subPose) + end + end + + for _, poseObj in pairs(poseKeyframe:GetPoses()) do + recurApplyPoses(nil, poseObj) + end +end + +local animationObjects = game:GetObjects(animationUrl) + +local poseAnimation = nil +local animations = {} +local rotateCharacter = false +local thumbnailCamera = nil + +local function getAnimations(model) + for _, child in pairs(model:GetChildren()) do + if child:IsA("Animation") then + if string.lower(model.Name) == "pose" then + poseAnimation = child + else + table.insert(animations, child) + end + else + getAnimations(child) + end + + if child:IsA("Camera") and child.Name == "ThumbnailCamera" then + thumbnailCamera = child:Clone() + end + + if child:IsA("StringValue") and child.Name == "swim" then + rotateCharacter = true + end + end +end + +for _, animationModel in pairs(animationObjects) do + getAnimations(animationModel) +end + +local KeyframeSequenceProvider = game:GetService("KeyframeSequenceProvider") +local keyframes = {} + +if poseAnimation then + local kfs = KeyframeSequenceProvider:GetKeyframeSequence(poseAnimation.AnimationId) + local animKeyframes = kfs:GetKeyframes() + for _, keyframe in pairs(animKeyframes) do + table.insert(keyframes, keyframe) + end +else + for _, animation in pairs(animations) do + local kfs = KeyframeSequenceProvider:GetKeyframeSequence(animation.AnimationId) + local animKeyframes = kfs:GetKeyframes() + for _, keyframe in pairs(animKeyframes) do + table.insert(keyframes, keyframe) + end + end +end + +ThumbnailGenerator:AddProfilingCheckpoint("AnimationsLoaded") + +local keyframe = keyframes[math.max(1, math.floor(#keyframes/2))] +applyR15Pose(player.Character, keyframe) + +if rotateCharacter then + local rootPart = player.Character:FindFirstChild("HumanoidRootPart") + if rootPart then + rootPart.CFrame = rootPart.CFrame * CFrame.Angles(math.rad(-90), 0, 0) + if not thumbnailCamera then + local camera = Instance.new("Camera") + camera.Name = "ThumbnailCamera" + + local rotatedCameraOffset = Vector3.new(6, 5, 7) * .7 + camera.CFrame = CFrame.new(Vector3.new(rootPart.Position.X, rootPart.Position.Y, rootPart.Position.Z) - rotatedCameraOffset, rootPart.Position) + camera.Parent = player.Character + end + end +end + +if thumbnailCamera then + thumbnailCamera.Parent = player.Character +end + +local result, requestedUrls = game:GetService("ThumbnailGenerator"):Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Avatar_R15_Action.lua b/services/grid-bot/scripts/thumbnails/Avatar_R15_Action.lua new file mode 100644 index 00000000..5cb633bc --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Avatar_R15_Action.lua @@ -0,0 +1,172 @@ +-- Avatar_R15_Action v1.1.1 +-- For R6, this generates the normal with/without gear pose. For R15 it positions their body in an action pose. +local baseUrl, characterAppearanceUrl, fileExtension, x, y = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local player = game:GetService("Players"):CreateLocalPlayer(0) +player.CharacterAppearance = characterAppearanceUrl +player:LoadCharacterBlocking() + +ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") + +local poseAnimationId = "http://www.roblox.com/asset/?id=532421348" + +local function getJointBetween(part0, part1) + for _, obj in pairs(part1:GetChildren()) do + if obj:IsA("Motor6D") and obj.Part0 == part0 then + return obj + end + end +end + +local function applyKeyframe(character, poseKeyframe) + local function recurApplyPoses(parentPose, poseObject) + if parentPose then + local joint = getJointBetween(character[parentPose.Name], character[poseObject.Name]) + if joint and poseObject.Weight ~= 0 then + joint.C1 = poseObject.CFrame:inverse() + joint.C1.p + end + end + for _, subPose in pairs(poseObject:GetSubPoses()) do + recurApplyPoses(poseObject, subPose) + end + end + + for _, poseObj in pairs(poseKeyframe:GetPoses()) do + recurApplyPoses(nil, poseObj) + end +end + +local function applyR15Pose(character) + local poseKeyframSequence = game:GetService("KeyframeSequenceProvider"):GetKeyframeSequence(poseAnimationId) + local poseKeyframe = poseKeyframSequence:GetKeyframes()[1] + + applyKeyframe(character, poseKeyframe) +end + +local function findAttachmentsRecur(parent, resultTable, returnDictionary) + for _, obj in pairs(parent:GetChildren()) do + if obj:IsA("Attachment") then + if returnDictionary then + resultTable[obj.Name] = obj + else + resultTable[#resultTable + 1] = obj + end + elseif not obj:IsA("Tool") and not obj:IsA("Accoutrement") then -- Leave out tools and accoutrements in the character + findAttachmentsRecur(obj, resultTable, returnDictionary) + end + end +end + +local function findAttachmentsInTool(tool) + local attachments = {} + findAttachmentsRecur(tool, attachments, false) + return attachments +end + +local function findAttachmentsInCharacter(character) + local attachments = {} + findAttachmentsRecur(character, attachments, true) + return attachments +end + +local function weldAttachments(attach1, attach2) + local weld = Instance.new("Weld") + weld.Part0 = attach1.Parent + weld.Part1 = attach2.Parent + weld.C0 = attach1.CFrame + weld.C1 = attach2.CFrame + weld.Parent = attach1.Parent + return weld +end + +local function findFirstMatchingAttachment(model, name) + for _, child in pairs(model:GetChildren()) do + if child:IsA("Attachment") and child.Name == name then + return child + elseif not child:IsA("Accoutrement") and not child:IsA("Tool") then + local foundAttachment = findFirstMatchingAttachment(child, name) + if foundAttachment then + return foundAttachment + end + end + end +end + +local function doR15ToolPose(character, humanoid, tool) + local characterAttachments = findAttachmentsInCharacter(character) + local toolAttachments = findAttachmentsInTool(tool) + local foundAttachments = false + -- If matching attachments exist in the gear then weld them and do the "action" R15 pose. + -- Otherwise keep the R15 in the T-Pose position and just raise the arm. + for _, attachment in pairs(toolAttachments) do + local matchingAttachment = characterAttachments[attachment.Name] + if matchingAttachment then + foundAttachments = true + weldAttachments(matchingAttachment, attachment) + end + end + + if foundAttachments then + tool.Parent = character + applyR15Pose(character) + + local toolPose = tool:FindFirstChild("ThumbnailPose") + if toolPose and toolPose:IsA("Keyframe") then + applyKeyframe(character, toolPose) + end + else + tool.Parent = nil + local rightShoulderJoint = getJointBetween(character.UpperTorso, character.RightUpperArm) + if rightShoulderJoint then + rightShoulderJoint.C1 = rightShoulderJoint.C1 * CFrame.new(0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 1, 0):inverse() + end + if tool:FindFirstChild("Handle") then + local attachment = findFirstMatchingAttachment(character, "RightGripAttachment") + if attachment then + tool.Handle.CFrame = attachment.Parent.CFrame * attachment.CFrame * tool.Grip:inverse() + end + end + humanoid:EquipTool(tool) + end +end + +local character = player.Character +if character then + local tool = character:FindFirstChildOfClass("Tool") + local humanoid = character:FindFirstChildOfClass("Humanoid") + local animateScript = character:FindFirstChild("Animate") + if animateScript then + local equippedPoseValue = animateScript:FindFirstChild("Pose") or animateScript:FindFirstChild("pose") + if equippedPoseValue then + local poseAnim = equippedPoseValue:FindFirstChildOfClass("Animation") + if poseAnim then + poseAnimationId = poseAnim.AnimationId + end + end + end + + if humanoid then + if humanoid.RigType == Enum.HumanoidRigType.R6 then + if tool then + character.Torso["Right Shoulder"].CurrentAngle = math.rad(90) + end + elseif humanoid.RigType == Enum.HumanoidRigType.R15 then + if tool then + doR15ToolPose(character, humanoid, tool) + else + applyR15Pose(character) + end + end + end +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Avatar_R15_Standard.lua b/services/grid-bot/scripts/thumbnails/Avatar_R15_Standard.lua new file mode 100644 index 00000000..554c60dc --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Avatar_R15_Standard.lua @@ -0,0 +1,56 @@ +-- Avatar_R15_Standard v1.0.2 +-- Pose R6 characters in the normal way. For R15, have them in the same pose, and raise their arm up if they have gear. +-- Sample params: +-- baseUrl: "http://www.roblox.com/" +-- characterAppearanceUrl: "http://www.roblox.com/Asset/AvatarAccoutrements.ashx?AvatarHash=98925edb8aa60e39ba8a4f0bf8b71d6f&AssetIDs=3372792,9255011,20418682,68258723,158066137,232503325,244097060,248286896,264611665,376530220,376531012,376531300,376531703,376532000,624157131&ResolvedAvatarType=R15&Height=1&Width=0.75&Head=0.95&Depth=0.88" +-- fileExtension: "Png" +-- x: 1260 +-- y: 1260 + +local baseUrl, characterAppearanceUrl, fileExtension, x, y = ... + +local ThumbnailGenerator = game:GetService('ThumbnailGenerator') +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local player = game:GetService("Players"):CreateLocalPlayer(0) +player.CharacterAppearance = characterAppearanceUrl +player:LoadCharacterBlocking() +ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") + +local function getJointBetween(part0, part1) + for _, obj in pairs(part1:GetChildren()) do + if obj:IsA("Motor6D") and obj.Part0 == part0 then + return obj + end + end +end + +local function doR15ToolPose(rig) + local rightShoulderJoint = getJointBetween(rig.UpperTorso, rig.RightUpperArm) + if rightShoulderJoint then + rightShoulderJoint.C1 = rightShoulderJoint.C1 * CFrame.new(0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 1, 0):inverse() + end +end + +-- Raise right arm up to hold gear. +local character = player.Character +if character then + if character:FindFirstChildOfClass("Tool") then + local humanoid = character:FindFirstChildOfClass("Humanoid") + if humanoid then + if humanoid.RigType == Enum.HumanoidRigType.R6 then + character.Torso['Right Shoulder'].CurrentAngle = math.rad(90) + elseif humanoid.RigType == Enum.HumanoidRigType.R15 then + doR15ToolPose(character) + end + end + end +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/BodyPart.lua b/services/grid-bot/scripts/thumbnails/BodyPart.lua new file mode 100644 index 00000000..8019c05d --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/BodyPart.lua @@ -0,0 +1,273 @@ +-- BodyPart v1.0.6 +-- See http://wiki.roblox.com/index.php?title=R15_Compatibility_Guide#Package_Parts for details on how body parts work with R15 + +local assetUrl, baseUrl, fileExtension, x, y, R6RigUrl, customUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +local DFFlagBodyPartFocusInThreeDThumbnails = settings():GetFFlag("BodyPartFocusInThreeDThumbnails") + +local CreateExtentsMinMax +local MannequinUtility +local ScaleUtility + +if DFFlagBodyPartFocusInThreeDThumbnails then + CreateExtentsMinMax = require(ThumbnailGenerator:GetThumbnailModule("CreateExtentsMinMax")) + MannequinUtility = require(ThumbnailGenerator:GetThumbnailModule("MannequinUtility")) + ScaleUtility = require(ThumbnailGenerator:GetThumbnailModule("ScaleUtility")) +end + +pcall(function() game:GetService('ContentProvider'):SetBaseUrl(baseUrl) end) +game:GetService('ScriptContext').ScriptsDisabled = true + +local objects = game:GetObjects(assetUrl) +ThumbnailGenerator:AddProfilingCheckpoint("BodyPartLoaded") + +local useR15 = false +local useR15NewNames = false +local bodyPartProportion = "Classic" +local floatMax = math.huge + +if DFFlagBodyPartFocusInThreeDThumbnails then + for _, object in pairs(objects) do + if object:IsA("Folder") then + if object.Name == "R15" then + useR15 = true + elseif object.Name == "R15ArtistIntent" then + useR15 = true + useR15NewNames = true + end + end + end + + bodyPartProportion = ScaleUtility.GetObjectsScaleType(objects) +else + for _, object in pairs(objects) do + if object:IsA("Folder") and (object.Name == "R15" or object.Name == "R15ArtistIntent") then + useR15 = true + + -- Check to see if there are native scale parts in this object + local partScaleType = object:FindFirstChild("AvatarPartScaleType", true) + if partScaleType then + bodyPartProportion = partScaleType.Value + end + break + end + end +end + +local mannequin +if DFFlagBodyPartFocusInThreeDThumbnails then + if useR15 then + mannequin = MannequinUtility.LoadMannequinForScaleType(bodyPartProportion) + else + mannequin = MannequinUtility.LoadR6Mannequin() + end +else + local R15RigUrl = "http://www.roblox.com/asset/?id=516159357" + for _, object in pairs(objects) do + if object:IsA("Folder") and object.Name == "R15ArtistIntent" then + useR15NewNames = true + if bodyPartProportion == "Classic" then + R15RigUrl = "http://www.roblox.com/asset/?id=1664543044" + else + R15RigUrl = "http://www.roblox.com/asset/?id=2337256345" + end + break + end + end + + if useR15 then + mannequin = game:GetObjects(R15RigUrl)[1] + else + mannequin = game:GetObjects(R6RigUrl)[1] + end + + mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None + mannequin.Parent = workspace +end + +ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + +game:GetObjects(customUrl)[1].Parent = mannequin + +ThumbnailGenerator:AddProfilingCheckpoint("CustomUrlLoaded") + +local function addFolderChildren(folder, focusPartNamesOut, focusPartsOut) + for _, child in pairs(folder:GetChildren()) do + local existingBodyPart = mannequin:FindFirstChild(child.Name) + if existingBodyPart then + existingBodyPart:Destroy() + end + child.Parent = mannequin + table.insert(focusPartNamesOut, child.name) + table.insert(focusPartsOut, child) + end +end + +local r15FolderName = "R15" +if (useR15 and useR15NewNames) then + r15FolderName = "R15ArtistIntent" +end + +local focusParts = {} +local focusPartNames = {} + +for _, object in pairs(objects) do + if useR15 and object:IsA("Folder") and object.Name == r15FolderName then + addFolderChildren(object, focusPartNames, focusParts) + elseif not useR15 and object:IsA("Folder") and object.Name == "R6" then + addFolderChildren(object, focusPartNames, focusParts) + elseif not (object:IsA("Folder") and string.find(object.Name, "R15")) then -- There will now be MULTIPLE R15 Folders. Ignore the ones we didn't search for. + object.Parent = mannequin + end +end + +local function buildJoint(parentAttachment, partForJointAttachment) + local jointName = parentAttachment.Name:gsub("RigAttachment", "") + local motor = partForJointAttachment.Parent:FindFirstChild(jointName) + if not motor then + motor = Instance.new("Motor6D") + end + motor.Name = jointName + + motor.Part0 = parentAttachment.Parent + motor.Part1 = partForJointAttachment.Parent + + motor.C0 = parentAttachment.CFrame + motor.C1 = partForJointAttachment.CFrame + + motor.Parent = partForJointAttachment.Parent +end + +-- Builds an R15 rig from the attachments in the parts +local function buildRigFromAttachments(currentPart, lastPart) + local validSiblings = {} + for _, sibling in pairs(currentPart.Parent:GetChildren()) do + -- Don't find matching attachment in the current part being processed. + -- Don't visit the last part visited again, this would cause an infinite loop. + if sibling:IsA("BasePart") and sibling ~= currentPart and sibling ~= lastPart then + table.insert(validSiblings, sibling) + end + end + + local function processRigAttachment(attachment) + for _, sibling in pairs(validSiblings) do + local matchingAttachment = sibling:FindFirstChild(attachment.Name) + if matchingAttachment then + buildJoint(attachment, matchingAttachment) + buildRigFromAttachments(matchingAttachment.Parent, currentPart) + end + end + end + + for _, object in pairs(currentPart:GetChildren()) do + if object:IsA("Attachment") and string.find(object.Name, "RigAttachment") then + processRigAttachment(object) + end + end +end + +if useR15 then + if DFFlagBodyPartFocusInThreeDThumbnails then + local humanoid = mannequin:FindFirstChild("Humanoid") + if humanoid then + ScaleUtility.CreateProportionScaleValues(humanoid, bodyPartProportion) + humanoid:BuildRigFromAttachments() + end + else + -- Build R15 rig + local humanoidRootPart = mannequin:WaitForChild("HumanoidRootPart") + humanoidRootPart.CFrame = CFrame.new(Vector3.new(0, 5, 0)) * CFrame.Angles(0, math.pi, 0) + humanoidRootPart.Anchored = true + buildRigFromAttachments(humanoidRootPart) + local humanoid = mannequin:WaitForChild("Humanoid") + if humanoid then + local typeObject = humanoid:FindFirstChild("BodyTypeScale") + + if typeObject == nil then + typeObject = Instance.new("NumberValue") + typeObject.Name = "BodyTypeScale" + typeObject.Value = 0 + typeObject.Parent = humanoid + end + + local proportionObject = humanoid:FindFirstChild("BodyProportionScale") + if proportionObject == nil then + proportionObject = Instance.new("NumberValue") + proportionObject.Name = "BodyProportionScale" + proportionObject.Value = 0 + proportionObject.Parent = humanoid + end + + if bodyPartProportion == "ProportionsNormal" then + typeObject.Value = 1 + proportionObject.Value = 0 + elseif bodyPartProportion == "ProportionsSlender" then + typeObject.Value = 1 + proportionObject.Value = 1 + end + end + end +end + +local function addToBounds(cornerPosition, focusExtentsOut) + focusExtentsOut["minx"] = math.min(focusExtentsOut["minx"], cornerPosition.x) + focusExtentsOut["miny"] = math.min(focusExtentsOut["miny"], cornerPosition.y) + focusExtentsOut["minz"] = math.min(focusExtentsOut["minz"], cornerPosition.z) + focusExtentsOut["maxx"] = math.max(focusExtentsOut["maxx"], cornerPosition.x) + focusExtentsOut["maxy"] = math.max(focusExtentsOut["maxy"], cornerPosition.y) + focusExtentsOut["maxz"] = math.max(focusExtentsOut["maxz"], cornerPosition.z) +end + +local function addCornerToBounds(partCFrame, cornerSelect, halfPartSize, focusExtentsOut) + local cornerPositionLocal = cornerSelect * halfPartSize + local cornerPositionWorld = partCFrame * cornerPositionLocal + addToBounds(cornerPositionWorld, focusExtentsOut) +end + +local extentsMinMax +local shouldCrop = false + +if DFFlagBodyPartFocusInThreeDThumbnails then + extentsMinMax = CreateExtentsMinMax(focusParts) + shouldCrop = #focusParts > 0 +else + local focusOnExtents = { minx = floatMax, miny = floatMax, minz = floatMax, maxx = -floatMax, maxy = -floatMax, maxz = -floatMax } + + local FFlagThumbnailSupportFocusOnPart = settings():GetFFlag("ThumbnailSupportFocusOnPart") + if FFlagThumbnailSupportFocusOnPart and string.lower(fileExtension) == "png" then + -- expand focusOnExtents to bound all the part(s) in the focusPartNames table + if #focusPartNames > 0 then + for _, focusPartName in pairs(focusPartNames) do + local focusPart = mannequin:FindFirstChild(focusPartName, --[[recursive = ]] true) + if focusPart then + local partPosition = focusPart.Position + local partRotation = focusPart.Rotation + local halfPartSize = focusPart.Size / 2.0 + local partCFrame = CFrame.Angles(math.rad(partRotation.x), math.rad(partRotation.y), math.rad(partRotation.z)) + partPosition + addCornerToBounds(partCFrame, Vector3.new( 1, 1, 1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new( 1, 1,-1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new( 1,-1, 1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new( 1,-1,-1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new(-1, 1, 1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new(-1, 1,-1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new(-1,-1, 1), halfPartSize, focusOnExtents) + addCornerToBounds(partCFrame, Vector3.new(-1,-1,-1), halfPartSize, focusOnExtents) + shouldCrop = true + end + end + end + end + + extentsMinMax = { + Vector3.new(focusOnExtents["minx"], focusOnExtents["miny"], focusOnExtents["minz"]), + Vector3.new(focusOnExtents["maxx"], focusOnExtents["maxy"], focusOnExtents["maxz"]) + } +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true, --[[crop = ]] shouldCrop, extentsMinMax) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Closeup.lua b/services/grid-bot/scripts/thumbnails/Closeup.lua new file mode 100644 index 00000000..456be696 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Closeup.lua @@ -0,0 +1,98 @@ +-- Closeup v1.0.3 +-- Used for avatar closeup (aka "headshot") +local baseUrl, characterAppearanceUrl, fileExtension, x, y, quadratic, baseHatZoom, maxHatZoom, cameraOffsetX, cameraOffsetY = ... + +local FFlagOnlyCheckHeadAccessoryInHeadShot = game:DefineFastFlag("OnlyCheckHeadAccessoryInHeadShot", false) +local FFlagNewHeadshotLighting = game:DefineFastFlag("NewHeadshotLighting", false) + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService('ContentProvider'):SetBaseUrl(baseUrl) end) +game:GetService('ScriptContext').ScriptsDisabled = true + +local player = game:GetService("Players"):CreateLocalPlayer(0) +player.CharacterAppearance = characterAppearanceUrl +player:LoadCharacterBlocking() + +ThumbnailGenerator:AddProfilingCheckpoint("PlayerCharacterLoaded") + +local headAttachments = {} +if FFlagOnlyCheckHeadAccessoryInHeadShot then + if player.Character:FindFirstChild("Head") then + for _,child in pairs(player.Character.Head:GetChildren()) do + if child:IsA("Attachment") then + headAttachments[child.Name] = true + end + end + end +end + +local maxDimension = 0 + +if player.Character then + -- Remove gear + for _, child in pairs(player.Character:GetChildren()) do + if child:IsA("Tool") then + child:Destroy() + elseif child:IsA("Accoutrement") then + local handle = child:FindFirstChild("Handle") + if handle then + local attachment = handle:FindFirstChildWhichIsA("Attachment") + --legacy hat does not have attachment in it and should be considered when zoom out camera + if not FFlagOnlyCheckHeadAccessoryInHeadShot or not attachment or headAttachments[attachment.Name] then + local size = handle.Size / 2 + handle.Position - player.Character.Head.Position + local xy = Vector2.new(size.x, size.y) + if xy.magnitude > maxDimension then + maxDimension = xy.magnitude + end + end + end + end + end + + -- Setup Camera + local maxHatOffset = 0.5 -- Maximum amount to move camera upward to accomodate large hats + maxDimension = math.min(1, maxDimension / 3) -- Confine maxdimension to specific bounds + + if quadratic then + maxDimension = maxDimension * maxDimension -- Zoom out on quadratic interpolation + end + + local viewOffset = player.Character.Head.CFrame * CFrame.new(cameraOffsetX, cameraOffsetY + maxHatOffset * maxDimension, 0.1) -- View vector offset from head + + local yAngle = -math.pi / 16 + if FFlagNewHeadshotLighting then + yAngle = 0 -- Camera is looking straight at avatar's face. + end + local positionOffset = player.Character.Head.CFrame + (CFrame.Angles(0, yAngle, 0).lookVector.unit * 3) -- Position vector offset from head + + local camera = Instance.new("Camera", player.Character) + camera.Name = "ThumbnailCamera" + camera.CameraType = Enum.CameraType.Scriptable + camera.CoordinateFrame = CFrame.new(positionOffset.p, viewOffset.p) + camera.FieldOfView = baseHatZoom + (maxHatZoom - baseHatZoom) * maxDimension + + if FFlagNewHeadshotLighting then + -- New lighting setup: we want a light slightly in front of, to the right, and above the character. + -- Adding Part to be anchor of light. For 3D thumbnails (like full avatar) we should be careful about adding parts as this can affect the bounds. + local part = Instance.new("Part") + part.Parent = game.Workspace + part.Anchored = true + part.Transparency = 1 + + local light = Instance.new("PointLight") + light.Color = Color3.new(255/255, 255/255, 255/255) + light.Brightness = 3 + light.Range = 10 + light.Parent = part + light.Shadows = true + + part.Position = Vector3.new(-5,110,-5) + end +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Decal.lua b/services/grid-bot/scripts/thumbnails/Decal.lua new file mode 100644 index 00000000..9cf03ec7 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Decal.lua @@ -0,0 +1,27 @@ +-- Decal v1.0.2 +-- Used for faces and decals + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService('ScriptContext').ScriptsDisabled = true + +local decal = game:GetObjects(assetUrl)[1] +ThumbnailGenerator:AddProfilingCheckpoint("DecalLoaded") + +local image, requestedUrls +local success = pcall(function() + image, requestedUrls = ThumbnailGenerator:ClickTexture(decal.Texture, fileExtension, x, y) +end) + +ThumbnailGenerator:AddProfilingCheckpoint("TextureGenerated") + +if success then + return image, requestedUrls +end + +-- if we fail return the hourglass, since we're probably in moderation. +return "/9j/4AAQSkZJRgABAQEASABIAAD//gATQ3JlYXRlZCB3aXRoIEdJTVD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wgARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQBAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADEAAAAX8P/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABBQJ//8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAwEBPwF//8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEBPwF//8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQAGPwJ//8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPyF//9oADAMBAAIAAwAAABAf/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAwEBPxB//8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEBPxB//8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxB//9k=", {decal.Texture} \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Gear.lua b/services/grid-bot/scripts/thumbnails/Gear.lua new file mode 100644 index 00000000..68e11256 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Gear.lua @@ -0,0 +1,20 @@ +-- Gear v1.0.3 + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +for _, object in pairs(game:GetObjects(assetUrl)) do + object.Parent = workspace +end + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true, --[[crop =]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Hat.lua b/services/grid-bot/scripts/thumbnails/Hat.lua new file mode 100644 index 00000000..507ca7df --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Hat.lua @@ -0,0 +1,72 @@ +-- Hat v1.1.0 + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +local DFFlagHatThumbnailMannequins = settings():GetFFlag("HatThumbnailMannequins") + +-- Modules +local CreateExtentsMinMax +local MannequinUtility +local ScaleUtility + +if DFFlagHatThumbnailMannequins then + CreateExtentsMinMax = require(ThumbnailGenerator:GetThumbnailModule("CreateExtentsMinMax")) + MannequinUtility = require(ThumbnailGenerator:GetThumbnailModule("MannequinUtility")) + ScaleUtility = require(ThumbnailGenerator:GetThumbnailModule("ScaleUtility")) +end + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local accoutrement = game:GetObjects(assetUrl)[1] +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local handle + +if DFFlagHatThumbnailMannequins then + local accoutrementScaleType = ScaleUtility.GetScaleTypeForAccessory(accoutrement) + local mannequin = MannequinUtility.LoadMannequinForScaleType(accoutrementScaleType) + + ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + + -- Rotate mannequin for back accessories + handle = accoutrement:FindFirstChild("Handle") + if handle and handle:FindFirstChild("BodyBackAttachment") then + MannequinUtility.RotateMannequin(mannequin, CFrame.Angles(0, math.pi, 0)) + end + + -- Scale mannequin based on accoutrement scale type + local humanoid = mannequin:FindFirstChild("Humanoid") + if humanoid then + ScaleUtility.CreateProportionScaleValues(humanoid, accoutrementScaleType) + humanoid:BuildRigFromAttachments() + end + + accoutrement.Parent = mannequin +else + accoutrement.Parent = workspace +end + +local focusParts = {} +local extentsMinMax + +if DFFlagHatThumbnailMannequins then + if handle then + focusParts[#focusParts + 1] = handle + + local connectedParts = handle:GetConnectedParts() + for _, part in pairs(connectedParts) do + focusParts[#focusParts + 1] = part + end + end + + extentsMinMax = CreateExtentsMinMax(focusParts) +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true, --[[crop =]] true, extentsMinMax) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Head.lua b/services/grid-bot/scripts/thumbnails/Head.lua new file mode 100644 index 00000000..238ffd3c --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Head.lua @@ -0,0 +1,177 @@ +-- Head v1.2.0 + +local assetUrl, fileExtension, x, y, baseUrl, mannequinId = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +local DFFlagHeadThumbnailMannequins = settings():GetFFlag("HeadThumbnailMannequins") +local FFlagMeshPartHeadOption = settings():GetFFlag("MeshPartHeadOption") + +-- Modules +local CreateExtentsMinMax +local MannequinUtility +local ScaleUtility + +if DFFlagHeadThumbnailMannequins or FFlagMeshPartHeadOption then + CreateExtentsMinMax = require(ThumbnailGenerator:GetThumbnailModule("CreateExtentsMinMax")) + MannequinUtility = require(ThumbnailGenerator:GetThumbnailModule("MannequinUtility")) + ScaleUtility = require(ThumbnailGenerator:GetThumbnailModule("ScaleUtility")) +end + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local objects = {} +local headScaleType +local mannequin + +if FFlagMeshPartHeadOption then + local HeadMannequin = MannequinUtility.LoadMannequinForScaleType("Classic") + HeadMannequin.Parent = workspace + local mcd = HeadMannequin.Humanoid:GetAppliedDescription() + + if mcd then + local HeadId = assetUrl:match("*?id=(%d+)") + mcd.Head = HeadId + + local gray = BrickColor.Gray() + mcd.HeadColor = Color3.new(gray.r, gray.g, gray.b) + HeadMannequin.Humanoid:ApplyDescriptionBlocking(mcd) + if HeadMannequin.Head:IsA("MeshPart") then + objects[1] = HeadMannequin.Head + else + objects[1] = HeadMannequin.Head.Mesh + end + end + HeadMannequin.Parent = nil +else + objects = game:GetObjects(assetUrl) +end + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +if DFFlagHeadThumbnailMannequins then + headScaleType = ScaleUtility.GetObjectsScaleType(objects) + mannequin = MannequinUtility.LoadMannequinForScaleType(headScaleType) +else + mannequin = game:GetObjects(baseUrl.. "asset/?id=" .. tostring(mannequinId))[1] + mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None + mannequin.Parent = workspace +end + +ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + +local function addFaceDecal(head) + if head:FindFirstChild("face") then + return + end + + local face = Instance.new("Decal") + face.Name = "face" + face.Texture = "rbxasset://textures/face.png" + face.Parent = head +end + +local function replaceMannequinMeshPartHead(meshPartHead, meshHead) + local newHead = Instance.new("Part") + newHead.Size = meshPartHead.Size + newHead.CFrame = meshPartHead.CFrame + newHead.Color = meshPartHead.Color + newHead.Name = "Head" + + addFaceDecal(newHead) + + local copiedAttachments = false + for _, child in pairs(meshHead:GetChildren()) do + if child:IsA("Vector3Value") and string.find(child.Name, "Attachment") then + copiedAttachments = true + + local newAttachment = Instance.new("Attachment") + newAttachment.Name = child.Name + newAttachment.Position = child.Value + newAttachment.Parent = newHead + end + end + + if not copiedAttachments then + for _, child in pairs(meshPartHead:GetChildren()) do + child.Parent = newHead + end + end + + meshPartHead:Destroy() + newHead.Parent = mannequin +end + +local function replaceMannequinHeadWithMeshHead() + for _, obj in pairs(objects) do + if obj:IsA("Folder") and obj.Name == "R15ArtistIntent" then + local head = obj.Head + addFaceDecal(head) + mannequin.Head:Destroy() + head.Parent = mannequin + end + end +end + +local headObject = objects[1] + +if headObject:IsA("Folder") then + replaceMannequinHeadWithMeshHead() +else + if DFFlagHeadThumbnailMannequins then + if mannequin.Head:IsA("MeshPart") then + replaceMannequinMeshPartHead(mannequin.Head, headObject) + end + else + if headObject:IsA("MeshPart") then + replaceMannequinMeshPartHead(mannequin.Head, headObject) + else + mannequin.Head.BrickColor = BrickColor.Gray() + end + end + + if not headObject:IsA("MeshPart") then + if mannequin.Head:FindFirstChild("Mesh") then + mannequin.Head.Mesh:Destroy() + end + headObject.Parent = mannequin.Head + else + addFaceDecal(headObject) + mannequin.Head:Destroy() + headObject.Parent = mannequin + end +end + +if DFFlagHeadThumbnailMannequins then + -- Scale mannequin based on the scale type of the Head + local humanoid = mannequin:FindFirstChild("Humanoid") + if humanoid then + ScaleUtility.CreateProportionScaleValues(humanoid, headScaleType) + humanoid:BuildRigFromAttachments() + end +else + for _, child in pairs(mannequin:GetChildren()) do + if child:IsA("BasePart") and child.Name ~= "Head" then + child:Destroy() + end + end +end + +local shouldCrop = false +local extentsMinMax + +if DFFlagHeadThumbnailMannequins then + local focusParts = { + mannequin:FindFirstChild("Head") + } + + shouldCrop = #focusParts > 0 + extentsMinMax = CreateExtentsMinMax(focusParts) +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true, shouldCrop, extentsMinMax) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Mesh.lua b/services/grid-bot/scripts/thumbnails/Mesh.lua new file mode 100644 index 00000000..7ac70cb8 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Mesh.lua @@ -0,0 +1,21 @@ +-- Mesh v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local part = Instance.new("Part") +part.Parent = workspace + +local specialMesh = Instance.new("SpecialMesh") +specialMesh.MeshId = assetUrl +specialMesh.Parent = part + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true, --[[crop = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/MeshPart.lua b/services/grid-bot/scripts/thumbnails/MeshPart.lua new file mode 100644 index 00000000..3f4ee6f5 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/MeshPart.lua @@ -0,0 +1,46 @@ +-- MeshPart v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true +game:DefineFastFlag("OnlyAllowMeshParts", false) + +for _, object in pairs(game:GetObjects(assetUrl)) do + if game:GetFastFlag("OnlyAllowMeshParts") then + if object:IsA("MeshPart") and #object:GetChildren() == 0 then + pcall(function() object.Parent = workspace end) + break + end + else + if object:IsA("Sky") then + local resultValues = nil + local success = pcall(function() resultValues = {ThumbnailGenerator:ClickTexture(object.SkyboxFt, fileExtension, x, y)} end) + if success then + return unpack(resultValues) + else + object.Parent = game:GetService("Lighting") + return ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] false) + end + elseif object:IsA("LuaSourceContainer") then + return ThumbnailGenerator:ClickTexture(baseUrl.. "Thumbs/Script.png", fileExtension, x, y) + elseif object:IsA("SpecialMesh") then + local part = Instance.new("Part") + part.Parent = workspace + object.Parent = part + return ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) + else + pcall(function() object.Parent = workspace end) + end + end +end + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Model.lua b/services/grid-bot/scripts/thumbnails/Model.lua new file mode 100644 index 00000000..ea4eeb58 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Model.lua @@ -0,0 +1,38 @@ +-- Model v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +for _, object in pairs(game:GetObjects(assetUrl)) do + if object:IsA("Sky") then + local resultValues = nil + local success = pcall(function() resultValues = {ThumbnailGenerator:ClickTexture(object.SkyboxFt, fileExtension, x, y)} end) + if success then + return unpack(resultValues) + else + object.Parent = game:GetService("Lighting") + return ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] false) + end + elseif object:IsA("LuaSourceContainer") then + return ThumbnailGenerator:ClickTexture(baseUrl.. "Thumbs/Script.png", fileExtension, x, y) + elseif object:IsA("SpecialMesh") then + local part = Instance.new("Part") + part.Parent = workspace + object.Parent = part + return ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) + else + pcall(function() object.Parent = workspace end) + end +end + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Package.lua b/services/grid-bot/scripts/thumbnails/Package.lua new file mode 100644 index 00000000..db01758d --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Package.lua @@ -0,0 +1,360 @@ +-- Package v1.1.7 +-- See http://wiki.roblox.com/index.php?title=R15_Compatibility_Guide#Package_Parts for details on how body parts work with R15 + +local assetUrls, baseUrl, fileExtension, x, y, R6RigUrl, customTextureUrls = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local function split(str, delim) + local results = {} + local lastMatchEnd = 0 + + local matchStart, matchEnd = string.find(str, delim, --[[init = ]] 1, --[[plain = ]] true) + while matchStart and matchEnd do + if matchStart - lastMatchEnd > 1 then + table.insert(results, string.sub(str, lastMatchEnd + 1, matchStart - 1)) + end + + lastMatchEnd = matchEnd + matchStart, matchEnd = string.find(str, delim, --[[init = ]] lastMatchEnd + 1, --[[plain = ]] true) + end + + if string.len(str) - lastMatchEnd > 1 then + table.insert(results, string.sub(str, lastMatchEnd + 1)) + end + return results +end + +local R15ArtistIntentAssets = {} +local R15Assets = {} +local R6Assets = {} +local bothAssets = {} + +local useR15ArtistIntent = false +local useR15 = true +local poseAnimationId = nil + +local poseValueFound = false +local function processR15Anim(animFolder) + local function processStrValue(strValue) + local animation = strValue:FindFirstChildOfClass("Animation") + if not animation then + return + end + + -- By default the pose animation will be used for the thumbnail + -- If the pose animation doesn't exist then the idle will be used, otherwise the first animation will be used. + if string.lower(strValue.Name) == "pose" then + poseValueFound = true + poseAnimationId = animation.AnimationId + elseif not poseValueFound and string.lower(strValue.Name) == "idle" then + poseAnimationId = animation.AnimationId + elseif not poseAnimationId then + poseAnimationId = animation.AnimationId + end + end + + + for _, obj in pairs(animFolder:GetChildren()) do + if obj:IsA("StringValue") then + processStrValue(obj) + end + end +end + +local assetUrlsList = split(assetUrls, ";") + +for _, assetUrl in pairs(assetUrlsList) do + local currObjects = game:GetObjects(assetUrl) + + for _, object in pairs(currObjects) do + if object:IsA("Folder") and object.Name == "R15ArtistIntent" then + for _, child in pairs(object:GetChildren()) do + table.insert(R15ArtistIntentAssets, child) + end + elseif object:IsA("Folder") and object.Name == "R15Fixed" then -- luacheck: ignore + -- Do nothing. We just don't want this to be dumped in bothAssets + elseif object:IsA("Folder") and object.Name == "R15" then + for _, child in pairs(object:GetChildren()) do + table.insert(R15Assets, child) + end + elseif object:IsA("Folder") and object.Name == "R6" then + for _, child in pairs(object:GetChildren()) do + table.insert(R6Assets, child) + end + elseif object:IsA("CharacterMesh") then + -- Legacy body part format using a CharacterMesh + table.insert(R6Assets, object) + elseif object:IsA("Folder") and object.Name == "R15Anim" then + processR15Anim(object) + else + table.insert(bothAssets, object) + end + end +end + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +-- if the package doesn't contain animations, use this pose. +poseAnimationId = poseAnimationId or "http://www.roblox.com/asset/?id=532421348" + +-- Only use R6 if we found body parts that are only compatible with R15 +if #R6Assets ~= 0 and #R15Assets == 0 and #R15ArtistIntentAssets == 0 then + useR15 = false +end + +local R15RigUrl = "http://www.roblox.com/asset/?id=516159357" +if useR15 and #R15ArtistIntentAssets > 0 then + useR15ArtistIntent = true + R15RigUrl = "http://www.roblox.com/asset/?id=1664543044" +end + +local mannequin +if useR15 then + mannequin = game:GetObjects(R15RigUrl)[1] + if (useR15ArtistIntent) then + for _, obj in pairs(R15ArtistIntentAssets) do + table.insert(bothAssets, obj) + end + else + for _,obj in pairs(R15Assets) do + table.insert(bothAssets, obj) + end + end +else + mannequin = game:GetObjects(R6RigUrl)[1] + for _,obj in pairs(R6Assets) do + table.insert(bothAssets, obj) + end +end +mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None +mannequin.Parent = workspace + +ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + +local tool = nil +local accoutrements = {} + +for _, currObject in pairs(bothAssets) do + if currObject:IsA("BasePart") then + local existingBodyPart = mannequin:FindFirstChild(currObject.Name) + if existingBodyPart ~= nil then + existingBodyPart:Destroy() + end + end + + if currObject:IsA("Tool") then + if useR15 then + tool = currObject + else + mannequin.Torso["Right Shoulder"].CurrentAngle = math.rad(90) + currObject.Parent = mannequin + end + elseif currObject:IsA("DataModelMesh") then + local headMesh = mannequin.Head:FindFirstChild("Mesh") + if headMesh then + headMesh:Destroy() + end + currObject.Parent = mannequin.Head + elseif currObject:IsA("Decal") then + local face = mannequin.Head:FindFirstChild("face") + if face then + face:Destroy() + end + currObject.Parent = mannequin.Head + elseif currObject:IsA("Accoutrement") then + table.insert(accoutrements, currObject) + else + currObject.Parent = mannequin + end +end + +local textureUrls = split(customTextureUrls, ";") +for _, url in pairs(textureUrls) do + local obj = game:GetObjects(url)[1] + if obj:IsA("Shirt") then + -- Don't add a texture Shirt if package already has a Shirt + if not mannequin:FindFirstChildOfClass("Shirt") then + obj.Parent = mannequin + end + elseif obj:IsA("Pants") then + -- Don't add a texture Pants if package already has a Pants + if not mannequin:FindFirstChildOfClass("Pants") then + obj.Parent = mannequin + end + else + obj.Parent = mannequin + end +end + +ThumbnailGenerator:AddProfilingCheckpoint("CustomUrlsLoaded") + +local function buildJoint(parentAttachment, partForJointAttachment) + local jointName = parentAttachment.Name:gsub("RigAttachment", "") + local motor = partForJointAttachment.Parent:FindFirstChild(jointName) + if not motor then + motor = Instance.new("Motor6D") + end + motor.Name = jointName + + motor.Part0 = parentAttachment.Parent + motor.Part1 = partForJointAttachment.Parent + + motor.C0 = parentAttachment.CFrame + motor.C1 = partForJointAttachment.CFrame + + motor.Parent = partForJointAttachment.Parent +end + +-- Builds an R15 rig from the attachments in the parts +local function buildRigFromAttachments(currentPart, lastPart) + local validSiblings = {} + for _, sibling in pairs(currentPart.Parent:GetChildren()) do + -- Don't find matching attachment in the current part being processed. + -- Don't visit the last part visited again, this would cause an infinite loop. + if sibling:IsA("BasePart") and sibling ~= currentPart and sibling ~= lastPart then + table.insert(validSiblings, sibling) + end + end + + local function processRigAttachment(attachment) + for _, sibling in pairs(validSiblings) do + local matchingAttachment = sibling:FindFirstChild(attachment.Name) + if matchingAttachment then + buildJoint(attachment, matchingAttachment) + buildRigFromAttachments(matchingAttachment.Parent, currentPart) + end + end + end + + for _, object in pairs(currentPart:GetChildren()) do + if object:IsA("Attachment") and string.find(object.Name, "RigAttachment") then + processRigAttachment(object) + end + end +end + +local function getJointBetween(part0, part1) + for _, obj in pairs(part1:GetChildren()) do + if obj:IsA("Motor6D") and obj.Part0 == part0 then + return obj + end + end +end + +local function applyR15ToolPose(rig) + local upperTorso = rig:FindFirstChild("UpperTorso") + local rightUpperArm = rig:FindFirstChild("RightUpperArm") + if upperTorso and rightUpperArm then + local rightShoulderJoint = getJointBetween(upperTorso, rightUpperArm) + if rightShoulderJoint then + rightShoulderJoint.C1 = rightShoulderJoint.C1 * CFrame.new(0, 0, 0, 1, 0, 0, 0, 0, -1, 0, 1, 0):inverse() + end + end +end + +-- Applies the middle keyframe of a pose to a given character. +local function applyPoseToCharacter(character, poseAnimId) + local poseKeyframSequence = game:GetService("KeyframeSequenceProvider"):GetKeyframeSequence(poseAnimId) + local keyframes = poseKeyframSequence:GetKeyframes() + local poseKeyframe = keyframes[math.max(1, math.floor(#keyframes/2))] + + local function recurApplyPoses(parentPose, poseObject) + if parentPose then + local joint = getJointBetween(character[parentPose.Name], character[poseObject.Name]) + if joint then + joint.C1 = joint.C1 * poseObject.CFrame:inverse() + end + end + + for _, subPose in pairs(poseObject:GetSubPoses()) do + recurApplyPoses(poseObject, subPose) + end + end + + for _, poseObj in pairs(poseKeyframe:GetPoses()) do + recurApplyPoses(nil, poseObj) + end +end + +if useR15 then + -- Build R15 rig + local humanoidRootPart = mannequin:WaitForChild("HumanoidRootPart") + humanoidRootPart.CFrame = CFrame.new(Vector3.new(0, 5, 0)) * CFrame.Angles(0, math.pi, 0) + humanoidRootPart.Anchored = true + buildRigFromAttachments(humanoidRootPart) + + if tool then + applyR15ToolPose(mannequin) + + local hand = mannequin:FindFirstChild("RightHand") + local handle = tool:FindFirstChild("Handle") + if hand and handle then + local handGrip = hand:FindFirstChild("RightGripAttachment") + if handGrip then + handle.CFrame = hand.CFrame * handGrip.CFrame * tool.Grip:inverse() + end + end + tool.Parent = mannequin + elseif poseAnimationId then + applyPoseToCharacter(mannequin, poseAnimationId) + end +end + +local function findFirstMatchingAttachment(model, name) + for _, child in pairs(model:GetChildren()) do + if child:IsA("Attachment") and child.Name == name then + return child + elseif not child:IsA("Accoutrement") and not child:IsA("Tool") then + local foundAttachment = findFirstMatchingAttachment(child, name) + if foundAttachment then + return foundAttachment + end + end + end +end + +for _, accoutrement in pairs(accoutrements) do + local handle = accoutrement:FindFirstChild("Handle") + if handle then + local accoutrementAttachment = handle:FindFirstChildOfClass("Attachment") + local characterAttachment = nil + if accoutrementAttachment then + characterAttachment = findFirstMatchingAttachment(mannequin, accoutrementAttachment.Name) + end + + local attachmentPart + if characterAttachment then + attachmentPart = characterAttachment.Parent + else + attachmentPart = mannequin:FindFirstChild("Head") + end + + local attachmentCFrame + if characterAttachment then + attachmentCFrame = characterAttachment.CFrame + else + attachmentCFrame = CFrame.new(0, 0.5, 0) + end + + local hatCFrame + if accoutrementAttachment then + hatCFrame = accoutrementAttachment.CFrame + else + hatCFrame = accoutrement.AttachmentPoint + end + + handle.CFrame = attachmentPart.CFrame * attachmentCFrame * hatCFrame:inverse() + handle.Anchored = true + handle.Parent = mannequin + end +end + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Pants.lua b/services/grid-bot/scripts/thumbnails/Pants.lua new file mode 100644 index 00000000..0f14ad77 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Pants.lua @@ -0,0 +1,38 @@ +-- Pants v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl, mannequinId = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local mannequin = game:GetObjects(baseUrl.. "asset/?id=" .. tostring(mannequinId))[1] +mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None +mannequin.Parent = workspace + +ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + +local pants = game:GetObjects(assetUrl)[1] +pants.Parent = mannequin + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +local DFFlagThrowErrorWhenRequestedURLFailed = settings():GetFFlag("ThrowErrorWhenRequestedURLFailed") +if DFFlagThrowErrorWhenRequestedURLFailed then + local ContentProvider = game:GetService("ContentProvider") + local failedRequests = ContentProvider:GetFailedRequests() + if #failedRequests > 0 then + local failedRequestString = "Asset failed to be requested:" + for _,failedString in pairs(failedRequests) do + failedRequestString = failedRequestString.." "..failedString + end + error(failedRequestString) + end +end + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Place.lua b/services/grid-bot/scripts/thumbnails/Place.lua new file mode 100644 index 00000000..ca37dc24 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Place.lua @@ -0,0 +1,27 @@ +-- Place v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl, universeId = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +if universeId ~= nil then + pcall(function() game:SetUniverseId(universeId) end) +end + +game:GetService("ScriptContext").ScriptsDisabled = true +game:GetService("StarterGui").ShowDevelopmentGui = false + +game:Load(assetUrl) + +ThumbnailGenerator:AddProfilingCheckpoint("GameLoaded") + +-- Do this after again loading the place file to ensure that these values aren't changed when the place file is loaded. +game:GetService("ScriptContext").ScriptsDisabled = true +game:GetService("StarterGui").ShowDevelopmentGui = false + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] false) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/Shirt.lua b/services/grid-bot/scripts/thumbnails/Shirt.lua new file mode 100644 index 00000000..e8001039 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/Shirt.lua @@ -0,0 +1,38 @@ +-- Shirt v1.0.2 + +local assetUrl, fileExtension, x, y, baseUrl, mannequinId = ... + +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailScriptStarted") + +pcall(function() game:GetService("ContentProvider"):SetBaseUrl(baseUrl) end) +game:GetService("ScriptContext").ScriptsDisabled = true + +local mannequin = game:GetObjects(baseUrl.. "asset/?id=" .. tostring(mannequinId))[1] +mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None +mannequin.Parent = workspace + +ThumbnailGenerator:AddProfilingCheckpoint("MannequinLoaded") + +local shirt = game:GetObjects(assetUrl)[1] +shirt.Parent = mannequin + +ThumbnailGenerator:AddProfilingCheckpoint("ObjectsLoaded") + +local result, requestedUrls = ThumbnailGenerator:Click(fileExtension, x, y, --[[hideSky = ]] true) +ThumbnailGenerator:AddProfilingCheckpoint("ThumbnailGenerated") + +local DFFlagThrowErrorWhenRequestedURLFailed = settings():GetFFlag("ThrowErrorWhenRequestedURLFailed") +if DFFlagThrowErrorWhenRequestedURLFailed then + local ContentProvider = game:GetService("ContentProvider") + local failedRequests = ContentProvider:GetFailedRequests() + if #failedRequests > 0 then + local failedRequestString = "Asset failed to be requested:" + for _,failedString in pairs(failedRequests) do + failedRequestString = failedRequestString.." "..failedString + end + error(failedRequestString) + end +end + +return result, requestedUrls \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/modules/BundleLoader.lua b/services/grid-bot/scripts/thumbnails/modules/BundleLoader.lua new file mode 100644 index 00000000..b3fe0606 --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/modules/BundleLoader.lua @@ -0,0 +1,128 @@ +local BundleLoader = {} + +local AssetService = game:GetService("AssetService") +local ThumbnailGenerator = game:GetService("ThumbnailGenerator") + +local MannequinUtility = require(ThumbnailGenerator:GetThumbnailModule("MannequinUtility")) +local ScaleUtility = require(ThumbnailGenerator:GetThumbnailModule("ScaleUtility")) + +local ARTIST_INTENT_FOLDER = "R15ArtistIntent" +local ASSET_URL = "asset/?id=" + +local function constructAssetUrl(baseUrl, assetId) + return baseUrl ..ASSET_URL.. tostring(assetId) +end + +function BundleLoader.LoadBundleAssets(baseUrl, bundleId) + local bundleInfo = AssetService:GetBundleDetailsSync(bundleId) + + local contentIdsList = {} + for _, itemInfo in pairs(bundleInfo.Items) do + if itemInfo.Type == "Asset" then + local assetId = itemInfo.Id + local assetUrl = constructAssetUrl(baseUrl, assetId) + + contentIdsList[#contentIdsList + 1] = assetUrl + end + end + + local objectsList = game:GetObjectsList(contentIdsList) + + local results = {} + for _, objects in pairs(objectsList) do + local assetFolder = Instance.new("Folder") + + for _, object in pairs(objects) do + object.Parent = assetFolder + end + + results[#results + 1] = assetFolder + end + + return results +end + +local function addPartsToCharacter(character, folder) + for _, part in pairs(folder:GetChildren()) do + local existingPart = character:FindFirstChild(part.Name) + part.Parent = character + + if existingPart then + existingPart:Destroy() + end + end +end + +local function addMeshHeadToCharacter(character, mesh) + local head = character:FindFirstChild("Head") + if not head then + return + end + + local existingMesh = head:FindFirstChild("Mesh") + if existingMesh then + existingMesh:Destroy() + end + + for _, child in pairs(mesh:GetChildren()) do + if child:IsA("Vector3Value") and string.find(child.Name, "Attachment") then + local attachment = head:FindFirstChild(child.Name) + + if not attachment then + attachment = Instance.new("Attachment") + end + attachment.Name = child.Name + attachment.Position = child.Value + attachment.Parent = head + end + end + + mesh.Parent = head +end + +local function addFaceToCharacter(character, face) + local head = character:FindFirstChild("Head") + if not head then + return + end + + local existingFace = head:FindFirstChild("face") + if existingFace then + existingFace:Destroy() + end + + face.Parent = head +end + +function BundleLoader.LoadBundleCharacter(baseUrl, bundleId) + local bundleAssets = BundleLoader.LoadBundleAssets(baseUrl, bundleId) + local character = MannequinUtility.LoadR15Mannequin() + + local scaleType = ScaleUtility.GetObjectsScaleType(bundleAssets) + + for _, loadedAsset in pairs(bundleAssets) do + for _, item in pairs(loadedAsset:GetChildren()) do + if item:IsA("Folder") and item.Name == ARTIST_INTENT_FOLDER then + addPartsToCharacter(character, item) + elseif not item:IsA("Folder") then + if item:IsA("DataModelMesh") then + addMeshHeadToCharacter(character, item) + elseif item:IsA("Decal") then + addFaceToCharacter(character, item) + else + item.Parent = character + end + end + end + end + + local humanoid = character:FindFirstChildOfClass("Humanoid") + if humanoid then + ScaleUtility.CreateProportionScaleValues(humanoid, scaleType) + humanoid:BuildRigFromAttachments() + end + + return character +end + +return BundleLoader \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/modules/CreateExtentsMinMax.lua b/services/grid-bot/scripts/thumbnails/modules/CreateExtentsMinMax.lua new file mode 100644 index 00000000..4185a04c --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/modules/CreateExtentsMinMax.lua @@ -0,0 +1,75 @@ +-- Utility function for focusing on a selection of parts in a thumbnail + +local FLOAT_MAX = math.huge + +-- 10% tolerance for meshes that are bigger than the part which contains them +local MESH_SIZE_TOLERANCE_MULTIPLIER = 1.1 + +local function addToBounds(cornerPosition, focusExtentsOut) + focusExtentsOut["minx"] = math.min(focusExtentsOut["minx"], cornerPosition.x) + focusExtentsOut["miny"] = math.min(focusExtentsOut["miny"], cornerPosition.y) + focusExtentsOut["minz"] = math.min(focusExtentsOut["minz"], cornerPosition.z) + focusExtentsOut["maxx"] = math.max(focusExtentsOut["maxx"], cornerPosition.x) + focusExtentsOut["maxy"] = math.max(focusExtentsOut["maxy"], cornerPosition.y) + focusExtentsOut["maxz"] = math.max(focusExtentsOut["maxz"], cornerPosition.z) +end + +local function addCornerToBounds(partCFrame, cornerSelect, halfPartSize, focusExtentsOut) + local cornerPositionLocal = cornerSelect * halfPartSize + local cornerPositionWorld = partCFrame * cornerPositionLocal + addToBounds(cornerPositionWorld, focusExtentsOut) +end + +-- Adds a tolerance for meshes in parts +local function getPartSizeBounds(part) + local mesh = part:FindFirstChildWhichIsA("DataModelMesh") + if not mesh then + return part.Size + end + + return part.Size * MESH_SIZE_TOLERANCE_MULTIPLIER +end + +local CORNERS = { + Vector3.new( 1, 1, 1), + Vector3.new( 1, 1, -1), + Vector3.new( 1, -1, 1), + Vector3.new( 1, -1, -1), + Vector3.new(-1, 1, 1), + Vector3.new(-1, 1, -1), + Vector3.new(-1, -1, 1), + Vector3.new(-1, -1, -1), +} + +local function CreateExtentsMinMax(focusParts) + local focusOnExtents = { + minx = FLOAT_MAX, + miny = FLOAT_MAX, + minz = FLOAT_MAX, + maxx = -FLOAT_MAX, + maxy = -FLOAT_MAX, + maxz = -FLOAT_MAX + } + + -- Expand focusOnExtents to bound all the parts in the focusParts table + for _, focusPart in ipairs(focusParts) do + if focusPart:IsA("BasePart") then + local partSize = getPartSizeBounds(focusPart) + local halfPartSize = partSize * 0.5 + local partCFrame = focusPart.CFrame + + for _, corner in ipairs(CORNERS) do + addCornerToBounds(partCFrame, corner, halfPartSize, focusOnExtents) + end + end + end + + local extentsMinMax = { + Vector3.new(focusOnExtents["minx"], focusOnExtents["miny"], focusOnExtents["minz"]), + Vector3.new(focusOnExtents["maxx"], focusOnExtents["maxy"], focusOnExtents["maxz"]) + } + + return extentsMinMax +end + +return CreateExtentsMinMax \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/modules/MannequinUtility.lua b/services/grid-bot/scripts/thumbnails/modules/MannequinUtility.lua new file mode 100644 index 00000000..5617351d --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/modules/MannequinUtility.lua @@ -0,0 +1,51 @@ +-- Utility module for functions related to loading mannequins for thumbnails + +local MannequinUtility = {} + +local InsertService = game:GetService("InsertService") + +local R6_MANNEQUIN_CONTENT_ID = "rbxasset://models/Thumbnails/Mannequins/R6.rbxmx" +local R15_MANNEQUIN_CONTENT_ID = "rbxasset://models/Thumbnails/Mannequins/R15.rbxm" +local RTHRO_MANNEQUIN_CONTENT_ID = "rbxasset://models/Thumbnails/Mannequins/Rthro.rbxm" + +local function loadMannequin(contentId) + local mannequin = InsertService:LoadLocalAsset(contentId) + mannequin.Humanoid.DisplayDistanceType = Enum.HumanoidDisplayDistanceType.None + mannequin.Parent = workspace + + return mannequin +end + +function MannequinUtility.LoadR15Mannequin() + return loadMannequin(R15_MANNEQUIN_CONTENT_ID) +end + +function MannequinUtility.LoadR6Mannequin() + return loadMannequin(R6_MANNEQUIN_CONTENT_ID) +end + +function MannequinUtility.LoadRthroMannequin() + return loadMannequin(RTHRO_MANNEQUIN_CONTENT_ID) +end + +function MannequinUtility.LoadMannequinForScaleType(scaleType) + if scaleType == "Classic" then + return MannequinUtility.LoadR15Mannequin() + else + return MannequinUtility.LoadRthroMannequin() + end +end + +function MannequinUtility.RotateMannequin(mannequin, cframe) + local humanoidRootPart = mannequin:FindFirstChild("HumanoidRootPart") + if not humanoidRootPart then + return + end + + local rootRigAttachment = humanoidRootPart:FindFirstChild("RootRigAttachment") + if rootRigAttachment then + rootRigAttachment.CFrame = rootRigAttachment.CFrame * cframe + end +end + +return MannequinUtility \ No newline at end of file diff --git a/services/grid-bot/scripts/thumbnails/modules/ScaleUtility.lua b/services/grid-bot/scripts/thumbnails/modules/ScaleUtility.lua new file mode 100644 index 00000000..2571d91d --- /dev/null +++ b/services/grid-bot/scripts/thumbnails/modules/ScaleUtility.lua @@ -0,0 +1,63 @@ +-- Utility module for managing the scale of mannequins and items in thumbnails + +local ScaleUtility = {} + +local CLASSIC_SCALE = "Classic" +local RTHRO_NORMAL = "ProportionsNormal" +local RTHRO_SLENDER = "ProportionsSlender" + +local function getPartScaleType(part) + local value = part:FindFirstChild("AvatarPartScaleType") + if value then + return value.Value + end + + return CLASSIC_SCALE +end + +function ScaleUtility.GetScaleTypeForAccessory(accessory) + local handle = accessory:FindFirstChild("Handle") + if not handle then + return CLASSIC_SCALE + end + + return getPartScaleType(handle) +end + +function ScaleUtility.GetObjectsScaleType(objects) + for _, object in pairs(objects) do + local partScaleType = object:FindFirstChild("AvatarPartScaleType", --[[ recursive = ]] true) + if partScaleType then + return partScaleType.Value + end + end +end + +local function getOrCreateScaleValue(humanoid, name, default) + local scaleValue = humanoid:FindFirstChild(name) + if scaleValue then + return scaleValue + end + + scaleValue = Instance.new("NumberValue") + scaleValue.Name = name + scaleValue.Value = default + scaleValue.Parent = humanoid + + return scaleValue +end + +function ScaleUtility.CreateProportionScaleValues(humanoid, scaleType) + local bodyTypeValue = getOrCreateScaleValue(humanoid, "BodyTypeScale", 0) + local bodyProportionValue = getOrCreateScaleValue(humanoid, "BodyProportionScale", 0) + + if scaleType == RTHRO_NORMAL then + bodyTypeValue.Value = 1 + bodyProportionValue.Value = 0 + elseif scaleType == RTHRO_SLENDER then + bodyTypeValue.Value = 1 + bodyProportionValue.Value = 1 + end +end + +return ScaleUtility \ No newline at end of file diff --git a/services/grid-bot/src/Extensions/IServiceCollectionExtensions.cs b/services/grid-bot/src/Extensions/IServiceCollectionExtensions.cs index 1e80d0b5..8cb5e476 100644 --- a/services/grid-bot/src/Extensions/IServiceCollectionExtensions.cs +++ b/services/grid-bot/src/Extensions/IServiceCollectionExtensions.cs @@ -28,6 +28,10 @@ namespace Grid.Bot.Extensions; using Events; using Utility; +using Grid.JobManagement; +using Grid.PortManagement; +using Grid.ProcessManagement; + using EnvironmentProvider = Grid.Bot.EnvironmentProvider; /// @@ -170,43 +174,35 @@ public static IServiceCollection AddJobManager(this IServiceCollection services) } #endif + gridSettings.GridServerAdditionalVolumeMappings = [ + ..gridSettings.GridServerAdditionalVolumeMappingsSetting, + $"{gridSettings.GridServerSharedDirectoryInternalScripts}:{gridSettings.GridServerInsideDirectoryInternalScripts}" + ]; + var logger = new Logger( name: gridSettings.JobManagerLoggerName, logLevelGetter: () => gridSettings.JobManagerLogLevel, logToConsole: gridSettings.JobManagerLogToConsole ); - var portAllocator = new PortAllocator(logger); - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var jobManager = new ProcessJobManager( - logger, - portAllocator, - gridSettings - ); - - jobManager.Start(); + var clientSettingsFactory = services + .BuildServiceProvider() + .GetRequiredService(); - services.AddSingleton(jobManager); - services.AddSingleton(_ => null as DockerJobManager); + var clientSettingsClient = new ClientSettingsFactoryProxyClient(clientSettingsFactory); - } - else - { - var jobManager = new DockerJobManager( - logger, - portAllocator, - gridSettings, - RandomFactory.GetDefaultRandom() - ); + var portAllocator = new PortAllocator(logger); + var jobManagerFactory = new JobManagerGridServerFactory(); - jobManager.Start(); + var jobManagerGridServer = jobManagerFactory.GetJobManager( + logger, + clientSettingsClient, + gridSettings + ); - services.AddSingleton(jobManager); - services.AddSingleton(_ => null as ProcessJobManager); - } + jobManagerGridServer.Start(); + services.AddSingleton(jobManagerGridServer); services.AddSingleton(); return services; @@ -371,4 +367,4 @@ public static IServiceCollection AddDiscordEventHandlers(this IServiceCollection return services; } -} \ No newline at end of file +} diff --git a/services/grid-bot/src/Grid.Bot.csproj b/services/grid-bot/src/Grid.Bot.csproj index 6201b394..597ca201 100755 --- a/services/grid-bot/src/Grid.Bot.csproj +++ b/services/grid-bot/src/Grid.Bot.csproj @@ -56,9 +56,13 @@ + + + + diff --git a/services/grid-bot/src/Runner.cs b/services/grid-bot/src/Runner.cs index 1be69cde..d46e061f 100755 --- a/services/grid-bot/src/Runner.cs +++ b/services/grid-bot/src/Runner.cs @@ -53,10 +53,10 @@ private static ServiceProvider InitializeServices() services.AddGlobalLogger(); services.AddUtilities(); + services.AddClientSettings(); services.AddJobManager(); services.AddFloodCheckersRedis(); services.AddHttpClients(); - services.AddClientSettings(); services.AddDiscord(); services.AddDiscordEventHandlers();