/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ using QuantConnect.Util; using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Interfaces; using QuantConnect.Configuration; using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants; namespace QuantConnect.DownloaderDataProvider.Launcher; public static class Program { /// /// Synchronizer in charge of guaranteeing a single operation per file path /// private readonly static KeyStringSynchronizer DiskSynchronizer = new(); /// /// The provider used to cache history data files /// private static readonly IDataCacheProvider _dataCacheProvider = new DiskDataCacheProvider(DiskSynchronizer); /// /// Represents the time interval of 5 seconds. /// private static TimeSpan _logDisplayInterval = TimeSpan.FromSeconds(5); /// /// The main entry point for the application. /// /// Command-line arguments passed to the application. public static void Main(string[] args) { // Parse report arguments and merge with config to use in the optimizer if (args.Length > 0) { Config.MergeCommandLineArgumentsWithConfiguration(DownloaderDataProviderArgumentParser.ParseArguments(args)); } InitializeConfigurations(); var dataDownloader = Composer.Instance.GetExportedValueByTypeName(Config.Get(DownloaderCommandArguments.CommandDownloaderDataDownloader)); var dataDownloadConfig = new DataDownloadConfig(); // Calculate the total number of seconds between the EndDate and StartDate var totalDataPerSymbolInSeconds = (dataDownloadConfig.EndDate - dataDownloadConfig.StartDate).TotalSeconds; var totalDataInSeconds = totalDataPerSymbolInSeconds * dataDownloadConfig.Symbols.Count; var completeSymbolCount = 0; var startUtcTime = DateTime.UtcNow; foreach (var symbol in dataDownloadConfig.Symbols) { var downloadParameters = new DataDownloaderGetParameters(symbol, dataDownloadConfig.Resolution, dataDownloadConfig.StartDate, dataDownloadConfig.EndDate, dataDownloadConfig.TickType); Log.Trace($"DownloaderDataProvider.Main(): Starting download {downloadParameters}"); var downloadedData = dataDownloader.Get(downloadParameters); if (downloadedData == null) { completeSymbolCount++; Log.Trace($"DownloaderDataProvider.Main(): No data available for the following parameters: {downloadParameters}"); continue; } var writer = new LeanDataWriter(dataDownloadConfig.Resolution, symbol, Globals.DataFolder, dataDownloadConfig.TickType, mapSymbol: true, dataCacheProvider: _dataCacheProvider); var lastLogStatusTime = DateTime.UtcNow; writer.Write(downloadedData.Select(data => { var utcNow = DateTime.UtcNow; if (utcNow - lastLogStatusTime >= _logDisplayInterval) { lastLogStatusTime = utcNow; var progressSoFar = (data.EndTime - dataDownloadConfig.StartDate).TotalSeconds + totalDataPerSymbolInSeconds * completeSymbolCount; var eta = CalculateETA(utcNow, startUtcTime, totalDataInSeconds, progressSoFar); var progress = CalculateProgress(data.EndTime, dataDownloadConfig.StartDate, dataDownloadConfig.EndDate); Log.Trace($"DownloaderDataProvider.Main(): Downloading {downloadParameters.Symbol} data: {progress:F2}%. ETA: {eta}"); } return data; })); completeSymbolCount++; Log.Trace($"DownloaderDataProvider.Main(): Download completed for {downloadParameters.Symbol} at {downloadParameters.Resolution} resolution, " + $"covering the period from {dataDownloadConfig.StartDate} to {dataDownloadConfig.EndDate}."); } } /// /// Calculates the Estimated Time of Arrival (ETA) based on the current progress of a download. /// /// The current UTC DateTime. /// The DateTime when the download started. /// /// /// A TimeSpan representing the Estimated Time of Arrival (ETA). /// /// The method calculates the time elapsed since the start of downloading and estimates /// the remaining time based on the current progress. It uses the difference between /// the end time and the end date to calculate missing data time, and the difference /// between the end time and the start date to calculate the progress so far. /// The ETA is then calculated based on these values. /// public static TimeSpan CalculateETA(DateTime utcNow, DateTime startUtcTime, double totalDataInSeconds, double currentProgressSecond) { // Calculate how much time has passed since the start of downloading TimeSpan howMuchItTookSoFar = utcNow - startUtcTime; var missingDataSecond = totalDataInSeconds - currentProgressSecond; // Calculate ETA in seconds var etaSeconds = (missingDataSecond / currentProgressSecond) * howMuchItTookSoFar.TotalSeconds; // Convert ETA from seconds to TimeSpan TimeSpan etaTimeSpan = TimeSpan.FromSeconds(etaSeconds); return etaTimeSpan; } /// /// Calculates the progress as a percentage based on the elapsed time between the start and end dates. /// /// The current date and time. /// The start date and time. /// The end date and time. /// A double representing the progress as a percentage. private static double CalculateProgress(DateTime currentTime, DateTime start, DateTime end) { double totalDays = (end - start).TotalDays; if (totalDays == 0) { return 0; } double elapsedDays = (currentTime - start).TotalDays; return (elapsedDays / totalDays) * 100; } /// /// Initializes various configurations for the application. /// This method sets up logging, data providers, map file providers, and factor file providers. /// /// /// The method reads configuration values to determine whether debugging is enabled, /// which log handler to use, and which data, map file, and factor file providers to initialize. /// /// /// /// /// /// /// /// public static void InitializeConfigurations() { Log.DebuggingEnabled = Config.GetBool("debug-mode", false); Log.LogHandler = Composer.Instance.GetExportedValueByTypeName(Config.Get("log-handler", "CompositeLogHandler")); var dataProvider = Composer.Instance.GetExportedValueByTypeName("DefaultDataProvider"); var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider")); var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider")); mapFileProvider.Initialize(dataProvider); factorFileProvider.Initialize(mapFileProvider, dataProvider); } }