/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using QuantConnect.Util;
using QuantConnect.Data;
using QuantConnect.Logging;
using QuantConnect.Interfaces;
using QuantConnect.Configuration;
using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants;
namespace QuantConnect.DownloaderDataProvider.Launcher;
public static class Program
{
///
/// Synchronizer in charge of guaranteeing a single operation per file path
///
private readonly static KeyStringSynchronizer DiskSynchronizer = new();
///
/// The provider used to cache history data files
///
private static readonly IDataCacheProvider _dataCacheProvider = new DiskDataCacheProvider(DiskSynchronizer);
///
/// Represents the time interval of 5 seconds.
///
private static TimeSpan _logDisplayInterval = TimeSpan.FromSeconds(5);
///
/// The main entry point for the application.
///
/// Command-line arguments passed to the application.
public static void Main(string[] args)
{
// Parse report arguments and merge with config to use in the optimizer
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(DownloaderDataProviderArgumentParser.ParseArguments(args));
}
InitializeConfigurations();
var dataDownloader = Composer.Instance.GetExportedValueByTypeName(Config.Get(DownloaderCommandArguments.CommandDownloaderDataDownloader));
var dataDownloadConfig = new DataDownloadConfig();
// Calculate the total number of seconds between the EndDate and StartDate
var totalDataPerSymbolInSeconds = (dataDownloadConfig.EndDate - dataDownloadConfig.StartDate).TotalSeconds;
var totalDataInSeconds = totalDataPerSymbolInSeconds * dataDownloadConfig.Symbols.Count;
var completeSymbolCount = 0;
var startUtcTime = DateTime.UtcNow;
foreach (var symbol in dataDownloadConfig.Symbols)
{
var downloadParameters = new DataDownloaderGetParameters(symbol, dataDownloadConfig.Resolution, dataDownloadConfig.StartDate, dataDownloadConfig.EndDate, dataDownloadConfig.TickType);
Log.Trace($"DownloaderDataProvider.Main(): Starting download {downloadParameters}");
var downloadedData = dataDownloader.Get(downloadParameters);
if (downloadedData == null)
{
completeSymbolCount++;
Log.Trace($"DownloaderDataProvider.Main(): No data available for the following parameters: {downloadParameters}");
continue;
}
var writer = new LeanDataWriter(dataDownloadConfig.Resolution, symbol, Globals.DataFolder, dataDownloadConfig.TickType, mapSymbol: true,
dataCacheProvider: _dataCacheProvider);
var lastLogStatusTime = DateTime.UtcNow;
writer.Write(downloadedData.Select(data =>
{
var utcNow = DateTime.UtcNow;
if (utcNow - lastLogStatusTime >= _logDisplayInterval)
{
lastLogStatusTime = utcNow;
var progressSoFar = (data.EndTime - dataDownloadConfig.StartDate).TotalSeconds + totalDataPerSymbolInSeconds * completeSymbolCount;
var eta = CalculateETA(utcNow, startUtcTime, totalDataInSeconds, progressSoFar);
var progress = CalculateProgress(data.EndTime, dataDownloadConfig.StartDate, dataDownloadConfig.EndDate);
Log.Trace($"DownloaderDataProvider.Main(): Downloading {downloadParameters.Symbol} data: {progress:F2}%. ETA: {eta}");
}
return data;
}));
completeSymbolCount++;
Log.Trace($"DownloaderDataProvider.Main(): Download completed for {downloadParameters.Symbol} at {downloadParameters.Resolution} resolution, " +
$"covering the period from {dataDownloadConfig.StartDate} to {dataDownloadConfig.EndDate}.");
}
}
///
/// Calculates the Estimated Time of Arrival (ETA) based on the current progress of a download.
///
/// The current UTC DateTime.
/// The DateTime when the download started.
///
///
/// A TimeSpan representing the Estimated Time of Arrival (ETA).
///
/// The method calculates the time elapsed since the start of downloading and estimates
/// the remaining time based on the current progress. It uses the difference between
/// the end time and the end date to calculate missing data time, and the difference
/// between the end time and the start date to calculate the progress so far.
/// The ETA is then calculated based on these values.
///
public static TimeSpan CalculateETA(DateTime utcNow, DateTime startUtcTime, double totalDataInSeconds, double currentProgressSecond)
{
// Calculate how much time has passed since the start of downloading
TimeSpan howMuchItTookSoFar = utcNow - startUtcTime;
var missingDataSecond = totalDataInSeconds - currentProgressSecond;
// Calculate ETA in seconds
var etaSeconds = (missingDataSecond / currentProgressSecond) * howMuchItTookSoFar.TotalSeconds;
// Convert ETA from seconds to TimeSpan
TimeSpan etaTimeSpan = TimeSpan.FromSeconds(etaSeconds);
return etaTimeSpan;
}
///
/// Calculates the progress as a percentage based on the elapsed time between the start and end dates.
///
/// The current date and time.
/// The start date and time.
/// The end date and time.
/// A double representing the progress as a percentage.
private static double CalculateProgress(DateTime currentTime, DateTime start, DateTime end)
{
double totalDays = (end - start).TotalDays;
if (totalDays == 0)
{
return 0;
}
double elapsedDays = (currentTime - start).TotalDays;
return (elapsedDays / totalDays) * 100;
}
///
/// Initializes various configurations for the application.
/// This method sets up logging, data providers, map file providers, and factor file providers.
///
///
/// The method reads configuration values to determine whether debugging is enabled,
/// which log handler to use, and which data, map file, and factor file providers to initialize.
///
///
///
///
///
///
///
///
public static void InitializeConfigurations()
{
Log.DebuggingEnabled = Config.GetBool("debug-mode", false);
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName(Config.Get("log-handler", "CompositeLogHandler"));
var dataProvider = Composer.Instance.GetExportedValueByTypeName("DefaultDataProvider");
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider"));
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
mapFileProvider.Initialize(dataProvider);
factorFileProvider.Initialize(mapFileProvider, dataProvider);
}
}