Gerardo Salazar
82c9b6ccb7
Build & Test Lean / build (push) Has been cancelled
* Adds processed data directory to read price data from
* Make coarse universe generator look at data directory before failing to find daily data
* Set coarse generator output of missing daily file to debug log
* Add CoarseUniverseGenerator logs
* Fixes 100 nanosecond increment lookahead bias when parsing large numbers
* Whenever we parse a number that is has precision greater than
DateTime ticks (sub-100 nanoseconds), if we have nanoseconds
between [0, 1000), excluding numbers divisible by 100,
we will have leftover nanoseconds between [0, 100) nanoseconds, but
they won't be factored in to the DateTime calculation, since casting
to `long` only takes the integer component of the number, so we lose
the extra nanoseconds that came with the decimal, and time is set to
the "floored" value without those nanoseconds.
Since .NET `DateTime` type has a limitation of only being able
to represent time in increments of 100 nanoseconds, by not
considering the sub-100 nanoseconds, we introduce a look-ahead
bias of at most 100 nanoseconds/1 tick
* Misc adjustment to make method use `decimal` instead of `double`
for increased precision when parsing large numbers
* Changes CoinAPI data converter to support processing raw files in original directory structure and file name
* Removes Market requirement from CoinAPI data converter
* Remove timeout on decompression of raw AlgoSeek futures data
* Updates SEC downloader to use HttpClient where requests were failing
* For some unknown reason, valid requests to a valid URL were
failing when using WebClient. Changing our requester to
HttpClient fixes the issue, and enables us to leverage
async capabilities where applicable.
* Added fault tolerance to index file downloads, including a
rate limit in case we've been rate limited
* Further refactoring; catches 429 errors, adds missing rategate calls
* Replace all usage of WebClient, force retry for all failures
* Adds optional config value for Benzinga News API key in downloader
* Modifies Estimize Downloader api config name and fixes directory not found bug
* Refactor Estimize to speed up processing time
* Adds ticker limits if desired
* Misc. bug fixes, performance improvements, code cleanup
* Remove debug log statements leftover from previous commit
* Add support for non-tick Index resolutions in LeanDataWriter
* Empty commit
* Empty commit
* Empty commit
* Empty commit
* Empty commit
* Empty commit
* Lower requests/second for SEC downloader, add missing rategate call
Co-authored-by: Martin-Molinero <martin@quantconnect.com>
353 lines
18 KiB
C#
353 lines
18 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using QuantConnect.Configuration;
|
|
using QuantConnect.Logging;
|
|
using QuantConnect.ToolBox.AlgoSeekFuturesConverter;
|
|
using QuantConnect.ToolBox.AlgoSeekOptionsConverter;
|
|
using QuantConnect.ToolBox.Benzinga;
|
|
using QuantConnect.ToolBox.BinanceDownloader;
|
|
using QuantConnect.ToolBox.BitfinexDownloader;
|
|
using QuantConnect.ToolBox.CoarseUniverseGenerator;
|
|
using QuantConnect.ToolBox.CoinApiDataConverter;
|
|
using QuantConnect.ToolBox.CryptoiqDownloader;
|
|
using QuantConnect.ToolBox.DukascopyDownloader;
|
|
using QuantConnect.ToolBox.EstimizeDataDownloader;
|
|
using QuantConnect.ToolBox.GDAXDownloader;
|
|
using QuantConnect.ToolBox.IBDownloader;
|
|
using QuantConnect.ToolBox.IEX;
|
|
using QuantConnect.ToolBox.IQFeedDownloader;
|
|
using QuantConnect.ToolBox.IVolatilityEquityConverter;
|
|
using QuantConnect.ToolBox.KaikoDataConverter;
|
|
using QuantConnect.ToolBox.KrakenDownloader;
|
|
using QuantConnect.ToolBox.NseMarketDataConverter;
|
|
using QuantConnect.ToolBox.OandaDownloader;
|
|
using QuantConnect.ToolBox.Polygon;
|
|
using QuantConnect.ToolBox.QuandlBitfinexDownloader;
|
|
using QuantConnect.ToolBox.QuantQuoteConverter;
|
|
using QuantConnect.ToolBox.RandomDataGenerator;
|
|
using QuantConnect.ToolBox.SECDataDownloader;
|
|
using QuantConnect.ToolBox.USTreasuryYieldCurve;
|
|
using QuantConnect.ToolBox.YahooDownloader;
|
|
using QuantConnect.Util;
|
|
using QuantConnect.ToolBox.SmartInsider;
|
|
using QuantConnect.ToolBox.TiingoNewsConverter;
|
|
using QuantConnect.ToolBox.ZerodhaDownloader;
|
|
|
|
namespace QuantConnect.ToolBox
|
|
{
|
|
public class Program
|
|
{
|
|
public static void Main(string[] args)
|
|
{
|
|
Log.DebuggingEnabled = Config.GetBool("debug-mode");
|
|
Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt");
|
|
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
|
|
|
|
var optionsObject = ToolboxArgumentParser.ParseArguments(args);
|
|
if (optionsObject.Count == 0)
|
|
{
|
|
PrintMessageAndExit();
|
|
}
|
|
|
|
var targetApp = GetParameterOrExit(optionsObject, "app").ToLowerInvariant();
|
|
if (targetApp.Contains("download") || targetApp.EndsWith("dl"))
|
|
{
|
|
var fromDate = Parse.DateTimeExact(GetParameterOrExit(optionsObject, "from-date"), "yyyyMMdd-HH:mm:ss");
|
|
var resolution = optionsObject.ContainsKey("resolution") ? optionsObject["resolution"].ToString() : "";
|
|
var market = optionsObject.ContainsKey("market") ? optionsObject["market"].ToString() : "";
|
|
var securityType = optionsObject.ContainsKey("security-type") ? optionsObject["security-type"].ToString() : "";
|
|
var tickers = ToolboxArgumentParser.GetTickers(optionsObject);
|
|
var toDate = optionsObject.ContainsKey("to-date")
|
|
? Parse.DateTimeExact(optionsObject["to-date"].ToString(), "yyyyMMdd-HH:mm:ss")
|
|
: DateTime.UtcNow;
|
|
switch (targetApp)
|
|
{
|
|
case "zdl":
|
|
case "zerodhadownloader":
|
|
ZerodhaDataDownloaderProgram.ZerodhaDataDownloader(tickers,market, resolution, securityType, fromDate, toDate);
|
|
break;
|
|
case "gdaxdl":
|
|
case "gdaxdownloader":
|
|
GDAXDownloaderProgram.GDAXDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "cdl":
|
|
case "cryptoiqdownloader":
|
|
CryptoiqDownloaderProgram.CryptoiqDownloader(tickers, GetParameterOrExit(optionsObject, "exchange"), fromDate, toDate);
|
|
break;
|
|
case "ddl":
|
|
case "dukascopydownloader":
|
|
DukascopyDownloaderProgram.DukascopyDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "ibdl":
|
|
case "ibdownloader":
|
|
IBDownloaderProgram.IBDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "iexdl":
|
|
case "iexdownloader":
|
|
IEXDownloaderProgram.IEXDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "iqfdl":
|
|
case "iqfeeddownloader":
|
|
IQFeedDownloaderProgram.IQFeedDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "kdl":
|
|
case "krakendownloader":
|
|
KrakenDownloaderProgram.KrakenDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "odl":
|
|
case "oandadownloader":
|
|
OandaDownloaderProgram.OandaDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "qbdl":
|
|
case "quandlbitfinexdownloader":
|
|
QuandlBitfinexDownloaderProgram.QuandlBitfinexDownloader(fromDate, GetParameterOrExit(optionsObject, "api-key"));
|
|
break;
|
|
case "ydl":
|
|
case "yahoodownloader":
|
|
YahooDownloaderProgram.YahooDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "bfxdl":
|
|
case "bitfinexdownloader":
|
|
BitfinexDownloaderProgram.BitfinexDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "mbxdl":
|
|
case "binancedownloader":
|
|
BinanceDownloaderProgram.DataDownloader(tickers, resolution, fromDate, toDate);
|
|
break;
|
|
case "secdl":
|
|
case "secdownloader":
|
|
SECDataDownloaderProgram.SECDataDownloader(
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
fromDate,
|
|
toDate
|
|
);
|
|
break;
|
|
case "ecdl":
|
|
case "estimizeconsensusdownloader":
|
|
EstimizeConsensusDataDownloaderProgram.EstimizeConsensusDataDownloader();
|
|
break;
|
|
case "eedl":
|
|
case "estimizeestimatedownloader":
|
|
EstimizeEstimateDataDownloaderProgram.EstimizeEstimateDataDownloader();
|
|
break;
|
|
case "erdl":
|
|
case "estimizereleasedownloader":
|
|
EstimizeReleaseDataDownloaderProgram.EstimizeReleaseDataDownloader();
|
|
break;
|
|
|
|
case "ustycdl":
|
|
case "ustreasuryyieldcurvedownloader":
|
|
USTreasuryYieldCurveProgram.USTreasuryYieldCurveRateDownloader(
|
|
fromDate,
|
|
toDate,
|
|
GetParameterOrExit(optionsObject, "destination-dir")
|
|
);
|
|
break;
|
|
|
|
case "bzndl":
|
|
case "benzinganewsdownloader":
|
|
BenzingaProgram.BenzingaNewsDataDownloader(
|
|
fromDate,
|
|
toDate,
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
GetParameterOrDefault(optionsObject, "api-key", string.Empty)
|
|
);
|
|
break;
|
|
|
|
case "tecdl":
|
|
case "tradingeconomicscalendardownloader":
|
|
TradingEconomicsDataDownloader.TradingEconomicsCalendarDownloaderProgram.TradingEconomicsCalendarDownloader();
|
|
break;
|
|
|
|
case "pdl":
|
|
case "polygondownloader":
|
|
PolygonDownloaderProgram.PolygonDownloader(
|
|
tickers,
|
|
GetParameterOrExit(optionsObject, "security-type"),
|
|
GetParameterOrExit(optionsObject, "market"),
|
|
resolution,
|
|
fromDate,
|
|
toDate);
|
|
break;
|
|
|
|
default:
|
|
PrintMessageAndExit(1, "ERROR: Unrecognized --app value");
|
|
break;
|
|
}
|
|
}
|
|
else if (targetApp.Contains("updater") || targetApp.EndsWith("spu"))
|
|
{
|
|
switch (targetApp)
|
|
{
|
|
case "mbxspu":
|
|
case "binancesymbolpropertiesupdater":
|
|
BinanceDownloaderProgram.ExchangeInfoDownloader();
|
|
break;
|
|
default:
|
|
PrintMessageAndExit(1, "ERROR: Unrecognized --app value");
|
|
break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
switch (targetApp)
|
|
{
|
|
case "asfc":
|
|
case "algoseekfuturesconverter":
|
|
AlgoSeekFuturesProgram.AlgoSeekFuturesConverter(GetParameterOrExit(optionsObject, "date"));
|
|
break;
|
|
case "asoc":
|
|
case "algoseekoptionsconverter":
|
|
AlgoSeekOptionsConverterProgram.AlgoSeekOptionsConverter(GetParameterOrExit(optionsObject, "date"));
|
|
break;
|
|
case "ivec":
|
|
case "ivolatilityequityconverter":
|
|
IVolatilityEquityConverterProgram.IVolatilityEquityConverter(GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "source-meta-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
GetParameterOrExit(optionsObject, "resolution"));
|
|
break;
|
|
case "kdc":
|
|
case "kaikodataconverter":
|
|
KaikoDataConverterProgram.KaikoDataConverter(GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "date"),
|
|
GetParameterOrDefault(optionsObject, "exchange", string.Empty));
|
|
break;
|
|
case "cadc":
|
|
case "coinapidataconverter":
|
|
CoinApiDataConverterProgram.CoinApiDataProgram(GetParameterOrExit(optionsObject, "date"),
|
|
GetParameterOrExit(optionsObject, "source-dir"), GetParameterOrExit(optionsObject, "destination-dir"));
|
|
break;
|
|
case "nmdc":
|
|
case "nsemarketdataconverter":
|
|
NseMarketDataConverterProgram.NseMarketDataConverter(GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"));
|
|
break;
|
|
case "qqc":
|
|
case "quantquoteconverter":
|
|
QuantQuoteConverterProgram.QuantQuoteConverter(GetParameterOrExit(optionsObject, "destination-dir"),
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "resolution"));
|
|
break;
|
|
case "cug":
|
|
case "coarseuniversegenerator":
|
|
CoarseUniverseGeneratorProgram.CoarseUniverseGenerator();
|
|
break;
|
|
case "rdg":
|
|
case "randomdatagenerator":
|
|
RandomDataGeneratorProgram.RandomDataGenerator(
|
|
GetParameterOrExit(optionsObject, "start"),
|
|
GetParameterOrExit(optionsObject, "end"),
|
|
GetParameterOrExit(optionsObject, "symbol-count"),
|
|
GetParameterOrDefault(optionsObject, "market", null),
|
|
GetParameterOrDefault(optionsObject, "security-type", "Equity"),
|
|
GetParameterOrDefault(optionsObject, "resolution", "Minute"),
|
|
GetParameterOrDefault(optionsObject, "data-density", "Dense"),
|
|
GetParameterOrDefault(optionsObject, "include-coarse", "true"),
|
|
GetParameterOrDefault(optionsObject, "quote-trade-ratio", "1"),
|
|
GetParameterOrDefault(optionsObject, "random-seed", null),
|
|
GetParameterOrDefault(optionsObject, "ipo-percentage", "5.0"),
|
|
GetParameterOrDefault(optionsObject, "rename-percentage", "30.0"),
|
|
GetParameterOrDefault(optionsObject, "splits-percentage", "15.0"),
|
|
GetParameterOrDefault(optionsObject, "dividends-percentage", "60.0"),
|
|
GetParameterOrDefault(optionsObject, "dividend-every-quarter-percentage", "30.0")
|
|
);
|
|
break;
|
|
case "seccv":
|
|
case "secconverter":
|
|
var start = Parse.DateTimeExact(GetParameterOrExit(optionsObject, "date"), "yyyyMMdd");
|
|
SECDataDownloaderProgram.SECDataConverter(
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrDefault(optionsObject, "destination-dir", Globals.DataFolder),
|
|
start);
|
|
break;
|
|
case "ustyccv":
|
|
case "ustreasuryyieldcurveconverter":
|
|
USTreasuryYieldCurveProgram.USTreasuryYieldCurveConverter(
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"));
|
|
break;
|
|
case "sidc":
|
|
case "smartinsiderconverter":
|
|
SmartInsiderProgram.SmartInsiderConverter(
|
|
DateTime.ParseExact(GetParameterOrExit(optionsObject, "date"), "yyyyMMdd", CultureInfo.InvariantCulture),
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
GetParameterOrDefault(optionsObject, "source-meta-dir", null));
|
|
break;
|
|
case "tiinc":
|
|
case "tiingonewsconverter":
|
|
var date = GetParameterOrDefault(optionsObject, "date", null);
|
|
TiingoNewsConverterProgram.TiingoNewsConverter(
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
date != null ? DateTime.ParseExact(date, "yyyyMMdd", CultureInfo.InvariantCulture) : (DateTime?) null);
|
|
break;
|
|
case "bzncv":
|
|
case "benzinganewsconverter":
|
|
BenzingaProgram.BenzingaNewsDataConverter(
|
|
GetParameterOrExit(optionsObject, "source-dir"),
|
|
GetParameterOrExit(optionsObject, "destination-dir"),
|
|
GetParameterOrDefault(optionsObject, "source-meta-dir", Path.Combine(Globals.DataFolder, "alternative", "benzinga")),
|
|
GetParameterOrExit(optionsObject, "date"));
|
|
break;
|
|
|
|
default:
|
|
PrintMessageAndExit(1, "ERROR: Unrecognized --app value");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void PrintMessageAndExit(int exitCode = 0, string message = "")
|
|
{
|
|
if (!message.IsNullOrEmpty())
|
|
{
|
|
Console.WriteLine("\n" + message);
|
|
}
|
|
Console.WriteLine("\nUse the '--help' parameter for more information");
|
|
Console.WriteLine("Press any key to quit");
|
|
Console.ReadLine();
|
|
Environment.Exit(exitCode);
|
|
}
|
|
|
|
private static string GetParameterOrExit(IReadOnlyDictionary<string, object> optionsObject, string parameter)
|
|
{
|
|
if (!optionsObject.ContainsKey(parameter))
|
|
{
|
|
PrintMessageAndExit(1, "ERROR: REQUIRED parameter --" + parameter + "= is missing");
|
|
}
|
|
return optionsObject[parameter].ToString();
|
|
}
|
|
|
|
private static string GetParameterOrDefault(IReadOnlyDictionary<string, object> optionsObject, string parameter, string defaultValue)
|
|
{
|
|
object value;
|
|
if (!optionsObject.TryGetValue(parameter, out value))
|
|
{
|
|
Console.WriteLine($"'{parameter}' was not specified. Using default value: '{defaultValue}'");
|
|
return defaultValue;
|
|
}
|
|
|
|
return value.ToString();
|
|
}
|
|
|
|
}
|
|
}
|