Gerardo Salazar
828 lines
41 KiB
C#
828 lines
41 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using Python.Runtime;
|
|
using QuantConnect.Algorithm;
|
|
using QuantConnect.Configuration;
|
|
using QuantConnect.Data;
|
|
using QuantConnect.Data.Fundamental;
|
|
using QuantConnect.Data.Market;
|
|
using QuantConnect.Indicators;
|
|
using QuantConnect.Interfaces;
|
|
using QuantConnect.Lean.Engine;
|
|
using QuantConnect.Lean.Engine.DataFeeds;
|
|
using QuantConnect.Securities;
|
|
using QuantConnect.Securities.Future;
|
|
using QuantConnect.Securities.Option;
|
|
using QuantConnect.Statistics;
|
|
using QuantConnect.Util;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using QuantConnect.Data.UniverseSelection;
|
|
using QuantConnect.Logging;
|
|
using QuantConnect.Packets;
|
|
using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace QuantConnect.Research
|
|
{
|
|
/// <summary>
|
|
/// Provides access to data for quantitative analysis
|
|
/// </summary>
|
|
public class QuantBook : QCAlgorithm
|
|
{
|
|
private dynamic _pandas;
|
|
private IDataCacheProvider _dataCacheProvider;
|
|
private IDataProvider _dataProvider;
|
|
private static bool _isPythonNotebook;
|
|
|
|
static QuantBook()
|
|
{
|
|
Logging.Log.LogHandler =
|
|
Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
|
|
|
|
//Determine if we are in a Python Notebook
|
|
try
|
|
{
|
|
using (Py.GIL())
|
|
{
|
|
var isPython = PythonEngine.ModuleFromString(Guid.NewGuid().ToString(),
|
|
"try:\n" +
|
|
" import IPython\n" +
|
|
" def IsPythonNotebook():\n" +
|
|
" return (IPython.get_ipython() != None)\n" +
|
|
"except:\n" +
|
|
" print('No IPython installed')\n" +
|
|
" def IsPythonNotebook():\n" +
|
|
" return false\n").GetAttr("IsPythonNotebook").Invoke();
|
|
isPython.TryConvert(out _isPythonNotebook);
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
//Default to false
|
|
_isPythonNotebook = false;
|
|
Logging.Log.Error("QuantBook failed to determine Notebook kernel language");
|
|
}
|
|
|
|
Logging.Log.Trace($"QuantBook started; Is Python: {_isPythonNotebook}");
|
|
}
|
|
|
|
/// <summary>
|
|
/// <see cref = "QuantBook" /> constructor.
|
|
/// Provides access to data for quantitative analysis
|
|
/// </summary>
|
|
public QuantBook() : base()
|
|
{
|
|
try
|
|
{
|
|
using (Py.GIL())
|
|
{
|
|
_pandas = Py.Import("pandas");
|
|
}
|
|
|
|
// Issue #4892 : Set start time relative to NY time
|
|
// when the data is available from the previous day
|
|
var newYorkTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
|
|
var hourThreshold = Config.GetInt("qb-data-hour", 9);
|
|
|
|
// If it is after our hour threshold; then we can use today
|
|
if (newYorkTime.Hour >= hourThreshold)
|
|
{
|
|
SetStartDate(newYorkTime);
|
|
}
|
|
else
|
|
{
|
|
SetStartDate(newYorkTime - TimeSpan.FromDays(1));
|
|
}
|
|
|
|
|
|
// Sets PandasConverter
|
|
SetPandasConverter();
|
|
|
|
// Initialize History Provider
|
|
var composer = new Composer();
|
|
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
|
|
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
|
|
// init the API
|
|
systemHandlers.Initialize();
|
|
systemHandlers.LeanManager.Initialize(systemHandlers,
|
|
algorithmHandlers,
|
|
new BacktestNodePacket(),
|
|
new AlgorithmManager(false));
|
|
systemHandlers.LeanManager.SetAlgorithm(this);
|
|
|
|
algorithmHandlers.ObjectStore.Initialize("QuantBook",
|
|
Config.GetInt("job-user-id"),
|
|
Config.GetInt("project-id"),
|
|
Config.Get("api-access-token"),
|
|
new Controls
|
|
{
|
|
// if <= 0 we disable periodic persistence and make it synchronous
|
|
PersistenceIntervalSeconds = -1,
|
|
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
|
|
StorageFileCount = Config.GetInt("storage-file-count", 100),
|
|
StoragePermissions = (FileAccess) Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
|
|
});
|
|
SetObjectStore(algorithmHandlers.ObjectStore);
|
|
|
|
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
|
|
_dataProvider = algorithmHandlers.DataProvider;
|
|
|
|
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
|
|
var registeredTypes = new RegisteredSecurityDataTypesProvider();
|
|
var securityService = new SecurityService(Portfolio.CashBook,
|
|
MarketHoursDatabase,
|
|
symbolPropertiesDataBase,
|
|
this,
|
|
registeredTypes,
|
|
new SecurityCacheProvider(Portfolio));
|
|
Securities.SetSecurityService(securityService);
|
|
SubscriptionManager.SetDataManager(
|
|
new DataManager(new NullDataFeed(),
|
|
new UniverseSelection(this, securityService, algorithmHandlers.DataPermissionsManager, algorithmHandlers.DataProvider),
|
|
this,
|
|
TimeKeeper,
|
|
MarketHoursDatabase,
|
|
false,
|
|
registeredTypes,
|
|
algorithmHandlers.DataPermissionsManager));
|
|
|
|
var mapFileProvider = algorithmHandlers.MapFileProvider;
|
|
HistoryProvider = composer.GetExportedValueByTypeName<IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
|
|
HistoryProvider.Initialize(
|
|
new HistoryProviderInitializeParameters(
|
|
null,
|
|
null,
|
|
algorithmHandlers.DataProvider,
|
|
_dataCacheProvider,
|
|
mapFileProvider,
|
|
algorithmHandlers.FactorFileProvider,
|
|
null,
|
|
true,
|
|
algorithmHandlers.DataPermissionsManager
|
|
)
|
|
);
|
|
|
|
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
|
|
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
throw new Exception("QuantBook.Main(): " + exception);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Python implementation of GetFundamental, get fundamental data for input symbols or tickers
|
|
/// </summary>
|
|
/// <param name="input">The symbols or tickers to retrieve fundamental data for</param>
|
|
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>pandas DataFrame</returns>
|
|
public PyObject GetFundamental(PyObject input, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
//Null selector is not allowed for Python DataFrame
|
|
if (string.IsNullOrWhiteSpace(selector))
|
|
{
|
|
throw new ArgumentException("Invalid selector. Cannot be None, empty or consist only of white-space characters");
|
|
}
|
|
|
|
//Covert to symbols
|
|
var symbols = PythonUtil.ConvertToSymbols(input);
|
|
|
|
//Fetch the data
|
|
var fundamentalData = GetAllFundamental(symbols, selector, start, end);
|
|
|
|
using (Py.GIL())
|
|
{
|
|
var data = new PyDict();
|
|
foreach (var day in fundamentalData.OrderBy(x => x.Key))
|
|
{
|
|
var orderedValues = day.Value.OrderBy(x => x.Key.ID.ToString()).ToList();
|
|
var columns = orderedValues.Select(x => x.Key.ID.ToString());
|
|
var values = orderedValues.Select(x => x.Value);
|
|
var row = _pandas.Series(values, columns);
|
|
data.SetItem(day.Key.ToPython(), row);
|
|
}
|
|
|
|
return _pandas.DataFrame.from_dict(data, orient:"index");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get fundamental data from given symbols
|
|
/// </summary>
|
|
/// <param name="symbols">The symbols to retrieve fundamental data for</param>
|
|
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>Enumerable collection of DataDictionaries, one dictionary for each day there is data</returns>
|
|
public IEnumerable<DataDictionary<dynamic>> GetFundamental(IEnumerable<Symbol> symbols, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
var data = GetAllFundamental(symbols, selector, start, end);
|
|
|
|
foreach (var kvp in data.OrderBy(kvp => kvp.Key))
|
|
{
|
|
yield return kvp.Value;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get fundamental data for a given symbol
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve fundamental data for</param>
|
|
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>Enumerable collection of DataDictionaries, one Dictionary for each day there is data.</returns>
|
|
public IEnumerable<DataDictionary<dynamic>> GetFundamental(Symbol symbol, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
var list = new List<Symbol>
|
|
{
|
|
symbol
|
|
};
|
|
|
|
return GetFundamental(list, selector, start, end);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get fundamental data for a given set of tickers
|
|
/// </summary>
|
|
/// <param name="tickers">The tickers to retrieve fundamental data for</param>
|
|
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>Enumerable collection of DataDictionaries, one dictionary for each day there is data.</returns>
|
|
public IEnumerable<DataDictionary<dynamic>> GetFundamental(IEnumerable<string> tickers, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
var list = new List<Symbol>();
|
|
foreach (var ticker in tickers)
|
|
{
|
|
list.Add(QuantConnect.Symbol.Create(ticker, SecurityType.Equity, Market.USA));
|
|
}
|
|
|
|
return GetFundamental(list, selector, start, end);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get fundamental data for a given ticker
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve fundamental data for</param>
|
|
/// <param name="selector">Selects a value from the Fundamental data to filter the request output</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>Enumerable collection of DataDictionaries, one Dictionary for each day there is data.</returns>
|
|
public dynamic GetFundamental(string ticker, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
//Check if its Python; PythonNet likes to convert the strings, but for python we want the DataFrame as the return object
|
|
//So we must route the function call to the Python version.
|
|
if (_isPythonNotebook)
|
|
{
|
|
return GetFundamental(ticker.ToPython(), selector, start, end);
|
|
}
|
|
|
|
var symbol = QuantConnect.Symbol.Create(ticker, SecurityType.Equity, Market.USA);
|
|
var list = new List<Symbol>
|
|
{
|
|
symbol
|
|
};
|
|
|
|
return GetFundamental(list, selector, start, end);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets <see cref="OptionHistory"/> object for a given symbol, date and resolution
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve historical option data for</param>
|
|
/// <param name="start">The history request start time</param>
|
|
/// <param name="end">The history request end time. Defaults to 1 day if null</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <returns>A <see cref="OptionHistory"/> object that contains historical option data.</returns>
|
|
public OptionHistory GetOptionHistory(Symbol symbol, DateTime start, DateTime? end = null, Resolution? resolution = null)
|
|
{
|
|
if (!end.HasValue || end.Value == start)
|
|
{
|
|
end = start.AddDays(1);
|
|
}
|
|
|
|
// Load a canonical option Symbol if the user provides us with an underlying Symbol
|
|
if (symbol.SecurityType != SecurityType.Option && symbol.SecurityType != SecurityType.FutureOption)
|
|
{
|
|
symbol = AddOption(symbol, resolution, symbol.ID.Market).Symbol;
|
|
}
|
|
|
|
IEnumerable<Symbol> symbols;
|
|
if (symbol.IsCanonical())
|
|
{
|
|
// canonical symbol, lets find the contracts
|
|
var option = Securities[symbol] as Option;
|
|
var resolutionToUseForUnderlying = resolution ?? SubscriptionManager.SubscriptionDataConfigService
|
|
.GetSubscriptionDataConfigs(symbol)
|
|
.GetHighestResolution();
|
|
if (!Securities.ContainsKey(symbol.Underlying))
|
|
{
|
|
// only add underlying if not present
|
|
AddEquity(symbol.Underlying.Value, resolutionToUseForUnderlying);
|
|
}
|
|
var allSymbols = new List<Symbol>();
|
|
for (var date = start; date < end; date = date.AddDays(1))
|
|
{
|
|
if (option.Exchange.DateIsOpen(date))
|
|
{
|
|
allSymbols.AddRange(OptionChainProvider.GetOptionContractList(symbol.Underlying, date));
|
|
}
|
|
}
|
|
|
|
var optionFilterUniverse = new OptionFilterUniverse();
|
|
var distinctSymbols = allSymbols.Distinct();
|
|
symbols = base.History(symbol.Underlying, start, end.Value, resolution)
|
|
.SelectMany(x =>
|
|
{
|
|
// the option chain symbols wont change so we can set 'exchangeDateChange' to false always
|
|
optionFilterUniverse.Refresh(distinctSymbols, x, exchangeDateChange:false);
|
|
return option.ContractFilter.Filter(optionFilterUniverse);
|
|
})
|
|
.Distinct().Concat(new[] { symbol.Underlying });
|
|
}
|
|
else
|
|
{
|
|
// the symbol is a contract
|
|
symbols = new List<Symbol>{ symbol };
|
|
}
|
|
|
|
return new OptionHistory(History(symbols, start, end.Value, resolution));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets <see cref="FutureHistory"/> object for a given symbol, date and resolution
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve historical future data for</param>
|
|
/// <param name="start">The history request start time</param>
|
|
/// <param name="end">The history request end time. Defaults to 1 day if null</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <returns>A <see cref="FutureHistory"/> object that contains historical future data.</returns>
|
|
public FutureHistory GetFutureHistory(Symbol symbol, DateTime start, DateTime? end = null, Resolution? resolution = null)
|
|
{
|
|
if (!end.HasValue || end.Value == start)
|
|
{
|
|
end = start.AddDays(1);
|
|
}
|
|
|
|
var allSymbols = new HashSet<Symbol>();
|
|
if (symbol.IsCanonical())
|
|
{
|
|
// canonical symbol, lets find the contracts
|
|
var future = Securities[symbol] as Future;
|
|
|
|
for (var date = start; date < end; date = date.AddDays(1))
|
|
{
|
|
if (future.Exchange.DateIsOpen(date))
|
|
{
|
|
var underlying = new Tick { Time = date };
|
|
var allList = FutureChainProvider.GetFutureContractList(future.Symbol, date);
|
|
|
|
allSymbols.UnionWith(future.ContractFilter.Filter(new FutureFilterUniverse(allList, underlying)));
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// the symbol is a contract
|
|
allSymbols.Add(symbol);
|
|
}
|
|
|
|
return new FutureHistory(History(allSymbols, start, end.Value, resolution));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of an indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="periods">The number of bars to request</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of an indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IndicatorDataPoint> indicator, Symbol symbol, int period, Resolution? resolution = null, Func<IBaseData, decimal> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, period, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="periods">The number of bars to request</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IBaseDataBar> indicator, Symbol symbol, int period, Resolution? resolution = null, Func<IBaseData, IBaseDataBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, period, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="periods">The number of bars to request</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<TradeBar> indicator, Symbol symbol, int period, Resolution? resolution = null, Func<IBaseData, TradeBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, period, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of an indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="span">The span over which to retrieve recent historical data</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of an indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IndicatorDataPoint> indicator, Symbol symbol, TimeSpan span, Resolution? resolution = null, Func<IBaseData, decimal> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, span, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="span">The span over which to retrieve recent historical data</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IBaseDataBar> indicator, Symbol symbol, TimeSpan span, Resolution? resolution = null, Func<IBaseData, IBaseDataBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, span, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="span">The span over which to retrieve recent historical data</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<TradeBar> indicator, Symbol symbol, TimeSpan span, Resolution? resolution = null, Func<IBaseData, TradeBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, span, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of an indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="start">The start time in the algorithm's time zone</param>
|
|
/// <param name="end">The end time in the algorithm's time zone</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of an indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IndicatorDataPoint> indicator, Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, Func<IBaseData, decimal> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, start, end, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="start">The start time in the algorithm's time zone</param>
|
|
/// <param name="end">The end time in the algorithm's time zone</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<IBaseDataBar> indicator, Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, Func<IBaseData, IBaseDataBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, start, end, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of a bar indicator for the specified symbol. The exact number of bars will be returned.
|
|
/// The symbol must exist in the Securities collection.
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="symbol">The symbol to retrieve historical data for</param>
|
|
/// <param name="start">The start time in the algorithm's time zone</param>
|
|
/// <param name="end">The end time in the algorithm's time zone</param>
|
|
/// <param name="resolution">The resolution to request</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame of historical data of a bar indicator</returns>
|
|
public PyObject Indicator(IndicatorBase<TradeBar> indicator, Symbol symbol, DateTime start, DateTime end, Resolution? resolution = null, Func<IBaseData, TradeBar> selector = null)
|
|
{
|
|
var history = History(new[] { symbol }, start, end, resolution);
|
|
return Indicator(indicator, history, selector);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets Portfolio Statistics from a pandas.DataFrame with equity and benchmark values
|
|
/// </summary>
|
|
/// <param name="dataFrame">pandas.DataFrame with the information required to compute the Portfolio statistics</param>
|
|
/// <returns><see cref="PortfolioStatistics"/> object wrapped in a <see cref="PyDict"/> with the portfolio statistics.</returns>
|
|
public PyDict GetPortfolioStatistics(PyObject dataFrame)
|
|
{
|
|
var dictBenchmark = new SortedDictionary<DateTime, double>();
|
|
var dictEquity = new SortedDictionary<DateTime, double>();
|
|
var dictPL = new SortedDictionary<DateTime, double>();
|
|
|
|
using (Py.GIL())
|
|
{
|
|
var result = new PyDict();
|
|
|
|
try
|
|
{
|
|
// Converts the data from pandas.DataFrame into dictionaries keyed by time
|
|
var df = ((dynamic)dataFrame).dropna();
|
|
dictBenchmark = GetDictionaryFromSeries((PyObject)df["benchmark"]);
|
|
dictEquity = GetDictionaryFromSeries((PyObject)df["equity"]);
|
|
dictPL = GetDictionaryFromSeries((PyObject)df["equity"].pct_change());
|
|
}
|
|
catch (PythonException e)
|
|
{
|
|
result.SetItem("Runtime Error", e.Message.ToPython());
|
|
return result;
|
|
}
|
|
|
|
// Convert the double into decimal
|
|
var equity = new SortedDictionary<DateTime, decimal>(dictEquity.ToDictionary(kvp => kvp.Key, kvp => (decimal)kvp.Value));
|
|
var profitLoss = new SortedDictionary<DateTime, decimal>(dictPL.ToDictionary(kvp => kvp.Key, kvp => double.IsNaN(kvp.Value) ? 0 : (decimal)kvp.Value));
|
|
|
|
// Gets the last value of the day of the benchmark and equity
|
|
var listBenchmark = CalculateDailyRateOfChange(dictBenchmark);
|
|
var listPerformance = CalculateDailyRateOfChange(dictEquity);
|
|
|
|
// Gets the startting capital
|
|
var startingCapital = Convert.ToDecimal(dictEquity.FirstOrDefault().Value);
|
|
|
|
// Compute portfolio statistics
|
|
var stats = new PortfolioStatistics(profitLoss, equity, listPerformance, listBenchmark, startingCapital);
|
|
|
|
result.SetItem("Average Win (%)", Convert.ToDouble(stats.AverageWinRate * 100).ToPython());
|
|
result.SetItem("Average Loss (%)", Convert.ToDouble(stats.AverageLossRate * 100).ToPython());
|
|
result.SetItem("Compounding Annual Return (%)", Convert.ToDouble(stats.CompoundingAnnualReturn * 100m).ToPython());
|
|
result.SetItem("Drawdown (%)", Convert.ToDouble(stats.Drawdown * 100).ToPython());
|
|
result.SetItem("Expectancy", Convert.ToDouble(stats.Expectancy).ToPython());
|
|
result.SetItem("Net Profit (%)", Convert.ToDouble(stats.TotalNetProfit * 100).ToPython());
|
|
result.SetItem("Sharpe Ratio", Convert.ToDouble(stats.SharpeRatio).ToPython());
|
|
result.SetItem("Win Rate (%)", Convert.ToDouble(stats.WinRate * 100).ToPython());
|
|
result.SetItem("Loss Rate (%)", Convert.ToDouble(stats.LossRate * 100).ToPython());
|
|
result.SetItem("Profit-Loss Ratio", Convert.ToDouble(stats.ProfitLossRatio).ToPython());
|
|
result.SetItem("Alpha", Convert.ToDouble(stats.Alpha).ToPython());
|
|
result.SetItem("Beta", Convert.ToDouble(stats.Beta).ToPython());
|
|
result.SetItem("Annual Standard Deviation", Convert.ToDouble(stats.AnnualStandardDeviation).ToPython());
|
|
result.SetItem("Annual Variance", Convert.ToDouble(stats.AnnualVariance).ToPython());
|
|
result.SetItem("Information Ratio", Convert.ToDouble(stats.InformationRatio).ToPython());
|
|
result.SetItem("Tracking Error", Convert.ToDouble(stats.TrackingError).ToPython());
|
|
result.SetItem("Treynor Ratio", Convert.ToDouble(stats.TreynorRatio).ToPython());
|
|
|
|
return result;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Converts a pandas.Series into a <see cref="SortedDictionary{DateTime, Double}"/>
|
|
/// </summary>
|
|
/// <param name="series">pandas.Series to be converted</param>
|
|
/// <returns><see cref="SortedDictionary{DateTime, Double}"/> with pandas.Series information</returns>
|
|
private SortedDictionary<DateTime, double> GetDictionaryFromSeries(PyObject series)
|
|
{
|
|
var dictionary = new SortedDictionary<DateTime, double>();
|
|
|
|
var pyDict = new PyDict(((dynamic)series).to_dict());
|
|
foreach (PyObject item in pyDict.Items())
|
|
{
|
|
var key = (DateTime)item[0].AsManagedObject(typeof(DateTime));
|
|
var value = (double)item[1].AsManagedObject(typeof(double));
|
|
dictionary.Add(key, value);
|
|
}
|
|
|
|
return dictionary;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calculates the daily rate of change
|
|
/// </summary>
|
|
/// <param name="dictionary"><see cref="IDictionary{DateTime, Double}"/> with prices keyed by time</param>
|
|
/// <returns><see cref="List{Double}"/> with daily rate of change</returns>
|
|
private List<double> CalculateDailyRateOfChange(IDictionary<DateTime, double> dictionary)
|
|
{
|
|
var daily = dictionary.GroupBy(kvp => kvp.Key.Date)
|
|
.ToDictionary(x => x.Key, v => v.LastOrDefault().Value)
|
|
.Values.ToArray();
|
|
|
|
var rocp = new double[daily.Length];
|
|
for (var i = 1; i < daily.Length; i++)
|
|
{
|
|
rocp[i] = (daily[i] - daily[i - 1]) / daily[i - 1];
|
|
}
|
|
rocp[0] = 0;
|
|
|
|
return rocp.ToList();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of an indicator and convert it into pandas.DataFrame
|
|
/// </summary>
|
|
/// <param name="indicator">Indicator</param>
|
|
/// <param name="history">Historical data used to calculate the indicator</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns>
|
|
private PyObject Indicator(IndicatorBase<IndicatorDataPoint> indicator, IEnumerable<Slice> history, Func<IBaseData, decimal> selector = null)
|
|
{
|
|
// Reset the indicator
|
|
indicator.Reset();
|
|
|
|
// Create a dictionary of the properties
|
|
var name = indicator.GetType().Name;
|
|
|
|
var properties = indicator.GetType().GetProperties()
|
|
.Where(x => x.PropertyType.IsGenericType)
|
|
.ToDictionary(x => x.Name, y => new List<IndicatorDataPoint>());
|
|
properties.Add(name, new List<IndicatorDataPoint>());
|
|
|
|
indicator.Updated += (s, e) =>
|
|
{
|
|
if (!indicator.IsReady)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var kvp in properties)
|
|
{
|
|
var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current");
|
|
kvp.Value.Add((IndicatorDataPoint)dataPoint);
|
|
}
|
|
};
|
|
|
|
selector = selector ?? (x => x.Value);
|
|
|
|
history.PushThrough(bar =>
|
|
{
|
|
var value = selector(bar);
|
|
indicator.Update(bar.EndTime, value);
|
|
});
|
|
|
|
return PandasConverter.GetIndicatorDataFrame(properties);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the historical data of an bar indicator and convert it into pandas.DataFrame
|
|
/// </summary>
|
|
/// <param name="indicator">Bar indicator</param>
|
|
/// <param name="history">Historical data used to calculate the indicator</param>
|
|
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
|
|
/// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns>
|
|
private PyObject Indicator<T>(IndicatorBase<T> indicator, IEnumerable<Slice> history, Func<IBaseData, T> selector = null)
|
|
where T : IBaseData
|
|
{
|
|
// Reset the indicator
|
|
indicator.Reset();
|
|
|
|
// Create a dictionary of the properties
|
|
var name = indicator.GetType().Name;
|
|
|
|
var properties = indicator.GetType().GetProperties()
|
|
.Where(x => x.PropertyType.IsGenericType)
|
|
.ToDictionary(x => x.Name, y => new List<IndicatorDataPoint>());
|
|
properties.Add(name, new List<IndicatorDataPoint>());
|
|
|
|
indicator.Updated += (s, e) =>
|
|
{
|
|
if (!indicator.IsReady)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var kvp in properties)
|
|
{
|
|
var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current");
|
|
kvp.Value.Add((IndicatorDataPoint)dataPoint);
|
|
}
|
|
};
|
|
|
|
selector = selector ?? (x => (T)x);
|
|
|
|
history.PushThrough(bar => indicator.Update(selector(bar)));
|
|
|
|
return PandasConverter.GetIndicatorDataFrame(properties);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets a value of a property
|
|
/// </summary>
|
|
/// <param name="baseData">Object with the desired property</param>
|
|
/// <param name="fullName">Property name</param>
|
|
/// <returns>Property value</returns>
|
|
private object GetPropertyValue(object baseData, string fullName)
|
|
{
|
|
foreach (var name in fullName.Split('.'))
|
|
{
|
|
if (baseData == null) return null;
|
|
|
|
var info = baseData.GetType().GetProperty(name);
|
|
|
|
baseData = info?.GetValue(baseData, null);
|
|
}
|
|
|
|
return baseData;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get all fundamental data for given symbols
|
|
/// </summary>
|
|
/// <param name="symbols">The symbols to retrieve fundamental data for</param>
|
|
/// <param name="start">The start date of selected data</param>
|
|
/// <param name="end">The end date of selected data</param>
|
|
/// <returns>DataDictionary of Enumerable IBaseData</returns>
|
|
private Dictionary<DateTime, DataDictionary<dynamic>> GetAllFundamental(IEnumerable<Symbol> symbols, string selector, DateTime? start = null, DateTime? end = null)
|
|
{
|
|
//SubscriptionRequest does not except nullable DateTimes, so set a startTime and endTime
|
|
var startTime = start.HasValue ? (DateTime)start : QuantConnect.Time.BeginningOfTime;
|
|
var endTime = end.HasValue ? (DateTime)end : QuantConnect.Time.EndOfTime;
|
|
|
|
//Collection to store our results
|
|
var data = new Dictionary<DateTime, DataDictionary<dynamic>>();
|
|
|
|
//Build factory
|
|
var factory = new FineFundamentalSubscriptionEnumeratorFactory(false);
|
|
|
|
//Get all data for each symbol and fill our dictionary
|
|
var options = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
|
|
Parallel.ForEach(symbols, options, symbol =>
|
|
{
|
|
var config = new SubscriptionDataConfig(
|
|
typeof(FineFundamental),
|
|
symbol,
|
|
Resolution.Daily,
|
|
TimeZones.NewYork,
|
|
TimeZones.NewYork,
|
|
false,
|
|
false,
|
|
false
|
|
);
|
|
var security = Securities.CreateSecurity(symbol, config);
|
|
var request = new SubscriptionRequest(false, null, security, config, startTime.ConvertToUtc(TimeZones.NewYork), endTime.ConvertToUtc(TimeZones.NewYork));
|
|
using (var enumerator = factory.CreateEnumerator(request, _dataProvider))
|
|
{
|
|
while (enumerator.MoveNext())
|
|
{
|
|
var dataPoint = string.IsNullOrWhiteSpace(selector)
|
|
? enumerator.Current
|
|
: GetPropertyValue(enumerator.Current, selector);
|
|
|
|
lock (data)
|
|
{
|
|
if (!data.ContainsKey(enumerator.Current.Time))
|
|
{
|
|
data[enumerator.Current.Time] = new DataDictionary<dynamic>(enumerator.Current.Time);
|
|
}
|
|
data[enumerator.Current.Time].Add(enumerator.Current.Symbol, dataPoint);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
return data;
|
|
}
|
|
}
|
|
}
|