Files
quantconnect--lean/Engine/LeanEngineAlgorithmHandlers.cs
T
Stefano Raggi d407307566 Add IObjectStore interface with LocalObjectStore implementation
This commit is squashed from iterative development:

- More consistent method naming
- Storage root path updated to be absolute and include algorithm name
- Storage root path created only if object store is actually used
- Implemented XML save/load
- Added missing unit tests
- Replaced Log.Trace with Log.Error calls
- Added the object store name logging in Engine.Main
- Read storage root from config
- Create algorithm storage root folder in Initialize
- Remove empty folder in Dispose
- Added null checks in all methods
- Added missing XML parameter docs
- make Initialize and Dispose virtual
- make AlgorithmStorageRoot protected

The IObjectStore abstraction provides algorithms with a persistent
storage mechanism. While the algorithm is running, data is maintained
in memory as a dictionary of raw bytes (string -> byte[]). This ensures
we avoid any reference type shenanigans. Periodically, the data in the
object store is persisted and additionally, when the algorithm shuts
down, the object store's data will again be persisted. This ensures that
when the algorithm starts up again, it will have access to any state
that has been saved into the object store.

A great use case for IObjectStore is saving a compute heavy model.
For example, computing the weights of a deep neural network is very
CPU intensive, but after the weights are computed, evaluation is fairly
quick. An initial backtest can be used to solved for the network's weights
and then subsequent backtests or even in live mode, the weights will be
available to the algorithm provided they were saved into the object store.

Also, some libraries require a file path to load model data. The object
store provides a `GetFilePath(key)` method which will copy the data for
the provided key to the disk and return that path so the library can load
the model data.
2019-12-17 22:21:11 -05:00

210 lines
9.3 KiB
C#

/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.ComponentModel.Composition;
using QuantConnect.Configuration;
using QuantConnect.Interfaces;
using QuantConnect.Lean.Engine.Alpha;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.RealTime;
using QuantConnect.Lean.Engine.Results;
using QuantConnect.Lean.Engine.Setup;
using QuantConnect.Lean.Engine.TransactionHandlers;
using QuantConnect.Logging;
using QuantConnect.Util;
namespace QuantConnect.Lean.Engine
{
/// <summary>
/// Provides a container for the algorithm specific handlers
/// </summary>
public class LeanEngineAlgorithmHandlers : IDisposable
{
/// <summary>
/// Gets the result handler used to communicate results from the algorithm
/// </summary>
public IResultHandler Results { get; }
/// <summary>
/// Gets the setup handler used to initialize the algorithm state
/// </summary>
public ISetupHandler Setup { get; }
/// <summary>
/// Gets the data feed handler used to provide data to the algorithm
/// </summary>
public IDataFeed DataFeed { get; }
/// <summary>
/// Gets the transaction handler used to process orders from the algorithm
/// </summary>
public ITransactionHandler Transactions { get; }
/// <summary>
/// Gets the real time handler used to process real time events
/// </summary>
public IRealTimeHandler RealTime { get; }
/// <summary>
/// Gets the map file provider used as a map file source for the data feed
/// </summary>
public IMapFileProvider MapFileProvider { get; }
/// <summary>
/// Gets the map file provider used as a map file source for the data feed
/// </summary>
public IFactorFileProvider FactorFileProvider { get; }
/// <summary>
/// Gets the data file provider used to retrieve security data if it is not on the file system
/// </summary>
public IDataProvider DataProvider { get; }
/// <summary>
/// Gets the alpha handler used to process algorithm generated insights
/// </summary>
public IAlphaHandler Alphas { get; }
/// <summary>
/// Gets the object store used for persistence
/// </summary>
public IObjectStore ObjectStore { get; }
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
/// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
/// <param name="dataProvider">file provider used to retrieve security data if it is not on the file system</param>
/// <param name="alphas">The alpha handler used to process generated insights</param>
/// <param name="objectStore">The object store used for persistence</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IAlphaHandler alphas,
IObjectStore objectStore
)
{
if (results == null)
{
throw new ArgumentNullException(nameof(results));
}
if (setup == null)
{
throw new ArgumentNullException(nameof(setup));
}
if (dataFeed == null)
{
throw new ArgumentNullException(nameof(dataFeed));
}
if (transactions == null)
{
throw new ArgumentNullException(nameof(transactions));
}
if (realTime == null)
{
throw new ArgumentNullException(nameof(realTime));
}
if (mapFileProvider == null)
{
throw new ArgumentNullException(nameof(mapFileProvider));
}
if (factorFileProvider == null)
{
throw new ArgumentNullException(nameof(factorFileProvider));
}
if (dataProvider == null)
{
throw new ArgumentNullException(nameof(dataProvider));
}
if (alphas == null)
{
throw new ArgumentNullException(nameof(alphas));
}
if (objectStore == null)
{
throw new ArgumentNullException(nameof(objectStore));
}
Results = results;
Setup = setup;
DataFeed = dataFeed;
Transactions = transactions;
RealTime = realTime;
MapFileProvider = mapFileProvider;
FactorFileProvider = factorFileProvider;
DataProvider = dataProvider;
Alphas = alphas;
ObjectStore = objectStore;
}
/// <summary>
/// Creates a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified composer using type names from configuration
/// </summary>
/// <param name="composer">The composer instance to obtain implementations from</param>
/// <returns>A fully hydrates <see cref="LeanEngineSystemHandlers"/> instance.</returns>
/// <exception cref="CompositionException">Throws a CompositionException during failure to load</exception>
public static LeanEngineAlgorithmHandlers FromConfiguration(Composer composer)
{
var setupHandlerTypeName = Config.Get("setup-handler", "ConsoleSetupHandler");
var transactionHandlerTypeName = Config.Get("transaction-handler", "BacktestingTransactionHandler");
var realTimeHandlerTypeName = Config.Get("real-time-handler", "BacktestingRealTimeHandler");
var dataFeedHandlerTypeName = Config.Get("data-feed-handler", "FileSystemDataFeed");
var resultHandlerTypeName = Config.Get("result-handler", "BacktestingResultHandler");
var mapFileProviderTypeName = Config.Get("map-file-provider", "LocalDiskMapFileProvider");
var factorFileProviderTypeName = Config.Get("factor-file-provider", "LocalDiskFactorFileProvider");
var dataProviderTypeName = Config.Get("data-provider", "DefaultDataProvider");
var alphaHandlerTypeName = Config.Get("alpha-handler", "DefaultAlphaHandler");
var objectStoreTypeName = Config.Get("object-store", "LocalObjectStore");
return new LeanEngineAlgorithmHandlers(
composer.GetExportedValueByTypeName<IResultHandler>(resultHandlerTypeName),
composer.GetExportedValueByTypeName<ISetupHandler>(setupHandlerTypeName),
composer.GetExportedValueByTypeName<IDataFeed>(dataFeedHandlerTypeName),
composer.GetExportedValueByTypeName<ITransactionHandler>(transactionHandlerTypeName),
composer.GetExportedValueByTypeName<IRealTimeHandler>(realTimeHandlerTypeName),
composer.GetExportedValueByTypeName<IMapFileProvider>(mapFileProviderTypeName),
composer.GetExportedValueByTypeName<IFactorFileProvider>(factorFileProviderTypeName),
composer.GetExportedValueByTypeName<IDataProvider>(dataProviderTypeName),
composer.GetExportedValueByTypeName<IAlphaHandler>(alphaHandlerTypeName),
composer.GetExportedValueByTypeName<IObjectStore>(objectStoreTypeName)
);
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
Setup.Dispose();
ObjectStore.Dispose();
Log.Trace("LeanEngineAlgorithmHandlers.Dispose(): Disposed of algorithm handlers.");
}
}
}