Files
quantconnect--lean/Engine/DataFeeds/Enumerators/LiveFillForwardEnumerator.cs
Martin-Molinero 9cdb4a91c5 Refactor live data feed (#4636)
* Live Coarse universe refactor

- Live trading will source Coarse and Fine fundamental data directly
  from disk. Updating unit tests.

* Adds ILiveDataProvider interface

  * Adds wrapper for IDataQueueHandler implementations

  * Replaces IDataQueueHandler with ILiveDataProvider in
    LiveTradingDataFeed

  * Edits IDataQueueHandler documentation

* Maintains aggregation for current IDQH impls and skips for ILDF impls

  * Note: No unit test was created for this method, go back and TODO

* Protobuf Market data

- Adding protobuf support for Ticks, TradeBars and QuoteBars. Adding
  unit tests.

* Adds unit tests for LiveDataAggregator changes

  * Fixes bug where custom data was not handled as it was before
  * Fixes race condition bug because of variable reuse in class

* Add protobuf extension serialization

* Fixes for protobuf serialization

* Refactor

* Fix OptionChainUniverse

* replace BaseDataExchange pumping ticks with consolidators

* AlpacaBrokerage

* BitfinexBrokerage

* GDAXBrokerage

* OandaBrokerage

* InteractiveBrokers

* TradierBrokerage

* FxcmBrokerage

* PaperBrokerage

* etc

* WIP fixes for existing LTDF unit tests

* Fixes more LTDF unit tests

* make IDataAggregator.Update recieving Generic BaseData rather than Tick

* Change IDataQueueHandler.Subscribe method

* Some fixes after adding new commits

* Adds protobuf (de)serialization support for Dividend and Split

* Serialize protobuf with length prefix

* Fix missing LTDF unit tests

* Adds TiingoNews protobuf definitions

* fix comments

* more fixes on IQFeedDataQueueHandler

* disallow putting ticks into enumerator directly

* ScannableEnumerator tests

* fix OandaBrokerage

* AggregationManager unit tests

* fix AlpacaBrokerage tests

* fix InteractiveBrokers

* fix FxcmBrokerage tests

* call AggregationManager.Remove method on unsubscribe

* fix GDAX existing tests

* Fixes, refactor adding more tests for AggregatorManager

* Adds BenzingaNews protobuf definitions and round trip unit test

* Adds missing TiingoNews unit test to Protobuf round trip tests

* Improve sleep sequence of LiveSynchronizer

* need start aggregating first, and then can subscribe

* More test fixes and refactor

- Refactoring AggregationManager and ScannableEnumerator so the last is
  the one that owns the consolidator
- Adding pulse on the main LiveSynchronizer

* Improve performance of LEquityDataSynchronizingEnu

* Add missing Set job packet method

* Minor performance improvements

* Improvements add test timeout

- Improvements adding test timeout to find blocking test in travis

* Improve aggregationManager performance

* Testing improvements for travis

* Remove test timeouts

* More test fixes

- Adding more missing dispose calls and improving determinism

* fix IEXDataQueueHandler and tests

* Final tweaks to LTDF tests

* more AggregationManager tests

* consume and log ticks

* fix test: couldn't subscribe to Forex tickers

* change Resolution for all bar configs

* Improve RealTimeScheduleEventServiceAccuracy

* refactoring: move common code to base class

* fixed bug; unsubscribe SubscriptionDataConfig

* Small performance improvement

* Minor fixes

* Avoid Symbol serialization

* Fixes coarse selection in live mode

* Fix for live coarse

* Adds protobuf (de)serialization support for Robintrack

  * Adds round-trip unit test

* Minor performance improvements

* More minor performance improvements

* pass LiveNodePacket through to OandaBrokerage

* Fixes empty list becoming null value when deserializing with protobuf

* Reverts BZ live trading exception removal and fixes tests

* Refactor WorkQueue making it abstract

* Add try catch for composer

* Adds optional data batching period to LiveFillForwardEnumerator

* Override data-queue-handler with config

* Improve PeriodCountConsolidator.Scan performance

* Move batching delay to main Synchornizer thread

* Reverts addition of Robintrack protobuf definitions

* Give priority to config history provider if set

* Add Estimize protobuffing

- Add Estimize protobuffing support. Adding unit tests

* Always dispose of data queue handler

Co-authored-by: Gerardo Salazar <gsalaz9800@gmail.com>
Co-authored-by: Adalyat Nazirov <aenazirov@gmail.com>
2020-08-18 20:21:10 -03:00

96 lines
5.2 KiB
C#

/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.Collections.Generic;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Securities;
using QuantConnect.Util;
namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
{
/// <summary>
/// An implementation of the <see cref="FillForwardEnumerator"/> that uses an <see cref="ITimeProvider"/>
/// to determine if a fill forward bar needs to be emitted
/// </summary>
public class LiveFillForwardEnumerator : FillForwardEnumerator
{
private readonly ITimeProvider _timeProvider;
/// <summary>
/// Initializes a new instance of the <see cref="LiveFillForwardEnumerator"/> class that accepts
/// a reference to the fill forward resolution, useful if the fill forward resolution is dynamic
/// and changing as the enumeration progresses
/// </summary>
/// <param name="timeProvider">The source of time used to gauage when this enumerator should emit extra bars when
/// null data is returned from the source enumerator</param>
/// <param name="enumerator">The source enumerator to be filled forward</param>
/// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
/// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
/// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
/// <param name="subscriptionEndTime">The end time of the subscription, once passing this date the enumerator will stop</param>
/// <param name="dataResolution">The source enumerator's data resolution</param>
/// <param name="dataTimeZone">Time zone of the underlying source data</param>
/// <param name="subscriptionStartTime">The start time of the subscription</param>
public LiveFillForwardEnumerator(ITimeProvider timeProvider, IEnumerator<BaseData> enumerator, SecurityExchange exchange, IReadOnlyRef<TimeSpan> fillForwardResolution, bool isExtendedMarketHours, DateTime subscriptionEndTime, TimeSpan dataResolution, DateTimeZone dataTimeZone, DateTime subscriptionStartTime)
: base(enumerator, exchange, fillForwardResolution, isExtendedMarketHours, subscriptionEndTime, dataResolution, dataTimeZone, subscriptionStartTime)
{
_timeProvider = timeProvider;
}
/// <summary>
/// Determines whether or not fill forward is required, and if true, will produce the new fill forward data
/// </summary>
/// <param name="fillForwardResolution"></param>
/// <param name="previous">The last piece of data emitted by this enumerator</param>
/// <param name="next">The next piece of data on the source enumerator, this may be null</param>
/// <param name="fillForward">When this function returns true, this will have a non-null value, null when the function returns false</param>
/// <returns>True when a new fill forward piece of data was produced and should be emitted by this enumerator</returns>
protected override bool RequiresFillForwardData(TimeSpan fillForwardResolution, BaseData previous, BaseData next, out BaseData fillForward)
{
// convert times to UTC for accurate comparisons and differences across DST changes
fillForward = null;
// Add a delay to the time we expect a data point if we've configured a delay for batching
var nextExpectedDataPointTimeUtc = previous.EndTime.ConvertToUtc(Exchange.TimeZone) + fillForwardResolution;
if (next != null)
{
// if not future data, just return the 'next'
if (next.EndTime.ConvertToUtc(Exchange.TimeZone) <= nextExpectedDataPointTimeUtc)
{
return false;
}
// next is future data, fill forward in between
var clone = previous.Clone(true);
clone.Time = previous.Time + fillForwardResolution;
fillForward = clone;
return true;
}
// the underlying enumerator returned null, check to see if time has passed for fill forwarding
if (nextExpectedDataPointTimeUtc <= _timeProvider.GetUtcNow())
{
var clone = previous.Clone(true);
clone.Time = previous.Time + fillForwardResolution;
fillForward = clone;
return true;
}
return false;
}
}
}