Files
quantconnect--lean/Common/Data/Custom/Estimize/EstimizeRelease.cs
Martin-Molinero 9cdb4a91c5 Refactor live data feed (#4636)
* Live Coarse universe refactor

- Live trading will source Coarse and Fine fundamental data directly
  from disk. Updating unit tests.

* Adds ILiveDataProvider interface

  * Adds wrapper for IDataQueueHandler implementations

  * Replaces IDataQueueHandler with ILiveDataProvider in
    LiveTradingDataFeed

  * Edits IDataQueueHandler documentation

* Maintains aggregation for current IDQH impls and skips for ILDF impls

  * Note: No unit test was created for this method, go back and TODO

* Protobuf Market data

- Adding protobuf support for Ticks, TradeBars and QuoteBars. Adding
  unit tests.

* Adds unit tests for LiveDataAggregator changes

  * Fixes bug where custom data was not handled as it was before
  * Fixes race condition bug because of variable reuse in class

* Add protobuf extension serialization

* Fixes for protobuf serialization

* Refactor

* Fix OptionChainUniverse

* replace BaseDataExchange pumping ticks with consolidators

* AlpacaBrokerage

* BitfinexBrokerage

* GDAXBrokerage

* OandaBrokerage

* InteractiveBrokers

* TradierBrokerage

* FxcmBrokerage

* PaperBrokerage

* etc

* WIP fixes for existing LTDF unit tests

* Fixes more LTDF unit tests

* make IDataAggregator.Update recieving Generic BaseData rather than Tick

* Change IDataQueueHandler.Subscribe method

* Some fixes after adding new commits

* Adds protobuf (de)serialization support for Dividend and Split

* Serialize protobuf with length prefix

* Fix missing LTDF unit tests

* Adds TiingoNews protobuf definitions

* fix comments

* more fixes on IQFeedDataQueueHandler

* disallow putting ticks into enumerator directly

* ScannableEnumerator tests

* fix OandaBrokerage

* AggregationManager unit tests

* fix AlpacaBrokerage tests

* fix InteractiveBrokers

* fix FxcmBrokerage tests

* call AggregationManager.Remove method on unsubscribe

* fix GDAX existing tests

* Fixes, refactor adding more tests for AggregatorManager

* Adds BenzingaNews protobuf definitions and round trip unit test

* Adds missing TiingoNews unit test to Protobuf round trip tests

* Improve sleep sequence of LiveSynchronizer

* need start aggregating first, and then can subscribe

* More test fixes and refactor

- Refactoring AggregationManager and ScannableEnumerator so the last is
  the one that owns the consolidator
- Adding pulse on the main LiveSynchronizer

* Improve performance of LEquityDataSynchronizingEnu

* Add missing Set job packet method

* Minor performance improvements

* Improvements add test timeout

- Improvements adding test timeout to find blocking test in travis

* Improve aggregationManager performance

* Testing improvements for travis

* Remove test timeouts

* More test fixes

- Adding more missing dispose calls and improving determinism

* fix IEXDataQueueHandler and tests

* Final tweaks to LTDF tests

* more AggregationManager tests

* consume and log ticks

* fix test: couldn't subscribe to Forex tickers

* change Resolution for all bar configs

* Improve RealTimeScheduleEventServiceAccuracy

* refactoring: move common code to base class

* fixed bug; unsubscribe SubscriptionDataConfig

* Small performance improvement

* Minor fixes

* Avoid Symbol serialization

* Fixes coarse selection in live mode

* Fix for live coarse

* Adds protobuf (de)serialization support for Robintrack

  * Adds round-trip unit test

* Minor performance improvements

* More minor performance improvements

* pass LiveNodePacket through to OandaBrokerage

* Fixes empty list becoming null value when deserializing with protobuf

* Reverts BZ live trading exception removal and fixes tests

* Refactor WorkQueue making it abstract

* Add try catch for composer

* Adds optional data batching period to LiveFillForwardEnumerator

* Override data-queue-handler with config

* Improve PeriodCountConsolidator.Scan performance

* Move batching delay to main Synchornizer thread

* Reverts addition of Robintrack protobuf definitions

* Give priority to config history provider if set

* Add Estimize protobuffing

- Add Estimize protobuffing support. Adding unit tests

* Always dispose of data queue handler

Co-authored-by: Gerardo Salazar <gsalaz9800@gmail.com>
Co-authored-by: Adalyat Nazirov <aenazirov@gmail.com>
2020-08-18 20:21:10 -03:00

226 lines
8.5 KiB
C#

/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using Newtonsoft.Json;
using System;
using System.IO;
using NodaTime;
using ProtoBuf;
using static QuantConnect.StringExtensions;
namespace QuantConnect.Data.Custom.Estimize
{
/// <summary>
/// Financial releases for the specified company
/// </summary>
[ProtoContract(SkipConstructor = true)]
public class EstimizeRelease : BaseData
{
/// <summary>
/// The unique identifier for the release
/// </summary>
[ProtoMember(10)]
[JsonProperty(PropertyName = "id")]
public string Id { get; set; }
/// <summary>
/// The fiscal year for the release
/// </summary>
[ProtoMember(11)]
[JsonProperty(PropertyName = "fiscal_year")]
public int FiscalYear { get; set; }
/// <summary>
/// The fiscal quarter for the release
/// </summary>
[ProtoMember(12)]
[JsonProperty(PropertyName = "fiscal_quarter")]
public int FiscalQuarter { get; set; }
/// <summary>
/// The date of the release
/// </summary>
[ProtoMember(13)]
[JsonProperty(PropertyName = "release_date")]
public DateTime ReleaseDate
{
get { return Time; }
set { Time = value; }
}
/// <summary>
/// The date of the release
/// </summary>
public override DateTime EndTime => ReleaseDate;
/// <summary>
/// The earnings per share for the specified fiscal quarter
/// </summary>
[ProtoMember(14)]
[JsonProperty(PropertyName = "eps")]
public decimal? Eps { get; set; }
/// <summary>
/// The earnings per share for the specified fiscal quarter
/// </summary>
public override decimal Value => Eps ?? 0m;
/// <summary>
/// The revenue for the specified fiscal quarter
/// </summary>
[ProtoMember(15)]
[JsonProperty(PropertyName = "revenue")]
public decimal? Revenue { get; set; }
/// <summary>
/// The estimated EPS from Wall Street
/// </summary>
[ProtoMember(16)]
[JsonProperty(PropertyName = "wallstreet_eps_estimate")]
public decimal? WallStreetEpsEstimate { get; set; }
/// <summary>
/// The estimated revenue from Wall Street
/// </summary>
[ProtoMember(17)]
[JsonProperty(PropertyName = "wallstreet_revenue_estimate")]
public decimal? WallStreetRevenueEstimate { get; set; }
/// <summary>
/// The mean EPS consensus by the Estimize community
/// </summary>
[ProtoMember(18)]
[JsonProperty(PropertyName = "consensus_eps_estimate")]
public decimal? ConsensusEpsEstimate { get; set; }
/// <summary>
/// The mean revenue consensus by the Estimize community
/// </summary>
[ProtoMember(19)]
[JsonProperty(PropertyName = "consensus_revenue_estimate")]
public decimal? ConsensusRevenueEstimate { get; set; }
/// <summary>
/// The weighted EPS consensus by the Estimize community
/// </summary>
[ProtoMember(20)]
[JsonProperty(PropertyName = "consensus_weighted_eps_estimate")]
public decimal? ConsensusWeightedEpsEstimate { get; set; }
/// <summary>
/// The weighted revenue consensus by the Estimize community
/// </summary>
[ProtoMember(21)]
[JsonProperty(PropertyName = "consensus_weighted_revenue_estimate")]
public decimal? ConsensusWeightedRevenueEstimate { get; set; }
/// <summary>
/// Without a default constructor, Json.NET will call the
/// other constructor with `null` for the string parameter
/// </summary>
public EstimizeRelease()
{
}
/// <summary>
/// Creates EstimizeRelease instance from a line of CSV
/// </summary>
/// <param name="csvLine">CSV line</param>
public EstimizeRelease(string csvLine)
{
// ReleaseDate[0], Id[1], FiscalYear[2], FiscalQuarter[3], Eps[4], Revenue[5], ConsensusEpsEstimate[6], ConsensusRevenueEstimate[7], WallStreetEpsEstimate[8], WallStreetRevenueEstimate[9], ConsensusWeightedEpsEstimate[10], ConsensusWeightedRevenueEstimate[11]");
var csv = csvLine.Split(',');
ReleaseDate = Parse.DateTimeExact(csv[0].Trim(), "yyyyMMdd HH:mm:ss");
Id = csv[1];
FiscalYear = Parse.Int(csv[2]);
FiscalQuarter = Parse.Int(csv[3]);
Eps = csv[4].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
Revenue = csv[5].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
ConsensusEpsEstimate = csv[6].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
ConsensusRevenueEstimate = csv[7].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
WallStreetEpsEstimate = csv[8].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
WallStreetRevenueEstimate = csv[9].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
ConsensusWeightedEpsEstimate = csv[10].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
ConsensusWeightedRevenueEstimate = csv[11].IfNotNullOrEmpty<decimal?>(s => Parse.Decimal(s));
}
/// <summary>
/// Return the Subscription Data Source gained from the URL
/// </summary>
/// <param name="config">Configuration object</param>
/// <param name="date">Date of this source file</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>Subscription Data Source.</returns>
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
var source = Path.Combine(
Globals.DataFolder,
"alternative",
"estimize",
"release",
$"{config.Symbol.Value.ToLowerInvariant()}.csv"
);
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects.
/// </summary>
/// <param name="config">Subscription data config setup object</param>
/// <param name="line">Content of the source document</param>
/// <param name="date">Date of the requested data</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>
/// Estimize Release object
/// </returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
return new EstimizeRelease(line)
{
Symbol = config.Symbol
};
}
/// <summary>
/// Formats a string with the Estimize Release information.
/// </summary>
public override string ToString()
{
return Invariant($"{Symbol}(Q{FiscalQuarter} {FiscalYear}) :: ") +
Invariant($"EPS: {Eps} ") +
Invariant($"Revenue: {Revenue} on ") +
Invariant($"{EndTime:yyyyMMdd}");
}
/// <summary>
/// Indicates if there is support for mapping
/// </summary>
/// <returns>True indicates mapping should be used</returns>
public override bool RequiresMapping()
{
return true;
}
/// <summary>
/// Specifies the data time zone for this data type. This is useful for custom data types
/// </summary>
/// <returns>The <see cref="DateTimeZone"/> of this data type</returns>
public override DateTimeZone DataTimeZone()
{
return TimeZones.Utc;
}
}
}