Jhonathan Abreu
1479a1e836
* Add Price, Value and Close aliases to BaseContract Expose Price, Value and Close on BaseContract as aliases of LastPrice to mimic the BaseData/TradeBar API without inheriting BaseData. Marked with PandasIgnore to avoid duplicating the LastPrice column in chain DataFrames. * Keep BaseContract chains DataFrame unchanged for Value alias The pandas converter force-includes the 'Value' member for non-Lean-data types (to preserve the value column of custom data types despite BaseData.Value being PandasIgnore'd). That override made the new BaseContract.Value alias leak into option/future chain DataFrames as a redundant 'value' column. Exclude BaseContract-derived types from the forced inclusion so their PandasIgnore'd aliases are honored and the chain DataFrames stay unchanged. * Isolate OptionContract alias test from shared price-model singleton OptionContract's default option data is the shared static OptionPriceModelResultData.Null singleton, which other tests mutate via Update. Reading LastPrice before setting up the contract's own data made the assertion depend on global test state (failing in CI with a leaked price). Assign a dedicated price model so the contract no longer reads from or writes to the shared singleton. * Give each OptionContract its own default option data instead of a shared singleton OptionContract initialized its option data to the shared static OptionPriceModelResultData.Null singleton. Update() mutates that data, so contracts without an explicit price model shared and clobbered each other's trade/quote/open-interest state (a latent bug, and the cause of order-dependent test failures). Each contract now gets its own default OptionPriceModelResultData instance, and the alias test no longer needs to work around the shared state. * Restore shared Null option data default and reset it in test SetUp Revert OptionContract back to the shared static OptionPriceModelResultData.Null default. To keep the alias test deterministic, reset that singleton in the fixture's SetUp by updating a throwaway contract with a zero-priced trade bar, and give the tested contract its own price model so its Update doesn't re-pollute the singleton.
754 lines
30 KiB
C#
754 lines
30 KiB
C#
/*
|
|
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
|
|
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
using Python.Runtime;
|
|
using QuantConnect.Data;
|
|
using QuantConnect.Data.Fundamental;
|
|
using QuantConnect.Data.Market;
|
|
using QuantConnect.Util;
|
|
using System;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.Linq;
|
|
using System.Reflection;
|
|
|
|
namespace QuantConnect.Python
|
|
{
|
|
/// <summary>
|
|
/// Organizes a list of data to create pandas.DataFrames
|
|
/// </summary>
|
|
public partial class PandasData
|
|
{
|
|
// we keep these so we don't need to ask for them each time
|
|
private static PyString _empty;
|
|
private static PyObject _pandas;
|
|
private static PyObject _pandasColumn;
|
|
private static PyObject _seriesFactory;
|
|
private static PyObject _dataFrameFactory;
|
|
private static PyObject _multiIndexFactory;
|
|
private static PyObject _multiIndex;
|
|
private static PyObject _indexFactory;
|
|
|
|
private static PyList _defaultNames;
|
|
private static PyList _level1Names;
|
|
private static PyList _level2Names;
|
|
private static PyList _level3Names;
|
|
|
|
private readonly static Dictionary<Type, List<DataTypeMember>> _membersCache = new();
|
|
|
|
private readonly static MemberInfo _tickLastPriceMember = typeof(Tick).GetProperty(nameof(Tick.LastPrice));
|
|
private readonly static MemberInfo _openInterestLastPriceMember = typeof(OpenInterest).GetProperty(nameof(Tick.LastPrice));
|
|
|
|
private static readonly string[] _nonLeanDataTypeForcedMemberNames = new[] { nameof(BaseData.Value) };
|
|
|
|
private readonly static string[] _quoteTickOnlyPropertes = new[] {
|
|
nameof(Tick.AskPrice),
|
|
nameof(Tick.AskSize),
|
|
nameof(Tick.BidPrice),
|
|
nameof(Tick.BidSize)
|
|
};
|
|
|
|
private static readonly Type PandasNonExpandableAttribute = typeof(PandasNonExpandableAttribute);
|
|
private static readonly Type PandasIgnoreAttribute = typeof(PandasIgnoreAttribute);
|
|
private static readonly Type PandasIgnoreMembersAttribute = typeof(PandasIgnoreMembersAttribute);
|
|
|
|
private static readonly IReadOnlyCollection<DateTime> EmptySeriesTimesKey = new List<DateTime>();
|
|
private static readonly List<DataTypeMember> EmptyDataTypeMembers = new List<DataTypeMember>();
|
|
|
|
private readonly Symbol _symbol;
|
|
private readonly bool _isFundamentalType;
|
|
private readonly bool _isBaseData;
|
|
private readonly bool _timeAsColumn;
|
|
private readonly Dictionary<string, Serie> _series;
|
|
|
|
private readonly Dictionary<Type, List<DataTypeMember>> _members = new();
|
|
|
|
/// <summary>
|
|
/// Gets true if this is a custom data request, false for normal QC data
|
|
/// </summary>
|
|
public bool IsCustomData { get; }
|
|
|
|
/// <summary>
|
|
/// Implied levels of a multi index pandas.Series (depends on the security type)
|
|
/// </summary>
|
|
public int Levels { get; } = 2;
|
|
|
|
/// <summary>
|
|
/// Initializes the static members of the <see cref="PandasData"/> class
|
|
/// </summary>
|
|
static PandasData()
|
|
{
|
|
using (Py.GIL())
|
|
{
|
|
// Use our PandasMapper class that modifies pandas indexing to support tickers, symbols and SIDs
|
|
_pandas = Py.Import("PandasMapper");
|
|
_pandasColumn = _pandas.GetAttr("PandasColumn");
|
|
_seriesFactory = _pandas.GetAttr("Series");
|
|
_dataFrameFactory = _pandas.GetAttr("DataFrame");
|
|
_multiIndex = _pandas.GetAttr("MultiIndex");
|
|
_multiIndexFactory = _multiIndex.GetAttr("from_tuples");
|
|
_indexFactory = _pandas.GetAttr("Index");
|
|
_empty = new PyString(string.Empty);
|
|
|
|
var time = new PyString("time");
|
|
var symbol = new PyString("symbol");
|
|
var expiry = new PyString("expiry");
|
|
_defaultNames = new PyList(new PyObject[] { expiry, new PyString("strike"), new PyString("type"), symbol, time });
|
|
_level1Names = new PyList(new PyObject[] { symbol });
|
|
_level2Names = new PyList(new PyObject[] { symbol, time });
|
|
_level3Names = new PyList(new PyObject[] { expiry, symbol, time });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Initializes an instance of <see cref="PandasData"/>
|
|
/// </summary>
|
|
public PandasData(object data, bool timeAsColumn = false)
|
|
{
|
|
_series = new();
|
|
var baseData = data as IBaseData;
|
|
|
|
// in the case we get a list/collection of data we take the first data point to determine the type
|
|
// but it's also possible to get a data which supports enumerating we don't care about those cases
|
|
if (baseData == null && data is IEnumerable enumerable)
|
|
{
|
|
foreach (var item in enumerable)
|
|
{
|
|
data = item;
|
|
baseData = data as IBaseData;
|
|
break;
|
|
}
|
|
}
|
|
|
|
var type = data.GetType();
|
|
_isFundamentalType = type == typeof(Fundamental);
|
|
_isBaseData = baseData != null;
|
|
_timeAsColumn = timeAsColumn && _isBaseData;
|
|
_symbol = _isBaseData ? baseData.Symbol : ((ISymbolProvider)data).Symbol;
|
|
IsCustomData = Extensions.IsCustomDataType(_symbol, type);
|
|
|
|
if (baseData == null)
|
|
{
|
|
Levels = 1;
|
|
}
|
|
else if (_symbol.SecurityType == SecurityType.Future)
|
|
{
|
|
Levels = 3;
|
|
}
|
|
else if (_symbol.SecurityType.IsOption())
|
|
{
|
|
Levels = 5;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds security data object to the end of the lists
|
|
/// </summary>
|
|
/// <param name="data"><see cref="IBaseData"/> object that contains security data</param>
|
|
public void Add(object data)
|
|
{
|
|
Add(data, false);
|
|
}
|
|
|
|
private void Add(object data, bool overrideValues)
|
|
{
|
|
if (data == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var typeMembers = GetInstanceDataTypeMembers(data);
|
|
|
|
var endTime = default(DateTime);
|
|
if (_isBaseData)
|
|
{
|
|
endTime = ((IBaseData)data).EndTime;
|
|
if (_timeAsColumn)
|
|
{
|
|
AddToSeries("time", endTime, endTime, overrideValues);
|
|
}
|
|
}
|
|
|
|
AddMembersData(data, typeMembers, endTime, overrideValues);
|
|
|
|
if (data is DynamicData dynamicData)
|
|
{
|
|
var storage = dynamicData.GetStorageDictionary();
|
|
var value = dynamicData.Value;
|
|
AddToSeries("value", endTime, value, overrideValues);
|
|
|
|
foreach (var kvp in storage.Where(x => x.Key != "value"
|
|
// if this is a PythonData instance we add in '__typename' which we don't want into the data frame
|
|
&& !x.Key.StartsWith("__", StringComparison.InvariantCulture)))
|
|
{
|
|
AddToSeries(kvp.Key, endTime, kvp.Value, overrideValues);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void AddMemberToSeries(object instance, DateTime endTime, DataTypeMember member, bool overrideValues)
|
|
{
|
|
var baseName = (string)null;
|
|
var tick = member.IsTickProperty ? instance as Tick : null;
|
|
if (tick != null && member.IsTickLastPrice && tick.TickType == TickType.OpenInterest)
|
|
{
|
|
baseName = "OpenInterest";
|
|
}
|
|
|
|
// TODO field/property.GetValue is expensive
|
|
var key = member.GetMemberName(baseName);
|
|
var value = member.GetValue(instance);
|
|
|
|
var memberType = member.GetMemberType();
|
|
// For DataDictionary instances, we only want to add the values
|
|
if (MemberIsDataDictionary(memberType))
|
|
{
|
|
value = memberType.GetProperty("Values").GetValue(value);
|
|
}
|
|
else if (member.IsProperty)
|
|
{
|
|
if (_isFundamentalType && value is FundamentalTimeDependentProperty timeDependentProperty)
|
|
{
|
|
value = timeDependentProperty.Clone(new FixedTimeProvider(endTime));
|
|
}
|
|
else if (member.IsTickProperty && tick != null)
|
|
{
|
|
if (tick.TickType != TickType.Quote && _quoteTickOnlyPropertes.Contains(member.Member.Name))
|
|
{
|
|
value = null;
|
|
}
|
|
else if (member.IsTickLastPrice)
|
|
{
|
|
var nullValueKey = tick.TickType != TickType.OpenInterest
|
|
? member.GetMemberName("OpenInterest")
|
|
: member.GetMemberName();
|
|
AddToSeries(nullValueKey, endTime, null, overrideValues);
|
|
}
|
|
}
|
|
}
|
|
|
|
AddToSeries(key, endTime, value, overrideValues);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds Lean data objects to the end of the lists
|
|
/// </summary>
|
|
/// <param name="tradeBar"><see cref="TradeBar"/> object that contains trade bar information of the security</param>
|
|
/// <param name="quoteBar"><see cref="QuoteBar"/> object that contains quote bar information of the security</param>
|
|
public void Add(TradeBar tradeBar, QuoteBar quoteBar)
|
|
{
|
|
// Quote bar first, so if there is a trade bar, OHLC will be overwritten
|
|
Add(quoteBar);
|
|
Add(tradeBar, overrideValues: true);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get the pandas.DataFrame of the current <see cref="PandasData"/> state
|
|
/// </summary>
|
|
/// <param name="levels">Number of levels of the multi index</param>
|
|
/// <param name="filterMissingValueColumns">If false, make sure columns with "missing" values only are still added to the dataframe</param>
|
|
/// <returns>pandas.DataFrame object</returns>
|
|
public PyObject ToPandasDataFrame(int levels = 2, bool filterMissingValueColumns = true)
|
|
{
|
|
using var _ = Py.GIL();
|
|
|
|
PyObject[] indexTemplate;
|
|
// Create the index labels
|
|
var names = _defaultNames;
|
|
|
|
if (levels == 1)
|
|
{
|
|
names = _level1Names;
|
|
indexTemplate = GetIndexTemplate(_symbol);
|
|
}
|
|
else if (levels == 2)
|
|
{
|
|
// symbol, time
|
|
names = _level2Names;
|
|
indexTemplate = GetIndexTemplate(_symbol, null);
|
|
}
|
|
else if (levels == 3)
|
|
{
|
|
// expiry, symbol, time
|
|
names = _level3Names;
|
|
indexTemplate = GetIndexTemplate(_symbol.ID.Date, _symbol, null);
|
|
}
|
|
else
|
|
{
|
|
if (_symbol.SecurityType == SecurityType.Future)
|
|
{
|
|
indexTemplate = GetIndexTemplate(_symbol.ID.Date, null, null, _symbol, null);
|
|
}
|
|
else if (_symbol.SecurityType.IsOption())
|
|
{
|
|
indexTemplate = GetIndexTemplate(_symbol.ID.Date, _symbol.ID.StrikePrice, _symbol.ID.OptionRight, _symbol, null);
|
|
}
|
|
else
|
|
{
|
|
indexTemplate = GetIndexTemplate(null, null, null, _symbol, null);
|
|
}
|
|
}
|
|
|
|
names = new PyList(names.SkipLast(names.Count() > 1 && _timeAsColumn ? 1 : 0).ToArray());
|
|
|
|
// creating the pandas MultiIndex is expensive so we keep a cash
|
|
var indexCache = new Dictionary<IReadOnlyCollection<DateTime>, PyObject>(new ListComparer<DateTime>());
|
|
// Returns a dictionary keyed by column name where values are pandas.Series objects
|
|
using var pyDict = new PyDict();
|
|
foreach (var (seriesName, serie) in _series)
|
|
{
|
|
if (filterMissingValueColumns && serie.ShouldFilter) continue;
|
|
|
|
var key = serie.Times ?? EmptySeriesTimesKey;
|
|
if (!indexCache.TryGetValue(key, out var index))
|
|
{
|
|
PyList indexSource;
|
|
if (_timeAsColumn)
|
|
{
|
|
indexSource = serie.Values.Select(_ => CreateIndexSourceValue(DateTime.MinValue, indexTemplate)).ToPyListUnSafe();
|
|
}
|
|
else
|
|
{
|
|
indexSource = serie.Times.Select(time => CreateIndexSourceValue(time, indexTemplate)).ToPyListUnSafe();
|
|
}
|
|
|
|
if (indexTemplate.Length == 1)
|
|
{
|
|
using var nameDic = Py.kw("name", names[0]);
|
|
index = _indexFactory.Invoke(new[] { indexSource }, nameDic);
|
|
}
|
|
else
|
|
{
|
|
using var namesDic = Py.kw("names", names);
|
|
index = _multiIndexFactory.Invoke(new[] { indexSource }, namesDic);
|
|
}
|
|
|
|
indexCache[key] = index;
|
|
|
|
foreach (var pyObject in indexSource)
|
|
{
|
|
pyObject.Dispose();
|
|
}
|
|
indexSource.Dispose();
|
|
}
|
|
|
|
// Adds pandas.Series value keyed by the column name
|
|
using var pyvalues = new PyList();
|
|
for (var i = 0; i < serie.Values.Count; i++)
|
|
{
|
|
using var pyObject = serie.Values[i].ToPython();
|
|
pyvalues.Append(pyObject);
|
|
}
|
|
using var series = _seriesFactory.Invoke(pyvalues, index);
|
|
using var pyStrKey = seriesName.ToPython();
|
|
using var pyKey = _pandasColumn.Invoke(pyStrKey);
|
|
pyDict.SetItem(pyKey, series);
|
|
}
|
|
_series.Clear();
|
|
foreach (var kvp in indexCache)
|
|
{
|
|
kvp.Value.Dispose();
|
|
}
|
|
|
|
for (var i = 0; i < indexTemplate.Length; i++)
|
|
{
|
|
DisposeIfNotEmpty(indexTemplate[i]);
|
|
}
|
|
names.Dispose();
|
|
|
|
// Create the DataFrame
|
|
var result = _dataFrameFactory.Invoke(pyDict);
|
|
|
|
foreach (var item in pyDict)
|
|
{
|
|
item.Dispose();
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper method to create a single pandas data frame indexed by symbol
|
|
/// </summary>
|
|
/// <remarks>Will add a single point per pandas data series (symbol)</remarks>
|
|
public static PyObject ToPandasDataFrame(IEnumerable<PandasData> pandasDatas, bool skipTimesColumn = false)
|
|
{
|
|
using var _ = Py.GIL();
|
|
|
|
using var list = pandasDatas.Select(x => x._symbol).ToPyListUnSafe();
|
|
|
|
using var namesDic = Py.kw("name", _level1Names[0]);
|
|
using var index = _indexFactory.Invoke(new[] { list }, namesDic);
|
|
|
|
var valuesPerSeries = new Dictionary<string, PyList>();
|
|
var seriesToSkip = new Dictionary<string, bool>();
|
|
foreach (var pandasData in pandasDatas)
|
|
{
|
|
foreach (var kvp in pandasData._series)
|
|
{
|
|
if (skipTimesColumn && kvp.Key == "time")
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (seriesToSkip.ContainsKey(kvp.Key))
|
|
{
|
|
seriesToSkip[kvp.Key] &= kvp.Value.ShouldFilter;
|
|
}
|
|
else
|
|
{
|
|
seriesToSkip[kvp.Key] = kvp.Value.ShouldFilter;
|
|
}
|
|
|
|
if (!valuesPerSeries.TryGetValue(kvp.Key, out PyList value))
|
|
{
|
|
// Adds pandas.Series value keyed by the column name
|
|
value = valuesPerSeries[kvp.Key] = new PyList();
|
|
}
|
|
|
|
if (kvp.Value.Values.Count > 0)
|
|
{
|
|
// taking only 1 value per symbol
|
|
using var valueOfSymbol = kvp.Value.Values[0].ToPython();
|
|
value.Append(valueOfSymbol);
|
|
}
|
|
else
|
|
{
|
|
value.Append(PyObject.None);
|
|
}
|
|
}
|
|
}
|
|
|
|
using var pyDict = new PyDict();
|
|
foreach (var kvp in valuesPerSeries)
|
|
{
|
|
if (seriesToSkip.TryGetValue(kvp.Key, out var skip) && skip)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
using var series = _seriesFactory.Invoke(kvp.Value, index);
|
|
using var pyStrKey = kvp.Key.ToPython();
|
|
using var pyKey = _pandasColumn.Invoke(pyStrKey);
|
|
pyDict.SetItem(pyKey, series);
|
|
|
|
kvp.Value.Dispose();
|
|
}
|
|
var result = _dataFrameFactory.Invoke(pyDict);
|
|
|
|
// Drop columns with only NaN or None values
|
|
using var dropnaKwargs = Py.kw("axis", 1, "inplace", true, "how", "all");
|
|
result.GetAttr("dropna").Invoke(Array.Empty<PyObject>(), dropnaKwargs);
|
|
|
|
return result;
|
|
}
|
|
|
|
private List<DataTypeMember> GetInstanceDataTypeMembers(object data)
|
|
{
|
|
var type = data.GetType();
|
|
if (!_members.TryGetValue(type, out var members))
|
|
{
|
|
HashSet<string> columnNames;
|
|
|
|
if (data is DynamicData dynamicData)
|
|
{
|
|
columnNames = (data as DynamicData)?.GetStorageDictionary()
|
|
// if this is a PythonData instance we add in '__typename' which we don't want into the data frame
|
|
.Where(x => !x.Key.StartsWith("__", StringComparison.InvariantCulture)).ToHashSet(x => x.Key);
|
|
columnNames.Add("value");
|
|
members = EmptyDataTypeMembers;
|
|
}
|
|
else
|
|
{
|
|
members = GetTypeMembers(type);
|
|
columnNames = members.SelectMany(x => x.GetMemberNames()).ToHashSet();
|
|
// We add openinterest key so the series is created: open interest tick LastPrice is renamed to OpenInterest
|
|
if (data is Tick)
|
|
{
|
|
columnNames.Add("openinterest");
|
|
}
|
|
}
|
|
|
|
_members[type] = members;
|
|
|
|
if (_timeAsColumn)
|
|
{
|
|
columnNames.Add("time");
|
|
}
|
|
|
|
foreach (var columnName in columnNames)
|
|
{
|
|
_series.TryAdd(columnName, new Serie(withTimeIndex: !_timeAsColumn));
|
|
}
|
|
}
|
|
|
|
return members;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets or create/adds the <see cref="DataTypeMember"/> instances corresponding to the members of the given type,
|
|
/// and returns the names of the members.
|
|
/// </summary>
|
|
private List<DataTypeMember> GetTypeMembers(Type type)
|
|
{
|
|
List<DataTypeMember> typeMembers;
|
|
lock (_membersCache)
|
|
{
|
|
if (!_membersCache.TryGetValue(type, out typeMembers))
|
|
{
|
|
// Contracts (e.g. OptionContract, FuturesContract) expose their own representative price members
|
|
// (LastPrice, BidPrice, ...) and mark the BaseData-like aliases (Value, Price, Close) with
|
|
// PandasIgnore, so we don't want to force the Value member in as we do for custom data types.
|
|
var forcedInclusionMembers = LeanData.IsCommonLeanDataType(type) || typeof(BaseContract).IsAssignableFrom(type)
|
|
? Array.Empty<string>()
|
|
: _nonLeanDataTypeForcedMemberNames;
|
|
typeMembers = GetDataTypeMembers(type, forcedInclusionMembers).ToList();
|
|
_membersCache[type] = typeMembers;
|
|
}
|
|
}
|
|
|
|
_members[type] = typeMembers;
|
|
return typeMembers;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the <see cref="DataTypeMember"/> instances corresponding to the members of the given type.
|
|
/// It will try to unwrap properties which types are classes unless they are marked either to be ignored or to be added as a whole
|
|
/// </summary>
|
|
private static IEnumerable<DataTypeMember> GetDataTypeMembers(Type type, string[] forcedInclusionMembers)
|
|
{
|
|
var members = type
|
|
.GetMembers(BindingFlags.Instance | BindingFlags.Public)
|
|
.Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property)
|
|
.Where(x => forcedInclusionMembers.Contains(x.Name)
|
|
|| (!x.IsDefined(PandasIgnoreAttribute) && !x.DeclaringType.IsDefined(PandasIgnoreMembersAttribute)));
|
|
|
|
return members
|
|
.Select(member =>
|
|
{
|
|
var dataTypeMember = CreateDataTypeMember(member);
|
|
var memberType = dataTypeMember.GetMemberType();
|
|
|
|
// Should we unpack its properties into columns?
|
|
if (memberType.IsClass
|
|
&& (memberType.Namespace == null
|
|
// We only expand members of types in the QuantConnect namespace,
|
|
// else we might be expanding types like System.String, NodaTime.DateTimeZone or any other external types
|
|
|| (memberType.Namespace.StartsWith("QuantConnect.", StringComparison.InvariantCulture)
|
|
&& !memberType.IsDefined(PandasNonExpandableAttribute)
|
|
&& !member.IsDefined(PandasNonExpandableAttribute))))
|
|
{
|
|
dataTypeMember = CreateDataTypeMember(member, GetDataTypeMembers(memberType, forcedInclusionMembers).ToArray());
|
|
}
|
|
|
|
return (memberType, dataTypeMember);
|
|
})
|
|
// Check if there are multiple properties/fields of the same type,
|
|
// in which case we add the property/field name as prefix for the inner members to avoid name conflicts
|
|
.GroupBy(x => x.memberType, x => x.dataTypeMember)
|
|
.SelectMany(grouping =>
|
|
{
|
|
var typeProperties = grouping.ToList();
|
|
if (typeProperties.Count > 1)
|
|
{
|
|
var propertiesToExpand = typeProperties.Where(x => x.ShouldBeUnwrapped).ToList();
|
|
if (propertiesToExpand.Count > 1)
|
|
{
|
|
foreach (var property in propertiesToExpand)
|
|
{
|
|
property.SetPrefix();
|
|
}
|
|
}
|
|
}
|
|
|
|
return typeProperties;
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds the member value to the corresponding series, making sure unwrapped values a properly added
|
|
/// by checking the children members and adding their values to their own series
|
|
/// </summary>
|
|
private void AddMembersData(object instance, IEnumerable<DataTypeMember> members, DateTime endTime, bool overrideValues)
|
|
{
|
|
foreach (var member in members)
|
|
{
|
|
if (!member.ShouldBeUnwrapped)
|
|
{
|
|
AddMemberToSeries(instance, endTime, member, overrideValues);
|
|
}
|
|
else
|
|
{
|
|
var memberValue = member.GetValue(instance);
|
|
if (memberValue != null)
|
|
{
|
|
AddMembersData(memberValue, member.Children, endTime, overrideValues);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Only dipose of the PyObject if it was set to something different than empty
|
|
/// </summary>
|
|
private static void DisposeIfNotEmpty(PyObject pyObject)
|
|
{
|
|
if (!ReferenceEquals(pyObject, _empty))
|
|
{
|
|
pyObject.Dispose();
|
|
}
|
|
}
|
|
|
|
private static bool MemberIsDataDictionary(Type memberType)
|
|
{
|
|
while (memberType != null && !memberType.IsValueType)
|
|
{
|
|
if (memberType.IsGenericType && memberType.GetGenericTypeDefinition() == typeof(DataDictionary<>))
|
|
{
|
|
return true;
|
|
}
|
|
memberType = memberType.BaseType;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private PyObject[] GetIndexTemplate(params object[] args)
|
|
{
|
|
return args.SkipLast(args.Length > 1 && _timeAsColumn ? 1 : 0).Select(x => x?.ToPython() ?? _empty).ToArray();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Create a new tuple index
|
|
/// </summary>
|
|
private PyObject CreateIndexSourceValue(DateTime index, PyObject[] list)
|
|
{
|
|
if (!_timeAsColumn && list.Length > 1)
|
|
{
|
|
DisposeIfNotEmpty(list[^1]);
|
|
list[^1] = index.ToPython();
|
|
}
|
|
|
|
if (list.Length > 1)
|
|
{
|
|
return new PyTuple(list.ToArray());
|
|
}
|
|
|
|
return list[0].ToPython();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds data to dictionary
|
|
/// </summary>
|
|
/// <param name="key">The key of the value to get</param>
|
|
/// <param name="time"><see cref="DateTime"/> object to add to the value associated with the specific key</param>
|
|
/// <param name="input"><see cref="Object"/> to add to the value associated with the specific key. Can be null.</param>
|
|
private void AddToSeries(string key, DateTime time, object input, bool overrideValues)
|
|
{
|
|
if (!_series.TryGetValue(key, out var serie))
|
|
{
|
|
throw new ArgumentException($"PandasData.AddToSeries(): {Messages.PandasData.KeyNotFoundInSeries(key)}");
|
|
}
|
|
|
|
serie.Add(time, input, overrideValues);
|
|
}
|
|
|
|
private class Serie
|
|
{
|
|
private static readonly IFormatProvider InvariantCulture = CultureInfo.InvariantCulture;
|
|
|
|
public bool ShouldFilter { get; private set; }
|
|
public List<DateTime> Times { get; }
|
|
public List<object> Values { get; }
|
|
|
|
public Serie(bool withTimeIndex = true)
|
|
{
|
|
ShouldFilter = true;
|
|
Values = new();
|
|
if (withTimeIndex)
|
|
{
|
|
Times = new();
|
|
}
|
|
}
|
|
|
|
public void Add(DateTime time, object input, bool overrideValues)
|
|
{
|
|
var value = input is decimal ? Convert.ToDouble(input, InvariantCulture) : input;
|
|
if (ShouldFilter)
|
|
{
|
|
// we need at least 1 valid entry for the series not to get filtered
|
|
if (value is double doubleValue)
|
|
{
|
|
if (!doubleValue.IsNaNOrZero())
|
|
{
|
|
ShouldFilter = false;
|
|
}
|
|
}
|
|
else if (value is string stringValue)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(stringValue))
|
|
{
|
|
ShouldFilter = false;
|
|
}
|
|
}
|
|
else if (value is bool boolValue)
|
|
{
|
|
if (boolValue)
|
|
{
|
|
ShouldFilter = false;
|
|
}
|
|
}
|
|
else if (value != null)
|
|
{
|
|
if (value is ICollection enumerable)
|
|
{
|
|
if (enumerable.Count != 0)
|
|
{
|
|
ShouldFilter = false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ShouldFilter = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (overrideValues && Times != null && Times.Count > 0 && Times[^1] == time)
|
|
{
|
|
// If the time is the same as the last one, we overwrite the value
|
|
Values[^1] = value;
|
|
}
|
|
else
|
|
{
|
|
Values.Add(value);
|
|
Times?.Add(time);
|
|
}
|
|
}
|
|
}
|
|
|
|
private class FixedTimeProvider : ITimeProvider
|
|
{
|
|
private readonly DateTime _time;
|
|
public DateTime GetUtcNow() => _time;
|
|
public FixedTimeProvider(DateTime time)
|
|
{
|
|
_time = time;
|
|
}
|
|
}
|
|
}
|
|
}
|