/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
using System;
using System.IO;
using System.Linq;
using Python.Runtime;
using QuantConnect.Util;
using QuantConnect.Logging;
using System.Collections.Generic;
using System.Runtime.InteropServices;
namespace QuantConnect.Python
{
///
/// Helper class for Python initialization
///
public static class PythonInitializer
{
// Used to allow multiple Python unit and regression tests to be run in the same test run
private static bool _isInitialized;
// Used to hold pending path additions before Initialize is called
private static List _pendingPathAdditions = new List();
///
/// Initialize the Python.NET library
///
public static void Initialize()
{
if (!_isInitialized)
{
Log.Trace("PythonInitializer.Initialize(): start...");
PythonEngine.Initialize();
// required for multi-threading usage
PythonEngine.BeginAllowThreads();
_isInitialized = true;
Log.Trace("PythonInitializer.Initialize(): ended");
AddPythonPaths(new []{ Environment.CurrentDirectory });
}
}
///
/// Adds directories to the python path at runtime
///
public static bool AddPythonPaths(IEnumerable paths)
{
// Filter out any paths that are already on our Python path
if (paths.IsNullOrEmpty())
{
return false;
}
// Add these paths to our pending additions
_pendingPathAdditions.AddRange(paths);
if (_isInitialized)
{
using (Py.GIL())
{
dynamic sys = Py.Import("sys");
using var locals = new PyDict();
locals.SetItem("sys", sys);
// Filter out any already paths that already exist on our current PythonPath
using var pythonCurrentPath = PythonEngine.Eval("sys.path", locals: locals);
var currentPath = pythonCurrentPath.As>();
_pendingPathAdditions = _pendingPathAdditions.Where(x => !currentPath.Contains(x.Replace('\\', '/'))).ToList();
// Insert any pending path additions
if (!_pendingPathAdditions.IsNullOrEmpty())
{
var code = string.Join(";", _pendingPathAdditions
.Select(s => $"sys.path.insert(0, '{s}')")).Replace('\\', '/');
PythonEngine.Exec(code, locals: locals);
_pendingPathAdditions.Clear();
}
}
}
return true;
}
///
/// "Activate" a virtual Python environment by prepending its library storage to Pythons
/// path. This allows the libraries in this venv to be selected prior to our base install.
/// Requires PYTHONNET_PYDLL to be set to base install.
///
/// If a module is already loaded, Python will use its cached version first
/// these modules must be reloaded by reload() from importlib library
public static bool ActivatePythonVirtualEnvironment(string pathToVirtualEnv)
{
if (string.IsNullOrEmpty(pathToVirtualEnv))
{
return false;
}
if(!Directory.Exists(pathToVirtualEnv))
{
Log.Error($"PythonIntializer.ActivatePythonVirtualEnvironment(): Path {pathToVirtualEnv} to virtual environment does not exist");
return false;
}
pathToVirtualEnv = pathToVirtualEnv.TrimEnd('/', '\\');
var pathsToPrepend = new List();
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
// For linux we need to know the python version to determine the lib folder containing our packages
// Compare our PyDLL to the directory names under the lib directory and get a match
var pyDll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL");
var version = Path.GetFileNameWithoutExtension(pyDll);
var libDir = Directory.GetDirectories($"{pathToVirtualEnv}/lib")
.Select(d => new DirectoryInfo(d).Name)
.First(x => version.Contains(x, StringComparison.InvariantCulture));
pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}");
pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}/site-packages");
}
else
{
pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib");
pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib\\site-packages");
}
Log.Trace($"PythonIntializer.ActivatePythonVirtualEnvironment(): Adding the following locations to Python Path: {string.Join(",", pathsToPrepend)}");
return AddPythonPaths(pathsToPrepend);
}
}
}