Add support for BigInteger in CPython engine (#10830)

mmisol · web-flow · commit af43ba9fe5b4 · 2020-07-02T12:26:41.000-04:00
Support is added for encoding decoding BigInteger values from Python
int that have a size that exceeds what can fit in a long.

Also a BigInteger encoder/decoder is added to the custom encoders of
Python.NET in order to support encoding and decoding in the context of
function calls.
diff --git a/src/Libraries/DSCPython/CPythonEvaluator.cs b/src/Libraries/DSCPython/CPythonEvaluator.cs
@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Reflection;
 using Autodesk.DesignScript.Runtime;
+using DSCPython.Encoders;
 using Dynamo.Utilities;
 using Python.Runtime;
 
@@ -24,6 +25,10 @@ internal delegate void EvaluationEventHandler(EvaluationState state,
     [IsVisibleInDynamoLibrary(false)]
     public static class CPythonEvaluator
     {
+        static CPythonEvaluator()
+        {
+            InitializeEncoders();
+        }
 
         /// <summary>
         ///     Executes a Python script with custom variable names. Script may be a string
@@ -101,6 +106,17 @@ public static object EvaluatePythonScript(
             }
         }
 
+        /// <summary>
+        /// Registers custom encoders and decoders with the Python.NET runtime
+        /// </summary>
+        private static void InitializeEncoders()
+        {
+            var encoders = new IPyObjectEncoder[] { new BigIntegerEncoder() };
+            var decoders = encoders.Cast<IPyObjectDecoder>().ToArray();
+            Array.ForEach(encoders, e => PyObjectConversions.RegisterEncoder(e));
+            Array.ForEach(decoders, d => PyObjectConversions.RegisterDecoder(d));
+        }
+
         /// <summary>
         /// Gets the trace back message from the exception, if it is a PythonException.
         /// </summary>
@@ -177,35 +193,47 @@ public static DataMarshaler OutputMarshaler
                         {
                             if (PyList.IsListType(pyObj))
                             {
-                                var pyList = new PyList(pyObj);
-                                var list = new List<object>();
-                                foreach (PyObject item in pyList)
+                                using (var pyList = new PyList(pyObj))
                                 {
-                                    list.Add(outputMarshaler.Marshal(item));
+                                    var list = new List<object>();
+                                    foreach (PyObject item in pyList)
+                                    {
+                                        list.Add(outputMarshaler.Marshal(item));
+                                    }
+                                    return list;
                                 }
-                                return list;
                             }
-                            else if (PyDict.IsDictType(pyObj))
+                            if (PyDict.IsDictType(pyObj))
                             {
-                                var pyDict = new PyDict(pyObj);
-                                var dict = new Dictionary<object, object>();
-                                foreach (PyObject item in pyDict.Items())
+                                using (var pyDict = new PyDict(pyObj))
                                 {
-                                    dict.Add(
-                                        outputMarshaler.Marshal(item.GetItem(0)),
-                                        outputMarshaler.Marshal(item.GetItem(1))
-                                    );
+                                    var dict = new Dictionary<object, object>();
+                                    foreach (PyObject item in pyDict.Items())
+                                    {
+                                        dict.Add(
+                                            outputMarshaler.Marshal(item.GetItem(0)),
+                                            outputMarshaler.Marshal(item.GetItem(1))
+                                        );
+                                    }
+                                    return dict;
                                 }
-                                return dict;
                             }
-                            else if (PyLong.IsLongType(pyObj))
+                            if (PyLong.IsLongType(pyObj))
                             {
-                                return PyLong.AsLong(pyObj).ToInt64();
-                            }
-                            else
-                            {
-                                return outputMarshaler.Marshal(pyObj.AsManagedObject(typeof(object)));
+                                using (var pyLong = PyLong.AsLong(pyObj))
+                                {
+                                    try
+                                    {
+                                        return pyLong.ToInt64();
+                                    }
+                                    catch (PythonException exc) when (exc.Message.StartsWith("OverflowError"))
+                                    {
+                                        return pyLong.ToBigInteger();
+                                    }
+                                }
                             }
+
+                            return outputMarshaler.Marshal(pyObj.AsManagedObject(typeof(object)));
                         });
                 }
                 return outputMarshaler;
diff --git a/src/Libraries/DSCPython/DSCPython.csproj b/src/Libraries/DSCPython/DSCPython.csproj
@@ -50,6 +50,7 @@
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Numerics" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -60,6 +61,7 @@
     <Compile Include="..\..\AssemblySharedInfoGenerator\AssemblySharedInfo.cs">
       <Link>Properties\AssemblySharedInfo.cs</Link>
     </Compile>
+    <Compile Include="Encoders\BigIntegerEncoder.cs" />
     <Compile Include="CPythonEvaluator.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\Resources.Designer.cs">
diff --git a/src/Libraries/DSCPython/Encoders/BigIntegerEncoder.cs b/src/Libraries/DSCPython/Encoders/BigIntegerEncoder.cs
@@ -0,0 +1,39 @@
+﻿using System;
+using System.Numerics;
+using Python.Runtime;
+
+namespace DSCPython.Encoders
+{
+    internal class BigIntegerEncoder : IPyObjectEncoder, IPyObjectDecoder
+    {
+        public bool CanDecode(PyObject objectType, Type targetType)
+        {
+            return targetType == typeof(BigInteger);
+        }
+
+        public bool CanEncode(Type type)
+        {
+            return type == typeof(BigInteger);
+        }
+
+        public bool TryDecode<T>(PyObject pyObj, out T value)
+        {
+            if (!PyLong.IsLongType(pyObj))
+            {
+                value = default;
+                return false;
+            }
+
+            using (var pyLong = PyLong.AsLong(pyObj))
+            {
+                value = (T)(object)pyLong.ToBigInteger();
+                return true;
+            }
+        }
+
+        public PyObject TryEncode(object value)
+        {
+            return new PyLong(value.ToString());
+        }
+    }
+}
diff --git a/test/Engine/FFITarget/DummyMath.cs b/test/Engine/FFITarget/DummyMath.cs
@@ -1,5 +1,6 @@
 ﻿using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 
 namespace FFITarget
 {
@@ -10,6 +11,11 @@ public static double Sum(IEnumerable<double> values)
             return values.Sum();
         }
 
+        public static BigInteger Sum(BigInteger bigNumber1, BigInteger bigNumber2)
+        {
+            return bigNumber1 + bigNumber2;
+        }
+
         public int a { get; set; }
 
         public static DummyMath ValueCtor(int _a)
diff --git a/test/Engine/FFITarget/FFITarget.csproj b/test/Engine/FFITarget/FFITarget.csproj
@@ -37,6 +37,7 @@
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Numerics" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
diff --git a/test/Libraries/DynamoPythonTests/IronPythonTests.csproj b/test/Libraries/DynamoPythonTests/IronPythonTests.csproj
@@ -78,12 +78,14 @@
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Numerics" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\..\..\src\AssemblySharedInfoGenerator\AssemblySharedInfo.cs">
       <Link>Properties\AssemblySharedInfo.cs</Link>
     </Compile>
     <Compile Include="CodeCompletionTests.cs" />
+    <Compile Include="PythonEvalTestsWithLibraries.cs" />
     <Compile Include="PythonEvalTests.cs" />
     <Compile Include="PythonEditTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
diff --git a/test/Libraries/DynamoPythonTests/PythonEditTests.cs b/test/Libraries/DynamoPythonTests/PythonEditTests.cs
@@ -432,9 +432,6 @@ public void ReturnIronPythonDictionary_AsDynamoDictionary()
         }
 
         [Test]
-        [Category("Failure")]
-        // This test is failing for CPython3 Engine. 
-        // The BigIntegers which use more than int64 type, are currently not being marshalled. 
         public void BigInteger_CanBeMarshaledAsInt64()
         {
             // open test graph
diff --git a/test/Libraries/DynamoPythonTests/PythonEvalTestsWithLibraries.cs b/test/Libraries/DynamoPythonTests/PythonEvalTestsWithLibraries.cs
@@ -0,0 +1,47 @@
+﻿using System.Collections;
+using System.Collections.Generic;
+using System.Numerics;
+using Dynamo;
+using NUnit.Framework;
+using static DSIronPythonTests.IronPythonTests;
+
+namespace DynamoPythonTests
+{
+    public class PythonEvalTestsWithLibraries : DynamoModelTestBase
+    {
+        protected override void GetLibrariesToPreload(List<string> libraries)
+        {
+            libraries.Add("FFITarget.dll");
+        }
+
+        public IEnumerable<PythonEvaluatorDelegate> Evaluators = new List<PythonEvaluatorDelegate> {
+            DSCPython.CPythonEvaluator.EvaluatePythonScript,
+            DSIronPython.IronPythonEvaluator.EvaluateIronPythonScript
+        };
+
+        [Test]
+        public void TestBigIntegerEncoding()
+        {
+            string code = @"
+import sys
+import clr
+clr.AddReference('FFITarget')
+from FFITarget import DummyMath
+
+# Provide Python int as arguments: Python => .NET
+sum = DummyMath.Sum(11111111111111111111, 11111111111111111111)
+# sum contains a BigInteger and we use it in Python + operation: .NET => Python
+sum = sum + 1
+
+OUT = sum
+";
+            var empty = new ArrayList();
+            var expected = BigInteger.Parse("22222222222222222223");
+            foreach (var pythonEvaluator in Evaluators)
+            {
+                var result = pythonEvaluator(code, empty, empty);
+                Assert.AreEqual(expected, result);
+            }
+        }
+    }
+}
diff --git a/test/core/python/BigIntegerToLong.dyn b/test/core/python/BigIntegerToLong.dyn
@@ -88,7 +88,7 @@
       "HasRunWithoutCrash": true,
       "IsVisibleInDynamoLibrary": true,
       "Version": "2.7.0.8544",
-      "RunType": "Automatic",
+      "RunType": "Manual",
       "RunPeriod": "1000"
     },
     "Camera": {

Original file line number	Diff line number	Diff line change
`@@ -432,9 +432,6 @@ public void ReturnIronPythonDictionary_AsDynamoDictionary()`
`432`	`432`	`}`
`433`	`433`
`434`	`434`	`[Test]`
`435`		`- [Category("Failure")]`
`436`		`- // This test is failing for CPython3 Engine.`
`437`		`- // The BigIntegers which use more than int64 type, are currently not being marshalled.`
`438`	`435`	`public void BigInteger_CanBeMarshaledAsInt64()`
`439`	`436`	`{`
`440`	`437`	`// open test graph`