Decode Python list into IList in function calls (#10865)

mmisol · web-flow · commit 001662b2e8b4 · 2020-07-08T15:10:48.000-04:00
Implements a decoder for Python sequence to transform them into generic
or non-generic IList. This relies on Python.NET 'PySequence.ToList'.

Also implements an encoder to perform a 'no-op'. This allows to treat
returned lists from function calls like .NET lists, which is the
behavior of Iron Python.
diff --git a/src/Libraries/DSCPython/CPythonEvaluator.cs b/src/Libraries/DSCPython/CPythonEvaluator.cs
@@ -111,7 +111,7 @@ public static object EvaluatePythonScript(
         /// </summary>
         private static void InitializeEncoders()
         {
-            var encoders = new IPyObjectEncoder[] { new BigIntegerEncoder() };
+            var encoders = new IPyObjectEncoder[] { new BigIntegerEncoder(), new ListEncoder() };
             var decoders = encoders.Cast<IPyObjectDecoder>().ToArray();
             Array.ForEach(encoders, e => PyObjectConversions.RegisterEncoder(e));
             Array.ForEach(decoders, d => PyObjectConversions.RegisterDecoder(d));
diff --git a/src/Libraries/DSCPython/DSCPython.csproj b/src/Libraries/DSCPython/DSCPython.csproj
@@ -63,6 +63,7 @@
     </Compile>
     <Compile Include="Encoders\BigIntegerEncoder.cs" />
     <Compile Include="CPythonEvaluator.cs" />
+    <Compile Include="Encoders\ListEncoder.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\Resources.Designer.cs">
       <AutoGen>True</AutoGen>
diff --git a/src/Libraries/DSCPython/Encoders/ListEncoder.cs b/src/Libraries/DSCPython/Encoders/ListEncoder.cs
@@ -0,0 +1,60 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using Dynamo.Utilities;
+using Python.Runtime;
+
+namespace DSCPython.Encoders
+{
+    internal class ListEncoder : IPyObjectEncoder, IPyObjectDecoder
+    {
+        private static readonly Type[] decodableTypes = new Type[]
+        {
+            typeof(IList), typeof(ArrayList),
+            typeof(IList<>), typeof(List<>)
+        };
+
+        public bool CanDecode(PyObject objectType, Type targetType)
+        {
+            if (targetType.IsGenericType)
+            {
+                targetType = targetType.GetGenericTypeDefinition();
+            }
+            return decodableTypes.IndexOf(targetType) >= 0;
+        }
+
+        public bool CanEncode(Type type)
+        {
+            return typeof(IList).IsAssignableFrom(type);
+        }
+
+        public bool TryDecode<T>(PyObject pyObj, out T value)
+        {
+            if (!PySequence.IsSequenceType(pyObj))
+            {
+                value = default;
+                return false;
+            }
+
+            using (var pyList = PyList.AsList(pyObj))
+            {
+                if (typeof(T).IsGenericType)
+                {
+                    value = pyList.ToList<T>();
+                }
+                else
+                {
+                    value = (T)pyList.ToList();
+                }
+                return true;
+            }
+        }
+
+        public PyObject TryEncode(object value)
+        {
+            // This is a no-op to prevent Python.NET from encoding generic lists
+            // https://github.com/pythonnet/pythonnet/pull/963#issuecomment-642938541
+            return PyObject.FromManagedObject(value);
+        }
+    }
+}
diff --git a/test/Libraries/DynamoPythonTests/PythonEvalTestsWithLibraries.cs b/test/Libraries/DynamoPythonTests/PythonEvalTestsWithLibraries.cs
@@ -21,7 +21,7 @@ protected override void GetLibrariesToPreload(List<string> libraries)
         };
 
         [Test]
-        public void TestBigIntegerEncoding()
+        public void TestBigIntegerEncodingDecoding()
         {
             string code = @"
 import sys
@@ -46,8 +46,7 @@ from FFITarget import DummyMath
         }
 
         [Test]
-        [Category("Failure")]
-        public void TestListEncoding()
+        public void TestListDecoding()
         {
             string code = @"
 import sys
@@ -56,16 +55,32 @@ import clr
 from DSCore import List
 
 l = ['a']
-# Python list => .NET IList - Does not work in CPython
-l = List.AddItemToEnd('b',l)
-# .NET IList => Python list - Does not work in IronPython. Couldn't test CPython because of previous bug
-# l.append('c')
-l.Add(c)
+# Python list => .NET IList
+untypedList = List.AddItemToEnd('b', l)
+untypedList.Add('c')
+
+l2 = ['a','b']
+# Python list => .NET IList<>
+typedList = List.SetDifference(l2, l)
+typedList.Add('b')
+
+l3 = [[1,2],[3,4]]
+# Python list (nested) => .NET IList<IList<>>
+flatennedList = List.Flatten(l3)
+
+l4 = []
+# Python list (empty) => .NET IList
+elementCount = List.Count(l4)
 
-OUT = l
+sum = 0
+# Python-wrapped .NET List can be iterated over
+for i in flatennedList:
+  sum = sum + i
+
+OUT = untypedList, typedList, flatennedList, elementCount, sum
 ";
             var empty = new ArrayList();
-            var expected = new ArrayList { "a", "b", "c" };
+            var expected = new ArrayList { new ArrayList { "a", "b", "c" }, new ArrayList { "b", "b" }, new ArrayList { 1, 2, 3, 4 }, 0, 10 };
             foreach (var pythonEvaluator in Evaluators)
             {
                 var result = pythonEvaluator(code, empty, empty);
@@ -86,11 +101,7 @@ from FFITarget import DummyCollection
 from DSCore import List
 from array import array
 
-# Python array => .NET IList - array is in a builtin library. This does not work in either engine
-# native = array('l', [1,2])
-# native = List.AddItemToEnd(3, native) 
-
-# .NET array => Python list - Works in both engines
+# .NET array => Python list
 a = DummyCollection.MakeArray(1,2)
 a[0] = a[1] + 1
 b = len(a)
@@ -110,8 +121,7 @@ from array import array
         }
 
         [Test]
-        [Category("Failure")]
-        public void TestTupleEncoding()
+        public void TestTupleDecoding()
         {
             string code = @"
 import sys
@@ -122,9 +132,9 @@ from FFITarget import DummyCollection
 from DSCore import List
 
 t = (1,2,3)
-# Python tuple => .NET array - Works in both
+# Python tuple => .NET array
 a = DummyCollection.MakeArray(t)
-# Python tuple => .NET IList - Does not work in CPython
+# Python tuple => .NET IList
 l = List.AddItemToEnd(4, t)
 
 OUT = a, l
@@ -140,8 +150,7 @@ from DSCore import List
         }
 
         [Test]
-        [Category("Failure")]
-        public void TestRangeEncoding()
+        public void TestRangeDecodingCPython()
         {
             string code = @"
 import sys
@@ -154,19 +163,16 @@ from DSCore import List
 r = range(0, 10, 2)
 # Python range => .NET array - Works in both
 a = DummyCollection.MakeArray(r)
-# Python range => .NET IList - Does not work in CPython
-l = List.AddItemToEnd(10, r)
+# Python range => .NET IList - Does not work in IronPython
+l = List.AddItemToEnd(10, range(0, 10, 2))
 
 OUT = a, l
 ";
             var empty = new ArrayList();
             var expected = new ArrayList { new ArrayList { 0, 2, 4, 6, 8 }, new ArrayList { 0, 2, 4, 6, 8, 10 } };
-            foreach (var pythonEvaluator in Evaluators)
-            {
-                var result = pythonEvaluator(code, empty, empty);
-                Assert.IsTrue(result is IEnumerable);
-                CollectionAssert.AreEqual(expected, result as IEnumerable);
-            }
+            var result = DSCPython.CPythonEvaluator.EvaluatePythonScript(code, empty, empty);
+            Assert.IsTrue(result is IEnumerable);
+            CollectionAssert.AreEqual(expected, result as IEnumerable);
         }
 
         [Test]

Original file line number	Diff line number	Diff line change
`@@ -111,7 +111,7 @@ public static object EvaluatePythonScript(`
`111`	`111`	`/// </summary>`
`112`	`112`	`private static void InitializeEncoders()`
`113`	`113`	`{`
`114`		`- var encoders = new IPyObjectEncoder[] { new BigIntegerEncoder() };`
	`114`	`+ var encoders = new IPyObjectEncoder[] { new BigIntegerEncoder(), new ListEncoder() };`
`115`	`115`	`var decoders = encoders.Cast<IPyObjectDecoder>().ToArray();`
`116`	`116`	`Array.ForEach(encoders, e => PyObjectConversions.RegisterEncoder(e));`
`117`	`117`	`Array.ForEach(decoders, d => PyObjectConversions.RegisterDecoder(d));`