Marshall out iterables as lists in CPython (#10894)

mmisol · web-flow · commit 0da94b4630da · 2020-07-15T18:04:16.000-04:00
Adds support for marshalling out iterables that are not lists or
dictionaries using the CPython engine. In order to achieve this, the
check PyIter.IsIterable was used, along with some extra considerations,
mentioned in the source with comments.
diff --git a/src/Libraries/DSCPython/CPythonEvaluator.cs b/src/Libraries/DSCPython/CPythonEvaluator.cs
@@ -195,18 +195,15 @@ public static DataMarshaler OutputMarshaler
                     outputMarshaler.RegisterMarshaler(
                         delegate (PyObject pyObj)
                         {
-                            if (PyList.IsListType(pyObj))
+                            // First, check if we are dealing with a wrapped .NET object.
+                            // This simplifies the cases that come afterwards, as wrapped
+                            // .NET collections pass some Python checks but not others. 
+                            var clrObj = pyObj.GetManagedObject();
+                            if (clrObj != null)
                             {
-                                using (var pyList = new PyList(pyObj))
-                                {
-                                    var list = new List<object>();
-                                    foreach (PyObject item in pyList)
-                                    {
-                                        list.Add(outputMarshaler.Marshal(item));
-                                    }
-                                    return list;
-                                }
+                                return outputMarshaler.Marshal(clrObj);
                             }
+                            // Dictionaries are iterable, so they should come first
                             if (PyDict.IsDictType(pyObj))
                             {
                                 using (var pyDict = new PyDict(pyObj))
@@ -222,6 +219,20 @@ public static DataMarshaler OutputMarshaler
                                     return dict;
                                 }
                             }
+                            // Other iterables should become lists, except for strings
+                            if (PyIter.IsIterable(pyObj) && !PyString.IsStringType(pyObj))
+                            {
+                                using (var pyList = PyList.AsList(pyObj))
+                                {
+                                    var list = new List<object>();
+                                    foreach (PyObject item in pyList)
+                                    {
+                                        list.Add(outputMarshaler.Marshal(item));
+                                    }
+                                    return list;
+                                }
+                            }
+                            // Special case for big long values: decode them as BigInteger
                             if (PyLong.IsLongType(pyObj))
                             {
                                 using (var pyLong = PyLong.AsLong(pyObj))
@@ -236,7 +247,7 @@ public static DataMarshaler OutputMarshaler
                                     }
                                 }
                             }
-
+                            // Default handling for other Python objects
                             var unmarshalled = pyObj.AsManagedObject(typeof(object));
                             if (unmarshalled is PyObject)
                             {
diff --git a/test/Libraries/DynamoPythonTests/PythonEvalTests.cs b/test/Libraries/DynamoPythonTests/PythonEvalTests.cs
@@ -195,5 +195,34 @@ class myobj:
                 Assert.AreEqual("Output could not be converted to a .NET value", exc.Message);
             }
         }
+
+        [Test]
+        public void NonListIterablesCanBeOutput()
+        {
+            var code = @"
+s = { 'hello' }
+fs = frozenset({ 'world' })
+d = { 'one': 1 }
+dk = d.keys()
+dv = d.values()
+di = d.items()
+
+OUT = s,fs,dk,dv,di
+";
+            var expected = new ArrayList
+            {
+                new ArrayList { "hello" },
+                new ArrayList { "world" },
+                new ArrayList { "one" },
+                new ArrayList { 1 },
+                new ArrayList { new ArrayList { "one", 1 } }
+            };
+            var empty = new ArrayList();
+            foreach (var pythonEvaluator in Evaluators)
+            {
+                var output = pythonEvaluator(code, empty, empty);
+                Assert.AreEqual(expected, output);
+            }
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -195,18 +195,15 @@ public static DataMarshaler OutputMarshaler`
`195`	`195`	`outputMarshaler.RegisterMarshaler(`
`196`	`196`	`delegate (PyObject pyObj)`
`197`	`197`	`{`
`198`		`- if (PyList.IsListType(pyObj))`
	`198`	`+ // First, check if we are dealing with a wrapped .NET object.`
	`199`	`+ // This simplifies the cases that come afterwards, as wrapped`
	`200`	`+ // .NET collections pass some Python checks but not others.`
	`201`	`+ var clrObj = pyObj.GetManagedObject();`
	`202`	`+ if (clrObj != null)`
`199`	`203`	`{`
`200`		`- using (var pyList = new PyList(pyObj))`
`201`		`- {`
`202`		`- var list = new List<object>();`
`203`		`- foreach (PyObject item in pyList)`
`204`		`- {`
`205`		`- list.Add(outputMarshaler.Marshal(item));`
`206`		`- }`
`207`		`- return list;`
`208`		`- }`
	`204`	`+ return outputMarshaler.Marshal(clrObj);`
`209`	`205`	`}`
	`206`	`+ // Dictionaries are iterable, so they should come first`
`210`	`207`	`if (PyDict.IsDictType(pyObj))`
`211`	`208`	`{`
`212`	`209`	`using (var pyDict = new PyDict(pyObj))`
`@@ -222,6 +219,20 @@ public static DataMarshaler OutputMarshaler`
`222`	`219`	`return dict;`
`223`	`220`	`}`
`224`	`221`	`}`
	`222`	`+ // Other iterables should become lists, except for strings`
	`223`	`+ if (PyIter.IsIterable(pyObj) && !PyString.IsStringType(pyObj))`
	`224`	`+ {`
	`225`	`+ using (var pyList = PyList.AsList(pyObj))`
	`226`	`+ {`
	`227`	`+ var list = new List<object>();`
	`228`	`+ foreach (PyObject item in pyList)`
	`229`	`+ {`
	`230`	`+ list.Add(outputMarshaler.Marshal(item));`
	`231`	`+ }`
	`232`	`+ return list;`
	`233`	`+ }`
	`234`	`+ }`
	`235`	`+ // Special case for big long values: decode them as BigInteger`
`225`	`236`	`if (PyLong.IsLongType(pyObj))`
`226`	`237`	`{`
`227`	`238`	`using (var pyLong = PyLong.AsLong(pyObj))`
`@@ -236,7 +247,7 @@ public static DataMarshaler OutputMarshaler`
`236`	`247`	`}`
`237`	`248`	`}`
`238`	`249`	`}`
`239`		`-`
	`250`	`+ // Default handling for other Python objects`
`240`	`251`	`var unmarshalled = pyObj.AsManagedObject(typeof(object));`
`241`	`252`	`if (unmarshalled is PyObject)`
`242`	`253`	`{`