bpo-29861: release references to multiprocessing Pool tasks (#743) (#801)

* bpo-29861: release references to multiprocessing Pool tasks (#743)

* bpo-29861: release references to multiprocessing Pool tasks

Release references to tasks, their arguments and their results as soon
as they are finished, instead of keeping them alive until another task
arrives.

* Comments in test

(cherry picked from commit 8988945cdc)

* Fix Misc/NEWS??

Lib/test/_test_multiprocessing.py

@@ -18,6 +18,7 @@ import random
 import logging
 import struct
 import operator
+import weakref
 import test.support
 import test.support.script_helper
 
@@ -1668,6 +1669,19 @@ def sqr(x, wait=0.0):
 def mul(x, y):
     return x*y
 
+def identity(x):
+    return x
+
+class CountedObject(object):
+    n_instances = 0
+
+    def __new__(cls):
+        cls.n_instances += 1
+        return object.__new__(cls)
+
+    def __del__(self):
+        type(self).n_instances -= 1
+
 class SayWhenError(ValueError): pass
 
 def exception_throwing_generator(total, when):
@@ -1676,6 +1690,7 @@ def exception_throwing_generator(total, when):
             raise SayWhenError("Somebody said when")
         yield i
 
+
 class _TestPool(BaseTestCase):
 
     @classmethod
@@ -1910,6 +1925,19 @@ class _TestPool(BaseTestCase):
             with self.assertRaises(RuntimeError):
                 p.apply(self._test_wrapped_exception)
 
+    def test_release_task_refs(self):
+        # Issue #29861: task arguments and results should not be kept
+        # alive after we are done with them.
+        objs = [CountedObject() for i in range(10)]
+        refs = [weakref.ref(o) for o in objs]
+        self.pool.map(identity, objs)
+
+        del objs
+        self.assertEqual(set(wr() for wr in refs), {None})
+        # With a process pool, copies of the objects are returned, check
+        # they were released too.
+        self.assertEqual(CountedObject.n_instances, 0)
+
 
 def raising():
     raise KeyError("key")