mirror of
https://github.com/python/cpython.git
synced 2025-08-04 08:59:19 +00:00
gh-129948: Add set() to multiprocessing.managers.SyncManager (#129949)
The SyncManager provided support for various data structures such as dict, list, and queue, but oddly, not set. This introduces support for set by defining SetProxy and registering it with SyncManager. --- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith <greg@krypto.org>
This commit is contained in:
mingyu
GitHub
parent
a65366ed87
commit
9f81f828c7
5 changed files with 199 additions and 3 deletions
|
|
@ -380,35 +380,40 @@ However, if you really do need to use some shared data then
|
|||
proxies.
|
||||
|
||||
A manager returned by :func:`Manager` will support types
|
||||
:class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`,
|
||||
:class:`list`, :class:`dict`, :class:`set`, :class:`~managers.Namespace`, :class:`Lock`,
|
||||
:class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`,
|
||||
:class:`Condition`, :class:`Event`, :class:`Barrier`,
|
||||
:class:`Queue`, :class:`Value` and :class:`Array`. For example, ::
|
||||
|
||||
from multiprocessing import Process, Manager
|
||||
|
||||
def f(d, l):
|
||||
def f(d, l, s):
|
||||
d[1] = '1'
|
||||
d['2'] = 2
|
||||
d[0.25] = None
|
||||
l.reverse()
|
||||
s.add('a')
|
||||
s.add('b')
|
||||
|
||||
if __name__ == '__main__':
|
||||
with Manager() as manager:
|
||||
d = manager.dict()
|
||||
l = manager.list(range(10))
|
||||
s = manager.set()
|
||||
|
||||
p = Process(target=f, args=(d, l))
|
||||
p = Process(target=f, args=(d, l, s))
|
||||
p.start()
|
||||
p.join()
|
||||
|
||||
print(d)
|
||||
print(l)
|
||||
print(s)
|
||||
|
||||
will print ::
|
||||
|
||||
{0.25: None, 1: '1', '2': 2}
|
||||
[9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
|
||||
{'a', 'b'}
|
||||
|
||||
Server process managers are more flexible than using shared memory objects
|
||||
because they can be made to support arbitrary object types. Also, a single
|
||||
|
|
@ -1942,6 +1947,15 @@ their parent process exits. The manager classes are defined in the
|
|||
|
||||
Create a shared :class:`list` object and return a proxy for it.
|
||||
|
||||
.. method:: set()
|
||||
set(sequence)
|
||||
set(mapping)
|
||||
|
||||
Create a shared :class:`set` object and return a proxy for it.
|
||||
|
||||
.. versionadded:: next
|
||||
:class:`set` support was added.
|
||||
|
||||
.. versionchanged:: 3.6
|
||||
Shared objects are capable of being nested. For example, a shared
|
||||
container object such as a shared list can contain other shared objects
|
||||
|
|
|
|||
|
|
@ -700,6 +700,11 @@ multiprocessing
|
|||
|
||||
(Contributed by Roy Hyunjin Han for :gh:`103134`.)
|
||||
|
||||
* Add support for shared :class:`set` objects via
|
||||
:meth:`SyncManager.set() <multiprocessing.managers.SyncManager.set>`.
|
||||
The :func:`set` in :func:`multiprocessing.Manager` method is now available.
|
||||
(Contributed by Mingyu Park in :gh:`129949`.)
|
||||
|
||||
|
||||
operator
|
||||
--------
|
||||
|
|
|
|||
|
|
@ -1195,6 +1195,36 @@ class DictProxy(_BaseDictProxy):
|
|||
|
||||
collections.abc.MutableMapping.register(_BaseDictProxy)
|
||||
|
||||
_BaseSetProxy = MakeProxyType("_BaseSetProxy", (
|
||||
'__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
|
||||
'__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
|
||||
'__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
|
||||
'__ge__', '__gt__', '__le__', '__lt__',
|
||||
'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
|
||||
'intersection', 'intersection_update', 'isdisjoint', 'issubset',
|
||||
'issuperset', 'pop', 'remove', 'symmetric_difference',
|
||||
'symmetric_difference_update', 'union', 'update',
|
||||
))
|
||||
|
||||
class SetProxy(_BaseSetProxy):
|
||||
def __ior__(self, value):
|
||||
self._callmethod('__ior__', (value,))
|
||||
return self
|
||||
def __iand__(self, value):
|
||||
self._callmethod('__iand__', (value,))
|
||||
return self
|
||||
def __ixor__(self, value):
|
||||
self._callmethod('__ixor__', (value,))
|
||||
return self
|
||||
def __isub__(self, value):
|
||||
self._callmethod('__isub__', (value,))
|
||||
return self
|
||||
|
||||
__class_getitem__ = classmethod(types.GenericAlias)
|
||||
|
||||
collections.abc.MutableMapping.register(_BaseSetProxy)
|
||||
|
||||
|
||||
ArrayProxy = MakeProxyType('ArrayProxy', (
|
||||
'__len__', '__getitem__', '__setitem__'
|
||||
))
|
||||
|
|
@ -1245,6 +1275,7 @@ SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
|
|||
SyncManager.register('Pool', pool.Pool, PoolProxy)
|
||||
SyncManager.register('list', list, ListProxy)
|
||||
SyncManager.register('dict', dict, DictProxy)
|
||||
SyncManager.register('set', set, SetProxy)
|
||||
SyncManager.register('Value', Value, ValueProxy)
|
||||
SyncManager.register('Array', Array, ArrayProxy)
|
||||
SyncManager.register('Namespace', Namespace, NamespaceProxy)
|
||||
|
|
|
|||
|
|
@ -6441,6 +6441,150 @@ class TestSyncManagerTypes(unittest.TestCase):
|
|||
o.y = 1
|
||||
self.run_worker(self._test_namespace, o)
|
||||
|
||||
@classmethod
|
||||
def _test_set_operator_symbols(cls, obj):
|
||||
case = unittest.TestCase()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
case.assertEqual(len(obj), 3)
|
||||
case.assertIn('a', obj)
|
||||
case.assertNotIn('d', obj)
|
||||
result = obj | {'d', 'e'}
|
||||
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
|
||||
result = {'d', 'e'} | obj
|
||||
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
|
||||
obj |= {'d', 'e'}
|
||||
case.assertSetEqual(obj, {'a', 'b', 'c', 'd', 'e'})
|
||||
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = {'a', 'b', 'd'} - obj
|
||||
case.assertSetEqual(result, {'d'})
|
||||
result = obj - {'a', 'b'}
|
||||
case.assertSetEqual(result, {'c'})
|
||||
obj -= {'a', 'b'}
|
||||
case.assertSetEqual(obj, {'c'})
|
||||
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = {'b', 'c', 'd'} ^ obj
|
||||
case.assertSetEqual(result, {'a', 'd'})
|
||||
result = obj ^ {'b', 'c', 'd'}
|
||||
case.assertSetEqual(result, {'a', 'd'})
|
||||
obj ^= {'b', 'c', 'd'}
|
||||
case.assertSetEqual(obj, {'a', 'd'})
|
||||
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = obj & {'b', 'c', 'd'}
|
||||
case.assertSetEqual(result, {'b', 'c'})
|
||||
result = {'b', 'c', 'd'} & obj
|
||||
case.assertSetEqual(result, {'b', 'c'})
|
||||
obj &= {'b', 'c', 'd'}
|
||||
case.assertSetEqual(obj, {'b', 'c'})
|
||||
case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
case.assertSetEqual(set(obj), {'a', 'b', 'c'})
|
||||
|
||||
@classmethod
|
||||
def _test_set_operator_methods(cls, obj):
|
||||
case = unittest.TestCase()
|
||||
obj.add('d')
|
||||
case.assertIn('d', obj)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
copy_obj = obj.copy()
|
||||
case.assertSetEqual(copy_obj, obj)
|
||||
obj.remove('a')
|
||||
case.assertNotIn('a', obj)
|
||||
case.assertRaises(KeyError, obj.remove, 'a')
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a'])
|
||||
obj.discard('a')
|
||||
case.assertNotIn('a', obj)
|
||||
obj.discard('a')
|
||||
case.assertNotIn('a', obj)
|
||||
obj.update(['a'])
|
||||
popped = obj.pop()
|
||||
case.assertNotIn(popped, obj)
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = obj.intersection({'b', 'c', 'd'})
|
||||
case.assertSetEqual(result, {'b', 'c'})
|
||||
obj.intersection_update({'b', 'c', 'd'})
|
||||
case.assertSetEqual(obj, {'b', 'c'})
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = obj.difference({'a', 'b'})
|
||||
case.assertSetEqual(result, {'c'})
|
||||
obj.difference_update({'a', 'b'})
|
||||
case.assertSetEqual(obj, {'c'})
|
||||
|
||||
obj.clear()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = obj.symmetric_difference({'b', 'c', 'd'})
|
||||
case.assertSetEqual(result, {'a', 'd'})
|
||||
obj.symmetric_difference_update({'b', 'c', 'd'})
|
||||
case.assertSetEqual(obj, {'a', 'd'})
|
||||
|
||||
@classmethod
|
||||
def _test_set_comparisons(cls, obj):
|
||||
case = unittest.TestCase()
|
||||
obj.update(['a', 'b', 'c'])
|
||||
result = obj.union({'d', 'e'})
|
||||
case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
|
||||
case.assertTrue(obj.isdisjoint({'d', 'e'}))
|
||||
case.assertFalse(obj.isdisjoint({'a', 'd'}))
|
||||
|
||||
case.assertTrue(obj.issubset({'a', 'b', 'c', 'd'}))
|
||||
case.assertFalse(obj.issubset({'a', 'b'}))
|
||||
case.assertLess(obj, {'a', 'b', 'c', 'd'})
|
||||
case.assertLessEqual(obj, {'a', 'b', 'c'})
|
||||
|
||||
case.assertTrue(obj.issuperset({'a', 'b'}))
|
||||
case.assertFalse(obj.issuperset({'a', 'b', 'd'}))
|
||||
case.assertGreater(obj, {'a'})
|
||||
case.assertGreaterEqual(obj, {'a', 'b'})
|
||||
|
||||
def test_set(self):
|
||||
o = self.manager.set()
|
||||
self.run_worker(self._test_set_operator_symbols, o)
|
||||
o = self.manager.set()
|
||||
self.run_worker(self._test_set_operator_methods, o)
|
||||
o = self.manager.set()
|
||||
self.run_worker(self._test_set_comparisons, o)
|
||||
|
||||
def test_set_init(self):
|
||||
o = self.manager.set({'a', 'b', 'c'})
|
||||
self.assertSetEqual(o, {'a', 'b', 'c'})
|
||||
o = self.manager.set(["a", "b", "c"])
|
||||
self.assertSetEqual(o, {"a", "b", "c"})
|
||||
o = self.manager.set({"a": 1, "b": 2, "c": 3})
|
||||
self.assertSetEqual(o, {"a", "b", "c"})
|
||||
self.assertRaises(RemoteError, self.manager.set, 1234)
|
||||
|
||||
def test_set_contain_all_method(self):
|
||||
o = self.manager.set()
|
||||
set_methods = {
|
||||
'__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
|
||||
'__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
|
||||
'__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
|
||||
'__ge__', '__gt__', '__le__', '__lt__',
|
||||
'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
|
||||
'intersection', 'intersection_update', 'isdisjoint', 'issubset',
|
||||
'issuperset', 'pop', 'remove', 'symmetric_difference',
|
||||
'symmetric_difference_update', 'union', 'update',
|
||||
}
|
||||
self.assertLessEqual(set_methods, set(dir(o)))
|
||||
|
||||
|
||||
class TestNamedResource(unittest.TestCase):
|
||||
@only_run_in_spawn_testsuite("spawn specific test.")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
Add support for shared :class:`set` to :class:`multiprocessing.managers.SyncManager`
|
||||
via :meth:`SyncManager.set() <multiprocessing.managers.SyncManager.set>`.
|
||||
Loading…
Add table
Add a link
Reference in a new issue