DEV: shared data with cycle processing

synergine2/cycle.py

@@ -1,17 +1,23 @@
 # coding: utf-8
 import multiprocessing
+import typing
 
 from synergine2.base import BaseObject
 from synergine2.config import Config
+from synergine2.exceptions import SynergineException
 from synergine2.log import SynergineLogger
 from synergine2.processing import ProcessManager
-from synergine2.simulation import SimulationMechanism
-from synergine2.simulation import SimulationBehaviour
+from synergine2.share import shared
+from synergine2.simulation import Subject
 from synergine2.simulation import Simulation
 from synergine2.simulation import SubjectBehaviour
 from synergine2.simulation import SubjectMechanism
 from synergine2.simulation import Event
-from synergine2.utils import ChunkManager, time_it
+from synergine2.utils import time_it
+
+
+JOB_TYPE_SUBJECTS = 0
+JOB_TYPE_SIMULATION = 1
 
 
 class CycleManager(BaseObject):
@@ -22,20 +28,107 @@ class CycleManager(BaseObject):
             simulation: Simulation,
             process_manager: ProcessManager=None,
     ):
+        # TODO: reproduire le mechanisme d'index de behaviour/etc pour simulation
+        self.config = config
+        self.logger = logger
+        self.simulation = simulation
+        self.current_cycle = -1
+        self.first_cycle = True
+
+        self.subject_mechanisms_cache = {}  # type: typing.Dict[int, typing.Dict[str, SubjectMechanism]]
+        self.subject_behaviours_cache = {}  # type: typing.Dict[int, typing.Dict[str, SubjectBehaviour]]
+
+        # TODO NOW: Les processes devront maintenir une liste des subjects qui sont nouveaux.ne connaissent pas
+        # Attention a ce qu'in ne soient pas "expose" quand on créer ces subjects au sein du process.
+        # Ces subjects ont vocation à adopter l'id du vrau subject tout de suite après leur instanciation
         if process_manager is None:
             process_manager = ProcessManager(
                 config=config,
                 # TODO: Changer de config de merde (core.use_x_cores)
                 process_count=config.get('core', {}).get('use_x_cores', multiprocessing.cpu_count()),
-                chunk_manager=ChunkManager(multiprocessing.cpu_count()),
+                job=self.job,
             )
-
-        self.config = config
-        self.logger = logger
-        self.simulation = simulation
         self.process_manager = process_manager
-        self.current_cycle = -1
-        self.first_cycle = True
+
+    def job(self, worker_id: int, process_count: int, job_type: str) -> 'TODO':
+        # ICI: (in process) on doit avoir:
+        # La tranche x:y de sujets à traiter
+        shared.refresh()
+        if job_type == JOB_TYPE_SUBJECTS:
+            return self._job_subjects(worker_id, process_count)
+        if job_type == JOB_TYPE_SIMULATION:
+            return self._job_simulation(worker_id, process_count)
+        raise SynergineException('Unknown job type "{}"'.format(job_type))
+
+    def _job_subjects(self, worker_id: int, process_count: int) -> typing.Dict[int, typing.Dict[str, typing.Any]]:
+        # Determine list of process subject to work with
+        subject_ids = shared.get('subject_ids')
+        chunk_length, rest = divmod(len(subject_ids), process_count)
+
+        from_ = chunk_length * worker_id
+        to_ = from_ + chunk_length
+
+        if worker_id + 1 == process_count:
+            to_ += rest
+
+        subject_ids_to_parse = subject_ids[from_:to_]
+
+        # Build list of subjects for compute them
+        subjects = []
+        for subject_id in subject_ids_to_parse:
+            subject = self.simulation.get_or_create_subject(subject_id)
+            subjects.append(subject)
+
+        results_by_subjects = self._subjects_computing(subjects)
+        return results_by_subjects
+
+    def _job_simulation(self, worker_id: int, process_count: int) -> typing.Dict[int, typing.Dict[str, typing.Any]]:
+        self.logger.info('Simulation computing (worker {})'.format(worker_id))
+
+        mechanisms = self.simulation.mechanisms.values()
+        mechanisms_data = {}
+        behaviours_data = {}
+
+        self.logger.info('{} mechanisms to compute'.format(str(len(mechanisms))))
+        if self.logger.is_debug:
+            self.logger.debug('Mechanisms are: {}'.format(
+                str([m.repr_debug() for m in mechanisms])
+            ))
+
+        for mechanism in mechanisms:
+            mechanism_data = mechanism.run(
+                process_number=worker_id,
+                process_count=process_count,
+            )
+
+            if self.logger.is_debug:
+                self.logger.debug('{} mechanism product data: {}'.format(
+                    type(mechanism).__name__,
+                    str(mechanism_data),
+                ))
+
+            mechanisms_data[type(mechanism)] = mechanism_data
+
+        behaviours = self.simulation.behaviours.values()
+        self.logger.info('{} behaviours to compute'.format(str(len(behaviours))))
+
+        if self.logger.is_debug:
+            self.logger.debug('Behaviours are: {}'.format(
+                str([b.repr_debug() for b in behaviours])
+            ))
+
+        for behaviour in behaviours:
+            behaviour_data = behaviour.run(mechanisms_data)  # TODO: Behaviours dependencies
+            if self.logger.is_debug:
+                self.logger.debug('{} behaviour produce data: {}'.format(
+                    type(behaviour).__name__,
+                    behaviour_data,
+                ))
+
+            if behaviour_data:
+                behaviours_data[type(behaviour)] = behaviour_data
+
+        return behaviours_data
 
     def next(self) -> [Event]:
         if self.first_cycle:
@@ -47,6 +140,8 @@ class CycleManager(BaseObject):
         self.logger.info('Process cycle {}'.format(self.current_cycle))
 
         events = []
+        shared.commit()
+
         # TODO: gestion des behaviours non parallelisables
         # TODO: Proposer des ordres d'execution
         with time_it() as elapsed_time:
@@ -69,10 +164,8 @@ class CycleManager(BaseObject):
 
         self.logger.info('Process simulation events')
 
-        results_by_processes = self.process_manager.execute_jobs(
-            data=self.simulation,
-            job_maker=self.simulation_computing,
-        )
+        # TODO: Think about compute simulation cycle in workers
+        results_by_processes = self.process_manager.make_them_work(JOB_TYPE_SIMULATION)
 
         for process_result in results_by_processes:
             for behaviour_class, behaviour_result in process_result.items():
@@ -107,36 +200,35 @@ class CycleManager(BaseObject):
         results = {}
 
         self.logger.info('Process subjects events')
-
-        results_by_processes = self.process_manager.chunk_and_execute_jobs(
-            data=self.simulation.subjects,
-            job_maker=self.subjects_computing,
-        )
+        results_by_processes = self.process_manager.make_them_work(JOB_TYPE_SUBJECTS)
 
         for process_results in results_by_processes:
             results.update(process_results)
 
         # Duplicate list to prevent conflicts with behaviours subjects manipulations
         for subject in self.simulation.subjects[:]:
-            subject_behaviours = results.get(subject.id, {})
+            subject_behaviours_results = results.get(subject.id, {})
             if subject.behaviour_selector:
                 # TODO: Looging
-                subject_behaviours = subject.behaviour_selector.reduce_behaviours(dict(subject_behaviours))
+                subject_behaviours_results = subject.behaviour_selector.reduce_behaviours(dict(
+                    subject_behaviours_results,
+                ))
 
-            for behaviour_class, behaviour_data in subject_behaviours.items():
+            subject_behaviours = self.get_subject_behaviours(subject)
+            for behaviour_class_name, behaviour_data in subject_behaviours_results.items():
                 # TODO: Ajouter une etape de selection des actions a faire (genre neuronnal)
                 # (genre se cacher et fuir son pas compatibles)
-                behaviour_events = subject.behaviours[behaviour_class].action(behaviour_data)
+                behaviour_events = subject_behaviours[behaviour_class_name].action(behaviour_data)
 
                 self.logger.info('{} behaviour for subject {} generate {} events'.format(
-                    str(behaviour_class),
+                    str(behaviour_class_name),
                     str(subject.id),
                     str(len(behaviour_events)),
                 ))
 
                 if self.logger.is_debug:
                     self.logger.debug('{} behaviour for subject {} generated events: {}'.format(
-                        str(behaviour_class),
+                        str(behaviour_class_name),
                         str(subject.id),
                         str([e.repr_debug() for e in behaviour_events]),
                     ))
@@ -146,71 +238,17 @@ class CycleManager(BaseObject):
         self.logger.info('Subjects behaviours generate {} events'.format(len(events)))
         return events
 
-    def simulation_computing(
-            self,
-            simulation,
-            process_number,
-            process_count,
-    ):
-        self.logger.info('Simulation computing')
-
-        # TODO: necessaire de passer simulation ?
-        mechanisms = self.get_mechanisms_to_compute(simulation)
-        mechanisms_data = {}
-        behaviours_data = {}
-
-        self.logger.info('{} mechanisms to compute'.format(str(len(mechanisms))))
-        if self.logger.is_debug:
-            self.logger.debug('Mechanisms are: {}'.format(
-                str([m.repr_debug() for m in mechanisms])
-            ))
-
-        for mechanism in mechanisms:
-            mechanism_data = mechanism.run(
-                process_number=process_number,
-                process_count=process_count,
-            )
-
-            if self.logger.is_debug:
-                self.logger.debug('{} mechanism product data: {}'.format(
-                    type(mechanism).__name__,
-                    str(mechanism_data),
-                ))
-
-            mechanisms_data[type(mechanism)] = mechanism_data
-
-        behaviours = self.get_behaviours_to_compute(simulation)
-        self.logger.info('{} behaviours to compute'.format(str(len(behaviours))))
-
-        if self.logger.is_debug:
-            self.logger.debug('Behaviours are: {}'.format(
-                str([b.repr_debug() for b in behaviours])
-            ))
-
-        for behaviour in behaviours:
-            behaviour_data = behaviour.run(mechanisms_data)  # TODO: Behaviours dependencies
-            if self.logger.is_debug:
-                self.logger.debug('{} behaviour produce data: {}'.format(
-                    type(behaviour).__name__,
-                    behaviour_data,
-                ))
-
-            if behaviour_data:
-                behaviours_data[type(behaviour)] = behaviour_data
-
-        return behaviours_data
-
-    def subjects_computing(
+    def _subjects_computing(
             self,
             subjects,
             process_number=None,
             process_count=None,
-    ):
+    ) -> typing.Dict[int, typing.Dict[str, typing.Any]]:
         results = {}
         self.logger.info('Subjects computing: {} subjects to compute'.format(str(len(subjects))))
 
         for subject in subjects:
-            mechanisms = self.get_mechanisms_to_compute(subject)
+            mechanisms = self.get_subject_mechanisms(subject)
 
             if mechanisms:
                 self.logger.info('Subject {}: {} mechanisms'.format(
@@ -221,13 +259,13 @@ class CycleManager(BaseObject):
                 if self.logger.is_debug:
                     self.logger.info('Subject {}: mechanisms are: {}'.format(
                         str(subject.id),
-                        str([m.repr_debug for m in mechanisms])
+                        str([m.repr_debug for n, m in mechanisms.items()])
                     ))
 
             mechanisms_data = {}
             behaviours_data = {}
 
-            for mechanism in mechanisms:
+            for mechanism_class_name, mechanism in mechanisms.items():
                 with time_it() as elapsed_time:
                     mechanism_data = mechanism.run()
                 if self.logger.is_debug:
@@ -238,7 +276,7 @@ class CycleManager(BaseObject):
                         elapsed_time.get_final_time(),
                     ))
 
-                mechanisms_data[type(mechanism)] = mechanism_data
+                mechanisms_data[mechanism_class_name] = mechanism_data
 
             if mechanisms:
                 if self.logger.is_debug:
@@ -247,7 +285,7 @@ class CycleManager(BaseObject):
                         str(mechanisms_data),
                     ))
 
-            subject_behaviours = self.get_behaviours_to_compute(subject)
+            subject_behaviours = self.get_subject_behaviours(subject)
             if not subject_behaviours:
                 break
 
@@ -256,7 +294,7 @@ class CycleManager(BaseObject):
                 str(len(subject_behaviours)),
             ))
 
-            for behaviour in subject_behaviours:
+            for behaviour_class_name, behaviour in subject_behaviours.items():
                 self.logger.info('Subject {}: run {} behaviour'.format(
                     str(subject.id),
                     str(type(behaviour)),
@@ -275,25 +313,46 @@ class CycleManager(BaseObject):
                     ))
 
                 if behaviour_data:
-                    behaviours_data[type(behaviour)] = behaviour_data
+                    behaviours_data[behaviour_class_name] = behaviour_data
 
             results[subject.id] = behaviours_data
         return results
 
-    def get_mechanisms_to_compute(self, mechanisable) -> [SubjectMechanism, SimulationMechanism]:
+    def get_subject_mechanisms(self, subject: Subject) -> typing.Dict[str, SubjectMechanism]:
         # TODO: Implementer un systeme qui inhibe des mechanisme (ex. someil inhibe l'ouie)
-        return mechanisable.mechanisms.values()
+        # Attention: c'est utilisé dans le main process aussi, pertinent de la faire là ?
+        try:
+            return self.subject_mechanisms_cache[subject.id]
+        except KeyError:
+            mechanisms = {}
+            for mechanism_class_id in shared.get('subject_mechanisms_index')[subject.id]:
+                mechanism_class = self.simulation.index[mechanism_class_id]
+                mechanism = mechanism_class(
+                    self.config,
+                    self.simulation,
+                    subject,
+                )
+                mechanisms[mechanism_class.__name__] = mechanism
+            self.subject_mechanisms_cache[subject.id] = mechanisms
+            return mechanisms
 
-    def get_behaviours_to_compute(self, mechanisable) -> [SubjectBehaviour, SimulationBehaviour]:
+    def get_subject_behaviours(self, subject: Subject) -> typing.Dict[str, SubjectBehaviour]:
         # TODO: Implementer un systeme qui inhibe des behaviours (ex. someil inhibe avoir faim)
-        behaviours = list(mechanisable.behaviours.values())
-
-        for behaviour in behaviours[:]:
-            if behaviour.frequency != 1:
-                if self.current_cycle % behaviour.frequency:
-                    behaviours.remove(behaviour)
-
-        return behaviours
+        # Attention: c'est utilisé dans le main process aussi, pertinent de la faire là ?
+        try:
+            return self.subject_behaviours_cache[subject.id]
+        except KeyError:
+            behaviours = {}
+            for behaviour_class_id in shared.get('subject_behaviours_index')[subject.id]:
+                behaviour_class = self.simulation.index[behaviour_class_id]
+                behaviour = behaviour_class(
+                    self.config,
+                    self.simulation,
+                    subject,
+                )
+                behaviours[behaviour_class.__name__] = behaviour
+            self.subject_behaviours_cache[subject.id] = behaviours
+            return behaviours
 
     def apply_actions(
             self,
@@ -364,3 +423,6 @@ class CycleManager(BaseObject):
 
         self.logger.info('{} events generated'.format(len(events)))
         return events
+
+    def stop(self) -> None:
+        self.process_manager.terminate()

synergine2/processing.py

@@ -48,11 +48,11 @@ class Worker(object):
 
     def work(self, *args, **kwargs):
         while True:
-            message = self.process_read_pipe.recv()
-            if message == STOP:
+            args = self.process_read_pipe.recv()
+            if args == STOP:
                 return
 
-            result = self.real_job(message)
+            result = self.real_job(*args)
             self.local_write_pipe.send(result)
 
 
@@ -76,8 +76,8 @@ class ProcessManager(BaseObject):
     def make_them_work(self, message: typing.Any) -> 'TODO':
         responses = []
 
-        for worker in self.workers:
-            worker.process_write_pipe.send(message)
+        for worker_id, worker in enumerate(self.workers):
+            worker.process_write_pipe.send((worker_id, self._process_count, message))
 
         for worker in self.workers:
             responses.append(worker.local_read_pipe.recv())
@@ -90,31 +90,3 @@ class ProcessManager(BaseObject):
 
         for worker in self.workers:
             worker.process.join()
-
-    #
-    # def chunk_and_execute_jobs(self, data: list, job_maker: types.FunctionType) -> list:
-    #     chunks = self._chunk_manager.make_chunks(data)
-    #
-    #     if self._process_count > 1:
-    #         print('USE POOL')
-    #         results = self.pool.starmap(job_maker, [(chunk, i, self._process_count) for i, chunk in enumerate(chunks)])
-    #     else:
-    #         print('USE MONO')
-    #         results = [job_maker(data, 0, 1)]
-    #
-    #     return results
-    #
-    # def execute_jobs(self, data: object, job_maker: types.FunctionType) -> list:
-    #     # TODO: Is there a reason to make multiprocessing here ? data is not chunked ...
-    #     if self._process_count > 1:
-    #         results = self.pool.starmap(job_maker, [(data, i, self._process_count) for i in range(self._process_count)])
-    #     else:
-    #         results = [job_maker(data, 0, 1)]
-    #
-    #     return results
-    #
-    # def __del__(self):
-    #     # TODO: DEV
-    #     return
-    #     if self.pool:
-    #         self.pool.terminate()

synergine2/share.py

@@ -25,6 +25,40 @@ class SharedDataIndex(object):
         raise NotImplementedError()
 
 
+class TrackedDict(dict):
+    base = dict
+
+    def __init__(self, seq=None, **kwargs):
+        self.key = kwargs.pop('key')
+        self.shared = kwargs.pop('shared')
+        super().__init__(seq, **kwargs)
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self.shared.set(self.key, dict(self))
+
+    def setdefault(self, k, d=None):
+        v = super().setdefault(k, d)
+        self.shared.set(self.key, dict(self))
+        return v
+    # TODO: Cover all methods
+
+
+class TrackedList(list):
+    base = list
+
+    def __init__(self, seq=(), **kwargs):
+        self.key = kwargs.pop('key')
+        self.shared = kwargs.pop('shared')
+        super().__init__(seq)
+
+    def append(self, p_object):
+        super().append(p_object)
+        self.shared.set(self.key, list(self))
+
+    # TODO: Cover all methods
+
+
 class SharedDataManager(object):
     """
     This object is designed to own shared memory between processes. It must be feed (with set method) before
@@ -35,29 +69,67 @@ class SharedDataManager(object):
 
         self._data = {}
         self._modified_keys = set()
+        self._default_values = {}
+        self._special_types = {}  # type: typing.Dict[str, typing.Union[typing.Type[TrackedDict], typing.Type[TrackedList]]]  # nopep8
 
         if clear:
-            self._r.flushdb()
+            self.clear()
+
+    def clear(self) -> None:
+        self._r.flushdb()
+        self._data = {}
+        self._modified_keys = set()
+
+    def reset(self) -> None:
+        for key, value in self._default_values.items():
+            self.set(key, value)
+        self.commit()
+        self._data = {}
 
     def set(self, key: str, value: typing.Any) -> None:
+        try:
+            special_type = self._special_types[key]
+            value = special_type(value, key=key, shared=self)
+        except KeyError:
+            pass
+
         self._data[key] = value
         self._modified_keys.add(key)
 
     def get(self, *key_args: typing.Union[str, float, int]) -> typing.Any:
         key = '_'.join([str(v) for v in key_args])
 
-        if key not in self._data:
+        try:
+            return self._data[key]
+        except KeyError:
             b_value = self._r.get(key)
             if b_value is None:
                 # We not allow None value storage
                 raise UnknownSharedData('No shared data for key "{}"'.format(key))
-            self._data[key] = pickle.loads(b_value)
+
+            value = pickle.loads(b_value)
+            special_type = None
+
+            try:
+                special_type = self._special_types[key]
+            except KeyError:
+                pass
+
+            if special_type:
+                self._data[key] = special_type(value, key=key, shared=self)
+            else:
+                self._data[key] = value
 
         return self._data[key]
 
     def commit(self) -> None:
         for key in self._modified_keys:
-            self._r.set(key, pickle.dumps(self.get(key)))
+            try:
+                special_type = self._special_types[key]
+                value = special_type.base(self.get(key))
+                self._r.set(key, pickle.dumps(value))
+            except KeyError:
+                self._r.set(key, pickle.dumps(self.get(key)))
         self._modified_keys = set()
 
     def refresh(self) -> None:
@@ -83,6 +155,13 @@ class SharedDataManager(object):
             key = '_'.join(key_args)
         indexes = indexes or []
 
+        if type(value) is dict:
+            value = TrackedDict(value, key=key, shared=shared)
+            self._special_types[key] = TrackedDict
+        elif type(value) is list:
+            value = TrackedList(value, key=key, shared=shared)
+            self._special_types[key] = TrackedList
+
         def get_key(obj):
             return key
 
@@ -93,6 +172,7 @@ class SharedDataManager(object):
             key_formatter = get_key_with_id
         else:
             self.set(key, value)
+            self._default_values[key] = value
             key_formatter = get_key
 
         def fget(self_):
@@ -122,6 +202,9 @@ class SharedDataManager(object):
 
         return shared_property
 
+# TODO: Does exist a way to permit overload of SharedDataManager class ?
+shared = SharedDataManager()
+
 
 class ListIndex(SharedDataIndex):
     def add(self, value):

synergine2/simulation.py

@@ -4,6 +4,7 @@ import typing
 
 from synergine2.base import BaseObject
 from synergine2.config import Config
+from synergine2.share import shared
 from synergine2.utils import get_mechanisms_classes
 
 
@@ -26,6 +27,7 @@ class Subject(BaseObject):
     collections = []
     behaviours_classes = []
     behaviour_selector_class = None  # type: typing.Type[SubjectBehaviourSelector]
+    intention_manager_class = None  # type: typing.Type[IntentionManager]
 
     def __init__(
         self,
@@ -36,19 +38,42 @@ class Subject(BaseObject):
         self.collections = self.collections[:]
 
         self.config = config
-        self.id = id(self)  # We store object id because it's lost between process
+        self._id = id(self)  # We store object id because it's lost between process
         self.simulation = simulation
-        self.behaviours = {}
-        self.mechanisms = {}
-        self.intentions = IntentionManager()
-        self.behaviour_selector = None  # type: SubjectBehaviourSelector
+        self.intentions = None
+
         if self.behaviour_selector_class:
             self.behaviour_selector = self.behaviour_selector_class()
+        else:
+            self.behaviour_selector = SubjectBehaviourSelector()
+
+        if self.intention_manager_class:
+            self.intentions = self.intention_manager_class()
+        else:
+            self.intentions = IntentionManager()
+
+        # TODO: Revoir le mechanisme de collection: utilité, usage avec les process, etc
+        # for collection in self.collections:
+        #     self.simulation.collections[collection].append(self)
+
+    @property
+    def id(self) -> int:
+        return self._id
 
-        for collection in self.collections:
-            self.simulation.collections[collection].append(self)
+    def change_id(self, id_: int) -> None:
+        self._id = id_
 
-        self.initialize()
+    def expose(self) -> None:
+        subject_behaviours_index = shared.get('subject_behaviours_index').setdefault(self._id, [])
+        subject_mechanisms_index = shared.get('subject_mechanisms_index').setdefault(self._id, [])
+        subject_classes = shared.get('subject_classes')
+
+        for behaviour_class in self.behaviours_classes:
+            subject_behaviours_index.append(id(behaviour_class))
+            for mechanism_class in behaviour_class.use:
+                subject_mechanisms_index.append(id(mechanism_class))
+
+        subject_classes[self._id] = id(type(self))
 
     def __str__(self):
         return self.__repr__()
@@ -59,37 +84,35 @@ class Subject(BaseObject):
             self.id,
         )
 
-    def initialize(self):
-        for mechanism_class in get_mechanisms_classes(self):
-            self.mechanisms[mechanism_class] = mechanism_class(
-                config=self.config,
-                simulation=self.simulation,
-                subject=self,
-            )
-
-        for behaviour_class in self.behaviours_classes:
-            self.behaviours[behaviour_class] = behaviour_class(
-                config=self.config,
-                simulation=self.simulation,
-                subject=self,
-            )
-
 
 class Subjects(list):
     """
     TODO: Manage other list methods
     """
+    subject_ids = shared.create('subject_ids', [])
+
     def __init__(self, *args, **kwargs):
         self.simulation = kwargs.pop('simulation')
         self.removes = []
         self.adds = []
         self.track_changes = False
         self.index = {}
+        self._auto_expose = True
         super().__init__(*args, **kwargs)
 
+    @property
+    def auto_expose(self) -> bool:
+        return self._auto_expose
+
+    @auto_expose.setter
+    def auto_expose(self, value: bool) -> None:
+        assert self._auto_expose
+        self._auto_expose = value
+
     def remove(self, value: Subject):
         # Remove from index
         del self.index[value.id]
+        self.subject_ids.remove(value.id)
         # Remove from subjects list
         super().remove(value)
         # Remove from collections
@@ -98,32 +121,67 @@ class Subjects(list):
         # Add to removed listing
         if self.track_changes:
             self.removes.append(value)
+        # TODO: Supprimer des choses du shared ! Sinon fuite mémoire dans la bdd
 
     def append(self, p_object):
         # Add to index
         self.index[p_object.id] = p_object
+        self.subject_ids.append(p_object.id)
         # Add to subjects list
         super().append(p_object)
         # Add to adds list
         if self.track_changes:
             self.adds.append(p_object)
+        if self.auto_expose:
+            p_object.expose()
 
 
 class Simulation(BaseObject):
     accepted_subject_class = Subjects
     behaviours_classes = []
 
+    subject_behaviours_index = shared.create('subject_behaviours_index', {})
+    subject_mechanisms_index = shared.create('subject_mechanisms_index', {})
+    subject_classes = shared.create('subject_classes', {})
+
     def __init__(
         self,
         config: Config,
     ):
         self.config = config
         self.collections = collections.defaultdict(list)
-        self._subjects = None
+        self._subjects = None  # type: Subjects
+
+        # Should contain all usable class of Behaviors, Mechanisms, SubjectBehaviourSelectors,
+        # IntentionManagers, Subject
+        self._index = {}  # type: typing.Dict[int, type]
+        self._index_locked = False
+
         self.behaviours = {}
         self.mechanisms = {}
 
-        self.initialize()
+        for mechanism_class in get_mechanisms_classes(self):
+            self.mechanisms[mechanism_class.__name__] = mechanism_class(
+                config=self.config,
+                simulation=self,
+            )
+
+        for behaviour_class in self.behaviours_classes:
+            self.behaviours[behaviour_class.__name__] = behaviour_class(
+                config=self.config,
+                simulation=self,
+            )
+
+    def add_to_index(self, class_: type) -> None:
+        assert not self._index_locked
+        self._index[id(class_)] = class_
+
+    @property
+    def index(self) -> typing.Dict[int, type]:
+        return self._index
+
+    def lock_index(self) -> None:
+        self._index_locked = True
 
     @property
     def subjects(self):
@@ -137,21 +195,24 @@ class Simulation(BaseObject):
             ))
         self._subjects = value
 
-    def initialize(self):
-        for mechanism_class in get_mechanisms_classes(self):
-            self.mechanisms[mechanism_class] = mechanism_class(
-                config=self.config,
-                simulation=self,
-            )
+    def get_or_create_subject(self, subject_id: int) -> Subject:
+        try:
+            return self._subjects.index[subject_id]
+        except KeyError:
+            # We should be in process context and subject have to been created
+            subject_class_id = shared.get('subject_classes')[subject_id]
+            subject_class = self.index[subject_class_id]
+            subject = subject_class(self.config, self)
+            subject.change_id(subject_id)
+            self.subjects.append(subject)
+            return subject
 
-        for behaviour_class in self.behaviours_classes:
-            self.behaviours[behaviour_class] = behaviour_class(
-                config=self.config,
-                simulation=self,
-            )
 
+class Mechanism(BaseObject):
+    pass
 
-class SubjectMechanism(BaseObject):
+
+class SubjectMechanism(Mechanism):
     def __init__(
             self,
             config: Config,
@@ -166,7 +227,7 @@ class SubjectMechanism(BaseObject):
         raise NotImplementedError()
 
 
-class SimulationMechanism(BaseObject):
+class SimulationMechanism(Mechanism):
     """If parallelizable behaviour, call """
     parallelizable = False
 
@@ -193,7 +254,12 @@ class Event(BaseObject):
         return self.__class__.__name__
 
 
-class SubjectBehaviour(BaseObject):
+class Behaviour(BaseObject):
+    def run(self, data):
+        raise NotImplementedError()
+
+
+class SubjectBehaviour(Behaviour):
     frequency = 1
     use = []  # type: typing.List[typing.Type[SubjectMechanism]]
 
@@ -226,7 +292,7 @@ class SubjectBehaviour(BaseObject):
         raise NotImplementedError()
 
 
-class SimulationBehaviour(BaseObject):
+class SimulationBehaviour(Behaviour):
     frequency = 1
     use = []
 
@@ -265,4 +331,4 @@ class SubjectBehaviourSelector(BaseObject):
         self,
         behaviours: typing.Dict[typing.Type[SubjectBehaviour], object],
     ) -> typing.Dict[typing.Type[SubjectBehaviour], object]:
-        raise NotImplementedError()
+        return behaviours

synergine2/terminals.py

@@ -18,6 +18,9 @@ STOP_SIGNAL = '__STOP_SIGNAL__'
 
 
 class TerminalPackage(BaseObject):
+    """
+    TODO: Update this class considering shared data across processes
+    """
     def __init__(
             self,
             subjects: [Subject]=None,

synergine2_xyz/physics.py

@@ -46,7 +46,7 @@ class Physics(object):
         self.move_cost_computer = self.move_cost_computer_class(config)
 
     def load(self) -> None:
-        raise NotImplementedError()
+        pass
 
     def position_to_key(self, position: typing.Tuple[int, int]) -> str:
         return '{}.{}'.format(*position)

synergine2_xyz/simulation.py

@@ -19,7 +19,7 @@ class XYZSimulation(BaseSimulation):
         self.physics.load()
 
     def create_physics(self) -> Physics:
-        raise NotImplementedError()
+        return Physics(self.config)
 
     def is_possible_subject_position(self, subject: XYZSubject, position: tuple) -> bool:
         return self.is_possible_position(position)

tests/test_cycle.py

@@ -0,0 +1,113 @@
+# coding: utf-8
+from synergine2.config import Config
+from synergine2.cycle import CycleManager
+from synergine2.log import SynergineLogger
+from synergine2.share import shared
+from synergine2.simulation import Simulation
+from synergine2.simulation import Event
+from synergine2.simulation import Subject
+from synergine2.simulation import Subjects
+from synergine2.simulation import SubjectMechanism
+from synergine2.simulation import SubjectBehaviour
+from tests import BaseTest
+
+
+class MyEvent(Event):
+    def __init__(self, value):
+        self.value = value
+
+
+class MySubjectMechanism(SubjectMechanism):
+    def run(self):
+        return 42
+
+
+class MySubjectBehavior(SubjectBehaviour):
+    use = [MySubjectMechanism]
+
+    def run(self, data):
+        class_name = MySubjectMechanism.__name__
+        if class_name in data and data[class_name] == 42:
+            return self.subject.id
+
+    def action(self, data) -> [Event]:
+        return [MyEvent(data * 2)]
+
+
+class MySubject(Subject):
+    behaviours_classes = [MySubjectBehavior]
+
+
+class MySubjects(Subjects):
+    pass
+
+
+class TestCycle(BaseTest):
+    # def test_subjects_cycle(self):
+    #     shared.reset()
+    #     config = Config({'core': {'use_x_cores': 2}})
+    #     logger = SynergineLogger(name='test')
+    #
+    #     simulation = Simulation(config)
+    #     subjects = MySubjects(simulation=simulation)
+    #     simulation.subjects = subjects
+    #
+    #     # Prepare simulation class index
+    #     simulation.add_to_index(MySubjectBehavior)
+    #     simulation.add_to_index(MySubjectMechanism)
+    #     simulation.add_to_index(MySubject)
+    #
+    #     for i in range(3):
+    #         subjects.append(MySubject(config, simulation=simulation))
+    #
+    #     cycle_manager = CycleManager(
+    #         config=config,
+    #         logger=logger,
+    #         simulation=simulation,
+    #     )
+    #
+    #     events = cycle_manager.next()
+    #     cycle_manager.stop()
+    #
+    #     assert 3 == len(events)
+    #     event_values = [e.value for e in events]
+    #     assert all([s.id * 2 in event_values for s in subjects])
+
+    def test_new_subject(self):
+        shared.reset()
+        subject_ids = shared.get('subject_ids')
+        config = Config({'core': {'use_x_cores': 1}})
+        logger = SynergineLogger(name='test')
+
+        simulation = Simulation(config)
+        subjects = MySubjects(simulation=simulation)
+        simulation.subjects = subjects
+
+        # Prepare simulation class index
+        simulation.add_to_index(MySubjectBehavior)
+        simulation.add_to_index(MySubjectMechanism)
+        simulation.add_to_index(MySubject)
+
+        for i in range(3):
+            subjects.append(MySubject(config, simulation=simulation))
+
+        cycle_manager = CycleManager(
+            config=config,
+            logger=logger,
+            simulation=simulation,
+        )
+
+        events = cycle_manager.next()
+
+        assert 3 == len(events)
+        event_values = [e.value for e in events]
+        assert all([s.id * 2 in event_values for s in subjects])
+
+        subjects.append(MySubject(config, simulation=simulation))
+        events = cycle_manager.next()
+        cycle_manager.stop()
+
+        assert 4 == len(events)
+        event_values = [e.value for e in events]
+        assert all([s.id * 2 in event_values for s in subjects])
+

tests/test_life_game.py

@@ -1,192 +0,0 @@
-# coding: utf-8
-import collections
-
-from sandbox.life_game.simulation import Cell
-from sandbox.life_game.simulation import Empty
-from sandbox.life_game.utils import get_subjects_from_str_representation
-from synergine2.config import Config
-from synergine2.cycle import CycleManager
-from synergine2.log import SynergineLogger
-from synergine2_xyz.simulation import XYZSimulation
-from synergine2_xyz.subjects import XYZSubjects
-from synergine2_xyz.utils import get_str_representation_from_positions
-from tests import BaseTest
-from tests import str_kwargs
-
-
-class LifeGameBaseTest(BaseTest):
-    def _get_str_representation_of_subjects(self, subjects: list):
-        items_positions = collections.defaultdict(list)
-
-        for subject in subjects:
-            if type(subject) == Cell:
-                items_positions['1'].append(subject.position)
-            if type(subject) == Empty:
-                items_positions['0'].append(subject.position)
-
-        return get_str_representation_from_positions(
-            items_positions,
-            complete_lines_with='0',
-            **str_kwargs
-        )
-
-
-class TestSimpleSimulation(LifeGameBaseTest):
-    def test_cycles_evolution(self):
-        simulation = XYZSimulation(Config())
-        subjects = self._get_subjects(simulation)
-        simulation.subjects = subjects
-
-        cycle_manager = CycleManager(
-            Config(),
-            SynergineLogger('test'),
-            simulation=simulation,
-        )
-
-        assert """
-            0 0 0 0 0
-            0 1 1 1 0
-            0 0 0 0 0
-        """ == self._get_str_representation_of_subjects(
-            subjects,
-        )
-
-        cycle_manager.next()
-
-        assert """
-            0 0 0 0 0
-            0 0 1 0 0
-            0 0 1 0 0
-            0 0 1 0 0
-            0 0 0 0 0
-        """ == self._get_str_representation_of_subjects(
-            subjects,
-        )
-
-        cycle_manager.next()
-
-        assert """
-            0 0 0 0 0
-            0 0 0 0 0
-            0 1 1 1 0
-            0 0 0 0 0
-            0 0 0 0 0
-        """ == self._get_str_representation_of_subjects(
-            subjects,
-        )
-
-    def _get_subjects(self, simulation: XYZSimulation):
-        cells = XYZSubjects(simulation=simulation)
-
-        for position in [
-            (-1, 0, 0),
-            (0, 0, 0),
-            (1, 0, 0),
-        ]:
-            cells.append(Cell(
-                Config(),
-                simulation=simulation,
-                position=position,
-            ))
-
-        for position in [
-            (-2, -1, 0),
-            (-1, -1, 0),
-            (0, -1, 0),
-            (1, -1, 0),
-            (2, -1, 0),
-            (-2, 0, 0),
-            (2, 0, 0),
-            (-2, 1, 0),
-            (-1, 1, 0),
-            (0, 1, 0),
-            (1, 1, 0),
-            (2, 1, 0),
-        ]:
-            cells.append(Empty(
-                Config(),
-                simulation=simulation,
-                position=position,
-            ))
-        return cells
-
-
-class TestMultipleSimulations(LifeGameBaseTest):
-    def test_cross(self):
-        str_representations = [
-            """
-            0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 1 0 0 1 0 0 0 0
-            0 1 1 1 0 0 1 1 1 0 0
-            0 1 0 0 0 0 0 0 1 0 0
-            0 1 0 0 0 0 0 0 1 0 0
-            0 1 1 1 0 0 1 1 1 0 0
-            0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0
-        """,
-            """
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 0 1 1 0 0 0 0 0
-            0 0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 1 0 1 0 0 1 0 1 0 0
-            0 1 1 0 0 0 0 0 0 1 1 0
-            0 1 1 0 0 0 0 0 0 1 1 0
-            0 0 1 0 1 0 0 1 0 1 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 0 0 1 1 0 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-        """,
-            """
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 1 1 0 0 1 1 0 0 0
-            0 1 1 1 0 0 0 0 1 1 1 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 1 1 1 0 0 0 0 1 1 1 0
-            0 0 0 1 1 0 0 1 1 0 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-        """,
-            """
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 1 1 1 0 0 1 1 1 0 0
-            0 0 1 0 0 0 0 0 0 1 0 0
-            0 0 1 0 0 0 0 0 0 1 0 0
-            0 0 1 1 1 0 0 1 1 1 0 0
-            0 0 0 0 1 0 0 1 0 0 0 0
-            0 0 0 0 1 1 1 1 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-            0 0 0 0 0 0 0 0 0 0 0 0
-        """,
-        ]
-
-        simulation = XYZSimulation(Config())
-        subjects = get_subjects_from_str_representation(
-            str_representations[0],
-            simulation,
-        )
-        simulation.subjects = subjects
-
-        cycle_manager = CycleManager(
-            config=Config(),
-            logger=SynergineLogger('test'),
-            simulation=simulation,
-        )
-
-        for str_representation in str_representations:
-            assert str_representation == \
-               self._get_str_representation_of_subjects(
-                    subjects,
-               )
-            cycle_manager.next()

tests/test_processing.py

@@ -24,6 +24,8 @@ class TestProcessing(BaseTest):
     @pytest.mark.timeout(10)
     def make_job_with_scalar(
             self,
+            worker_id: int,
+            process_count: int,
             data: list,
     ):
         result = sum(data)
@@ -32,6 +34,8 @@ class TestProcessing(BaseTest):
     @pytest.mark.timeout(10)
     def make_job_with_object(
             self,
+            worker_id: int,
+            process_count: int,
             data: list,
     ):
         data = [o.value for o in data]
@@ -149,7 +153,7 @@ class TestProcessing(BaseTest):
 
         shared.set('foo_1', 0)
 
-        def job(key):
+        def job(worker_id, processes_count, key):
             shared.refresh()
             value = shared.get('foo_{}'.format(key)) or 0
             return value + 1

tests/test_share.py

@@ -1,4 +1,6 @@
 # coding: utf-8
+import pickle
+
 import pytest
 
 from synergine2.exceptions import UnknownSharedData
@@ -99,6 +101,38 @@ class TestShare(BaseTest):
         foo.data['foo'] = 'buz'
         assert shared.get('data') == {'foo': 'buz'}
 
+        shared.commit()
+        assert shared.get('data') == {'foo': 'buz'}
+        assert pickle.loads(shared._r.get('data')) == {'foo': 'buz'}
+
+        foo.data['foo'] = 'bAz'
+        shared.commit()
+        assert shared.get('data') == {'foo': 'bAz'}
+        assert pickle.loads(shared._r.get('data')) == {'foo': 'bAz'}
+
+    def test_update_list_with_pointer(self):
+        shared = share.SharedDataManager()
+
+        class Foo(object):
+            data = shared.create('data', [])
+
+        foo = Foo()
+        foo.data = ['foo']
+
+        assert shared.get('data') == ['foo']
+
+        foo.data.append('bar')
+        assert shared.get('data') == ['foo', 'bar']
+
+        shared.commit()
+        assert shared.get('data') == ['foo', 'bar']
+        assert pickle.loads(shared._r.get('data')) == ['foo', 'bar']
+
+        foo.data.append('bAr')
+        shared.commit()
+        assert shared.get('data') == ['foo', 'bar', 'bAr']
+        assert pickle.loads(shared._r.get('data')) == ['foo', 'bar', 'bAr']
+
     def test_refresh_without_commit(self):
         shared = share.SharedDataManager()