Browse Source

Merge pull request #360 from kalliope-project/refacto_LIFO

refactor lifobuffer. now a singleton which is testable
Monf 7 years ago
parent
commit
320d8602c0

+ 12 - 13
Tests/test_lifo_buffer.py

@@ -335,19 +335,18 @@ class TestLIFOBuffer(unittest.TestCase):
                 self.lifo_buffer._process_neuron_list(matched_synapse=matched_synapse)
 
         # test with a neuron that want to add a synapse list to the LIFO
-        # TODO refactor the LIFO to be a syncable thread. This test cannot work because python split the memory
-        # self.lifo_buffer.clean()
-        # synapse = BrainLoader().brain.get_synapse_by_name("synapse6")
-        # order = "synapse6"
-        # matched_synapse = MatchedSynapse(matched_synapse=synapse,
-        #                                  user_order=order,
-        #                                  matched_order=order)
-        #
-        # self.lifo_buffer.set_api_call(True)
-        # self.lifo_buffer.set_answer("synapse 6 answer")
-        # with mock.patch("kalliope.core.TTS.TTSModule.generate_and_play"):
-        #     with self.assertRaises(SynapseListAddedToLIFO):
-        #         self.lifo_buffer._process_neuron_list(matched_synapse=matched_synapse)
+        self.lifo_buffer.clean()
+        synapse = BrainLoader().brain.get_synapse_by_name("synapse6")
+        order = "synapse6"
+        matched_synapse = MatchedSynapse(matched_synapse=synapse,
+                                         user_order=order,
+                                         matched_order=order)
+
+        self.lifo_buffer.set_api_call(True)
+        self.lifo_buffer.set_answer("synapse 6 answer")
+        with mock.patch("kalliope.core.TTS.TTSModule.generate_and_play"):
+            self.assertRaises(SynapseListAddedToLIFO,
+                              self.lifo_buffer._process_neuron_list(matched_synapse=matched_synapse))
 
 
 if __name__ == '__main__':

+ 27 - 13
Tests/test_synapse_launcher.py

@@ -3,7 +3,7 @@ import unittest
 import mock
 
 from kalliope.core import LIFOBuffer
-from kalliope.core.Models import Brain, Signal
+from kalliope.core.Models import Brain, Signal, Singleton
 from kalliope.core.Models.MatchedSynapse import MatchedSynapse
 from kalliope.core.Models.Settings import Settings
 from kalliope.core.SynapseLauncher import SynapseLauncher, SynapseNameNotFound
@@ -40,7 +40,7 @@ class TestSynapseLauncher(unittest.TestCase):
         self.settings_test = Settings(default_synapse="Synapse3")
 
         # clean the LiFO
-        LIFOBuffer.lifo_list = list()
+        Singleton._instances = dict()
 
     def test_start_synapse_by_name(self):
         # existing synapse in the brain
@@ -49,11 +49,12 @@ class TestSynapseLauncher(unittest.TestCase):
             SynapseLauncher.start_synapse_by_name("Synapse1", brain=self.brain_test)
             # we expect that the lifo has been loaded with the synapse to run
             expected_result = [[should_be_created_matched_synapse]]
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            lifo_buffer = LIFOBuffer()
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
 
             # we expect that the lifo has been loaded with the synapse to run and overwritten parameters
-            # clean the LiFO
-            LIFOBuffer.lifo_list = list()
+            Singleton._instances = dict()
+            lifo_buffer = LIFOBuffer()
             overriding_param = {
                 "val1": "val"
             }
@@ -63,7 +64,7 @@ class TestSynapseLauncher(unittest.TestCase):
                                                                overriding_parameter=overriding_param)
             # we expect that the lifo has been loaded with the synapse to run
             expected_result = [[should_be_created_matched_synapse]]
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
 
         # non existing synapse in the brain
         with self.assertRaises(SynapseNameNotFound):
@@ -84,13 +85,14 @@ class TestSynapseLauncher(unittest.TestCase):
                                                             brain=self.brain_test,
                                                             settings=self.settings_test)
 
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            lifo_buffer = LIFOBuffer()
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
 
         # -------------------------
         # test_match_synapse1_and_2
         # -------------------------
         # clean LIFO
-        LIFOBuffer.lifo_list = list()
+        Singleton._instances = dict()
         with mock.patch("kalliope.core.LIFOBuffer.execute"):
             order_to_match = "this is the second sentence"
             should_be_created_matched_synapse1 = MatchedSynapse(matched_synapse=self.synapse1,
@@ -104,13 +106,14 @@ class TestSynapseLauncher(unittest.TestCase):
             SynapseLauncher.run_matching_synapse_from_order(order_to_match,
                                                             brain=self.brain_test,
                                                             settings=self.settings_test)
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            lifo_buffer = LIFOBuffer()
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
 
         # -------------------------
         # test_match_default_synapse
         # -------------------------
         # clean LIFO
-        LIFOBuffer.lifo_list = list()
+        Singleton._instances = dict()
         with mock.patch("kalliope.core.LIFOBuffer.execute"):
             order_to_match = "not existing sentence"
             should_be_created_matched_synapse = MatchedSynapse(matched_synapse=self.synapse3,
@@ -121,13 +124,14 @@ class TestSynapseLauncher(unittest.TestCase):
             SynapseLauncher.run_matching_synapse_from_order(order_to_match,
                                                             brain=self.brain_test,
                                                             settings=self.settings_test)
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            lifo_buffer = LIFOBuffer()
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
 
         # -------------------------
         # test_no_match_and_no_default_synapse
         # -------------------------
         # clean LIFO
-        LIFOBuffer.lifo_list = list()
+        Singleton._instances = dict()
         with mock.patch("kalliope.core.LIFOBuffer.execute"):
             order_to_match = "not existing sentence"
             new_settings = Settings()
@@ -135,4 +139,14 @@ class TestSynapseLauncher(unittest.TestCase):
             SynapseLauncher.run_matching_synapse_from_order(order_to_match,
                                                             brain=self.brain_test,
                                                             settings=new_settings)
-            self.assertEqual(expected_result, LIFOBuffer.lifo_list)
+            lifo_buffer = LIFOBuffer()
+            self.assertEqual(expected_result, lifo_buffer.lifo_list)
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+    # suite = unittest.TestSuite()
+    # suite.addTest(TestSynapseLauncher("test_run_matching_synapse_from_order"))
+    # runner = unittest.TextTestRunner()
+    # runner.run(suite)

+ 13 - 0
docker/clone_and_test_python3.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# install dev version
+git clone https://github.com/kalliope-project/kalliope.git kalliope;
+cd kalliope;
+git checkout dev;
+
+# install
+sudo python3 setup.py install
+
+# tests
+export LANG=C.UTF-8
+python3 -m unittest discover

+ 1 - 0
docker/ubuntu_16_04.dockerfile

@@ -21,6 +21,7 @@ RUN pip install --upgrade pip six
 RUN pip install --upgrade pip pyyaml
 RUN pip install --upgrade pip SpeechRecognition
 RUN pip install --upgrade pip Werkzeug
+RUN pip install --upgrade pip gitdb
 
 # add a standart user. tests must not be ran as root
 RUN useradd -m -u 1000 tester

+ 38 - 0
docker/ubuntu_16_04_python3.dockerfile

@@ -0,0 +1,38 @@
+# used to build the last kalliope dev version with python 3
+# docker build --force-rm=true -t kalliope-ubuntu1604-python3 -f docker/ubuntu_16_04_python3.dockerfile .
+# docker run -it --rm kalliope-ubuntu1604-python3
+
+FROM ubuntu:16.04
+
+ENV no_proxy="127.0.0.1,localhost,kalliope.fr"
+
+# set UTF-8 to the terminal
+ENV LANG en_US.UTF-8
+
+# pico2wav is a multiverse package
+RUN echo "deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse" >> /etc/apt/sources.list
+
+# install packages
+RUN apt-get update && apt-get install -y \
+    git python3-dev libsmpeg0 libttspico-utils libsmpeg0 flac dialog \
+    libffi-dev libffi-dev libssl-dev portaudio19-dev build-essential \
+    sox libatlas3-base mplayer wget vim sudo\
+    && rm -rf /var/lib/apt/lists/*
+
+# Install the last PIP
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python3 get-pip.py
+
+# add a standart user. tests must not be ran as root
+RUN useradd -m -u 1000 tester
+RUN usermod -aG sudo tester
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+ADD docker/clone_and_test_python3.sh /home/tester/clone_and_test_python3.sh
+RUN chown tester /home/tester/clone_and_test_python3.sh
+
+USER tester
+WORKDIR /home/tester
+
+# run tests
+CMD ./clone_and_test_python3.sh

+ 58 - 65
kalliope/core/LIFOBuffer.py

@@ -1,4 +1,5 @@
 import logging
+from six import with_metaclass
 
 from kalliope.core.Cortex import Cortex
 from kalliope.core.NeuronLauncher import NeuronLauncher
@@ -24,7 +25,7 @@ class SynapseListAddedToLIFO(Exception):
     pass
 
 
-class LIFOBuffer(object):
+class LIFOBuffer(with_metaclass(Singleton, object)):
     """
     This class is a LIFO list of synapse to process where the last synapse list to enter will be the first synapse
     list to be processed.
@@ -33,27 +34,24 @@ class LIFOBuffer(object):
     like with the Neurotransmitter neuron.
     
     """
-    __metaclass__ = Singleton
-
-    api_response = APIResponse()
-    lifo_list = list()
-    logger.debug("[LIFOBuffer] LIFO buffer created")
-    answer = None
-    is_api_call = False
-    no_voice = False
-    is_running = False
-    reset_lifo = False
-
-    @classmethod
-    def set_answer(cls, value):
-        cls.answer = value
-
-    @classmethod
-    def set_api_call(cls, value):
-        cls.is_api_call = value
-
-    @classmethod
-    def add_synapse_list_to_lifo(cls, matched_synapse_list, high_priority=False):
+
+    def __init__(self):
+        logger.debug("[LIFOBuffer] LIFO buffer created")
+        self.api_response = APIResponse()
+        self.lifo_list = list()
+        self.answer = None
+        self.is_api_call = False
+        self.no_voice = False
+        self.is_running = False
+        self.reset_lifo = False
+
+    def set_answer(self, value):
+        self.answer = value
+
+    def set_api_call(self, value):
+        self.is_api_call = value
+
+    def add_synapse_list_to_lifo(self, matched_synapse_list, high_priority=False):
         """
         Add a synapse list to process to the lifo
         :param matched_synapse_list: List of Matched Synapse
@@ -61,33 +59,30 @@ class LIFOBuffer(object):
         :return: 
         """
         logger.debug("[LIFOBuffer] Add a new synapse list to process to the LIFO")
-        cls.lifo_list.append(matched_synapse_list)
+        self.lifo_list.append(matched_synapse_list)
         if high_priority:
-            cls.reset_lifo = True
+            self.reset_lifo = True
 
-    @classmethod
-    def clean(cls):
+    def clean(self):
         """
         Clean the LIFO by creating a new list
         """
-        cls.lifo_list = list()
-        cls.api_response = APIResponse()
+        self.lifo_list = list()
+        self.api_response = APIResponse()
 
-    @classmethod
-    def _return_serialized_api_response(cls):
+    def _return_serialized_api_response(self):
         """
         Serialize Exception has been raised by the execute process somewhere, return the serialized API response
         to the caller. Clean up the APIResponse object for the next call
         :return: 
         """
         # we prepare a json response
-        returned_api_response = cls.api_response.serialize()
+        returned_api_response = self.api_response.serialize()
         # we clean up the API response object for the next call
-        cls.api_response = APIResponse()
+        self.api_response = APIResponse()
         return returned_api_response
 
-    @classmethod
-    def execute(cls, answer=None, is_api_call=False, no_voice=False):
+    def execute(self, answer=None, is_api_call=False, no_voice=False):
         """
         Process the LIFO list.
         
@@ -103,35 +98,34 @@ class LIFOBuffer(object):
         :return: serialized APIResponse object
         """
         # store the answer if present
-        cls.answer = answer
-        cls.is_api_call = is_api_call
-        cls.no_voice = no_voice
+        self.answer = answer
+        self.is_api_call = is_api_call
+        self.no_voice = no_voice
 
-        if not cls.is_running:
-            cls.is_running = True
+        if not self.is_running:
+            self.is_running = True
 
             try:
                 # we keep looping over the LIFO til we have synapse list to process in it
-                while cls.lifo_list:
-                    logger.debug("[LIFOBuffer] number of synapse list to process: %s" % len(cls.lifo_list))
+                while self.lifo_list:
+                    logger.debug("[LIFOBuffer] number of synapse list to process: %s" % len(self.lifo_list))
                     try:
                         # get the last list of matched synapse in the LIFO
-                        last_synapse_fifo_list = cls.lifo_list[-1]
-                        cls._process_synapse_list(last_synapse_fifo_list)
+                        last_synapse_fifo_list = self.lifo_list[-1]
+                        self._process_synapse_list(last_synapse_fifo_list)
                     except SynapseListAddedToLIFO:
                         continue
                     # remove the synapse list from the LIFO
-                    cls.lifo_list.remove(last_synapse_fifo_list)
+                    self.lifo_list.remove(last_synapse_fifo_list)
                     # clean the cortex from value loaded from order as all synapses have been processed
                     Cortex.clean_parameter_from_order()
-                cls.is_running = False
+                self.is_running = False
                 raise Serialize
 
             except Serialize:
-                return cls._return_serialized_api_response()
+                return self._return_serialized_api_response()
 
-    @classmethod
-    def _process_synapse_list(cls, synapse_list):
+    def _process_synapse_list(self, synapse_list):
         """
         Process a list of matched synapse.
         Execute each neuron list for each synapse.
@@ -145,16 +139,15 @@ class LIFOBuffer(object):
             matched_synapse = synapse_list[0]
             # add the synapse to the API response so the user will get the status if the synapse was not already
             # in the response
-            if matched_synapse not in cls.api_response.list_processed_matched_synapse:
-                cls.api_response.list_processed_matched_synapse.append(matched_synapse)
+            if matched_synapse not in self.api_response.list_processed_matched_synapse:
+                self.api_response.list_processed_matched_synapse.append(matched_synapse)
 
-            cls._process_neuron_list(matched_synapse=matched_synapse)
+            self._process_neuron_list(matched_synapse=matched_synapse)
 
             # The synapse has been processed we can remove it from the list.
             synapse_list.remove(matched_synapse)
 
-    @classmethod
-    def _process_neuron_list(cls, matched_synapse):
+    def _process_neuron_list(self, matched_synapse):
         """
         Process the neuron list of the matched_synapse
         Execute the Neuron
@@ -174,26 +167,26 @@ class LIFOBuffer(object):
             # get the first neuron in the FIFO neuron list
             neuron = matched_synapse.neuron_fifo_list[0]
             # from here, we are back into the last neuron we were processing.
-            if cls.answer is not None:  # we give the answer if exist to the first neuron
-                neuron.parameters["answer"] = cls.answer
+            if self.answer is not None:  # we give the answer if exist to the first neuron
+                neuron.parameters["answer"] = self.answer
                 # the next neuron should not get this answer
-                cls.answer = None
+                self.answer = None
             # todo fix this when we have a full client/server call. The client would be the voice or api call
-            neuron.parameters["is_api_call"] = cls.is_api_call
-            neuron.parameters["no_voice"] = cls.no_voice
-            logger.debug("[LIFOBuffer] process_neuron_list: is_api_call: %s, no_voice: %s" % (cls.is_api_call,
-                                                                                              cls.no_voice))
+            neuron.parameters["is_api_call"] = self.is_api_call
+            neuron.parameters["no_voice"] = self.no_voice
+            logger.debug("[LIFOBuffer] process_neuron_list: is_api_call: %s, no_voice: %s" % (self.is_api_call,
+                                                                                              self.no_voice))
             # execute the neuron
             instantiated_neuron = NeuronLauncher.start_neuron(neuron=neuron,
                                                               parameters_dict=matched_synapse.parameters)
 
             # the status of an execution is "complete" if no neuron are waiting for an answer
-            cls.api_response.status = "complete"
+            self.api_response.status = "complete"
             if instantiated_neuron is not None:
                 if instantiated_neuron.is_waiting_for_answer:  # the neuron is waiting for an answer
                     logger.debug("[LIFOBuffer] Wait for answer mode")
-                    cls.api_response.status = "waiting_for_answer"
-                    cls.is_running = False
+                    self.api_response.status = "waiting_for_answer"
+                    self.is_running = False
                     raise Serialize
                 else:
                     logger.debug("[LIFOBuffer] complete mode")
@@ -203,11 +196,11 @@ class LIFOBuffer(object):
                     # the neuron is fully processed we can remove it from the list
                     matched_synapse.neuron_fifo_list.remove(neuron)
 
-                if cls.reset_lifo:  # the last executed neuron want to run a synapse
+                if self.reset_lifo:  # the last executed neuron want to run a synapse
                     logger.debug("[LIFOBuffer] Last executed neuron want to run a synapse. Restart the LIFO")
                     # we have added a list of synapse to the LIFO ! this one must start over.
                     # break all while loop until the execution is back to the LIFO loop
-                    cls.reset_lifo = False
+                    self.reset_lifo = False
                     raise SynapseListAddedToLIFO
             else:
                 raise Serialize

+ 4 - 2
kalliope/core/NeuronModule.py

@@ -249,8 +249,10 @@ class NeuronModule(object):
 
         list_synapse_to_process = list()
         list_synapse_to_process.append(matched_synapse)
-        LIFOBuffer.add_synapse_list_to_lifo(list_synapse_to_process, high_priority=high_priority)
-        LIFOBuffer.execute(is_api_call=is_api_call)
+        # get the singleton
+        lifo_buffer = LIFOBuffer()
+        lifo_buffer.add_synapse_list_to_lifo(list_synapse_to_process, high_priority=high_priority)
+        lifo_buffer.execute(is_api_call=is_api_call)
 
     @staticmethod
     def is_order_matching(order_said, order_match):