ResourcesManager.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. import getpass
  2. import logging
  3. import os
  4. import shutil
  5. import re
  6. import tempfile
  7. from git import Repo
  8. from packaging import version
  9. from kalliope.core.ConfigurationManager import SettingLoader
  10. from kalliope.core.ConfigurationManager.DnaLoader import DnaLoader
  11. from kalliope.core.Models import Neuron
  12. from kalliope.core.NeuronLauncher import NeuronLauncher
  13. from kalliope.core.Utils import Utils
  14. logging.basicConfig()
  15. logger = logging.getLogger("kalliope")
  16. # Global values for processing:
  17. # LOCAL_TMP_FOLDER = "/tmp/kalliope/resources/"
  18. TMP_GIT_FOLDER = "kalliope_new_module_temp_name"
  19. DNA_FILE_NAME = "dna.yml"
  20. INSTALL_FILE_NAME = "install.yml"
  21. # Global values for required parameters in DNA:
  22. DNA_NAME = "name"
  23. DNA_TYPE = "type"
  24. # Global_Names for 'types' to match:
  25. TYPE_NEURON = "neuron"
  26. TYPE_TTS = "tts"
  27. TYPE_STT = "stt"
  28. TYPE_TRIGGER = "trigger"
  29. class ResourcesManagerException(Exception):
  30. pass
  31. class ResourcesManager(object):
  32. def __init__(self, **kwargs):
  33. """
  34. This class is used to manage community resources.
  35. :param kwargs:
  36. git-url: the url of the module to clone and install
  37. """
  38. super(ResourcesManager, self).__init__()
  39. # get settings
  40. sl = SettingLoader()
  41. self.settings = sl.settings
  42. # in case of update or install, url where
  43. self.git_url = kwargs.get('git_url', None)
  44. # temp path where we install the new module
  45. self.tmp_path = tempfile.gettempdir() + "/kalliope/resources/" +\
  46. TMP_GIT_FOLDER
  47. self.dna_file_path = self.tmp_path + os.sep + DNA_FILE_NAME
  48. self.install_file_path = self.tmp_path + os.sep + INSTALL_FILE_NAME
  49. self.dna = None
  50. def install(self):
  51. """
  52. Module installation method.
  53. """
  54. # first, we clone the repo
  55. self._clone_repo(path=self.tmp_path,
  56. git_url=self.git_url)
  57. # check the content of the cloned repo
  58. if self.is_repo_ok(dna_file_path=self.dna_file_path,
  59. install_file_path=self.install_file_path):
  60. # Load the dna.yml file
  61. self.dna = DnaLoader(self.dna_file_path).get_dna()
  62. if self.dna is not None:
  63. logger.debug("[ResourcesManager] DNA file content: " + str(self.dna))
  64. if self.is_settings_ok(resources=self.settings.resources, dna=self.dna):
  65. # the dna file is ok, check the supported version
  66. if self._check_supported_version(current_version=self.settings.kalliope_version,
  67. supported_versions=self.dna.kalliope_supported_version):
  68. # Let's find the target folder depending the type
  69. module_type = self.dna.module_type.lower()
  70. target_folder = self._get_target_folder(resources=self.settings.resources,
  71. module_type=module_type)
  72. if target_folder is not None:
  73. # let's move the tmp folder in the right folder and get a new path for the module
  74. module_name = self.dna.name.lower()
  75. target_path = self._rename_temp_folder(name=self.dna.name.lower(),
  76. target_folder=target_folder,
  77. tmp_path=self.tmp_path)
  78. # if the target_path exists, then run the install file within the new repository
  79. if target_path is not None:
  80. self.install_file_path = target_path + os.sep + INSTALL_FILE_NAME
  81. self.run_ansible_playbook_module(install_file_path=self.install_file_path)
  82. Utils.print_success("Module: %s installed" % module_name)
  83. else:
  84. logger.debug("[ResourcesManager] installation cancelled, deleting temp repo %s"
  85. % str(self.tmp_path))
  86. shutil.rmtree(self.tmp_path)
  87. @staticmethod
  88. def is_settings_ok(resources, dna):
  89. """
  90. Test if required settings files in config of Kalliope are ok.
  91. The resource object must not be empty
  92. Check id the use have set the an installation path in his settings for the target module type
  93. :param resources: the Resources model
  94. :param dna: DNA info about the module to install
  95. :return:
  96. """
  97. settings_ok = True
  98. if resources is None:
  99. message = "Resources folder not set in settings, cannot install."
  100. logger.debug(message)
  101. Utils.print_danger(message)
  102. settings_ok = False
  103. else:
  104. if dna.module_type == "neuron" and resources.neuron_folder is None:
  105. message = "Resources folder for neuron installation not set in settings, cannot install."
  106. logger.debug(message)
  107. Utils.print_danger(message)
  108. settings_ok = False
  109. if dna.module_type == "stt" and resources.stt_folder is None:
  110. message = "Resources folder for stt installation not set in settings, cannot install."
  111. logger.debug(message)
  112. Utils.print_danger(message)
  113. settings_ok = False
  114. if dna.module_type == "tts" and resources.tts_folder is None:
  115. message = "Resources folder for tts installation not set in settings, cannot install."
  116. logger.debug(message)
  117. Utils.print_danger(message)
  118. settings_ok = False
  119. if dna.module_type == "trigger" and resources.trigger_folder is None:
  120. message = "Resources folder for trigger installation not set in settings, cannot install."
  121. logger.debug(message)
  122. Utils.print_danger(message)
  123. settings_ok = False
  124. return settings_ok
  125. @staticmethod
  126. def is_repo_ok(dna_file_path, install_file_path):
  127. """
  128. Check if the git cloned repo is fine to be installed
  129. :return: True if repo is ok to be installed, False otherwise
  130. """
  131. Utils.print_info("Checking repository...")
  132. repo_ok = True
  133. # check that a install.yml file is present
  134. if not os.path.exists(install_file_path):
  135. Utils.print_danger("Missing %s file" % INSTALL_FILE_NAME)
  136. repo_ok = False
  137. if not os.path.exists(dna_file_path):
  138. Utils.print_danger("Missing %s file" % DNA_FILE_NAME)
  139. repo_ok = False
  140. return repo_ok
  141. @staticmethod
  142. def _get_target_folder(resources, module_type):
  143. """
  144. Return the folder from the resources and given a module type
  145. :param resources: Resource object
  146. :type resources: Resources
  147. :param module_type: type of the module
  148. :return: path of the folder
  149. """
  150. # dict to get the path behind a type of resource
  151. module_type_converter = {
  152. TYPE_NEURON: resources.neuron_folder,
  153. TYPE_STT: resources.stt_folder,
  154. TYPE_TTS: resources.tts_folder,
  155. TYPE_TRIGGER: resources.trigger_folder
  156. }
  157. # Let's find the right path depending of the type
  158. try:
  159. folder_path = module_type_converter[module_type]
  160. except KeyError:
  161. folder_path = None
  162. # No folder_path has been found
  163. message = "No %s folder set in settings, cannot install." % module_type
  164. if folder_path is None:
  165. logger.debug(message)
  166. Utils.print_danger(message)
  167. return folder_path
  168. @staticmethod
  169. def _clone_repo(path, git_url):
  170. """
  171. Use git to clone locally the neuron in a temp folder
  172. :return:
  173. """
  174. # clone the repo
  175. logger.debug("[ResourcesManager] GIT clone into folder: %s" % path)
  176. Utils.print_info("Cloning repository...")
  177. # if the folder already exist we remove it
  178. if os.path.exists(path):
  179. shutil.rmtree(path)
  180. else:
  181. os.makedirs(path)
  182. Repo.clone_from(git_url, path)
  183. @staticmethod
  184. def _rename_temp_folder(name, target_folder, tmp_path):
  185. """
  186. Rename the temp folder of the cloned repo
  187. Return the name of the path to install
  188. :return: path to install, None if already exists
  189. """
  190. logger.debug("[ResourcesManager] Rename temp folder")
  191. new_absolute_neuron_path = target_folder + os.sep + name
  192. try:
  193. shutil.move(tmp_path, new_absolute_neuron_path)
  194. return new_absolute_neuron_path
  195. except OSError:
  196. # the folder already exist
  197. Utils.print_warning("The module %s already exist in the path %s" % (name, target_folder))
  198. # remove the cloned repo
  199. logger.debug("[ResourcesManager] Deleting temp folder %s" % str(tmp_path))
  200. shutil.rmtree(tmp_path)
  201. @staticmethod
  202. def run_ansible_playbook_module(install_file_path):
  203. """
  204. Run the install.yml file through an Ansible playbook using the dedicated neuron !
  205. :param install_file_path: the path of the Ansible playbook to run.
  206. :return:
  207. """
  208. logger.debug("[ResourcesManager] Run ansible playbook")
  209. Utils.print_info("Starting neuron installation")
  210. # ask the sudo password
  211. pswd = getpass.getpass('Sudo password:')
  212. ansible_neuron_parameters = {
  213. "task_file": install_file_path,
  214. "sudo": True,
  215. "sudo_user": "root",
  216. "sudo_password": pswd
  217. }
  218. neuron = Neuron(name="ansible_playbook", parameters=ansible_neuron_parameters)
  219. NeuronLauncher.start_neuron(neuron)
  220. @staticmethod
  221. def _check_supported_version(current_version, supported_versions):
  222. """
  223. The dna file contains supported Kalliope version for the module to install.
  224. Check if supported versions are match the current installed version. If not, ask the user to confirm the
  225. installation anyway
  226. :param current_version: current version installed of Kalliope. E.g 0.4.0
  227. :param supported_versions: list of supported version
  228. :return: True if the version is supported or user has confirmed the installation
  229. """
  230. logger.debug("[ResourcesManager] Current installed version of Kalliope: %s" % str(current_version))
  231. logger.debug("[ResourcesManager] Module supported version: %s" % str(supported_versions))
  232. supported_version_found = False
  233. # Extract major version
  234. match_current_version = re.search('^[\d]*[.][\d]*', current_version)
  235. if match_current_version:
  236. current_version = match_current_version.group(0)
  237. for supported_version in supported_versions:
  238. if version.parse(str(current_version)) == version.parse(str(supported_version)):
  239. # we found the exact version
  240. supported_version_found = True
  241. break
  242. if not supported_version_found:
  243. # we ask the user if we want to install the module even if the version doesn't match
  244. Utils.print_info("Current installed version of Kalliope: %s" % current_version)
  245. Utils.print_info("Module supported versions: %s" % str(supported_versions))
  246. Utils.print_warning("The neuron seems to be not supported by your current version of Kalliope")
  247. supported_version_found = Utils.query_yes_no("install it anyway?")
  248. logger.debug("[ResourcesManager] install it anyway user answer: %s" % supported_version_found)
  249. logger.debug("[ResourcesManager] check_supported_version: %s" % str(supported_version_found))
  250. return supported_version_found