XapianIndexer.class.php 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. <?php
  2. /* For licensing terms, see /license.txt */
  3. /**
  4. * @package chamilo.include.search
  5. */
  6. /**
  7. * Code
  8. */
  9. require_once 'xapian.php';
  10. require_once dirname(__FILE__).'/../IndexableChunk.class.php';
  11. /**
  12. * Abstract helper class
  13. * @package chamilo.include.search
  14. */
  15. abstract class XapianIndexer
  16. {
  17. /* XapianWritableDatabase */
  18. protected $db;
  19. /* IndexableChunk[] */
  20. protected $chunks;
  21. /* XapianTermGenerator */
  22. public $indexer;
  23. /* XapianStem */
  24. public $stemmer;
  25. /**
  26. * Generates a list of languages Xapian manages
  27. *
  28. * This method enables the definition of more matches between
  29. * Chamilo languages and Xapian languages (through hardcoding)
  30. * @return array Array of languages codes -> Xapian languages
  31. */
  32. public final function xapian_languages()
  33. {
  34. /* http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html */
  35. return array(
  36. 'none' => 'none', //don't stem terms
  37. 'da' => 'danish',
  38. 'nl' => 'dutch',
  39. /* Martin Porter's 2002 revision of his stemmer */
  40. 'en' => 'english',
  41. /* Lovin's stemmer */
  42. 'lovins' => 'english_lovins',
  43. /* Porter's stemmer as described in his 1980 paper */
  44. 'porter' => 'english_porter',
  45. 'fi' => 'finnish',
  46. 'fr' => 'french',
  47. 'de' => 'german',
  48. 'it' => 'italian',
  49. 'no' => 'norwegian',
  50. 'pt' => 'portuguese',
  51. 'ru' => 'russian',
  52. 'es' => 'spanish',
  53. 'sv' => 'swedish',
  54. );
  55. }
  56. /**
  57. * Connect to the database, and create it if it doesn't exist
  58. */
  59. function connectDb($path = null, $dbMode = null, $lang = 'english')
  60. {
  61. if ($this->db != null) {
  62. return $this->db;
  63. }
  64. if ($dbMode == null) {
  65. $dbMode = Xapian::DB_CREATE_OR_OPEN;
  66. }
  67. if ($path == null) {
  68. $path = api_get_path(SYS_DATA_PATH).'searchdb/';
  69. }
  70. try {
  71. $this->db = new XapianWritableDatabase($path, $dbMode);
  72. $this->indexer = new XapianTermGenerator();
  73. if (!in_array($lang, $this->xapian_languages())) {
  74. $lang = 'english';
  75. }
  76. $this->stemmer = new XapianStem($lang);
  77. $this->indexer->set_stemmer($this->stemmer);
  78. return $this->db;
  79. } catch (Exception $e) {
  80. Display::display_error_message($e->getMessage());
  81. return 1;
  82. }
  83. }
  84. /**
  85. * Simple getter for the db attribute
  86. * @return object The db attribute
  87. */
  88. function getDb()
  89. {
  90. return $this->db;
  91. }
  92. /**
  93. * Add this chunk to the chunk array attribute
  94. * @param string Chunk of text
  95. * @return void
  96. */
  97. function addChunk($chunk)
  98. {
  99. $this->chunks[] = $chunk;
  100. }
  101. /**
  102. * Actually index the current data
  103. *
  104. * @return integer New Xapian document ID or NULL upon failure
  105. */
  106. function index()
  107. {
  108. try {
  109. if (!empty($this->chunks)) {
  110. foreach ($this->chunks as $chunk) {
  111. $doc = new XapianDocument();
  112. $this->indexer->set_document($doc);
  113. if (!empty($chunk->terms)) {
  114. foreach ($chunk->terms as $term) {
  115. /* FIXME: think of getting weight */
  116. $doc->add_term($term['flag'].$term['name'], 1);
  117. }
  118. }
  119. // free-form index all data array (title, content, etc)
  120. if (!empty($chunk->data)) {
  121. foreach ($chunk->data as $key => $value) {
  122. $this->indexer->index_text($value, 1);
  123. }
  124. }
  125. $doc->set_data($chunk->xapian_data, 1);
  126. // $did = $this->db->FileManager::add_document($doc);
  127. //write to disk
  128. $this->db->flush();
  129. return $did;
  130. }
  131. }
  132. } catch (Exception $e) {
  133. Display::display_error_message($e->getMessage());
  134. exit(1);
  135. }
  136. }
  137. /**
  138. * Get a specific document from xapian db
  139. *
  140. * @param int did Xapian::docid
  141. * @return mixed XapianDocument, or false on error
  142. */
  143. function get_document($did)
  144. {
  145. if ($this->db == null) {
  146. $this->connectDb();
  147. }
  148. try {
  149. $docid = $this->db->get_document($did);
  150. } catch (Exception $e) {
  151. //Display::display_error_message($e->getMessage());
  152. return false;
  153. }
  154. return $docid;
  155. }
  156. /**
  157. * Get document data on a xapian document
  158. *
  159. * @param XapianDocument $doc xapian document to push into the db
  160. * @return mixed xapian document data or FALSE if error
  161. */
  162. function get_document_data($doc)
  163. {
  164. if ($this->db == null) {
  165. $this->connectDb();
  166. }
  167. try {
  168. if (!is_a($doc, 'XapianDocument')) {
  169. return false;
  170. }
  171. $doc_data = $doc->get_data();
  172. return $doc_data;
  173. } catch (Exception $e) {
  174. //Display::display_error_message($e->getMessage());
  175. return false;
  176. }
  177. }
  178. /**
  179. * Replace all terms of a document in xapian db
  180. *
  181. * @param int did Xapian::docid
  182. * @param array terms New terms of the document
  183. * @return boolean false on error
  184. */
  185. function update_terms($did, $terms, $prefix)
  186. {
  187. $doc = $this->get_document($did);
  188. if ($doc === false) {
  189. return false;
  190. }
  191. $doc->clear_terms();
  192. foreach ($terms as $term) {
  193. //add directly
  194. $doc->add_term($prefix.$term, 1);
  195. }
  196. $this->db->replace_document($did, $doc);
  197. $this->db->flush();
  198. return true;
  199. }
  200. /**
  201. * Remove a document from xapian db
  202. *
  203. * @param int did Xapian::docid
  204. */
  205. function remove_document($did)
  206. {
  207. if ($this->db == null) {
  208. $this->connectDb();
  209. }
  210. if (is_numeric($did) && $did > 0) {
  211. $doc = $this->get_document($did);
  212. if ($doc !== false) {
  213. $this->db->delete_document($did);
  214. $this->db->flush();
  215. }
  216. }
  217. }
  218. /**
  219. * Adds a term to the document specified
  220. *
  221. * @param string $term The term to add
  222. * @param XapianDocument $doc The xapian document where to add the term
  223. * @return mixed XapianDocument, or false on error
  224. */
  225. function add_term_to_doc($term, $doc)
  226. {
  227. if (!is_a($doc, 'XapianDocument')) {
  228. return false;
  229. }
  230. try {
  231. $doc->add_term($term);
  232. } catch (Exception $e) {
  233. Display::display_error_message($e->getMessage());
  234. return 1;
  235. }
  236. }
  237. /**
  238. * Remove a term from the document specified
  239. *
  240. * @param string $term The term to add
  241. * @param XapianDocument $doc The xapian document where to add the term
  242. * @return mixed XapianDocument, or false on error
  243. */
  244. function remove_term_from_doc($term, $doc)
  245. {
  246. if (!is_a($doc, 'XapianDocument')) {
  247. return false;
  248. }
  249. try {
  250. $doc->remove_term($term);
  251. } catch (Exception $e) {
  252. Display::display_error_message($e->getMessage());
  253. return 1;
  254. }
  255. }
  256. /**
  257. * Replace a document in the actual db
  258. *
  259. * @param XapianDocument $doc xapian document to push into the db
  260. * @param Xapian::docid $did xapian document id of the document to replace
  261. */
  262. function replace_document($doc, $did)
  263. {
  264. if (!is_a($doc, 'XapianDocument')) {
  265. return false;
  266. }
  267. if ($this->db == null) {
  268. $this->connectDb();
  269. }
  270. try {
  271. $this->getDb()->replace_document((int) $did, $doc);
  272. $this->getDb()->flush();
  273. } catch (Exception $e) {
  274. Display::display_error_message($e->getMessage());
  275. return 1;
  276. }
  277. }
  278. /**
  279. * Class contructor
  280. */
  281. public function __construct()
  282. {
  283. $this->db = null;
  284. $this->stemmer = null;
  285. }
  286. /**
  287. * Class destructor
  288. */
  289. public function __destruct()
  290. {
  291. unset($this->db);
  292. unset($this->stemmer);
  293. }
  294. }