XapianIndexer.class.php 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. <?php
  2. /* For licensing terms, see /license.txt */
  3. /**
  4. * @package chamilo.include.search
  5. */
  6. require_once 'xapian.php';
  7. require_once dirname(__FILE__) . '/../IndexableChunk.class.php';
  8. /**
  9. * Abstract helper class
  10. * @package chamilo.include.search
  11. */
  12. abstract class XapianIndexer
  13. {
  14. /* XapianWritableDatabase */
  15. protected $db;
  16. /* IndexableChunk[] */
  17. protected $chunks;
  18. /* XapianTermGenerator */
  19. public $indexer;
  20. /* XapianStem */
  21. public $stemmer;
  22. /**
  23. * Class contructor
  24. */
  25. function __construct()
  26. {
  27. $this->db = null;
  28. $this->stemmer = null;
  29. }
  30. /**
  31. * Generates a list of languages Xapian manages
  32. *
  33. * This method enables the definition of more matches between
  34. * Chamilo languages and Xapian languages (through hardcoding)
  35. * @return array Array of languages codes -> Xapian languages
  36. */
  37. public final function xapian_languages()
  38. {
  39. /* http://xapian.org/docs/apidoc/html/classXapian_1_1Stem.html */
  40. return array(
  41. 'none' => 'none', //don't stem terms
  42. 'da' => 'danish',
  43. 'nl' => 'dutch',
  44. /* Martin Porter's 2002 revision of his stemmer */
  45. 'en' => 'english',
  46. /* Lovin's stemmer */
  47. 'lovins' => 'english_lovins',
  48. /* Porter's stemmer as described in his 1980 paper */
  49. 'porter' => 'english_porter',
  50. 'fi' => 'finnish',
  51. 'fr' => 'french',
  52. 'de' => 'german',
  53. 'it' => 'italian',
  54. 'no' => 'norwegian',
  55. 'pt' => 'portuguese',
  56. 'ru' => 'russian',
  57. 'es' => 'spanish',
  58. 'sv' => 'swedish',
  59. );
  60. }
  61. /**
  62. * Connect to the database, and create it if it doesn't exist
  63. */
  64. function connectDb($path = null, $dbMode = null, $lang = 'english')
  65. {
  66. if ($this->db != null)
  67. return $this->db;
  68. if ($dbMode == null)
  69. $dbMode = Xapian::DB_CREATE_OR_OPEN;
  70. if ($path == null)
  71. $path = api_get_path(SYS_UPLOAD_PATH).'plugins/xapian/searchdb/';
  72. try {
  73. $this->db = new XapianWritableDatabase($path, $dbMode);
  74. $this->indexer = new XapianTermGenerator();
  75. if (!in_array($lang, $this->xapian_languages())) {
  76. $lang = 'english';
  77. }
  78. $this->stemmer = new XapianStem($lang);
  79. $this->indexer->set_stemmer($this->stemmer);
  80. return $this->db;
  81. } catch (Exception $e) {
  82. Display::display_error_message($e->getMessage());
  83. return 1;
  84. }
  85. }
  86. /**
  87. * Simple getter for the db attribute
  88. * @return object The db attribute
  89. */
  90. function getDb()
  91. {
  92. return $this->db;
  93. }
  94. /**
  95. * Add this chunk to the chunk array attribute
  96. * @param string Chunk of text
  97. * @return void
  98. */
  99. function addChunk($chunk)
  100. {
  101. $this->chunks[] = $chunk;
  102. }
  103. /**
  104. * Actually index the current data
  105. *
  106. * @return integer New Xapian document ID or null upon failure
  107. */
  108. function index()
  109. {
  110. try {
  111. if (!empty($this->chunks)) {
  112. foreach ($this->chunks as $chunk) {
  113. $doc = new XapianDocument();
  114. $this->indexer->set_document($doc);
  115. if (!empty($chunk->terms)) {
  116. foreach ($chunk->terms as $term) {
  117. /* FIXME: think of getting weight */
  118. $doc->add_term($term['flag'] . $term['name'], 1);
  119. }
  120. }
  121. // free-form index all data array (title, content, etc)
  122. if (!empty($chunk->data)) {
  123. foreach ($chunk->data as $key => $value) {
  124. $this->indexer->index_text($value, 1);
  125. }
  126. }
  127. $doc->set_data($chunk->xapian_data, 1);
  128. $did = $this->db->add_document($doc);
  129. //write to disk
  130. $this->db->flush();
  131. return $did;
  132. }
  133. }
  134. } catch (Exception $e) {
  135. Display::display_error_message($e->getMessage());
  136. exit(1);
  137. }
  138. }
  139. /**
  140. * Get a specific document from xapian db
  141. *
  142. * @param int did Xapian::docid
  143. * @return mixed XapianDocument, or false on error
  144. */
  145. function get_document($did)
  146. {
  147. if ($this->db == null) {
  148. $this->connectDb();
  149. }
  150. try {
  151. $docid = $this->db->get_document($did);
  152. } catch (Exception $e) {
  153. //Display::display_error_message($e->getMessage());
  154. return false;
  155. }
  156. return $docid;
  157. }
  158. /**
  159. * Get document data on a xapian document
  160. *
  161. * @param XapianDocument $doc xapian document to push into the db
  162. * @return mixed xapian document data or FALSE if error
  163. */
  164. function get_document_data($doc)
  165. {
  166. if ($this->db == null) {
  167. $this->connectDb();
  168. }
  169. try {
  170. if (!is_a($doc, 'XapianDocument')) {
  171. return FALSE;
  172. }
  173. $doc_data = $doc->get_data();
  174. return $doc_data;
  175. } catch (Exception $e) {
  176. //Display::display_error_message($e->getMessage());
  177. return false;
  178. }
  179. }
  180. /**
  181. * Replace all terms of a document in xapian db
  182. *
  183. * @param int $did Xapian::docid
  184. * @param array $terms New terms of the document
  185. * @param string $prefix Prefix used to categorize the doc (usually 'T' for title, 'A' for author)
  186. * @return boolean false on error
  187. */
  188. function update_terms($did, $terms, $prefix)
  189. {
  190. $doc = $this->get_document($did);
  191. if ($doc === false) {
  192. return false;
  193. }
  194. $doc->clear_terms();
  195. foreach ($terms as $term) {
  196. //add directly
  197. $doc->add_term($prefix . $term, 1);
  198. }
  199. $this->db->replace_document($did, $doc);
  200. $this->db->flush();
  201. return true;
  202. }
  203. /**
  204. * Remove a document from xapian db
  205. *
  206. * @param int did Xapian::docid
  207. */
  208. function remove_document($did)
  209. {
  210. if ($this->db == null) {
  211. $this->connectDb();
  212. }
  213. if (is_numeric($did) && $did > 0) {
  214. $doc = $this->get_document($did);
  215. if ($doc !== FALSE) {
  216. $this->db->delete_document($did);
  217. $this->db->flush();
  218. }
  219. }
  220. }
  221. /**
  222. * Adds a term to the document specified
  223. *
  224. * @param string $term The term to add
  225. * @param XapianDocument $doc The xapian document where to add the term
  226. * @return mixed XapianDocument, or false on error
  227. */
  228. function add_term_to_doc($term, $doc)
  229. {
  230. if (!is_a($doc, 'XapianDocument')) {
  231. return FALSE;
  232. }
  233. try {
  234. $doc->add_term($term);
  235. } catch (Exception $e) {
  236. Display::display_error_message($e->getMessage());
  237. return 1;
  238. }
  239. }
  240. /**
  241. * Remove a term from the document specified
  242. *
  243. * @param string $term The term to add
  244. * @param XapianDocument $doc The xapian document where to add the term
  245. * @return mixed XapianDocument, or false on error
  246. */
  247. function remove_term_from_doc($term, $doc)
  248. {
  249. if (!is_a($doc, 'XapianDocument')) {
  250. return FALSE;
  251. }
  252. try {
  253. $doc->remove_term($term);
  254. } catch (Exception $e) {
  255. Display::display_error_message($e->getMessage());
  256. return 1;
  257. }
  258. }
  259. /**
  260. * Replace a document in the actual db
  261. *
  262. * @param XapianDocument $doc xapian document to push into the db
  263. * @param Xapian::docid $did xapian document id of the document to replace
  264. */
  265. function replace_document($doc, $did)
  266. {
  267. if (!is_a($doc, 'XapianDocument')) {
  268. return FALSE;
  269. }
  270. if ($this->db == null) {
  271. $this->connectDb();
  272. }
  273. try {
  274. $this->getDb()->replace_document((int) $did, $doc);
  275. $this->getDb()->flush();
  276. } catch (Exception $e) {
  277. Display::display_error_message($e->getMessage());
  278. return 1;
  279. }
  280. }
  281. /**
  282. * Class destructor
  283. */
  284. function __destruct()
  285. {
  286. unset($this->db);
  287. unset($this->stemmer);
  288. }
  289. }