md_phpdig.php 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. <?php /* <!-- Dokeos metadata/md_phpdig.php -->
  2. <!-- 2005/03/24 -->
  3. <!-- Copyright (C) 2005 rene.haentjens@UGent.be - see metadata/md_funcs.php -->
  4. */
  5. /**
  6. ==============================================================================
  7. * Dokeos Metadata: PhpDig connection
  8. *
  9. * If PhpDig 1.8.3 is installed in a Dokeos course site, then MD items
  10. * can be indexed for search (via PhpDig's search screen search.php).
  11. *
  12. * The functions below inject the words of metadata/indexabletext directly
  13. * into PhpDig's tables. Affected tables:
  14. *
  15. * keywords: key_id, twoletters, keyword (lowercase, accents removed)
  16. *
  17. * sites: site_id, site_url (e.g. http://xx.yy.zz/), upddate, ...
  18. *
  19. * spider: spider_id, site_id, upddate, num_words, first_words,
  20. * path (e.g. uu/vv/ww/), file (e.g. index.php?sid=xxx), ...
  21. *
  22. * engine: spider_id, key_id, weight
  23. *
  24. * Most of the function code is a simplified version of real PhpDig code
  25. * released under the GNU GPL V2, see www.phpdig.net.
  26. *
  27. * @package dokeos.metadata
  28. ==============================================================================
  29. */
  30. // PHPDIG CONNECTION ---------------------------------------------------------->
  31. $phpDigInc = get_course_path() . $_course['path'] . '/phpdig-1.8.6/includes/';
  32. $phpDigIncCn = $phpDigInc. 'connect.php'; // to connect to PhpDig's database
  33. $phpDigIncCw = $phpDigInc. 'common_words.txt'; // stopwords
  34. // if (!file_exists($phpDigIncCn)) return(); doesn't seem to work properly...
  35. if (file_exists($phpDigIncCw))
  36. if (is_array($lines = @file($phpDigIncCw)))
  37. while (list($id,$word) = each($lines))
  38. $common_words[trim($word)] = 1;
  39. define('SUMMARY_DISPLAY_LENGTH', 700);
  40. //define('PHPDIG_ENCODING', 'iso-8859-1');
  41. define('PHPDIG_ENCODING', strtolower($charset));
  42. define('SMALL_WORDS_SIZE', 2);
  43. define('MAX_WORDS_SIZE',50);
  44. define('WORDS_CHARS_LATIN1', '[:alnum:]��ߵ');
  45. foreach (array( 'A'=>'������', 'a'=>'������', 'O'=>'������', 'o'=>'������',
  46. 'E'=>'����', 'e'=>'����', 'C'=>'�', 'c'=>'�', 'I'=>'����',
  47. 'i'=>'����', 'U'=>'����', 'u'=>'����', 'Y'=>'�', 'y'=>'��',
  48. 'N'=>'�', 'n'=>'�') as $without => $allwith)
  49. foreach (explode('!', chunk_split($allwith, 1, '!')) as $with)
  50. if ($with) // because last one will be empty!
  51. {
  52. $letterswithout .= $without; $letterswith .= $with;
  53. }
  54. define('LETTERS_WITH_ACCENTS', $letterswith);
  55. define('SAME_WITHOUT_ACCENTS', $letterswithout);
  56. (strlen(LETTERS_WITH_ACCENTS) == strlen(SAME_WITHOUT_ACCENTS))
  57. or give_up('LETTERS_WITH_ACCENTS problem in md_phpdig.php');
  58. function find_site($url)
  59. {
  60. $site_url = "site_url = '" . addslashes($url) . "'";
  61. $result = Database::query("SELECT site_id FROM " . PHPDIG_DB_PREFIX .
  62. "sites WHERE " . $site_url, __FILE__, __LINE__); // find site
  63. if (Database::num_rows($result) == 1)
  64. {
  65. $row = Database::fetch_array($result); return (int) $row['site_id'];
  66. }
  67. else
  68. {
  69. $result = Database::query("INSERT INTO " . PHPDIG_DB_PREFIX .
  70. "sites SET " . $site_url, __FILE__, __LINE__); // new site
  71. $site_id = Database::insert_id();
  72. $result = Database::query("INSERT INTO " . PHPDIG_DB_PREFIX .
  73. "site_page (site_id,num_page) VALUES ('$site_id', '0')");
  74. return $site_id;
  75. }
  76. }
  77. function remove_engine_entries($url, $path, $file = '')
  78. {
  79. global $charset;
  80. $and_path = " AND path = '" . addslashes($path) . "'";
  81. if ($file) $and_path .= " AND file LIKE '" . addslashes(
  82. str_replace(array('_', '%'), array('\_', '\%'), $file)) . "%'";
  83. $result = Database::query("SELECT spider_id FROM " . PHPDIG_DB_PREFIX .
  84. "spider WHERE site_id=" . ($site_id = find_site($url)) . $and_path,
  85. __FILE__, __LINE__); // find page(s)
  86. while ($row = Database::fetch_array($result))
  87. {
  88. Database::query("DELETE FROM " . PHPDIG_DB_PREFIX .
  89. "engine WHERE spider_id=" . (int)$row['spider_id'],
  90. __FILE__, __LINE__); // delete all references to keywords
  91. $aff .= ' +' . Database::affected_rows();
  92. }
  93. Database::query("DELETE FROM " . PHPDIG_DB_PREFIX .
  94. "spider WHERE site_id=" . $site_id . $and_path,
  95. __FILE__, __LINE__); // delete page
  96. echo htmlspecialchars($url . $path . $file, ENT_QUOTES, $charset), ' (site_id ',
  97. $site_id, '): ', Database::affected_rows(), $aff,
  98. ' pages + word references removed from index.<br />';
  99. return $site_id;
  100. }
  101. function index_words($site_id, $path, $file, $first_words, $keywords)
  102. {
  103. global $common_words;
  104. $spider_set_path_etc = "spider SET path='" . addslashes($path) .
  105. "',file='" . addslashes($file) . "',first_words='" .
  106. addslashes($first_words) . "',site_id='$site_id'";
  107. // do not set upddate,md5,num_words,last_modified,filesize
  108. Database::query("INSERT INTO " . PHPDIG_DB_PREFIX . $spider_set_path_etc,
  109. __FILE__, __LINE__);
  110. $spider_id = Database::insert_id(); $new = 0;
  111. foreach ($keywords as $key => $w)
  112. if (strlen($key) > SMALL_WORDS_SIZE and strlen($key) <= MAX_WORDS_SIZE and
  113. !isset($common_words[$key]) and
  114. ereg('^['.WORDS_CHARS_LATIN1.'#$]', $key))
  115. {
  116. $result = Database::query("SELECT key_id FROM " . PHPDIG_DB_PREFIX .
  117. "keywords WHERE keyword = '" . addslashes($key) . "'",
  118. __FILE__, __LINE__);
  119. if (Database::num_rows($result) == 0)
  120. {
  121. Database::query("INSERT INTO " . PHPDIG_DB_PREFIX .
  122. "keywords (keyword,twoletters) VALUES ('" . addslashes($key) .
  123. "','" .addslashes(substr(str_replace('\\','',$key),0,2)) ."')",
  124. __FILE__, __LINE__);
  125. $key_id = Database::insert_id(); $new++;
  126. }
  127. else
  128. {
  129. $keyid = Database::fetch_row($result); $key_id = $keyid[0];
  130. }
  131. Database::query("INSERT INTO " . PHPDIG_DB_PREFIX .
  132. "engine (spider_id,key_id,weight) VALUES ($spider_id,$key_id,$w)",
  133. __FILE__, __LINE__);
  134. }
  135. echo '<tr><td>', htmlspecialchars($file, ENT_QUOTES, $charset), '</td><td>(spider_id ',
  136. $spider_id, '):</td><td align="right">', count($keywords), ' kwds, ',
  137. $new , ' new</td></tr>', "\n";
  138. }
  139. function get_first_words($text, $path, $file)
  140. {
  141. $db_some_text = preg_replace("/([ ]{2}|\n|\r|\r\n)/" ," ", $text);
  142. if (strlen($db_some_text) > SUMMARY_DISPLAY_LENGTH) {
  143. $db_some_text = substr($db_some_text, 0, SUMMARY_DISPLAY_LENGTH) . "...";
  144. }
  145. $titre_resume = $path . $file;
  146. if (($psc = strpos($titre_resume, 'scorm/')) !== FALSE)
  147. $titre_resume = substr($titre_resume, $psc + 6);
  148. if (($pth = strpos($titre_resume, '&thumb')) !== FALSE)
  149. $titre_resume = substr($titre_resume, 0, $pth);
  150. return $titre_resume."\n".$db_some_text;
  151. }
  152. function get_keywords($text)
  153. {
  154. if (($token = strtok(phpdigEpureText($text), ' '))) $nbre_mots[$token] = 1;
  155. while (($token = strtok(' ')))
  156. $nbre_mots[$token] = ($nm = $nbre_mots[$token]) ? $nm + 1 : 1;
  157. return $nbre_mots;
  158. }
  159. function phpdigEpureText($text)
  160. {
  161. $text = strtr(phpdigStripAccents(strtolower($text)), '��', '��');
  162. $text = ereg_replace('[^'.WORDS_CHARS_LATIN1.' \'._~@#$&%/=-]+',' ',$text); // RH: was ' \'._~@#$:&%/;,=-]+', also below
  163. $text = ereg_replace('(['.WORDS_CHARS_LATIN1.'])[\'._~@#$&%/=-]+($|[[:space:]]$|[[:space:]]['.WORDS_CHARS_LATIN1.'])','\1\2',$text);
  164. // the next two repeated lines needed
  165. if (SMALL_WORDS_SIZE >= 1) {
  166. $text = ereg_replace('[[:space:]][^ ]{1,'.SMALL_WORDS_SIZE.'}[[:space:]]',' ',' '.$text.' ');
  167. $text = ereg_replace('[[:space:]][^ ]{1,'.SMALL_WORDS_SIZE.'}[[:space:]]',' ',' '.$text.' ');
  168. }
  169. //$text = ereg_replace('\.+[[:space:]]|\.+$|\.{2,}',' ',$text);
  170. $text = ereg_replace('\.{2,}',' ',$text);
  171. $text = ereg_replace('^[[:space:]]*\.+',' ',$text);
  172. return trim(ereg_replace("[[:space:]]+"," ",$text));
  173. }
  174. function phpdigStripAccents($chaine)
  175. {
  176. $chaine = str_replace('�','ae',str_replace('�','ae',$chaine));
  177. return strtr($chaine, LETTERS_WITH_ACCENTS, SAME_WITHOUT_ACCENTS);
  178. }
  179. ?>