Prechádzať zdrojové kódy

Adding more funcionality to the cron import csv process (dump all, add extra fields) see BT#6369

Julio Montoya 11 rokov pred
rodič
commit
fc452e9d3e
2 zmenil súbory, kde vykonal 287 pridanie a 54 odobranie
  1. 243 47
      main/cron/import_csv.php
  2. 44 7
      main/inc/lib/sessionmanager.lib.php

+ 243 - 47
main/cron/import_csv.php

@@ -7,15 +7,40 @@ if (PHP_SAPI!='cli') {
 require_once __DIR__.'/../inc/global.inc.php';
 require_once api_get_path(LIBRARY_PATH).'log.class.php';
 
+/**
+ * Class ImportCsv
+ */
 class ImportCsv
 {
     private $logger;
+    private $dumpValues;
+    public $test;
+    public $defaultLanguage = 'english';
 
+    /**
+     * @param Logger $logger
+     */
     public function __construct($logger)
     {
         $this->logger = $logger;
     }
 
+    /**
+     * @param bool $dump
+     */
+    function setDumpValues($dump)
+    {
+        $this->dumpValues = $dump;
+    }
+
+    function getDumpValues()
+    {
+        return $this->dumpValues;
+    }
+
+    /**
+     * Runs the import process
+     */
     public function run()
     {
         $path = api_get_path(SYS_CODE_PATH).'cron/incoming/';
@@ -24,34 +49,134 @@ class ImportCsv
             exit;
         }
 
+        if ($this->getDumpValues()) {
+
+            $table = Database::get_main_table(TABLE_MAIN_USER);
+            $sql = "DELETE FROM $table WHERE username NOT IN ('admin') AND lastname <> 'Anonymous' ";
+            Database::query($sql);
+
+            echo 'Dumping tables'.PHP_EOL;
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_COURSE);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_COURSE_USER);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_ACCESS_URL_REL_USER);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_SESSION);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_SESSION_CATEGORY);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_SESSION_COURSE);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+            $table = Database::get_main_table(TABLE_MAIN_SESSION_USER);
+            $sql = "DELETE FROM $table";
+            Database::query($sql);
+            echo $sql.PHP_EOL;
+
+        }
+
+        echo "Starting with reading the files: ".PHP_EOL.PHP_EOL;
+
         $files = scandir($path);
+        $fileToProcess = array();
+
         if (!empty($files)) {
             foreach ($files as $file) {
                 $fileInfo = pathinfo($file);
                 if ($fileInfo['extension'] == 'csv') {
                     // teachers_yyyymmdd.csv, courses_yyyymmdd.csv, students_yyyymmdd.csv and sessions_yyyymmdd.csv
                     $parts = explode('_', $fileInfo['filename']);
-                    $method = 'import'.ucwords($parts[0]);
+                    $method = 'import'.ucwords($parts[1]);
 
                     if (method_exists($this, $method)) {
-                        $this->$method($path.$fileInfo['basename']);
+                         $fileToProcess[$parts[1]][] = array(
+                            'method' => $method,
+                            'file' => $path.$fileInfo['basename']
+                        );
+                        //$this->$method($path.$fileInfo['basename']);
                     } else {
-                        echo "Error - This file can't be processed.";
+                        echo "Error - This file '$file' can't be processed.".PHP_EOL;
+                        echo "The file have to has this format:".PHP_EOL;
+                        echo "prefix_students_ddmmyyyy.csv, prefix_teachers_ddmmyyyy.csv, prefix_courses_ddmmyyyy.csv, prefix_sessions_ddmmyyyy.csv ".PHP_EOL;
                         exit;
                     }
                 }
             }
+
+            if (empty($fileToProcess)) {
+                echo 'Error - no files to process.';
+                exit;
+            }
+
+            $sections = array('students', 'teachers', 'courses', 'sessions');
+
+
+            $this->prepareImport();
+
+            foreach ($sections as $section) {
+                $this->logger->addInfo("-- Import $section --");
+
+                if (isset($fileToProcess[$section]) && !empty($fileToProcess[$section])) {
+                    $files = $fileToProcess[$section];
+                    foreach ($files as $fileInfo) {
+                        $method = $fileInfo['method'];
+                        $file = $fileInfo['file'];
+
+                        echo 'Reading file: '.$file.PHP_EOL;
+                        $this->logger->addInfo("Reading file: $file");
+
+                        $this->$method($file);
+                    }
+
+                }
+            }
         }
     }
 
+    private function prepareImport()
+    {
+        // Create user extra field: extra_external_user_id
+        UserManager::create_extra_field('external_user_id', 1, 'External user id', null);
+
+        // Create course extra field: extra_external_course_id
+        CourseManager::create_course_extra_field('external_course_id', 1, 'External course id');
+        // Create session extra field extra_external_session_id
+        SessionManager::create_session_extra_field('external_session_id', 1, 'External session id');
+    }
+
     /**
      * @param string $file
      */
     function moveFile($file)
     {
         $moved = str_replace('incoming', 'treated', $file);
-        // $result = rename($file, $moved);
-        $result = 1;
+
+        if ($this->test) {
+            $result = 1;
+        } else {
+            $result = rename($file, $moved);
+        }
+
         if ($result) {
             $this->logger->addInfo("Moving file to the treated folder: $file");
         } else {
@@ -59,15 +184,63 @@ class ImportCsv
         }
     }
 
+    /**
+     * @param array $row
+     *
+     * @return array
+     */
+    private function cleanUserRow($row)
+    {
+        $row['lastname'] = $row['LastName'];
+        $row['firstname'] = $row['FirstName'];
+        $row['email'] = $row['Email'];
+        $row['username'] = $row['UserName'];
+        $row['password'] = $row['Password'];
+        $row['auth_srouce'] = $row['AuthSource'];
+        $row['official_code'] = $row['OfficialCode'];
+        $row['phone'] = $row['PhoneNumber'];
+
+        if (isset($row['StudentID'])) {
+            $row['extra_external_user_id'] = $row['StudentID'];
+        }
+
+        if (isset($row['TeacherID'])) {
+            $row['extra_external_user_id'] = $row['TeacherID'];
+        }
+
+        //$row['lastname'] =  Status
+        return $row;
+    }
+
+
+    /**
+     * @param array $row
+     * @return array
+     */
+    private function cleanCourseRow($row)
+    {
+        $row['title'] = $row['Title'];
+        $row['course_code'] = $row['Code'];
+        $row['category_code'] = $row['CourseCategory'];
+        $row['email'] = $row['Teacher'];
+        $row['language'] = $row['Language'];
+        $row['external_id'] = $row['CourseID'];
+
+        if (isset($row['CourseID'])) {
+            $row['extra_external_course_id'] = $row['CourseID'];
+        }
+
+        return $row;
+    }
+
+
     /**
      * File to import
      * @param string $file
      */
-    function importTeachers($file)
+    private function importTeachers($file)
     {
         $data = Import::csv_to_array($file);
-        $this->logger->addInfo("-- Import Teachers --");
-        $this->logger->addInfo("Reading file: $file");
 
         /* Unique identifier: official-code username.
         Email address and password should never get updated. *ok
@@ -77,15 +250,18 @@ class ImportCsv
         We’ll handle that manually if applicable.
         No delete!
         */
+        $language = $this->defaultLanguage;
 
         if (!empty($data)) {
+            $this->logger->addInfo(count($data)." records found.");
             foreach ($data as $row) {
+                $row = $this->cleanUserRow($row);
                 $userInfo = api_get_user_info_from_username($row['username']);
                 $userInfoByOfficialCode = api_get_user_info_from_official_code($row['official_code']);
 
                 if (empty($userInfo) && empty($userInfoByOfficialCode)) {
                     // Create user
-                    $result = UserManager::create_user(
+                    $userId = UserManager::create_user(
                         $row['firstname'],
                         $row['lastname'],
                         COURSEMANAGER,
@@ -93,11 +269,11 @@ class ImportCsv
                         $row['username'],
                         $row['password'],
                         $row['official_code'],
-                        $row['language'],
+                        $language, //$row['language'],
                         $row['phone'],
-                        $row['picture'], //picture
+                        null, //$row['picture'], //picture
                         PLATFORM_AUTH_SOURCE, // ?
-                        $row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
+                        '0000-00-00 00:00:00', //$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
                         1, //active
                         0,
                         null, // extra
@@ -105,14 +281,19 @@ class ImportCsv
                         false //$send_mail = false
                     );
 
-                    if ($result) {
-                        $this->logger->addInfo("Info - Teachers - User created: ".$row['username']);
+                    if ($userId) {
+                        foreach ($row as $key => $value) {
+                            if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                UserManager::update_extra_field_value($userId, substr($key, 6), $value);
+                            }
+                        }
+                        $this->logger->addInfo("Teachers - User created: ".$row['username']);
                     } else {
-                        $this->logger->addError("Error - Teachers - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
+                        $this->logger->addError("Teachers - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
                     }
                 } else {
                     if (empty($userInfo)) {
-                        $this->logger->addError("Error - Teachers - Can't update user :".$row['username']);
+                        $this->logger->addError("Teachers - Can't update user :".$row['username']);
                         continue;
                     }
 
@@ -141,26 +322,27 @@ class ImportCsv
                     );
 
                     if ($result) {
+                        foreach ($row as $key => $value) {
+                            if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                UserManager::update_extra_field_value($userInfo['user_id'], substr($key, 6), $value);
+                            }
+                        }
                         $this->logger->addInfo("Teachers - User updated: ".$row['username']);
                     } else {
                         $this->logger->addError("Teachers - User not updated: ".$row['username']);
                     }
                 }
-
-                // UserManager::delete_user();
-                $this->moveFile($file);
             }
         }
+        $this->moveFile($file);
     }
 
     /**
      * @param string $file
      */
-    function importStudents($file)
+    private function importStudents($file)
     {
         $data = Import::csv_to_array($file);
-        $this->logger->addInfo("-- Import Students --");
-        $this->logger->addInfo("Reading file: $file");
 
         /*
          * Another users import.
@@ -172,7 +354,10 @@ class ImportCsv
         He should be set inactive one year after the current date. So I presume you’ll just update the expiration date. We want to grant access to courses up to a year after deletion.
          */
         if (!empty($data)) {
+            $language = $this->defaultLanguage;
+            $this->logger->addInfo(count($data)." records found.");
             foreach ($data as $row) {
+                $row = $this->cleanUserRow($row);
                 $userInfo = api_get_user_info_from_username($row['username']);
                 $userInfoByOfficialCode = api_get_user_info_from_official_code($row['official_code']);
 
@@ -181,16 +366,16 @@ class ImportCsv
                     $result = UserManager::create_user(
                         $row['firstname'],
                         $row['lastname'],
-                        STUDENT,
+                        COURSEMANAGER,
                         $row['email'],
                         $row['username'],
                         $row['password'],
                         $row['official_code'],
-                        $row['language'],
+                        $language, //$row['language'],
                         $row['phone'],
-                        $row['picture'], //picture
+                        null, //$row['picture'], //picture
                         PLATFORM_AUTH_SOURCE, // ?
-                        $row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
+                        '0000-00-00 00:00:00', //$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
                         1, //active
                         0,
                         null, // extra
@@ -199,6 +384,11 @@ class ImportCsv
                     );
 
                     if ($result) {
+                        foreach ($row as $key => $value) {
+                            if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                UserManager::update_extra_field_value($result, substr($key, 6), $value);
+                            }
+                        }
                         $this->logger->addInfo("Students - User created: ".$row['username']);
                     } else {
                         $this->logger->addError("Students - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
@@ -239,14 +429,17 @@ class ImportCsv
                     );
 
                     if ($result) {
+                        foreach ($row as $key => $value) {
+                            if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                UserManager::update_extra_field_value($userInfo['user_id'], substr($key, 6), $value);
+                            }
+                        }
+
                         $this->logger->addInfo("Students - User updated: ".$row['username']);
                     } else {
                         $this->logger->addError("Students - User NOT updated: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
                     }
                 }
-
-                // UserManager::delete_user();
-                $this->moveFile($file);
             }
         }
 
@@ -256,13 +449,16 @@ class ImportCsv
     /**
      * @param string $file
      */
-    function importCourses($file)
+    private function importCourses($file)
     {
         $data = Import::csv_to_array($file);
-        $this->logger->addInfo("Reading file: $file");
+
+        $language = $this->defaultLanguage;
 
         if (!empty($data)) {
+            $this->logger->addInfo(count($data)." records found.");
             foreach ($data as $row) {
+                $row = $this->cleanCourseRow($row);
                 $courseInfo = api_get_course_info($row['course_code']);
                 if (empty($courseInfo)) {
                     // Create
@@ -276,6 +472,7 @@ class ImportCsv
                     $courseInfo = CourseManager::create_course($params);
 
                     if (!empty($courseInfo)) {
+                        CourseManager::update_course_extra_field_value($courseInfo['code'], 'external_course_id', $row['extra_external_course_id']);
                         $this->logger->addInfo("Courses - Course created ".$courseInfo['code']);
                     } else {
                         $this->logger->addError("Courses - Can't create course:".$row['title']);
@@ -294,18 +491,6 @@ class ImportCsv
                     } else {
                         $this->logger->addError("Courses - Course NOT updated ".$courseInfo['code']);
                     }
-
-                    /*course_language='".Database::escape_string($course_language)."',
-                    title='".Database::escape_string($title)."',
-                    category_code='".Database::escape_string($category_code)."',
-                    tutor_name='".Database::escape_string($tutor_name)."',
-                    visual_code='".Database::escape_string($visual_code)."',
-                    department_name='".Database::escape_string($department_name)."',
-                    department_url='".Database::escape_string($department_url)."',
-                    disk_quota='".Database::escape_string($disk_quota)."',
-                    visibility = '".Database::escape_string($visibility)."',
-                    subscribe = '".Database::escape_string($subscribe)."',
-                    unsubscribe='*/
                 }
             }
         }
@@ -315,9 +500,9 @@ class ImportCsv
     /**
      * @param string $file
      */
-    function importSessions($file)
+    private function importSessions($file)
     {
-        $result = SessionManager::importCSV($file, true, 1, $this->logger);
+        $result = SessionManager::importCSV($file, true, 1, $this->logger, array('SessionId' => 'extra_external_session_id'));
 
         if (!empty($result['error_message'])) {
             $this->logger->addError($result['error_message']);
@@ -326,6 +511,7 @@ class ImportCsv
         $this->moveFile($file);
     }
 }
+
 use Monolog\Logger;
 use Monolog\Handler\StreamHandler;
 use Monolog\Handler\NativeMailerHandler;
@@ -345,8 +531,8 @@ $from = api_get_setting('emailAdministrator');
 
 if (!empty($emails)) {
     foreach ($emails as $email) {
-        $stream = new NativeMailerHandler($email, $subject, $from, $minLevel);
-        $logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
+        //$stream = new NativeMailerHandler($email, $subject, $from, $minLevel);
+        //$logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
     }
 }
 
@@ -355,4 +541,14 @@ $logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
 $logger->pushHandler(new RotatingFileHandler('import_csv', 5, $minLevel));
 
 $import = new ImportCsv($logger);
+$dump = false;
+
+if (isset($argv[1]) && $argv[1] = '--dump') {
+    $dump = true;
+}
+
+$import->setDumpValues($dump);
+// Do not moves the files to treated
+$import->test = true;
+
 $import->run();

+ 44 - 7
main/inc/lib/sessionmanager.lib.php

@@ -11,8 +11,13 @@
  * Code
  */
 
-class SessionManager {
-	private function __construct() {
+/**
+ * Class SessionManager
+ */
+class SessionManager
+{
+	private function __construct()
+    {
 	}
 
     /**
@@ -20,7 +25,8 @@ class SessionManager {
      * @param   int     Session ID
      * @return  array   Session details (id, id_coach, name, nbr_courses, nbr_users, nbr_classes, date_start, date_end, nb_days_access_before_beginning,nb_days_access_after_end, session_admin_id)
      */
-    public static function fetch($id) {
+    public static function fetch($id)
+    {
     	$t = Database::get_main_table(TABLE_MAIN_SESSION);
         if ($id != strval(intval($id))) { return array(); }
         $s = "SELECT * FROM $t WHERE id = $id";
@@ -1863,11 +1869,14 @@ class SessionManager {
      * false: if session exists a new session will be created adding a counter session1, session2, etc
      * @param int $user_id
      * @param $logger
+     * @param array convert a file row to an extra field. Example in CSV file there's a SessionID then it will
+     * converted to extra_external_session_id if you set this: array('SessionId' => 'extra_external_session_id')
      * @return array
      */
-    static function importCSV($file, $updatesession, $user_id = null, $logger = null)
+    static function importCSV($file, $updatesession, $user_id = null, $logger = null, $extraFields = array())
     {
         $content = file($file);
+
         $error_message = null;
         $session_counter = 0;
 
@@ -1885,12 +1894,15 @@ class SessionManager {
         $tbl_session_course         = Database::get_main_table(TABLE_MAIN_SESSION_COURSE);
         $tbl_session_course_user    = Database::get_main_table(TABLE_MAIN_SESSION_COURSE_USER);
 
+        $sessions = array();
+
         if (!api_strstr($content[0], ';')) {
             $error_message = get_lang('NotCSV');
         } else {
             $tag_names = array();
 
             foreach ($content as $key => $enreg) {
+
                 $enreg = explode(';', trim($enreg));
                 if ($key) {
                     foreach ($tag_names as $tag_key => $tag_name) {
@@ -1912,7 +1924,13 @@ class SessionManager {
                 $user_counter = 0;
                 $course_counter = 0;
 
-                $session_name           = $enreg['SessionName'];
+                if (isset($extraFields) && !empty($extraFields)) {
+                    foreach ($extraFields as $original => $to) {
+                        $enreg[$to] = $enreg[$original];
+                    }
+                }
+
+                $session_name           = Database::escape_string($enreg['SessionName']);
                 $date_start             = $enreg['DateStart'];
                 $date_end               = $enreg['DateEnd'];
                 $visibility             = $enreg['Visibility'];
@@ -1938,7 +1956,7 @@ class SessionManager {
                         if ($i > 1) {
                             $suffix = ' - '.$i;
                         }
-                        $sql = 'SELECT 1 FROM '.$tbl_session.' WHERE name="'.Database::escape_string($session_name.$suffix).'"';
+                        $sql = 'SELECT 1 FROM '.$tbl_session.' WHERE name="'.$session_name.$suffix.'"';
                         $rs = Database::query($sql);
 
                         if (Database::result($rs, 0, 0)) {
@@ -1951,7 +1969,7 @@ class SessionManager {
 
                     // Creating the session.
                     $sql_session = "INSERT IGNORE INTO $tbl_session SET
-                            name = '".Database::escape_string($session_name)."',
+                            name = '".$session_name."',
                             id_coach = '$coach_id',
                             date_start = '$date_start',
                             date_end = '$date_end',
@@ -1963,6 +1981,13 @@ class SessionManager {
 
                     if ($debug) {
                         if ($session_id) {
+
+                            foreach ($enreg as $key => $value) {
+                                if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                    self::update_session_extra_field_value($session_id, substr($key, 6), $value);
+                                }
+                            }
+
                             $logger->addInfo("Sessions - Session created: #$session_id - $session_name");
                         } else {
                             $logger->addError("Sessions - Session NOT created: $session_name");
@@ -1989,6 +2014,11 @@ class SessionManager {
 
                         if ($debug) {
                             if ($session_id) {
+                                foreach ($enreg as $key => $value) {
+                                    if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                        self::update_session_extra_field_value($session_id, substr($key, 6), $value);
+                                    }
+                                }
                                 $logger->addInfo("Sessions - #$session_id created: $session_name");
                             } else {
                                 $logger->addError("Sessions - Session NOT created: $session_name");
@@ -2004,7 +2034,14 @@ class SessionManager {
                                 session_category_id = '$session_category_id'
                             WHERE name = '$session_name'";
                         Database::query($sql_session);
+
                         $session_id = Database::query("SELECT id FROM $tbl_session WHERE name='$session_name'");
+
+                        foreach ($enreg as $key => $value) {
+                            if (substr($key, 0, 6) == 'extra_') { //an extra field
+                                self::update_session_extra_field_value($session_id, substr($key, 6), $value);
+                            }
+                        }
                         list($session_id) = Database::fetch_array($session_id);
                         Database::query("DELETE FROM $tbl_session_user WHERE id_session='$session_id'");
                         Database::query("DELETE FROM $tbl_session_course WHERE id_session='$session_id'");