library/standard_tables_capture.inc.php

   1 <?php
   2
   3 /**
   4  * This library contains functions that implement the database load processing
   5  * of external database files into openEMR
   6  *
   7  * @package   OpenEMR
   8  * @link      https://www.open-emr.org
   9  * @author    Rohit Kumar <pandit.rohit@netsity.com>
  10  * @author    (Mac) Kevin McAloon <mcaloon@patienthealthcareanalytics.com>
  11  * @author    Brady Miller <brady.g.miller@gmail.com>
  12  * @author    Roberto Vasquez <robertogagliotta@gmail.com>
  13  * @author    Stephen Waite <stephen.waite@cmsvt.com>
  14  * @copyright Copyright (c) 2011 Phyaura, LLC <info@phyaura.com>
  15  * @copyright Copyright (c) 2019-2022 Brady Miller <brady.g.miller@gmail.com>
  16  * @copyright Copyright (c) 2019-2022 Stephen Waite <stephen.waite@cmsvt.com>
  17  * @license   https://github.com/openemr/openemr/blob/master/LICENSE GNU General Public License 3
  18  */
  19
  20 use OpenEMR\Events\Codes\CodeTypeInstalledEvent;
  21
  22 // Function to copy a package to temp
  23 // $type (RXNORM, SNOMED etc.)
  24 function temp_copy($filename, $type)
  25 {
  26
  27     if (!file_exists($filename)) {
  28         return false;
  29     }
  30
  31     if (!file_exists($GLOBALS['temporary_files_dir'] . "/" . $type)) {
  32         if (!mkdir($GLOBALS['temporary_files_dir'] . "/" . $type, 0777, true)) {
  33                 return false;
  34         }
  35     }
  36
  37     if (copy($filename, $GLOBALS['temporary_files_dir'] . "/" . $type . "/" . basename($filename))) {
  38         return true;
  39     } else {
  40         return false;
  41     }
  42 }
  43
  44 // Function to unarchive a package
  45 // $type (RXNORM, SNOMED etc.)
  46 function temp_unarchive($filename, $type)
  47 {
  48     $filename = $GLOBALS['temporary_files_dir'] . "/" . $type . "/" . basename($filename);
  49     if (!file_exists($filename)) {
  50         return false;
  51     } elseif ($type == "ICD10") {
  52         // copy zip file contents to /tmp/ICD10 due to CMS zip file
  53         $zip = new ZipArchive();
  54         $path = $GLOBALS['temporary_files_dir'] . "/" . $type;
  55         if ($zip->open($filename) === true) {
  56             for ($i = 0; $i < $zip->numFiles; $i++) {
  57                 $sub_dir_filename = $zip->getNameIndex($i);
  58                 $fileinfo = pathinfo($sub_dir_filename);
  59                 if (!(copy("zip://" . $filename . "#" . $sub_dir_filename, "$path/" . $fileinfo['basename']))) {
  60                     return false;
  61                 }
  62             }
  63             $zip->close();
  64             return true;
  65         } else {
  66             return false;
  67         }
  68     } else {
  69         // unzip the file
  70         $zip = new ZipArchive();
  71         if ($zip->open($filename) === true) {
  72             if (!($zip->extractTo($GLOBALS['temporary_files_dir'] . "/" . $type))) {
  73                 return false;
  74             }
  75             $zip->close();
  76             return true;
  77         }
  78     }
  79 }
  80
  81 // Function to import the RXNORM tables
  82 // $is_windows_flag - pass the IS_WINDOWS constant
  83 function rxnorm_import($is_windows_flag)
  84 {
  85     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
  86     // or any manual processing that needs to occur.
  87     if (!empty($GLOBALS['kernel'])) {
  88         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('RXNORM', ['is_windows_flag' => $is_windows_flag]);
  89         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_PRE);
  90     }
  91     // set paths
  92     $dirScripts = $GLOBALS['temporary_files_dir'] . "/RXNORM/scripts/mysql";
  93     $dir = $GLOBALS['temporary_files_dir'] . "/RXNORM/rrf";
  94     $dir = str_replace('\\', '/', $dir);
  95
  96     $rx_info = array();
  97     $rx_info['rxnatomarchive'] = array('title' => "Archive Data", 'dir' => "$dir", 'origin' => "RXNATOMARCHIVE.RRF", 'filename' => "RXNATOMARCHIVE.RRF", 'table' => "rxnatomarchive", 'required' => 0);
  98     $rx_info['rxnconso'] = array('title' => "Concept Names and Sources", 'dir' => "$dir", 'origin' => "RXNCONSO.RRF", 'filename' => "RXNCONSO.RRF", 'table' => "rxnconso",  'required' => 1);
  99     $rx_info['rxncui'] = array('title' => "Retired RXCUI Data", 'dir' => "$dir", 'origin' => "RXNCUI.RRF", 'filename' => "RXNCUI.RRF", 'table' => "rxncui", 'required' => 1);
 100     $rx_info['rxncuichanges'] = array('title' => "Concept Changes", 'dir' => "$dir", 'origin' => "RXNCUICHANGES.RRF", 'filename' => "RXNCUICHANGES.RRF", 'table' => "rxncuichanges", 'required' => 1);
 101     $rx_info['rxndoc'] = array('title' => "Documentation for Abbreviated Values", 'dir' => "$dir", 'origin' => "RXNDOC.RRF", 'filename' => "RXNDOC.RRF", 'table' => "rxndoc", 'required' => 1);
 102     $rx_info['rxnrel'] = array('title' => "Relationships", 'dir' => "$dir", 'origin' => "RXNREL.RRF", 'filename' => "RXNREL.RRF", 'table' => "rxnrel", 'required' => 1);
 103     $rx_info['rxnsab'] = array('title' => "Source Information", 'dir' => "$dir", 'origin' => "RXNSAB.RRF", 'filename' => "RXNSAB.RRF", 'table' => "rxnsab", 'required' => 0);
 104     $rx_info['rxnsat'] = array('title' => "Simple Concept and Atom Attributes", 'dir' => "$dir", 'origin' => "RXNSAT.RRF", 'filename' => "RXNSAT.RRF", 'table' => "rxnsat", 'required' => 0);
 105     $rx_info['rxnsty'] = array('title' => "Semantic Types ", 'dir' => "$dir", 'origin' => "RXNSTY.RRF", 'filename' => "RXNSTY.RRF", 'table' => "rxnsty", 'required' => 1);
 106
 107     // load scripts
 108     $file_load = file_get_contents($dirScripts . '/Table_scripts_mysql_rxn.sql', true);
 109     if ($is_windows_flag) {
 110         $data_load = file_get_contents($dirScripts . '/Load_scripts_mysql_rxn_win.sql', true);
 111     } else {
 112         $data_load = file_get_contents($dirScripts . '/Load_scripts_mysql_rxn_unix.sql', true);
 113     }
 114
 115     $indexes_load = file_get_contents($dirScripts . '/Indexes_mysql_rxn.sql', true);
 116
 117     //
 118     // Creating the structure for table and applying indexes
 119     //
 120
 121     $file_array = explode(";", $file_load);
 122     foreach ($file_array as $val) {
 123         if (trim($val) != '') {
 124             sqlStatementNoLog($val);
 125         }
 126     }
 127
 128     $indexes_array = explode(";", $indexes_load);
 129
 130     foreach ($indexes_array as $val1) {
 131         if (trim($val1) != '') {
 132             sqlStatementNoLog($val1);
 133         }
 134     }
 135
 136
 137     // Settings to drastically speed up import with InnoDB
 138     sqlStatementNoLog("SET autocommit=0");
 139     sqlStatementNoLog("START TRANSACTION");
 140     $data = explode(";", $data_load);
 141     foreach ($data as $val) {
 142         foreach ($rx_info as $key => $value) {
 143             $file_name = $value['origin'];
 144             $replacement = $dir . "/" . $file_name;
 145
 146             $pattern = '/' . $file_name . '/';
 147             if (strpos($val, $file_name) !== false) {
 148                 $val1 = str_replace($file_name, $replacement, $val);
 149                 if (trim($val1) != '') {
 150                     sqlStatementNoLog($val1);
 151                 }
 152             }
 153         }
 154     }
 155
 156     // Settings to drastically speed up import with InnoDB
 157     sqlStatementNoLog("COMMIT");
 158     sqlStatementNoLog("SET autocommit=1");
 159
 160     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 161     // or any manual processing that needs to occur.
 162     if (!empty($GLOBALS['kernel'])) {
 163         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('RXNORM', ['is_windows_flag' => $is_windows_flag]);
 164         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 165     }
 166
 167     return true;
 168 }
 169
 170 // Function to import SNOMED tables
 171 function snomed_import($us_extension = false)
 172 {
 173     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 174     // or any manual processing that needs to occur.
 175     if (!empty($GLOBALS['kernel'])) {
 176         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('SNOMED', ['us_extension' => $us_extension]);
 177         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 178     }
 179
 180     // set up array
 181     $table_array_for_snomed = array(
 182         "sct_concepts_drop" => "DROP TABLE IF EXISTS `sct_concepts`",
 183         "sct_concepts_structure" => "CREATE TABLE IF NOT EXISTS `sct_concepts` (
 184             `ConceptId` bigint(20) NOT NULL,
 185             `ConceptStatus` int(11) NOT NULL,
 186             `FullySpecifiedName` varchar(255) NOT NULL,
 187             `CTV3ID` varchar(5) NOT NULL,
 188             `SNOMEDID` varchar(8) NOT NULL,
 189             `IsPrimitive` tinyint(1) NOT NULL,
 190             PRIMARY KEY (`ConceptId`)
 191             ) ENGINE=InnoDB",
 192         "sct_descriptions_drop" => "DROP TABLE IF EXISTS `sct_descriptions`",
 193         "sct_descriptions_structure" => "CREATE TABLE IF NOT EXISTS `sct_descriptions` (
 194             `DescriptionId` bigint(20) NOT NULL,
 195             `DescriptionStatus` int(11) NOT NULL,
 196             `ConceptId` bigint(20) NOT NULL,
 197             `Term` varchar(255) NOT NULL,
 198             `InitialCapitalStatus` tinyint(1) NOT NULL,
 199             `DescriptionType` int(11) NOT NULL,
 200             `LanguageCode` varchar(8) NOT NULL,
 201             PRIMARY KEY (`DescriptionId`),
 202             KEY `idx_concept_id` (`ConceptId`)
 203             ) ENGINE=InnoDB",
 204         "sct_relationships_drop" => "DROP TABLE IF EXISTS `sct_relationships`",
 205         "sct_relationships_structure" => "CREATE TABLE IF NOT EXISTS `sct_relationships` (
 206             `RelationshipId` bigint(20) NOT NULL,
 207             `ConceptId1` bigint(20) NOT NULL,
 208             `RelationshipType` bigint(20) NOT NULL,
 209             `ConceptId2` bigint(20) NOT NULL,
 210             `CharacteristicType` int(11) NOT NULL,
 211             `Refinability` int(11) NOT NULL,
 212             `RelationshipGroup` int(11) NOT NULL,
 213             PRIMARY KEY (`RelationshipId`)
 214             ) ENGINE=InnoDB"
 215     );
 216
 217     // set up paths
 218     $dir_snomed = $GLOBALS['temporary_files_dir'] . "/SNOMED/";
 219     $sub_path = "Terminology/Content/";
 220     $dir = $dir_snomed;
 221     $dir = str_replace('\\', '/', $dir);
 222
 223     // executing the create statement for tables, these are defined in snomed_capture.inc.php file
 224     // this is skipped if the US extension is being added
 225     if (!$us_extension) {
 226         foreach ($table_array_for_snomed as $val) {
 227             if (trim($val) != '') {
 228                 sqlStatement($val);
 229             }
 230         }
 231     }
 232
 233     // reading the SNOMED directory and identifying the files to import and replacing the variables by originals values.
 234     if (is_dir($dir) && $handle = opendir($dir)) {
 235         while (false !== ($filename = readdir($handle))) {
 236             if ($filename != "." && $filename != ".." && !strpos($filename, "zip")) {
 237                 $path = $dir . "" . $filename . "/" . $sub_path;
 238                 if (!(is_dir($path))) {
 239                     $path = $dir . "" . $filename . "/RF1Release/" . $sub_path;
 240                 }
 241
 242                 if (is_dir($path) && $handle1 = opendir($path)) {
 243                     while (false !== ($filename1 = readdir($handle1))) {
 244                         $load_script = "Load data local infile '#FILENAME#' into table #TABLE# fields terminated by '\\t' ESCAPED BY '' lines terminated by '\\n' ignore 1 lines   ";
 245                         $array_replace = array("#FILENAME#","#TABLE#");
 246                         if ($filename1 != "." && $filename1 != "..") {
 247                             $file_replace = $path . $filename1;
 248                             if (strpos($filename1, "Concepts") !== false) {
 249                                 $new_str = str_replace($array_replace, array($file_replace,"sct_concepts"), $load_script);
 250                             }
 251
 252                             if (strpos($filename1, "Descriptions") !== false) {
 253                                 $new_str = str_replace($array_replace, array($file_replace,"sct_descriptions"), $load_script);
 254                             }
 255
 256                             if (strpos($filename1, "Relationships") !== false) {
 257                                 $new_str = str_replace($array_replace, array($file_replace,"sct_relationships"), $load_script);
 258                             }
 259
 260                             if ($new_str != '') {
 261                                 sqlStatement($new_str);
 262                             }
 263                         }
 264                     }
 265                 }
 266
 267                 closedir($handle1);
 268             }
 269         }
 270
 271         closedir($handle);
 272     }
 273
 274     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 275     // or any manual processing that needs to occur.
 276     if (!empty($GLOBALS['kernel'])) {
 277         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('SNOMED', ['us_extension' => $us_extension]);
 278         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 279     }
 280
 281     return true;
 282 }
 283
 284 function drop_old_sct()
 285 {
 286     $array_to_truncate = array(
 287         "sct_concepts_drop" => "DROP TABLE IF EXISTS `sct_concepts`",
 288         "sct_descriptions_drop" => "DROP TABLE IF EXISTS `sct_descriptions`",
 289         "sct_relationships_drop" => "DROP TABLE IF EXISTS `sct_relationships`"
 290     );
 291     foreach ($array_to_truncate as $val) {
 292         if (trim($val) != '') {
 293             sqlStatement($val);
 294         }
 295     }
 296 }
 297
 298 function drop_old_sct2()
 299 {
 300     $array_to_truncate = array(
 301         "sct2_concept_drop" => "DROP TABLE IF EXISTS `sct2_concept`",
 302         "sct2_description_drop" => "DROP TABLE IF EXISTS `sct2_description`",
 303         "sct2_identifier_drop" => "DROP TABLE IF EXISTS `sct2_identifier`",
 304         "sct2_relationship_drop" => "DROP TABLE IF EXISTS `sct2_relationship`",
 305         "sct2_statedrelationship_drop" => "DROP TABLE IF EXISTS `sct2_statedrelationship`",
 306         "sct2_textdefinition_drop" => "DROP TABLE IF EXISTS `sct2_textdefinition`"
 307     );
 308     foreach ($array_to_truncate as $val) {
 309         if (trim($val) != '') {
 310             sqlStatement($val);
 311         }
 312     }
 313 }
 314
 315 function chg_ct_external_torf1()
 316 {
 317     sqlStatement("UPDATE code_types SET ct_external = 2 WHERE ct_key = 'SNOMED'");
 318     sqlStatement("UPDATE code_types SET ct_external = 7 WHERE ct_key = 'SNOMED-CT'");
 319     sqlStatement("UPDATE code_types SET ct_external = 9 WHERE ct_key = 'SNOMED-PR'");
 320 }
 321
 322 function chg_ct_external_torf2()
 323 {
 324     sqlStatement("UPDATE code_types SET ct_external = 10 WHERE ct_key = 'SNOMED'");
 325     sqlStatement("UPDATE code_types SET ct_external = 11 WHERE ct_key = 'SNOMED-CT'");
 326     sqlStatement("UPDATE code_types SET ct_external = 12 WHERE ct_key = 'SNOMED-PR'");
 327 }
 328
 329 function snomedRF2_import()
 330 {
 331     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 332     // or any manual processing that needs to occur.
 333     if (!empty($GLOBALS['kernel'])) {
 334         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('SNOMED', []);
 335         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_PRE);
 336     }
 337
 338     // set up array
 339     $table_array_for_snomed = array(
 340         "sct2_concept_drop" => "DROP TABLE IF EXISTS `sct2_concept`",
 341         "sct2_concept_structure" => "CREATE TABLE IF NOT EXISTS `sct2_concept` (
 342             `id` bigint(20) NOT NULL,
 343             `effectiveTime` date NOT NULL,
 344             `active` int(11) NOT NULL,
 345             `moduleId` bigint(20) NOT NULL,
 346             `definitionStatusId` bigint(25) NOT NULL,
 347              PRIMARY KEY (`id`)
 348             ) ENGINE=InnoDB",
 349         "sct2_description_drop" => "DROP TABLE IF EXISTS `sct2_description`",
 350         "sct2_description_structure" => "CREATE TABLE IF NOT EXISTS `sct2_description` (
 351             `id` bigint(20) NOT NULL,
 352             `effectiveTime` date NOT NULL,
 353             `active` bigint(11) NOT NULL,
 354             `moduleId` bigint(25) NOT NULL,
 355             `conceptId` bigint(20) NOT NULL,
 356             `languageCode` varchar(8) NOT NULL,
 357             `typeId` bigint(25) NOT NULL,
 358             `term` varchar(255) NOT NULL,
 359             `caseSignificanceId` bigint(25) NOT NULL,
 360              PRIMARY KEY (`id`, `active`, `conceptId`),
 361              KEY `idx_concept_id` (`conceptId`)
 362             ) ENGINE=InnoDB",
 363         "sct2_identifier_drop" => "DROP TABLE IF EXISTS `sct2_identifier`",
 364         "sct2_identifier_structure" => "CREATE TABLE IF NOT EXISTS `sct2_identifier` (
 365             `identifierSchemeId` bigint(25) NOT NULL,
 366             `alternateIdentifier` bigint(25) NOT NULL,
 367             `effectiveTime` date NOT NULL,
 368             `active` int(11) NOT NULL,
 369             `moduleId` bigint(25) NOT NULL,
 370             `referencedComponentId` bigint(25) NOT NULL,
 371              PRIMARY KEY (`identifierSchemeId`)
 372             ) ENGINE=InnoDB",
 373         "sct2_relationship_drop" => "DROP TABLE IF EXISTS `sct2_relationship`",
 374         "sct2_relationship_structure" => "CREATE TABLE IF NOT EXISTS `sct2_relationship` (
 375             `id` bigint(20) NOT NULL,
 376             `effectiveTime` date NOT NULL,
 377             `active` int(11) NOT NULL,
 378             `moduleId` bigint(25) NOT NULL,
 379             `sourceId` bigint(20) NOT NULL,
 380             `destinationId` bigint(20) NOT NULL,
 381             `typeId` bigint(25) NOT NULL,
 382             `characteristicTypeId` bigint(25) NOT NULL,
 383             `modifierId` bigint(25) NOT NULL,
 384              PRIMARY KEY (`id`)
 385             ) ENGINE=InnoDB",
 386         "sct2_statedrelationship_drop" => "DROP TABLE IF EXISTS `sct2_statedrelationship`",
 387         "sct2_statedrelationship_structure" => "CREATE TABLE IF NOT EXISTS `sct2_statedrelationship` (
 388             `id` bigint(20) NOT NULL,
 389             `effectiveTime` date NOT NULL,
 390             `active` int(11) NOT NULL,
 391             `moduleId` bigint(25) NOT NULL,
 392             `sourceId` bigint(20) NOT NULL,
 393             `destinationId` bigint(20) NOT NULL,
 394             `relationshipGroup` int(11) NOT NULL,
 395             `typeId` bigint(25) NOT NULL,
 396              PRIMARY KEY (`id`)
 397             ) ENGINE=InnoDB",
 398         "sct2_textdefinition_drop" => "DROP TABLE IF EXISTS `sct2_textdefinition`",
 399         "sct2_textdefinition_structure" => "CREATE TABLE IF NOT EXISTS `sct2_textdefinition` (
 400             `id` bigint(20) NOT NULL,
 401             `effectiveTime` date NOT NULL,
 402             `active` int(11) NOT NULL,
 403             `moduleId` bigint(25) NOT NULL,
 404             `conceptId` bigint(20) NOT NULL,
 405             `languageCode` varchar(8) NOT NULL,
 406             `typeId` bigint(25) NOT NULL,
 407             `term` varchar(655) NOT NULL,
 408              PRIMARY KEY (`id`)
 409             ) ENGINE=InnoDB"
 410     );
 411
 412     // set up paths
 413     $dir_snomed = $GLOBALS['temporary_files_dir'] . "/SNOMED/";
 414     // $sub_path="Terminology/Content/";
 415     $sub_path = "Full/Terminology/";
 416     $dir = $dir_snomed;
 417     $dir = str_replace('\\', '/', $dir);
 418
 419     // executing the create statement for tables, these are defined in snomed_capture.inc file
 420     // this is skipped if the US extension is being added
 421     //if (!$us_extension) {
 422         //var_dump($us_extension);
 423     foreach ($table_array_for_snomed as $val) {
 424         if (trim($val) != '') {
 425             sqlStatement($val);
 426         }
 427     }
 428     //}
 429
 430     // reading the SNOMED directory and identifying the files to import and replacing the variables by originals values.
 431     if (is_dir($dir) && $handle = opendir($dir)) {
 432         while (false !== ($filename = readdir($handle))) {
 433             if ($filename != "." && $filename != ".." && !strpos($filename, "zip")) {
 434                 $path = $dir . "" . $filename . "/" . $sub_path;
 435                 if (!(is_dir($path))) {
 436                     $path = $dir . "" . $filename . "/RF2Release/" . $sub_path;
 437                 }
 438                 if (is_dir($path) && $handle1 = opendir($path)) {
 439                     while (false !== ($filename1 = readdir($handle1))) {
 440                         $load_script = "Load data local infile '#FILENAME#' into table #TABLE# fields terminated by '\\t' ESCAPED BY '' lines terminated by '\\n' ignore 1 lines   ";
 441                         $array_replace = array("#FILENAME#","#TABLE#");
 442                         if ($filename1 != "." && $filename1 != "..") {
 443                             $file_replace = $path . $filename1;
 444                             if (strpos($filename1, "Concept") !== false) {
 445                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_concept"), $load_script);
 446                             }
 447                             if (strpos($filename1, "Description") !== false) {
 448                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_description"), $load_script);
 449                             }
 450                             if (strpos($filename1, "Identifier") !== false) {
 451                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_identifier"), $load_script);
 452                             }
 453                             if (strpos($filename1, "Relationship") !== false) {
 454                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_relationship"), $load_script);
 455                             }
 456                             if (strpos($filename1, "StatedRelationship") !== false) {
 457                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_statedrelationship"), $load_script);
 458                             }
 459                             if (strpos($filename1, "TextDefinition") !== false) {
 460                                 $new_str = str_replace($array_replace, array($file_replace,"sct2_textdefinition"), $load_script);
 461                             }
 462                             if ($new_str != '') {
 463                                 sqlStatement($new_str);
 464                             }
 465                         }
 466                     }
 467                 }
 468                 closedir($handle1);
 469             }
 470         }
 471         closedir($handle);
 472     }
 473
 474     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 475     // or any manual processing that needs to occur.
 476     if (!empty($GLOBALS['kernel'])) {
 477         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('SNOMED', []);
 478         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 479     }
 480
 481     return true;
 482 }
 483
 484 // Function to import ICD tables $type differentiates ICD 9, 10 and eventually 11 (circa 2018 :-) etc.
 485 //
 486 function icd_import($type)
 487 {
 488     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 489     // or any manual processing that needs to occur.
 490     if (!empty($GLOBALS['kernel'])) {
 491         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('ICD', ['type' => $type]);
 492         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_PRE);
 493     }
 494
 495     // set up paths
 496     $dir_icd = $GLOBALS['temporary_files_dir'] . "/" . $type . "/";
 497     $dir = str_replace('\\', '/', $dir_icd);
 498     $db_load = '';
 499     $db_update = '';
 500
 501     // the incoming array is a metadata array containing keys that substr match to the incoming filename
 502     // followed by the field name, position and length of each fixed length text record in the incoming
 503     // flat files. There are separate definitions for ICD 9 and 10 based on the type passed in
 504     $incoming = array();
 505
 506     // find active revision
 507     $res = sqlQueryNoLog("SELECT max(revision) rev FROM icd10_pcs_order_code");
 508     $next_rev = ($res['rev'] ?? 0) + 1;
 509     $incoming['icd10pcs_codes_'] = array(
 510         'TABLENAME' => "icd10_pcs_order_code",
 511         'FLD1' => "pcs_code", 'POS1' => 0, 'LEN1' => 7,
 512         'FLD2' => "long_desc", 'POS2' => 8, 'LEN2' => 300,
 513         'REV' => $next_rev
 514     );
 515
 516     $res = sqlQueryNoLog("SELECT max(revision) rev FROM icd10_dx_order_code");
 517     $next_rev = ($res['rev'] ?? 0) + 1;
 518     $incoming['icd10cm_order_'] = array(
 519         'TABLENAME' => "icd10_dx_order_code",
 520         'FLD1' => "dx_code", 'POS1' => 6, 'LEN1' => 7,
 521         'FLD2' => "valid_for_coding", 'POS2' => 14, 'LEN2' => 1,
 522         'FLD3' => "short_desc", 'POS3' => 16, 'LEN3' => 60,
 523         'FLD4' => "long_desc", 'POS4' => 77, 'LEN4' => 300,
 524         'REV' => $next_rev
 525     );
 526
 527     // Settings to drastically speed up import with InnoDB
 528     sqlStatementNoLog("SET autocommit=0");
 529     sqlStatementNoLog("START TRANSACTION");
 530
 531     // first inactivate older set(s)
 532     sqlStatementNoLog("UPDATE icd10_pcs_order_code SET active = 0");
 533     sqlStatementNoLog("UPDATE icd10_dx_order_code SET active = 0");
 534
 535     if (is_dir($dir) && $handle = opendir($dir)) {
 536         while (false !== ($filename = readdir($handle))) {
 537             // bypass unwanted entries
 538             if (!stripos($filename, ".txt") || stripos($filename, "addenda")) {
 539                 continue;
 540             }
 541
 542             $keys = array_keys($incoming);
 543             while ($this_key = array_pop($keys)) {
 544                 if (stripos($filename, $this_key) !== false) {
 545                     $generator = getFileData($dir . $filename);
 546                     foreach ($generator as $value) {
 547                         $run_sql = "INSERT INTO `" . $incoming[$this_key]['TABLENAME'] . "` (";
 548                         $sql_place = "(";
 549                         $sql_values = [];
 550                         foreach (range(1, 4) as $field) {
 551                             $fld = "FLD" . $field;
 552                             $nxtfld = "FLD" . ($field + 1);
 553                             $pos = "POS" . $field;
 554                             $len = "LEN" . $field;
 555                             $run_sql .= $incoming[$this_key][$fld] . ", ";
 556                             $sql_place .= "?, ";
 557                             // concat this fields template in the sql string
 558                             array_push($sql_values, substr($value, $incoming[$this_key][$pos], $incoming[$this_key][$len]));
 559                             if (!array_key_exists($nxtfld, $incoming[$this_key])) {
 560                                 $run_sql .= "active, revision) VALUES ";
 561                                 $sql_place .= "?, ?)";
 562                                 array_push($sql_values, 1);
 563                                 array_push($sql_values, $incoming[$this_key]['REV']);
 564                                 sqlStatementNoLog($run_sql . $sql_place, $sql_values);
 565                                 break;
 566                             } else {
 567                                 $run_sql .= " ";
 568                                 $sql_place .= " ";
 569                             }
 570                         }
 571                     }
 572                 }
 573             }
 574         }
 575         // Settings to drastically speed up import with InnoDB
 576         sqlStatementNoLog("COMMIT");
 577         sqlStatementNoLog("SET autocommit=1");
 578         closedir($handle);
 579     } else {
 580         echo htmlspecialchars(xl('ERROR: No ICD import directory.'), ENT_NOQUOTES) . "<br />";
 581         return;
 582     }
 583
 584     // now update the tables where necessary
 585     sqlStatement("update `icd10_dx_order_code` SET formatted_dx_code = dx_code");
 586     sqlStatement("update `icd10_dx_order_code` SET formatted_dx_code = concat(concat(left(dx_code, 3), '.'), substr(dx_code, 4)) WHERE LENGTH(dx_code) > 3");
 587
 588     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 589     // or any manual processing that needs to occur.
 590     if (!empty($GLOBALS['kernel'])) {
 591         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('ICD', ['type' => $type]);
 592         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 593     }
 594
 595     return true;
 596 }
 597
 598 function valueset_import($type)
 599 {
 600     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 601     // or any manual processing that needs to occur.
 602     if (!empty($GLOBALS['kernel'])) {
 603         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('CQM_VALUESET', ['type' => $type]);
 604         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_PRE);
 605     }
 606
 607     $dir_valueset = $GLOBALS['temporary_files_dir'] . "/" . $type . "/";
 608     $dir = str_replace('\\', '/', $dir_valueset);
 609
 610     // Settings to drastically speed up import with InnoDB
 611     sqlStatementNoLog("SET autocommit=0");
 612     sqlStatementNoLog("START TRANSACTION");
 613     if (is_dir($dir) && $handle = opendir($dir)) {
 614         while (false !== ($filename = readdir($handle))) {
 615             // skip the zip file that's in the tmp file dir
 616             if (stripos($filename, ".zip")) {
 617                 continue;
 618             }
 619             if (stripos($filename, ".xml")) {
 620                     $abs_path = $dir . $filename;
 621                     $xml  = simplexml_load_file($abs_path, null, 0, 'ns0', true);
 622                 foreach ($xml->DescribedValueSet as $vset) {
 623                     $vset_attr = $vset->attributes();
 624                     $nqf = $vset->xpath('ns0:Group[@displayName="NQF Number"]/ns0:Keyword');
 625                     foreach ($vset->ConceptList as $cp) {
 626                         foreach ($nqf as $nqf_code) {
 627                             foreach ($cp->Concept as $con) {
 628                                 $con_attr = $con->attributes();
 629                                 sqlStatementNoLog(
 630                                     "INSERT INTO valueset values(?,?,?,?,?,?,?) on DUPLICATE KEY UPDATE
 631                                     code_system = values(code_system),
 632                                     description = values(description),
 633                                     valueset_name = values(valueset_name)",
 634                                     array(
 635                                         (string) $nqf_code,
 636                                         (string) $con_attr->code,
 637                                         (string) $con_attr->codeSystem,
 638                                         (string) $con_attr->codeSystemName,
 639                                         (string) $vset_attr->ID,
 640                                         (string) $con_attr->displayName,
 641                                         (string) $vset_attr->displayName
 642                                     )
 643                                 );
 644                                 sqlStatementNoLog(
 645                                     "INSERT INTO valueset_oid values(?,?,?,?,?,?,?) on DUPLICATE KEY UPDATE
 646                                     code_system = values(code_system),
 647                                     description = values(description),
 648                                     valueset_name = values(valueset_name)",
 649                                     array(
 650                                         (string) $nqf_code,
 651                                         (string) $vset_attr->ID,
 652                                         (string) $con_attr->codeSystem,
 653                                          'OID',
 654                                         (string) $vset_attr->ID,
 655                                         (string) $vset_attr->displayName,
 656                                         (string) $vset_attr->displayName
 657                                     )
 658                                 );
 659                             }
 660                         }
 661                     }
 662                 }
 663
 664                 sqlStatementNoLog("UPDATE valueset set code_type='SNOMED CT' where code_type='SNOMEDCT'");
 665                 sqlStatementNoLog("UPDATE valueset set code_type='ICD9' where code_type='ICD9CM'");
 666                 sqlStatementNoLog("UPDATE valueset set code_type='ICD10' where code_type='ICD10CM'");
 667             }
 668         }
 669     }
 670
 671     // Settings to drastically speed up import with InnoDB
 672     sqlStatementNoLog("COMMIT");
 673     sqlStatementNoLog("SET autocommit=1");
 674
 675     // let's fire off an event so people can listen if needed and handle any module upgrading, version checks,
 676     // or any manual processing that needs to occur.
 677     if (!empty($GLOBALS['kernel'])) {
 678         $codeTypeInstalledEvent = new CodeTypeInstalledEvent('CQM_VALUESET', ['type' => $type]);
 679         $GLOBALS['kernel']->getEventDispatcher()->dispatch($codeTypeInstalledEvent, CodeTypeInstalledEvent::EVENT_INSTALLED_POST);
 680     }
 681         return true;
 682 }
 683
 684 // Function to clean up temp files
 685 // $type (RXNORM etc.)
 686 function temp_dir_cleanup($type)
 687 {
 688     if (is_dir($GLOBALS['temporary_files_dir'] . "/" . $type)) {
 689         rmdir_recursive($GLOBALS['temporary_files_dir'] . "/" . $type);
 690     }
 691 }
 692
 693 // Function to update version tracker table if successful
 694 // $type (RXNORM etc.)
 695 function update_tracker_table($type, $revision, $version, $file_checksum)
 696 {
 697     if ($type == 'RXNORM') {
 698         sqlStatement("INSERT INTO `standardized_tables_track` (`imported_date`,`name`,`revision_date`, `revision_version`, `file_checksum`) VALUES (NOW(),'RXNORM',?,?,?)", array($revision, $version, $file_checksum));
 699         return true;
 700     } elseif ($type == 'SNOMED') {
 701         sqlStatement("INSERT INTO `standardized_tables_track` (`imported_date`,`name`,`revision_date`, `revision_version`, `file_checksum`) VALUES (NOW(),'SNOMED',?,?,?)", array($revision, $version, $file_checksum));
 702         return true;
 703     } elseif ($type == 'ICD9') {
 704         sqlStatement("INSERT INTO `standardized_tables_track` (`imported_date`,`name`,`revision_date`, `revision_version`, `file_checksum`) VALUES (NOW(),'ICD9',?,?,?)", array($revision, $version, $file_checksum));
 705         return true;
 706     } elseif ($type == 'CQM_VALUESET') {
 707         sqlStatement("INSERT INTO `standardized_tables_track` (`imported_date`,`name`,`revision_date`, `revision_version`, `file_checksum`) VALUES (NOW(),'CQM_VALUESET',?,?,?)", array($revision, $version, $file_checksum));
 708         return true;
 709     } else { // $type == 'ICD10')
 710         sqlStatement("INSERT INTO `standardized_tables_track` (`imported_date`,`name`,`revision_date`, `revision_version`, `file_checksum`) VALUES (NOW(),'ICD10',?,?,?)", array($revision, $version, $file_checksum));
 711         return true;
 712     }
 713
 714     return false;
 715 }
 716
 717 // Function to delete an entire directory
 718 function rmdir_recursive($dir)
 719 {
 720     $files = scandir($dir);
 721     array_shift($files);    // remove '.' from array
 722     array_shift($files);    // remove '..' from array
 723
 724     foreach ($files as $file) {
 725         $file = $dir . '/' . $file;
 726         if (is_dir($file)) {
 727             rmdir_recursive($file);
 728             continue;
 729         }
 730
 731         unlink($file);
 732     }
 733
 734     rmdir($dir);
 735 }
 736
 737 // function to cleanup temp, copy and unarchive the zip file
 738 function handle_zip_file($mode, $file)
 739 {
 740         // 1. copy the file to temp directory
 741     if (!temp_copy($file, $mode)) {
 742         echo htmlspecialchars(xl('ERROR: Unable to copy the file.'), ENT_NOQUOTES) . "<br />";
 743         temp_dir_cleanup($mode);
 744         exit;
 745     }
 746         // 2. unarchive the file
 747     if (!temp_unarchive($file, $mode)) {
 748         echo htmlspecialchars(xl('ERROR: Unable to extract the file.'), ENT_NOQUOTES) . "<br />";
 749         temp_dir_cleanup($mode);
 750         exit;
 751     }
 752 }
 753
 754 /**
 755  * @return Generator
 756  */
 757 function getFileData($fn)
 758 {
 759     $file = fopen($fn, 'r');
 760
 761     if (!$file) {
 762         return;
 763     }
 764
 765     while (($line = fgets($file)) !== false) {
 766         yield $line;
 767     }
 768
 769     fclose($file);
 770 }