t/unit_fixture/CXGN/Analysis.t

   1
   2 use strict;
   3 use Test::More 'no_plan';
   4 use Data::Dumper;
   5 use lib 't/lib';
   6 use SGN::Test::Fixture;
   7 use CXGN::Dataset;
   8 use CXGN::Analysis;
   9 use CXGN::Analysis::AnalysisCreate;
  10 use CXGN::AnalysisModel::GetModel;
  11 use CXGN::BreederSearch;
  12 use JSON;
  13 use Test::WWW::Mechanize;
  14 use File::Basename;
  15
  16 print STDERR "Starting test...\n";
  17 my $t = SGN::Test::Fixture->new();
  18 my $schema = $t->bcs_schema();
  19 my $dbh = $t->dbh();
  20 my $people_schema = $t->people_schema();
  21 my $metadata_schema = $t->metadata_schema();
  22 my $phenome_schema = $t->phenome_schema();
  23
  24 for my $extension ("xls", "xlsx") {
  25
  26     $dbh->begin_work();
  27
  28     my $mech = Test::WWW::Mechanize->new;
  29
  30     my $ds = CXGN::Dataset->new(people_schema => $t->people_schema(), schema => $t->bcs_schema());
  31     $ds->accessions([ 38913, 38914, 38915 ]);
  32     $ds->years([ '2012', '2013' ]);
  33     $ds->traits([ 70666, 70741 ]);
  34     $ds->trials([ 139, 144 ]);
  35     $ds->plots([ 40034, 40035 ]);
  36     $ds->name("test");
  37     $ds->description("test description");
  38
  39     $ds->name("test");
  40     $ds->description("test description");
  41     $ds->sp_person_id(41);
  42
  43     my $sp_dataset_id = $ds->store();
  44
  45     print STDERR "Dataset_id = $sp_dataset_id\n";
  46
  47     $mech->post_ok('http://localhost:3010/brapi/v1/token', [ "username" => "janedoe", "password" => "secretpw", "grant_type" => "password" ]);
  48     my $response = decode_json $mech->content;
  49     print STDERR Dumper $response;
  50     is($response->{'metadata'}->{'status'}->[2]->{'message'}, 'Login Successfull');
  51     my $sgn_session_id = $response->{access_token};
  52     print STDERR $sgn_session_id . "\n";
  53
  54     my $field_trial_id = 165;
  55     my $trait_ids = [ '77559', '77557' ];
  56     my $analysis_breeding_program_id = $schema->resultset("Project::Project")->find({ name => 'test' })->project_id();
  57
  58     $mech->post_ok('http://localhost:3010/api/drone_imagery/calculate_statistics', [ "statistics_select" => "lmer_germplasmname_replicate", "field_trial_id_list" => encode_json [ $field_trial_id ], "observation_variable_id_list" => encode_json $trait_ids ]);
  59     my $response = decode_json $mech->content;
  60     print STDERR Dumper $response;
  61     ok($response->{stats_out_tempfile});
  62     is($response->{analysis_model_type}, 'lmer_germplasmname_replicate');
  63     is_deeply($response->{unique_accessions}, [
  64         'IITA-TMS-IBA011412',
  65         'IITA-TMS-IBA30572',
  66         'IITA-TMS-IBA980002',
  67         'IITA-TMS-IBA980581',
  68         'TMEB419',
  69         'TMEB693'
  70     ]);
  71     is_deeply($response->{unique_traits}, [
  72         'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013',
  73         'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'
  74     ]);
  75     ok($response->{analysis_model_training_data_file_type});
  76     is($response->{statistical_ontology_term}, 'Univariate linear mixed model genetic BLUPs using germplasmName computed using LMER R|SGNSTAT:0000002');
  77     is($response->{application_name}, 'NickMorales Mixed Models');
  78     ok($response->{application_version});
  79     is($response->{analysis_model_language}, 'R');
  80     is($response->{analysis_result_values_type}, 'analysis_result_values_match_accession_names');
  81     ok($response->{stats_tempfile});
  82     my %result_blup_genetic_data_no_timestamps;
  83     while (my ($k, $v) = each %{$response->{result_blup_genetic_data}}) {
  84         while (my ($k1, $v1) = each %$v) {
  85             ok(defined($v1->[0]));
  86             $result_blup_genetic_data_no_timestamps{$k}->{$k1} = $v1->[0];
  87         }
  88     }
  89     is_deeply(\%result_blup_genetic_data_no_timestamps, {
  90         'IITA-TMS-IBA980002' => {
  91             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '-66.8437119136587',
  92             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0'
  93         },
  94         'TMEB419'            => {
  95             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0',
  96             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '-63.0930280648434'
  97         },
  98         'IITA-TMS-IBA30572'  => {
  99             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0',
 100             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '73.0440483006472'
 101         },
 102         'IITA-TMS-IBA980581' => {
 103             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '0.533200041783611',
 104             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0'
 105         },
 106         'IITA-TMS-IBA011412' => {
 107             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '77.8812579916274',
 108             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0'
 109         },
 110         'TMEB693'            => {
 111             'cass sink leaf|ADP alpha-D-glucoside|ug/g|week 16|COMP:0000011'  => '0',
 112             'cass sink leaf|3-phosphoglyceric acid|ug/g|week 16|COMP:0000013' => '-21.5217663555557'
 113         }
 114     });
 115
 116     my $protocol = 'lme(t1~replicate + 1|germplasmName, data=mat, na.action = na.omit)';
 117
 118     my @allowed_composed_cvs = split ',', $t->config->{composable_cvs};
 119     my $composable_cvterm_delimiter = $t->config->{composable_cvterm_delimiter};
 120     my $composable_cvterm_format = $t->config->{composable_cvterm_format};
 121
 122     print STDERR "Start add analysis...\n";
 123
 124     my $m = CXGN::Analysis::AnalysisCreate->new({
 125         bcs_schema                              => $schema,
 126         people_schema                           => $people_schema,
 127         metadata_schema                         => $metadata_schema,
 128         phenome_schema                          => $phenome_schema,
 129         archive_path                            => $t->config->{archive_path},
 130         tempfile_for_deleting_nd_experiment_ids => "/tmp/nd_experiment_id_del_file",
 131         base_path                               => $t->config->{basepath},
 132         dbhost                                  => $t->config->{dbhost},
 133         dbname                                  => $t->config->{dbname},
 134         dbuser                                  => $t->config->{dbuser},
 135         dbpass                                  => $t->config->{dbpass},
 136         analysis_to_save_boolean                => 'yes',
 137         analysis_name                           => 'analysis1',
 138         analysis_description                    => 'analysis description',
 139         analysis_year                           => '2020',
 140         analysis_breeding_program_id            => $analysis_breeding_program_id,
 141         analysis_protocol                       => $protocol,
 142         analysis_dataset_id                     => undef,
 143         analysis_accession_names                => $response->{unique_accessions},
 144         analysis_trait_names                    => $response->{unique_traits},
 145         analysis_statistical_ontology_term      => $response->{statistical_ontology_term},
 146         analysis_precomputed_design_optional    => undef,
 147         analysis_result_values                  => $response->{result_blup_genetic_data},
 148         analysis_result_values_type             => $response->{analysis_result_values_type},
 149         analysis_result_summary                 => {
 150             'genetic_variance' => 0.1,
 151             'res1'             => 1
 152         },
 153         analysis_model_name                     => 'analysismodel1',
 154         analysis_model_description              => 'analysis model description',
 155         analysis_model_is_public                => 'yes',
 156         analysis_model_language                 => $response->{analysis_model_language},
 157         analysis_model_type                     => $response->{analysis_model_type},
 158         analysis_model_properties               => {
 159             'protocol'           => $protocol,
 160             'arbitrary_property' => 0.001
 161         },
 162         analysis_model_application_name         => $response->{application_name},
 163         analysis_model_application_version      => $response->{application_version},
 164         analysis_model_file                     => undef,
 165         analysis_model_file_type                => undef,
 166         analysis_model_training_data_file       => $response->{stats_tempfile},
 167         analysis_model_training_data_file_type  => $response->{analysis_model_training_data_file_type},
 168         analysis_model_auxiliary_files          => [],
 169         allowed_composed_cvs                    => \@allowed_composed_cvs,
 170         composable_cvterm_delimiter             => $composable_cvterm_delimiter,
 171         composable_cvterm_format                => $composable_cvterm_format,
 172         user_id                                 => 41,
 173         user_name                               => 'janedoe',
 174         user_role                               => 'curator'
 175     });
 176
 177     print STDERR "Storing Analysis...\n";
 178     my $saved_analysis_object = $m->store();
 179
 180     print STDERR Dumper $saved_analysis_object;
 181     is_deeply($saved_analysis_object, {
 182         'model_id'    => 3 + ($extension eq 'xlsx' ? 2 : 0),
 183         'success'     => 1,
 184         'analysis_id' => 166 + ($extension eq 'xlsx' ? 2 : 0),
 185     });
 186     print STDERR "End add analysis...\n";
 187
 188     my $m = CXGN::AnalysisModel::GetModel->new({
 189         bcs_schema      => $schema,
 190         metadata_schema => $metadata_schema,
 191         phenome_schema  => $phenome_schema,
 192         nd_protocol_id  => $saved_analysis_object->{model_id}
 193     });
 194     my $saved_model_object = $m->get_model();
 195     print STDERR Dumper $saved_model_object;
 196     is($saved_model_object->{model_name}, 'analysismodel1');
 197     is($saved_model_object->{model_description}, 'analysis model description');
 198     is_deeply($saved_model_object->{model_properties}, {
 199         'model_is_public'     => 'yes',
 200         'dataset_id'          => undef,
 201         'protocol'            => 'lme(t1~replicate + 1|germplasmName, data=mat, na.action = na.omit)',
 202         'application_version' => 'V1.01',
 203         'application_name'    => 'NickMorales Mixed Models',
 204         'model_language'      => 'R',
 205         'arbitrary_property'  => '0.001'
 206     });
 207
 208     sleep(5);
 209     my $a = CXGN::Analysis->new({
 210         bcs_schema      => $schema,
 211         people_schema   => $people_schema,
 212         metadata_schema => $metadata_schema,
 213         phenome_schema  => $phenome_schema,
 214         trial_id        => $saved_analysis_object->{analysis_id}
 215     });
 216     my $stored_analysis_phenotypes = $a->get_phenotype_matrix();
 217     print STDERR Dumper $stored_analysis_phenotypes;
 218     is(scalar(@$stored_analysis_phenotypes), 7);
 219
 220     my $analysis_result_file = $t->config->{basepath} . "/t/data/analysis/analysis_results_accessions.csv";
 221     my $analysis_training_data_file_dummy = $t->config->{basepath} . "/t/data/trial/upload_phenotypin_spreadsheet.$extension";
 222     my $ua = LWP::UserAgent->new;
 223     my $response_upload_analysis = $ua->post(
 224         'http://localhost:3010/ajax/analysis/store/spreadsheet',
 225         Content_Type => 'form-data',
 226         Content      => [
 227             "sgn_session_id"                                    => $sgn_session_id,
 228             "upload_new_analysis_name"                          => "upload_analysis_01",
 229             "upload_new_analysis_description"                   => "analysis description",
 230             "upload_new_analysis_year"                          => "2020",
 231             "upload_new_analysis_breeding_program_id"           => $analysis_breeding_program_id,
 232             "upload_new_analysis_protocol"                      => "lm(t1 ~ some formula to describe analysis)",
 233             "upload_new_analysis_dataset_id"                    => "",
 234             upload_new_analysis_file                            => [ $analysis_result_file, basename($analysis_result_file) ],
 235             "upload_new_analysis_statistical_ontology_term"     => "Univariate linear mixed model genetic BLUPs using germplasmName computed using LMER R|SGNSTAT:0000002",
 236             "upload_new_analysis_result_values_type"            => "analysis_result_values_match_accession_names",
 237             "upload_new_analysis_result_summary_string"         => "variance:0.9,total:1.0",
 238             "upload_new_analysis_model_name"                    => "upload_analysis_model_1",
 239             "upload_new_analysis_model_description"             => "upload_analysis_model_1 description",
 240             "upload_new_analysis_model_is_public"               => "yes",
 241             "upload_new_analysis_model_language"                => "R",
 242             "upload_new_analysis_model_properties_string"       => "arbitraryprop:1,protocol_id:1",
 243             "upload_new_analysis_model_application_name"        => "myPipeline",
 244             "upload_new_analysis_model_application_version"     => "v1",
 245             upload_new_analysis_model_training_data_file        => [ $analysis_training_data_file_dummy, basename($analysis_training_data_file_dummy) ],
 246             "upload_new_analysis_model_training_data_file_type" => "myPipelinev1_training_pheno_file",
 247             upload_new_analysis_model_auxiliary_file_1          => [ $analysis_training_data_file_dummy, basename($analysis_training_data_file_dummy) ],
 248             "upload_new_analysis_model_auxiliary_file_type_1"   => "myPipelinev1_log_file",
 249             upload_new_analysis_model_auxiliary_file_2          => [ $analysis_training_data_file_dummy, basename($analysis_training_data_file_dummy) ],
 250             "upload_new_analysis_model_auxiliary_file_type_2"   => "myPipelinev1_grm_file",
 251             upload_new_analysis_model_file                      => [ $analysis_training_data_file_dummy, basename($analysis_training_data_file_dummy) ],
 252             "upload_new_analysis_model_file_type"               => "myPipelinev1_model_weights_file",
 253         ]
 254     );
 255
 256     ok($response_upload_analysis->is_success);
 257     my $message_uploaded_analysis = $response_upload_analysis->decoded_content;
 258     my $message_hash_uploaded_analysis = decode_json $message_uploaded_analysis;
 259     print STDERR Dumper $message_hash_uploaded_analysis;
 260     ok($message_hash_uploaded_analysis->{analysis_id});
 261
 262     print STDERR "ANALYSIS IDS: $message_hash_uploaded_analysis->{analysis_id} $saved_analysis_object->{analysis_id}\n";
 263
 264     sleep(5);
 265     my $a = CXGN::Analysis->new({
 266         bcs_schema      => $schema,
 267         people_schema   => $people_schema,
 268         metadata_schema => $metadata_schema,
 269         phenome_schema  => $phenome_schema,
 270         trial_id        => $message_hash_uploaded_analysis->{analysis_id}
 271     });
 272     my $stored_analysis_phenotypes = $a->get_phenotype_matrix();
 273     print STDERR Dumper $stored_analysis_phenotypes;
 274     is(scalar(@$stored_analysis_phenotypes), 6);
 275
 276     print STDERR "DELETE ANALYSIS & DATASET...\n";
 277
 278     $a->delete_phenotype_metadata($metadata_schema, $phenome_schema);
 279
 280     $a->delete_phenotype_data($t->config->{basepath}, $t->config->{dbhost}, $t->config->{dbname}, $t->config->{dbuser}, $t->config->{dbpass}, $t->config->{cluster_shared_tempdir} . "/test_temp_nd_experiment_id_delete");
 281
 282     $a->delete_field_layout();
 283
 284     $a->delete_project_entry();
 285
 286     my $b = CXGN::Analysis->new({
 287         bcs_schema      => $schema,
 288         people_schema   => $people_schema,
 289         metadata_schema => $metadata_schema,
 290         phenome_schema  => $phenome_schema,
 291         trial_id        => $saved_analysis_object->{analysis_id}
 292     });
 293
 294     $b->delete_phenotype_metadata($metadata_schema, $phenome_schema);
 295
 296     $b->delete_phenotype_data($t->config->{basepath}, $t->config->{dbhost}, $t->config->{dbname}, $t->config->{dbuser}, $t->config->{dbpass}, $t->config->{cluster_shared_tempdir} . "/test_temp_nd_experiment_id_delete");
 297
 298     $b->delete_field_layout();
 299
 300     $b->delete_project_entry();
 301
 302     $ds->delete();
 303
 304     print STDERR "REFRESHING MATVIEWS...\n";
 305
 306     CXGN::BreederSearch->new({ dbh => $dbh })->refresh_matviews($t->config->{dbhost}, $t->config->{dbname}, $t->config->{dbuser}, $t->config->{dbpass}, 'fullview', 'basic', $t->config->{basepath});
 307
 308     CXGN::BreederSearch->new({ dbh => $dbh })->refresh_matviews($t->config->{dbhost}, $t->config->{dbname}, $t->config->{dbuser}, $t->config->{dbpass}, 'stockprops', 'basic', $t->config->{basepath});
 309
 310     print STDERR "Rolling back...\n";
 311     $dbh->rollback();
 312
 313     $t->clean_up_db();
 314 }
 315 done_testing();
 316