start fixing test for multi cat phenotype upload.
[sgn.git] / t / unit_fixture / CXGN / Uploading / ValidateNIRS.t
blobe3b723f9f156c7f6418d6ddfd62067782adacd00
2 use strict;
3 use warnings;
5 use lib 't/lib';
6 use SGN::Test::Fixture;
7 use Test::More;
8 use Test::WWW::Mechanize;
9 use LWP::UserAgent;
10 use CXGN::Dataset;
11 use CXGN::Phenotypes::ParseUpload;
12 use CXGN::Phenotypes::StorePhenotypes;
14 #Needed to update IO::Socket::SSL
15 use Data::Dumper;
16 use JSON;
17 local $Data::Dumper::Indent = 0;
19 my $f = SGN::Test::Fixture->new();
21 for my $extension ("xls", "xlsx") {
23     my $schema = $f->bcs_schema;
24     my $dbh = $schema->storage->dbh;
25     my $people_schema = $f->people_schema;
27     my $mech = Test::WWW::Mechanize->new;
29     $mech->post_ok('http://localhost:3010/brapi/v1/token', [ "username" => "janedoe", "password" => "secretpw", "grant_type" => "password" ]);
30     my $response = decode_json $mech->content;
31     print STDERR Dumper $response;
32     is($response->{'metadata'}->{'status'}->[2]->{'message'}, 'Login Successfull');
33     my $sgn_session_id = $response->{access_token};
34     print STDERR $sgn_session_id . "\n";
36     my $location_rs = $schema->resultset('NaturalDiversity::NdGeolocation')->search({ description => 'Cornell Biotech' });
37     my $location_id = $location_rs->first->nd_geolocation_id;
39     my $bp_rs = $schema->resultset('Project::Project')->search({ name => 'test' });
40     my $breeding_program_id = $bp_rs->first->project_id;
42     my $tn = CXGN::Trial->new({ bcs_schema => $f->bcs_schema(), trial_id => 137 });
43     $tn->create_plant_entities(2);
45     my $parser = CXGN::Phenotypes::ParseUpload->new();
46     my $filename = "t/data/trial/upload_phenotypin_spreadsheet_large.$extension";
47     my $parsed_file = $parser->parse('phenotype spreadsheet', $filename, 0, 'plots', $f->bcs_schema);
48     ok($parsed_file, "Check if parse parse phenotype spreadsheet works");
50     my %phenotype_metadata;
51     $phenotype_metadata{'archived_file'} = $filename;
52     $phenotype_metadata{'archived_file_type'} = "spreadsheet phenotype file";
53     $phenotype_metadata{'operator'} = "janedoe";
54     $phenotype_metadata{'date'} = "2016-02-17_05:15:21";
55     my %parsed_data = %{$parsed_file->{'data'}};
56     my @plots = @{$parsed_file->{'units'}};
57     my @traits = @{$parsed_file->{'variables'}};
59     my $store_phenotypes = CXGN::Phenotypes::StorePhenotypes->new(
60         basepath                   => $f->config->{basepath},
61         dbhost                     => $f->config->{dbhost},
62         dbname                     => $f->config->{dbname},
63         dbuser                     => $f->config->{dbuser},
64         dbpass                     => $f->config->{dbpass},
65         temp_file_nd_experiment_id => $f->config->{cluster_shared_tempdir} . "/test_temp_nd_experiment_id_delete",
66         bcs_schema                 => $f->bcs_schema,
67         metadata_schema            => $f->metadata_schema,
68         phenome_schema             => $f->phenome_schema,
69         user_id                    => 41,
70         stock_list                 => \@plots,
71         trait_list                 => \@traits,
72         values_hash                => \%parsed_data,
73         has_timestamps             => 0,
74         overwrite_values           => 0,
75         metadata_hash              => \%phenotype_metadata,
76         composable_validation_check_name => $f->config->{composable_validation_check_name}
77     );
78     my ($verified_warning, $verified_error) = $store_phenotypes->verify();
79     ok(!$verified_error);
80     my ($stored_phenotype_error_msg, $store_success) = $store_phenotypes->store();
81     ok(!$stored_phenotype_error_msg, "check that store large pheno spreadsheet works");
83     print STDERR "Uploading NIRS\n";
85     my $file = $f->config->{basepath} . "/t/data/NIRS/C16Mval_spectra.csv";
87     my $ua = LWP::UserAgent->new;
88     $response = $ua->post(
89         'http://localhost:3010/ajax/highdimensionalphenotypes/nirs_upload_verify',
90         Content_Type => 'form-data',
91         Content      => [
92             upload_nirs_spreadsheet_file_input             => [ $file, 'nirs_data_upload' ],
93             "sgn_session_id"                               => $sgn_session_id,
94             "upload_nirs_spreadsheet_data_level"           => "plants",
95             "upload_nirs_spreadsheet_protocol_name"        => "NIRS SCIO Protocol",
96             "upload_nirs_spreadsheet_protocol_desc"        => "description",
97             "upload_nirs_spreadsheet_protocol_device_type" => "SCIO"
98         ]
99     );
101     #print STDERR Dumper $response;
102     ok($response->is_success);
103     my $message = $response->decoded_content;
104     print STDERR Dumper $message;
105     my $message_hash = decode_json $message;
106     print STDERR Dumper $message_hash;
107     ok($message_hash->{figure});
108     is_deeply($message_hash->{success}, [ 'File nirs_data_upload saved in archive.', 'File valid: nirs_data_upload.', 'File data successfully parsed.', 'Aggregated file data successfully parsed.', 'Aggregated file data verified. Plot names and trait names are valid.' ]);
110     my $ua = LWP::UserAgent->new;
111     $response = $ua->post(
112         'http://localhost:3010/ajax/highdimensionalphenotypes/nirs_upload_store',
113         Content_Type => 'form-data',
114         Content      => [
115             upload_nirs_spreadsheet_file_input             => [ $file, 'nirs_data_upload' ],
116             "sgn_session_id"                               => $sgn_session_id,
117             "upload_nirs_spreadsheet_data_level"           => "plants",
118             "upload_nirs_spreadsheet_protocol_name"        => "NIRS SCIO Protocol",
119             "upload_nirs_spreadsheet_protocol_desc"        => "description",
120             "upload_nirs_spreadsheet_protocol_device_type" => "SCIO"
121         ]
122     );
124     #print STDERR Dumper $response;
125     ok($response->is_success);
126     my $message = $response->decoded_content;
127     my $message_hash = decode_json $message;
128     print STDERR Dumper $message_hash;
129     ok($message_hash->{figure});
130     is(scalar(@{$message_hash->{success}}), 8);
131     is($message_hash->{success}->[6], 'All values in your file have been successfully processed!<br><br>30 new values stored<br>0 previously stored values skipped<br>0 previously stored values overwritten<br>0 previously stored values removed<br><br>');
132     my $nirs_protocol_id = $message_hash->{nd_protocol_id};
134     my $dry_matter_trait_id = $f->bcs_schema()->resultset("Cv::Cvterm")->find({ name => 'dry matter content percentage' })->cvterm_id();
136     my $ds = CXGN::Dataset->new(people_schema => $f->people_schema(), schema => $f->bcs_schema());
137     $ds->plots([
138         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial21' })->stock_id(),
139         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial22' })->stock_id(),
140         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial23' })->stock_id(),
141         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial24' })->stock_id(),
142         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial25' })->stock_id(),
143         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial26' })->stock_id(),
144         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial27' })->stock_id(),
145         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial28' })->stock_id(),
146         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial29' })->stock_id(),
147         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial210' })->stock_id(),
148         $f->bcs_schema()->resultset("Stock::Stock")->find({ uniquename => 'test_trial211' })->stock_id()
149     ]);
150     $ds->traits([
151         $dry_matter_trait_id
152     ]);
153     $ds->name("nirs_dataset_test");
154     $ds->description("test nirs description");
155     $ds->sp_person_id(41);
156     my $sp_dataset_id = $ds->store();
158     my $ua = LWP::UserAgent->new;
159     $response = $ua->post(
160         'http://localhost:3010/ajax/Nirs/generate_spectral_plot',
161         Content_Type => 'form-data',
162         Content      => [
163             dataset_id                => $sp_dataset_id,
164             "sgn_session_id"          => $sgn_session_id,
165             "nd_protocol_id"          => $nirs_protocol_id,
166             "query_associated_stocks" => "yes"
167         ]
168     );
170     #print STDERR Dumper $response;
171     ok($response->is_success);
172     my $message = $response->decoded_content;
173     my $message_hash = decode_json $message;
174     print STDERR Dumper $message_hash;
175     ok($message_hash->{figure});
177     $ua = LWP::UserAgent->new;
178     $response = $ua->post(
179         'http://localhost:3010/ajax/highdimensionalphenotypes/download_file',
180         Content_Type => 'form-data',
181         Content      => [
182             dataset_id                        => $sp_dataset_id,
183             nd_protocol_id                    => $nirs_protocol_id,
184             "sgn_session_id"                  => $sgn_session_id,
185             "high_dimensional_phenotype_type" => "NIRS",
186             "query_associated_stocks"         => "yes",
187             "download_file_type"              => "data_matrix"
188         ]
189     );
191     #print STDERR Dumper $response;
192     ok($response->is_success);
193     $message = $response->decoded_content;
194     $message_hash = decode_json $message;
195     print STDERR Dumper $message_hash;
196     ok($message_hash->{download_file_link});
198     $ua = LWP::UserAgent->new;
199     $response = $ua->post(
200         'http://localhost:3010/ajax/highdimensionalphenotypes/download_file',
201         Content_Type => 'form-data',
202         Content      => [
203             dataset_id                        => $sp_dataset_id,
204             nd_protocol_id                    => $nirs_protocol_id,
205             "sgn_session_id"                  => $sgn_session_id,
206             "high_dimensional_phenotype_type" => "NIRS",
207             "query_associated_stocks"         => "yes",
208             "download_file_type"              => "identifier_metadata"
209         ]
210     );
212     #print STDERR Dumper $response;
213     ok($response->is_success);
214     $message = $response->decoded_content;
215     $message_hash = decode_json $message;
216     print STDERR Dumper $message_hash;
217     ok($message_hash->{download_file_link});
219     $ua = LWP::UserAgent->new;
220     $response = $ua->post(
221         'http://localhost:3010/ajax/highdimensionalphenotypes/download_relationship_matrix_file',
222         Content_Type => 'form-data',
223         Content      => [
224             dataset_id                        => $sp_dataset_id,
225             nd_protocol_id                    => $nirs_protocol_id,
226             "sgn_session_id"                  => $sgn_session_id,
227             "high_dimensional_phenotype_type" => "NIRS",
228             "query_associated_stocks"         => "yes",
229         ]
230     );
232     #print STDERR Dumper $response;
233     ok($response->is_success);
234     $message = $response->decoded_content;
235     $message_hash = decode_json $message;
236     print STDERR Dumper $message_hash;
237     ok($message_hash->{download_file_link});
239     # $ua = LWP::UserAgent->new;
240     # $response = $ua->post(
241     #         'http://localhost:3010/ajax/Nirs/generate_results',
242     #         Content_Type => 'form-data',
243     #         Content => [
244     #             train_dataset_id => $sp_dataset_id,
245     #             trait_id => $dry_matter_trait_id,
246     #             "format"=>"SCIO",
247     #             "cv"=>"random",
248     #             "algorithm"=>"pls",
249     #             "niter"=>10,
250     #             "tune"=>10,
251     #             "preprocessing"=>"1",
252     #             "rf"=>0,
253     #             "sgn_session_id"=>$sgn_session_id
254     #         ]
255     #     );
256     #
257     # print STDERR Dumper $response;
258     # ok($response->is_success);
259     # $message = $response->decoded_content;
260     # $message_hash = decode_json $message;
261     # print STDERR Dumper $message_hash;
262     # ok($message_hash->{model_properties});
263     # ok($message_hash->{model_file});
264     # ok($message_hash->{training_data_file});
265     # ok($message_hash->{performance_output});
266     $f->clean_up_db();
269 done_testing();