?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Added eval; site now shows clean dataset missing message instead of server error...
[sgn.git] / lib / SGN / Controller / AJAX / DroneImagery / DroneImageryAnalytics.pm
blobde77321b1e0341123ade5d0cbacaa1bd9a03492a
1
2 =head1 NAME
3
4 SGN::Controller::AJAX::DroneImagery::DroneImageryAnalytics - a REST controller class to provide the
5 functions for drone imagery analytics
6
7 =head1 DESCRIPTION
8
9 =head1 AUTHOR
10
11 =cut
12
13 package SGN::Controller::AJAX::DroneImagery::DroneImageryAnalytics;
14
15 use strict;
16 use warnings;
17 use Moose;
18 use Data::Dumper;
19 use LWP::UserAgent;
20 use JSON;
21 use SGN::Model::Cvterm;
22 use DateTime;
23 use CXGN::UploadFile;
24 use SGN::Image;
25 use CXGN::DroneImagery::ImagesSearch;
26 use URI::Encode qw(uri_encode uri_decode);
27 use File::Basename qw | basename dirname|;
28 use File::Slurp qw(write_file);
29 use File::Temp 'tempfile';
30 use CXGN::Calendar;
31 use Image::Size;
32 use Text::CSV;
33 use CXGN::Phenotypes::StorePhenotypes;
34 use CXGN::Phenotypes::PhenotypeMatrix;
35 use CXGN::BrAPI::FileResponse;
36 use CXGN::Onto;
37 use R::YapRI::Base;
38 use R::YapRI::Data::Matrix;
39 use CXGN::Tag;
40 use CXGN::DroneImagery::ImageTypes;
41 use CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation;
42 use Time::Piece;
43 use POSIX;
44 use Math::Round;
45 use Parallel::ForkManager;
46 use CXGN::NOAANCDC;
47 use CXGN::BreederSearch;
48 use CXGN::Phenotypes::SearchFactory;
49 use CXGN::BreedersToolbox::Accessions;
50 use CXGN::Genotype::GRM;
51 use CXGN::Pedigree::ARM;
52 use CXGN::AnalysisModel::SaveModel;
53 use CXGN::AnalysisModel::GetModel;
54 use Math::Polygon;
55 use Math::Trig;
56 use List::MoreUtils qw(first_index);
57 use List::Util qw(sum);
58 use Scalar::Util qw(looks_like_number);
59 use SGN::Controller::AJAX::DroneImagery::DroneImagery;
60 use Storable qw(dclone);
61 use Statistics::Descriptive;
62
63 BEGIN { extends 'Catalyst::Controller::REST' }
64
65 __PACKAGE__->config(
66 default => 'application/json',
67 stash_key => 'rest',
68 map => { 'application/json' => 'JSON', 'text/html' => 'JSON' },
69 );
70
71 sub drone_imagery_calculate_analytics : Path('/api/drone_imagery/calculate_analytics') : ActionClass('REST') { }
72 sub drone_imagery_calculate_analytics_POST : Args(0) {
73 my $self = shift;
74 my $c = shift;
75 print STDERR Dumper $c->req->params();
76 my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef;
77 my $schema = $c->dbic_schema("Bio::Chado::Schema", undef, $sp_person_id);
78 my $people_schema = $c->dbic_schema("CXGN::People::Schema", undef, $sp_person_id);
79 my $metadata_schema = $c->dbic_schema("CXGN::Metadata::Schema", undef, $sp_person_id);
80 my $phenome_schema = $c->dbic_schema("CXGN::Phenome::Schema", undef, $sp_person_id);
81 my ($user_id, $user_name, $user_role) = _check_user_login($c);
82
83 my $statistics_select_original = $c->req->param('statistics_select');
84 my $analytics_protocol_id = $c->req->param('analytics_protocol_id');
85 my $analytics_protocol_name = $c->req->param('analytics_protocol_name');
86 my $analytics_protocol_desc = $c->req->param('analytics_protocol_desc');
87 my $sim_env_change_over_time = $c->req->param('sim_env_change_over_time') || '';
88
89 my $field_trial_id_list = $c->req->param('field_trial_id_list') ? decode_json $c->req->param('field_trial_id_list') : [];
90 my $field_trial_id_list_string = join ',', @$field_trial_id_list;
91
92 if (scalar(@$field_trial_id_list) != 1) {
93 $c->stash->{rest} = { error => "Please select one field trial!"};
94 return;
95 }
96
97 my $protocol_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'drone_imagery_analytics_env_simulation_protocol', 'protocol_type')->cvterm_id();
98 my $protocolprop_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'analytics_protocol_properties', 'protocol_property')->cvterm_id();
99 my $protocolprop_result_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'analytics_protocol_result_summary', 'protocol_property')->cvterm_id();
100 my $analytics_experiment_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'analytics_protocol_experiment', 'experiment_type')->cvterm_id();
101
102 my $protocol_properties;
103 my $analytics_nd_experiment_id;
104 my $protocol_result_summary = [];
105 my $protocol_result_summary_id;
106 if (!$analytics_protocol_id) {
107 my $q = "INSERT INTO nd_protocol (name, description, type_id) VALUES (?,?,?) RETURNING nd_protocol_id;";
108 my $h = $schema->storage->dbh()->prepare($q);
109 $h->execute($analytics_protocol_name, $analytics_protocol_desc, $protocol_type_cvterm_id);
110 ($analytics_protocol_id) = $h->fetchrow_array();
111
112 my $number_iterations = $c->req->param('number_iterations') || 2;
113 my $env_iterations;
114 for my $iterations (1..$number_iterations) {
115 my $a_env = rand(1);
116 my $b_env = rand(1);
117 my $ro_env = rand(1);
118 my $row_ro_env = rand(1);
119
120 $env_iterations->{$iterations} = {
121 a_env => $a_env,
122 b_env => $b_env,
123 ro_env => $ro_env,
124 row_ro_env => $row_ro_env,
125 };
126 }
127
128 $protocol_properties = {
129 analytics_select => $c->req->param('analytics_select') || 'minimize_local_env_effect',
130 observation_variable_id_list => $c->req->param('observation_variable_id_list') ? decode_json $c->req->param('observation_variable_id_list') : [],
131 relationship_matrix_type => $c->req->param('relationship_matrix_type') || 'genotypes',
132 htp_pheno_rel_matrix_type => $c->req->param('htp_pheno_rel_matrix_type'),
133 htp_pheno_rel_matrix_time_points => $c->req->param('htp_pheno_rel_matrix_time_points'),
134 htp_pheno_rel_matrix_blues_inversion => $c->req->param('htp_pheno_rel_matrix_blues_inversion'),
135 genotype_compute_from_parents => $c->req->param('compute_from_parents') eq 'yes' ? 1 : 0,
136 include_pedgiree_info_if_compute_from_parents => $c->req->param('include_pedgiree_info_if_compute_from_parents') eq 'yes' ? 1 : 0,
137 use_parental_grms_if_compute_from_parents => $c->req->param('use_parental_grms_if_compute_from_parents') eq 'yes' ? 1 : 0,
138 use_area_under_curve => $c->req->param('use_area_under_curve') eq 'yes' ? 1 : 0,
139 genotyping_protocol_id => $c->req->param('protocol_id'),
140 tolparinv => $c->req->param('tolparinv'),
141 legendre_order_number => $c->req->param('legendre_order_number'),
142 permanent_environment_structure => $c->req->param('permanent_environment_structure'),
143 permanent_environment_structure_phenotype_correlation_traits => decode_json $c->req->param('permanent_environment_structure_phenotype_correlation_traits'),
144 permanent_environment_structure_phenotype_trait_ids => decode_json $c->req->param('permanent_environment_structure_phenotype_trait_ids'),
145 env_variance_percent => $c->req->param('env_variance_percent') || "0.2,0.1,0.05,0.01,0.3",
146 number_iterations => $number_iterations,
147 simulated_environment_real_data_trait_id => $c->req->param('simulated_environment_real_data_trait_id'),
148 sim_env_change_over_time_correlation => $c->req->param('sim_env_change_over_time_correlation') || '0.9',
149 fixed_effect_type => $c->req->param('fixed_effect_type'),
150 fixed_effect_trait_id => $c->req->param('fixed_effect_trait_id'),
151 fixed_effect_quantiles => $c->req->param('fixed_effect_quantiles'),
152 env_iterations => $env_iterations,
153 perform_cv => $c->req->param('drone_imagery_analytics_select_perform_cv') || 0
154 };
155 my $q2 = "INSERT INTO nd_protocolprop (nd_protocol_id, value, type_id) VALUES (?,?,?);";
156 my $h2 = $schema->storage->dbh()->prepare($q2);
157 $h2->execute($analytics_protocol_id, encode_json $protocol_properties, $protocolprop_type_cvterm_id);
158
159 my $q3 = "INSERT INTO nd_protocolprop (nd_protocol_id, value, type_id) VALUES (?,?,?) RETURNING nd_protocolprop_id;";
160 my $h3 = $schema->storage->dbh()->prepare($q3);
161 $h3->execute($analytics_protocol_id, encode_json $protocol_result_summary, $protocolprop_result_type_cvterm_id);
162 ($protocol_result_summary_id) = $h3->fetchrow_array();
163
164 my $location_id = $schema->resultset("NaturalDiversity::NdGeolocation")->search({description=>'[Computation]'})->first->nd_geolocation_id();
165 my $experiment = $schema->resultset('NaturalDiversity::NdExperiment')->create({
166 nd_geolocation_id => $location_id,
167 type_id => $analytics_experiment_type_cvterm_id,
168 nd_experiment_protocols => [{nd_protocol_id => $analytics_protocol_id}]
169 });
170 $analytics_nd_experiment_id = $experiment->nd_experiment_id();
171 }
172 else {
173 my $q = "SELECT value FROM nd_protocolprop WHERE nd_protocol_id=? AND type_id=?;";
174 my $h = $schema->storage->dbh()->prepare($q);
175 $h->execute($analytics_protocol_id, $protocolprop_type_cvterm_id);
176 my ($value) = $h->fetchrow_array();
177 $protocol_properties = decode_json $value;
178
179 my $q3 = "SELECT nd_experiment.nd_experiment_id
180 FROM nd_experiment_protocol
181 JOIN nd_experiment ON(nd_experiment_protocol.nd_experiment_id = nd_experiment.nd_experiment_id)
182 WHERE nd_protocol_id=? AND nd_experiment.type_id = ?;";
183 my $h3 = $schema->storage->dbh()->prepare($q3);
184 $h3->execute($analytics_protocol_id, $analytics_experiment_type_cvterm_id);
185 ($analytics_nd_experiment_id) = $h3->fetchrow_array();
186 }
187
188 my $analytics_select = $protocol_properties->{analytics_select};
189 my $trait_id_list = $protocol_properties->{observation_variable_id_list};
190 my $compute_relationship_matrix_from_htp_phenotypes = $protocol_properties->{relationship_matrix_type};
191 my $compute_relationship_matrix_from_htp_phenotypes_type = $protocol_properties->{htp_pheno_rel_matrix_type};
192 my $compute_relationship_matrix_from_htp_phenotypes_time_points = $protocol_properties->{htp_pheno_rel_matrix_time_points};
193 my $compute_relationship_matrix_from_htp_phenotypes_blues_inversion = $protocol_properties->{htp_pheno_rel_matrix_blues_inversion};
194 my $compute_from_parents = $protocol_properties->{genotype_compute_from_parents};
195 my $include_pedgiree_info_if_compute_from_parents = $protocol_properties->{include_pedgiree_info_if_compute_from_parents};
196 my $use_parental_grms_if_compute_from_parents = $protocol_properties->{use_parental_grms_if_compute_from_parents};
197 my $use_area_under_curve = $protocol_properties->{use_area_under_curve};
198 my $protocol_id = $protocol_properties->{genotyping_protocol_id};
199 my $tolparinv = $protocol_properties->{tolparinv};
200 my $legendre_order_number = $protocol_properties->{legendre_order_number};
201 my $permanent_environment_structure = $protocol_properties->{permanent_environment_structure};
202 my $permanent_environment_structure_phenotype_correlation_traits = $protocol_properties->{permanent_environment_structure_phenotype_correlation_traits};
203 my $permanent_environment_structure_phenotype_trait_ids = $protocol_properties->{permanent_environment_structure_phenotype_trait_ids};
204 my @env_variance_percents = split ',', $protocol_properties->{env_variance_percent};
205 my $number_iterations = $protocol_properties->{number_iterations};
206 my $simulated_environment_real_data_trait_id = $protocol_properties->{simulated_environment_real_data_trait_id};
207 my $correlation_between_times = $protocol_properties->{sim_env_change_over_time_correlation} || 0.9;
208 my $fixed_effect_type = $protocol_properties->{fixed_effect_type} || 'replicate';
209 my $fixed_effect_trait_id = $protocol_properties->{fixed_effect_trait_id};
210 my $fixed_effect_quantiles = $protocol_properties->{fixed_effect_quantiles};
211 my $env_iterations = $protocol_properties->{env_iterations};
212 my $perform_cv = $protocol_properties->{perform_cv} || 0;
213
214 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
215 my $tmp_stats_dir = $shared_cluster_dir_config."/tmp_drone_statistics";
216 mkdir $tmp_stats_dir if ! -d $tmp_stats_dir;
217 my ($grm_rename_tempfile_fh, $grm_rename_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
218 $grm_rename_tempfile .= '.GRM';
219 my ($minimization_iterations_tempfile_fh, $minimization_iterations_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
220
221 my $dir = $c->tempfiles_subdir('/tmp_drone_statistics');
222 my $minimization_iterations_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
223 $minimization_iterations_tempfile_string .= '.png';
224 my $minimization_iterations_figure_tempfile = $c->config->{basepath}."/".$minimization_iterations_tempfile_string;
225
226 my $env_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
227 $env_effects_figure_tempfile_string .= '.png';
228 my $env_effects_figure_tempfile = $c->config->{basepath}."/".$env_effects_figure_tempfile_string;
229
230 my $statistics_select;
231 my $blupf90_solutions_tempfile;
232 my $yhat_residual_tempfile;
233 my $grm_file;
234
235 my $field_trial_design;
236
237 eval {
238 foreach my $field_trial_id (@$field_trial_id_list) {
239 my $field_trial_design_full = CXGN::Trial->new({bcs_schema => $schema, trial_id=>$field_trial_id})->get_layout()->get_design();
240 while (my($plot_number, $plot_obj) = each %$field_trial_design_full) {
241 my $plot_number_unique = $field_trial_id."_".$plot_number;
242 $field_trial_design->{$plot_number_unique} = {
243 stock_name => $plot_obj->{accession_name},
244 block_number => $plot_obj->{block_number},
245 col_number => $plot_obj->{col_number},
246 row_number => $plot_obj->{row_number},
247 plot_name => $plot_obj->{plot_name},
248 plot_number => $plot_number_unique,
249 rep_number => $plot_obj->{rep_number},
250 is_a_control => $plot_obj->{is_a_control}
251 };
252 }
253 }
254
255 my $drone_run_related_time_cvterms_json_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'drone_run_related_time_cvterms_json', 'project_property')->cvterm_id();
256 my $drone_run_field_trial_project_relationship_type_id_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'drone_run_on_field_trial', 'project_relationship')->cvterm_id();
257 my $drone_run_band_drone_run_project_relationship_type_id_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($schema, 'drone_run_band_on_drone_run', 'project_relationship')->cvterm_id();
258 my $drone_run_time_q = "SELECT drone_run_project.project_id, project_relationship.object_project_id, projectprop.value
259 FROM project AS drone_run_band_project
260 JOIN project_relationship AS drone_run_band_rel ON (drone_run_band_rel.subject_project_id = drone_run_band_project.project_id AND drone_run_band_rel.type_id = $drone_run_band_drone_run_project_relationship_type_id_cvterm_id)
261 JOIN project AS drone_run_project ON (drone_run_project.project_id = drone_run_band_rel.object_project_id)
262 JOIN project_relationship ON (drone_run_project.project_id = project_relationship.subject_project_id AND project_relationship.type_id=$drone_run_field_trial_project_relationship_type_id_cvterm_id)
263 LEFT JOIN projectprop ON (drone_run_band_project.project_id = projectprop.project_id AND projectprop.type_id=$drone_run_related_time_cvterms_json_cvterm_id)
264 WHERE project_relationship.object_project_id IN ($field_trial_id_list_string) ;";
265 my $h = $schema->storage->dbh()->prepare($drone_run_time_q);
266 $h->execute();
267 my $refresh_mat_views = 0;
268 while( my ($drone_run_project_id, $field_trial_project_id, $related_time_terms_json) = $h->fetchrow_array()) {
269 my $related_time_terms;
270 if (!$related_time_terms_json) {
271 $related_time_terms = SGN::Controller::AJAX::DroneImagery::DroneImagery::_perform_gdd_calculation_and_drone_run_time_saving($c, $schema, $field_trial_project_id, $drone_run_project_id, $c->config->{noaa_ncdc_access_token}, 50, 'average_daily_temp_sum');
272 $refresh_mat_views = 1;
273 }
274 else {
275 $related_time_terms = decode_json $related_time_terms_json;
276 }
277 if (!exists($related_time_terms->{gdd_average_temp})) {
278 $related_time_terms = SGN::Controller::AJAX::DroneImagery::DroneImagery::_perform_gdd_calculation_and_drone_run_time_saving($c, $schema, $field_trial_project_id, $drone_run_project_id, $c->config->{noaa_ncdc_access_token}, 50, 'average_daily_temp_sum');
279 $refresh_mat_views = 1;
280 }
281 }
282 if ($refresh_mat_views) {
283 my $bs = CXGN::BreederSearch->new( { dbh=>$c->dbc->dbh, dbname=>$c->config->{dbname}, } );
284 my $refresh = $bs->refresh_matviews($c->config->{dbhost}, $c->config->{dbname}, $c->config->{dbuser}, $c->config->{dbpass}, 'fullview', 'nonconcurrent', $c->config->{basepath});
285 sleep(10);
286 }
287 };
288
289 my ($permanent_environment_structure_tempfile_fh, $permanent_environment_structure_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
290 my ($permanent_environment_structure_env_tempfile_fh, $permanent_environment_structure_env_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
291 my ($permanent_environment_structure_env_tempfile2_fh, $permanent_environment_structure_env_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
292 my ($permanent_environment_structure_env_tempfile_mat_fh, $permanent_environment_structure_env_tempfile_mat) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
293 my ($sim_env_changing_mat_tempfile_fh, $sim_env_changing_mat_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
294 my ($sim_env_changing_mat_full_tempfile_fh, $sim_env_changing_mat_full_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
295 my ($stats_tempfile_fh, $stats_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
296 my ($stats_tempfile_2_fh, $stats_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
297 $stats_tempfile_2 .= '.dat';
298 my ($stats_tempfile_fixed_effect_binning_fh, $stats_tempfile_fixed_effect_binning) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
299 my ($stats_tempfile_fixed_effect_binned_fh, $stats_tempfile_fixed_effect_binned) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
300 my ($stats_prep_tempfile_fh, $stats_prep_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
301 my ($stats_prep_factor_tempfile_fh, $stats_prep_factor_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
302 my ($parameter_tempfile_fh, $parameter_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
303 $parameter_tempfile .= '.f90';
304 my ($parameter_asreml_tempfile_fh, $parameter_asreml_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
305 $parameter_asreml_tempfile .= '.as';
306 my ($coeff_genetic_tempfile_fh, $coeff_genetic_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
307 $coeff_genetic_tempfile .= '_genetic_coefficients.csv';
308 my ($coeff_pe_tempfile_fh, $coeff_pe_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
309 $coeff_pe_tempfile .= '_permanent_environment_coefficients.csv';
310
311 my $stats_out_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/drone_stats_XXXXX');
312 my $stats_out_tempfile = $c->config->{basepath}."/".$stats_out_tempfile_string;
313
314 my ($stats_prep2_tempfile_fh, $stats_prep2_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
315 my ($stats_out_htp_rel_tempfile_input_fh, $stats_out_htp_rel_tempfile_input) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
316 my ($stats_out_htp_rel_tempfile_fh, $stats_out_htp_rel_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
317
318 my $stats_out_htp_rel_tempfile_out_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/drone_stats_XXXXX');
319 my $stats_out_htp_rel_tempfile_out = $c->config->{basepath}."/".$stats_out_htp_rel_tempfile_out_string;
320
321 my ($stats_out_pe_pheno_rel_tempfile_fh, $stats_out_pe_pheno_rel_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
322 my ($stats_out_pe_pheno_rel_tempfile2_fh, $stats_out_pe_pheno_rel_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
323
324 my ($stats_out_pe_pheno_rel_2dspline_grm_tempfile_fh, $stats_out_pe_pheno_rel_2dspline_grm_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
325 my ($stats_out_pe_pheno_rel_tempfile3_fh, $stats_out_pe_pheno_rel_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
326 my ($stats_out_pe_pheno_rel_tempfile4_fh, $stats_out_pe_pheno_rel_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
327 my ($stats_out_pe_pheno_rel_tempfile5_fh, $stats_out_pe_pheno_rel_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
328
329 my ($stats_out_param_tempfile_fh, $stats_out_param_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
330 my ($stats_out_tempfile_row_fh, $stats_out_tempfile_row) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
331 my ($stats_out_tempfile_col_fh, $stats_out_tempfile_col) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
332 my ($stats_out_tempfile_2dspl_fh, $stats_out_tempfile_2dspl) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
333 my ($stats_out_tempfile_residual_fh, $stats_out_tempfile_residual) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
334 my ($stats_out_tempfile_genetic_fh, $stats_out_tempfile_genetic) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
335 my ($stats_out_tempfile_permanent_environment_fh, $stats_out_tempfile_permanent_environment) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
336 my ($stats_out_tempfile_varcomp_fh, $stats_out_tempfile_varcomp) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
337
338 my $time = DateTime->now();
339 my $timestamp = $time->ymd()."_".$time->hms();
340
341 my $csv = Text::CSV->new({ sep_char => "\t" });
342
343 my @legendre_coeff_exec = (
344 '1 * $b',
345 '$time * $b',
346 '(1/2*(3*$time**2 - 1)*$b)',
347 '1/2*(5*$time**3 - 3*$time)*$b',
348 '1/8*(35*$time**4 - 30*$time**2 + 3)*$b',
349 '1/16*(63*$time**5 - 70*$time**2 + 15*$time)*$b',
350 '1/16*(231*$time**6 - 315*$time**4 + 105*$time**2 - 5)*$b'
351 );
352
353 my $env_sim_exec = {
354 "linear_gradient" => '( ($a_env-$a_env_adjustment)*$row_number/$max_row + ($b_env-$b_env_adjustment)*$col_number/$max_col )',
355 "random_1d_normal_gradient" => '( (1/(2*3.14159)) * exp(-1*(( ($row_number-$row_number_adjustment) /$max_row)**2)/2) )',
356 "random_2d_normal_gradient" => '( exp( (-1/(2*(1-$ro_env**2))) * ( ( (( ($row_number-$row_number_adjustment) - $mean_row)/$max_row)**2)/($sig_row**2) + ( (( ($col_number-$col_number_adjustment) - $mean_col)/$max_col)**2)/($sig_col**2) - ((2*$ro_env)*(( ($row_number-$row_number_adjustment) - $mean_row)/$max_row)*(( ($col_number-$col_number_adjustment) - $mean_col)/$max_col) )/($sig_row*$sig_col) ) ) / (2*3.14159*$sig_row*$sig_col*sqrt(1-$ro_env**2)) )',
357 "random" => 'rand(1)'
358 };
359
360 my @plot_ids_ordered;
361 my $F;
362 my $q_time;
363 my $h_time;
364 my @seen_rows_array;
365 my @seen_cols_array;
366 my $row_stat;
367 my $mean_row;
368 my $sig_row;
369 my $col_stat;
370 my $mean_col;
371 my $sig_col;
372
373 my $spatial_effects_plots;
374 my $spatial_effects_files_store;
375 my $env_corr_res;
376 my $env_varcomps;
377
378 my (@sorted_trait_names, @unique_accession_names, @unique_plot_names, %trait_name_encoder, %trait_to_time_map);
379
380 foreach my $env_variance_percent (@env_variance_percents) {
381 for my $iterations (1..$number_iterations) {
382 print STDERR "ITERATION $iterations\n";
383
384 my $a_env = $env_iterations->{$iterations}->{a_env};
385 my $b_env = $env_iterations->{$iterations}->{b_env};
386 my $ro_env = $env_iterations->{$iterations}->{ro_env};
387 my $row_ro_env = $env_iterations->{$iterations}->{row_ro_env};
388
389 my $fixed_effect_trait_data;
390 if ($fixed_effect_type eq 'fixed_effect_trait') {
391 my $phenotypes_search_fixed_effect = CXGN::Phenotypes::SearchFactory->instantiate(
392 'MaterializedViewTable',
393 {
394 bcs_schema=>$schema,
395 data_level=>'plot',
396 trait_list=>[$fixed_effect_trait_id],
397 trial_list=>$field_trial_id_list,
398 include_timestamp=>0,
399 exclude_phenotype_outlier=>0
400 }
401 );
402 my ($fixed_effect_data, $fixed_effect_unique_traits) = $phenotypes_search_fixed_effect->search();
403
404 open(my $F, ">", $stats_tempfile_fixed_effect_binning) || die "Can't open file ".$stats_tempfile_fixed_effect_binning;
405 print $F "stock_id\tvalue_continuous\tvalue_binned\n";
406 foreach my $obs_unit (@$fixed_effect_data){
407 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
408 my $observations = $obs_unit->{observations};
409 foreach (@$observations){
410 my $value = $_->{value};
411 print $F "$obsunit_stock_id\t$value\t0\n";
412 }
413 }
414 close($F);
415
416 my $cmd_factor = 'R -e "library(data.table); library(dplyr);
417 mat <- fread(\''.$stats_tempfile_fixed_effect_binning.'\', header=TRUE, sep=\'\t\');
418 mat <- mat %>% mutate(value_binned = cut(value_continuous, breaks = unique(quantile(value_continuous,probs=seq.int(0,1, by=1/'.$fixed_effect_quantiles.'))), include.lowest=TRUE));
419 mat\$value_binned <- as.numeric(as.factor(mat\$value_binned));
420 write.table(mat, file=\''.$stats_tempfile_fixed_effect_binned.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');"';
421 my $status_factor = system($cmd_factor);
422
423 open(my $fh_factor, '<', $stats_tempfile_fixed_effect_binned) or die "Could not open file '$stats_tempfile_fixed_effect_binned' $!";
424 print STDERR "Opened $stats_tempfile_fixed_effect_binned\n";
425 my $header = <$fh_factor>;
426 if ($csv->parse($header)) {
427 my @header_cols = $csv->fields();
428 }
429
430 while (my $row = <$fh_factor>) {
431 my @columns;
432 if ($csv->parse($row)) {
433 @columns = $csv->fields();
434 }
435 my $plot_id = $columns[0];
436 my $value = $columns[1];
437 my $value_binned = $columns[2];
438 $fixed_effect_trait_data->{$plot_id} = $value_binned;
439 }
440 close($fh_factor);
441 }
442
443 my (%phenotype_data_original, @data_matrix_original, @data_matrix_phenotypes_original);
444 my (%trait_name_encoder_rev, %stock_info, %unique_accessions, %seen_days_after_plantings, %seen_times, %obsunit_row_col, %stock_row_col, %stock_name_row_col, %stock_row_col_id, %seen_rows, %seen_cols, %seen_plots, %seen_plot_names, %plot_id_map, %trait_composing_info, %seen_trial_ids, %seen_trait_names, %unique_traits_ids, @phenotype_header, $header_string);
445 my (@sorted_scaled_ln_times, %plot_id_factor_map_reverse, %plot_id_count_map_reverse, %accession_id_factor_map, %accession_id_factor_map_reverse, %time_count_map_reverse, @rep_time_factors, @ind_rep_factors, %plot_rep_time_factor_map, %seen_rep_times, %seen_ind_reps, @legs_header, %polynomial_map);
446 my $time_min = 100000000;
447 my $time_max = 0;
448 my $min_row = 10000000000;
449 my $max_row = 0;
450 my $min_col = 10000000000;
451 my $max_col = 0;
452 my $phenotype_min_original = 1000000000;
453 my $phenotype_max_original = -1000000000;
454
455 if ($statistics_select_original eq 'airemlf90_grm_random_regression_dap_blups') {
456 $statistics_select = 'airemlf90_grm_random_regression_dap_blups';
457
458 print STDERR "PREPARE RELATIONSHIP MATRIX\n";
459 eval {
460 my %seen_accession_stock_ids;
461 foreach my $trial_id (@$field_trial_id_list) {
462 my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id });
463 my $accessions = $trial->get_accessions();
464 foreach (@$accessions) {
465 $seen_accession_stock_ids{$_->{stock_id}}++;
466 }
467 }
468 my @accession_ids = keys %seen_accession_stock_ids;
469
470 if ($compute_relationship_matrix_from_htp_phenotypes eq 'genotypes') {
471
472 if ($include_pedgiree_info_if_compute_from_parents) {
473 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
474 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
475 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
476 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
477 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
478 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
479 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
480 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
481
482 if (!$protocol_id) {
483 $protocol_id = undef;
484 }
485
486 my $pedigree_arm = CXGN::Pedigree::ARM->new({
487 bcs_schema=>$schema,
488 arm_temp_file=>$arm_tempfile,
489 people_schema=>$people_schema,
490 accession_id_list=>\@accession_ids,
491 # plot_id_list=>\@plot_id_list,
492 cache_root=>$c->config->{cache_file_path},
493 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
494 });
495 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
496 $shared_cluster_dir_config,
497 $c->config->{backend},
498 $c->config->{cluster_host},
499 $c->config->{'web_cluster_queue'},
500 $c->config->{basepath}
501 );
502 # print STDERR Dumper $parent_hash;
503
504 my $female_geno = CXGN::Genotype::GRM->new({
505 bcs_schema=>$schema,
506 grm_temp_file=>$grm1_tempfile,
507 people_schema=>$people_schema,
508 cache_root=>$c->config->{cache_file_path},
509 accession_id_list=>$female_stock_ids,
510 protocol_id=>$protocol_id,
511 get_grm_for_parental_accessions=>0,
512 download_format=>'three_column_reciprocal'
513 # minor_allele_frequency=>$minor_allele_frequency,
514 # marker_filter=>$marker_filter,
515 # individuals_filter=>$individuals_filter
516 });
517 my $female_grm_data = $female_geno->download_grm(
518 'data',
519 $shared_cluster_dir_config,
520 $c->config->{backend},
521 $c->config->{cluster_host},
522 $c->config->{'web_cluster_queue'},
523 $c->config->{basepath}
524 );
525 my @fl = split '\n', $female_grm_data;
526 my %female_parent_grm;
527 foreach (@fl) {
528 my @l = split '\t', $_;
529 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
530 }
531 # print STDERR Dumper \%female_parent_grm;
532
533 my $male_geno = CXGN::Genotype::GRM->new({
534 bcs_schema=>$schema,
535 grm_temp_file=>$grm1_tempfile,
536 people_schema=>$people_schema,
537 cache_root=>$c->config->{cache_file_path},
538 accession_id_list=>$male_stock_ids,
539 protocol_id=>$protocol_id,
540 get_grm_for_parental_accessions=>0,
541 download_format=>'three_column_reciprocal'
542 # minor_allele_frequency=>$minor_allele_frequency,
543 # marker_filter=>$marker_filter,
544 # individuals_filter=>$individuals_filter
545 });
546 my $male_grm_data = $male_geno->download_grm(
547 'data',
548 $shared_cluster_dir_config,
549 $c->config->{backend},
550 $c->config->{cluster_host},
551 $c->config->{'web_cluster_queue'},
552 $c->config->{basepath}
553 );
554 my @ml = split '\n', $male_grm_data;
555 my %male_parent_grm;
556 foreach (@ml) {
557 my @l = split '\t', $_;
558 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
559 }
560 # print STDERR Dumper \%male_parent_grm;
561
562 my %rel_result_hash;
563 foreach my $a1 (@accession_ids) {
564 foreach my $a2 (@accession_ids) {
565 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
566 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
567 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
568 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
569
570 my $female_rel = 0;
571 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
572 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
573 }
574 elsif ($female_parent1 && $female_parent2 && $female_parent1 == $female_parent2) {
575 $female_rel = 1;
576 }
577 elsif ($a1 == $a2) {
578 $female_rel = 1;
579 }
580
581 my $male_rel = 0;
582 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
583 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
584 }
585 elsif ($male_parent1 && $male_parent2 && $male_parent1 == $male_parent2) {
586 $male_rel = 1;
587 }
588 elsif ($a1 == $a2) {
589 $male_rel = 1;
590 }
591 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
592
593 my $rel = 0.5*($female_rel + $male_rel);
594 $rel_result_hash{$a1}->{$a2} = $rel;
595 }
596 }
597 # print STDERR Dumper \%rel_result_hash;
598
599 my $data = '';
600 my %result_hash;
601 foreach my $s (sort @accession_ids) {
602 foreach my $c (sort @accession_ids) {
603 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
604 my $val = $rel_result_hash{$s}->{$c};
605 if (defined $val and length $val) {
606 $result_hash{$s}->{$c} = $val;
607 $data .= "S$s\tS$c\t$val\n";
608 }
609 }
610 }
611 }
612
613 # print STDERR Dumper $data;
614 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
615 print $F2 $data;
616 close($F2);
617
618 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
619 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
620 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
621 A_1 <- A_wide[,-1];
622 A_1[is.na(A_1)] <- 0;
623 A <- A_1 + t(A_1);
624 diag(A) <- diag(as.matrix(A_1));
625 E = eigen(A);
626 ev = E\$values;
627 U = E\$vectors;
628 no = dim(A)[1];
629 nev = which(ev < 0);
630 wr = 0;
631 k=length(nev);
632 if(k > 0){
633 p = ev[no - k];
634 B = sum(ev[nev])*2.0;
635 wr = (B*B*100.0)+1;
636 val = ev[nev];
637 ev[nev] = p*(B-val)*(B-val)/wr;
638 A = U%*%diag(ev)%*%t(U);
639 }
640 A <- as.data.frame(A);
641 colnames(A) <- A_wide[,1];
642 A\$stock_id <- A_wide[,1];
643 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
644 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
645 A_threecol\$variable <- substring(A_threecol\$variable, 2);
646 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
647 print STDERR $cmd."\n";
648 my $status = system($cmd);
649
650 my %rel_pos_def_result_hash;
651 open(my $F3, '<', $grm_out_tempfile) or die "Could not open file '$grm_out_tempfile' $!";
652 print STDERR "Opened $grm_out_tempfile\n";
653
654 while (my $row = <$F3>) {
655 my @columns;
656 if ($csv->parse($row)) {
657 @columns = $csv->fields();
658 }
659 my $stock_id1 = $columns[0];
660 my $stock_id2 = $columns[1];
661 my $val = $columns[2];
662 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
663 }
664 close($F3);
665
666 my $data_pos_def = '';
667 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
668 my %result_hash;
669 foreach my $s (sort @accession_ids) {
670 foreach my $c (sort @accession_ids) {
671 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
672 my $val = $rel_pos_def_result_hash{$s}->{$c};
673 if (defined $val and length $val) {
674 $result_hash{$s}->{$c} = $val;
675 $data_pos_def .= "$s\t$c\t$val\n";
676 }
677 }
678 }
679 }
680 }
681 else {
682 my %result_hash;
683 foreach my $s (sort @accession_ids) {
684 foreach my $c (sort @accession_ids) {
685 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
686 my $val = $rel_pos_def_result_hash{$s}->{$c};
687 if (defined $val and length $val) {
688 $result_hash{$s}->{$c} = $val;
689 $result_hash{$c}->{$s} = $val;
690 $data_pos_def .= "S$s\tS$c\t$val\n";
691 if ($s != $c) {
692 $data_pos_def .= "S$c\tS$s\t$val\n";
693 }
694 }
695 }
696 }
697 }
698 }
699
700 my %result_hash_2dspline_rr_structure;
701 my $data_2dspline_rr = '';
702 foreach my $s (sort @accession_ids) {
703 foreach my $c (sort @accession_ids) {
704 if (!exists($result_hash_2dspline_rr_structure{$s}->{$c}) && !exists($result_hash_2dspline_rr_structure{$c}->{$s})) {
705 my $val = $rel_pos_def_result_hash{$s}->{$c};
706 if (defined $val and length $val) {
707 $result_hash_2dspline_rr_structure{$s}->{$c} = $val;
708 $result_hash_2dspline_rr_structure{$c}->{$s} = $val;
709 $data_2dspline_rr .= "S$s\tS$c\t$val\n";
710 if ($s != $c) {
711 $data_2dspline_rr .= "S$c\tS$s\t$val\n";
712 }
713 }
714 }
715 }
716 }
717
718 open(my $F5, ">", $stats_out_pe_pheno_rel_2dspline_grm_tempfile) || die "Can't open file ".$stats_out_pe_pheno_rel_2dspline_grm_tempfile;
719 print $F5 $data_2dspline_rr;
720 close($F5);
721
722 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
723 print $F4 $data_pos_def;
724 close($F4);
725
726 $grm_file = $grm_out_posdef_tempfile;
727 }
728 elsif ($use_parental_grms_if_compute_from_parents) {
729 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
730 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
731 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
732 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
733 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
734 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
735 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
736 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
737
738 if (!$protocol_id) {
739 $protocol_id = undef;
740 }
741
742 my $pedigree_arm = CXGN::Pedigree::ARM->new({
743 bcs_schema=>$schema,
744 arm_temp_file=>$arm_tempfile,
745 people_schema=>$people_schema,
746 accession_id_list=>\@accession_ids,
747 # plot_id_list=>\@plot_id_list,
748 cache_root=>$c->config->{cache_file_path},
749 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
750 });
751 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
752 $shared_cluster_dir_config,
753 $c->config->{backend},
754 $c->config->{cluster_host},
755 $c->config->{'web_cluster_queue'},
756 $c->config->{basepath}
757 );
758 # print STDERR Dumper $parent_hash;
759
760 my $female_geno = CXGN::Genotype::GRM->new({
761 bcs_schema=>$schema,
762 grm_temp_file=>$grm1_tempfile,
763 people_schema=>$people_schema,
764 cache_root=>$c->config->{cache_file_path},
765 accession_id_list=>$female_stock_ids,
766 protocol_id=>$protocol_id,
767 get_grm_for_parental_accessions=>0,
768 download_format=>'three_column_reciprocal'
769 # minor_allele_frequency=>$minor_allele_frequency,
770 # marker_filter=>$marker_filter,
771 # individuals_filter=>$individuals_filter
772 });
773 my $female_grm_data = $female_geno->download_grm(
774 'data',
775 $shared_cluster_dir_config,
776 $c->config->{backend},
777 $c->config->{cluster_host},
778 $c->config->{'web_cluster_queue'},
779 $c->config->{basepath}
780 );
781 my @fl = split '\n', $female_grm_data;
782 my %female_parent_grm;
783 foreach (@fl) {
784 my @l = split '\t', $_;
785 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
786 }
787 # print STDERR Dumper \%female_parent_grm;
788
789 my $male_geno = CXGN::Genotype::GRM->new({
790 bcs_schema=>$schema,
791 grm_temp_file=>$grm1_tempfile,
792 people_schema=>$people_schema,
793 cache_root=>$c->config->{cache_file_path},
794 accession_id_list=>$male_stock_ids,
795 protocol_id=>$protocol_id,
796 get_grm_for_parental_accessions=>0,
797 download_format=>'three_column_reciprocal'
798 # minor_allele_frequency=>$minor_allele_frequency,
799 # marker_filter=>$marker_filter,
800 # individuals_filter=>$individuals_filter
801 });
802 my $male_grm_data = $male_geno->download_grm(
803 'data',
804 $shared_cluster_dir_config,
805 $c->config->{backend},
806 $c->config->{cluster_host},
807 $c->config->{'web_cluster_queue'},
808 $c->config->{basepath}
809 );
810 my @ml = split '\n', $male_grm_data;
811 my %male_parent_grm;
812 foreach (@ml) {
813 my @l = split '\t', $_;
814 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
815 }
816 # print STDERR Dumper \%male_parent_grm;
817
818 my %rel_result_hash;
819 foreach my $a1 (@accession_ids) {
820 foreach my $a2 (@accession_ids) {
821 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
822 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
823 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
824 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
825
826 my $female_rel = 0;
827 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
828 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
829 }
830 elsif ($a1 == $a2) {
831 $female_rel = 1;
832 }
833
834 my $male_rel = 0;
835 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
836 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
837 }
838 elsif ($a1 == $a2) {
839 $male_rel = 1;
840 }
841 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
842
843 my $rel = 0.5*($female_rel + $male_rel);
844 $rel_result_hash{$a1}->{$a2} = $rel;
845 }
846 }
847 # print STDERR Dumper \%rel_result_hash;
848
849 my $data = '';
850 my %result_hash;
851 foreach my $s (sort @accession_ids) {
852 foreach my $c (sort @accession_ids) {
853 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
854 my $val = $rel_result_hash{$s}->{$c};
855 if (defined $val and length $val) {
856 $result_hash{$s}->{$c} = $val;
857 $data .= "S$s\tS$c\t$val\n";
858 }
859 }
860 }
861 }
862
863 # print STDERR Dumper $data;
864 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
865 print $F2 $data;
866 close($F2);
867
868 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
869 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
870 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
871 A_1 <- A_wide[,-1];
872 A_1[is.na(A_1)] <- 0;
873 A <- A_1 + t(A_1);
874 diag(A) <- diag(as.matrix(A_1));
875 E = eigen(A);
876 ev = E\$values;
877 U = E\$vectors;
878 no = dim(A)[1];
879 nev = which(ev < 0);
880 wr = 0;
881 k=length(nev);
882 if(k > 0){
883 p = ev[no - k];
884 B = sum(ev[nev])*2.0;
885 wr = (B*B*100.0)+1;
886 val = ev[nev];
887 ev[nev] = p*(B-val)*(B-val)/wr;
888 A = U%*%diag(ev)%*%t(U);
889 }
890 A <- as.data.frame(A);
891 colnames(A) <- A_wide[,1];
892 A\$stock_id <- A_wide[,1];
893 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
894 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
895 A_threecol\$variable <- substring(A_threecol\$variable, 2);
896 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
897 print STDERR $cmd."\n";
898 my $status = system($cmd);
899
900 my %rel_pos_def_result_hash;
901 open(my $F3, '<', $grm_out_tempfile) or die "Could not open file '$grm_out_tempfile' $!";
902 print STDERR "Opened $grm_out_tempfile\n";
903
904 while (my $row = <$F3>) {
905 my @columns;
906 if ($csv->parse($row)) {
907 @columns = $csv->fields();
908 }
909 my $stock_id1 = $columns[0];
910 my $stock_id2 = $columns[1];
911 my $val = $columns[2];
912 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
913 }
914 close($F3);
915
916 my $data_pos_def = '';
917 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
918 my %result_hash;
919 foreach my $s (sort @accession_ids) {
920 foreach my $c (sort @accession_ids) {
921 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
922 my $val = $rel_pos_def_result_hash{$s}->{$c};
923 if (defined $val and length $val) {
924 $result_hash{$s}->{$c} = $val;
925 $data_pos_def .= "$s\t$c\t$val\n";
926 }
927 }
928 }
929 }
930 }
931 else {
932 my %result_hash;
933 foreach my $s (sort @accession_ids) {
934 foreach my $c (sort @accession_ids) {
935 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
936 my $val = $rel_pos_def_result_hash{$s}->{$c};
937 if (defined $val and length $val) {
938 $result_hash{$s}->{$c} = $val;
939 $result_hash{$c}->{$s} = $val;
940 $data_pos_def .= "S$s\tS$c\t$val\n";
941 if ($s != $c) {
942 $data_pos_def .= "S$c\tS$s\t$val\n";
943 }
944 }
945 }
946 }
947 }
948 }
949
950 my %result_hash_2dspline_rr_structure;
951 my $data_2dspline_rr = '';
952 foreach my $s (sort @accession_ids) {
953 foreach my $c (sort @accession_ids) {
954 if (!exists($result_hash_2dspline_rr_structure{$s}->{$c}) && !exists($result_hash_2dspline_rr_structure{$c}->{$s})) {
955 my $val = $rel_pos_def_result_hash{$s}->{$c};
956 if (defined $val and length $val) {
957 $result_hash_2dspline_rr_structure{$s}->{$c} = $val;
958 $result_hash_2dspline_rr_structure{$c}->{$s} = $val;
959 $data_2dspline_rr .= "S$s\tS$c\t$val\n";
960 if ($s != $c) {
961 $data_2dspline_rr .= "S$c\tS$s\t$val\n";
962 }
963 }
964 }
965 }
966 }
967
968 open(my $F5, ">", $stats_out_pe_pheno_rel_2dspline_grm_tempfile) || die "Can't open file ".$stats_out_pe_pheno_rel_2dspline_grm_tempfile;
969 print $F5 $data_2dspline_rr;
970 close($F5);
971
972 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
973 print $F4 $data_pos_def;
974 close($F4);
975
976 $grm_file = $grm_out_posdef_tempfile;
977 }
978 else {
979 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
980 my $tmp_grm_dir = $shared_cluster_dir_config."/tmp_genotype_download_grm";
981 mkdir $tmp_grm_dir if ! -d $tmp_grm_dir;
982 my ($grm_tempfile_fh, $grm_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
983 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
984
985 if (!$protocol_id) {
986 $protocol_id = undef;
987 }
988
989 my $grm_search_params = {
990 bcs_schema=>$schema,
991 grm_temp_file=>$grm_tempfile,
992 people_schema=>$people_schema,
993 cache_root=>$c->config->{cache_file_path},
994 accession_id_list=>\@accession_ids,
995 protocol_id=>$protocol_id,
996 get_grm_for_parental_accessions=>$compute_from_parents,
997 # minor_allele_frequency=>$minor_allele_frequency,
998 # marker_filter=>$marker_filter,
999 # individuals_filter=>$individuals_filter
1000 };
1001
1002 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
1003 $grm_search_params->{download_format} = 'three_column_stock_id_integer';
1004 }
1005 else {
1006 $grm_search_params->{download_format} = 'three_column_reciprocal';
1007 }
1008
1009 my $geno = CXGN::Genotype::GRM->new($grm_search_params);
1010 my $grm_data = $geno->download_grm(
1011 'data',
1012 $shared_cluster_dir_config,
1013 $c->config->{backend},
1014 $c->config->{cluster_host},
1015 $c->config->{'web_cluster_queue'},
1016 $c->config->{basepath}
1017 );
1018
1019 open(my $F2, ">", $grm_out_tempfile) || die "Can't open file ".$grm_out_tempfile;
1020 print $F2 $grm_data;
1021 close($F2);
1022 $grm_file = $grm_out_tempfile;
1023
1024 $grm_search_params->{download_format} = 'three_column_reciprocal';
1025 my $geno_rr_2dspline_env = CXGN::Genotype::GRM->new($grm_search_params);
1026 my $grm_data_rr_2dspline_env = $geno_rr_2dspline_env->download_grm(
1027 'data',
1028 $shared_cluster_dir_config,
1029 $c->config->{backend},
1030 $c->config->{cluster_host},
1031 $c->config->{'web_cluster_queue'},
1032 $c->config->{basepath}
1033 );
1034
1035 open(my $F5, ">", $stats_out_pe_pheno_rel_2dspline_grm_tempfile) || die "Can't open file ".$stats_out_pe_pheno_rel_2dspline_grm_tempfile;
1036 print $F5 $grm_data_rr_2dspline_env;
1037 close($F5);
1038 }
1039
1040 }
1041 elsif ($compute_relationship_matrix_from_htp_phenotypes eq 'htp_phenotypes') {
1042
1043 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
1044 'MaterializedViewTable',
1045 {
1046 bcs_schema=>$schema,
1047 data_level=>'plot',
1048 trial_list=>$field_trial_id_list,
1049 include_timestamp=>0,
1050 exclude_phenotype_outlier=>0
1051 }
1052 );
1053 my ($data, $unique_traits) = $phenotypes_search->search();
1054
1055 if (scalar(@$data) == 0) {
1056 $c->stash->{rest} = { error => "There are no phenotypes for the trial you have selected!"};
1057 return;
1058 }
1059
1060 my $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
1061 my $h_time = $schema->storage->dbh()->prepare($q_time);
1062
1063 my %seen_plot_names_htp_rel;
1064 my %phenotype_data_htp_rel;
1065 my %seen_times_htp_rel;
1066 foreach my $obs_unit (@$data){
1067 my $germplasm_name = $obs_unit->{germplasm_uniquename};
1068 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
1069 my $row_number = $obs_unit->{obsunit_row_number} || '';
1070 my $col_number = $obs_unit->{obsunit_col_number} || '';
1071 my $rep = $obs_unit->{obsunit_rep};
1072 my $block = $obs_unit->{obsunit_block};
1073 $seen_plot_names_htp_rel{$obs_unit->{observationunit_uniquename}} = $obs_unit;
1074 my $observations = $obs_unit->{observations};
1075 foreach (@$observations){
1076 if ($_->{associated_image_project_time_json}) {
1077 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
1078
1079 my $time_days_cvterm = $related_time_terms_json->{day};
1080 my $time_days_term_string = $time_days_cvterm;
1081 my $time_days = (split '\|', $time_days_cvterm)[0];
1082 my $time_days_value = (split ' ', $time_days)[1];
1083
1084 my $time_gdd_value = $related_time_terms_json->{gdd_average_temp} + 0;
1085 my $gdd_term_string = "GDD $time_gdd_value";
1086 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
1087 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
1088 if (!$gdd_cvterm_id) {
1089 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
1090 name => $gdd_term_string,
1091 cv => 'cxgn_time_ontology'
1092 });
1093 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
1094 }
1095 my $time_gdd_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
1096
1097 $phenotype_data_htp_rel{$obs_unit->{observationunit_uniquename}}->{$_->{trait_name}} = $_->{value};
1098 $seen_times_htp_rel{$_->{trait_name}} = [$time_days_value, $time_days_term_string, $time_gdd_value, $time_gdd_term_string];
1099 }
1100 }
1101 }
1102
1103 my @allowed_standard_htp_values = ('Nonzero Pixel Count', 'Total Pixel Sum', 'Mean Pixel Value', 'Harmonic Mean Pixel Value', 'Median Pixel Value', 'Pixel Variance', 'Pixel Standard Deviation', 'Pixel Population Standard Deviation', 'Minimum Pixel Value', 'Maximum Pixel Value', 'Minority Pixel Value', 'Minority Pixel Count', 'Majority Pixel Value', 'Majority Pixel Count', 'Pixel Group Count');
1104 my %filtered_seen_times_htp_rel;
1105 while (my ($t, $time) = each %seen_times_htp_rel) {
1106 my $allowed = 0;
1107 foreach (@allowed_standard_htp_values) {
1108 if (index($t, $_) != -1) {
1109 $allowed = 1;
1110 last;
1111 }
1112 }
1113 if ($allowed) {
1114 $filtered_seen_times_htp_rel{$t} = $time;
1115 }
1116 }
1117
1118 my @seen_plot_names_htp_rel_sorted = sort keys %seen_plot_names_htp_rel;
1119 my @filtered_seen_times_htp_rel_sorted = sort keys %filtered_seen_times_htp_rel;
1120
1121 my @header_htp = ('plot_id', 'plot_name', 'accession_id', 'accession_name', 'rep', 'block');
1122
1123 my %trait_name_encoder_htp;
1124 my %trait_name_encoder_rev_htp;
1125 my $trait_name_encoded_htp = 1;
1126 my @header_traits_htp;
1127 foreach my $trait_name (@filtered_seen_times_htp_rel_sorted) {
1128 if (!exists($trait_name_encoder_htp{$trait_name})) {
1129 my $trait_name_e = 't'.$trait_name_encoded_htp;
1130 $trait_name_encoder_htp{$trait_name} = $trait_name_e;
1131 $trait_name_encoder_rev_htp{$trait_name_e} = $trait_name;
1132 push @header_traits_htp, $trait_name_e;
1133 $trait_name_encoded_htp++;
1134 }
1135 }
1136
1137 my @htp_pheno_matrix;
1138 if ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'all') {
1139 push @header_htp, @header_traits_htp;
1140 push @htp_pheno_matrix, \@header_htp;
1141
1142 foreach my $p (@seen_plot_names_htp_rel_sorted) {
1143 my $obj = $seen_plot_names_htp_rel{$p};
1144 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
1145 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
1146 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
1147 push @row, $val;
1148 }
1149 push @htp_pheno_matrix, \@row;
1150 }
1151 }
1152 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'latest_trait') {
1153 my $max_day = 0;
1154 foreach (keys %seen_days_after_plantings) {
1155 if ($_ + 0 > $max_day) {
1156 $max_day = $_;
1157 }
1158 }
1159
1160 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
1161 my $day = $filtered_seen_times_htp_rel{$t}->[0];
1162 if ($day <= $max_day) {
1163 push @header_htp, $t;
1164 }
1165 }
1166 push @htp_pheno_matrix, \@header_htp;
1167
1168 foreach my $p (@seen_plot_names_htp_rel_sorted) {
1169 my $obj = $seen_plot_names_htp_rel{$p};
1170 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
1171 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
1172 my $day = $filtered_seen_times_htp_rel{$t}->[0];
1173 if ($day <= $max_day) {
1174 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
1175 push @row, $val;
1176 }
1177 }
1178 push @htp_pheno_matrix, \@row;
1179 }
1180 }
1181 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'vegetative') {
1182
1183 }
1184 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'reproductive') {
1185
1186 }
1187 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'mature') {
1188
1189 }
1190 else {
1191 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_time_points htp_pheno_rel_matrix_time_points is not valid!" };
1192 return;
1193 }
1194
1195 open(my $htp_pheno_f, ">", $stats_out_htp_rel_tempfile_input) || die "Can't open file ".$stats_out_htp_rel_tempfile_input;
1196 foreach (@htp_pheno_matrix) {
1197 my $line = join "\t", @$_;
1198 print $htp_pheno_f $line."\n";
1199 }
1200 close($htp_pheno_f);
1201
1202 my %rel_htp_result_hash;
1203 if ($compute_relationship_matrix_from_htp_phenotypes_type eq 'correlations') {
1204 my $htp_cmd = 'R -e "library(lme4); library(data.table);
1205 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
1206 mat_agg <- aggregate(mat[, 7:ncol(mat)], list(mat\$accession_id), mean);
1207 mat_pheno <- mat_agg[,2:ncol(mat_agg)];
1208 cor_mat <- cor(t(mat_pheno));
1209 rownames(cor_mat) <- mat_agg[,1];
1210 colnames(cor_mat) <- mat_agg[,1];
1211 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
1212 cor_mat <- range01(cor_mat);
1213 write.table(cor_mat, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
1214 print STDERR Dumper $htp_cmd;
1215 my $status = system($htp_cmd);
1216 }
1217 elsif ($compute_relationship_matrix_from_htp_phenotypes_type eq 'blues') {
1218 my $htp_cmd = 'R -e "library(lme4); library(data.table);
1219 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
1220 blues <- data.frame(id = seq(1,length(unique(mat\$accession_id))));
1221 varlist <- names(mat)[7:ncol(mat)];
1222 blues.models <- lapply(varlist, function(x) {
1223 tryCatch(
1224 lmer(substitute(i ~ 1 + (1|accession_id), list(i = as.name(x))), data = mat, REML = FALSE, control = lmerControl(optimizer =\'Nelder_Mead\', boundary.tol='.$compute_relationship_matrix_from_htp_phenotypes_blues_inversion.' ) ), error=function(e) {}
1225 )
1226 });
1227 counter = 1;
1228 for (m in blues.models) {
1229 if (!is.null(m)) {
1230 blues\$accession_id <- row.names(ranef(m)\$accession_id);
1231 blues[,ncol(blues) + 1] <- ranef(m)\$accession_id\$\`(Intercept)\`;
1232 colnames(blues)[ncol(blues)] <- varlist[counter];
1233 }
1234 counter = counter + 1;
1235 }
1236 blues_vals <- as.matrix(blues[,3:ncol(blues)]);
1237 blues_vals <- apply(blues_vals, 2, function(y) (y - mean(y)) / sd(y) ^ as.logical(sd(y)));
1238 rel <- (1/ncol(blues_vals)) * (blues_vals %*% t(blues_vals));
1239 rownames(rel) <- blues[,2];
1240 colnames(rel) <- blues[,2];
1241 write.table(rel, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
1242 print STDERR Dumper $htp_cmd;
1243 my $status = system($htp_cmd);
1244 }
1245 else {
1246 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_type htp_pheno_rel_matrix_type is not valid!" };
1247 return;
1248 }
1249
1250 open(my $htp_rel_res, '<', $stats_out_htp_rel_tempfile) or die "Could not open file '$stats_out_htp_rel_tempfile' $!";
1251 print STDERR "Opened $stats_out_htp_rel_tempfile\n";
1252 my $header_row = <$htp_rel_res>;
1253 my @header;
1254 if ($csv->parse($header_row)) {
1255 @header = $csv->fields();
1256 }
1257
1258 while (my $row = <$htp_rel_res>) {
1259 my @columns;
1260 if ($csv->parse($row)) {
1261 @columns = $csv->fields();
1262 }
1263 my $stock_id1 = $columns[0];
1264 my $counter = 1;
1265 foreach my $stock_id2 (@header) {
1266 my $val = $columns[$counter];
1267 $rel_htp_result_hash{$stock_id1}->{$stock_id2} = $val;
1268 $counter++;
1269 }
1270 }
1271 close($htp_rel_res);
1272
1273 my $data_rel_htp = '';
1274 my %result_hash;
1275 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
1276 foreach my $s (sort @accession_ids) {
1277 foreach my $c (sort @accession_ids) {
1278 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
1279 my $val = $rel_htp_result_hash{$s}->{$c};
1280 if (defined $val and length $val) {
1281 $result_hash{$s}->{$c} = $val;
1282 $data_rel_htp .= "$s\t$c\t$val\n";
1283 }
1284 }
1285 }
1286 }
1287 }
1288 else {
1289 foreach my $s (sort @accession_ids) {
1290 foreach my $c (sort @accession_ids) {
1291 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
1292 my $val = $rel_htp_result_hash{$s}->{$c};
1293 if (defined $val and length $val) {
1294 $result_hash{$s}->{$c} = $val;
1295 $result_hash{$c}->{$s} = $val;
1296 $data_rel_htp .= "S$s\tS$c\t$val\n";
1297 if ($s != $c) {
1298 $data_rel_htp .= "S$c\tS$s\t$val\n";
1299 }
1300 }
1301 }
1302 }
1303 }
1304 }
1305
1306 my %result_hash_2dspline_rr_structure;
1307 my $data_2dspline_rr = '';
1308 foreach my $s (sort @accession_ids) {
1309 foreach my $c (sort @accession_ids) {
1310 if (!exists($result_hash_2dspline_rr_structure{$s}->{$c}) && !exists($result_hash_2dspline_rr_structure{$c}->{$s})) {
1311 my $val = $rel_htp_result_hash{$s}->{$c};
1312 if (defined $val and length $val) {
1313 $result_hash_2dspline_rr_structure{$s}->{$c} = $val;
1314 $result_hash_2dspline_rr_structure{$c}->{$s} = $val;
1315 $data_2dspline_rr .= "S$s\tS$c\t$val\n";
1316 if ($s != $c) {
1317 $data_2dspline_rr .= "S$c\tS$s\t$val\n";
1318 }
1319 }
1320 }
1321 }
1322 }
1323
1324 open(my $F5, ">", $stats_out_pe_pheno_rel_2dspline_grm_tempfile) || die "Can't open file ".$stats_out_pe_pheno_rel_2dspline_grm_tempfile;
1325 print $F5 $data_2dspline_rr;
1326 close($F5);
1327
1328 open(my $htp_rel_out, ">", $stats_out_htp_rel_tempfile_out) || die "Can't open file ".$stats_out_htp_rel_tempfile_out;
1329 print $htp_rel_out $data_rel_htp;
1330 close($htp_rel_out);
1331
1332 $grm_file = $stats_out_htp_rel_tempfile_out;
1333 }
1334 else {
1335 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes is not valid!" };
1336 return;
1337 }
1338 };
1339
1340 print STDERR "PREPARE ORIGINAL PHENOTYPE FILES\n";
1341 eval {
1342 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
1343 'MaterializedViewTable',
1344 {
1345 bcs_schema=>$schema,
1346 data_level=>'plot',
1347 trait_list=>$trait_id_list,
1348 trial_list=>$field_trial_id_list,
1349 include_timestamp=>0,
1350 exclude_phenotype_outlier=>0
1351 }
1352 );
1353 my ($data, $unique_traits) = $phenotypes_search->search();
1354 @sorted_trait_names = sort keys %$unique_traits;
1355
1356 if (scalar(@$trait_id_list) < 2) {
1357 $c->stash->{rest} = { error => "Select more than 2 time points!"};
1358 return;
1359 }
1360
1361 if (scalar(@$data) == 0) {
1362 $c->stash->{rest} = { error => "There are no phenotypes for the trials and traits you have selected!"};
1363 return;
1364 }
1365
1366 $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
1367 $h_time = $schema->storage->dbh()->prepare($q_time);
1368
1369 foreach my $obs_unit (@$data){
1370 my $germplasm_name = $obs_unit->{germplasm_uniquename};
1371 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
1372 my $replicate_number = $obs_unit->{obsunit_rep} || '';
1373 my $block_number = $obs_unit->{obsunit_block} || '';
1374 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
1375 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
1376 my $row_number = $obs_unit->{obsunit_row_number} || '';
1377 my $col_number = $obs_unit->{obsunit_col_number} || '';
1378 push @plot_ids_ordered, $obsunit_stock_id;
1379
1380 if ($row_number < $min_row) {
1381 $min_row = $row_number;
1382 }
1383 elsif ($row_number >= $max_row) {
1384 $max_row = $row_number;
1385 }
1386 if ($col_number < $min_col) {
1387 $min_col = $col_number;
1388 }
1389 elsif ($col_number >= $max_col) {
1390 $max_col = $col_number;
1391 }
1392
1393 $obsunit_row_col{$row_number}->{$col_number} = {
1394 stock_id => $obsunit_stock_id,
1395 stock_uniquename => $obsunit_stock_uniquename
1396 };
1397 $seen_rows{$row_number}++;
1398 $seen_cols{$col_number}++;
1399 $plot_id_map{$obsunit_stock_id} = $obsunit_stock_uniquename;
1400 $seen_plot_names{$obsunit_stock_uniquename}++;
1401 $seen_plots{$obsunit_stock_id} = $obsunit_stock_uniquename;
1402 $stock_row_col{$obsunit_stock_id} = {
1403 row_number => $row_number,
1404 col_number => $col_number,
1405 obsunit_stock_id => $obsunit_stock_id,
1406 obsunit_name => $obsunit_stock_uniquename,
1407 rep => $replicate_number,
1408 block => $block_number,
1409 germplasm_stock_id => $germplasm_stock_id,
1410 germplasm_name => $germplasm_name
1411 };
1412 $stock_name_row_col{$obsunit_stock_uniquename} = {
1413 row_number => $row_number,
1414 col_number => $col_number,
1415 obsunit_stock_id => $obsunit_stock_id,
1416 obsunit_name => $obsunit_stock_uniquename,
1417 rep => $replicate_number,
1418 block => $block_number,
1419 germplasm_stock_id => $germplasm_stock_id,
1420 germplasm_name => $germplasm_name
1421 };
1422 $stock_row_col_id{$row_number}->{$col_number} = $obsunit_stock_id;
1423 $unique_accessions{$germplasm_name}++;
1424 $stock_info{$germplasm_stock_id} = {
1425 uniquename => $germplasm_name
1426 };
1427 my $observations = $obs_unit->{observations};
1428 foreach (@$observations){
1429 if ($_->{associated_image_project_time_json}) {
1430 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
1431 my $time;
1432 my $time_term_string = '';
1433 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups') {
1434 $time = $related_time_terms_json->{gdd_average_temp} + 0;
1435
1436 my $gdd_term_string = "GDD $time";
1437 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
1438 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
1439
1440 if (!$gdd_cvterm_id) {
1441 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
1442 name => $gdd_term_string,
1443 cv => 'cxgn_time_ontology'
1444 });
1445 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
1446 }
1447 $time_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
1448 }
1449 elsif ($statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
1450 my $time_days_cvterm = $related_time_terms_json->{day};
1451 $time_term_string = $time_days_cvterm;
1452 my $time_days = (split '\|', $time_days_cvterm)[0];
1453 $time = (split ' ', $time_days)[1] + 0;
1454
1455 $seen_days_after_plantings{$time}++;
1456 }
1457
1458 my $value = $_->{value};
1459 my $trait_name = $_->{trait_name};
1460 $phenotype_data_original{$obsunit_stock_uniquename}->{$time} = $value;
1461 $seen_times{$time} = $trait_name;
1462 $seen_trait_names{$trait_name} = $time_term_string;
1463 $trait_to_time_map{$trait_name} = $time;
1464
1465 if ($value < $phenotype_min_original) {
1466 $phenotype_min_original = $value;
1467 }
1468 elsif ($value >= $phenotype_max_original) {
1469 $phenotype_max_original = $value;
1470 }
1471 }
1472 }
1473 }
1474 if (scalar(keys %seen_times) == 0) {
1475 $c->stash->{rest} = { error => "There are no phenotypes with associated days after planting time associated to the traits you have selected!"};
1476 return;
1477 }
1478
1479 @sorted_trait_names = sort {$a <=> $b} keys %seen_times;
1480 # print STDERR Dumper \@sorted_trait_names;
1481
1482 my $trait_name_encoded = 1;
1483 foreach my $trait_name (@sorted_trait_names) {
1484 if (!exists($trait_name_encoder{$trait_name})) {
1485 my $trait_name_e = 't'.$trait_name_encoded;
1486 $trait_name_encoder{$trait_name} = $trait_name_e;
1487 $trait_name_encoder_rev{$trait_name_e} = $trait_name;
1488 $trait_name_encoded++;
1489 }
1490 }
1491
1492 foreach (@sorted_trait_names) {
1493 if ($_ < $time_min) {
1494 $time_min = $_;
1495 }
1496 if ($_ >= $time_max) {
1497 $time_max = $_;
1498 }
1499 }
1500 print STDERR Dumper [$time_min, $time_max];
1501
1502 while ( my ($trait_name, $time_term) = each %seen_trait_names) {
1503 push @{$trait_composing_info{$trait_name}}, $time_term;
1504 }
1505
1506 @unique_plot_names = sort keys %seen_plot_names;
1507 if ($legendre_order_number >= scalar(@sorted_trait_names)) {
1508 $legendre_order_number = scalar(@sorted_trait_names) - 1;
1509 }
1510
1511 my @sorted_trait_names_scaled;
1512 my $leg_pos_counter = 0;
1513 foreach (@sorted_trait_names) {
1514 my $scaled_time = ($_ - $time_min)/($time_max - $time_min);
1515 push @sorted_trait_names_scaled, $scaled_time;
1516 if ($leg_pos_counter < $legendre_order_number+1) {
1517 push @sorted_scaled_ln_times, log($scaled_time+0.0001);
1518 }
1519 $leg_pos_counter++;
1520 }
1521 my $sorted_trait_names_scaled_string = join ',', @sorted_trait_names_scaled;
1522
1523 my $cmd = 'R -e "library(sommer); library(orthopolynom);
1524 polynomials <- leg(c('.$sorted_trait_names_scaled_string.'), n='.$legendre_order_number.', intercept=TRUE);
1525 write.table(polynomials, file=\''.$stats_out_tempfile.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');"';
1526 my $status = system($cmd);
1527
1528 open(my $fh, '<', $stats_out_tempfile) or die "Could not open file '$stats_out_tempfile' $!";
1529 print STDERR "Opened $stats_out_tempfile\n";
1530 my $header = <$fh>;
1531 my @header_cols;
1532 if ($csv->parse($header)) {
1533 @header_cols = $csv->fields();
1534 }
1535
1536 my $p_counter = 0;
1537 while (my $row = <$fh>) {
1538 my @columns;
1539 if ($csv->parse($row)) {
1540 @columns = $csv->fields();
1541 }
1542 my $time = $sorted_trait_names[$p_counter];
1543 $polynomial_map{$time} = \@columns;
1544 $p_counter++;
1545 }
1546 close($fh);
1547
1548 open(my $F_prep, ">", $stats_prep_tempfile) || die "Can't open file ".$stats_prep_tempfile;
1549 print $F_prep "accession_id,accession_id_factor,plot_id,plot_id_factor,replicate,time,replicate_time,ind_replicate,fixed_effect_trait_binned\n";
1550 foreach my $p (@unique_plot_names) {
1551 my $replicate = $stock_name_row_col{$p}->{rep};
1552 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
1553 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
1554 my $fixed_effect_trait_value = defined($fixed_effect_trait_data->{$obsunit_stock_id}) ? $fixed_effect_trait_data->{$obsunit_stock_id} : 0;
1555 foreach my $t (@sorted_trait_names) {
1556 print $F_prep "$germplasm_stock_id,,$obsunit_stock_id,,$replicate,$t,$replicate"."_"."$t,$germplasm_stock_id"."_"."$replicate,$fixed_effect_trait_value\n";
1557 }
1558 }
1559 close($F_prep);
1560
1561 my $cmd_factor = 'R -e "library(data.table); library(dplyr);
1562 mat <- fread(\''.$stats_prep_tempfile.'\', header=TRUE, sep=\',\');
1563 mat\$replicate_time <- as.numeric(as.factor(mat\$replicate_time));
1564 mat\$ind_replicate <- as.numeric(as.factor(mat\$ind_replicate));
1565 mat\$accession_id_factor <- as.numeric(as.factor(mat\$accession_id));
1566 mat\$plot_id_factor <- as.numeric(as.factor(mat\$plot_id));
1567 ';
1568 if ($fixed_effect_type eq 'fixed_effect_trait') {
1569 $cmd_factor .= 'mat\$replicate_time <- as.numeric(as.factor(paste(mat\$fixed_effect_trait_binned, mat\$time, sep=\'_\')));';
1570 }
1571 $cmd_factor .= 'write.table(mat, file=\''.$stats_prep_factor_tempfile.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');"';
1572 my $status_factor = system($cmd_factor);
1573
1574 open(my $fh_factor, '<', $stats_prep_factor_tempfile) or die "Could not open file '$stats_prep_factor_tempfile' $!";
1575 print STDERR "Opened $stats_prep_factor_tempfile\n";
1576 $header = <$fh_factor>;
1577 if ($csv->parse($header)) {
1578 @header_cols = $csv->fields();
1579 }
1580
1581 my $line_factor_count = 0;
1582 while (my $row = <$fh_factor>) {
1583 my @columns;
1584 if ($csv->parse($row)) {
1585 @columns = $csv->fields();
1586 }
1587 my $accession_id = $columns[0];
1588 my $accession_id_factor = $columns[1];
1589 my $plot_id = $columns[2];
1590 my $plot_id_factor = $columns[3];
1591 my $rep = $columns[4];
1592 my $time = $columns[5];
1593 my $rep_time = $columns[6];
1594 my $ind_rep = $columns[7];
1595 $stock_row_col{$plot_id}->{plot_id_factor} = $plot_id_factor;
1596 $stock_name_row_col{$plot_id_map{$plot_id}}->{plot_id_factor} = $plot_id_factor;
1597 $plot_rep_time_factor_map{$plot_id}->{$rep}->{$time} = $rep_time;
1598 $seen_rep_times{$rep_time}++;
1599 $seen_ind_reps{$plot_id_factor}++;
1600 $accession_id_factor_map{$accession_id} = $accession_id_factor;
1601 $accession_id_factor_map_reverse{$accession_id_factor} = $stock_info{$accession_id}->{uniquename};
1602 $plot_id_factor_map_reverse{$plot_id_factor} = $seen_plots{$plot_id};
1603 $plot_id_count_map_reverse{$line_factor_count} = $seen_plots{$plot_id};
1604 $time_count_map_reverse{$line_factor_count} = $time;
1605 $line_factor_count++;
1606 }
1607 close($fh_factor);
1608 @rep_time_factors = sort keys %seen_rep_times;
1609 @ind_rep_factors = sort keys %seen_ind_reps;
1610
1611 foreach my $p (@unique_plot_names) {
1612 my $row_number = $stock_name_row_col{$p}->{row_number};
1613 my $col_number = $stock_name_row_col{$p}->{col_number};
1614 my $replicate = $stock_name_row_col{$p}->{rep};
1615 my $block = $stock_name_row_col{$p}->{block};
1616 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
1617 my $germplasm_name = $stock_name_row_col{$p}->{germplasm_name};
1618 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
1619
1620 my @data_matrix_phenotypes_row;
1621 my $current_trait_index = 0;
1622 foreach my $t (@sorted_trait_names) {
1623 my @row = (
1624 $accession_id_factor_map{$germplasm_stock_id},
1625 $obsunit_stock_id,
1626 $replicate,
1627 $t,
1628 $plot_rep_time_factor_map{$obsunit_stock_id}->{$replicate}->{$t},
1629 $stock_row_col{$obsunit_stock_id}->{plot_id_factor}
1630 );
1631
1632 my $polys = $polynomial_map{$t};
1633 push @row, @$polys;
1634
1635 if (defined($phenotype_data_original{$p}->{$t})) {
1636 if ($use_area_under_curve) {
1637 my $val = 0;
1638 foreach my $counter (0..$current_trait_index) {
1639 if ($counter == 0) {
1640 $val = $val + $phenotype_data_original{$p}->{$sorted_trait_names[$counter]} + 0;
1641 }
1642 else {
1643 my $t1 = $sorted_trait_names[$counter-1];
1644 my $t2 = $sorted_trait_names[$counter];
1645 my $p1 = $phenotype_data_original{$p}->{$t1} + 0;
1646 my $p2 = $phenotype_data_original{$p}->{$t2} + 0;
1647 my $neg = 1;
1648 my $min_val = $p1;
1649 if ($p2 < $p1) {
1650 $neg = -1;
1651 $min_val = $p2;
1652 }
1653 $val = $val + (($neg*($p2-$p1)*($t2-$t1))/2)+($t2-$t1)*$min_val;
1654 }
1655 }
1656
1657 push @row, $val;
1658 push @data_matrix_phenotypes_row, $val;
1659 }
1660 else {
1661 push @row, $phenotype_data_original{$p}->{$t} + 0;
1662 push @data_matrix_phenotypes_row, $phenotype_data_original{$p}->{$t} + 0;
1663 }
1664 } else {
1665 print STDERR $p." : $t : $germplasm_name : NA \n";
1666 push @row, '';
1667 push @data_matrix_phenotypes_row, 'NA';
1668 }
1669
1670 push @data_matrix_original, \@row;
1671 push @data_matrix_phenotypes_original, \@data_matrix_phenotypes_row;
1672
1673 $current_trait_index++;
1674 }
1675 }
1676
1677 for (0..$legendre_order_number) {
1678 push @legs_header, "legendre$_";
1679 }
1680 @phenotype_header = ("id", "plot_id", "replicate", "time", "replicate_time", "ind_replicate", @legs_header, "phenotype");
1681 open($F, ">", $stats_tempfile_2) || die "Can't open file ".$stats_tempfile_2;
1682 foreach (@data_matrix_original) {
1683 my $line = join ' ', @$_;
1684 print $F "$line\n";
1685 }
1686 close($F);
1687
1688 open(my $F2, ">", $stats_prep2_tempfile) || die "Can't open file ".$stats_prep2_tempfile;
1689 foreach (@data_matrix_phenotypes_original) {
1690 my $line = join ',', @$_;
1691 print $F2 "$line\n";
1692 }
1693 close($F2);
1694
1695 if ($permanent_environment_structure eq 'euclidean_rows_and_columns') {
1696 my $data = '';
1697 my %euclidean_distance_hash;
1698 my $min_euc_dist = 10000000000000000000;
1699 my $max_euc_dist = 0;
1700 foreach my $s (sort { $a <=> $b } @plot_ids_ordered) {
1701 foreach my $r (sort { $a <=> $b } @plot_ids_ordered) {
1702 my $s_factor = $stock_name_row_col{$plot_id_map{$s}}->{plot_id_factor};
1703 my $r_factor = $stock_name_row_col{$plot_id_map{$r}}->{plot_id_factor};
1704 if (!exists($euclidean_distance_hash{$s_factor}->{$r_factor}) && !exists($euclidean_distance_hash{$r_factor}->{$s_factor})) {
1705 my $row_1 = $stock_name_row_col{$plot_id_map{$s}}->{row_number};
1706 my $col_1 = $stock_name_row_col{$plot_id_map{$s}}->{col_number};
1707 my $row_2 = $stock_name_row_col{$plot_id_map{$r}}->{row_number};
1708 my $col_2 = $stock_name_row_col{$plot_id_map{$r}}->{col_number};
1709 my $dist = sqrt( ($row_2 - $row_1)**2 + ($col_2 - $col_1)**2 );
1710 if ($dist != 0) {
1711 $dist = 1/$dist;
1712 }
1713 if (defined $dist and length $dist) {
1714 $euclidean_distance_hash{$s_factor}->{$r_factor} = $dist;
1715
1716 if ($dist < $min_euc_dist) {
1717 $min_euc_dist = $dist;
1718 }
1719 elsif ($dist > $max_euc_dist) {
1720 $max_euc_dist = $dist;
1721 }
1722 }
1723 else {
1724 $c->stash->{rest} = { error => "There are not rows and columns for all of the plots! Do not try to use a Euclidean distance between plots for the permanent environment structure"};
1725 return;
1726 }
1727 }
1728 }
1729 }
1730
1731 foreach my $r (sort { $a <=> $b } keys %euclidean_distance_hash) {
1732 foreach my $s (sort { $a <=> $b } keys %{$euclidean_distance_hash{$r}}) {
1733 my $val = $euclidean_distance_hash{$r}->{$s};
1734 if (defined $val and length $val) {
1735 my $val_scaled = ($val-$min_euc_dist)/($max_euc_dist-$min_euc_dist);
1736 $data .= "$r\t$s\t$val_scaled\n";
1737 }
1738 }
1739 }
1740
1741 open(my $F3, ">", $permanent_environment_structure_tempfile) || die "Can't open file ".$permanent_environment_structure_tempfile;
1742 print $F3 $data;
1743 close($F3);
1744 }
1745 elsif ($permanent_environment_structure eq 'phenotype_correlation') {
1746 my $phenotypes_search_permanent_environment_structure = CXGN::Phenotypes::SearchFactory->instantiate(
1747 'MaterializedViewTable',
1748 {
1749 bcs_schema=>$schema,
1750 data_level=>'plot',
1751 trial_list=>$field_trial_id_list,
1752 trait_list=>$permanent_environment_structure_phenotype_correlation_traits,
1753 include_timestamp=>0,
1754 exclude_phenotype_outlier=>0
1755 }
1756 );
1757 my ($data_permanent_environment_structure, $unique_traits_permanent_environment_structure) = $phenotypes_search_permanent_environment_structure->search();
1758
1759 if (scalar(@$data_permanent_environment_structure) == 0) {
1760 $c->stash->{rest} = { error => "There are no phenotypes for the permanent environment structure traits you have selected!"};
1761 return;
1762 }
1763
1764 my %seen_plot_names_pe_rel;
1765 my %phenotype_data_pe_rel;
1766 my %seen_traits_pe_rel;
1767 foreach my $obs_unit (@$data_permanent_environment_structure){
1768 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
1769 my $germplasm_name = $obs_unit->{germplasm_uniquename};
1770 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
1771 my $row_number = $obs_unit->{obsunit_row_number} || '';
1772 my $col_number = $obs_unit->{obsunit_col_number} || '';
1773 my $rep = $obs_unit->{obsunit_rep};
1774 my $block = $obs_unit->{obsunit_block};
1775 $seen_plot_names_pe_rel{$obsunit_stock_uniquename} = $obs_unit;
1776 my $observations = $obs_unit->{observations};
1777 foreach (@$observations){
1778 $phenotype_data_pe_rel{$obsunit_stock_uniquename}->{$_->{trait_name}} = $_->{value};
1779 $seen_traits_pe_rel{$_->{trait_name}}++;
1780 }
1781 }
1782
1783 my @seen_plot_names_pe_rel_sorted = sort keys %seen_plot_names_pe_rel;
1784 my @seen_traits_pe_rel_sorted = sort keys %seen_traits_pe_rel;
1785
1786 my @header_pe = ('plot_id');
1787
1788 my %trait_name_encoder_pe;
1789 my %trait_name_encoder_rev_pe;
1790 my $trait_name_encoded_pe = 1;
1791 my @header_traits_pe;
1792 foreach my $trait_name (@seen_traits_pe_rel_sorted) {
1793 if (!exists($trait_name_encoder_pe{$trait_name})) {
1794 my $trait_name_e = 't'.$trait_name_encoded_pe;
1795 $trait_name_encoder_pe{$trait_name} = $trait_name_e;
1796 $trait_name_encoder_rev_pe{$trait_name_e} = $trait_name;
1797 push @header_traits_pe, $trait_name_e;
1798 $trait_name_encoded_pe++;
1799 }
1800 }
1801
1802 my @pe_pheno_matrix;
1803 push @header_pe, @header_traits_pe;
1804 push @pe_pheno_matrix, \@header_pe;
1805
1806 foreach my $p (@seen_plot_names_pe_rel_sorted) {
1807 my @row = ($stock_name_row_col{$p}->{plot_id_factor});
1808 foreach my $t (@seen_traits_pe_rel_sorted) {
1809 my $val = $phenotype_data_pe_rel{$p}->{$t} + 0;
1810 push @row, $val;
1811 }
1812 push @pe_pheno_matrix, \@row;
1813 }
1814
1815 open(my $pe_pheno_f, ">", $stats_out_pe_pheno_rel_tempfile) || die "Can't open file ".$stats_out_pe_pheno_rel_tempfile;
1816 foreach (@pe_pheno_matrix) {
1817 my $line = join "\t", @$_;
1818 print $pe_pheno_f $line."\n";
1819 }
1820 close($pe_pheno_f);
1821
1822 my %rel_pe_result_hash;
1823 my $pe_rel_cmd = 'R -e "library(lme4); library(data.table);
1824 mat_agg <- fread(\''.$stats_out_pe_pheno_rel_tempfile.'\', header=TRUE, sep=\'\t\');
1825 mat_pheno <- mat_agg[,-1];
1826 cor_mat <- cor(t(mat_pheno));
1827 rownames(cor_mat) <- mat_agg\$plot_id;
1828 colnames(cor_mat) <- mat_agg\$plot_id;
1829 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
1830 cor_mat <- range01(cor_mat);
1831 write.table(cor_mat, file=\''.$stats_out_pe_pheno_rel_tempfile2.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
1832 # print STDERR Dumper $pe_rel_cmd;
1833 my $status_pe_rel = system($pe_rel_cmd);
1834
1835 open(my $pe_rel_res, '<', $stats_out_pe_pheno_rel_tempfile2) or die "Could not open file '$stats_out_pe_pheno_rel_tempfile2' $!";
1836 print STDERR "Opened $stats_out_pe_pheno_rel_tempfile2\n";
1837 my $header_row = <$pe_rel_res>;
1838 my @header;
1839 if ($csv->parse($header_row)) {
1840 @header = $csv->fields();
1841 }
1842
1843 while (my $row = <$pe_rel_res>) {
1844 my @columns;
1845 if ($csv->parse($row)) {
1846 @columns = $csv->fields();
1847 }
1848 my $stock_id1 = $columns[0];
1849 my $counter = 1;
1850 foreach my $stock_id2 (@header) {
1851 my $val = $columns[$counter];
1852 $rel_pe_result_hash{$stock_id1}->{$stock_id2} = $val;
1853 $counter++;
1854 }
1855 }
1856 close($pe_rel_res);
1857
1858 my $data_rel_pe = '';
1859 my %result_hash_pe;
1860 foreach my $s (sort { $a <=> $b } @plot_ids_ordered) {
1861 foreach my $r (sort { $a <=> $b } @plot_ids_ordered) {
1862 my $s_factor = $stock_name_row_col{$plot_id_map{$s}}->{plot_id_factor};
1863 my $r_factor = $stock_name_row_col{$plot_id_map{$r}}->{plot_id_factor};
1864 if (!exists($result_hash_pe{$s_factor}->{$r_factor}) && !exists($result_hash_pe{$r_factor}->{$s_factor})) {
1865 $result_hash_pe{$s_factor}->{$r_factor} = $rel_pe_result_hash{$s_factor}->{$r_factor};
1866 }
1867 }
1868 }
1869 foreach my $r (sort { $a <=> $b } keys %result_hash_pe) {
1870 foreach my $s (sort { $a <=> $b } keys %{$result_hash_pe{$r}}) {
1871 my $val = $result_hash_pe{$r}->{$s};
1872 if (defined $val and length $val) {
1873 $data_rel_pe .= "$r\t$s\t$val\n";
1874 }
1875 }
1876 }
1877
1878 open(my $pe_rel_out, ">", $permanent_environment_structure_tempfile) || die "Can't open file ".$permanent_environment_structure_tempfile;
1879 print $pe_rel_out $data_rel_pe;
1880 close($pe_rel_out);
1881 }
1882 elsif ($permanent_environment_structure eq 'phenotype_2dspline_effect') {
1883 my $phenotypes_search_permanent_environment_structure = CXGN::Phenotypes::SearchFactory->instantiate(
1884 'MaterializedViewTable',
1885 {
1886 bcs_schema=>$schema,
1887 data_level=>'plot',
1888 trial_list=>$field_trial_id_list,
1889 trait_list=>$permanent_environment_structure_phenotype_trait_ids,
1890 include_timestamp=>0,
1891 exclude_phenotype_outlier=>0
1892 }
1893 );
1894 my ($data_permanent_environment_structure, $unique_traits_permanent_environment_structure) = $phenotypes_search_permanent_environment_structure->search();
1895 my @sorted_trait_names_permanent_environment_structure = sort keys %$unique_traits_permanent_environment_structure;
1896
1897 if (scalar(@$data_permanent_environment_structure) == 0) {
1898 $c->stash->{rest} = { error => "There are no phenotypes for the permanent environment structure traits you have selected!"};
1899 return;
1900 }
1901
1902 my %seen_plot_names_pe;
1903 my %phenotype_data_pe;
1904 my %stock_name_row_col_pe;
1905 foreach my $obs_unit (@$data_permanent_environment_structure){
1906 my $germplasm_name = $obs_unit->{germplasm_uniquename};
1907 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
1908 my $replicate_number = $obs_unit->{obsunit_rep} || '';
1909 my $block_number = $obs_unit->{obsunit_block} || '';
1910 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
1911 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
1912 my $row_number = $obs_unit->{obsunit_row_number} || '';
1913 my $col_number = $obs_unit->{obsunit_col_number} || '';
1914 $seen_plot_names_pe{$obsunit_stock_uniquename}++;
1915 $stock_name_row_col_pe{$obsunit_stock_uniquename} = {
1916 row_number => $row_number,
1917 col_number => $col_number,
1918 obsunit_stock_id => $obsunit_stock_id,
1919 obsunit_name => $obsunit_stock_uniquename,
1920 rep => $replicate_number,
1921 block => $block_number,
1922 germplasm_stock_id => $germplasm_stock_id,
1923 germplasm_name => $germplasm_name
1924 };
1925 my $observations = $obs_unit->{observations};
1926 foreach (@$observations){
1927 my $value = $_->{value};
1928 my $trait_name = $_->{trait_name};
1929 $phenotype_data_pe{$obsunit_stock_uniquename}->{$trait_name} = $value;
1930 }
1931 }
1932
1933 my @unique_plot_names_pe = sort keys %seen_plot_names_pe;
1934
1935 my %trait_name_encoder_permanent_environment_structure;
1936 my %trait_name_encoder_rev_permanent_environment_structure;
1937 my $trait_name_encoded_pe = 1;
1938 foreach my $trait_name (@sorted_trait_names_permanent_environment_structure) {
1939 if (!exists($trait_name_encoder_permanent_environment_structure{$trait_name})) {
1940 my $trait_name_e = 't'.$trait_name_encoded_pe;
1941 $trait_name_encoder_permanent_environment_structure{$trait_name} = $trait_name_e;
1942 $trait_name_encoder_rev_permanent_environment_structure{$trait_name_e} = $trait_name;
1943 $trait_name_encoded_pe++;
1944 }
1945 }
1946
1947 my @data_matrix_pe;
1948 foreach my $p (@unique_plot_names_pe) {
1949 my $row_number = $stock_name_row_col_pe{$p}->{row_number};
1950 my $col_number = $stock_name_row_col_pe{$p}->{col_number};
1951 my $replicate = $stock_name_row_col_pe{$p}->{rep};
1952 my $block = $stock_name_row_col_pe{$p}->{block};
1953 my $germplasm_stock_id = $stock_name_row_col_pe{$p}->{germplasm_stock_id};
1954 my $germplasm_name = $stock_name_row_col_pe{$p}->{germplasm_name};
1955 my $obsunit_stock_id = $stock_name_row_col_pe{$p}->{obsunit_stock_id};
1956
1957 my @row = ($replicate, $block, "S".$germplasm_stock_id, $obsunit_stock_id, $row_number, $col_number, $row_number, $col_number);
1958
1959 foreach my $t (@sorted_trait_names_permanent_environment_structure) {
1960 if (defined($phenotype_data_pe{$p}->{$t})) {
1961 push @row, $phenotype_data_pe{$p}->{$t};
1962 } else {
1963 print STDERR $p." : $t : $germplasm_name : NA \n";
1964 push @row, 'NA';
1965 }
1966 }
1967 push @data_matrix_pe, \@row;
1968 }
1969
1970 my @phenotype_header_pe = ("replicate", "block", "id", "plot_id", "rowNumber", "colNumber", "rowNumberFactor", "colNumberFactor");
1971 foreach (@sorted_trait_names_permanent_environment_structure) {
1972 push @phenotype_header_pe, $trait_name_encoder_permanent_environment_structure{$_};
1973 }
1974 my $header_string_pe = join ',', @phenotype_header_pe;
1975
1976 open($F, ">", $stats_out_pe_pheno_rel_tempfile3) || die "Can't open file ".$stats_out_pe_pheno_rel_tempfile3;
1977 print $F $header_string_pe."\n";
1978 foreach (@data_matrix_pe) {
1979 my $line = join ',', @$_;
1980 print $F "$line\n";
1981 }
1982 close($F);
1983
1984 my @encoded_traits_pe = values %trait_name_encoder_permanent_environment_structure;
1985 my $encoded_trait_string_pe = join ',', @encoded_traits_pe;
1986 my $number_traits_pe = scalar(@encoded_traits_pe);
1987 my $cbind_string_pe = $number_traits_pe > 1 ? "cbind($encoded_trait_string_pe)" : $encoded_trait_string_pe;
1988
1989 my $statistics_cmd_pe = 'R -e "library(sommer); library(data.table); library(reshape2);
1990 mat <- data.frame(fread(\''.$stats_out_pe_pheno_rel_tempfile3.'\', header=TRUE, sep=\',\'));
1991 geno_mat_3col <- data.frame(fread(\''.$stats_out_pe_pheno_rel_2dspline_grm_tempfile.'\', header=FALSE, sep=\'\t\'));
1992 geno_mat <- acast(geno_mat_3col, V1~V2, value.var=\'V3\');
1993 geno_mat[is.na(geno_mat)] <- 0;
1994 mat\$rowNumber <- as.numeric(mat\$rowNumber);
1995 mat\$colNumber <- as.numeric(mat\$colNumber);
1996 mat\$rowNumberFactor <- as.factor(mat\$rowNumberFactor);
1997 mat\$colNumberFactor <- as.factor(mat\$colNumberFactor);
1998 mix <- mmer('.$cbind_string_pe.'~1 + replicate, random=~vs(id, Gu=geno_mat, Gtc=unsm('.$number_traits_pe.')) +vs(spl2D(rowNumber, colNumber), Gtc=diag('.$number_traits_pe.')), rcov=~vs(units, Gtc=unsm('.$number_traits_pe.')), data=mat, tolparinv='.$tolparinv.');
1999 if (!is.null(mix\$U)) {
2000 #gen_cor <- cov2cor(mix\$sigma\$\`u:id\`);
2001 X <- with(mat, spl2D(rowNumber, colNumber));
2002 spatial_blup_results <- data.frame(plot_id = mat\$plot_id);
2003 ';
2004 my $trait_index = 1;
2005 foreach my $enc_trait_name (@encoded_traits_pe) {
2006 $statistics_cmd_pe .= '
2007 blups'.$trait_index.' <- mix\$U\$\`u:rowNumber\`\$'.$enc_trait_name.';
2008 spatial_blup_results\$'.$enc_trait_name.' <- data.matrix(X) %*% data.matrix(blups'.$trait_index.');
2009 ';
2010 $trait_index++;
2011 }
2012 $statistics_cmd_pe .= 'cor_mat <- cor(t(spatial_blup_results[,-1]));
2013 rownames(cor_mat) <- spatial_blup_results\$plot_id;
2014 colnames(cor_mat) <- spatial_blup_results\$plot_id;
2015 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
2016 cor_mat <- range01(cor_mat);
2017 write.table(spatial_blup_results, file=\''.$stats_out_pe_pheno_rel_tempfile5.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
2018 write.table(cor_mat, file=\''.$stats_out_pe_pheno_rel_tempfile4.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');
2019 }
2020 "';
2021 # print STDERR Dumper $statistics_cmd;
2022 eval {
2023 my $status = system($statistics_cmd_pe);
2024 };
2025 my $run_stats_fault = 0;
2026 my %rel_pe_result_hash;
2027 my $current_env_row_count = 0;
2028 if ($@) {
2029 print STDERR "R ERROR\n";
2030 print STDERR Dumper $@;
2031 $run_stats_fault = 1;
2032 }
2033 else {
2034 open(my $pe_rel_res, '<', $stats_out_pe_pheno_rel_tempfile4) or die "Could not open file '$stats_out_pe_pheno_rel_tempfile4' $!";
2035 print STDERR "Opened $stats_out_pe_pheno_rel_tempfile4\n";
2036 my $header_row = <$pe_rel_res>;
2037 my @header;
2038 if ($csv->parse($header_row)) {
2039 @header = $csv->fields();
2040 }
2041
2042 while (my $row = <$pe_rel_res>) {
2043 my @columns;
2044 if ($csv->parse($row)) {
2045 @columns = $csv->fields();
2046 }
2047 my $stock_id1 = $columns[0];
2048 my $counter = 1;
2049 foreach my $stock_id2 (@header) {
2050 my $val = $columns[$counter];
2051 $rel_pe_result_hash{$stock_id1}->{$stock_id2} = $val;
2052 $counter++;
2053 }
2054 $current_env_row_count++;
2055 }
2056 close($pe_rel_res);
2057 # print STDERR Dumper \%rel_pe_result_hash;
2058
2059 my $data_rel_pe = '';
2060 my %result_hash_pe;
2061 foreach my $s (sort { $a <=> $b } @plot_ids_ordered) {
2062 foreach my $r (sort { $a <=> $b } @plot_ids_ordered) {
2063 my $s_factor = $stock_name_row_col{$plot_id_map{$s}}->{plot_id_factor};
2064 my $r_factor = $stock_name_row_col{$plot_id_map{$r}}->{plot_id_factor};
2065 if (!exists($result_hash_pe{$s_factor}->{$r_factor}) && !exists($result_hash_pe{$r_factor}->{$s_factor})) {
2066 $result_hash_pe{$s_factor}->{$r_factor} = $rel_pe_result_hash{$s}->{$r};
2067 }
2068 }
2069 }
2070 foreach my $r (sort { $a <=> $b } keys %result_hash_pe) {
2071 foreach my $s (sort { $a <=> $b } keys %{$result_hash_pe{$r}}) {
2072 my $val = $result_hash_pe{$r}->{$s};
2073 if (defined $val and length $val) {
2074 $data_rel_pe .= "$r\t$s\t$val\n";
2075 }
2076 }
2077 }
2078
2079 open(my $pe_rel_out, ">", $permanent_environment_structure_tempfile) || die "Can't open file ".$permanent_environment_structure_tempfile;
2080 print STDERR "Opened $permanent_environment_structure_tempfile\n";
2081 print $pe_rel_out $data_rel_pe;
2082 close($pe_rel_out);
2083 }
2084
2085 if ($run_stats_fault || $current_env_row_count == 0) {
2086 $c->stash->{rest} = { error => "There was a problem running the 2D-spline model for the permanent environment structure!"};
2087 return;
2088 }
2089 }
2090
2091 print STDERR Dumper [$phenotype_min_original, $phenotype_max_original];
2092
2093 @unique_accession_names = sort keys %unique_accessions;
2094 @unique_plot_names = sort keys %seen_plot_names;
2095 };
2096
2097 @seen_rows_array = keys %seen_rows;
2098 @seen_cols_array = keys %seen_cols;
2099 $row_stat = Statistics::Descriptive::Full->new();
2100 $row_stat->add_data(@seen_rows_array);
2101 $mean_row = $row_stat->mean();
2102 $sig_row = $row_stat->variance();
2103 $col_stat = Statistics::Descriptive::Full->new();
2104 $col_stat->add_data(@seen_cols_array);
2105 $mean_col = $col_stat->mean();
2106 $sig_col = $col_stat->variance();
2107
2108 my $result_1 = CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation::perform_drone_imagery_analytics($schema, $a_env, $b_env, $ro_env, $row_ro_env, $env_variance_percent, $protocol_id, $statistics_select, $analytics_select, $tolparinv, $use_area_under_curve, $legendre_order_number, $permanent_environment_structure, \@legendre_coeff_exec, \%trait_name_encoder, \%trait_name_encoder_rev, \%stock_info, \%plot_id_map, \@sorted_trait_names, \%accession_id_factor_map, \@rep_time_factors, \@ind_rep_factors, \@unique_accession_names, \%plot_id_count_map_reverse, \@sorted_scaled_ln_times, \%time_count_map_reverse, \%accession_id_factor_map_reverse, \%seen_times, \%plot_id_factor_map_reverse, \%trait_to_time_map, \@unique_plot_names, \%stock_name_row_col, \%phenotype_data_original, \%plot_rep_time_factor_map, \%stock_row_col, \%stock_row_col_id, \%polynomial_map, \@plot_ids_ordered, $csv, $timestamp, $user_name, $stats_tempfile, $grm_file, $grm_rename_tempfile, $tmp_stats_dir, $stats_out_tempfile, $stats_out_tempfile_row, $stats_out_tempfile_col, $stats_out_tempfile_residual, $stats_out_tempfile_2dspl, $stats_prep2_tempfile, $stats_out_param_tempfile, $parameter_tempfile, $parameter_asreml_tempfile, $stats_tempfile_2, $permanent_environment_structure_tempfile, $permanent_environment_structure_env_tempfile, $permanent_environment_structure_env_tempfile2, $permanent_environment_structure_env_tempfile_mat, $sim_env_changing_mat_tempfile, $sim_env_changing_mat_full_tempfile, $yhat_residual_tempfile, $blupf90_solutions_tempfile, $coeff_genetic_tempfile, $coeff_pe_tempfile, $stats_out_tempfile_varcomp, $time_min, $time_max, $header_string, $env_sim_exec, $min_row, $max_row, $min_col, $max_col, $mean_row, $sig_row, $mean_col, $sig_col, $sim_env_change_over_time, $correlation_between_times, $field_trial_id_list, $simulated_environment_real_data_trait_id, $fixed_effect_type, $perform_cv);
2109 if (ref($result_1) eq 'HASH') {
2110 $c->stash->{rest} = $result_1;
2111 $c->detach();
2112 }
2113 my ($statistical_ontology_term_1, $analysis_model_training_data_file_type_1, $analysis_model_language_1, $sorted_residual_trait_names_array_1, $rr_unique_traits_hash_1, $rr_residual_unique_traits_hash_1, $statistics_cmd_1, $cmd_f90_1, $number_traits_1, $trait_to_time_map_hash_1,
2114
2115 $result_blup_data_original_1, $result_blup_data_delta_original_1, $result_blup_spatial_data_original_1, $result_blup_pe_data_original_1, $result_blup_pe_data_delta_original_1, $result_residual_data_original_1, $result_fitted_data_original_1, $fixed_effects_original_hash_1,
2116 $rr_genetic_coefficients_original_hash_1, $rr_temporal_coefficients_original_hash_1,
2117 $rr_coeff_genetic_covariance_original_array_1, $rr_coeff_env_covariance_original_array_1, $rr_coeff_genetic_correlation_original_array_1, $rr_coeff_env_correlation_original_array_1, $rr_residual_variance_original_1, $varcomp_original_array_1,
2118 $model_sum_square_residual_original_1, $genetic_effect_min_original_1, $genetic_effect_max_original_1, $env_effect_min_original_1, $env_effect_max_original_1, $genetic_effect_sum_square_original_1, $genetic_effect_sum_original_1, $env_effect_sum_square_original_1, $env_effect_sum_original_1, $residual_sum_square_original_1, $residual_sum_original_1, $result_cv_original_1, $result_cv_2_original_1,
2119
2120 $phenotype_data_altered_hash_1, $data_matrix_altered_array_1, $data_matrix_phenotypes_altered_array_1, $phenotype_min_altered_1, $phenotype_max_altered_1,
2121 $result_blup_data_altered_1, $result_blup_data_delta_altered_1, $result_blup_spatial_data_altered_1, $result_blup_pe_data_altered_1, $result_blup_pe_data_delta_altered_1, $result_residual_data_altered_1, $result_fitted_data_altered_1, $fixed_effects_altered_hash_1,
2122 $rr_genetic_coefficients_altered_hash_1, $rr_temporal_coefficients_altered_hash_1,
2123 $rr_coeff_genetic_covariance_altered_array_1, $rr_coeff_env_covariance_altered_array_1, $rr_coeff_genetic_correlation_altered_array_1, $rr_coeff_env_correlation_altered_array_1, $rr_residual_variance_altered_1, $varcomp_altered_array_1,
2124 $model_sum_square_residual_altered_1, $genetic_effect_min_altered_1, $genetic_effect_max_altered_1, $env_effect_min_altered_1, $env_effect_max_altered_1, $genetic_effect_sum_square_altered_1, $genetic_effect_sum_altered_1, $env_effect_sum_square_altered_1, $env_effect_sum_altered_1, $residual_sum_square_altered_1, $residual_sum_altered_1, $result_cv_altered_1, $result_cv_2_altered_1,
2125
2126 $phenotype_data_altered_env_hash_1_1, $data_matrix_altered_env_array_1_1, $data_matrix_phenotypes_altered_env_array_1_1, $phenotype_min_altered_env_1_1, $phenotype_max_altered_env_1_1, $env_sim_min_1_1, $env_sim_max_1_1, $sim_data_hash_1_1,
2127 $result_blup_data_altered_env_1_1, $result_blup_data_delta_altered_env_1_1, $result_blup_spatial_data_altered_env_1_1, $result_blup_pe_data_altered_env_1_1, $result_blup_pe_data_delta_altered_env_1_1, $result_residual_data_altered_env_1_1, $result_fitted_data_altered_env_1_1, $fixed_effects_altered_env_hash_1_1,
2128 $rr_genetic_coefficients_altered_env_hash_1_1, $rr_temporal_coefficients_altered_env_hash_1_1,
2129 $rr_coeff_genetic_covariance_altered_env_array_1_1, $rr_coeff_env_covariance_altered_env_array_1_1, $rr_coeff_genetic_correlation_altered_env_array_1_1, $rr_coeff_env_correlation_altered_env_array_1_1, $rr_residual_variance_altered_env_1_1, $varcomp_altered_array_env_1_1,
2130 $model_sum_square_residual_altered_env_1_1, $genetic_effect_min_altered_env_1_1, $genetic_effect_max_altered_env_1_1, $env_effect_min_altered_env_1_1, $env_effect_max_altered_env_1_1, $genetic_effect_sum_square_altered_env_1_1, $genetic_effect_sum_altered_env_1_1, $env_effect_sum_square_altered_env_1_1, $env_effect_sum_altered_env_1_1, $residual_sum_square_altered_env_1_1, $residual_sum_altered_env_1_1, $result_cv_altered_env_1_1, $result_cv_2_altered_env_1_1,
2131
2132 $phenotype_data_altered_env_hash_2_1, $data_matrix_altered_env_array_2_1, $data_matrix_phenotypes_altered_env_array_2_1, $phenotype_min_altered_env_2_1, $phenotype_max_altered_env_2_1, $env_sim_min_2_1, $env_sim_max_2_1, $sim_data_hash_2_1,
2133 $result_blup_data_altered_env_2_1, $result_blup_data_delta_altered_env_2_1, $result_blup_spatial_data_altered_env_2_1, $result_blup_pe_data_altered_env_2_1, $result_blup_pe_data_delta_altered_env_2_1, $result_residual_data_altered_env_2_1, $result_fitted_data_altered_env_2_1, $fixed_effects_altered_env_hash_2_1, $rr_genetic_coefficients_altered_env_hash_2_1, $rr_temporal_coefficients_altered_env_hash_2_1,
2134 $rr_coeff_genetic_covariance_altered_env_array_2_1, $rr_coeff_env_covariance_altered_env_array_2_1, $rr_coeff_genetic_correlation_altered_env_array_2_1, $rr_coeff_env_correlation_altered_env_array_2_1, $rr_residual_variance_altered_env_2_1, $varcomp_altered_array_env_2_1,
2135 $model_sum_square_residual_altered_env_2_1, $genetic_effect_min_altered_env_2_1, $genetic_effect_max_altered_env_2_1, $env_effect_min_altered_env_2_1, $env_effect_max_altered_env_2_1, $genetic_effect_sum_square_altered_env_2_1, $genetic_effect_sum_altered_env_2_1, $env_effect_sum_square_altered_env_2_1, $env_effect_sum_altered_env_2_1, $residual_sum_square_altered_env_2_1, $residual_sum_altered_env_2_1, $result_cv_altered_env_2_1, $result_cv_2_altered_env_2_1,
2136
2137 $phenotype_data_altered_env_hash_3_1, $data_matrix_altered_env_array_3_1, $data_matrix_phenotypes_altered_env_array_3_1, $phenotype_min_altered_env_3_1, $phenotype_max_altered_env_3_1, $env_sim_min_3_1, $env_sim_max_3_1, $sim_data_hash_3_1,
2138 $result_blup_data_altered_env_3_1, $result_blup_data_delta_altered_env_3_1, $result_blup_spatial_data_altered_env_3_1, $result_blup_pe_data_altered_env_3_1, $result_blup_pe_data_delta_altered_env_3_1, $result_residual_data_altered_env_3_1, $result_fitted_data_altered_env_3_1, $fixed_effects_altered_env_hash_3_1, $rr_genetic_coefficients_altered_env_hash_3_1, $rr_temporal_coefficients_altered_env_hash_3_1,
2139 $rr_coeff_genetic_covariance_altered_env_array_3_1, $rr_coeff_env_covariance_altered_env_array_3_1, $rr_coeff_genetic_correlation_altered_env_array_3_1, $rr_coeff_env_correlation_altered_env_array_3_1, $rr_residual_variance_altered_env_3_1, $varcomp_altered_array_env_3_1,
2140 $model_sum_square_residual_altered_env_3_1, $genetic_effect_min_altered_env_3_1, $genetic_effect_max_altered_env_3_1, $env_effect_min_altered_env_3_1, $env_effect_max_altered_env_3_1, $genetic_effect_sum_square_altered_env_3_1, $genetic_effect_sum_altered_env_3_1, $env_effect_sum_square_altered_env_3_1, $env_effect_sum_altered_env_3_1, $residual_sum_square_altered_env_3_1, $residual_sum_altered_env_3_1, $result_cv_altered_env_3_1, $result_cv_2_altered_env_3_1,
2141
2142 $phenotype_data_altered_env_hash_4_1, $data_matrix_altered_env_array_4_1, $data_matrix_phenotypes_altered_env_array_4_1, $phenotype_min_altered_env_4_1, $phenotype_max_altered_env_4_1, $env_sim_min_4_1, $env_sim_max_4_1, $sim_data_hash_4_1,
2143 $result_blup_data_altered_env_4_1, $result_blup_data_delta_altered_env_4_1, $result_blup_spatial_data_altered_env_4_1, $result_blup_pe_data_altered_env_4_1, $result_blup_pe_data_delta_altered_env_4_1, $result_residual_data_altered_env_4_1, $result_fitted_data_altered_env_4_1, $fixed_effects_altered_env_hash_4_1, $rr_genetic_coefficients_altered_env_hash_4_1, $rr_temporal_coefficients_altered_env_hash_4_1,
2144 $rr_coeff_genetic_covariance_altered_env_array_4_1, $rr_coeff_env_covariance_altered_env_array_4_1, $rr_coeff_genetic_correlation_altered_env_array_4_1, $rr_coeff_env_correlation_altered_env_array_4_1, $rr_residual_variance_altered_env_4_1, $varcomp_altered_array_env_4_1,
2145 $model_sum_square_residual_altered_env_4_1, $genetic_effect_min_altered_env_4_1, $genetic_effect_max_altered_env_4_1, $env_effect_min_altered_env_4_1, $env_effect_max_altered_env_4_1, $genetic_effect_sum_square_altered_env_4_1, $genetic_effect_sum_altered_env_4_1, $env_effect_sum_square_altered_env_4_1, $env_effect_sum_altered_env_4_1, $residual_sum_square_altered_env_4_1, $residual_sum_altered_env_4_1, $result_cv_altered_env_4_1, $result_cv_2_altered_env_4_1,
2146
2147 $phenotype_data_altered_env_hash_5_1, $data_matrix_altered_env_array_5_1, $data_matrix_phenotypes_altered_env_array_5_1, $phenotype_min_altered_env_5_1, $phenotype_max_altered_env_5_1, $env_sim_min_5_1, $env_sim_max_5_1, $sim_data_hash_5_1,
2148 $result_blup_data_altered_env_5_1, $result_blup_data_delta_altered_env_5_1, $result_blup_spatial_data_altered_env_5_1, $result_blup_pe_data_altered_env_5_1, $result_blup_pe_data_delta_altered_env_5_1, $result_residual_data_altered_env_5_1, $result_fitted_data_altered_env_5_1, $fixed_effects_altered_env_hash_5_1, $rr_genetic_coefficients_altered_env_hash_5_1, $rr_temporal_coefficients_altered_env_hash_5_1,
2149 $rr_coeff_genetic_covariance_altered_env_array_5_1, $rr_coeff_env_covariance_altered_env_array_5_1, $rr_coeff_genetic_correlation_altered_env_array_5_1, $rr_coeff_env_correlation_altered_env_array_5_1, $rr_residual_variance_altered_env_5_1, $varcomp_altered_array_env_5_1,
2150 $model_sum_square_residual_altered_env_5_1, $genetic_effect_min_altered_env_5_1, $genetic_effect_max_altered_env_5_1, $env_effect_min_altered_env_5_1, $env_effect_max_altered_env_5_1, $genetic_effect_sum_square_altered_env_5_1, $genetic_effect_sum_altered_env_5_1, $env_effect_sum_square_altered_env_5_1, $env_effect_sum_altered_env_5_1, $residual_sum_square_altered_env_5_1, $residual_sum_altered_env_5_1, $result_cv_altered_env_5_1, $result_cv_2_altered_env_5_1,
2151
2152 $phenotype_data_altered_env_hash_6_1, $data_matrix_altered_env_array_6_1, $data_matrix_phenotypes_altered_env_array_6_1, $phenotype_min_altered_env_6_1, $phenotype_max_altered_env_6_1, $env_sim_min_6_1, $env_sim_max_6_1, $sim_data_hash_6_1,
2153 $result_blup_data_altered_env_6_1, $result_blup_data_delta_altered_env_6_1, $result_blup_spatial_data_altered_env_6_1, $result_blup_pe_data_altered_env_6_1, $result_blup_pe_data_delta_altered_env_6_1, $result_residual_data_altered_env_6_1, $result_fitted_data_altered_env_6_1, $fixed_effects_altered_env_hash_6_1, $rr_genetic_coefficients_altered_env_hash_6_1, $rr_temporal_coefficients_altered_env_hash_6_1,
2154 $rr_coeff_genetic_covariance_altered_env_array_6_1, $rr_coeff_env_covariance_altered_env_array_6_1, $rr_coeff_genetic_correlation_altered_env_array_6_1, $rr_coeff_env_correlation_altered_env_array_6_1, $rr_residual_variance_altered_env_6_1, $varcomp_altered_array_env_6_1,
2155 $model_sum_square_residual_altered_env_6_1, $genetic_effect_min_altered_env_6_1, $genetic_effect_max_altered_env_6_1, $env_effect_min_altered_env_6_1, $env_effect_max_altered_env_6_1, $genetic_effect_sum_square_altered_env_6_1, $genetic_effect_sum_altered_env_6_1, $env_effect_sum_square_altered_env_6_1, $env_effect_sum_altered_env_6_1, $residual_sum_square_altered_env_6_1, $residual_sum_altered_env_6_1, $result_cv_altered_env_6_1, $result_cv_2_altered_env_6_1
2156 ) = @$result_1;
2157
2158 eval {
2159 print STDERR "PLOTTING CORRELATION\n";
2160 my ($full_plot_level_correlation_tempfile_fh, $full_plot_level_correlation_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2161 open(my $F_fullplot, ">", $full_plot_level_correlation_tempfile) || die "Can't open file ".$full_plot_level_correlation_tempfile;
2162 print STDERR "OPENED PLOTCORR FILE $full_plot_level_correlation_tempfile\n";
2163
2164 my @header_full_plot_corr = ('plot_name, plot_id, row_number, col_number, rep, block, germplasm_name, germplasm_id');
2165 my @types_full_plot_corr = ('pheno_orig_', 'pheno_postm1_', 'eff_origm1_', 'eff_postm1_',
2166 'sim_env1_', 'simm1_pheno1_', 'effm1_sim1_',
2167 'sim_env2_', 'simm1_pheno2_', 'effm1_sim2_',
2168 'sim_env3_', 'simm1_pheno3_', 'effm1_sim3_',
2169 'sim_env4_', 'simm1_pheno4_', 'effm1_sim4_',
2170 'sim_env5_', 'simm1_pheno5_', 'effm1_sim5_',
2171 'sim_env6_', 'simm1_pheno6_', 'effm1_sim6_');
2172 foreach my $t (@sorted_trait_names) {
2173 foreach my $type (@types_full_plot_corr) {
2174 push @header_full_plot_corr, $type.$trait_name_encoder{$t};
2175 }
2176 }
2177 my $header_string_full_plot_corr = join ',', @header_full_plot_corr;
2178 print $F_fullplot "$header_string_full_plot_corr\n";
2179 foreach my $p (@unique_plot_names) {
2180 my @row = ($p, $stock_name_row_col{$p}->{obsunit_stock_id}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $stock_name_row_col{$p}->{rep}, $stock_name_row_col{$p}->{block}, $stock_name_row_col{$p}->{germplasm_name}, $stock_name_row_col{$p}->{germplasm_stock_id});
2181 foreach my $t (@sorted_trait_names) {
2182 my $phenotype_original = $phenotype_data_original{$p}->{$t};
2183 my $phenotype_post_1 = $phenotype_data_altered_hash_1->{$p}->{$t};
2184 my $effect_original_1 = $result_blup_pe_data_delta_original_1->{$p}->{$t}->[0];
2185 my $effect_post_1 = $result_blup_pe_data_delta_altered_1->{$p}->{$t}->[0];
2186 push @row, ($phenotype_original, $phenotype_post_1, $effect_original_1, $effect_post_1);
2187
2188 my $sim_env = $sim_data_hash_1_1->{$p}->{$t};
2189 my $pheno_sim_1 = $phenotype_data_altered_env_hash_1_1->{$p}->{$t};
2190 my $effect_sim_1 = $result_blup_pe_data_delta_altered_env_1_1->{$p}->{$t}->[0];
2191 push @row, ($sim_env, $pheno_sim_1, $effect_sim_1);
2192
2193 my $sim_env2 = $sim_data_hash_2_1->{$p}->{$t};
2194 my $pheno_sim2_1 = $phenotype_data_altered_env_hash_2_1->{$p}->{$t};
2195 my $effect_sim2_1 = $result_blup_pe_data_delta_altered_env_2_1->{$p}->{$t}->[0];
2196 push @row, ($sim_env2, $pheno_sim2_1, $effect_sim2_1);
2197
2198 my $sim_env3 = $sim_data_hash_3_1->{$p}->{$t};
2199 my $pheno_sim3_1 = $phenotype_data_altered_env_hash_3_1->{$p}->{$t};
2200 my $effect_sim3_1 = $result_blup_pe_data_delta_altered_env_3_1->{$p}->{$t}->[0];
2201 push @row, ($sim_env3, $pheno_sim3_1, $effect_sim3_1);
2202
2203 my $sim_env4 = $sim_data_hash_4_1->{$p}->{$t};
2204 my $pheno_sim4_1 = $phenotype_data_altered_env_hash_4_1->{$p}->{$t};
2205 my $effect_sim4_1 = $result_blup_pe_data_delta_altered_env_4_1->{$p}->{$t}->[0];
2206 push @row, ($sim_env4, $pheno_sim4_1, $effect_sim4_1);
2207
2208 my $sim_env5 = $sim_data_hash_5_1->{$p}->{$t};
2209 my $pheno_sim5_1 = $phenotype_data_altered_env_hash_5_1->{$p}->{$t};
2210 my $effect_sim5_1 = $result_blup_pe_data_delta_altered_env_5_1->{$p}->{$t}->[0];
2211 push @row, ($sim_env5, $pheno_sim5_1, $effect_sim5_1);
2212
2213 my $sim_env6 = $sim_data_hash_6_1->{$p}->{$t};
2214 my $pheno_sim6_1 = $phenotype_data_altered_env_hash_6_1->{$p}->{$t};
2215 my $effect_sim6_1 = $result_blup_pe_data_delta_altered_env_6_1->{$p}->{$t}->[0];
2216 push @row, ($sim_env6, $pheno_sim6_1, $effect_sim6_1);
2217 }
2218 my $line = join ',', @row;
2219 print $F_fullplot "$line\n";
2220 }
2221 close($F_fullplot);
2222
2223 my $plot_corr_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2224 $plot_corr_figure_tempfile_string .= '.png';
2225 my $plot_corr_figure_tempfile = $c->config->{basepath}."/".$plot_corr_figure_tempfile_string;
2226
2227 my $cmd_plotcorr_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
2228 mat_orig <- fread(\''.$full_plot_level_correlation_tempfile.'\', header=TRUE, sep=\',\');
2229 gg <- ggcorr(data=mat_orig[,-seq(1,8)], hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE);
2230 ggsave(\''.$plot_corr_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, limitsize = FALSE, units=\'in\');
2231 "';
2232 # print STDERR Dumper $cmd;
2233 my $status_plotcorr_plot = system($cmd_plotcorr_plot);
2234 push @$spatial_effects_plots, [$plot_corr_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2235 push @$spatial_effects_files_store, [$full_plot_level_correlation_tempfile, "datafile_".$statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2236 };
2237
2238 eval {
2239 my @plot_corr_full_vals;
2240
2241 my @original_pheno_vals;
2242 my ($phenotypes_original_heatmap_tempfile_fh, $phenotypes_original_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2243 open(my $F_pheno, ">", $phenotypes_original_heatmap_tempfile) || die "Can't open file ".$phenotypes_original_heatmap_tempfile;
2244 print $F_pheno "trait_type,row,col,value\n";
2245 foreach my $p (@unique_plot_names) {
2246 foreach my $t (@sorted_trait_names) {
2247 my $val = $phenotype_data_original{$p}->{$t};
2248 my @row = ("pheno_orig_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2249 push @original_pheno_vals, $val;
2250 my $line = join ',', @row;
2251 print $F_pheno "$line\n";
2252 }
2253 }
2254 close($F_pheno);
2255 push @plot_corr_full_vals, \@original_pheno_vals;
2256
2257 my $original_pheno_stat = Statistics::Descriptive::Full->new();
2258 $original_pheno_stat->add_data(@original_pheno_vals);
2259 my $sig_original_pheno = $original_pheno_stat->variance();
2260
2261 #PHENO POST M START
2262
2263 my @altered_pheno_vals;
2264 my ($phenotypes_post_heatmap_tempfile_fh, $phenotypes_post_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2265 open($F_pheno, ">", $phenotypes_post_heatmap_tempfile) || die "Can't open file ".$phenotypes_post_heatmap_tempfile;
2266 print $F_pheno "trait_type,row,col,value\n";
2267 foreach my $p (@unique_plot_names) {
2268 foreach my $t (@sorted_trait_names) {
2269 my $val = $phenotype_data_altered_hash_1->{$p}->{$t};
2270 my @row = ("pheno_postm1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2271 push @altered_pheno_vals, $val;
2272 my $line = join ',', @row;
2273 print $F_pheno "$line\n";
2274 }
2275 }
2276 close($F_pheno);
2277 push @plot_corr_full_vals, \@altered_pheno_vals;
2278
2279 my $altered_pheno_stat = Statistics::Descriptive::Full->new();
2280 $altered_pheno_stat->add_data(@altered_pheno_vals);
2281 my $sig_altered_pheno = $altered_pheno_stat->variance();
2282
2283 # EFFECT ORIGINAL M
2284
2285 my @original_effect_vals;
2286 my ($effects_heatmap_tempfile_fh, $effects_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2287 open(my $F_eff, ">", $effects_heatmap_tempfile) || die "Can't open file ".$effects_heatmap_tempfile;
2288 print $F_eff "trait_type,row,col,value\n";
2289 foreach my $p (@unique_plot_names) {
2290 foreach my $t (@sorted_trait_names) {
2291 my $val = $result_blup_pe_data_delta_original_1->{$p}->{$t}->[0];
2292 my @row = ("eff_origm1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2293 my $line = join ',', @row;
2294 print $F_eff "$line\n";
2295 push @original_effect_vals, $val;
2296 }
2297 }
2298 close($F_eff);
2299 push @plot_corr_full_vals, \@original_effect_vals;
2300
2301 my $original_effect_stat = Statistics::Descriptive::Full->new();
2302 $original_effect_stat->add_data(@original_effect_vals);
2303 my $sig_original_effect = $original_effect_stat->variance();
2304
2305 # EFFECT POST M MIN
2306
2307 my @altered_effect_vals;
2308 my ($effects_post_heatmap_tempfile_fh, $effects_post_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2309 open($F_eff, ">", $effects_post_heatmap_tempfile) || die "Can't open file ".$effects_post_heatmap_tempfile;
2310 print $F_eff "trait_type,row,col,value\n";
2311 foreach my $p (@unique_plot_names) {
2312 foreach my $t (@sorted_trait_names) {
2313 my $val = $result_blup_pe_data_delta_altered_1->{$p}->{$t}->[0];
2314 my @row = ("eff_postm1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2315 my $line = join ',', @row;
2316 print $F_eff "$line\n";
2317 push @altered_effect_vals, $val;
2318 }
2319 }
2320 close($F_eff);
2321 push @plot_corr_full_vals, \@altered_effect_vals;
2322
2323 my $altered_effect_stat = Statistics::Descriptive::Full->new();
2324 $altered_effect_stat->add_data(@altered_effect_vals);
2325 my $sig_altered_effect = $altered_effect_stat->variance();
2326
2327 # SIM ENV 1: ALTERED PHENO + EFFECT
2328
2329 my ($phenotypes_env_heatmap_tempfile_fh, $phenotypes_env_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2330 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile) || die "Can't open file ".$phenotypes_env_heatmap_tempfile;
2331 print $F_pheno "trait_type,row,col,value\n";
2332 foreach my $p (@unique_plot_names) {
2333 foreach my $t (@sorted_trait_names) {
2334 my @row = ("sim_env1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_1_1->{$p}->{$t});
2335 my $line = join ',', @row;
2336 print $F_pheno "$line\n";
2337 }
2338 }
2339 close($F_pheno);
2340
2341 my @sim_pheno1_vals;
2342 my ($phenotypes_pheno_sim_heatmap_tempfile_fh, $phenotypes_pheno_sim_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2343 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile;
2344 print $F_pheno "trait_type,row,col,value\n";
2345 foreach my $p (@unique_plot_names) {
2346 foreach my $t (@sorted_trait_names) {
2347 my $val = $phenotype_data_altered_env_hash_1_1->{$p}->{$t};
2348 my @row = ("simm1_pheno1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2349 my $line = join ',', @row;
2350 print $F_pheno "$line\n";
2351 push @sim_pheno1_vals, $val;
2352 }
2353 }
2354 close($F_pheno);
2355 push @plot_corr_full_vals, \@sim_pheno1_vals;
2356
2357 my $sim_pheno1_stat = Statistics::Descriptive::Full->new();
2358 $sim_pheno1_stat->add_data(@sim_pheno1_vals);
2359 my $sig_sim_pheno1 = $sim_pheno1_stat->variance();
2360
2361 my @sim_effect1_vals;
2362 my ($effects_sim_heatmap_tempfile_fh, $effects_sim_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2363 open($F_eff, ">", $effects_sim_heatmap_tempfile) || die "Can't open file ".$effects_sim_heatmap_tempfile;
2364 print $F_eff "trait_type,row,col,value\n";
2365 foreach my $p (@unique_plot_names) {
2366 foreach my $t (@sorted_trait_names) {
2367 my $val = $result_blup_pe_data_delta_altered_env_1_1->{$p}->{$t}->[0];
2368 my @row = ("effm1_sim1_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2369 my $line = join ',', @row;
2370 print $F_eff "$line\n";
2371 push @sim_effect1_vals, $val;
2372 }
2373 }
2374 close($F_eff);
2375 push @plot_corr_full_vals, \@sim_effect1_vals;
2376
2377 my $sim_effect1_stat = Statistics::Descriptive::Full->new();
2378 $sim_effect1_stat->add_data(@sim_effect1_vals);
2379 my $sig_sim_effect1 = $sim_effect1_stat->variance();
2380
2381 # SIM ENV 2: ALTERED PHENO + EFFECT
2382
2383 my ($phenotypes_env_heatmap_tempfile2_fh, $phenotypes_env_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2384 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile2) || die "Can't open file ".$phenotypes_env_heatmap_tempfile2;
2385 print $F_pheno "trait_type,row,col,value\n";
2386 foreach my $p (@unique_plot_names) {
2387 foreach my $t (@sorted_trait_names) {
2388 my @row = ("sim_env2_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_2_1->{$p}->{$t});
2389 my $line = join ',', @row;
2390 print $F_pheno "$line\n";
2391 }
2392 }
2393 close($F_pheno);
2394
2395 my @sim_pheno2_vals;
2396 my ($phenotypes_pheno_sim_heatmap_tempfile2_fh, $phenotypes_pheno_sim_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2397 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile2;
2398 print $F_pheno "trait_type,row,col,value\n";
2399 foreach my $p (@unique_plot_names) {
2400 foreach my $t (@sorted_trait_names) {
2401 my $val = $phenotype_data_altered_env_hash_2_1->{$p}->{$t};
2402 my @row = ("simm1_pheno2_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2403 my $line = join ',', @row;
2404 print $F_pheno "$line\n";
2405 push @sim_pheno2_vals, $val;
2406 }
2407 }
2408 close($F_pheno);
2409 push @plot_corr_full_vals, \@sim_pheno2_vals;
2410
2411 my $sim_pheno2_stat = Statistics::Descriptive::Full->new();
2412 $sim_pheno2_stat->add_data(@sim_pheno2_vals);
2413 my $sig_sim_pheno2 = $sim_pheno2_stat->variance();
2414
2415 my @sim_effect2_vals;
2416 my ($effects_sim_heatmap_tempfile2_fh, $effects_sim_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2417 open($F_eff, ">", $effects_sim_heatmap_tempfile2) || die "Can't open file ".$effects_sim_heatmap_tempfile2;
2418 print $F_eff "trait_type,row,col,value\n";
2419 foreach my $p (@unique_plot_names) {
2420 foreach my $t (@sorted_trait_names) {
2421 my $val = $result_blup_pe_data_delta_altered_env_2_1->{$p}->{$t}->[0];
2422 my @row = ("effm1_sim2_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2423 my $line = join ',', @row;
2424 print $F_eff "$line\n";
2425 push @sim_effect2_vals, $val;
2426 }
2427 }
2428 close($F_eff);
2429 push @plot_corr_full_vals, \@sim_effect2_vals;
2430
2431 # SIM ENV 3: ALTERED PHENO + EFFECT
2432
2433 my ($phenotypes_env_heatmap_tempfile3_fh, $phenotypes_env_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2434 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile3) || die "Can't open file ".$phenotypes_env_heatmap_tempfile3;
2435 print $F_pheno "trait_type,row,col,value\n";
2436 foreach my $p (@unique_plot_names) {
2437 foreach my $t (@sorted_trait_names) {
2438 my @row = ("sim_env3_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_3_1->{$p}->{$t});
2439 my $line = join ',', @row;
2440 print $F_pheno "$line\n";
2441 }
2442 }
2443 close($F_pheno);
2444
2445 my @sim_pheno3_vals;
2446 my ($phenotypes_pheno_sim_heatmap_tempfile3_fh, $phenotypes_pheno_sim_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2447 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile3;
2448 print $F_pheno "trait_type,row,col,value\n";
2449 foreach my $p (@unique_plot_names) {
2450 foreach my $t (@sorted_trait_names) {
2451 my $val = $phenotype_data_altered_env_hash_3_1->{$p}->{$t};
2452 my @row = ("simm1_pheno3_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2453 my $line = join ',', @row;
2454 print $F_pheno "$line\n";
2455 push @sim_pheno3_vals, $val;
2456 }
2457 }
2458 close($F_pheno);
2459 push @plot_corr_full_vals, \@sim_pheno3_vals;
2460
2461 my $sim_pheno3_stat = Statistics::Descriptive::Full->new();
2462 $sim_pheno3_stat->add_data(@sim_pheno3_vals);
2463 my $sig_sim_pheno3 = $sim_pheno3_stat->variance();
2464
2465 my @sim_effect3_vals;
2466 my ($effects_sim_heatmap_tempfile3_fh, $effects_sim_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2467 open($F_eff, ">", $effects_sim_heatmap_tempfile3) || die "Can't open file ".$effects_sim_heatmap_tempfile3;
2468 print $F_eff "trait_type,row,col,value\n";
2469 foreach my $p (@unique_plot_names) {
2470 foreach my $t (@sorted_trait_names) {
2471 my $val = $result_blup_pe_data_delta_altered_env_3_1->{$p}->{$t}->[0];
2472 my @row = ("effm1_sim3_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2473 my $line = join ',', @row;
2474 print $F_eff "$line\n";
2475 push @sim_effect3_vals, $val;
2476 }
2477 }
2478 close($F_eff);
2479 push @plot_corr_full_vals, \@sim_effect3_vals;
2480
2481 my $sim_effect3_stat = Statistics::Descriptive::Full->new();
2482 $sim_effect3_stat->add_data(@sim_effect3_vals);
2483 my $sig_sim_effect3 = $sim_effect3_stat->variance();
2484
2485 # SIM ENV 4: ALTERED PHENO + EFFECT
2486
2487 my ($phenotypes_env_heatmap_tempfile4_fh, $phenotypes_env_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2488 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile4) || die "Can't open file ".$phenotypes_env_heatmap_tempfile4;
2489 print $F_pheno "trait_type,row,col,value\n";
2490 foreach my $p (@unique_plot_names) {
2491 foreach my $t (@sorted_trait_names) {
2492 my @row = ("sim_env4_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_4_1->{$p}->{$t});
2493 my $line = join ',', @row;
2494 print $F_pheno "$line\n";
2495 }
2496 }
2497 close($F_pheno);
2498
2499 my @sim_pheno4_vals;
2500 my ($phenotypes_pheno_sim_heatmap_tempfile4_fh, $phenotypes_pheno_sim_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2501 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile4) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile4;
2502 print $F_pheno "trait_type,row,col,value\n";
2503 foreach my $p (@unique_plot_names) {
2504 foreach my $t (@sorted_trait_names) {
2505 my $val = $phenotype_data_altered_env_hash_4_1->{$p}->{$t};
2506 my @row = ("simm1_pheno4_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2507 my $line = join ',', @row;
2508 print $F_pheno "$line\n";
2509 push @sim_pheno4_vals, $val;
2510 }
2511 }
2512 close($F_pheno);
2513 push @plot_corr_full_vals, \@sim_pheno4_vals;
2514
2515 my $sim_pheno4_stat = Statistics::Descriptive::Full->new();
2516 $sim_pheno4_stat->add_data(@sim_pheno4_vals);
2517 my $sig_sim_pheno4 = $sim_pheno4_stat->variance();
2518
2519 my @sim_effect4_vals;
2520 my ($effects_sim_heatmap_tempfile4_fh, $effects_sim_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2521 open($F_eff, ">", $effects_sim_heatmap_tempfile4) || die "Can't open file ".$effects_sim_heatmap_tempfile4;
2522 print $F_eff "trait_type,row,col,value\n";
2523 foreach my $p (@unique_plot_names) {
2524 foreach my $t (@sorted_trait_names) {
2525 my $val = $result_blup_pe_data_delta_altered_env_4_1->{$p}->{$t}->[0];
2526 my @row = ("effm1_sim4_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2527 my $line = join ',', @row;
2528 print $F_eff "$line\n";
2529 push @sim_effect4_vals, $val;
2530 }
2531 }
2532 close($F_eff);
2533 push @plot_corr_full_vals, \@sim_effect4_vals;
2534
2535 my $sim_effect4_stat = Statistics::Descriptive::Full->new();
2536 $sim_effect4_stat->add_data(@sim_effect4_vals);
2537 my $sig_sim_effect4 = $sim_effect4_stat->variance();
2538
2539 # SIM ENV 5: ALTERED PHENO + EFFECT
2540
2541 my ($phenotypes_env_heatmap_tempfile5_fh, $phenotypes_env_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2542 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile5) || die "Can't open file ".$phenotypes_env_heatmap_tempfile5;
2543 print $F_pheno "trait_type,row,col,value\n";
2544 foreach my $p (@unique_plot_names) {
2545 foreach my $t (@sorted_trait_names) {
2546 my @row = ("sim_env5_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_5_1->{$p}->{$t});
2547 my $line = join ',', @row;
2548 print $F_pheno "$line\n";
2549 }
2550 }
2551 close($F_pheno);
2552
2553 my @sim_pheno5_vals;
2554 my ($phenotypes_pheno_sim_heatmap_tempfile5_fh, $phenotypes_pheno_sim_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2555 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile5;
2556 print $F_pheno "trait_type,row,col,value\n";
2557 foreach my $p (@unique_plot_names) {
2558 foreach my $t (@sorted_trait_names) {
2559 my $val = $phenotype_data_altered_env_hash_5_1->{$p}->{$t};
2560 my @row = ("simm1_pheno5_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2561 my $line = join ',', @row;
2562 print $F_pheno "$line\n";
2563 push @sim_pheno5_vals, $val;
2564 }
2565 }
2566 close($F_pheno);
2567 push @plot_corr_full_vals, \@sim_pheno5_vals;
2568
2569 my $sim_pheno5_stat = Statistics::Descriptive::Full->new();
2570 $sim_pheno5_stat->add_data(@sim_pheno5_vals);
2571 my $sig_sim_pheno5 = $sim_pheno5_stat->variance();
2572
2573 my @sim_effect5_vals;
2574 my ($effects_sim_heatmap_tempfile5_fh, $effects_sim_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2575 open($F_eff, ">", $effects_sim_heatmap_tempfile5) || die "Can't open file ".$effects_sim_heatmap_tempfile5;
2576 print $F_eff "trait_type,row,col,value\n";
2577 foreach my $p (@unique_plot_names) {
2578 foreach my $t (@sorted_trait_names) {
2579 my $val = $result_blup_pe_data_delta_altered_env_5_1->{$p}->{$t}->[0];
2580 my @row = ("effm1_sim5_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2581 my $line = join ',', @row;
2582 print $F_eff "$line\n";
2583 push @sim_effect5_vals, $val;
2584 }
2585 }
2586 close($F_eff);
2587 push @plot_corr_full_vals, \@sim_effect5_vals;
2588
2589 my $sim_effect5_stat = Statistics::Descriptive::Full->new();
2590 $sim_effect5_stat->add_data(@sim_effect5_vals);
2591 my $sig_sim_effect5 = $sim_effect5_stat->variance();
2592
2593 # SIM ENV 6: ALTERED PHENO + EFFECT
2594
2595 my ($phenotypes_env_heatmap_tempfile6_fh, $phenotypes_env_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2596 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile6) || die "Can't open file ".$phenotypes_env_heatmap_tempfile6;
2597 print $F_pheno "trait_type,row,col,value\n";
2598 foreach my $p (@unique_plot_names) {
2599 foreach my $t (@sorted_trait_names) {
2600 my @row = ("sim_env6_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_6_1->{$p}->{$t});
2601 my $line = join ',', @row;
2602 print $F_pheno "$line\n";
2603 }
2604 }
2605 close($F_pheno);
2606
2607 my @sim_pheno6_vals;
2608 my ($phenotypes_pheno_sim_heatmap_tempfile6_fh, $phenotypes_pheno_sim_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2609 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile6;
2610 print $F_pheno "trait_type,row,col,value\n";
2611 foreach my $p (@unique_plot_names) {
2612 foreach my $t (@sorted_trait_names) {
2613 my $val = $phenotype_data_altered_env_hash_6_1->{$p}->{$t};
2614 my @row = ("simm1_pheno6_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2615 my $line = join ',', @row;
2616 print $F_pheno "$line\n";
2617 push @sim_pheno6_vals, $val;
2618 }
2619 }
2620 close($F_pheno);
2621 push @plot_corr_full_vals, \@sim_pheno6_vals;
2622
2623 my $sim_pheno6_stat = Statistics::Descriptive::Full->new();
2624 $sim_pheno6_stat->add_data(@sim_pheno6_vals);
2625 my $sig_sim_pheno6 = $sim_pheno6_stat->variance();
2626
2627 my @sim_effect6_vals;
2628 my ($effects_sim_heatmap_tempfile6_fh, $effects_sim_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2629 open($F_eff, ">", $effects_sim_heatmap_tempfile6) || die "Can't open file ".$effects_sim_heatmap_tempfile6;
2630 print $F_eff "trait_type,row,col,value\n";
2631 foreach my $p (@unique_plot_names) {
2632 foreach my $t (@sorted_trait_names) {
2633 my $val = $result_blup_pe_data_delta_altered_env_6_1->{$p}->{$t}->[0];
2634 my @row = ("effm1_sim6_".$trait_name_encoder{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
2635 my $line = join ',', @row;
2636 print $F_eff "$line\n";
2637 push @sim_effect6_vals, $val;
2638 }
2639 }
2640 close($F_eff);
2641 push @plot_corr_full_vals, \@sim_effect6_vals;
2642
2643 my $sim_effect6_stat = Statistics::Descriptive::Full->new();
2644 $sim_effect6_stat->add_data(@sim_effect6_vals);
2645 my $sig_sim_effect6 = $sim_effect6_stat->variance();
2646
2647 my $plot_corr_summary_figure_inputfile_tempfile = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'tmp_drone_statistics/fileXXXX');
2648 open($F_eff, ">", $plot_corr_summary_figure_inputfile_tempfile) || die "Can't open file ".$plot_corr_summary_figure_inputfile_tempfile;
2649 foreach (@plot_corr_full_vals) {
2650 my $line = join ',', @$_;
2651 print $F_eff $line."\n";
2652 }
2653 close($F_eff);
2654
2655 my $plot_corr_summary_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2656 $plot_corr_summary_figure_tempfile_string .= '.png';
2657 my $plot_corr_summary_figure_tempfile = $c->config->{basepath}."/".$plot_corr_summary_figure_tempfile_string;
2658
2659 my $cmd_plotcorrsum_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
2660 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
2661 mat_full <- data.frame(t(mat_full_t));
2662 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_1\', \'mat_eff_1\', \'mat_eff_altered_1\',
2663 \'mat_p_sim1_1\', \'mat_eff_sim1_1\',
2664 \'mat_p_sim2_1\', \'mat_eff_sim2_1\',
2665 \'mat_p_sim3_1\', \'mat_eff_sim3_1\',
2666 \'mat_p_sim4_1\', \'mat_eff_sim4_1\',
2667 \'mat_p_sim5_1\', \'mat_eff_sim5_1\',
2668 \'mat_p_sim6_1\', \'mat_eff_sim6_1\');
2669 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2670 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
2671 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
2672 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
2673 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
2674 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
2675 mat <- data.frame(pheno_orig = mat_full\$mat_orig, pheno_altm1 = mat_full\$mat_altered_1, eff_origm1 = mat_full\$mat_eff_1, eff_altm1 = mat_full\$mat_eff_altered_1, env_lin = mat_env\$value, pheno_linm1 = mat_full\$mat_p_sim1_1, lin_effm1 = mat_full\$mat_eff_sim1_1, env_n1d = mat_env2\$value, pheno_n1dm1 = mat_full\$mat_p_sim2_1, n1d_effm1 = mat_full\$mat_eff_sim2_1, env_n2d = mat_env3\$value, pheno_n2dm1 = mat_full\$mat_p_sim3_1, n2d_effm1 = mat_full\$mat_eff_sim3_1, env_rand = mat_env4\$value, pheno_randm1 = mat_full\$mat_p_sim4_1, rand_effm1 = mat_full\$mat_eff_sim4_1, env_ar1 = mat_env5\$value, pheno_ar1m1 = mat_full\$mat_p_sim5_1, ar1_effm1 = mat_full\$mat_eff_sim5_1, env_realdata = mat_env6\$value, pheno_realdatam1 = mat_full\$mat_p_sim6_1, realdata_effm1 = mat_full\$mat_eff_sim6_1);
2676 gg <- ggcorr(data=mat, hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE, label_round = 2);
2677 ggsave(\''.$plot_corr_summary_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, units=\'in\');
2678 "';
2679 # print STDERR Dumper $cmd_plotcorrsum_plot;
2680
2681 my $status_plotcorrsum_plot = system($cmd_plotcorrsum_plot);
2682 push @$spatial_effects_plots, [$plot_corr_summary_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_envsimscorr_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2683
2684 my $env_effects_first_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2685 $env_effects_first_figure_tempfile_string .= '.png';
2686 my $env_effects_first_figure_tempfile = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string;
2687
2688 my $env_effects_first_figure_tempfile_string_2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2689 $env_effects_first_figure_tempfile_string_2 .= '.png';
2690 my $env_effects_first_figure_tempfile_2 = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string_2;
2691
2692 my $output_plot_row = 'row';
2693 my $output_plot_col = 'col';
2694 if ($max_col > $max_row) {
2695 $output_plot_row = 'col';
2696 $output_plot_col = 'row';
2697 }
2698
2699 my $cmd_spatialfirst_plot_2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2700 mat_orig <- fread(\''.$phenotypes_original_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2701 mat_altered_1 <- fread(\''.$phenotypes_post_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2702 pheno_mat <- rbind(mat_orig, mat_altered_1);
2703 options(device=\'png\');
2704 par();
2705 gg <- ggplot(pheno_mat, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2706 geom_tile() +
2707 scale_fill_viridis(discrete=FALSE) +
2708 coord_equal() +
2709 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2710 ggsave(\''.$env_effects_first_figure_tempfile_2.'\', gg, device=\'png\', width=20, height=20, units=\'in\');
2711 "';
2712 # print STDERR Dumper $cmd;
2713 my $status_spatialfirst_plot_2 = system($cmd_spatialfirst_plot_2);
2714 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string_2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_origheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2715
2716 my ($sim_effects_corr_results_fh, $sim_effects_corr_results) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2717
2718 my $cmd_spatialfirst_plot = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2719 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
2720 mat_full <- data.frame(t(mat_full_t));
2721 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_1\', \'mat_eff_1\', \'mat_eff_altered_1\',
2722 \'mat_p_sim1_1\', \'mat_eff_sim1_1\',
2723 \'mat_p_sim2_1\', \'mat_eff_sim2_1\',
2724 \'mat_p_sim3_1\', \'mat_eff_sim3_1\',
2725 \'mat_p_sim4_1\', \'mat_eff_sim4_1\',
2726 \'mat_p_sim5_1\', \'mat_eff_sim5_1\',
2727 \'mat_p_sim6_1\', \'mat_eff_sim6_1\');
2728 mat_eff_1 <- fread(\''.$effects_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2729 mat_eff_altered_1 <- fread(\''.$effects_post_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2730 effect_mat_1 <- rbind(mat_eff_1, mat_eff_altered_1);
2731 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2732 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
2733 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
2734 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
2735 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
2736 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
2737 options(device=\'png\');
2738 par();
2739 gg_eff_1 <- ggplot(effect_mat_1, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2740 geom_tile() +
2741 scale_fill_viridis(discrete=FALSE) +
2742 coord_equal() +
2743 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2744 ggsave(\''.$env_effects_first_figure_tempfile.'\', arrangeGrob(gg_eff_1, nrow=1), device=\'png\', width=20, height=20, units=\'in\');
2745 write.table(data.frame(airemlf90_grm_random_regression_dap_blups_env_linear = c(cor(mat_env\$value, mat_full\$mat_eff_sim1_1)), airemlf90_grm_random_regression_dap_blups_env_1DN = c(cor(mat_env2\$value, mat_full\$mat_eff_sim2_1)), airemlf90_grm_random_regression_dap_blups_env_2DN = c(cor(mat_env3\$value, mat_full\$mat_eff_sim3_1)), airemlf90_grm_random_regression_dap_blups_env_random = c(cor(mat_env4\$value, mat_full\$mat_eff_sim4_1)), airemlf90_grm_random_regression_dap_blups_env_ar1xar1 = c(cor(mat_env5\$value, mat_full\$mat_eff_sim5_1)), airemlf90_grm_random_regression_dap_blups_env_realdata = c(cor(mat_env6\$value, mat_full\$mat_eff_sim6_1)) ), file=\''.$sim_effects_corr_results.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
2746 "';
2747 # print STDERR Dumper $cmd;
2748 my $status_spatialfirst_plot = system($cmd_spatialfirst_plot);
2749 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_originaleffheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2750
2751 open(my $fh_corr_result, '<', $sim_effects_corr_results) or die "Could not open file '$sim_effects_corr_results' $!";
2752 print STDERR "Opened $sim_effects_corr_results\n";
2753
2754 my $header = <$fh_corr_result>;
2755 my @header;
2756 if ($csv->parse($header)) {
2757 @header = $csv->fields();
2758 }
2759
2760 while (my $row = <$fh_corr_result>) {
2761 my @columns;
2762 my $counter = 0;
2763 if ($csv->parse($row)) {
2764 @columns = $csv->fields();
2765 }
2766 foreach (@columns) {
2767 push @{$env_corr_res->{$header[$counter]."_corrtime_".$sim_env_change_over_time.$correlation_between_times."_envvar_".$env_variance_percent."_".$permanent_environment_structure}->{values}}, $_;
2768 $counter++;
2769 }
2770 }
2771 close($fh_corr_result);
2772
2773 my $env_effects_sim_figure_tempfile_string_env1 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2774 $env_effects_sim_figure_tempfile_string_env1 .= '.png';
2775 my $env_effects_sim_figure_tempfile_env1 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env1;
2776
2777 my $cmd_spatialenvsim_plot_env1 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2778 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2779 mat_p_sim <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2780 mat_eff_sim <- fread(\''.$effects_sim_heatmap_tempfile.'\', header=TRUE, sep=\',\');
2781 options(device=\'png\');
2782 par();
2783 gg_env <- ggplot(mat_env, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2784 geom_tile() +
2785 scale_fill_viridis(discrete=FALSE) +
2786 coord_equal() +
2787 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2788 gg_p_sim <- ggplot(mat_p_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2789 geom_tile() +
2790 scale_fill_viridis(discrete=FALSE) +
2791 coord_equal() +
2792 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2793 gg_eff_sim <- ggplot(mat_eff_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2794 geom_tile() +
2795 scale_fill_viridis(discrete=FALSE) +
2796 coord_equal() +
2797 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2798 ggsave(\''.$env_effects_sim_figure_tempfile_env1.'\', arrangeGrob(gg_env, gg_p_sim, gg_eff_sim, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2799 "';
2800 # print STDERR Dumper $cmd_spatialenvsim_plot_env1;
2801 my $status_spatialenvsim_plot_env1 = system($cmd_spatialenvsim_plot_env1);
2802 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env1, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env1effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2803
2804 my $env_effects_sim_figure_tempfile_string_env2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2805 $env_effects_sim_figure_tempfile_string_env2 .= '.png';
2806 my $env_effects_sim_figure_tempfile_env2 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env2;
2807
2808 my $cmd_spatialenvsim_plot_env2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2809 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
2810 mat_p_sim2 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
2811 mat_eff_sim2 <- fread(\''.$effects_sim_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
2812 options(device=\'png\');
2813 par();
2814 gg_env2 <- ggplot(mat_env2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2815 geom_tile() +
2816 scale_fill_viridis(discrete=FALSE) +
2817 coord_equal() +
2818 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2819 gg_p_sim2 <- ggplot(mat_p_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2820 geom_tile() +
2821 scale_fill_viridis(discrete=FALSE) +
2822 coord_equal() +
2823 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2824 gg_eff_sim2 <- ggplot(mat_eff_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2825 geom_tile() +
2826 scale_fill_viridis(discrete=FALSE) +
2827 coord_equal() +
2828 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2829 ggsave(\''.$env_effects_sim_figure_tempfile_env2.'\', arrangeGrob(gg_env2, gg_p_sim2, gg_eff_sim2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2830 "';
2831 # print STDERR Dumper $cmd_spatialenvsim_plot_env2;
2832 my $status_spatialenvsim_plot_env2 = system($cmd_spatialenvsim_plot_env2);
2833 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env2effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2834
2835 my $env_effects_sim_figure_tempfile_string_env3 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2836 $env_effects_sim_figure_tempfile_string_env3 .= '.png';
2837 my $env_effects_sim_figure_tempfile_env3 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env3;
2838
2839 my $cmd_spatialenvsim_plot_env3 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2840 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
2841 mat_p_sim3 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
2842 mat_eff_sim3 <- fread(\''.$effects_sim_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
2843 options(device=\'png\');
2844 par();
2845 gg_env3 <- ggplot(mat_env3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2846 geom_tile() +
2847 scale_fill_viridis(discrete=FALSE) +
2848 coord_equal() +
2849 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2850 gg_p_sim3 <- ggplot(mat_p_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2851 geom_tile() +
2852 scale_fill_viridis(discrete=FALSE) +
2853 coord_equal() +
2854 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2855 gg_eff_sim3 <- ggplot(mat_eff_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2856 geom_tile() +
2857 scale_fill_viridis(discrete=FALSE) +
2858 coord_equal() +
2859 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2860 ggsave(\''.$env_effects_sim_figure_tempfile_env3.'\', arrangeGrob(gg_env3, gg_p_sim3, gg_eff_sim3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2861 "';
2862 # print STDERR Dumper $cmd_spatialenvsim_plot_env3;
2863 my $status_spatialenvsim_plot_env3 = system($cmd_spatialenvsim_plot_env3);
2864 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env3, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env3effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2865
2866 my $env_effects_sim_figure_tempfile_string_env4 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2867 $env_effects_sim_figure_tempfile_string_env4 .= '.png';
2868 my $env_effects_sim_figure_tempfile_env4 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env4;
2869
2870 my $cmd_spatialenvsim_plot_env4 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2871 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
2872 mat_p_sim4 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
2873 mat_eff_sim4 <- fread(\''.$effects_sim_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
2874 options(device=\'png\');
2875 par();
2876 gg_env4 <- ggplot(mat_env4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2877 geom_tile() +
2878 scale_fill_viridis(discrete=FALSE) +
2879 coord_equal() +
2880 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2881 gg_p_sim4 <- ggplot(mat_p_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2882 geom_tile() +
2883 scale_fill_viridis(discrete=FALSE) +
2884 coord_equal() +
2885 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2886 gg_eff_sim4 <- ggplot(mat_eff_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2887 geom_tile() +
2888 scale_fill_viridis(discrete=FALSE) +
2889 coord_equal() +
2890 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2891 ggsave(\''.$env_effects_sim_figure_tempfile_env4.'\', arrangeGrob(gg_env4, gg_p_sim4, gg_eff_sim4, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2892 "';
2893 # print STDERR Dumper $cmd_spatialenvsim_plot_env4;
2894 my $status_spatialenvsim_plot_env4 = system($cmd_spatialenvsim_plot_env4);
2895 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env4, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env4effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2896
2897 my $env_effects_sim_figure_tempfile_string_env5 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2898 $env_effects_sim_figure_tempfile_string_env5 .= '.png';
2899 my $env_effects_sim_figure_tempfile_env5 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env5;
2900
2901 my $cmd_spatialenvsim_plot_env5 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2902 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
2903 mat_p_sim5 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
2904 mat_eff_sim5 <- fread(\''.$effects_sim_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
2905 options(device=\'png\');
2906 par();
2907 gg_env5 <- ggplot(mat_env5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2908 geom_tile() +
2909 scale_fill_viridis(discrete=FALSE) +
2910 coord_equal() +
2911 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2912 gg_p_sim5 <- ggplot(mat_p_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2913 geom_tile() +
2914 scale_fill_viridis(discrete=FALSE) +
2915 coord_equal() +
2916 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2917 gg_eff_sim5 <- ggplot(mat_eff_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2918 geom_tile() +
2919 scale_fill_viridis(discrete=FALSE) +
2920 coord_equal() +
2921 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2922 ggsave(\''.$env_effects_sim_figure_tempfile_env5.'\', arrangeGrob(gg_env5, gg_p_sim5, gg_eff_sim5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2923 "';
2924 # print STDERR Dumper $cmd_spatialenvsim_plot_env5;
2925 my $status_spatialenvsim_plot_env5 = system($cmd_spatialenvsim_plot_env5);
2926 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env5, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env5effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2927
2928 my $env_effects_sim_figure_tempfile_string_env6 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
2929 $env_effects_sim_figure_tempfile_string_env6 .= '.png';
2930 my $env_effects_sim_figure_tempfile_env6 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_env6;
2931
2932 my $cmd_spatialenvsim_plot_env6 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
2933 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
2934 mat_p_sim6 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
2935 mat_eff_sim6 <- fread(\''.$effects_sim_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
2936 options(device=\'png\');
2937 par();
2938 gg_env6 <- ggplot(mat_env6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2939 geom_tile() +
2940 scale_fill_viridis(discrete=FALSE) +
2941 coord_equal() +
2942 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2943 gg_p_sim6 <- ggplot(mat_p_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2944 geom_tile() +
2945 scale_fill_viridis(discrete=FALSE) +
2946 coord_equal() +
2947 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2948 gg_eff_sim6 <- ggplot(mat_eff_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
2949 geom_tile() +
2950 scale_fill_viridis(discrete=FALSE) +
2951 coord_equal() +
2952 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names).');
2953 ggsave(\''.$env_effects_sim_figure_tempfile_env6.'\', arrangeGrob(gg_env6, gg_p_sim6, gg_eff_sim6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
2954 "';
2955 # print STDERR Dumper $cmd_spatialenvsim_plot_env6;
2956 my $status_spatialenvsim_plot_env6 = system($cmd_spatialenvsim_plot_env6);
2957 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_env6, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env6effheatmap_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
2958 };
2959
2960 eval {
2961 my @sorted_germplasm_names = sort keys %unique_accessions;
2962 @sorted_trait_names = sort keys %$rr_unique_traits_hash_1;
2963
2964 my @original_blup_vals;
2965 my ($effects_original_line_chart_tempfile_fh, $effects_original_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2966 open(my $F_pheno, ">", $effects_original_line_chart_tempfile) || die "Can't open file ".$effects_original_line_chart_tempfile;
2967 print $F_pheno "germplasmName,time,value\n";
2968 foreach my $p (@sorted_germplasm_names) {
2969 foreach my $t (@sorted_trait_names) {
2970 my $val = $result_blup_data_original_1->{$p}->{$t}->[0];
2971 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
2972 push @original_blup_vals, $val;
2973 my $line = join ',', @row;
2974 print $F_pheno "$line\n";
2975 }
2976 }
2977 close($F_pheno);
2978
2979 my $original_blup_stat = Statistics::Descriptive::Full->new();
2980 $original_blup_stat->add_data(@original_blup_vals);
2981 my $sig_original_blup = $original_blup_stat->variance();
2982
2983 my @altered_blups_vals;
2984 my ($effects_altered_line_chart_tempfile_fh, $effects_altered_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
2985 open($F_pheno, ">", $effects_altered_line_chart_tempfile) || die "Can't open file ".$effects_altered_line_chart_tempfile;
2986 print $F_pheno "germplasmName,time,value\n";
2987 foreach my $p (@sorted_germplasm_names) {
2988 foreach my $t (@sorted_trait_names) {
2989 my $val = $result_blup_data_altered_1->{$p}->{$t}->[0];
2990 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
2991 my $line = join ',', @row;
2992 print $F_pheno "$line\n";
2993 push @altered_blups_vals, $val;
2994 }
2995 }
2996 close($F_pheno);
2997
2998 my $altered_blup_stat = Statistics::Descriptive::Full->new();
2999 $altered_blup_stat->add_data(@altered_blups_vals);
3000 my $sig_altered_blup = $altered_blup_stat->variance();
3001
3002 my @sim1_blup_vals;
3003 my ($effects_altered_env1_line_chart_tempfile_fh, $effects_altered_env1_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3004 open($F_pheno, ">", $effects_altered_env1_line_chart_tempfile) || die "Can't open file ".$effects_altered_env1_line_chart_tempfile;
3005 print $F_pheno "germplasmName,time,value\n";
3006 foreach my $p (@sorted_germplasm_names) {
3007 foreach my $t (@sorted_trait_names) {
3008 my $val = $result_blup_data_altered_env_1_1->{$p}->{$t}->[0];
3009 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3010 my $line = join ',', @row;
3011 print $F_pheno "$line\n";
3012 push @sim1_blup_vals, $val;
3013 }
3014 }
3015 close($F_pheno);
3016
3017 my $sim1_blup_stat = Statistics::Descriptive::Full->new();
3018 $sim1_blup_stat->add_data(@sim1_blup_vals);
3019 my $sig_sim1_blup = $sim1_blup_stat->variance();
3020
3021 my @sim2_blup_vals;
3022 my ($effects_altered_env2_line_chart_tempfile_fh, $effects_altered_env2_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3023 open($F_pheno, ">", $effects_altered_env2_line_chart_tempfile) || die "Can't open file ".$effects_altered_env2_line_chart_tempfile;
3024 print $F_pheno "germplasmName,time,value\n";
3025 foreach my $p (@sorted_germplasm_names) {
3026 foreach my $t (@sorted_trait_names) {
3027 my $val = $result_blup_data_altered_env_2_1->{$p}->{$t}->[0];
3028 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3029 my $line = join ',', @row;
3030 print $F_pheno "$line\n";
3031 push @sim2_blup_vals, $val;
3032 }
3033 }
3034 close($F_pheno);
3035
3036 my $sim2_blup_stat = Statistics::Descriptive::Full->new();
3037 $sim2_blup_stat->add_data(@sim2_blup_vals);
3038 my $sig_sim2_blup = $sim2_blup_stat->variance();
3039
3040 my @sim3_blup_vals;
3041 my ($effects_altered_env3_line_chart_tempfile_fh, $effects_altered_env3_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3042 open($F_pheno, ">", $effects_altered_env3_line_chart_tempfile) || die "Can't open file ".$effects_altered_env3_line_chart_tempfile;
3043 print $F_pheno "germplasmName,time,value\n";
3044 foreach my $p (@sorted_germplasm_names) {
3045 foreach my $t (@sorted_trait_names) {
3046 my $val = $result_blup_data_altered_env_3_1->{$p}->{$t}->[0];
3047 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3048 my $line = join ',', @row;
3049 print $F_pheno "$line\n";
3050 push @sim3_blup_vals, $val;
3051 }
3052 }
3053 close($F_pheno);
3054
3055 my $sim3_blup_stat = Statistics::Descriptive::Full->new();
3056 $sim3_blup_stat->add_data(@sim3_blup_vals);
3057 my $sig_sim3_blup = $sim3_blup_stat->variance();
3058
3059 my @sim4_blup_vals;
3060 my ($effects_altered_env4_line_chart_tempfile_fh, $effects_altered_env4_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3061 open($F_pheno, ">", $effects_altered_env4_line_chart_tempfile) || die "Can't open file ".$effects_altered_env4_line_chart_tempfile;
3062 print $F_pheno "germplasmName,time,value\n";
3063 foreach my $p (@sorted_germplasm_names) {
3064 foreach my $t (@sorted_trait_names) {
3065 my $val = $result_blup_data_altered_env_4_1->{$p}->{$t}->[0];
3066 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3067 my $line = join ',', @row;
3068 print $F_pheno "$line\n";
3069 push @sim4_blup_vals, $val;
3070 }
3071 }
3072 close($F_pheno);
3073
3074 my $sim4_blup_stat = Statistics::Descriptive::Full->new();
3075 $sim4_blup_stat->add_data(@sim4_blup_vals);
3076 my $sig_sim4_blup = $sim4_blup_stat->variance();
3077
3078 my @sim5_blup_vals;
3079 my ($effects_altered_env5_line_chart_tempfile_fh, $effects_altered_env5_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3080 open($F_pheno, ">", $effects_altered_env5_line_chart_tempfile) || die "Can't open file ".$effects_altered_env5_line_chart_tempfile;
3081 print $F_pheno "germplasmName,time,value\n";
3082 foreach my $p (@sorted_germplasm_names) {
3083 foreach my $t (@sorted_trait_names) {
3084 my $val = $result_blup_data_altered_env_5_1->{$p}->{$t}->[0];
3085 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3086 my $line = join ',', @row;
3087 print $F_pheno "$line\n";
3088 push @sim5_blup_vals, $val;
3089 }
3090 }
3091 close($F_pheno);
3092
3093 my $sim5_blup_stat = Statistics::Descriptive::Full->new();
3094 $sim5_blup_stat->add_data(@sim5_blup_vals);
3095 my $sig_sim5_blup = $sim5_blup_stat->variance();
3096
3097 my @sim6_blup_vals;
3098 my ($effects_altered_env6_line_chart_tempfile_fh, $effects_altered_env6_line_chart_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
3099 open($F_pheno, ">", $effects_altered_env6_line_chart_tempfile) || die "Can't open file ".$effects_altered_env6_line_chart_tempfile;
3100 print $F_pheno "germplasmName,time,value\n";
3101 foreach my $p (@sorted_germplasm_names) {
3102 foreach my $t (@sorted_trait_names) {
3103 my $val = $result_blup_data_altered_env_6_1->{$p}->{$t}->[0];
3104 my @row = ($p, $trait_to_time_map_hash_1->{$t}, $val);
3105 my $line = join ',', @row;
3106 print $F_pheno "$line\n";
3107 push @sim6_blup_vals, $val;
3108 }
3109 }
3110 close($F_pheno);
3111
3112 my $sim6_blup_stat = Statistics::Descriptive::Full->new();
3113 $sim6_blup_stat->add_data(@sim6_blup_vals);
3114 my $sig_sim6_blup = $sim6_blup_stat->variance();
3115
3116 my @set = ('0' ..'9', 'A' .. 'F');
3117 my @colors;
3118 for (1..scalar(@sorted_germplasm_names)) {
3119 my $str = join '' => map $set[rand @set], 1 .. 6;
3120 push @colors, '#'.$str;
3121 }
3122 my $color_string = join '\',\'', @colors;
3123
3124 my $genetic_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3125 $genetic_effects_figure_tempfile_string .= '.png';
3126 my $genetic_effects_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_figure_tempfile_string;
3127
3128 my $genetic_effects_alt_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3129 $genetic_effects_alt_figure_tempfile_string .= '.png';
3130 my $genetic_effects_alt_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_figure_tempfile_string;
3131
3132 my $genetic_effects_alt_env1_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3133 $genetic_effects_alt_env1_figure_tempfile_string .= '.png';
3134 my $genetic_effects_alt_env1_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env1_figure_tempfile_string;
3135
3136 my $genetic_effects_alt_env2_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3137 $genetic_effects_alt_env2_figure_tempfile_string .= '.png';
3138 my $genetic_effects_alt_env2_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env2_figure_tempfile_string;
3139
3140 my $genetic_effects_alt_env3_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3141 $genetic_effects_alt_env3_figure_tempfile_string .= '.png';
3142 my $genetic_effects_alt_env3_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env3_figure_tempfile_string;
3143
3144 my $genetic_effects_alt_env4_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3145 $genetic_effects_alt_env4_figure_tempfile_string .= '.png';
3146 my $genetic_effects_alt_env4_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env4_figure_tempfile_string;
3147
3148 my $genetic_effects_alt_env5_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3149 $genetic_effects_alt_env5_figure_tempfile_string .= '.png';
3150 my $genetic_effects_alt_env5_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env5_figure_tempfile_string;
3151
3152 my $genetic_effects_alt_env6_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
3153 $genetic_effects_alt_env6_figure_tempfile_string .= '.png';
3154 my $genetic_effects_alt_env6_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env6_figure_tempfile_string;
3155
3156 my $cmd_gen_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3157 mat <- fread(\''.$effects_original_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3158 mat\$time <- as.numeric(as.character(mat\$time));
3159 options(device=\'png\');
3160 par();
3161 sp <- ggplot(mat, aes(x = time, y = value)) +
3162 geom_line(aes(color = germplasmName), size = 1) +
3163 scale_fill_manual(values = c(\''.$color_string.'\')) +
3164 theme_minimal();
3165 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3166 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3167 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3168 sp <- sp + labs(title = \'Original Genetic Effects\');';
3169 if (scalar(@sorted_germplasm_names) > 100) {
3170 $cmd_gen_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3171 }
3172 $cmd_gen_plot .= 'ggsave(\''.$genetic_effects_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3173 "';
3174 print STDERR Dumper $cmd_gen_plot;
3175 my $status_gen_plot = system($cmd_gen_plot);
3176 push @$spatial_effects_plots, [$genetic_effects_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_efforigline_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3177
3178 my $cmd_gen_alt_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3179 mat <- fread(\''.$effects_altered_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3180 mat\$time <- as.numeric(as.character(mat\$time));
3181 options(device=\'png\');
3182 par();
3183 sp <- ggplot(mat, aes(x = time, y = value)) +
3184 geom_line(aes(color = germplasmName), size = 1) +
3185 scale_fill_manual(values = c(\''.$color_string.'\')) +
3186 theme_minimal();
3187 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3188 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3189 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3190 sp <- sp + labs(title = \'Altered Genetic Effects\');';
3191 if (scalar(@sorted_germplasm_names) > 100) {
3192 $cmd_gen_alt_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3193 }
3194 $cmd_gen_alt_plot .= 'ggsave(\''.$genetic_effects_alt_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3195 "';
3196 print STDERR Dumper $cmd_gen_alt_plot;
3197 my $status_gen_alt_plot = system($cmd_gen_alt_plot);
3198 push @$spatial_effects_plots, [$genetic_effects_alt_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltline_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3199
3200 my $cmd_gen_env1_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3201 mat <- fread(\''.$effects_altered_env1_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3202 mat\$time <- as.numeric(as.character(mat\$time));
3203 options(device=\'png\');
3204 par();
3205 sp <- ggplot(mat, aes(x = time, y = value)) +
3206 geom_line(aes(color = germplasmName), size = 1) +
3207 scale_fill_manual(values = c(\''.$color_string.'\')) +
3208 theme_minimal();
3209 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3210 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3211 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3212 sp <- sp + labs(title = \'SimLinear Genetic Effects\');';
3213 if (scalar(@sorted_germplasm_names) > 100) {
3214 $cmd_gen_env1_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3215 }
3216 $cmd_gen_env1_plot .= 'ggsave(\''.$genetic_effects_alt_env1_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3217 "';
3218 print STDERR Dumper $cmd_gen_env1_plot;
3219 my $status_gen_env1_plot = system($cmd_gen_env1_plot);
3220 push @$spatial_effects_plots, [$genetic_effects_alt_env1_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv1line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3221
3222 my $cmd_gen_env2_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3223 mat <- fread(\''.$effects_altered_env2_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3224 mat\$time <- as.numeric(as.character(mat\$time));
3225 options(device=\'png\');
3226 par();
3227 sp <- ggplot(mat, aes(x = time, y = value)) +
3228 geom_line(aes(color = germplasmName), size = 1) +
3229 scale_fill_manual(values = c(\''.$color_string.'\')) +
3230 theme_minimal();
3231 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3232 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3233 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3234 sp <- sp + labs(title = \'Sim1DN Genetic Effects\');';
3235 if (scalar(@sorted_germplasm_names) > 100) {
3236 $cmd_gen_env2_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3237 }
3238 $cmd_gen_env2_plot .= 'ggsave(\''.$genetic_effects_alt_env2_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3239 "';
3240 print STDERR Dumper $cmd_gen_env2_plot;
3241 my $status_gen_env2_plot = system($cmd_gen_env2_plot);
3242 push @$spatial_effects_plots, [$genetic_effects_alt_env2_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv2line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3243
3244 my $cmd_gen_env3_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3245 mat <- fread(\''.$effects_altered_env3_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3246 mat\$time <- as.numeric(as.character(mat\$time));
3247 options(device=\'png\');
3248 par();
3249 sp <- ggplot(mat, aes(x = time, y = value)) +
3250 geom_line(aes(color = germplasmName), size = 1) +
3251 scale_fill_manual(values = c(\''.$color_string.'\')) +
3252 theme_minimal();
3253 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3254 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3255 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3256 sp <- sp + labs(title = \'Sim2DN Genetic Effects\');';
3257 if (scalar(@sorted_germplasm_names) > 100) {
3258 $cmd_gen_env3_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3259 }
3260 $cmd_gen_env3_plot .= 'ggsave(\''.$genetic_effects_alt_env3_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3261 "';
3262 print STDERR Dumper $cmd_gen_env3_plot;
3263 my $status_gen_env3_plot = system($cmd_gen_env3_plot);
3264 push @$spatial_effects_plots, [$genetic_effects_alt_env3_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv3line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3265
3266 my $cmd_gen_env4_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3267 mat <- fread(\''.$effects_altered_env4_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3268 mat\$time <- as.numeric(as.character(mat\$time));
3269 options(device=\'png\');
3270 par();
3271 sp <- ggplot(mat, aes(x = time, y = value)) +
3272 geom_line(aes(color = germplasmName), size = 1) +
3273 scale_fill_manual(values = c(\''.$color_string.'\')) +
3274 theme_minimal();
3275 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3276 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3277 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3278 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
3279 if (scalar(@sorted_germplasm_names) > 100) {
3280 $cmd_gen_env4_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3281 }
3282 $cmd_gen_env4_plot .= 'ggsave(\''.$genetic_effects_alt_env4_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3283 "';
3284 print STDERR Dumper $cmd_gen_env4_plot;
3285 my $status_gen_env4_plot = system($cmd_gen_env4_plot);
3286 push @$spatial_effects_plots, [$genetic_effects_alt_env4_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv4line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3287
3288 my $cmd_gen_env5_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3289 mat <- fread(\''.$effects_altered_env5_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3290 mat\$time <- as.numeric(as.character(mat\$time));
3291 options(device=\'png\');
3292 par();
3293 sp <- ggplot(mat, aes(x = time, y = value)) +
3294 geom_line(aes(color = germplasmName), size = 1) +
3295 scale_fill_manual(values = c(\''.$color_string.'\')) +
3296 theme_minimal();
3297 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3298 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3299 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3300 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
3301 if (scalar(@sorted_germplasm_names) > 100) {
3302 $cmd_gen_env5_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3303 }
3304 $cmd_gen_env5_plot .= 'ggsave(\''.$genetic_effects_alt_env5_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3305 "';
3306 print STDERR Dumper $cmd_gen_env5_plot;
3307 my $status_gen_env5_plot = system($cmd_gen_env5_plot);
3308 push @$spatial_effects_plots, [$genetic_effects_alt_env5_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv5line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3309
3310 my $cmd_gen_env6_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
3311 mat <- fread(\''.$effects_altered_env6_line_chart_tempfile.'\', header=TRUE, sep=\',\');
3312 mat\$time <- as.numeric(as.character(mat\$time));
3313 options(device=\'png\');
3314 par();
3315 sp <- ggplot(mat, aes(x = time, y = value)) +
3316 geom_line(aes(color = germplasmName), size = 1) +
3317 scale_fill_manual(values = c(\''.$color_string.'\')) +
3318 theme_minimal();
3319 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
3320 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
3321 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
3322 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
3323 if (scalar(@sorted_germplasm_names) > 100) {
3324 $cmd_gen_env6_plot .= 'sp <- sp + theme(legend.position = \'none\');';
3325 }
3326 $cmd_gen_env6_plot .= 'ggsave(\''.$genetic_effects_alt_env6_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
3327 "';
3328 print STDERR Dumper $cmd_gen_env6_plot;
3329 my $status_gen_env6_plot = system($cmd_gen_env6_plot);
3330 push @$spatial_effects_plots, [$genetic_effects_alt_env6_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv6line_"."envvar_".$env_variance_percent."_".$permanent_environment_structure."_".$iterations];
3331 };
3332
3333 push @$env_varcomps, {
3334 type => "$statistics_select: Env Variance $env_variance_percent : PE Structure: $permanent_environment_structure : SimCorrelation: $correlation_between_times : Iteration $iterations",
3335 statistics_select => "$statistics_select: Env Variance $env_variance_percent : PE Structure: $permanent_environment_structure : SimCorrelation: $correlation_between_times",
3336 correlation_between_times => $correlation_between_times,
3337 env_variance => $env_variance_percent,
3338 original => {
3339 genetic_covariance => $rr_coeff_genetic_covariance_original_array_1,
3340 env_covariance => $rr_coeff_env_covariance_original_array_1,
3341 residual => $rr_residual_variance_original_1,
3342 genetic_correlation => $rr_coeff_genetic_correlation_original_array_1,
3343 env_correlation => $rr_coeff_env_correlation_original_array_1,
3344 cv_1 => $result_cv_original_1,
3345 cv_2 => $result_cv_2_original_1
3346 },
3347 altered => {
3348 genetic_covariance => $rr_coeff_genetic_covariance_altered_array_1,
3349 env_covariance => $rr_coeff_env_covariance_altered_array_1,
3350 residual => $rr_residual_variance_altered_1,
3351 genetic_correlation => $rr_coeff_genetic_correlation_altered_array_1,
3352 env_correlation => $rr_coeff_env_correlation_altered_array_1,
3353 cv_1 => $result_cv_altered_1,
3354 cv_2 => $result_cv_2_altered_1
3355 },
3356 env_linear => {
3357 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_1_1,
3358 env_covariance => $rr_coeff_env_covariance_altered_env_array_1_1,
3359 residual => $rr_residual_variance_altered_env_1_1,
3360 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_1_1,
3361 env_correlation => $rr_coeff_env_correlation_altered_env_array_1_1,
3362 cv_1 => $result_cv_altered_env_1_1,
3363 cv_2 => $result_cv_2_altered_env_1_1
3364 },
3365 env_1DN => {
3366 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_2_1,
3367 env_covariance => $rr_coeff_env_covariance_altered_env_array_2_1,
3368 residual => $rr_residual_variance_altered_env_2_1,
3369 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_2_1,
3370 env_correlation => $rr_coeff_env_correlation_altered_env_array_2_1,
3371 cv_1 => $result_cv_altered_env_2_1,
3372 cv_2 => $result_cv_2_altered_env_2_1
3373 },
3374 env_2DN => {
3375 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_3_1,
3376 env_covariance => $rr_coeff_env_covariance_altered_env_array_3_1,
3377 residual => $rr_residual_variance_altered_env_3_1,
3378 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_3_1,
3379 env_correlation => $rr_coeff_env_correlation_altered_env_array_3_1,
3380 cv_1 => $result_cv_altered_env_3_1,
3381 cv_2 => $result_cv_2_altered_env_3_1
3382 },
3383 env_random => {
3384 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_4_1,
3385 env_covariance => $rr_coeff_env_covariance_altered_env_array_4_1,
3386 residual => $rr_residual_variance_altered_env_4_1,
3387 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_4_1,
3388 env_correlation => $rr_coeff_env_correlation_altered_env_array_4_1,
3389 cv_1 => $result_cv_altered_env_4_1,
3390 cv_2 => $result_cv_2_altered_env_4_1
3391 },
3392 env_ar1xar1 => {
3393 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_5_1,
3394 env_covariance => $rr_coeff_env_covariance_altered_env_array_5_1,
3395 residual => $rr_residual_variance_altered_env_5_1,
3396 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_5_1,
3397 env_correlation => $rr_coeff_env_correlation_altered_env_array_5_1,
3398 cv_1 => $result_cv_altered_env_5_1,
3399 cv_2 => $result_cv_2_altered_env_5_1
3400 },
3401 env_realdata => {
3402 genetic_covariance => $rr_coeff_genetic_covariance_altered_env_array_6_1,
3403 env_covariance => $rr_coeff_env_covariance_altered_env_array_6_1,
3404 residual => $rr_residual_variance_altered_env_6_1,
3405 genetic_correlation => $rr_coeff_genetic_correlation_altered_env_array_6_1,
3406 env_correlation => $rr_coeff_env_correlation_altered_env_array_6_1,
3407 cv_1 => $result_cv_altered_env_6_1,
3408 cv_2 => $result_cv_2_altered_env_6_1
3409 }
3410 };
3411 }
3412
3413 my (%phenotype_data_original_2, @data_matrix_original_2, @data_matrix_phenotypes_original_2);
3414 my (%trait_name_encoder_2, %trait_name_encoder_rev_2, %seen_days_after_plantings_2, %stock_info_2, %seen_times_2, %seen_trial_ids_2, %trait_to_time_map_2, %trait_composing_info_2, @sorted_trait_names_2, %seen_trait_names_2, %unique_traits_ids_2, @phenotype_header_2, $header_string_2);
3415 my (@sorted_scaled_ln_times_2, %plot_id_factor_map_reverse_2, %plot_id_count_map_reverse_2, %accession_id_factor_map_2, %accession_id_factor_map_reverse_2, %time_count_map_reverse_2, @rep_time_factors_2, @ind_rep_factors_2, %plot_rep_time_factor_map_2, %seen_rep_times_2, %seen_ind_reps_2, @legs_header_2, %polynomial_map_2);
3416 my $time_min_2 = 100000000;
3417 my $time_max_2 = 0;
3418 my $phenotype_min_original_2 = 1000000000;
3419 my $phenotype_max_original_2 = -1000000000;
3420
3421 if ($statistics_select_original eq 'sommer_grm_spatial_genetic_blups' || $statistics_select_original eq 'sommer_grm_spatial_pure_2dspl_genetic_blups') {
3422 $statistics_select = $statistics_select_original eq 'sommer_grm_spatial_genetic_blups' ? 'sommer_grm_spatial_genetic_blups' : 'sommer_grm_spatial_pure_2dspl_genetic_blups';
3423
3424 print STDERR "PREPARE ORIGINAL PHENOTYPE FILES 2\n";
3425 eval {
3426 my $phenotypes_search_2 = CXGN::Phenotypes::SearchFactory->instantiate(
3427 'MaterializedViewTable',
3428 {
3429 bcs_schema=>$schema,
3430 data_level=>'plot',
3431 trait_list=>$trait_id_list,
3432 trial_list=>$field_trial_id_list,
3433 include_timestamp=>0,
3434 exclude_phenotype_outlier=>0
3435 }
3436 );
3437 my ($data_2, $unique_traits_2) = $phenotypes_search_2->search();
3438 @sorted_trait_names_2 = sort keys %$unique_traits_2;
3439
3440 if (scalar(@$data_2) == 0) {
3441 $c->stash->{rest} = { error => "There are no phenotypes for the trials and traits you have selected!"};
3442 return;
3443 }
3444
3445 foreach my $obs_unit (@$data_2){
3446 my $germplasm_name = $obs_unit->{germplasm_uniquename};
3447 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
3448 my $replicate_number = $obs_unit->{obsunit_rep} || '';
3449 my $block_number = $obs_unit->{obsunit_block} || '';
3450 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
3451 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
3452 my $row_number = $obs_unit->{obsunit_row_number} || '';
3453 my $col_number = $obs_unit->{obsunit_col_number} || '';
3454 $seen_trial_ids_2{$obs_unit->{trial_id}}++;
3455 push @plot_ids_ordered, $obsunit_stock_id;
3456
3457 if ($row_number < $min_row) {
3458 $min_row = $row_number;
3459 }
3460 elsif ($row_number >= $max_row) {
3461 $max_row = $row_number;
3462 }
3463 if ($col_number < $min_col) {
3464 $min_col = $col_number;
3465 }
3466 elsif ($col_number >= $max_col) {
3467 $max_col = $col_number;
3468 }
3469
3470 $obsunit_row_col{$row_number}->{$col_number} = {
3471 stock_id => $obsunit_stock_id,
3472 stock_uniquename => $obsunit_stock_uniquename
3473 };
3474 $seen_rows{$row_number}++;
3475 $seen_cols{$col_number}++;
3476 $plot_id_map{$obsunit_stock_id} = $obsunit_stock_uniquename;
3477 $seen_plot_names{$obsunit_stock_uniquename}++;
3478 $seen_plots{$obsunit_stock_id} = $obsunit_stock_uniquename;
3479 $stock_row_col{$obsunit_stock_id} = {
3480 row_number => $row_number,
3481 col_number => $col_number,
3482 obsunit_stock_id => $obsunit_stock_id,
3483 obsunit_name => $obsunit_stock_uniquename,
3484 rep => $replicate_number,
3485 block => $block_number,
3486 germplasm_stock_id => $germplasm_stock_id,
3487 germplasm_name => $germplasm_name
3488 };
3489 $stock_name_row_col{$obsunit_stock_uniquename} = {
3490 row_number => $row_number,
3491 col_number => $col_number,
3492 obsunit_stock_id => $obsunit_stock_id,
3493 obsunit_name => $obsunit_stock_uniquename,
3494 rep => $replicate_number,
3495 block => $block_number,
3496 germplasm_stock_id => $germplasm_stock_id,
3497 germplasm_name => $germplasm_name
3498 };
3499 $stock_row_col_id{$row_number}->{$col_number} = $obsunit_stock_id;
3500 $unique_accessions{$germplasm_name}++;
3501 $stock_info_2{"S".$germplasm_stock_id} = {
3502 uniquename => $germplasm_name
3503 };
3504 my $observations = $obs_unit->{observations};
3505 foreach (@$observations){
3506 my $value = $_->{value};
3507 my $trait_name = $_->{trait_name};
3508 $phenotype_data_original_2{$obsunit_stock_uniquename}->{$trait_name} = $value;
3509 $seen_trait_names_2{$trait_name}++;
3510
3511 if ($value < $phenotype_min_original_2) {
3512 $phenotype_min_original_2 = $value;
3513 }
3514 elsif ($value >= $phenotype_max_original_2) {
3515 $phenotype_max_original_2 = $value;
3516 }
3517
3518 if ($_->{associated_image_project_time_json}) {
3519 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
3520 my $time_days_cvterm = $related_time_terms_json->{day};
3521 my $time_term_string = $time_days_cvterm;
3522 my $time_days = (split '\|', $time_days_cvterm)[0];
3523 my $time_value = (split ' ', $time_days)[1];
3524 $seen_days_after_plantings_2{$time_value}++;
3525 $trait_to_time_map_2{$trait_name} = $time_value;
3526 }
3527 }
3528 }
3529
3530 @unique_plot_names = sort keys %seen_plot_names;
3531
3532 my $trait_name_encoded_2 = 1;
3533 foreach my $trait_name (@sorted_trait_names_2) {
3534 if (!exists($trait_name_encoder_2{$trait_name})) {
3535 my $trait_name_e = 't'.$trait_name_encoded_2;
3536 $trait_name_encoder_2{$trait_name} = $trait_name_e;
3537 $trait_name_encoder_rev_2{$trait_name_e} = $trait_name;
3538 $trait_name_encoded_2++;
3539 }
3540 }
3541
3542 foreach my $p (@unique_plot_names) {
3543 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
3544 if ($fixed_effect_type eq 'fixed_effect_trait') {
3545 $stock_name_row_col{$p}->{rep} = defined($fixed_effect_trait_data->{$obsunit_stock_id}) ? $fixed_effect_trait_data->{$obsunit_stock_id} : 0;
3546 }
3547 my $row_number = $stock_name_row_col{$p}->{row_number};
3548 my $col_number = $stock_name_row_col{$p}->{col_number};
3549 my $replicate = $stock_name_row_col{$p}->{rep};
3550 my $block = $stock_name_row_col{$p}->{block};
3551 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
3552 my $germplasm_name = $stock_name_row_col{$p}->{germplasm_name};
3553
3554 my @row = ($replicate, $block, "S".$germplasm_stock_id, $obsunit_stock_id, $row_number, $col_number, $row_number, $col_number);
3555
3556 foreach my $t (@sorted_trait_names_2) {
3557 if (defined($phenotype_data_original_2{$p}->{$t})) {
3558 push @row, $phenotype_data_original_2{$p}->{$t};
3559 } else {
3560 print STDERR $p." : $t : $germplasm_name : NA \n";
3561 push @row, 'NA';
3562 }
3563 }
3564 push @data_matrix_original_2, \@row;
3565 }
3566
3567 foreach (keys %seen_trial_ids_2){
3568 my $trial = CXGN::Trial->new({bcs_schema=>$schema, trial_id=>$_});
3569 my $traits_assayed = $trial->get_traits_assayed('plot', undef, 'time_ontology');
3570 foreach (@$traits_assayed) {
3571 $unique_traits_ids_2{$_->[0]} = $_;
3572 }
3573 }
3574 foreach (values %unique_traits_ids_2) {
3575 foreach my $component (@{$_->[2]}) {
3576 if (exists($seen_trait_names_2{$_->[1]}) && $component->{cv_type} && $component->{cv_type} eq 'time_ontology') {
3577 my $time_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $component->{cvterm_id}, 'extended');
3578 push @{$trait_composing_info_2{$_->[1]}}, $time_term_string;
3579 }
3580 }
3581 }
3582
3583 @phenotype_header_2 = ("replicate", "block", "id", "plot_id", "rowNumber", "colNumber", "rowNumberFactor", "colNumberFactor");
3584 foreach (@sorted_trait_names_2) {
3585 push @phenotype_header_2, $trait_name_encoder_2{$_};
3586 }
3587 $header_string_2 = join ',', @phenotype_header_2;
3588
3589 open($F, ">", $stats_tempfile) || die "Can't open file ".$stats_tempfile;
3590 print $F $header_string_2."\n";
3591 foreach (@data_matrix_original_2) {
3592 my $line = join ',', @$_;
3593 print $F "$line\n";
3594 }
3595 close($F);
3596
3597 print STDERR Dumper [$phenotype_min_original_2, $phenotype_max_original_2];
3598 };
3599
3600 @seen_rows_array = keys %seen_rows;
3601 @seen_cols_array = keys %seen_cols;
3602 $row_stat = Statistics::Descriptive::Full->new();
3603 $row_stat->add_data(@seen_rows_array);
3604 $mean_row = $row_stat->mean();
3605 $sig_row = $row_stat->variance();
3606 $col_stat = Statistics::Descriptive::Full->new();
3607 $col_stat->add_data(@seen_cols_array);
3608 $mean_col = $col_stat->mean();
3609 $sig_col = $col_stat->variance();
3610
3611 print STDERR "PREPARE RELATIONSHIP MATRIX\n";
3612 eval {
3613 my %seen_accession_stock_ids;
3614 foreach my $trial_id (@$field_trial_id_list) {
3615 my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id });
3616 my $accessions = $trial->get_accessions();
3617 foreach (@$accessions) {
3618 $seen_accession_stock_ids{$_->{stock_id}}++;
3619 }
3620 }
3621 my @accession_ids = keys %seen_accession_stock_ids;
3622
3623 if ($compute_relationship_matrix_from_htp_phenotypes eq 'genotypes') {
3624
3625 if ($include_pedgiree_info_if_compute_from_parents) {
3626 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
3627 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
3628 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
3629 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
3630 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
3631 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
3632 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
3633 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
3634
3635 if (!$protocol_id) {
3636 $protocol_id = undef;
3637 }
3638
3639 my $pedigree_arm = CXGN::Pedigree::ARM->new({
3640 bcs_schema=>$schema,
3641 arm_temp_file=>$arm_tempfile,
3642 people_schema=>$people_schema,
3643 accession_id_list=>\@accession_ids,
3644 # plot_id_list=>\@plot_id_list,
3645 cache_root=>$c->config->{cache_file_path},
3646 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
3647 });
3648 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
3649 $shared_cluster_dir_config,
3650 $c->config->{backend},
3651 $c->config->{cluster_host},
3652 $c->config->{'web_cluster_queue'},
3653 $c->config->{basepath}
3654 );
3655 # print STDERR Dumper $parent_hash;
3656
3657 my $female_geno = CXGN::Genotype::GRM->new({
3658 bcs_schema=>$schema,
3659 grm_temp_file=>$grm1_tempfile,
3660 people_schema=>$people_schema,
3661 cache_root=>$c->config->{cache_file_path},
3662 accession_id_list=>$female_stock_ids,
3663 protocol_id=>$protocol_id,
3664 get_grm_for_parental_accessions=>0,
3665 download_format=>'three_column_reciprocal'
3666 # minor_allele_frequency=>$minor_allele_frequency,
3667 # marker_filter=>$marker_filter,
3668 # individuals_filter=>$individuals_filter
3669 });
3670 my $female_grm_data = $female_geno->download_grm(
3671 'data',
3672 $shared_cluster_dir_config,
3673 $c->config->{backend},
3674 $c->config->{cluster_host},
3675 $c->config->{'web_cluster_queue'},
3676 $c->config->{basepath}
3677 );
3678 my @fl = split '\n', $female_grm_data;
3679 my %female_parent_grm;
3680 foreach (@fl) {
3681 my @l = split '\t', $_;
3682 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
3683 }
3684 # print STDERR Dumper \%female_parent_grm;
3685
3686 my $male_geno = CXGN::Genotype::GRM->new({
3687 bcs_schema=>$schema,
3688 grm_temp_file=>$grm1_tempfile,
3689 people_schema=>$people_schema,
3690 cache_root=>$c->config->{cache_file_path},
3691 accession_id_list=>$male_stock_ids,
3692 protocol_id=>$protocol_id,
3693 get_grm_for_parental_accessions=>0,
3694 download_format=>'three_column_reciprocal'
3695 # minor_allele_frequency=>$minor_allele_frequency,
3696 # marker_filter=>$marker_filter,
3697 # individuals_filter=>$individuals_filter
3698 });
3699 my $male_grm_data = $male_geno->download_grm(
3700 'data',
3701 $shared_cluster_dir_config,
3702 $c->config->{backend},
3703 $c->config->{cluster_host},
3704 $c->config->{'web_cluster_queue'},
3705 $c->config->{basepath}
3706 );
3707 my @ml = split '\n', $male_grm_data;
3708 my %male_parent_grm;
3709 foreach (@ml) {
3710 my @l = split '\t', $_;
3711 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
3712 }
3713 # print STDERR Dumper \%male_parent_grm;
3714
3715 my %rel_result_hash;
3716 foreach my $a1 (@accession_ids) {
3717 foreach my $a2 (@accession_ids) {
3718 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
3719 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
3720 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
3721 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
3722
3723 my $female_rel = 0;
3724 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
3725 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
3726 }
3727 elsif ($female_parent1 && $female_parent2 && $female_parent1 == $female_parent2) {
3728 $female_rel = 1;
3729 }
3730 elsif ($a1 == $a2) {
3731 $female_rel = 1;
3732 }
3733
3734 my $male_rel = 0;
3735 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
3736 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
3737 }
3738 elsif ($male_parent1 && $male_parent2 && $male_parent1 == $male_parent2) {
3739 $male_rel = 1;
3740 }
3741 elsif ($a1 == $a2) {
3742 $male_rel = 1;
3743 }
3744 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
3745
3746 my $rel = 0.5*($female_rel + $male_rel);
3747 $rel_result_hash{$a1}->{$a2} = $rel;
3748 }
3749 }
3750 # print STDERR Dumper \%rel_result_hash;
3751
3752 my $data = '';
3753 my %result_hash;
3754 foreach my $s (sort @accession_ids) {
3755 foreach my $c (sort @accession_ids) {
3756 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
3757 my $val = $rel_result_hash{$s}->{$c};
3758 if (defined $val and length $val) {
3759 $result_hash{$s}->{$c} = $val;
3760 $data .= "S$s\tS$c\t$val\n";
3761 }
3762 }
3763 }
3764 }
3765
3766 # print STDERR Dumper $data;
3767 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
3768 print $F2 $data;
3769 close($F2);
3770
3771 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
3772 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
3773 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
3774 A_1 <- A_wide[,-1];
3775 A_1[is.na(A_1)] <- 0;
3776 A <- A_1 + t(A_1);
3777 diag(A) <- diag(as.matrix(A_1));
3778 E = eigen(A);
3779 ev = E\$values;
3780 U = E\$vectors;
3781 no = dim(A)[1];
3782 nev = which(ev < 0);
3783 wr = 0;
3784 k=length(nev);
3785 if(k > 0){
3786 p = ev[no - k];
3787 B = sum(ev[nev])*2.0;
3788 wr = (B*B*100.0)+1;
3789 val = ev[nev];
3790 ev[nev] = p*(B-val)*(B-val)/wr;
3791 A = U%*%diag(ev)%*%t(U);
3792 }
3793 A <- as.data.frame(A);
3794 colnames(A) <- A_wide[,1];
3795 A\$stock_id <- A_wide[,1];
3796 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
3797 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
3798 A_threecol\$variable <- substring(A_threecol\$variable, 2);
3799 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
3800 print STDERR $cmd."\n";
3801 my $status = system($cmd);
3802
3803 my %rel_pos_def_result_hash;
3804 open(my $F3, '<', $grm_out_tempfile)
3805 or die "Could not open file '$grm_out_tempfile' $!";
3806
3807 print STDERR "Opened $grm_out_tempfile\n";
3808
3809 while (my $row = <$F3>) {
3810 my @columns;
3811 if ($csv->parse($row)) {
3812 @columns = $csv->fields();
3813 }
3814 my $stock_id1 = $columns[0];
3815 my $stock_id2 = $columns[1];
3816 my $val = $columns[2];
3817 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
3818 }
3819 close($F3);
3820
3821 my $data_pos_def = '';
3822 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
3823 my %result_hash;
3824 foreach my $s (sort @accession_ids) {
3825 foreach my $c (sort @accession_ids) {
3826 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
3827 my $val = $rel_pos_def_result_hash{$s}->{$c};
3828 if (defined $val and length $val) {
3829 $result_hash{$s}->{$c} = $val;
3830 $data_pos_def .= "$s\t$c\t$val\n";
3831 }
3832 }
3833 }
3834 }
3835 }
3836 else {
3837 my %result_hash;
3838 foreach my $s (sort @accession_ids) {
3839 foreach my $c (sort @accession_ids) {
3840 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
3841 my $val = $rel_pos_def_result_hash{$s}->{$c};
3842 if (defined $val and length $val) {
3843 $result_hash{$s}->{$c} = $val;
3844 $result_hash{$c}->{$s} = $val;
3845 $data_pos_def .= "S$s\tS$c\t$val\n";
3846 if ($s != $c) {
3847 $data_pos_def .= "S$c\tS$s\t$val\n";
3848 }
3849 }
3850 }
3851 }
3852 }
3853 }
3854
3855 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
3856 print $F4 $data_pos_def;
3857 close($F4);
3858
3859 $grm_file = $grm_out_posdef_tempfile;
3860 }
3861 elsif ($use_parental_grms_if_compute_from_parents) {
3862 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
3863 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
3864 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
3865 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
3866 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
3867 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
3868 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
3869 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
3870
3871 if (!$protocol_id) {
3872 $protocol_id = undef;
3873 }
3874
3875 my $pedigree_arm = CXGN::Pedigree::ARM->new({
3876 bcs_schema=>$schema,
3877 arm_temp_file=>$arm_tempfile,
3878 people_schema=>$people_schema,
3879 accession_id_list=>\@accession_ids,
3880 # plot_id_list=>\@plot_id_list,
3881 cache_root=>$c->config->{cache_file_path},
3882 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
3883 });
3884 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
3885 $shared_cluster_dir_config,
3886 $c->config->{backend},
3887 $c->config->{cluster_host},
3888 $c->config->{'web_cluster_queue'},
3889 $c->config->{basepath}
3890 );
3891 # print STDERR Dumper $parent_hash;
3892
3893 my $female_geno = CXGN::Genotype::GRM->new({
3894 bcs_schema=>$schema,
3895 grm_temp_file=>$grm1_tempfile,
3896 people_schema=>$people_schema,
3897 cache_root=>$c->config->{cache_file_path},
3898 accession_id_list=>$female_stock_ids,
3899 protocol_id=>$protocol_id,
3900 get_grm_for_parental_accessions=>0,
3901 download_format=>'three_column_reciprocal'
3902 # minor_allele_frequency=>$minor_allele_frequency,
3903 # marker_filter=>$marker_filter,
3904 # individuals_filter=>$individuals_filter
3905 });
3906 my $female_grm_data = $female_geno->download_grm(
3907 'data',
3908 $shared_cluster_dir_config,
3909 $c->config->{backend},
3910 $c->config->{cluster_host},
3911 $c->config->{'web_cluster_queue'},
3912 $c->config->{basepath}
3913 );
3914 my @fl = split '\n', $female_grm_data;
3915 my %female_parent_grm;
3916 foreach (@fl) {
3917 my @l = split '\t', $_;
3918 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
3919 }
3920 # print STDERR Dumper \%female_parent_grm;
3921
3922 my $male_geno = CXGN::Genotype::GRM->new({
3923 bcs_schema=>$schema,
3924 grm_temp_file=>$grm1_tempfile,
3925 people_schema=>$people_schema,
3926 cache_root=>$c->config->{cache_file_path},
3927 accession_id_list=>$male_stock_ids,
3928 protocol_id=>$protocol_id,
3929 get_grm_for_parental_accessions=>0,
3930 download_format=>'three_column_reciprocal'
3931 # minor_allele_frequency=>$minor_allele_frequency,
3932 # marker_filter=>$marker_filter,
3933 # individuals_filter=>$individuals_filter
3934 });
3935 my $male_grm_data = $male_geno->download_grm(
3936 'data',
3937 $shared_cluster_dir_config,
3938 $c->config->{backend},
3939 $c->config->{cluster_host},
3940 $c->config->{'web_cluster_queue'},
3941 $c->config->{basepath}
3942 );
3943 my @ml = split '\n', $male_grm_data;
3944 my %male_parent_grm;
3945 foreach (@ml) {
3946 my @l = split '\t', $_;
3947 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
3948 }
3949 # print STDERR Dumper \%male_parent_grm;
3950
3951 my %rel_result_hash;
3952 foreach my $a1 (@accession_ids) {
3953 foreach my $a2 (@accession_ids) {
3954 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
3955 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
3956 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
3957 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
3958
3959 my $female_rel = 0;
3960 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
3961 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
3962 }
3963 elsif ($a1 == $a2) {
3964 $female_rel = 1;
3965 }
3966
3967 my $male_rel = 0;
3968 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
3969 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
3970 }
3971 elsif ($a1 == $a2) {
3972 $male_rel = 1;
3973 }
3974 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
3975
3976 my $rel = 0.5*($female_rel + $male_rel);
3977 $rel_result_hash{$a1}->{$a2} = $rel;
3978 }
3979 }
3980 # print STDERR Dumper \%rel_result_hash;
3981
3982 my $data = '';
3983 my %result_hash;
3984 foreach my $s (sort @accession_ids) {
3985 foreach my $c (sort @accession_ids) {
3986 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
3987 my $val = $rel_result_hash{$s}->{$c};
3988 if (defined $val and length $val) {
3989 $result_hash{$s}->{$c} = $val;
3990 $data .= "S$s\tS$c\t$val\n";
3991 }
3992 }
3993 }
3994 }
3995
3996 # print STDERR Dumper $data;
3997 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
3998 print $F2 $data;
3999 close($F2);
4000
4001 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
4002 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
4003 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
4004 A_1 <- A_wide[,-1];
4005 A_1[is.na(A_1)] <- 0;
4006 A <- A_1 + t(A_1);
4007 diag(A) <- diag(as.matrix(A_1));
4008 E = eigen(A);
4009 ev = E\$values;
4010 U = E\$vectors;
4011 no = dim(A)[1];
4012 nev = which(ev < 0);
4013 wr = 0;
4014 k=length(nev);
4015 if(k > 0){
4016 p = ev[no - k];
4017 B = sum(ev[nev])*2.0;
4018 wr = (B*B*100.0)+1;
4019 val = ev[nev];
4020 ev[nev] = p*(B-val)*(B-val)/wr;
4021 A = U%*%diag(ev)%*%t(U);
4022 }
4023 A <- as.data.frame(A);
4024 colnames(A) <- A_wide[,1];
4025 A\$stock_id <- A_wide[,1];
4026 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
4027 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
4028 A_threecol\$variable <- substring(A_threecol\$variable, 2);
4029 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
4030 print STDERR $cmd."\n";
4031 my $status = system($cmd);
4032
4033 my %rel_pos_def_result_hash;
4034 open(my $F3, '<', $grm_out_tempfile)
4035 or die "Could not open file '$grm_out_tempfile' $!";
4036
4037 print STDERR "Opened $grm_out_tempfile\n";
4038
4039 while (my $row = <$F3>) {
4040 my @columns;
4041 if ($csv->parse($row)) {
4042 @columns = $csv->fields();
4043 }
4044 my $stock_id1 = $columns[0];
4045 my $stock_id2 = $columns[1];
4046 my $val = $columns[2];
4047 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
4048 }
4049 close($F3);
4050
4051 my $data_pos_def = '';
4052 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
4053 my %result_hash;
4054 foreach my $s (sort @accession_ids) {
4055 foreach my $c (sort @accession_ids) {
4056 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
4057 my $val = $rel_pos_def_result_hash{$s}->{$c};
4058 if (defined $val and length $val) {
4059 $result_hash{$s}->{$c} = $val;
4060 $data_pos_def .= "$s\t$c\t$val\n";
4061 }
4062 }
4063 }
4064 }
4065 }
4066 else {
4067 my %result_hash;
4068 foreach my $s (sort @accession_ids) {
4069 foreach my $c (sort @accession_ids) {
4070 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
4071 my $val = $rel_pos_def_result_hash{$s}->{$c};
4072 if (defined $val and length $val) {
4073 $result_hash{$s}->{$c} = $val;
4074 $result_hash{$c}->{$s} = $val;
4075 $data_pos_def .= "S$s\tS$c\t$val\n";
4076 if ($s != $c) {
4077 $data_pos_def .= "S$c\tS$s\t$val\n";
4078 }
4079 }
4080 }
4081 }
4082 }
4083 }
4084
4085 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
4086 print $F4 $data_pos_def;
4087 close($F4);
4088
4089 $grm_file = $grm_out_posdef_tempfile;
4090 }
4091 else {
4092 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
4093 my $tmp_grm_dir = $shared_cluster_dir_config."/tmp_genotype_download_grm";
4094 mkdir $tmp_grm_dir if ! -d $tmp_grm_dir;
4095 my ($grm_tempfile_fh, $grm_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
4096 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
4097
4098 if (!$protocol_id) {
4099 $protocol_id = undef;
4100 }
4101
4102 my $grm_search_params = {
4103 bcs_schema=>$schema,
4104 grm_temp_file=>$grm_tempfile,
4105 people_schema=>$people_schema,
4106 cache_root=>$c->config->{cache_file_path},
4107 accession_id_list=>\@accession_ids,
4108 protocol_id=>$protocol_id,
4109 get_grm_for_parental_accessions=>$compute_from_parents,
4110 # minor_allele_frequency=>$minor_allele_frequency,
4111 # marker_filter=>$marker_filter,
4112 # individuals_filter=>$individuals_filter
4113 };
4114
4115 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
4116 $grm_search_params->{download_format} = 'three_column_stock_id_integer';
4117 }
4118 else {
4119 $grm_search_params->{download_format} = 'three_column_reciprocal';
4120 }
4121
4122 my $geno = CXGN::Genotype::GRM->new($grm_search_params);
4123 my $grm_data = $geno->download_grm(
4124 'data',
4125 $shared_cluster_dir_config,
4126 $c->config->{backend},
4127 $c->config->{cluster_host},
4128 $c->config->{'web_cluster_queue'},
4129 $c->config->{basepath}
4130 );
4131
4132 open(my $F2, ">", $grm_out_tempfile) || die "Can't open file ".$grm_out_tempfile;
4133 print $F2 $grm_data;
4134 close($F2);
4135 $grm_file = $grm_out_tempfile;
4136 }
4137
4138 }
4139 elsif ($compute_relationship_matrix_from_htp_phenotypes eq 'htp_phenotypes') {
4140
4141 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
4142 'MaterializedViewTable',
4143 {
4144 bcs_schema=>$schema,
4145 data_level=>'plot',
4146 trial_list=>$field_trial_id_list,
4147 include_timestamp=>0,
4148 exclude_phenotype_outlier=>0
4149 }
4150 );
4151 my ($data, $unique_traits) = $phenotypes_search->search();
4152
4153 if (scalar(@$data) == 0) {
4154 $c->stash->{rest} = { error => "There are no phenotypes for the trial you have selected!"};
4155 return;
4156 }
4157
4158 my $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
4159 my $h_time = $schema->storage->dbh()->prepare($q_time);
4160
4161 my %seen_plot_names_htp_rel;
4162 my %phenotype_data_htp_rel;
4163 my %seen_times_htp_rel;
4164 foreach my $obs_unit (@$data){
4165 my $germplasm_name = $obs_unit->{germplasm_uniquename};
4166 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
4167 my $row_number = $obs_unit->{obsunit_row_number} || '';
4168 my $col_number = $obs_unit->{obsunit_col_number} || '';
4169 my $rep = $obs_unit->{obsunit_rep};
4170 my $block = $obs_unit->{obsunit_block};
4171 $seen_plot_names_htp_rel{$obs_unit->{observationunit_uniquename}} = $obs_unit;
4172 my $observations = $obs_unit->{observations};
4173 foreach (@$observations){
4174 if ($_->{associated_image_project_time_json}) {
4175 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
4176
4177 my $time_days_cvterm = $related_time_terms_json->{day};
4178 my $time_days_term_string = $time_days_cvterm;
4179 my $time_days = (split '\|', $time_days_cvterm)[0];
4180 my $time_days_value = (split ' ', $time_days)[1];
4181
4182 my $time_gdd_value = $related_time_terms_json->{gdd_average_temp} + 0;
4183 my $gdd_term_string = "GDD $time_gdd_value";
4184 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
4185 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
4186 if (!$gdd_cvterm_id) {
4187 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
4188 name => $gdd_term_string,
4189 cv => 'cxgn_time_ontology'
4190 });
4191 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
4192 }
4193 my $time_gdd_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
4194
4195 $phenotype_data_htp_rel{$obs_unit->{observationunit_uniquename}}->{$_->{trait_name}} = $_->{value};
4196 $seen_times_htp_rel{$_->{trait_name}} = [$time_days_value, $time_days_term_string, $time_gdd_value, $time_gdd_term_string];
4197 }
4198 }
4199 }
4200
4201 my @allowed_standard_htp_values = ('Nonzero Pixel Count', 'Total Pixel Sum', 'Mean Pixel Value', 'Harmonic Mean Pixel Value', 'Median Pixel Value', 'Pixel Variance', 'Pixel Standard Deviation', 'Pixel Population Standard Deviation', 'Minimum Pixel Value', 'Maximum Pixel Value', 'Minority Pixel Value', 'Minority Pixel Count', 'Majority Pixel Value', 'Majority Pixel Count', 'Pixel Group Count');
4202 my %filtered_seen_times_htp_rel;
4203 while (my ($t, $time) = each %seen_times_htp_rel) {
4204 my $allowed = 0;
4205 foreach (@allowed_standard_htp_values) {
4206 if (index($t, $_) != -1) {
4207 $allowed = 1;
4208 last;
4209 }
4210 }
4211 if ($allowed) {
4212 $filtered_seen_times_htp_rel{$t} = $time;
4213 }
4214 }
4215
4216 my @seen_plot_names_htp_rel_sorted = sort keys %seen_plot_names_htp_rel;
4217 my @filtered_seen_times_htp_rel_sorted = sort keys %filtered_seen_times_htp_rel;
4218
4219 my @header_htp = ('plot_id', 'plot_name', 'accession_id', 'accession_name', 'rep', 'block');
4220
4221 my %trait_name_encoder_htp;
4222 my %trait_name_encoder_rev_htp;
4223 my $trait_name_encoded_htp = 1;
4224 my @header_traits_htp;
4225 foreach my $trait_name (@filtered_seen_times_htp_rel_sorted) {
4226 if (!exists($trait_name_encoder_htp{$trait_name})) {
4227 my $trait_name_e = 't'.$trait_name_encoded_htp;
4228 $trait_name_encoder_htp{$trait_name} = $trait_name_e;
4229 $trait_name_encoder_rev_htp{$trait_name_e} = $trait_name;
4230 push @header_traits_htp, $trait_name_e;
4231 $trait_name_encoded_htp++;
4232 }
4233 }
4234
4235 my @htp_pheno_matrix;
4236 if ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'all') {
4237 push @header_htp, @header_traits_htp;
4238 push @htp_pheno_matrix, \@header_htp;
4239
4240 foreach my $p (@seen_plot_names_htp_rel_sorted) {
4241 my $obj = $seen_plot_names_htp_rel{$p};
4242 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
4243 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
4244 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
4245 push @row, $val;
4246 }
4247 push @htp_pheno_matrix, \@row;
4248 }
4249 }
4250 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'latest_trait') {
4251 my $max_day = 0;
4252 foreach (keys %seen_days_after_plantings) {
4253 if ($_ + 0 > $max_day) {
4254 $max_day = $_;
4255 }
4256 }
4257
4258 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
4259 my $day = $filtered_seen_times_htp_rel{$t}->[0];
4260 if ($day <= $max_day) {
4261 push @header_htp, $t;
4262 }
4263 }
4264 push @htp_pheno_matrix, \@header_htp;
4265
4266 foreach my $p (@seen_plot_names_htp_rel_sorted) {
4267 my $obj = $seen_plot_names_htp_rel{$p};
4268 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
4269 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
4270 my $day = $filtered_seen_times_htp_rel{$t}->[0];
4271 if ($day <= $max_day) {
4272 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
4273 push @row, $val;
4274 }
4275 }
4276 push @htp_pheno_matrix, \@row;
4277 }
4278 }
4279 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'vegetative') {
4280
4281 }
4282 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'reproductive') {
4283
4284 }
4285 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'mature') {
4286
4287 }
4288 else {
4289 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_time_points htp_pheno_rel_matrix_time_points is not valid!" };
4290 return;
4291 }
4292
4293 open(my $htp_pheno_f, ">", $stats_out_htp_rel_tempfile_input) || die "Can't open file ".$stats_out_htp_rel_tempfile_input;
4294 foreach (@htp_pheno_matrix) {
4295 my $line = join "\t", @$_;
4296 print $htp_pheno_f $line."\n";
4297 }
4298 close($htp_pheno_f);
4299
4300 my %rel_htp_result_hash;
4301 if ($compute_relationship_matrix_from_htp_phenotypes_type eq 'correlations') {
4302 my $htp_cmd = 'R -e "library(lme4); library(data.table);
4303 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
4304 mat_agg <- aggregate(mat[, 7:ncol(mat)], list(mat\$accession_id), mean);
4305 mat_pheno <- mat_agg[,2:ncol(mat_agg)];
4306 cor_mat <- cor(t(mat_pheno));
4307 rownames(cor_mat) <- mat_agg[,1];
4308 colnames(cor_mat) <- mat_agg[,1];
4309 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
4310 cor_mat <- range01(cor_mat);
4311 write.table(cor_mat, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
4312 print STDERR Dumper $htp_cmd;
4313 my $status = system($htp_cmd);
4314 }
4315 elsif ($compute_relationship_matrix_from_htp_phenotypes_type eq 'blues') {
4316 my $htp_cmd = 'R -e "library(lme4); library(data.table);
4317 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
4318 blues <- data.frame(id = seq(1,length(unique(mat\$accession_id))));
4319 varlist <- names(mat)[7:ncol(mat)];
4320 blues.models <- lapply(varlist, function(x) {
4321 tryCatch(
4322 lmer(substitute(i ~ 1 + (1|accession_id), list(i = as.name(x))), data = mat, REML = FALSE, control = lmerControl(optimizer =\'Nelder_Mead\', boundary.tol='.$compute_relationship_matrix_from_htp_phenotypes_blues_inversion.' ) ), error=function(e) {}
4323 )
4324 });
4325 counter = 1;
4326 for (m in blues.models) {
4327 if (!is.null(m)) {
4328 blues\$accession_id <- row.names(ranef(m)\$accession_id);
4329 blues[,ncol(blues) + 1] <- ranef(m)\$accession_id\$\`(Intercept)\`;
4330 colnames(blues)[ncol(blues)] <- varlist[counter];
4331 }
4332 counter = counter + 1;
4333 }
4334 blues_vals <- as.matrix(blues[,3:ncol(blues)]);
4335 blues_vals <- apply(blues_vals, 2, function(y) (y - mean(y)) / sd(y) ^ as.logical(sd(y)));
4336 rel <- (1/ncol(blues_vals)) * (blues_vals %*% t(blues_vals));
4337 rownames(rel) <- blues[,2];
4338 colnames(rel) <- blues[,2];
4339 write.table(rel, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
4340 print STDERR Dumper $htp_cmd;
4341 my $status = system($htp_cmd);
4342 }
4343 else {
4344 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_type htp_pheno_rel_matrix_type is not valid!" };
4345 return;
4346 }
4347
4348 open(my $htp_rel_res, '<', $stats_out_htp_rel_tempfile)
4349 or die "Could not open file '$stats_out_htp_rel_tempfile' $!";
4350
4351 print STDERR "Opened $stats_out_htp_rel_tempfile\n";
4352 my $header_row = <$htp_rel_res>;
4353 my @header;
4354 if ($csv->parse($header_row)) {
4355 @header = $csv->fields();
4356 }
4357
4358 while (my $row = <$htp_rel_res>) {
4359 my @columns;
4360 if ($csv->parse($row)) {
4361 @columns = $csv->fields();
4362 }
4363 my $stock_id1 = $columns[0];
4364 my $counter = 1;
4365 foreach my $stock_id2 (@header) {
4366 my $val = $columns[$counter];
4367 $rel_htp_result_hash{$stock_id1}->{$stock_id2} = $val;
4368 $counter++;
4369 }
4370 }
4371 close($htp_rel_res);
4372
4373 my $data_rel_htp = '';
4374 my %result_hash;
4375 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
4376 foreach my $s (sort @accession_ids) {
4377 foreach my $c (sort @accession_ids) {
4378 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
4379 my $val = $rel_htp_result_hash{$s}->{$c};
4380 if (defined $val and length $val) {
4381 $result_hash{$s}->{$c} = $val;
4382 $data_rel_htp .= "$s\t$c\t$val\n";
4383 }
4384 }
4385 }
4386 }
4387 }
4388 else {
4389 foreach my $s (sort @accession_ids) {
4390 foreach my $c (sort @accession_ids) {
4391 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
4392 my $val = $rel_htp_result_hash{$s}->{$c};
4393 if (defined $val and length $val) {
4394 $result_hash{$s}->{$c} = $val;
4395 $result_hash{$c}->{$s} = $val;
4396 $data_rel_htp .= "S$s\tS$c\t$val\n";
4397 if ($s != $c) {
4398 $data_rel_htp .= "S$c\tS$s\t$val\n";
4399 }
4400 }
4401 }
4402 }
4403 }
4404 }
4405
4406 open(my $htp_rel_out, ">", $stats_out_htp_rel_tempfile_out) || die "Can't open file ".$stats_out_htp_rel_tempfile_out;
4407 print $htp_rel_out $data_rel_htp;
4408 close($htp_rel_out);
4409
4410 $grm_file = $stats_out_htp_rel_tempfile_out;
4411 }
4412 else {
4413 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes is not valid!" };
4414 return;
4415 }
4416 };
4417
4418 my $result_2 = CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation::perform_drone_imagery_analytics($schema, $a_env, $b_env, $ro_env, $row_ro_env, $env_variance_percent, $protocol_id, $statistics_select, $analytics_select, $tolparinv, $use_area_under_curve, $legendre_order_number, $permanent_environment_structure, \@legendre_coeff_exec, \%trait_name_encoder_2, \%trait_name_encoder_rev_2, \%stock_info_2, \%plot_id_map, \@sorted_trait_names_2, \%accession_id_factor_map, \@rep_time_factors, \@ind_rep_factors, \@unique_accession_names, \%plot_id_count_map_reverse, \@sorted_scaled_ln_times, \%time_count_map_reverse, \%accession_id_factor_map_reverse, \%seen_times, \%plot_id_factor_map_reverse, \%trait_to_time_map_2, \@unique_plot_names, \%stock_name_row_col, \%phenotype_data_original_2, \%plot_rep_time_factor_map, \%stock_row_col, \%stock_row_col_id, \%polynomial_map, \@plot_ids_ordered, $csv, $timestamp, $user_name, $stats_tempfile, $grm_file, $grm_rename_tempfile, $tmp_stats_dir, $stats_out_tempfile, $stats_out_tempfile_row, $stats_out_tempfile_col, $stats_out_tempfile_residual, $stats_out_tempfile_2dspl, $stats_prep2_tempfile, $stats_out_param_tempfile, $parameter_tempfile, $parameter_asreml_tempfile, $stats_tempfile_2, $permanent_environment_structure_tempfile, $permanent_environment_structure_env_tempfile, $permanent_environment_structure_env_tempfile2, $permanent_environment_structure_env_tempfile_mat, $sim_env_changing_mat_tempfile, $sim_env_changing_mat_full_tempfile, $yhat_residual_tempfile, $blupf90_solutions_tempfile, $coeff_genetic_tempfile, $coeff_pe_tempfile, $stats_out_tempfile_varcomp, $time_min, $time_max, $header_string_2, $env_sim_exec, $min_row, $max_row, $min_col, $max_col, $mean_row, $sig_row, $mean_col, $sig_col, $sim_env_change_over_time, $correlation_between_times, $field_trial_id_list, $simulated_environment_real_data_trait_id, $fixed_effect_type, $perform_cv);
4419 if (ref($result_2) eq 'HASH') {
4420 $c->stash->{rest} = $result_2;
4421 $c->detach();
4422 }
4423 my ($statistical_ontology_term_2, $analysis_model_training_data_file_type_2, $analysis_model_language_2, $sorted_residual_trait_names_array_2, $rr_unique_traits_hash_2, $rr_residual_unique_traits_hash_2, $statistics_cmd_2, $cmd_f90_2, $number_traits_2, $trait_to_time_map_hash_2,
4424
4425 $result_blup_data_original_2, $result_blup_data_delta_original_2, $result_blup_spatial_data_original_2, $result_blup_pe_data_original_2, $result_blup_pe_data_delta_original_2, $result_residual_data_original_2, $result_fitted_data_original_2, $fixed_effects_original_hash_2,
4426 $rr_genetic_coefficients_original_hash_2, $rr_temporal_coefficients_original_hash_2,
4427 $rr_coeff_genetic_covariance_original_array_2, $rr_coeff_env_covariance_original_array_2, $rr_coeff_genetic_correlation_original_array_2, $rr_coeff_env_correlation_original_array_2, $rr_residual_variance_original_2, $varcomp_original_array_2,
4428 $model_sum_square_residual_original_2, $genetic_effect_min_original_2, $genetic_effect_max_original_2, $env_effect_min_original_2, $env_effect_max_original_2, $genetic_effect_sum_square_original_2, $genetic_effect_sum_original_2, $env_effect_sum_square_original_2, $env_effect_sum_original_2, $residual_sum_square_original_2, $residual_sum_original_2, $result_cv_original_2, $result_cv_2_original_2,
4429
4430 $phenotype_data_altered_hash_2, $data_matrix_altered_array_2, $data_matrix_phenotypes_altered_array_2, $phenotype_min_altered_2, $phenotype_max_altered_2,
4431 $result_blup_data_altered_2, $result_blup_data_delta_altered_2, $result_blup_spatial_data_altered_2, $result_blup_pe_data_altered_2, $result_blup_pe_data_delta_altered_2, $result_residual_data_altered_2, $result_fitted_data_altered_2, $fixed_effects_altered_hash_2,
4432 $rr_genetic_coefficients_altered_hash_2, $rr_temporal_coefficients_altered_hash_2,
4433 $rr_coeff_genetic_covariance_altered_array_2, $rr_coeff_env_covariance_altered_array_2, $rr_coeff_genetic_correlation_altered_array_2, $rr_coeff_env_correlation_altered_array_2, $rr_residual_variance_altered_2, $varcomp_altered_array_2,
4434 $model_sum_square_residual_altered_2, $genetic_effect_min_altered_2, $genetic_effect_max_altered_2, $env_effect_min_altered_2, $env_effect_max_altered_2, $genetic_effect_sum_square_altered_2, $genetic_effect_sum_altered_2, $env_effect_sum_square_altered_2, $env_effect_sum_altered_2, $residual_sum_square_altered_2, $residual_sum_altered_2, $result_cv_altered_2, $result_cv_2_altered_2,
4435
4436 $phenotype_data_altered_env_hash_1_2, $data_matrix_altered_env_array_1_2, $data_matrix_phenotypes_altered_env_array_1_2, $phenotype_min_altered_env_1_2, $phenotype_max_altered_env_1_2, $env_sim_min_1_2, $env_sim_max_1_2, $sim_data_hash_1_2,
4437 $result_blup_data_altered_env_1_2, $result_blup_data_delta_altered_env_1_2, $result_blup_spatial_data_altered_env_1_2, $result_blup_pe_data_altered_env_1_2, $result_blup_pe_data_delta_altered_env_1_2, $result_residual_data_altered_env_1_2, $result_fitted_data_altered_env_1_2, $fixed_effects_altered_env_hash_1_2, $rr_genetic_coefficients_altered_env_hash_1_2, $rr_temporal_coefficients_altered_env_hash_1_2,
4438 $rr_coeff_genetic_covariance_altered_env_array_1_2, $rr_coeff_env_covariance_altered_env_array_1_2, $rr_coeff_genetic_correlation_altered_env_array_1_2, $rr_coeff_env_correlation_altered_env_array_1_2, $rr_residual_variance_altered_env_1_2, $varcomp_altered_array_env_1_2,
4439 $model_sum_square_residual_altered_env_1_2, $genetic_effect_min_altered_env_1_2, $genetic_effect_max_altered_env_1_2, $env_effect_min_altered_env_1_2, $env_effect_max_altered_env_1_2, $genetic_effect_sum_square_altered_env_1_2, $genetic_effect_sum_altered_env_1_2, $env_effect_sum_square_altered_env_1_2, $env_effect_sum_altered_env_1_2, $residual_sum_square_altered_env_1_2, $residual_sum_altered_env_1_2, $result_cv_altered_env_1_2, $result_cv_2_altered_env_1_2,
4440
4441 $phenotype_data_altered_env_hash_2_2, $data_matrix_altered_env_array_2_2, $data_matrix_phenotypes_altered_env_array_2_2, $phenotype_min_altered_env_2_2, $phenotype_max_altered_env_2_2, $env_sim_min_2_2, $env_sim_max_2_2, $sim_data_hash_2_2,
4442 $result_blup_data_altered_env_2_2, $result_blup_data_delta_altered_env_2_2, $result_blup_spatial_data_altered_env_2_2, $result_blup_pe_data_altered_env_2_2, $result_blup_pe_data_delta_altered_env_2_2, $result_residual_data_altered_env_2_2, $result_fitted_data_altered_env_2_2, $fixed_effects_altered_env_hash_2_2, $rr_genetic_coefficients_altered_env_hash_2_2, $rr_temporal_coefficients_altered_env_hash_2_2,
4443 $rr_coeff_genetic_covariance_altered_env_array_2_2, $rr_coeff_env_covariance_altered_env_array_2_2, $rr_coeff_genetic_correlation_altered_env_array_2_2, $rr_coeff_env_correlation_altered_env_array_2_2, $rr_residual_variance_altered_env_2_2, $varcomp_altered_array_env_2_2,
4444 $model_sum_square_residual_altered_env_2_2, $genetic_effect_min_altered_env_2_2, $genetic_effect_max_altered_env_2_2, $env_effect_min_altered_env_2_2, $env_effect_max_altered_env_2_2, $genetic_effect_sum_square_altered_env_2_2, $genetic_effect_sum_altered_env_2_2, $env_effect_sum_square_altered_env_2_2, $env_effect_sum_altered_env_2_2, $residual_sum_square_altered_env_2_2, $residual_sum_altered_env_2_2, $result_cv_altered_env_2_2, $result_cv_2_altered_env_2_2,
4445
4446 $phenotype_data_altered_env_hash_3_2, $data_matrix_altered_env_array_3_2, $data_matrix_phenotypes_altered_env_array_3_2, $phenotype_min_altered_env_3_2, $phenotype_max_altered_env_3_2, $env_sim_min_3_2, $env_sim_max_3_2, $sim_data_hash_3_2,
4447 $result_blup_data_altered_env_3_2, $result_blup_data_delta_altered_env_3_2, $result_blup_spatial_data_altered_env_3_2, $result_blup_pe_data_altered_env_3_2, $result_blup_pe_data_delta_altered_env_3_2, $result_residual_data_altered_env_3_2, $result_fitted_data_altered_env_3_2, $fixed_effects_altered_env_hash_3_2, $rr_genetic_coefficients_altered_env_hash_3_2, $rr_temporal_coefficients_altered_env_hash_3_2,
4448 $rr_coeff_genetic_covariance_altered_env_array_3_2, $rr_coeff_env_covariance_altered_env_array_3_2, $rr_coeff_genetic_correlation_altered_env_array_3_2, $rr_coeff_env_correlation_altered_env_array_3_2, $rr_residual_variance_altered_env_3_2, $varcomp_altered_array_env_3_2,
4449 $model_sum_square_residual_altered_env_3_2, $genetic_effect_min_altered_env_3_2, $genetic_effect_max_altered_env_3_2, $env_effect_min_altered_env_3_2, $env_effect_max_altered_env_3_2, $genetic_effect_sum_square_altered_env_3_2, $genetic_effect_sum_altered_env_3_2, $env_effect_sum_square_altered_env_3_2, $env_effect_sum_altered_env_3_2, $residual_sum_square_altered_env_3_2, $residual_sum_altered_env_3_2, $result_cv_altered_env_3_2, $result_cv_2_altered_env_3_2,
4450
4451 $phenotype_data_altered_env_hash_4_2, $data_matrix_altered_env_array_4_2, $data_matrix_phenotypes_altered_env_array_4_2, $phenotype_min_altered_env_4_2, $phenotype_max_altered_env_4_2, $env_sim_min_4_2, $env_sim_max_4_2, $sim_data_hash_4_2,
4452 $result_blup_data_altered_env_4_2, $result_blup_data_delta_altered_env_4_2, $result_blup_spatial_data_altered_env_4_2, $result_blup_pe_data_altered_env_4_2, $result_blup_pe_data_delta_altered_env_4_2, $result_residual_data_altered_env_4_2, $result_fitted_data_altered_env_4_2, $fixed_effects_altered_env_hash_4_2, $rr_genetic_coefficients_altered_env_hash_4_2, $rr_temporal_coefficients_altered_env_hash_4_2,
4453 $rr_coeff_genetic_covariance_altered_env_array_4_2, $rr_coeff_env_covariance_altered_env_array_4_2, $rr_coeff_genetic_correlation_altered_env_array_4_2, $rr_coeff_env_correlation_altered_env_array_4_2, $rr_residual_variance_altered_env_4_2, $varcomp_altered_array_env_4_2,
4454 $model_sum_square_residual_altered_env_4_2, $genetic_effect_min_altered_env_4_2, $genetic_effect_max_altered_env_4_2, $env_effect_min_altered_env_4_2, $env_effect_max_altered_env_4_2, $genetic_effect_sum_square_altered_env_4_2, $genetic_effect_sum_altered_env_4_2, $env_effect_sum_square_altered_env_4_2, $env_effect_sum_altered_env_4_2, $residual_sum_square_altered_env_4_2, $residual_sum_altered_env_4_2, $result_cv_altered_env_4_2, $result_cv_2_altered_env_4_2,
4455
4456 $phenotype_data_altered_env_hash_5_2, $data_matrix_altered_env_array_5_2, $data_matrix_phenotypes_altered_env_array_5_2, $phenotype_min_altered_env_5_2, $phenotype_max_altered_env_5_2, $env_sim_min_5_2, $env_sim_max_5_2, $sim_data_hash_5_2,
4457 $result_blup_data_altered_env_5_2, $result_blup_data_delta_altered_env_5_2, $result_blup_spatial_data_altered_env_5_2, $result_blup_pe_data_altered_env_5_2, $result_blup_pe_data_delta_altered_env_5_2, $result_residual_data_altered_env_5_2, $result_fitted_data_altered_env_5_2, $fixed_effects_altered_env_hash_5_2, $rr_genetic_coefficients_altered_env_hash_5_2, $rr_temporal_coefficients_altered_env_hash_5_2,
4458 $rr_coeff_genetic_covariance_altered_env_array_5_2, $rr_coeff_env_covariance_altered_env_array_5_2, $rr_coeff_genetic_correlation_altered_env_array_5_2, $rr_coeff_env_correlation_altered_env_array_5_2, $rr_residual_variance_altered_env_5_2, $varcomp_altered_array_env_5_2,
4459 $model_sum_square_residual_altered_env_5_2, $genetic_effect_min_altered_env_5_2, $genetic_effect_max_altered_env_5_2, $env_effect_min_altered_env_5_2, $env_effect_max_altered_env_5_2, $genetic_effect_sum_square_altered_env_5_2, $genetic_effect_sum_altered_env_5_2, $env_effect_sum_square_altered_env_5_2, $env_effect_sum_altered_env_5_2, $residual_sum_square_altered_env_5_2, $residual_sum_altered_env_5_2, $result_cv_altered_env_5_2, $result_cv_2_altered_env_5_2,
4460
4461 $phenotype_data_altered_env_hash_6_2, $data_matrix_altered_env_array_6_2, $data_matrix_phenotypes_altered_env_array_6_2, $phenotype_min_altered_env_6_2, $phenotype_max_altered_env_6_2, $env_sim_min_6_2, $env_sim_max_6_2, $sim_data_hash_6_2,
4462 $result_blup_data_altered_env_6_2, $result_blup_data_delta_altered_env_6_2, $result_blup_spatial_data_altered_env_6_2, $result_blup_pe_data_altered_env_6_2, $result_blup_pe_data_delta_altered_env_6_2, $result_residual_data_altered_env_6_2, $result_fitted_data_altered_env_6_2, $fixed_effects_altered_env_hash_6_2, $rr_genetic_coefficients_altered_env_hash_6_2, $rr_temporal_coefficients_altered_env_hash_6_2,
4463 $rr_coeff_genetic_covariance_altered_env_array_6_2, $rr_coeff_env_covariance_altered_env_array_6_2, $rr_coeff_genetic_correlation_altered_env_array_6_2, $rr_coeff_env_correlation_altered_env_array_6_2, $rr_residual_variance_altered_env_6_2, $varcomp_altered_array_env_6_2,
4464 $model_sum_square_residual_altered_env_6_2, $genetic_effect_min_altered_env_6_2, $genetic_effect_max_altered_env_6_2, $env_effect_min_altered_env_6_2, $env_effect_max_altered_env_6_2, $genetic_effect_sum_square_altered_env_6_2, $genetic_effect_sum_altered_env_6_2, $env_effect_sum_square_altered_env_6_2, $env_effect_sum_altered_env_6_2, $residual_sum_square_altered_env_6_2, $residual_sum_altered_env_6_2, $result_cv_altered_env_6_2, $result_cv_2_altered_env_6_2
4465 ) = @$result_2;
4466
4467 eval {
4468 print STDERR "PLOTTING CORRELATION\n";
4469 my ($full_plot_level_correlation_tempfile_fh, $full_plot_level_correlation_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4470 open(my $F_fullplot, ">", $full_plot_level_correlation_tempfile) || die "Can't open file ".$full_plot_level_correlation_tempfile;
4471
4472 my @header_full_plot_corr = ('plot_name, plot_id, row_number, col_number, rep, block, germplasm_name, germplasm_id');
4473 my @types_full_plot_corr = ('pheno_orig_', 'pheno_postm2_', 'eff_origm2_', 'eff_postm2_',
4474 'sim_env1_', 'simm2_pheno1_', 'effm2_sim1_',
4475 'sim_env2_', 'simm2_pheno2_', 'effm2_sim2_',
4476 'sim_env3_', 'simm2_pheno3_', 'effm2_sim3_',
4477 'sim_env4_', 'simm2_pheno4_', 'effm2_sim4_',
4478 'sim_env5_', 'simm2_pheno5_', 'effm2_sim5_',
4479 'sim_env6_', 'simm2_pheno6_', 'effm2_sim6_');
4480 foreach my $t (@sorted_trait_names_2) {
4481 foreach my $type (@types_full_plot_corr) {
4482 push @header_full_plot_corr, $type.$trait_name_encoder_2{$t};
4483 }
4484 }
4485 my $header_string_full_plot_corr = join ',', @header_full_plot_corr;
4486 print $F_fullplot "$header_string_full_plot_corr\n";
4487 foreach my $p (@unique_plot_names) {
4488 my @row = ($p, $stock_name_row_col{$p}->{obsunit_stock_id}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $stock_name_row_col{$p}->{rep}, $stock_name_row_col{$p}->{block}, $stock_name_row_col{$p}->{germplasm_name}, $stock_name_row_col{$p}->{germplasm_stock_id});
4489 foreach my $t (@sorted_trait_names_2) {
4490 my $t_conv = $trait_name_encoder_rev_2{$trait_name_encoder_2{$t}};
4491
4492 my $phenotype_original = $phenotype_data_original_2{$p}->{$t};
4493 my $phenotype_post_2 = $phenotype_data_altered_hash_2->{$p}->{$t_conv};
4494 my $effect_original_2 = $result_blup_spatial_data_original_2->{$p}->{$t_conv}->[0];
4495 my $effect_post_2 = $result_blup_spatial_data_altered_2->{$p}->{$t_conv}->[0];
4496 push @row, ($phenotype_original, $phenotype_post_2, $effect_original_2, $effect_post_2);
4497
4498 my $sim_env = $sim_data_hash_1_2->{$p}->{$t};
4499 my $pheno_sim_2 = $phenotype_data_altered_env_hash_1_2->{$p}->{$t_conv};
4500 my $effect_sim_2 = $result_blup_spatial_data_altered_env_1_2->{$p}->{$t_conv}->[0];
4501 push @row, ($sim_env, $pheno_sim_2, $effect_sim_2);
4502
4503 my $sim_env2 = $sim_data_hash_2_2->{$p}->{$t};
4504 my $pheno_sim2_2 = $phenotype_data_altered_env_hash_2_2->{$p}->{$t_conv};
4505 my $effect_sim2_2 = $result_blup_spatial_data_altered_env_2_2->{$p}->{$t_conv}->[0];
4506 push @row, ($sim_env2, $pheno_sim2_2, $effect_sim2_2);
4507
4508 my $sim_env3 = $sim_data_hash_3_2->{$p}->{$t};
4509 my $pheno_sim3_2 = $phenotype_data_altered_env_hash_3_2->{$p}->{$t_conv};
4510 my $effect_sim3_2 = $result_blup_spatial_data_altered_env_3_2->{$p}->{$t_conv}->[0];
4511 push @row, ($sim_env3, $pheno_sim3_2, $effect_sim3_2);
4512
4513 my $sim_env4 = $sim_data_hash_4_2->{$p}->{$t};
4514 my $pheno_sim4_2 = $phenotype_data_altered_env_hash_4_2->{$p}->{$t_conv};
4515 my $effect_sim4_2 = $result_blup_spatial_data_altered_env_4_2->{$p}->{$t_conv}->[0];
4516 push @row, ($sim_env4, $pheno_sim4_2, $effect_sim4_2);
4517
4518 my $sim_env5 = $sim_data_hash_5_2->{$p}->{$t};
4519 my $pheno_sim5_2 = $phenotype_data_altered_env_hash_5_2->{$p}->{$t_conv};
4520 my $effect_sim5_2 = $result_blup_spatial_data_altered_env_5_2->{$p}->{$t_conv}->[0];
4521 push @row, ($sim_env5, $pheno_sim5_2, $effect_sim5_2);
4522
4523 my $sim_env6 = $sim_data_hash_6_2->{$p}->{$t};
4524 my $pheno_sim6_2 = $phenotype_data_altered_env_hash_6_2->{$p}->{$t_conv};
4525 my $effect_sim6_2 = $result_blup_spatial_data_altered_env_6_2->{$p}->{$t_conv}->[0];
4526 push @row, ($sim_env6, $pheno_sim6_2, $effect_sim6_2);
4527 }
4528 my $line = join ',', @row;
4529 print $F_fullplot "$line\n";
4530 }
4531 close($F_fullplot);
4532
4533 my $plot_corr_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
4534 $plot_corr_figure_tempfile_string .= '.png';
4535 my $plot_corr_figure_tempfile = $c->config->{basepath}."/".$plot_corr_figure_tempfile_string;
4536
4537 my $cmd_plotcorr_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
4538 mat_orig <- fread(\''.$full_plot_level_correlation_tempfile.'\', header=TRUE, sep=\',\');
4539 gg <- ggcorr(data=mat_orig[,-seq(1,8)], hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE);
4540 ggsave(\''.$plot_corr_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, limitsize = FALSE, units=\'in\');
4541 "';
4542 # print STDERR Dumper $cmd;
4543 my $status_plotcorr_plot = system($cmd_plotcorr_plot);
4544 push @$spatial_effects_plots, [$plot_corr_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
4545 push @$spatial_effects_files_store, [$full_plot_level_correlation_tempfile, "datafile_".$statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
4546 };
4547
4548 eval {
4549 my @plot_corr_full_vals;
4550
4551 my @original_pheno_vals;
4552 my ($phenotypes_original_heatmap_tempfile_fh, $phenotypes_original_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4553 open(my $F_pheno, ">", $phenotypes_original_heatmap_tempfile) || die "Can't open file ".$phenotypes_original_heatmap_tempfile;
4554 print $F_pheno "trait_type,row,col,value\n";
4555 foreach my $p (@unique_plot_names) {
4556 foreach my $t (@sorted_trait_names_2) {
4557 my $val = $phenotype_data_original_2{$p}->{$t};
4558 my @row = ("pheno_orig_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4559 push @original_pheno_vals, $val;
4560 my $line = join ',', @row;
4561 print $F_pheno "$line\n";
4562 }
4563 }
4564 close($F_pheno);
4565 push @plot_corr_full_vals, \@original_pheno_vals;
4566
4567 my $original_pheno_stat = Statistics::Descriptive::Full->new();
4568 $original_pheno_stat->add_data(@original_pheno_vals);
4569 my $sig_original_pheno = $original_pheno_stat->variance();
4570
4571 #PHENO POST M START
4572
4573 my @altered_pheno_vals_2;
4574 my ($phenotypes_post_heatmap_tempfile_fh_2, $phenotypes_post_heatmap_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4575 open($F_pheno, ">", $phenotypes_post_heatmap_tempfile_2) || die "Can't open file ".$phenotypes_post_heatmap_tempfile_2;
4576 print $F_pheno "trait_type,row,col,value\n";
4577 foreach my $p (@unique_plot_names) {
4578 foreach my $t (@sorted_trait_names_2) {
4579 my $val = $phenotype_data_altered_hash_2->{$p}->{$t};
4580 my @row = ("pheno_postm2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4581 push @altered_pheno_vals_2, $val;
4582 my $line = join ',', @row;
4583 print $F_pheno "$line\n";
4584 }
4585 }
4586 close($F_pheno);
4587 push @plot_corr_full_vals, \@altered_pheno_vals_2;
4588
4589 my $altered_pheno_stat_2 = Statistics::Descriptive::Full->new();
4590 $altered_pheno_stat_2->add_data(@altered_pheno_vals_2);
4591 my $sig_altered_pheno_2 = $altered_pheno_stat_2->variance();
4592
4593 # EFFECT ORIGINAL M
4594
4595 my @original_effect_vals_2;
4596 my ($effects_heatmap_tempfile_fh_2, $effects_heatmap_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4597 open(my $F_eff, ">", $effects_heatmap_tempfile_2) || die "Can't open file ".$effects_heatmap_tempfile_2;
4598 print $F_eff "trait_type,row,col,value\n";
4599 foreach my $p (@unique_plot_names) {
4600 foreach my $t (@sorted_trait_names_2) {
4601 my $val = $result_blup_spatial_data_original_2->{$p}->{$t}->[0];
4602 my @row = ("eff_origm2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4603 my $line = join ',', @row;
4604 print $F_eff "$line\n";
4605 push @original_effect_vals_2, $val;
4606 }
4607 }
4608 close($F_eff);
4609 push @plot_corr_full_vals, \@original_effect_vals_2;
4610
4611 my $original_effect_stat_2 = Statistics::Descriptive::Full->new();
4612 $original_effect_stat_2->add_data(@original_effect_vals_2);
4613 my $sig_original_effect_2 = $original_effect_stat_2->variance();
4614
4615 # EFFECT POST M MIN
4616
4617 my @altered_effect_vals_2;
4618 my ($effects_post_heatmap_tempfile_fh_2, $effects_post_heatmap_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4619 open($F_eff, ">", $effects_post_heatmap_tempfile_2) || die "Can't open file ".$effects_post_heatmap_tempfile_2;
4620 print $F_eff "trait_type,row,col,value\n";
4621 foreach my $p (@unique_plot_names) {
4622 foreach my $t (@sorted_trait_names_2) {
4623 my $val = $result_blup_spatial_data_altered_2->{$p}->{$t}->[0];
4624 my @row = ("eff_postm2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4625 my $line = join ',', @row;
4626 print $F_eff "$line\n";
4627 push @altered_effect_vals_2, $val;
4628 }
4629 }
4630 close($F_eff);
4631 push @plot_corr_full_vals, \@altered_effect_vals_2;
4632
4633 my $altered_effect_stat_2 = Statistics::Descriptive::Full->new();
4634 $altered_effect_stat_2->add_data(@altered_effect_vals_2);
4635 my $sig_altered_effect_2 = $altered_effect_stat_2->variance();
4636
4637 # SIM ENV 1: ALTERED PHENO + EFFECT
4638
4639 my ($phenotypes_env_heatmap_tempfile_fh, $phenotypes_env_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4640 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile) || die "Can't open file ".$phenotypes_env_heatmap_tempfile;
4641 print $F_pheno "trait_type,row,col,value\n";
4642 foreach my $p (@unique_plot_names) {
4643 foreach my $t (@sorted_trait_names_2) {
4644 my @row = ("sim_env1_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_1_2->{$p}->{$t});
4645 my $line = join ',', @row;
4646 print $F_pheno "$line\n";
4647 }
4648 }
4649 close($F_pheno);
4650
4651 my @sim_pheno1_vals_2;
4652 my ($phenotypes_pheno_sim_heatmap_tempfile_fh_2, $phenotypes_pheno_sim_heatmap_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4653 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile_2;
4654 print $F_pheno "trait_type,row,col,value\n";
4655 foreach my $p (@unique_plot_names) {
4656 foreach my $t (@sorted_trait_names_2) {
4657 my $val = $phenotype_data_altered_env_hash_1_2->{$p}->{$t};
4658 my @row = ("simm2_pheno1_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4659 my $line = join ',', @row;
4660 print $F_pheno "$line\n";
4661 push @sim_pheno1_vals_2, $val;
4662 }
4663 }
4664 close($F_pheno);
4665 push @plot_corr_full_vals, \@sim_pheno1_vals_2;
4666
4667 my $sim_pheno1_stat_2 = Statistics::Descriptive::Full->new();
4668 $sim_pheno1_stat_2->add_data(@sim_pheno1_vals_2);
4669 my $sig_sim2_pheno1 = $sim_pheno1_stat_2->variance();
4670
4671 my @sim_effect1_vals_2;
4672 my ($effects_sim_heatmap_tempfile_fh_2, $effects_sim_heatmap_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4673 open($F_eff, ">", $effects_sim_heatmap_tempfile_2) || die "Can't open file ".$effects_sim_heatmap_tempfile_2;
4674 print $F_eff "trait_type,row,col,value\n";
4675 foreach my $p (@unique_plot_names) {
4676 foreach my $t (@sorted_trait_names_2) {
4677 my $val = $result_blup_spatial_data_altered_env_1_2->{$p}->{$t}->[0];
4678 my @row = ("effm2_sim1_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4679 my $line = join ',', @row;
4680 print $F_eff "$line\n";
4681 push @sim_effect1_vals_2, $val;
4682 }
4683 }
4684 close($F_eff);
4685 push @plot_corr_full_vals, \@sim_effect1_vals_2;
4686
4687 my $sim_effect1_stat_2 = Statistics::Descriptive::Full->new();
4688 $sim_effect1_stat_2->add_data(@sim_effect1_vals_2);
4689 my $sig_sim2_effect1 = $sim_effect1_stat_2->variance();
4690
4691 # SIM ENV 2: ALTERED PHENO + EFFECT
4692
4693 my ($phenotypes_env_heatmap_tempfile2_fh, $phenotypes_env_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4694 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile2) || die "Can't open file ".$phenotypes_env_heatmap_tempfile2;
4695 print $F_pheno "trait_type,row,col,value\n";
4696 foreach my $p (@unique_plot_names) {
4697 foreach my $t (@sorted_trait_names_2) {
4698 my @row = ("sim_env2_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_2_2->{$p}->{$t});
4699 my $line = join ',', @row;
4700 print $F_pheno "$line\n";
4701 }
4702 }
4703 close($F_pheno);
4704
4705 my @sim_pheno2_vals_2;
4706 my ($phenotypes_pheno_sim_heatmap_tempfile2_fh_2, $phenotypes_pheno_sim_heatmap_tempfile2_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4707 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile2_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile2_2;
4708 print $F_pheno "trait_type,row,col,value\n";
4709 foreach my $p (@unique_plot_names) {
4710 foreach my $t (@sorted_trait_names_2) {
4711 my $val = $phenotype_data_altered_env_hash_2_2->{$p}->{$t};
4712 my @row = ("simm2_pheno2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4713 my $line = join ',', @row;
4714 print $F_pheno "$line\n";
4715 push @sim_pheno2_vals_2, $val;
4716 }
4717 }
4718 close($F_pheno);
4719 push @plot_corr_full_vals, \@sim_pheno2_vals_2;
4720
4721 my $sim_pheno2_stat_2 = Statistics::Descriptive::Full->new();
4722 $sim_pheno2_stat_2->add_data(@sim_pheno2_vals_2);
4723 my $sig_sim_pheno2_2 = $sim_pheno2_stat_2->variance();
4724
4725 my @sim_effect2_vals_2;
4726 my ($effects_sim_heatmap_tempfile2_fh_2, $effects_sim_heatmap_tempfile2_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4727 open($F_eff, ">", $effects_sim_heatmap_tempfile2_2) || die "Can't open file ".$effects_sim_heatmap_tempfile2_2;
4728 print $F_eff "trait_type,row,col,value\n";
4729 foreach my $p (@unique_plot_names) {
4730 foreach my $t (@sorted_trait_names_2) {
4731 my $val = $result_blup_spatial_data_altered_env_2_2->{$p}->{$t}->[0];
4732 my @row = ("effm2_sim2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4733 my $line = join ',', @row;
4734 print $F_eff "$line\n";
4735 push @sim_effect2_vals_2, $val;
4736 }
4737 }
4738 close($F_eff);
4739 push @plot_corr_full_vals, \@sim_effect2_vals_2;
4740
4741 my $sim_effect2_stat_2 = Statistics::Descriptive::Full->new();
4742 $sim_effect2_stat_2->add_data(@sim_effect2_vals_2);
4743 my $sig_sim_effect2_2 = $sim_effect2_stat_2->variance();
4744
4745 # SIM ENV 3: ALTERED PHENO + EFFECT
4746
4747 my ($phenotypes_env_heatmap_tempfile3_fh, $phenotypes_env_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4748 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile3) || die "Can't open file ".$phenotypes_env_heatmap_tempfile3;
4749 print $F_pheno "trait_type,row,col,value\n";
4750 foreach my $p (@unique_plot_names) {
4751 foreach my $t (@sorted_trait_names_2) {
4752 my @row = ("sim_env3_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_3_2->{$p}->{$t});
4753 my $line = join ',', @row;
4754 print $F_pheno "$line\n";
4755 }
4756 }
4757 close($F_pheno);
4758
4759 my @sim_pheno3_vals_2;
4760 my ($phenotypes_pheno_sim_heatmap_tempfile3_fh_2, $phenotypes_pheno_sim_heatmap_tempfile3_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4761 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile3_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile3_2;
4762 print $F_pheno "trait_type,row,col,value\n";
4763 foreach my $p (@unique_plot_names) {
4764 foreach my $t (@sorted_trait_names_2) {
4765 my $val = $phenotype_data_altered_env_hash_3_2->{$p}->{$t};
4766 my @row = ("simm2_pheno3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4767 my $line = join ',', @row;
4768 print $F_pheno "$line\n";
4769 push @sim_pheno3_vals_2, $val;
4770 }
4771 }
4772 close($F_pheno);
4773 push @plot_corr_full_vals, \@sim_pheno3_vals_2;
4774
4775 my $sim_pheno3_stat_2 = Statistics::Descriptive::Full->new();
4776 $sim_pheno3_stat_2->add_data(@sim_pheno3_vals_2);
4777 my $sig_sim_pheno3_2 = $sim_pheno3_stat_2->variance();
4778
4779 my @sim_effect3_vals_2;
4780 my ($effects_sim_heatmap_tempfile3_fh_2, $effects_sim_heatmap_tempfile3_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4781 open($F_eff, ">", $effects_sim_heatmap_tempfile3_2) || die "Can't open file ".$effects_sim_heatmap_tempfile3_2;
4782 print $F_eff "trait_type,row,col,value\n";
4783 foreach my $p (@unique_plot_names) {
4784 foreach my $t (@sorted_trait_names_2) {
4785 my $val = $result_blup_spatial_data_altered_env_3_2->{$p}->{$t}->[0];
4786 my @row = ("effm2_sim3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4787 my $line = join ',', @row;
4788 print $F_eff "$line\n";
4789 push @sim_effect3_vals_2, $val;
4790 }
4791 }
4792 close($F_eff);
4793 push @plot_corr_full_vals, \@sim_effect3_vals_2;
4794
4795 my $sim_effect3_stat_2 = Statistics::Descriptive::Full->new();
4796 $sim_effect3_stat_2->add_data(@sim_effect3_vals_2);
4797 my $sig_sim_effect3_2 = $sim_effect3_stat_2->variance();
4798
4799 # SIM ENV 4: ALTERED PHENO + EFFECT
4800
4801 my ($phenotypes_env_heatmap_tempfile4_fh, $phenotypes_env_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4802 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile4) || die "Can't open file ".$phenotypes_env_heatmap_tempfile4;
4803 print $F_pheno "trait_type,row,col,value\n";
4804 foreach my $p (@unique_plot_names) {
4805 foreach my $t (@sorted_trait_names_2) {
4806 my @row = ("sim_env4_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_4_2->{$p}->{$t});
4807 my $line = join ',', @row;
4808 print $F_pheno "$line\n";
4809 }
4810 }
4811 close($F_pheno);
4812
4813 my @sim_pheno4_vals_2;
4814 my ($phenotypes_pheno_sim_heatmap_tempfile4_fh_2, $phenotypes_pheno_sim_heatmap_tempfile4_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4815 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile4_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile4_2;
4816 print $F_pheno "trait_type,row,col,value\n";
4817 foreach my $p (@unique_plot_names) {
4818 foreach my $t (@sorted_trait_names_2) {
4819 my $val = $phenotype_data_altered_env_hash_4_2->{$p}->{$t};
4820 my @row = ("simm2_pheno4_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4821 my $line = join ',', @row;
4822 print $F_pheno "$line\n";
4823 push @sim_pheno4_vals_2, $val;
4824 }
4825 }
4826 close($F_pheno);
4827 push @plot_corr_full_vals, \@sim_pheno4_vals_2;
4828
4829 my $sim_pheno4_stat_2 = Statistics::Descriptive::Full->new();
4830 $sim_pheno4_stat_2->add_data(@sim_pheno4_vals_2);
4831 my $sig_sim_pheno4_2 = $sim_pheno4_stat_2->variance();
4832
4833 my @sim_effect4_vals_2;
4834 my ($effects_sim_heatmap_tempfile4_fh_2, $effects_sim_heatmap_tempfile4_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4835 open($F_eff, ">", $effects_sim_heatmap_tempfile4_2) || die "Can't open file ".$effects_sim_heatmap_tempfile4_2;
4836 print $F_eff "trait_type,row,col,value\n";
4837 foreach my $p (@unique_plot_names) {
4838 foreach my $t (@sorted_trait_names_2) {
4839 my $val = $result_blup_spatial_data_altered_env_4_2->{$p}->{$t}->[0];
4840 my @row = ("effm2_sim4_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4841 my $line = join ',', @row;
4842 print $F_eff "$line\n";
4843 push @sim_effect4_vals_2, $val;
4844 }
4845 }
4846 close($F_eff);
4847 push @plot_corr_full_vals, \@sim_effect4_vals_2;
4848
4849 my $sim_effect4_stat_2 = Statistics::Descriptive::Full->new();
4850 $sim_effect4_stat_2->add_data(@sim_effect4_vals_2);
4851 my $sig_sim_effect4_2 = $sim_effect4_stat_2->variance();
4852
4853 # SIM ENV 5: ALTERED PHENO + EFFECT
4854
4855 my ($phenotypes_env_heatmap_tempfile5_fh, $phenotypes_env_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4856 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile5) || die "Can't open file ".$phenotypes_env_heatmap_tempfile5;
4857 print $F_pheno "trait_type,row,col,value\n";
4858 foreach my $p (@unique_plot_names) {
4859 foreach my $t (@sorted_trait_names_2) {
4860 my @row = ("sim_env5_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_5_2->{$p}->{$t});
4861 my $line = join ',', @row;
4862 print $F_pheno "$line\n";
4863 }
4864 }
4865 close($F_pheno);
4866
4867 my @sim_pheno5_vals_2;
4868 my ($phenotypes_pheno_sim_heatmap_tempfile5_fh_2, $phenotypes_pheno_sim_heatmap_tempfile5_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4869 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile5_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile5_2;
4870 print $F_pheno "trait_type,row,col,value\n";
4871 foreach my $p (@unique_plot_names) {
4872 foreach my $t (@sorted_trait_names_2) {
4873 my $val = $phenotype_data_altered_env_hash_5_2->{$p}->{$t};
4874 my @row = ("simm2_pheno5_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4875 my $line = join ',', @row;
4876 print $F_pheno "$line\n";
4877 push @sim_pheno5_vals_2, $val;
4878 }
4879 }
4880 close($F_pheno);
4881 push @plot_corr_full_vals, \@sim_pheno5_vals_2;
4882
4883 my $sim_pheno5_stat_2 = Statistics::Descriptive::Full->new();
4884 $sim_pheno5_stat_2->add_data(@sim_pheno5_vals_2);
4885 my $sig_sim_pheno5_2 = $sim_pheno5_stat_2->variance();
4886
4887 my @sim_effect5_vals_2;
4888 my ($effects_sim_heatmap_tempfile5_fh_2, $effects_sim_heatmap_tempfile5_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4889 open($F_eff, ">", $effects_sim_heatmap_tempfile5_2) || die "Can't open file ".$effects_sim_heatmap_tempfile5_2;
4890 print $F_eff "trait_type,row,col,value\n";
4891 foreach my $p (@unique_plot_names) {
4892 foreach my $t (@sorted_trait_names_2) {
4893 my $val = $result_blup_spatial_data_altered_env_5_2->{$p}->{$t}->[0];
4894 my @row = ("effm2_sim5_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4895 my $line = join ',', @row;
4896 print $F_eff "$line\n";
4897 push @sim_effect5_vals_2, $val;
4898 }
4899 }
4900 close($F_eff);
4901 push @plot_corr_full_vals, \@sim_effect5_vals_2;
4902
4903 my $sim_effect5_stat_2 = Statistics::Descriptive::Full->new();
4904 $sim_effect5_stat_2->add_data(@sim_effect5_vals_2);
4905 my $sig_sim_effect5_2 = $sim_effect5_stat_2->variance();
4906
4907 # SIM ENV 6: ALTERED PHENO + EFFECT
4908
4909 my ($phenotypes_env_heatmap_tempfile6_fh, $phenotypes_env_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4910 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile6) || die "Can't open file ".$phenotypes_env_heatmap_tempfile6;
4911 print $F_pheno "trait_type,row,col,value\n";
4912 foreach my $p (@unique_plot_names) {
4913 foreach my $t (@sorted_trait_names_2) {
4914 my @row = ("sim_env6_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_6_2->{$p}->{$t});
4915 my $line = join ',', @row;
4916 print $F_pheno "$line\n";
4917 }
4918 }
4919 close($F_pheno);
4920
4921 my @sim_pheno6_vals_2;
4922 my ($phenotypes_pheno_sim_heatmap_tempfile6_fh_2, $phenotypes_pheno_sim_heatmap_tempfile6_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4923 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile6_2) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile6_2;
4924 print $F_pheno "trait_type,row,col,value\n";
4925 foreach my $p (@unique_plot_names) {
4926 foreach my $t (@sorted_trait_names_2) {
4927 my $val = $phenotype_data_altered_env_hash_6_2->{$p}->{$t};
4928 my @row = ("simm2_pheno6_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4929 my $line = join ',', @row;
4930 print $F_pheno "$line\n";
4931 push @sim_pheno6_vals_2, $val;
4932 }
4933 }
4934 close($F_pheno);
4935 push @plot_corr_full_vals, \@sim_pheno6_vals_2;
4936
4937 my $sim_pheno6_stat_2 = Statistics::Descriptive::Full->new();
4938 $sim_pheno6_stat_2->add_data(@sim_pheno6_vals_2);
4939 my $sig_sim_pheno6_2 = $sim_pheno6_stat_2->variance();
4940
4941 my @sim_effect6_vals_2;
4942 my ($effects_sim_heatmap_tempfile6_fh_2, $effects_sim_heatmap_tempfile6_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
4943 open($F_eff, ">", $effects_sim_heatmap_tempfile6_2) || die "Can't open file ".$effects_sim_heatmap_tempfile6_2;
4944 print $F_eff "trait_type,row,col,value\n";
4945 foreach my $p (@unique_plot_names) {
4946 foreach my $t (@sorted_trait_names_2) {
4947 my $val = $result_blup_spatial_data_altered_env_6_2->{$p}->{$t}->[0];
4948 my @row = ("effm2_sim6_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
4949 my $line = join ',', @row;
4950 print $F_eff "$line\n";
4951 push @sim_effect6_vals_2, $val;
4952 }
4953 }
4954 close($F_eff);
4955 push @plot_corr_full_vals, \@sim_effect6_vals_2;
4956
4957 my $sim_effect6_stat_2 = Statistics::Descriptive::Full->new();
4958 $sim_effect6_stat_2->add_data(@sim_effect6_vals_2);
4959 my $sig_sim_effect6_2 = $sim_effect6_stat_2->variance();
4960
4961 my $plot_corr_summary_figure_inputfile_tempfile = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'tmp_drone_statistics/fileXXXX');
4962 open($F_eff, ">", $plot_corr_summary_figure_inputfile_tempfile) || die "Can't open file ".$plot_corr_summary_figure_inputfile_tempfile;
4963 foreach (@plot_corr_full_vals) {
4964 my $line = join ',', @$_;
4965 print $F_eff $line."\n";
4966 }
4967 close($F_eff);
4968
4969 my $plot_corr_summary_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
4970 $plot_corr_summary_figure_tempfile_string .= '.png';
4971 my $plot_corr_summary_figure_tempfile = $c->config->{basepath}."/".$plot_corr_summary_figure_tempfile_string;
4972
4973 my $cmd_plotcorrsum_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
4974 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
4975 mat_full <- data.frame(t(mat_full_t));
4976 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_2\', \'mat_eff_2\', \'mat_eff_altered_2\',
4977 \'mat_p_sim1_2\', \'mat_eff_sim1_2\',
4978 \'mat_p_sim2_2\', \'mat_eff_sim2_2\',
4979 \'mat_p_sim3_2\', \'mat_eff_sim3_2\',
4980 \'mat_p_sim4_2\', \'mat_eff_sim4_2\',
4981 \'mat_p_sim5_2\', \'mat_eff_sim5_2\',
4982 \'mat_p_sim6_2\', \'mat_eff_sim6_2\');
4983 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
4984 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
4985 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
4986 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
4987 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
4988 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
4989 mat <- data.frame(pheno_orig = mat_full\$mat_orig, pheno_altm2 = mat_full\$mat_altered_2, eff_origm2 = mat_full\$mat_eff_2, eff_altm2 = mat_full\$mat_eff_altered_2, env_lin = mat_env\$value, pheno_linm2 = mat_full\$mat_p_sim1_2, lin_effm2 = mat_full\$mat_eff_sim1_2, env_n1d = mat_env2\$value, pheno_n1dm2 = mat_full\$mat_p_sim2_2, n1d_effm2 = mat_full\$mat_eff_sim2_2, env_n2d = mat_env3\$value, pheno_n2dm2 = mat_full\$mat_p_sim3_2, n2d_effm2 = mat_full\$mat_eff_sim3_2, env_rand = mat_env4\$value, pheno_randm2 = mat_full\$mat_p_sim4_2, rand_effm2 = mat_full\$mat_eff_sim4_2, env_ar1 = mat_env5\$value, pheno_ar1m2 = mat_full\$mat_p_sim5_2, ar1_effm2 = mat_full\$mat_eff_sim5_2, env_realdata = mat_env6\$value, pheno_realdatam2 = mat_full\$mat_p_sim6_2, realdata_effm2 = mat_full\$mat_eff_sim6_2);
4990 gg <- ggcorr(data=mat, hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE, label_round = 2);
4991 ggsave(\''.$plot_corr_summary_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, units=\'in\');
4992 "';
4993 # print STDERR Dumper $cmd_plotcorrsum_plot;
4994 my $status_plotcorrsum_plot = system($cmd_plotcorrsum_plot);
4995 push @$spatial_effects_plots, [$plot_corr_summary_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_envsimscorr_"."envvar_".$env_variance_percent."_".$iterations];
4996
4997 my $env_effects_first_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
4998 $env_effects_first_figure_tempfile_string .= '.png';
4999 my $env_effects_first_figure_tempfile = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string;
5000
5001 my $env_effects_first_figure_tempfile_string_2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5002 $env_effects_first_figure_tempfile_string_2 .= '.png';
5003 my $env_effects_first_figure_tempfile_2 = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string_2;
5004
5005 my $output_plot_row = 'row';
5006 my $output_plot_col = 'col';
5007 if ($max_col > $max_row) {
5008 $output_plot_row = 'col';
5009 $output_plot_col = 'row';
5010 }
5011
5012 my $cmd_spatialfirst_plot_2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5013 mat_orig <- fread(\''.$phenotypes_original_heatmap_tempfile.'\', header=TRUE, sep=\',\');
5014 mat_altered_2 <- fread(\''.$phenotypes_post_heatmap_tempfile_2.'\', header=TRUE, sep=\',\');
5015 pheno_mat <- rbind(mat_orig, mat_altered_2);
5016 options(device=\'png\');
5017 par();
5018 gg <- ggplot(pheno_mat, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5019 geom_tile() +
5020 scale_fill_viridis(discrete=FALSE) +
5021 coord_equal() +
5022 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5023 ggsave(\''.$env_effects_first_figure_tempfile_2.'\', gg, device=\'png\', width=20, height=20, units=\'in\');
5024 "';
5025 # print STDERR Dumper $cmd;
5026 my $status_spatialfirst_plot_2 = system($cmd_spatialfirst_plot_2);
5027 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string_2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_origheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5028
5029 my ($sim_effects_corr_results_fh, $sim_effects_corr_results) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5030
5031 my $cmd_spatialfirst_plot = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5032 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
5033 mat_full <- data.frame(t(mat_full_t));
5034 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_2\', \'mat_eff_2\', \'mat_eff_altered_2\',
5035 \'mat_p_sim1_2\', \'mat_eff_sim1_2\',
5036 \'mat_p_sim2_2\', \'mat_eff_sim2_2\',
5037 \'mat_p_sim3_2\', \'mat_eff_sim3_2\',
5038 \'mat_p_sim4_2\', \'mat_eff_sim4_2\',
5039 \'mat_p_sim5_2\', \'mat_eff_sim5_2\',
5040 \'mat_p_sim6_2\', \'mat_eff_sim6_2\');
5041 mat_eff_2 <- fread(\''.$effects_heatmap_tempfile_2.'\', header=TRUE, sep=\',\');
5042 mat_eff_altered_2 <- fread(\''.$effects_post_heatmap_tempfile_2.'\', header=TRUE, sep=\',\');
5043 effect_mat_2 <- rbind(mat_eff_2, mat_eff_altered_2);
5044 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
5045 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
5046 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
5047 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
5048 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
5049 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
5050 options(device=\'png\');
5051 par();
5052 gg_eff_2 <- ggplot(effect_mat_2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5053 geom_tile() +
5054 scale_fill_viridis(discrete=FALSE) +
5055 coord_equal() +
5056 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5057 ggsave(\''.$env_effects_first_figure_tempfile.'\', arrangeGrob(gg_eff_2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5058 write.table(data.frame(env_linear = c(cor(mat_env\$value, mat_full\$mat_eff_sim1_2)), env_1DN = c(cor(mat_env2\$value, mat_full\$mat_eff_sim2_2)), env_2DN = c(cor(mat_env3\$value, mat_full\$mat_eff_sim3_2)), env_random = c(cor(mat_env4\$value, mat_full\$mat_eff_sim4_2)), env_ar1xar1 = c(cor(mat_env5\$value, mat_full\$mat_eff_sim5_2)), env_realdata = c(cor(mat_env6\$value, mat_full\$mat_eff_sim6_2)) ), file=\''.$sim_effects_corr_results.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
5059 "';
5060 # print STDERR Dumper $cmd;
5061 my $status_spatialfirst_plot = system($cmd_spatialfirst_plot);
5062 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_originaleffheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5063
5064 open(my $fh_corr_result, '<', $sim_effects_corr_results) or die "Could not open file '$sim_effects_corr_results' $!";
5065 print STDERR "Opened $sim_effects_corr_results\n";
5066
5067 my $header = <$fh_corr_result>;
5068 my @header;
5069 if ($csv->parse($header)) {
5070 @header = $csv->fields();
5071 }
5072
5073 while (my $row = <$fh_corr_result>) {
5074 my @columns;
5075 my $counter = 0;
5076 if ($csv->parse($row)) {
5077 @columns = $csv->fields();
5078 }
5079 foreach (@columns) {
5080 push @{$env_corr_res->{$statistics_select."_".$header[$counter]."_corrtime_".$sim_env_change_over_time.$correlation_between_times."_envvar_".$env_variance_percent}->{values}}, $_;
5081 $counter++;
5082 }
5083 }
5084 close($fh_corr_result);
5085
5086 my $env_effects_sim_figure_tempfile_string_2_env1 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5087 $env_effects_sim_figure_tempfile_string_2_env1 .= '.png';
5088 my $env_effects_sim_figure_tempfile_2_env1 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env1;
5089
5090 my $cmd_spatialenvsim_plot_2_env1 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5091 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
5092 mat_p_sim <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile_2.'\', header=TRUE, sep=\',\');
5093 mat_eff_sim <- fread(\''.$effects_sim_heatmap_tempfile_2.'\', header=TRUE, sep=\',\');
5094 options(device=\'png\');
5095 par();
5096 gg_env <- ggplot(mat_env, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5097 geom_tile() +
5098 scale_fill_viridis(discrete=FALSE) +
5099 coord_equal() +
5100 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5101 gg_p_sim <- ggplot(mat_p_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5102 geom_tile() +
5103 scale_fill_viridis(discrete=FALSE) +
5104 coord_equal() +
5105 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5106 gg_eff_sim <- ggplot(mat_eff_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5107 geom_tile() +
5108 scale_fill_viridis(discrete=FALSE) +
5109 coord_equal() +
5110 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5111 ggsave(\''.$env_effects_sim_figure_tempfile_2_env1.'\', arrangeGrob(gg_env, gg_p_sim, gg_eff_sim, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5112 "';
5113 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env1;
5114 my $status_spatialenvsim_plot_2_env1 = system($cmd_spatialenvsim_plot_2_env1);
5115 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env1, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env1effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5116
5117 my $env_effects_sim_figure_tempfile_string_2_env2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5118 $env_effects_sim_figure_tempfile_string_2_env2 .= '.png';
5119 my $env_effects_sim_figure_tempfile_2_env2 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env2;
5120
5121 my $cmd_spatialenvsim_plot_2_env2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5122 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
5123 mat_p_sim2 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile2_2.'\', header=TRUE, sep=\',\');
5124 mat_eff_sim2 <- fread(\''.$effects_sim_heatmap_tempfile2_2.'\', header=TRUE, sep=\',\');
5125 options(device=\'png\');
5126 par();
5127 gg_env2 <- ggplot(mat_env2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5128 geom_tile() +
5129 scale_fill_viridis(discrete=FALSE) +
5130 coord_equal() +
5131 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5132 gg_p_sim2 <- ggplot(mat_p_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5133 geom_tile() +
5134 scale_fill_viridis(discrete=FALSE) +
5135 coord_equal() +
5136 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5137 gg_eff_sim2 <- ggplot(mat_eff_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5138 geom_tile() +
5139 scale_fill_viridis(discrete=FALSE) +
5140 coord_equal() +
5141 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5142 ggsave(\''.$env_effects_sim_figure_tempfile_2_env2.'\', arrangeGrob(gg_env2, gg_p_sim2, gg_eff_sim2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5143 "';
5144 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env2;
5145 my $status_spatialenvsim_plot_2_env2 = system($cmd_spatialenvsim_plot_2_env2);
5146 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env2effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5147
5148 my $env_effects_sim_figure_tempfile_string_2_env3 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5149 $env_effects_sim_figure_tempfile_string_2_env3 .= '.png';
5150 my $env_effects_sim_figure_tempfile_2_env3 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env3;
5151
5152 my $cmd_spatialenvsim_plot_2_env3 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5153 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
5154 mat_p_sim3 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile3_2.'\', header=TRUE, sep=\',\');
5155 mat_eff_sim3 <- fread(\''.$effects_sim_heatmap_tempfile3_2.'\', header=TRUE, sep=\',\');
5156 options(device=\'png\');
5157 par();
5158 gg_env3 <- ggplot(mat_env3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5159 geom_tile() +
5160 scale_fill_viridis(discrete=FALSE) +
5161 coord_equal() +
5162 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5163 gg_p_sim3 <- ggplot(mat_p_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5164 geom_tile() +
5165 scale_fill_viridis(discrete=FALSE) +
5166 coord_equal() +
5167 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5168 gg_eff_sim3 <- ggplot(mat_eff_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5169 geom_tile() +
5170 scale_fill_viridis(discrete=FALSE) +
5171 coord_equal() +
5172 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5173 ggsave(\''.$env_effects_sim_figure_tempfile_2_env3.'\', arrangeGrob(gg_env3, gg_p_sim3, gg_eff_sim3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5174 "';
5175 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env3;
5176 my $status_spatialenvsim_plot_2_env3 = system($cmd_spatialenvsim_plot_2_env3);
5177 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env3, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env3effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5178
5179 my $env_effects_sim_figure_tempfile_string_2_env4 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5180 $env_effects_sim_figure_tempfile_string_2_env4 .= '.png';
5181 my $env_effects_sim_figure_tempfile_2_env4 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env4;
5182
5183 my $cmd_spatialenvsim_plot_2_env4 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5184 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
5185 mat_p_sim4 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile4_2.'\', header=TRUE, sep=\',\');
5186 mat_eff_sim4 <- fread(\''.$effects_sim_heatmap_tempfile4_2.'\', header=TRUE, sep=\',\');
5187 options(device=\'png\');
5188 par();
5189 gg_env4 <- ggplot(mat_env4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5190 geom_tile() +
5191 scale_fill_viridis(discrete=FALSE) +
5192 coord_equal() +
5193 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5194 gg_p_sim4 <- ggplot(mat_p_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5195 geom_tile() +
5196 scale_fill_viridis(discrete=FALSE) +
5197 coord_equal() +
5198 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5199 gg_eff_sim4 <- ggplot(mat_eff_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5200 geom_tile() +
5201 scale_fill_viridis(discrete=FALSE) +
5202 coord_equal() +
5203 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5204 ggsave(\''.$env_effects_sim_figure_tempfile_2_env4.'\', arrangeGrob(gg_env4, gg_p_sim4, gg_eff_sim4, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5205 "';
5206 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env4;
5207 my $status_spatialenvsim_plot_2_env4 = system($cmd_spatialenvsim_plot_2_env4);
5208 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env4, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env4effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5209
5210 my $env_effects_sim_figure_tempfile_string_2_env5 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5211 $env_effects_sim_figure_tempfile_string_2_env5 .= '.png';
5212 my $env_effects_sim_figure_tempfile_2_env5 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env5;
5213
5214 my $cmd_spatialenvsim_plot_2_env5 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5215 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
5216 mat_p_sim5 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile5_2.'\', header=TRUE, sep=\',\');
5217 mat_eff_sim5 <- fread(\''.$effects_sim_heatmap_tempfile5_2.'\', header=TRUE, sep=\',\');
5218 options(device=\'png\');
5219 par();
5220 gg_env5 <- ggplot(mat_env5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5221 geom_tile() +
5222 scale_fill_viridis(discrete=FALSE) +
5223 coord_equal() +
5224 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5225 gg_p_sim5 <- ggplot(mat_p_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5226 geom_tile() +
5227 scale_fill_viridis(discrete=FALSE) +
5228 coord_equal() +
5229 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5230 gg_eff_sim5 <- ggplot(mat_eff_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5231 geom_tile() +
5232 scale_fill_viridis(discrete=FALSE) +
5233 coord_equal() +
5234 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5235 ggsave(\''.$env_effects_sim_figure_tempfile_2_env5.'\', arrangeGrob(gg_env5, gg_p_sim5, gg_eff_sim5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5236 "';
5237 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env5;
5238 my $status_spatialenvsim_plot_2_env5 = system($cmd_spatialenvsim_plot_2_env5);
5239 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env5, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env5effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5240
5241 my $env_effects_sim_figure_tempfile_string_2_env6 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5242 $env_effects_sim_figure_tempfile_string_2_env6 .= '.png';
5243 my $env_effects_sim_figure_tempfile_2_env6 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_2_env6;
5244
5245 my $cmd_spatialenvsim_plot_2_env6 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
5246 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
5247 mat_p_sim6 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile6_2.'\', header=TRUE, sep=\',\');
5248 mat_eff_sim6 <- fread(\''.$effects_sim_heatmap_tempfile6_2.'\', header=TRUE, sep=\',\');
5249 options(device=\'png\');
5250 par();
5251 gg_env6 <- ggplot(mat_env6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5252 geom_tile() +
5253 scale_fill_viridis(discrete=FALSE) +
5254 coord_equal() +
5255 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5256 gg_p_sim6 <- ggplot(mat_p_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5257 geom_tile() +
5258 scale_fill_viridis(discrete=FALSE) +
5259 coord_equal() +
5260 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5261 gg_eff_sim6 <- ggplot(mat_eff_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
5262 geom_tile() +
5263 scale_fill_viridis(discrete=FALSE) +
5264 coord_equal() +
5265 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
5266 ggsave(\''.$env_effects_sim_figure_tempfile_2_env6.'\', arrangeGrob(gg_env6, gg_p_sim6, gg_eff_sim6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
5267 "';
5268 # print STDERR Dumper $cmd_spatialenvsim_plot_2_env6;
5269 my $status_spatialenvsim_plot_2_env6 = system($cmd_spatialenvsim_plot_2_env6);
5270 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_2_env6, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env6effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
5271 };
5272
5273 eval {
5274 my @sorted_germplasm_names = sort keys %unique_accessions;
5275
5276 my @original_blup_vals_2;
5277 my ($effects_original_line_chart_tempfile_fh_2, $effects_original_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5278 open(my $F_pheno, ">", $effects_original_line_chart_tempfile_2) || die "Can't open file ".$effects_original_line_chart_tempfile_2;
5279 print $F_pheno "germplasmName,time,value\n";
5280 foreach my $p (@sorted_germplasm_names) {
5281 foreach my $t (@sorted_trait_names_2) {
5282 my $val = $result_blup_data_original_2->{$p}->{$t}->[0];
5283 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5284 push @original_blup_vals_2, $val;
5285 my $line = join ',', @row;
5286 print $F_pheno "$line\n";
5287 }
5288 }
5289 close($F_pheno);
5290
5291 my $original_blup_stat_2 = Statistics::Descriptive::Full->new();
5292 $original_blup_stat_2->add_data(@original_blup_vals_2);
5293 my $sig_original_blup_2 = $original_blup_stat_2->variance();
5294
5295 my @altered_blups_vals_2;
5296 my ($effects_altered_line_chart_tempfile_fh_2, $effects_altered_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5297 open($F_pheno, ">", $effects_altered_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_line_chart_tempfile_2;
5298 print $F_pheno "germplasmName,time,value\n";
5299 foreach my $p (@sorted_germplasm_names) {
5300 foreach my $t (@sorted_trait_names_2) {
5301 my $val = $result_blup_data_altered_2->{$p}->{$t}->[0];
5302 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5303 my $line = join ',', @row;
5304 print $F_pheno "$line\n";
5305 push @altered_blups_vals_2, $val;
5306 }
5307 }
5308 close($F_pheno);
5309
5310 my $altered_blup_stat_2 = Statistics::Descriptive::Full->new();
5311 $altered_blup_stat_2->add_data(@altered_blups_vals_2);
5312 my $sig_altered_blup_2 = $altered_blup_stat_2->variance();
5313
5314 my @sim1_blup_vals_2;
5315 my ($effects_altered_env1_line_chart_tempfile_fh_2, $effects_altered_env1_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5316 open($F_pheno, ">", $effects_altered_env1_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env1_line_chart_tempfile_2;
5317 print $F_pheno "germplasmName,time,value\n";
5318 foreach my $p (@sorted_germplasm_names) {
5319 foreach my $t (@sorted_trait_names_2) {
5320 my $val = $result_blup_data_altered_env_1_2->{$p}->{$t}->[0];
5321 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5322 my $line = join ',', @row;
5323 print $F_pheno "$line\n";
5324 push @sim1_blup_vals_2, $val;
5325 }
5326 }
5327 close($F_pheno);
5328
5329 my $sim1_blup_stat_2 = Statistics::Descriptive::Full->new();
5330 $sim1_blup_stat_2->add_data(@sim1_blup_vals_2);
5331 my $sig_sim1_blup_2 = $sim1_blup_stat_2->variance();
5332
5333 my @sim2_blup_vals_2;
5334 my ($effects_altered_env2_line_chart_tempfile_fh_2, $effects_altered_env2_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5335 open($F_pheno, ">", $effects_altered_env2_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env2_line_chart_tempfile_2;
5336 print $F_pheno "germplasmName,time,value\n";
5337 foreach my $p (@sorted_germplasm_names) {
5338 foreach my $t (@sorted_trait_names_2) {
5339 my $val = $result_blup_data_altered_env_2_2->{$p}->{$t}->[0];
5340 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5341 my $line = join ',', @row;
5342 print $F_pheno "$line\n";
5343 push @sim2_blup_vals_2, $val;
5344 }
5345 }
5346 close($F_pheno);
5347
5348 my $sim2_blup_stat_2 = Statistics::Descriptive::Full->new();
5349 $sim2_blup_stat_2->add_data(@sim2_blup_vals_2);
5350 my $sig_sim2_blup_2 = $sim2_blup_stat_2->variance();
5351
5352 my @sim3_blup_vals_2;
5353 my ($effects_altered_env3_line_chart_tempfile_fh_2, $effects_altered_env3_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5354 open($F_pheno, ">", $effects_altered_env3_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env3_line_chart_tempfile_2;
5355 print $F_pheno "germplasmName,time,value\n";
5356 foreach my $p (@sorted_germplasm_names) {
5357 foreach my $t (@sorted_trait_names_2) {
5358 my $val = $result_blup_data_altered_env_3_2->{$p}->{$t}->[0];
5359 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5360 my $line = join ',', @row;
5361 print $F_pheno "$line\n";
5362 push @sim3_blup_vals_2, $val;
5363 }
5364 }
5365 close($F_pheno);
5366
5367 my $sim3_blup_stat_2 = Statistics::Descriptive::Full->new();
5368 $sim3_blup_stat_2->add_data(@sim3_blup_vals_2);
5369 my $sig_sim3_blup_2 = $sim3_blup_stat_2->variance();
5370
5371 my @sim4_blup_vals_2;
5372 my ($effects_altered_env4_line_chart_tempfile_fh_2, $effects_altered_env4_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5373 open($F_pheno, ">", $effects_altered_env4_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env4_line_chart_tempfile_2;
5374 print $F_pheno "germplasmName,time,value\n";
5375 foreach my $p (@sorted_germplasm_names) {
5376 foreach my $t (@sorted_trait_names_2) {
5377 my $val = $result_blup_data_altered_env_4_2->{$p}->{$t}->[0];
5378 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5379 my $line = join ',', @row;
5380 print $F_pheno "$line\n";
5381 push @sim4_blup_vals_2, $val;
5382 }
5383 }
5384 close($F_pheno);
5385
5386 my $sim4_blup_stat_2 = Statistics::Descriptive::Full->new();
5387 $sim4_blup_stat_2->add_data(@sim4_blup_vals_2);
5388 my $sig_sim4_blup_2 = $sim4_blup_stat_2->variance();
5389
5390 my @sim5_blup_vals_2;
5391 my ($effects_altered_env5_line_chart_tempfile_fh_2, $effects_altered_env5_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5392 open($F_pheno, ">", $effects_altered_env5_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env5_line_chart_tempfile_2;
5393 print $F_pheno "germplasmName,time,value\n";
5394 foreach my $p (@sorted_germplasm_names) {
5395 foreach my $t (@sorted_trait_names_2) {
5396 my $val = $result_blup_data_altered_env_5_2->{$p}->{$t}->[0];
5397 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5398 my $line = join ',', @row;
5399 print $F_pheno "$line\n";
5400 push @sim5_blup_vals_2, $val;
5401 }
5402 }
5403 close($F_pheno);
5404
5405 my $sim5_blup_stat_2 = Statistics::Descriptive::Full->new();
5406 $sim5_blup_stat_2->add_data(@sim5_blup_vals_2);
5407 my $sig_sim5_blup_2 = $sim5_blup_stat_2->variance();
5408
5409 my @sim6_blup_vals_2;
5410 my ($effects_altered_env6_line_chart_tempfile_fh_2, $effects_altered_env6_line_chart_tempfile_2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
5411 open($F_pheno, ">", $effects_altered_env6_line_chart_tempfile_2) || die "Can't open file ".$effects_altered_env6_line_chart_tempfile_2;
5412 print $F_pheno "germplasmName,time,value\n";
5413 foreach my $p (@sorted_germplasm_names) {
5414 foreach my $t (@sorted_trait_names_2) {
5415 my $val = $result_blup_data_altered_env_6_2->{$p}->{$t}->[0];
5416 my @row = ($p, $trait_to_time_map_hash_2->{$t}, $val);
5417 my $line = join ',', @row;
5418 print $F_pheno "$line\n";
5419 push @sim6_blup_vals_2, $val;
5420 }
5421 }
5422 close($F_pheno);
5423
5424 my $sim6_blup_stat_2 = Statistics::Descriptive::Full->new();
5425 $sim6_blup_stat_2->add_data(@sim6_blup_vals_2);
5426 my $sig_sim6_blup_2 = $sim6_blup_stat_2->variance();
5427
5428 my @set = ('0' ..'9', 'A' .. 'F');
5429 my @colors;
5430 for (1..scalar(@sorted_germplasm_names)) {
5431 my $str = join '' => map $set[rand @set], 1 .. 6;
5432 push @colors, '#'.$str;
5433 }
5434 my $color_string = join '\',\'', @colors;
5435
5436 my $genetic_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5437 $genetic_effects_figure_tempfile_string .= '.png';
5438 my $genetic_effects_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_figure_tempfile_string;
5439
5440 my $genetic_effects_alt_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5441 $genetic_effects_alt_figure_tempfile_string .= '.png';
5442 my $genetic_effects_alt_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_figure_tempfile_string;
5443
5444 my $genetic_effects_alt_env1_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5445 $genetic_effects_alt_env1_figure_tempfile_string .= '.png';
5446 my $genetic_effects_alt_env1_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env1_figure_tempfile_string;
5447
5448 my $genetic_effects_alt_env2_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5449 $genetic_effects_alt_env2_figure_tempfile_string .= '.png';
5450 my $genetic_effects_alt_env2_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env2_figure_tempfile_string;
5451
5452 my $genetic_effects_alt_env3_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5453 $genetic_effects_alt_env3_figure_tempfile_string .= '.png';
5454 my $genetic_effects_alt_env3_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env3_figure_tempfile_string;
5455
5456 my $genetic_effects_alt_env4_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5457 $genetic_effects_alt_env4_figure_tempfile_string .= '.png';
5458 my $genetic_effects_alt_env4_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env4_figure_tempfile_string;
5459
5460 my $genetic_effects_alt_env5_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5461 $genetic_effects_alt_env5_figure_tempfile_string .= '.png';
5462 my $genetic_effects_alt_env5_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env5_figure_tempfile_string;
5463
5464 my $genetic_effects_alt_env6_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
5465 $genetic_effects_alt_env6_figure_tempfile_string .= '.png';
5466 my $genetic_effects_alt_env6_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env6_figure_tempfile_string;
5467
5468 my $cmd_gen_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5469 mat <- fread(\''.$effects_original_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5470 mat\$time <- as.numeric(as.character(mat\$time));
5471 options(device=\'png\');
5472 par();
5473 sp <- ggplot(mat, aes(x = time, y = value)) +
5474 geom_line(aes(color = germplasmName), size = 1) +
5475 scale_fill_manual(values = c(\''.$color_string.'\')) +
5476 theme_minimal();
5477 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5478 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5479 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5480 sp <- sp + labs(title = \'Original Genetic Effects\');';
5481 if (scalar(@sorted_germplasm_names) > 100) {
5482 $cmd_gen_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5483 }
5484 $cmd_gen_plot .= 'ggsave(\''.$genetic_effects_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5485 "';
5486 print STDERR Dumper $cmd_gen_plot;
5487 my $status_gen_plot = system($cmd_gen_plot);
5488 push @$spatial_effects_plots, [$genetic_effects_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_efforigline_"."envvar_".$env_variance_percent."_".$iterations];
5489
5490 my $cmd_gen_alt_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5491 mat <- fread(\''.$effects_altered_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5492 mat\$time <- as.numeric(as.character(mat\$time));
5493 options(device=\'png\');
5494 par();
5495 sp <- ggplot(mat, aes(x = time, y = value)) +
5496 geom_line(aes(color = germplasmName), size = 1) +
5497 scale_fill_manual(values = c(\''.$color_string.'\')) +
5498 theme_minimal();
5499 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5500 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5501 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5502 sp <- sp + labs(title = \'Altered Genetic Effects\');';
5503 if (scalar(@sorted_germplasm_names) > 100) {
5504 $cmd_gen_alt_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5505 }
5506 $cmd_gen_alt_plot .= 'ggsave(\''.$genetic_effects_alt_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5507 "';
5508 print STDERR Dumper $cmd_gen_alt_plot;
5509 my $status_gen_alt_plot = system($cmd_gen_alt_plot);
5510 push @$spatial_effects_plots, [$genetic_effects_alt_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltline_"."envvar_".$env_variance_percent."_".$iterations];
5511
5512 my $cmd_gen_env1_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5513 mat <- fread(\''.$effects_altered_env1_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5514 mat\$time <- as.numeric(as.character(mat\$time));
5515 options(device=\'png\');
5516 par();
5517 sp <- ggplot(mat, aes(x = time, y = value)) +
5518 geom_line(aes(color = germplasmName), size = 1) +
5519 scale_fill_manual(values = c(\''.$color_string.'\')) +
5520 theme_minimal();
5521 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5522 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5523 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5524 sp <- sp + labs(title = \'SimLinear Genetic Effects\');';
5525 if (scalar(@sorted_germplasm_names) > 100) {
5526 $cmd_gen_env1_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5527 }
5528 $cmd_gen_env1_plot .= 'ggsave(\''.$genetic_effects_alt_env1_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5529 "';
5530 print STDERR Dumper $cmd_gen_env1_plot;
5531 my $status_gen_env1_plot = system($cmd_gen_env1_plot);
5532 push @$spatial_effects_plots, [$genetic_effects_alt_env1_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv1line_"."envvar_".$env_variance_percent."_".$iterations];
5533
5534 my $cmd_gen_env2_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5535 mat <- fread(\''.$effects_altered_env2_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5536 mat\$time <- as.numeric(as.character(mat\$time));
5537 options(device=\'png\');
5538 par();
5539 sp <- ggplot(mat, aes(x = time, y = value)) +
5540 geom_line(aes(color = germplasmName), size = 1) +
5541 scale_fill_manual(values = c(\''.$color_string.'\')) +
5542 theme_minimal();
5543 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5544 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5545 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5546 sp <- sp + labs(title = \'Sim1DN Genetic Effects\');';
5547 if (scalar(@sorted_germplasm_names) > 100) {
5548 $cmd_gen_env2_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5549 }
5550 $cmd_gen_env2_plot .= 'ggsave(\''.$genetic_effects_alt_env2_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5551 "';
5552 print STDERR Dumper $cmd_gen_env2_plot;
5553 my $status_gen_env2_plot = system($cmd_gen_env2_plot);
5554 push @$spatial_effects_plots, [$genetic_effects_alt_env2_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv2line_"."envvar_".$env_variance_percent."_".$iterations];
5555
5556 my $cmd_gen_env3_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5557 mat <- fread(\''.$effects_altered_env3_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5558 mat\$time <- as.numeric(as.character(mat\$time));
5559 options(device=\'png\');
5560 par();
5561 sp <- ggplot(mat, aes(x = time, y = value)) +
5562 geom_line(aes(color = germplasmName), size = 1) +
5563 scale_fill_manual(values = c(\''.$color_string.'\')) +
5564 theme_minimal();
5565 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5566 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5567 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5568 sp <- sp + labs(title = \'Sim2DN Genetic Effects\');';
5569 if (scalar(@sorted_germplasm_names) > 100) {
5570 $cmd_gen_env3_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5571 }
5572 $cmd_gen_env3_plot .= 'ggsave(\''.$genetic_effects_alt_env3_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5573 "';
5574 print STDERR Dumper $cmd_gen_env3_plot;
5575 my $status_gen_env3_plot = system($cmd_gen_env3_plot);
5576 push @$spatial_effects_plots, [$genetic_effects_alt_env3_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv3line_"."envvar_".$env_variance_percent."_".$iterations];
5577
5578 my $cmd_gen_env4_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5579 mat <- fread(\''.$effects_altered_env4_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5580 mat\$time <- as.numeric(as.character(mat\$time));
5581 options(device=\'png\');
5582 par();
5583 sp <- ggplot(mat, aes(x = time, y = value)) +
5584 geom_line(aes(color = germplasmName), size = 1) +
5585 scale_fill_manual(values = c(\''.$color_string.'\')) +
5586 theme_minimal();
5587 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5588 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5589 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5590 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
5591 if (scalar(@sorted_germplasm_names) > 100) {
5592 $cmd_gen_env4_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5593 }
5594 $cmd_gen_env4_plot .= 'ggsave(\''.$genetic_effects_alt_env4_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5595 "';
5596 print STDERR Dumper $cmd_gen_env4_plot;
5597 my $status_gen_env4_plot = system($cmd_gen_env4_plot);
5598 push @$spatial_effects_plots, [$genetic_effects_alt_env4_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv4line_"."envvar_".$env_variance_percent."_".$iterations];
5599
5600 my $cmd_gen_env5_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5601 mat <- fread(\''.$effects_altered_env5_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5602 mat\$time <- as.numeric(as.character(mat\$time));
5603 options(device=\'png\');
5604 par();
5605 sp <- ggplot(mat, aes(x = time, y = value)) +
5606 geom_line(aes(color = germplasmName), size = 1) +
5607 scale_fill_manual(values = c(\''.$color_string.'\')) +
5608 theme_minimal();
5609 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5610 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5611 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5612 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
5613 if (scalar(@sorted_germplasm_names) > 100) {
5614 $cmd_gen_env5_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5615 }
5616 $cmd_gen_env5_plot .= 'ggsave(\''.$genetic_effects_alt_env5_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5617 "';
5618 print STDERR Dumper $cmd_gen_env5_plot;
5619 my $status_gen_env5_plot = system($cmd_gen_env5_plot);
5620 push @$spatial_effects_plots, [$genetic_effects_alt_env5_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv5line_"."envvar_".$env_variance_percent."_".$iterations];
5621
5622 my $cmd_gen_env6_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
5623 mat <- fread(\''.$effects_altered_env6_line_chart_tempfile_2.'\', header=TRUE, sep=\',\');
5624 mat\$time <- as.numeric(as.character(mat\$time));
5625 options(device=\'png\');
5626 par();
5627 sp <- ggplot(mat, aes(x = time, y = value)) +
5628 geom_line(aes(color = germplasmName), size = 1) +
5629 scale_fill_manual(values = c(\''.$color_string.'\')) +
5630 theme_minimal();
5631 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
5632 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
5633 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
5634 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
5635 if (scalar(@sorted_germplasm_names) > 100) {
5636 $cmd_gen_env6_plot .= 'sp <- sp + theme(legend.position = \'none\');';
5637 }
5638 $cmd_gen_env6_plot .= 'ggsave(\''.$genetic_effects_alt_env6_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
5639 "';
5640 print STDERR Dumper $cmd_gen_env6_plot;
5641 my $status_gen_env6_plot = system($cmd_gen_env6_plot);
5642 push @$spatial_effects_plots, [$genetic_effects_alt_env6_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv6line_"."envvar_".$env_variance_percent."_".$iterations];
5643 };
5644
5645 %trait_name_encoder = %trait_name_encoder_2;
5646 %trait_to_time_map = %trait_to_time_map_2;
5647
5648 push @$env_varcomps, {
5649 type => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times : Iteration $iterations",
5650 statistics_select => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times",
5651 correlation_between_times => $correlation_between_times,
5652 env_variance => $env_variance_percent,
5653 original => {
5654 covariance => $varcomp_original_array_2,
5655 cv_1 => $result_cv_original_2,
5656 cv_2 => $result_cv_2_original_2
5657 },
5658 altered => {
5659 covariance => $varcomp_altered_array_2,
5660 cv_1 => $result_cv_altered_2,
5661 cv_2 => $result_cv_2_altered_2
5662 },
5663 env_linear => {
5664 covariance => $varcomp_altered_array_env_1_2,
5665 cv_1 => $result_cv_altered_env_1_2,
5666 cv_2 => $result_cv_2_altered_env_1_2
5667 },
5668 env_1DN => {
5669 covariance => $varcomp_altered_array_env_2_2,
5670 cv_1 => $result_cv_altered_env_2_2,
5671 cv_2 => $result_cv_2_altered_env_2_2
5672 },
5673 env_2DN => {
5674 covariance => $varcomp_altered_array_env_3_2,
5675 cv_1 => $result_cv_altered_env_3_2,
5676 cv_2 => $result_cv_2_altered_env_3_2
5677 },
5678 env_random => {
5679 covariance => $varcomp_altered_array_env_4_2,
5680 cv_1 => $result_cv_altered_env_4_2,
5681 cv_2 => $result_cv_2_altered_env_4_2
5682 },
5683 env_ar1xar1 => {
5684 covariance => $varcomp_altered_array_env_5_2,
5685 cv_1 => $result_cv_altered_env_5_2,
5686 cv_2 => $result_cv_2_altered_env_5_2
5687 },
5688 env_realdata => {
5689 covariance => $varcomp_altered_array_env_6_2,
5690 cv_1 => $result_cv_altered_env_6_2,
5691 cv_2 => $result_cv_2_altered_env_6_2
5692 }
5693 };
5694 }
5695
5696 if ($statistics_select_original eq 'sommer_grm_univariate_spatial_genetic_blups' || $statistics_select_original eq 'sommer_grm_univariate_spatial_pure_2dspl_genetic_blups') {
5697 $statistics_select = $statistics_select_original eq 'sommer_grm_univariate_spatial_genetic_blups' ? 'sommer_grm_univariate_spatial_genetic_blups' : 'sommer_grm_univariate_spatial_pure_2dspl_genetic_blups';
5698
5699 print STDERR "PREPARE ORIGINAL PHENOTYPE FILES 2\n";
5700 eval {
5701 my $phenotypes_search_2 = CXGN::Phenotypes::SearchFactory->instantiate(
5702 'MaterializedViewTable',
5703 {
5704 bcs_schema=>$schema,
5705 data_level=>'plot',
5706 trait_list=>$trait_id_list,
5707 trial_list=>$field_trial_id_list,
5708 include_timestamp=>0,
5709 exclude_phenotype_outlier=>0
5710 }
5711 );
5712 my ($data_2, $unique_traits_2) = $phenotypes_search_2->search();
5713 @sorted_trait_names_2 = sort keys %$unique_traits_2;
5714
5715 if (scalar(@$data_2) == 0) {
5716 $c->stash->{rest} = { error => "There are no phenotypes for the trials and traits you have selected!"};
5717 return;
5718 }
5719
5720 foreach my $obs_unit (@$data_2){
5721 my $germplasm_name = $obs_unit->{germplasm_uniquename};
5722 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
5723 my $replicate_number = $obs_unit->{obsunit_rep} || '';
5724 my $block_number = $obs_unit->{obsunit_block} || '';
5725 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
5726 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
5727 my $row_number = $obs_unit->{obsunit_row_number} || '';
5728 my $col_number = $obs_unit->{obsunit_col_number} || '';
5729 $seen_trial_ids_2{$obs_unit->{trial_id}}++;
5730 push @plot_ids_ordered, $obsunit_stock_id;
5731
5732 if ($row_number < $min_row) {
5733 $min_row = $row_number;
5734 }
5735 elsif ($row_number >= $max_row) {
5736 $max_row = $row_number;
5737 }
5738 if ($col_number < $min_col) {
5739 $min_col = $col_number;
5740 }
5741 elsif ($col_number >= $max_col) {
5742 $max_col = $col_number;
5743 }
5744
5745 $obsunit_row_col{$row_number}->{$col_number} = {
5746 stock_id => $obsunit_stock_id,
5747 stock_uniquename => $obsunit_stock_uniquename
5748 };
5749 $seen_rows{$row_number}++;
5750 $seen_cols{$col_number}++;
5751 $plot_id_map{$obsunit_stock_id} = $obsunit_stock_uniquename;
5752 $seen_plot_names{$obsunit_stock_uniquename}++;
5753 $seen_plots{$obsunit_stock_id} = $obsunit_stock_uniquename;
5754 $stock_row_col{$obsunit_stock_id} = {
5755 row_number => $row_number,
5756 col_number => $col_number,
5757 obsunit_stock_id => $obsunit_stock_id,
5758 obsunit_name => $obsunit_stock_uniquename,
5759 rep => $replicate_number,
5760 block => $block_number,
5761 germplasm_stock_id => $germplasm_stock_id,
5762 germplasm_name => $germplasm_name
5763 };
5764 $stock_name_row_col{$obsunit_stock_uniquename} = {
5765 row_number => $row_number,
5766 col_number => $col_number,
5767 obsunit_stock_id => $obsunit_stock_id,
5768 obsunit_name => $obsunit_stock_uniquename,
5769 rep => $replicate_number,
5770 block => $block_number,
5771 germplasm_stock_id => $germplasm_stock_id,
5772 germplasm_name => $germplasm_name
5773 };
5774 $stock_row_col_id{$row_number}->{$col_number} = $obsunit_stock_id;
5775 $unique_accessions{$germplasm_name}++;
5776 $stock_info_2{"S".$germplasm_stock_id} = {
5777 uniquename => $germplasm_name
5778 };
5779 my $observations = $obs_unit->{observations};
5780 foreach (@$observations){
5781 my $value = $_->{value};
5782 my $trait_name = $_->{trait_name};
5783 $phenotype_data_original_2{$obsunit_stock_uniquename}->{$trait_name} = $value;
5784 $seen_trait_names_2{$trait_name}++;
5785
5786 if ($value < $phenotype_min_original_2) {
5787 $phenotype_min_original_2 = $value;
5788 }
5789 elsif ($value >= $phenotype_max_original_2) {
5790 $phenotype_max_original_2 = $value;
5791 }
5792
5793 if ($_->{associated_image_project_time_json}) {
5794 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
5795 my $time_days_cvterm = $related_time_terms_json->{day};
5796 my $time_term_string = $time_days_cvterm;
5797 my $time_days = (split '\|', $time_days_cvterm)[0];
5798 my $time_value = (split ' ', $time_days)[1];
5799 $seen_days_after_plantings_2{$time_value}++;
5800 $trait_to_time_map_2{$trait_name} = $time_value;
5801 }
5802 }
5803 }
5804
5805 @unique_plot_names = sort keys %seen_plot_names;
5806
5807 my $trait_name_encoded_2 = 1;
5808 foreach my $trait_name (@sorted_trait_names_2) {
5809 if (!exists($trait_name_encoder_2{$trait_name})) {
5810 my $trait_name_e = 't'.$trait_name_encoded_2;
5811 $trait_name_encoder_2{$trait_name} = $trait_name_e;
5812 $trait_name_encoder_rev_2{$trait_name_e} = $trait_name;
5813 $trait_name_encoded_2++;
5814 }
5815 }
5816
5817 foreach my $p (@unique_plot_names) {
5818 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
5819 if ($fixed_effect_type eq 'fixed_effect_trait') {
5820 $stock_name_row_col{$p}->{rep} = defined($fixed_effect_trait_data->{$obsunit_stock_id}) ? $fixed_effect_trait_data->{$obsunit_stock_id} : 0;
5821 }
5822 my $row_number = $stock_name_row_col{$p}->{row_number};
5823 my $col_number = $stock_name_row_col{$p}->{col_number};
5824 my $replicate = $stock_name_row_col{$p}->{rep};
5825 my $block = $stock_name_row_col{$p}->{block};
5826 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
5827 my $germplasm_name = $stock_name_row_col{$p}->{germplasm_name};
5828
5829 my @row = ($replicate, $block, "S".$germplasm_stock_id, $obsunit_stock_id, $row_number, $col_number, $row_number, $col_number);
5830
5831 foreach my $t (@sorted_trait_names_2) {
5832 if (defined($phenotype_data_original_2{$p}->{$t})) {
5833 push @row, $phenotype_data_original_2{$p}->{$t};
5834 } else {
5835 print STDERR $p." : $t : $germplasm_name : NA \n";
5836 push @row, 'NA';
5837 }
5838 }
5839 push @data_matrix_original_2, \@row;
5840 }
5841
5842 foreach (keys %seen_trial_ids_2){
5843 my $trial = CXGN::Trial->new({bcs_schema=>$schema, trial_id=>$_});
5844 my $traits_assayed = $trial->get_traits_assayed('plot', undef, 'time_ontology');
5845 foreach (@$traits_assayed) {
5846 $unique_traits_ids_2{$_->[0]} = $_;
5847 }
5848 }
5849 foreach (values %unique_traits_ids_2) {
5850 foreach my $component (@{$_->[2]}) {
5851 if (exists($seen_trait_names_2{$_->[1]}) && $component->{cv_type} && $component->{cv_type} eq 'time_ontology') {
5852 my $time_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $component->{cvterm_id}, 'extended');
5853 push @{$trait_composing_info_2{$_->[1]}}, $time_term_string;
5854 }
5855 }
5856 }
5857
5858 @phenotype_header_2 = ("replicate", "block", "id", "plot_id", "rowNumber", "colNumber", "rowNumberFactor", "colNumberFactor");
5859 foreach (@sorted_trait_names_2) {
5860 push @phenotype_header_2, $trait_name_encoder_2{$_};
5861 }
5862 $header_string_2 = join ',', @phenotype_header_2;
5863
5864 open($F, ">", $stats_tempfile) || die "Can't open file ".$stats_tempfile;
5865 print $F $header_string_2."\n";
5866 foreach (@data_matrix_original_2) {
5867 my $line = join ',', @$_;
5868 print $F "$line\n";
5869 }
5870 close($F);
5871
5872 print STDERR Dumper [$phenotype_min_original_2, $phenotype_max_original_2];
5873 };
5874
5875 @seen_rows_array = keys %seen_rows;
5876 @seen_cols_array = keys %seen_cols;
5877 $row_stat = Statistics::Descriptive::Full->new();
5878 $row_stat->add_data(@seen_rows_array);
5879 $mean_row = $row_stat->mean();
5880 $sig_row = $row_stat->variance();
5881 $col_stat = Statistics::Descriptive::Full->new();
5882 $col_stat->add_data(@seen_cols_array);
5883 $mean_col = $col_stat->mean();
5884 $sig_col = $col_stat->variance();
5885
5886 print STDERR "PREPARE RELATIONSHIP MATRIX\n";
5887 eval {
5888 my %seen_accession_stock_ids;
5889 foreach my $trial_id (@$field_trial_id_list) {
5890 my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id });
5891 my $accessions = $trial->get_accessions();
5892 foreach (@$accessions) {
5893 $seen_accession_stock_ids{$_->{stock_id}}++;
5894 }
5895 }
5896 my @accession_ids = keys %seen_accession_stock_ids;
5897
5898 if ($compute_relationship_matrix_from_htp_phenotypes eq 'genotypes') {
5899
5900 if ($include_pedgiree_info_if_compute_from_parents) {
5901 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
5902 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
5903 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
5904 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
5905 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
5906 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
5907 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
5908 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
5909
5910 if (!$protocol_id) {
5911 $protocol_id = undef;
5912 }
5913
5914 my $pedigree_arm = CXGN::Pedigree::ARM->new({
5915 bcs_schema=>$schema,
5916 arm_temp_file=>$arm_tempfile,
5917 people_schema=>$people_schema,
5918 accession_id_list=>\@accession_ids,
5919 # plot_id_list=>\@plot_id_list,
5920 cache_root=>$c->config->{cache_file_path},
5921 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
5922 });
5923 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
5924 $shared_cluster_dir_config,
5925 $c->config->{backend},
5926 $c->config->{cluster_host},
5927 $c->config->{'web_cluster_queue'},
5928 $c->config->{basepath}
5929 );
5930 # print STDERR Dumper $parent_hash;
5931
5932 my $female_geno = CXGN::Genotype::GRM->new({
5933 bcs_schema=>$schema,
5934 grm_temp_file=>$grm1_tempfile,
5935 people_schema=>$people_schema,
5936 cache_root=>$c->config->{cache_file_path},
5937 accession_id_list=>$female_stock_ids,
5938 protocol_id=>$protocol_id,
5939 get_grm_for_parental_accessions=>0,
5940 download_format=>'three_column_reciprocal'
5941 # minor_allele_frequency=>$minor_allele_frequency,
5942 # marker_filter=>$marker_filter,
5943 # individuals_filter=>$individuals_filter
5944 });
5945 my $female_grm_data = $female_geno->download_grm(
5946 'data',
5947 $shared_cluster_dir_config,
5948 $c->config->{backend},
5949 $c->config->{cluster_host},
5950 $c->config->{'web_cluster_queue'},
5951 $c->config->{basepath}
5952 );
5953 my @fl = split '\n', $female_grm_data;
5954 my %female_parent_grm;
5955 foreach (@fl) {
5956 my @l = split '\t', $_;
5957 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
5958 }
5959 # print STDERR Dumper \%female_parent_grm;
5960
5961 my $male_geno = CXGN::Genotype::GRM->new({
5962 bcs_schema=>$schema,
5963 grm_temp_file=>$grm1_tempfile,
5964 people_schema=>$people_schema,
5965 cache_root=>$c->config->{cache_file_path},
5966 accession_id_list=>$male_stock_ids,
5967 protocol_id=>$protocol_id,
5968 get_grm_for_parental_accessions=>0,
5969 download_format=>'three_column_reciprocal'
5970 # minor_allele_frequency=>$minor_allele_frequency,
5971 # marker_filter=>$marker_filter,
5972 # individuals_filter=>$individuals_filter
5973 });
5974 my $male_grm_data = $male_geno->download_grm(
5975 'data',
5976 $shared_cluster_dir_config,
5977 $c->config->{backend},
5978 $c->config->{cluster_host},
5979 $c->config->{'web_cluster_queue'},
5980 $c->config->{basepath}
5981 );
5982 my @ml = split '\n', $male_grm_data;
5983 my %male_parent_grm;
5984 foreach (@ml) {
5985 my @l = split '\t', $_;
5986 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
5987 }
5988 # print STDERR Dumper \%male_parent_grm;
5989
5990 my %rel_result_hash;
5991 foreach my $a1 (@accession_ids) {
5992 foreach my $a2 (@accession_ids) {
5993 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
5994 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
5995 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
5996 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
5997
5998 my $female_rel = 0;
5999 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
6000 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
6001 }
6002 elsif ($female_parent1 && $female_parent2 && $female_parent1 == $female_parent2) {
6003 $female_rel = 1;
6004 }
6005 elsif ($a1 == $a2) {
6006 $female_rel = 1;
6007 }
6008
6009 my $male_rel = 0;
6010 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
6011 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
6012 }
6013 elsif ($male_parent1 && $male_parent2 && $male_parent1 == $male_parent2) {
6014 $male_rel = 1;
6015 }
6016 elsif ($a1 == $a2) {
6017 $male_rel = 1;
6018 }
6019 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
6020
6021 my $rel = 0.5*($female_rel + $male_rel);
6022 $rel_result_hash{$a1}->{$a2} = $rel;
6023 }
6024 }
6025 # print STDERR Dumper \%rel_result_hash;
6026
6027 my $data = '';
6028 my %result_hash;
6029 foreach my $s (sort @accession_ids) {
6030 foreach my $c (sort @accession_ids) {
6031 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6032 my $val = $rel_result_hash{$s}->{$c};
6033 if (defined $val and length $val) {
6034 $result_hash{$s}->{$c} = $val;
6035 $data .= "S$s\tS$c\t$val\n";
6036 }
6037 }
6038 }
6039 }
6040
6041 # print STDERR Dumper $data;
6042 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
6043 print $F2 $data;
6044 close($F2);
6045
6046 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
6047 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
6048 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
6049 A_1 <- A_wide[,-1];
6050 A_1[is.na(A_1)] <- 0;
6051 A <- A_1 + t(A_1);
6052 diag(A) <- diag(as.matrix(A_1));
6053 E = eigen(A);
6054 ev = E\$values;
6055 U = E\$vectors;
6056 no = dim(A)[1];
6057 nev = which(ev < 0);
6058 wr = 0;
6059 k=length(nev);
6060 if(k > 0){
6061 p = ev[no - k];
6062 B = sum(ev[nev])*2.0;
6063 wr = (B*B*100.0)+1;
6064 val = ev[nev];
6065 ev[nev] = p*(B-val)*(B-val)/wr;
6066 A = U%*%diag(ev)%*%t(U);
6067 }
6068 A <- as.data.frame(A);
6069 colnames(A) <- A_wide[,1];
6070 A\$stock_id <- A_wide[,1];
6071 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
6072 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
6073 A_threecol\$variable <- substring(A_threecol\$variable, 2);
6074 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
6075 print STDERR $cmd."\n";
6076 my $status = system($cmd);
6077
6078 my %rel_pos_def_result_hash;
6079 open(my $F3, '<', $grm_out_tempfile)
6080 or die "Could not open file '$grm_out_tempfile' $!";
6081
6082 print STDERR "Opened $grm_out_tempfile\n";
6083
6084 while (my $row = <$F3>) {
6085 my @columns;
6086 if ($csv->parse($row)) {
6087 @columns = $csv->fields();
6088 }
6089 my $stock_id1 = $columns[0];
6090 my $stock_id2 = $columns[1];
6091 my $val = $columns[2];
6092 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
6093 }
6094 close($F3);
6095
6096 my $data_pos_def = '';
6097 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
6098 my %result_hash;
6099 foreach my $s (sort @accession_ids) {
6100 foreach my $c (sort @accession_ids) {
6101 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6102 my $val = $rel_pos_def_result_hash{$s}->{$c};
6103 if (defined $val and length $val) {
6104 $result_hash{$s}->{$c} = $val;
6105 $data_pos_def .= "$s\t$c\t$val\n";
6106 }
6107 }
6108 }
6109 }
6110 }
6111 else {
6112 my %result_hash;
6113 foreach my $s (sort @accession_ids) {
6114 foreach my $c (sort @accession_ids) {
6115 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6116 my $val = $rel_pos_def_result_hash{$s}->{$c};
6117 if (defined $val and length $val) {
6118 $result_hash{$s}->{$c} = $val;
6119 $result_hash{$c}->{$s} = $val;
6120 $data_pos_def .= "S$s\tS$c\t$val\n";
6121 if ($s != $c) {
6122 $data_pos_def .= "S$c\tS$s\t$val\n";
6123 }
6124 }
6125 }
6126 }
6127 }
6128 }
6129
6130 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
6131 print $F4 $data_pos_def;
6132 close($F4);
6133
6134 $grm_file = $grm_out_posdef_tempfile;
6135 }
6136 elsif ($use_parental_grms_if_compute_from_parents) {
6137 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
6138 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
6139 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
6140 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
6141 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
6142 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
6143 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
6144 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
6145
6146 if (!$protocol_id) {
6147 $protocol_id = undef;
6148 }
6149
6150 my $pedigree_arm = CXGN::Pedigree::ARM->new({
6151 bcs_schema=>$schema,
6152 arm_temp_file=>$arm_tempfile,
6153 people_schema=>$people_schema,
6154 accession_id_list=>\@accession_ids,
6155 # plot_id_list=>\@plot_id_list,
6156 cache_root=>$c->config->{cache_file_path},
6157 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
6158 });
6159 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
6160 $shared_cluster_dir_config,
6161 $c->config->{backend},
6162 $c->config->{cluster_host},
6163 $c->config->{'web_cluster_queue'},
6164 $c->config->{basepath}
6165 );
6166 # print STDERR Dumper $parent_hash;
6167
6168 my $female_geno = CXGN::Genotype::GRM->new({
6169 bcs_schema=>$schema,
6170 grm_temp_file=>$grm1_tempfile,
6171 people_schema=>$people_schema,
6172 cache_root=>$c->config->{cache_file_path},
6173 accession_id_list=>$female_stock_ids,
6174 protocol_id=>$protocol_id,
6175 get_grm_for_parental_accessions=>0,
6176 download_format=>'three_column_reciprocal'
6177 # minor_allele_frequency=>$minor_allele_frequency,
6178 # marker_filter=>$marker_filter,
6179 # individuals_filter=>$individuals_filter
6180 });
6181 my $female_grm_data = $female_geno->download_grm(
6182 'data',
6183 $shared_cluster_dir_config,
6184 $c->config->{backend},
6185 $c->config->{cluster_host},
6186 $c->config->{'web_cluster_queue'},
6187 $c->config->{basepath}
6188 );
6189 my @fl = split '\n', $female_grm_data;
6190 my %female_parent_grm;
6191 foreach (@fl) {
6192 my @l = split '\t', $_;
6193 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
6194 }
6195 # print STDERR Dumper \%female_parent_grm;
6196
6197 my $male_geno = CXGN::Genotype::GRM->new({
6198 bcs_schema=>$schema,
6199 grm_temp_file=>$grm1_tempfile,
6200 people_schema=>$people_schema,
6201 cache_root=>$c->config->{cache_file_path},
6202 accession_id_list=>$male_stock_ids,
6203 protocol_id=>$protocol_id,
6204 get_grm_for_parental_accessions=>0,
6205 download_format=>'three_column_reciprocal'
6206 # minor_allele_frequency=>$minor_allele_frequency,
6207 # marker_filter=>$marker_filter,
6208 # individuals_filter=>$individuals_filter
6209 });
6210 my $male_grm_data = $male_geno->download_grm(
6211 'data',
6212 $shared_cluster_dir_config,
6213 $c->config->{backend},
6214 $c->config->{cluster_host},
6215 $c->config->{'web_cluster_queue'},
6216 $c->config->{basepath}
6217 );
6218 my @ml = split '\n', $male_grm_data;
6219 my %male_parent_grm;
6220 foreach (@ml) {
6221 my @l = split '\t', $_;
6222 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
6223 }
6224 # print STDERR Dumper \%male_parent_grm;
6225
6226 my %rel_result_hash;
6227 foreach my $a1 (@accession_ids) {
6228 foreach my $a2 (@accession_ids) {
6229 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
6230 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
6231 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
6232 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
6233
6234 my $female_rel = 0;
6235 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
6236 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
6237 }
6238 elsif ($a1 == $a2) {
6239 $female_rel = 1;
6240 }
6241
6242 my $male_rel = 0;
6243 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
6244 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
6245 }
6246 elsif ($a1 == $a2) {
6247 $male_rel = 1;
6248 }
6249 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
6250
6251 my $rel = 0.5*($female_rel + $male_rel);
6252 $rel_result_hash{$a1}->{$a2} = $rel;
6253 }
6254 }
6255 # print STDERR Dumper \%rel_result_hash;
6256
6257 my $data = '';
6258 my %result_hash;
6259 foreach my $s (sort @accession_ids) {
6260 foreach my $c (sort @accession_ids) {
6261 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6262 my $val = $rel_result_hash{$s}->{$c};
6263 if (defined $val and length $val) {
6264 $result_hash{$s}->{$c} = $val;
6265 $data .= "S$s\tS$c\t$val\n";
6266 }
6267 }
6268 }
6269 }
6270
6271 # print STDERR Dumper $data;
6272 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
6273 print $F2 $data;
6274 close($F2);
6275
6276 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
6277 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
6278 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
6279 A_1 <- A_wide[,-1];
6280 A_1[is.na(A_1)] <- 0;
6281 A <- A_1 + t(A_1);
6282 diag(A) <- diag(as.matrix(A_1));
6283 E = eigen(A);
6284 ev = E\$values;
6285 U = E\$vectors;
6286 no = dim(A)[1];
6287 nev = which(ev < 0);
6288 wr = 0;
6289 k=length(nev);
6290 if(k > 0){
6291 p = ev[no - k];
6292 B = sum(ev[nev])*2.0;
6293 wr = (B*B*100.0)+1;
6294 val = ev[nev];
6295 ev[nev] = p*(B-val)*(B-val)/wr;
6296 A = U%*%diag(ev)%*%t(U);
6297 }
6298 A <- as.data.frame(A);
6299 colnames(A) <- A_wide[,1];
6300 A\$stock_id <- A_wide[,1];
6301 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
6302 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
6303 A_threecol\$variable <- substring(A_threecol\$variable, 2);
6304 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
6305 print STDERR $cmd."\n";
6306 my $status = system($cmd);
6307
6308 my %rel_pos_def_result_hash;
6309 open(my $F3, '<', $grm_out_tempfile)
6310 or die "Could not open file '$grm_out_tempfile' $!";
6311
6312 print STDERR "Opened $grm_out_tempfile\n";
6313
6314 while (my $row = <$F3>) {
6315 my @columns;
6316 if ($csv->parse($row)) {
6317 @columns = $csv->fields();
6318 }
6319 my $stock_id1 = $columns[0];
6320 my $stock_id2 = $columns[1];
6321 my $val = $columns[2];
6322 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
6323 }
6324 close($F3);
6325
6326 my $data_pos_def = '';
6327 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
6328 my %result_hash;
6329 foreach my $s (sort @accession_ids) {
6330 foreach my $c (sort @accession_ids) {
6331 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6332 my $val = $rel_pos_def_result_hash{$s}->{$c};
6333 if (defined $val and length $val) {
6334 $result_hash{$s}->{$c} = $val;
6335 $data_pos_def .= "$s\t$c\t$val\n";
6336 }
6337 }
6338 }
6339 }
6340 }
6341 else {
6342 my %result_hash;
6343 foreach my $s (sort @accession_ids) {
6344 foreach my $c (sort @accession_ids) {
6345 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6346 my $val = $rel_pos_def_result_hash{$s}->{$c};
6347 if (defined $val and length $val) {
6348 $result_hash{$s}->{$c} = $val;
6349 $result_hash{$c}->{$s} = $val;
6350 $data_pos_def .= "S$s\tS$c\t$val\n";
6351 if ($s != $c) {
6352 $data_pos_def .= "S$c\tS$s\t$val\n";
6353 }
6354 }
6355 }
6356 }
6357 }
6358 }
6359
6360 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
6361 print $F4 $data_pos_def;
6362 close($F4);
6363
6364 $grm_file = $grm_out_posdef_tempfile;
6365 }
6366 else {
6367 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
6368 my $tmp_grm_dir = $shared_cluster_dir_config."/tmp_genotype_download_grm";
6369 mkdir $tmp_grm_dir if ! -d $tmp_grm_dir;
6370 my ($grm_tempfile_fh, $grm_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
6371 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
6372
6373 if (!$protocol_id) {
6374 $protocol_id = undef;
6375 }
6376
6377 my $grm_search_params = {
6378 bcs_schema=>$schema,
6379 grm_temp_file=>$grm_tempfile,
6380 people_schema=>$people_schema,
6381 cache_root=>$c->config->{cache_file_path},
6382 accession_id_list=>\@accession_ids,
6383 protocol_id=>$protocol_id,
6384 get_grm_for_parental_accessions=>$compute_from_parents,
6385 # minor_allele_frequency=>$minor_allele_frequency,
6386 # marker_filter=>$marker_filter,
6387 # individuals_filter=>$individuals_filter
6388 };
6389
6390 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
6391 $grm_search_params->{download_format} = 'three_column_stock_id_integer';
6392 }
6393 else {
6394 $grm_search_params->{download_format} = 'three_column_reciprocal';
6395 }
6396
6397 my $geno = CXGN::Genotype::GRM->new($grm_search_params);
6398 my $grm_data = $geno->download_grm(
6399 'data',
6400 $shared_cluster_dir_config,
6401 $c->config->{backend},
6402 $c->config->{cluster_host},
6403 $c->config->{'web_cluster_queue'},
6404 $c->config->{basepath}
6405 );
6406
6407 open(my $F2, ">", $grm_out_tempfile) || die "Can't open file ".$grm_out_tempfile;
6408 print $F2 $grm_data;
6409 close($F2);
6410 $grm_file = $grm_out_tempfile;
6411 }
6412
6413 }
6414 elsif ($compute_relationship_matrix_from_htp_phenotypes eq 'htp_phenotypes') {
6415
6416 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
6417 'MaterializedViewTable',
6418 {
6419 bcs_schema=>$schema,
6420 data_level=>'plot',
6421 trial_list=>$field_trial_id_list,
6422 include_timestamp=>0,
6423 exclude_phenotype_outlier=>0
6424 }
6425 );
6426 my ($data, $unique_traits) = $phenotypes_search->search();
6427
6428 if (scalar(@$data) == 0) {
6429 $c->stash->{rest} = { error => "There are no phenotypes for the trial you have selected!"};
6430 return;
6431 }
6432
6433 my $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
6434 my $h_time = $schema->storage->dbh()->prepare($q_time);
6435
6436 my %seen_plot_names_htp_rel;
6437 my %phenotype_data_htp_rel;
6438 my %seen_times_htp_rel;
6439 foreach my $obs_unit (@$data){
6440 my $germplasm_name = $obs_unit->{germplasm_uniquename};
6441 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
6442 my $row_number = $obs_unit->{obsunit_row_number} || '';
6443 my $col_number = $obs_unit->{obsunit_col_number} || '';
6444 my $rep = $obs_unit->{obsunit_rep};
6445 my $block = $obs_unit->{obsunit_block};
6446 $seen_plot_names_htp_rel{$obs_unit->{observationunit_uniquename}} = $obs_unit;
6447 my $observations = $obs_unit->{observations};
6448 foreach (@$observations){
6449 if ($_->{associated_image_project_time_json}) {
6450 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
6451
6452 my $time_days_cvterm = $related_time_terms_json->{day};
6453 my $time_days_term_string = $time_days_cvterm;
6454 my $time_days = (split '\|', $time_days_cvterm)[0];
6455 my $time_days_value = (split ' ', $time_days)[1];
6456
6457 my $time_gdd_value = $related_time_terms_json->{gdd_average_temp} + 0;
6458 my $gdd_term_string = "GDD $time_gdd_value";
6459 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
6460 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
6461 if (!$gdd_cvterm_id) {
6462 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
6463 name => $gdd_term_string,
6464 cv => 'cxgn_time_ontology'
6465 });
6466 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
6467 }
6468 my $time_gdd_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
6469
6470 $phenotype_data_htp_rel{$obs_unit->{observationunit_uniquename}}->{$_->{trait_name}} = $_->{value};
6471 $seen_times_htp_rel{$_->{trait_name}} = [$time_days_value, $time_days_term_string, $time_gdd_value, $time_gdd_term_string];
6472 }
6473 }
6474 }
6475
6476 my @allowed_standard_htp_values = ('Nonzero Pixel Count', 'Total Pixel Sum', 'Mean Pixel Value', 'Harmonic Mean Pixel Value', 'Median Pixel Value', 'Pixel Variance', 'Pixel Standard Deviation', 'Pixel Population Standard Deviation', 'Minimum Pixel Value', 'Maximum Pixel Value', 'Minority Pixel Value', 'Minority Pixel Count', 'Majority Pixel Value', 'Majority Pixel Count', 'Pixel Group Count');
6477 my %filtered_seen_times_htp_rel;
6478 while (my ($t, $time) = each %seen_times_htp_rel) {
6479 my $allowed = 0;
6480 foreach (@allowed_standard_htp_values) {
6481 if (index($t, $_) != -1) {
6482 $allowed = 1;
6483 last;
6484 }
6485 }
6486 if ($allowed) {
6487 $filtered_seen_times_htp_rel{$t} = $time;
6488 }
6489 }
6490
6491 my @seen_plot_names_htp_rel_sorted = sort keys %seen_plot_names_htp_rel;
6492 my @filtered_seen_times_htp_rel_sorted = sort keys %filtered_seen_times_htp_rel;
6493
6494 my @header_htp = ('plot_id', 'plot_name', 'accession_id', 'accession_name', 'rep', 'block');
6495
6496 my %trait_name_encoder_htp;
6497 my %trait_name_encoder_rev_htp;
6498 my $trait_name_encoded_htp = 1;
6499 my @header_traits_htp;
6500 foreach my $trait_name (@filtered_seen_times_htp_rel_sorted) {
6501 if (!exists($trait_name_encoder_htp{$trait_name})) {
6502 my $trait_name_e = 't'.$trait_name_encoded_htp;
6503 $trait_name_encoder_htp{$trait_name} = $trait_name_e;
6504 $trait_name_encoder_rev_htp{$trait_name_e} = $trait_name;
6505 push @header_traits_htp, $trait_name_e;
6506 $trait_name_encoded_htp++;
6507 }
6508 }
6509
6510 my @htp_pheno_matrix;
6511 if ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'all') {
6512 push @header_htp, @header_traits_htp;
6513 push @htp_pheno_matrix, \@header_htp;
6514
6515 foreach my $p (@seen_plot_names_htp_rel_sorted) {
6516 my $obj = $seen_plot_names_htp_rel{$p};
6517 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
6518 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
6519 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
6520 push @row, $val;
6521 }
6522 push @htp_pheno_matrix, \@row;
6523 }
6524 }
6525 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'latest_trait') {
6526 my $max_day = 0;
6527 foreach (keys %seen_days_after_plantings) {
6528 if ($_ + 0 > $max_day) {
6529 $max_day = $_;
6530 }
6531 }
6532
6533 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
6534 my $day = $filtered_seen_times_htp_rel{$t}->[0];
6535 if ($day <= $max_day) {
6536 push @header_htp, $t;
6537 }
6538 }
6539 push @htp_pheno_matrix, \@header_htp;
6540
6541 foreach my $p (@seen_plot_names_htp_rel_sorted) {
6542 my $obj = $seen_plot_names_htp_rel{$p};
6543 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
6544 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
6545 my $day = $filtered_seen_times_htp_rel{$t}->[0];
6546 if ($day <= $max_day) {
6547 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
6548 push @row, $val;
6549 }
6550 }
6551 push @htp_pheno_matrix, \@row;
6552 }
6553 }
6554 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'vegetative') {
6555
6556 }
6557 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'reproductive') {
6558
6559 }
6560 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'mature') {
6561
6562 }
6563 else {
6564 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_time_points htp_pheno_rel_matrix_time_points is not valid!" };
6565 return;
6566 }
6567
6568 open(my $htp_pheno_f, ">", $stats_out_htp_rel_tempfile_input) || die "Can't open file ".$stats_out_htp_rel_tempfile_input;
6569 foreach (@htp_pheno_matrix) {
6570 my $line = join "\t", @$_;
6571 print $htp_pheno_f $line."\n";
6572 }
6573 close($htp_pheno_f);
6574
6575 my %rel_htp_result_hash;
6576 if ($compute_relationship_matrix_from_htp_phenotypes_type eq 'correlations') {
6577 my $htp_cmd = 'R -e "library(lme4); library(data.table);
6578 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
6579 mat_agg <- aggregate(mat[, 7:ncol(mat)], list(mat\$accession_id), mean);
6580 mat_pheno <- mat_agg[,2:ncol(mat_agg)];
6581 cor_mat <- cor(t(mat_pheno));
6582 rownames(cor_mat) <- mat_agg[,1];
6583 colnames(cor_mat) <- mat_agg[,1];
6584 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
6585 cor_mat <- range01(cor_mat);
6586 write.table(cor_mat, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
6587 print STDERR Dumper $htp_cmd;
6588 my $status = system($htp_cmd);
6589 }
6590 elsif ($compute_relationship_matrix_from_htp_phenotypes_type eq 'blues') {
6591 my $htp_cmd = 'R -e "library(lme4); library(data.table);
6592 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
6593 blues <- data.frame(id = seq(1,length(unique(mat\$accession_id))));
6594 varlist <- names(mat)[7:ncol(mat)];
6595 blues.models <- lapply(varlist, function(x) {
6596 tryCatch(
6597 lmer(substitute(i ~ 1 + (1|accession_id), list(i = as.name(x))), data = mat, REML = FALSE, control = lmerControl(optimizer =\'Nelder_Mead\', boundary.tol='.$compute_relationship_matrix_from_htp_phenotypes_blues_inversion.' ) ), error=function(e) {}
6598 )
6599 });
6600 counter = 1;
6601 for (m in blues.models) {
6602 if (!is.null(m)) {
6603 blues\$accession_id <- row.names(ranef(m)\$accession_id);
6604 blues[,ncol(blues) + 1] <- ranef(m)\$accession_id\$\`(Intercept)\`;
6605 colnames(blues)[ncol(blues)] <- varlist[counter];
6606 }
6607 counter = counter + 1;
6608 }
6609 blues_vals <- as.matrix(blues[,3:ncol(blues)]);
6610 blues_vals <- apply(blues_vals, 2, function(y) (y - mean(y)) / sd(y) ^ as.logical(sd(y)));
6611 rel <- (1/ncol(blues_vals)) * (blues_vals %*% t(blues_vals));
6612 rownames(rel) <- blues[,2];
6613 colnames(rel) <- blues[,2];
6614 write.table(rel, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
6615 print STDERR Dumper $htp_cmd;
6616 my $status = system($htp_cmd);
6617 }
6618 else {
6619 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_type htp_pheno_rel_matrix_type is not valid!" };
6620 return;
6621 }
6622
6623 open(my $htp_rel_res, '<', $stats_out_htp_rel_tempfile)
6624 or die "Could not open file '$stats_out_htp_rel_tempfile' $!";
6625
6626 print STDERR "Opened $stats_out_htp_rel_tempfile\n";
6627 my $header_row = <$htp_rel_res>;
6628 my @header;
6629 if ($csv->parse($header_row)) {
6630 @header = $csv->fields();
6631 }
6632
6633 while (my $row = <$htp_rel_res>) {
6634 my @columns;
6635 if ($csv->parse($row)) {
6636 @columns = $csv->fields();
6637 }
6638 my $stock_id1 = $columns[0];
6639 my $counter = 1;
6640 foreach my $stock_id2 (@header) {
6641 my $val = $columns[$counter];
6642 $rel_htp_result_hash{$stock_id1}->{$stock_id2} = $val;
6643 $counter++;
6644 }
6645 }
6646 close($htp_rel_res);
6647
6648 my $data_rel_htp = '';
6649 my %result_hash;
6650 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups') {
6651 foreach my $s (sort @accession_ids) {
6652 foreach my $c (sort @accession_ids) {
6653 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6654 my $val = $rel_htp_result_hash{$s}->{$c};
6655 if (defined $val and length $val) {
6656 $result_hash{$s}->{$c} = $val;
6657 $data_rel_htp .= "$s\t$c\t$val\n";
6658 }
6659 }
6660 }
6661 }
6662 }
6663 else {
6664 foreach my $s (sort @accession_ids) {
6665 foreach my $c (sort @accession_ids) {
6666 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
6667 my $val = $rel_htp_result_hash{$s}->{$c};
6668 if (defined $val and length $val) {
6669 $result_hash{$s}->{$c} = $val;
6670 $result_hash{$c}->{$s} = $val;
6671 $data_rel_htp .= "S$s\tS$c\t$val\n";
6672 if ($s != $c) {
6673 $data_rel_htp .= "S$c\tS$s\t$val\n";
6674 }
6675 }
6676 }
6677 }
6678 }
6679 }
6680
6681 open(my $htp_rel_out, ">", $stats_out_htp_rel_tempfile_out) || die "Can't open file ".$stats_out_htp_rel_tempfile_out;
6682 print $htp_rel_out $data_rel_htp;
6683 close($htp_rel_out);
6684
6685 $grm_file = $stats_out_htp_rel_tempfile_out;
6686 }
6687 else {
6688 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes is not valid!" };
6689 return;
6690 }
6691 };
6692
6693 my $result_3 = CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation::perform_drone_imagery_analytics($schema, $a_env, $b_env, $ro_env, $row_ro_env, $env_variance_percent, $protocol_id, $statistics_select, $analytics_select, $tolparinv, $use_area_under_curve, $legendre_order_number, $permanent_environment_structure, \@legendre_coeff_exec, \%trait_name_encoder_2, \%trait_name_encoder_rev_2, \%stock_info_2, \%plot_id_map, \@sorted_trait_names_2, \%accession_id_factor_map, \@rep_time_factors, \@ind_rep_factors, \@unique_accession_names, \%plot_id_count_map_reverse, \@sorted_scaled_ln_times, \%time_count_map_reverse, \%accession_id_factor_map_reverse, \%seen_times, \%plot_id_factor_map_reverse, \%trait_to_time_map_2, \@unique_plot_names, \%stock_name_row_col, \%phenotype_data_original_2, \%plot_rep_time_factor_map, \%stock_row_col, \%stock_row_col_id, \%polynomial_map, \@plot_ids_ordered, $csv, $timestamp, $user_name, $stats_tempfile, $grm_file, $grm_rename_tempfile, $tmp_stats_dir, $stats_out_tempfile, $stats_out_tempfile_row, $stats_out_tempfile_col, $stats_out_tempfile_residual, $stats_out_tempfile_2dspl, $stats_prep2_tempfile, $stats_out_param_tempfile, $parameter_tempfile, $parameter_asreml_tempfile, $stats_tempfile_2, $permanent_environment_structure_tempfile, $permanent_environment_structure_env_tempfile, $permanent_environment_structure_env_tempfile2, $permanent_environment_structure_env_tempfile_mat, $sim_env_changing_mat_tempfile, $sim_env_changing_mat_full_tempfile, $yhat_residual_tempfile, $blupf90_solutions_tempfile, $coeff_genetic_tempfile, $coeff_pe_tempfile, $stats_out_tempfile_varcomp, $time_min, $time_max, $header_string_2, $env_sim_exec, $min_row, $max_row, $min_col, $max_col, $mean_row, $sig_row, $mean_col, $sig_col, $sim_env_change_over_time, $correlation_between_times, $field_trial_id_list, $simulated_environment_real_data_trait_id, $fixed_effect_type, $perform_cv);
6694 if (ref($result_3) eq 'HASH') {
6695 $c->stash->{rest} = $result_3;
6696 $c->detach();
6697 }
6698 my ($statistical_ontology_term_3, $analysis_model_training_data_file_type_3, $analysis_model_language_3, $sorted_residual_trait_names_array_3, $rr_unique_traits_hash_3, $rr_residual_unique_traits_hash_3, $statistics_cmd_3, $cmd_f90_3, $number_traits_3, $trait_to_time_map_hash_3,
6699
6700 $result_blup_data_original_3, $result_blup_data_delta_original_3, $result_blup_spatial_data_original_3, $result_blup_pe_data_original_3, $result_blup_pe_data_delta_original_3, $result_residual_data_original_3, $result_fitted_data_original_3, $fixed_effects_original_hash_3,
6701 $rr_genetic_coefficients_original_hash_3, $rr_temporal_coefficients_original_hash_3,
6702 $rr_coeff_genetic_covariance_original_array_3, $rr_coeff_env_covariance_original_array_3, $rr_coeff_genetic_correlation_original_array_3, $rr_coeff_env_correlation_original_array_3, $rr_residual_variance_original_3, $varcomp_original_array_3,
6703 $model_sum_square_residual_original_3, $genetic_effect_min_original_3, $genetic_effect_max_original_3, $env_effect_min_original_3, $env_effect_max_original_3, $genetic_effect_sum_square_original_3, $genetic_effect_sum_original_3, $env_effect_sum_square_original_3, $env_effect_sum_original_3, $residual_sum_square_original_3, $residual_sum_original_3, $result_cv_original_3, $result_cv_2_original_3,
6704
6705 $phenotype_data_altered_hash_3, $data_matrix_altered_array_3, $data_matrix_phenotypes_altered_array_3, $phenotype_min_altered_3, $phenotype_max_altered_3,
6706 $result_blup_data_altered_3, $result_blup_data_delta_altered_3, $result_blup_spatial_data_altered_3, $result_blup_pe_data_altered_3, $result_blup_pe_data_delta_altered_3, $result_residual_data_altered_3, $result_fitted_data_altered_3, $fixed_effects_altered_hash_3,
6707 $rr_genetic_coefficients_altered_hash_3, $rr_temporal_coefficients_altered_hash_3,
6708 $rr_coeff_genetic_covariance_altered_array_3, $rr_coeff_env_covariance_altered_array_3, $rr_coeff_genetic_correlation_altered_array_3, $rr_coeff_env_correlation_altered_array_3, $rr_residual_variance_altered_3, $varcomp_altered_array_3,
6709 $model_sum_square_residual_altered_3, $genetic_effect_min_altered_3, $genetic_effect_max_altered_3, $env_effect_min_altered_3, $env_effect_max_altered_3, $genetic_effect_sum_square_altered_3, $genetic_effect_sum_altered_3, $env_effect_sum_square_altered_3, $env_effect_sum_altered_3, $residual_sum_square_altered_3, $residual_sum_altered_3, $result_cv_altered_3, $result_cv_2_altered_3,
6710
6711 $phenotype_data_altered_env_hash_1_3, $data_matrix_altered_env_array_1_3, $data_matrix_phenotypes_altered_env_array_1_3, $phenotype_min_altered_env_1_3, $phenotype_max_altered_env_1_3, $env_sim_min_1_3, $env_sim_max_1_3, $sim_data_hash_1_3,
6712 $result_blup_data_altered_env_1_3, $result_blup_data_delta_altered_env_1_3, $result_blup_spatial_data_altered_env_1_3, $result_blup_pe_data_altered_env_1_3, $result_blup_pe_data_delta_altered_env_1_3, $result_residual_data_altered_env_1_3, $result_fitted_data_altered_env_1_3, $fixed_effects_altered_env_hash_1_3, $rr_genetic_coefficients_altered_env_hash_1_3, $rr_temporal_coefficients_altered_env_hash_1_3,
6713 $rr_coeff_genetic_covariance_altered_env_array_1_3, $rr_coeff_env_covariance_altered_env_array_1_3, $rr_coeff_genetic_correlation_altered_env_array_1_3, $rr_coeff_env_correlation_altered_env_array_1_3, $rr_residual_variance_altered_env_1_3, $varcomp_altered_array_env_1_3,
6714 $model_sum_square_residual_altered_env_1_3, $genetic_effect_min_altered_env_1_3, $genetic_effect_max_altered_env_1_3, $env_effect_min_altered_env_1_3, $env_effect_max_altered_env_1_3, $genetic_effect_sum_square_altered_env_1_3, $genetic_effect_sum_altered_env_1_3, $env_effect_sum_square_altered_env_1_3, $env_effect_sum_altered_env_1_3, $residual_sum_square_altered_env_1_3, $residual_sum_altered_env_1_3, $result_cv_altered_env_1_3, $result_cv_2_altered_env_1_3,
6715
6716 $phenotype_data_altered_env_hash_2_3, $data_matrix_altered_env_array_2_3, $data_matrix_phenotypes_altered_env_array_2_3, $phenotype_min_altered_env_2_3, $phenotype_max_altered_env_2_3, $env_sim_min_2_3, $env_sim_max_2_3, $sim_data_hash_2_3,
6717 $result_blup_data_altered_env_2_3, $result_blup_data_delta_altered_env_2_3, $result_blup_spatial_data_altered_env_2_3, $result_blup_pe_data_altered_env_2_3, $result_blup_pe_data_delta_altered_env_2_3, $result_residual_data_altered_env_2_3, $result_fitted_data_altered_env_2_3, $fixed_effects_altered_env_hash_2_3, $rr_genetic_coefficients_altered_env_hash_2_3, $rr_temporal_coefficients_altered_env_hash_2_3,
6718 $rr_coeff_genetic_covariance_altered_env_array_2_3, $rr_coeff_env_covariance_altered_env_array_2_3, $rr_coeff_genetic_correlation_altered_env_array_2_3, $rr_coeff_env_correlation_altered_env_array_2_3, $rr_residual_variance_altered_env_2_3, $varcomp_altered_array_env_2_3,
6719 $model_sum_square_residual_altered_env_2_3, $genetic_effect_min_altered_env_2_3, $genetic_effect_max_altered_env_2_3, $env_effect_min_altered_env_2_3, $env_effect_max_altered_env_2_3, $genetic_effect_sum_square_altered_env_2_3, $genetic_effect_sum_altered_env_2_3, $env_effect_sum_square_altered_env_2_3, $env_effect_sum_altered_env_2_3, $residual_sum_square_altered_env_2_3, $residual_sum_altered_env_2_3, $result_cv_altered_env_2_3, $result_cv_2_altered_env_2_3,
6720
6721 $phenotype_data_altered_env_hash_3_3, $data_matrix_altered_env_array_3_3, $data_matrix_phenotypes_altered_env_array_3_3, $phenotype_min_altered_env_3_3, $phenotype_max_altered_env_3_3, $env_sim_min_3_3, $env_sim_max_3_3, $sim_data_hash_3_3,
6722 $result_blup_data_altered_env_3_3, $result_blup_data_delta_altered_env_3_3, $result_blup_spatial_data_altered_env_3_3, $result_blup_pe_data_altered_env_3_3, $result_blup_pe_data_delta_altered_env_3_3, $result_residual_data_altered_env_3_3, $result_fitted_data_altered_env_3_3, $fixed_effects_altered_env_hash_3_3, $rr_genetic_coefficients_altered_env_hash_3_3, $rr_temporal_coefficients_altered_env_hash_3_3,
6723 $rr_coeff_genetic_covariance_altered_env_array_3_3, $rr_coeff_env_covariance_altered_env_array_3_3, $rr_coeff_genetic_correlation_altered_env_array_3_3, $rr_coeff_env_correlation_altered_env_array_3_3, $rr_residual_variance_altered_env_3_3, $varcomp_altered_array_env_3_3,
6724 $model_sum_square_residual_altered_env_3_3, $genetic_effect_min_altered_env_3_3, $genetic_effect_max_altered_env_3_3, $env_effect_min_altered_env_3_3, $env_effect_max_altered_env_3_3, $genetic_effect_sum_square_altered_env_3_3, $genetic_effect_sum_altered_env_3_3, $env_effect_sum_square_altered_env_3_3, $env_effect_sum_altered_env_3_3, $residual_sum_square_altered_env_3_3, $residual_sum_altered_env_3_3, $result_cv_altered_env_3_3, $result_cv_2_altered_env_3_3,
6725
6726 $phenotype_data_altered_env_hash_4_3, $data_matrix_altered_env_array_4_3, $data_matrix_phenotypes_altered_env_array_4_3, $phenotype_min_altered_env_4_3, $phenotype_max_altered_env_4_3, $env_sim_min_4_3, $env_sim_max_4_3, $sim_data_hash_4_3,
6727 $result_blup_data_altered_env_4_3, $result_blup_data_delta_altered_env_4_3, $result_blup_spatial_data_altered_env_4_3, $result_blup_pe_data_altered_env_4_3, $result_blup_pe_data_delta_altered_env_4_3, $result_residual_data_altered_env_4_3, $result_fitted_data_altered_env_4_3, $fixed_effects_altered_env_hash_4_3, $rr_genetic_coefficients_altered_env_hash_4_3, $rr_temporal_coefficients_altered_env_hash_4_3,
6728 $rr_coeff_genetic_covariance_altered_env_array_4_3, $rr_coeff_env_covariance_altered_env_array_4_3, $rr_coeff_genetic_correlation_altered_env_array_4_3, $rr_coeff_env_correlation_altered_env_array_4_3, $rr_residual_variance_altered_env_4_3, $varcomp_altered_array_env_4_3,
6729 $model_sum_square_residual_altered_env_4_3, $genetic_effect_min_altered_env_4_3, $genetic_effect_max_altered_env_4_3, $env_effect_min_altered_env_4_3, $env_effect_max_altered_env_4_3, $genetic_effect_sum_square_altered_env_4_3, $genetic_effect_sum_altered_env_4_3, $env_effect_sum_square_altered_env_4_3, $env_effect_sum_altered_env_4_3, $residual_sum_square_altered_env_4_3, $residual_sum_altered_env_4_3, $result_cv_altered_env_4_3, $result_cv_2_altered_env_4_3,
6730
6731 $phenotype_data_altered_env_hash_5_3, $data_matrix_altered_env_array_5_3, $data_matrix_phenotypes_altered_env_array_5_3, $phenotype_min_altered_env_5_3, $phenotype_max_altered_env_5_3, $env_sim_min_5_3, $env_sim_max_5_3, $sim_data_hash_5_3,
6732 $result_blup_data_altered_env_5_3, $result_blup_data_delta_altered_env_5_3, $result_blup_spatial_data_altered_env_5_3, $result_blup_pe_data_altered_env_5_3, $result_blup_pe_data_delta_altered_env_5_3, $result_residual_data_altered_env_5_3, $result_fitted_data_altered_env_5_3, $fixed_effects_altered_env_hash_5_3, $rr_genetic_coefficients_altered_env_hash_5_3, $rr_temporal_coefficients_altered_env_hash_5_3,
6733 $rr_coeff_genetic_covariance_altered_env_array_5_3, $rr_coeff_env_covariance_altered_env_array_5_3, $rr_coeff_genetic_correlation_altered_env_array_5_3, $rr_coeff_env_correlation_altered_env_array_5_3, $rr_residual_variance_altered_env_5_3, $varcomp_altered_array_env_5_3,
6734 $model_sum_square_residual_altered_env_5_3, $genetic_effect_min_altered_env_5_3, $genetic_effect_max_altered_env_5_3, $env_effect_min_altered_env_5_3, $env_effect_max_altered_env_5_3, $genetic_effect_sum_square_altered_env_5_3, $genetic_effect_sum_altered_env_5_3, $env_effect_sum_square_altered_env_5_3, $env_effect_sum_altered_env_5_3, $residual_sum_square_altered_env_5_3, $residual_sum_altered_env_5_3, $result_cv_altered_env_5_3, $result_cv_2_altered_env_5_3,
6735
6736 $phenotype_data_altered_env_hash_6_3, $data_matrix_altered_env_array_6_3, $data_matrix_phenotypes_altered_env_array_6_3, $phenotype_min_altered_env_6_3, $phenotype_max_altered_env_6_3, $env_sim_min_6_3, $env_sim_max_6_3, $sim_data_hash_6_3,
6737 $result_blup_data_altered_env_6_3, $result_blup_data_delta_altered_env_6_3, $result_blup_spatial_data_altered_env_6_3, $result_blup_pe_data_altered_env_6_3, $result_blup_pe_data_delta_altered_env_6_3, $result_residual_data_altered_env_6_3, $result_fitted_data_altered_env_6_3, $fixed_effects_altered_env_hash_6_3, $rr_genetic_coefficients_altered_env_hash_6_3, $rr_temporal_coefficients_altered_env_hash_6_3,
6738 $rr_coeff_genetic_covariance_altered_env_array_6_3, $rr_coeff_env_covariance_altered_env_array_6_3, $rr_coeff_genetic_correlation_altered_env_array_6_3, $rr_coeff_env_correlation_altered_env_array_6_3, $rr_residual_variance_altered_env_6_3, $varcomp_altered_array_env_6_3,
6739 $model_sum_square_residual_altered_env_6_3, $genetic_effect_min_altered_env_6_3, $genetic_effect_max_altered_env_6_3, $env_effect_min_altered_env_6_3, $env_effect_max_altered_env_6_3, $genetic_effect_sum_square_altered_env_6_3, $genetic_effect_sum_altered_env_6_3, $env_effect_sum_square_altered_env_6_3, $env_effect_sum_altered_env_6_3, $residual_sum_square_altered_env_6_3, $residual_sum_altered_env_6_3, $result_cv_altered_env_6_3, $result_cv_2_altered_env_6_3
6740 ) = @$result_3;
6741
6742 eval {
6743 print STDERR "PLOTTING CORRELATION\n";
6744 my ($full_plot_level_correlation_tempfile_fh, $full_plot_level_correlation_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6745 open(my $F_fullplot, ">", $full_plot_level_correlation_tempfile) || die "Can't open file ".$full_plot_level_correlation_tempfile;
6746 print STDERR "OPENED PLOTCORR FILE $full_plot_level_correlation_tempfile\n";
6747
6748 my @header_full_plot_corr = ('plot_name, plot_id, row_number, col_number, rep, block, germplasm_name, germplasm_id');
6749 my @types_full_plot_corr = ('pheno_orig_', 'pheno_postm3_', 'eff_origm3_', 'eff_postm3_',
6750 'sim_env1_', 'simm3_pheno1_', 'effm3_sim1_',
6751 'sim_env2_', 'simm3_pheno2_', 'effm3_sim2_',
6752 'sim_env3_', 'simm3_pheno3_', 'effm3_sim3_',
6753 'sim_env4_', 'simm3_pheno4_', 'effm3_sim4_',
6754 'sim_env5_', 'simm3_pheno5_', 'effm3_sim5_',
6755 'sim_env6_', 'simm3_pheno6_', 'effm3_sim6_');
6756 foreach my $t (@sorted_trait_names_2) {
6757 foreach my $type (@types_full_plot_corr) {
6758 push @header_full_plot_corr, $type.$trait_name_encoder_2{$t};
6759 }
6760 }
6761 my $header_string_full_plot_corr = join ',', @header_full_plot_corr;
6762 print $F_fullplot "$header_string_full_plot_corr\n";
6763 foreach my $p (@unique_plot_names) {
6764 my @row = ($p, $stock_name_row_col{$p}->{obsunit_stock_id}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $stock_name_row_col{$p}->{rep}, $stock_name_row_col{$p}->{block}, $stock_name_row_col{$p}->{germplasm_name}, $stock_name_row_col{$p}->{germplasm_stock_id});
6765 foreach my $t (@sorted_trait_names_2) {
6766 my $t_conv = $t;
6767
6768 my $phenotype_original = $phenotype_data_original_2{$p}->{$t};
6769 my $phenotype_post_3 = $phenotype_data_altered_hash_3->{$p}->{$t_conv};
6770 my $effect_original_3 = $result_blup_spatial_data_original_3->{$p}->{$t_conv}->[0];
6771 my $effect_post_3 = $result_blup_spatial_data_altered_3->{$p}->{$t_conv}->[0] || 'NA';
6772 push @row, ($phenotype_original, $phenotype_post_3, $effect_original_3, $effect_post_3);
6773
6774 my $sim_env = $sim_data_hash_1_3->{$p}->{$t};
6775 my $pheno_sim_3 = $phenotype_data_altered_env_hash_1_3->{$p}->{$t_conv};
6776 my $effect_sim_3 = $result_blup_spatial_data_altered_env_1_3->{$p}->{$t_conv}->[0];
6777 push @row, ($sim_env, $pheno_sim_3, $effect_sim_3);
6778
6779 my $sim_env2 = $sim_data_hash_2_3->{$p}->{$t};
6780 my $pheno_sim2_3 = $phenotype_data_altered_env_hash_2_3->{$p}->{$t_conv};
6781 my $effect_sim2_3 = $result_blup_spatial_data_altered_env_2_3->{$p}->{$t_conv}->[0];
6782 push @row, ($sim_env2, $pheno_sim2_3, $effect_sim2_3);
6783
6784 my $sim_env3 = $sim_data_hash_3_3->{$p}->{$t};
6785 my $pheno_sim3_3 = $phenotype_data_altered_env_hash_3_3->{$p}->{$t_conv};
6786 my $effect_sim3_3 = $result_blup_spatial_data_altered_env_3_3->{$p}->{$t_conv}->[0];
6787 push @row, ($sim_env3, $pheno_sim3_3, $effect_sim3_3);
6788
6789 my $sim_env4 = $sim_data_hash_4_3->{$p}->{$t};
6790 my $pheno_sim4_3 = $phenotype_data_altered_env_hash_4_3->{$p}->{$t_conv};
6791 my $effect_sim4_3 = $result_blup_spatial_data_altered_env_4_3->{$p}->{$t_conv}->[0];
6792 push @row, ($sim_env4, $pheno_sim4_3, $effect_sim4_3);
6793
6794 my $sim_env5 = $sim_data_hash_5_3->{$p}->{$t};
6795 my $pheno_sim5_3 = $phenotype_data_altered_env_hash_5_3->{$p}->{$t_conv};
6796 my $effect_sim5_3 = $result_blup_spatial_data_altered_env_5_3->{$p}->{$t_conv}->[0];
6797 push @row, ($sim_env5, $pheno_sim5_3, $effect_sim5_3);
6798
6799 my $sim_env6 = $sim_data_hash_6_3->{$p}->{$t};
6800 my $pheno_sim6_3 = $phenotype_data_altered_env_hash_6_3->{$p}->{$t_conv};
6801 my $effect_sim6_3 = $result_blup_spatial_data_altered_env_6_3->{$p}->{$t_conv}->[0];
6802 push @row, ($sim_env6, $pheno_sim6_3, $effect_sim6_3);
6803 }
6804 my $line = join ',', @row;
6805 print $F_fullplot "$line\n";
6806 }
6807 close($F_fullplot);
6808
6809 my $plot_corr_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
6810 $plot_corr_figure_tempfile_string .= '.png';
6811 my $plot_corr_figure_tempfile = $c->config->{basepath}."/".$plot_corr_figure_tempfile_string;
6812
6813 my $cmd_plotcorr_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
6814 mat_orig <- fread(\''.$full_plot_level_correlation_tempfile.'\', header=TRUE, sep=\',\');
6815 gg <- ggcorr(data=mat_orig[,-seq(1,8)], hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE);
6816 ggsave(\''.$plot_corr_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, limitsize = FALSE, units=\'in\');
6817 "';
6818 # print STDERR Dumper $cmd;
6819 my $status_plotcorr_plot = system($cmd_plotcorr_plot);
6820 push @$spatial_effects_plots, [$plot_corr_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
6821 push @$spatial_effects_files_store, [$full_plot_level_correlation_tempfile, "datafile_".$statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
6822 };
6823
6824 eval {
6825 my @plot_corr_full_vals;
6826
6827 my @original_pheno_vals;
6828 my ($phenotypes_original_heatmap_tempfile_fh, $phenotypes_original_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6829 open(my $F_pheno, ">", $phenotypes_original_heatmap_tempfile) || die "Can't open file ".$phenotypes_original_heatmap_tempfile;
6830 print $F_pheno "trait_type,row,col,value\n";
6831 foreach my $p (@unique_plot_names) {
6832 foreach my $t (@sorted_trait_names_2) {
6833 my $val = $phenotype_data_original_2{$p}->{$t};
6834 my @row = ("pheno_orig_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6835 push @original_pheno_vals, $val;
6836 my $line = join ',', @row;
6837 print $F_pheno "$line\n";
6838 }
6839 }
6840 close($F_pheno);
6841 push @plot_corr_full_vals, \@original_pheno_vals;
6842
6843 my $original_pheno_stat = Statistics::Descriptive::Full->new();
6844 $original_pheno_stat->add_data(@original_pheno_vals);
6845 my $sig_original_pheno = $original_pheno_stat->variance();
6846
6847 #PHENO POST M START
6848
6849 my @altered_pheno_vals_3;
6850 my ($phenotypes_post_heatmap_tempfile_fh_3, $phenotypes_post_heatmap_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6851 open($F_pheno, ">", $phenotypes_post_heatmap_tempfile_3) || die "Can't open file ".$phenotypes_post_heatmap_tempfile_3;
6852 print $F_pheno "trait_type,row,col,value\n";
6853 foreach my $p (@unique_plot_names) {
6854 foreach my $t (@sorted_trait_names_2) {
6855 my $val = $phenotype_data_altered_hash_3->{$p}->{$t};
6856 my @row = ("pheno_postm3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6857 push @altered_pheno_vals_3, $val;
6858 my $line = join ',', @row;
6859 print $F_pheno "$line\n";
6860 }
6861 }
6862 close($F_pheno);
6863 push @plot_corr_full_vals, \@altered_pheno_vals_3;
6864
6865 my $altered_pheno_stat_3 = Statistics::Descriptive::Full->new();
6866 $altered_pheno_stat_3->add_data(@altered_pheno_vals_3);
6867 my $sig_altered_pheno_3 = $altered_pheno_stat_3->variance();
6868
6869 # EFFECT ORIGINAL M
6870
6871 my @original_effect_vals_3;
6872 my ($effects_heatmap_tempfile_fh_3, $effects_heatmap_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6873 open(my $F_eff, ">", $effects_heatmap_tempfile_3) || die "Can't open file ".$effects_heatmap_tempfile_3;
6874 print $F_eff "trait_type,row,col,value\n";
6875 foreach my $p (@unique_plot_names) {
6876 foreach my $t (@sorted_trait_names_2) {
6877 my $val = $result_blup_spatial_data_original_3->{$p}->{$t}->[0];
6878 my @row = ("eff_origm3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6879 my $line = join ',', @row;
6880 print $F_eff "$line\n";
6881 push @original_effect_vals_3, $val;
6882 }
6883 }
6884 close($F_eff);
6885 push @plot_corr_full_vals, \@original_effect_vals_3;
6886
6887 my $original_effect_stat_3 = Statistics::Descriptive::Full->new();
6888 $original_effect_stat_3->add_data(@original_effect_vals_3);
6889 my $sig_original_effect_3 = $original_effect_stat_3->variance();
6890
6891 # EFFECT POST M MIN
6892
6893 my @altered_effect_vals_3;
6894 my ($effects_post_heatmap_tempfile_fh_3, $effects_post_heatmap_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6895 open($F_eff, ">", $effects_post_heatmap_tempfile_3) || die "Can't open file ".$effects_post_heatmap_tempfile_3;
6896 print $F_eff "trait_type,row,col,value\n";
6897 foreach my $p (@unique_plot_names) {
6898 foreach my $t (@sorted_trait_names_2) {
6899 my $val = $result_blup_spatial_data_altered_3->{$p}->{$t}->[0];
6900 my @row = ("eff_postm3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6901 my $line = join ',', @row;
6902 print $F_eff "$line\n";
6903 push @altered_effect_vals_3, $val;
6904 }
6905 }
6906 close($F_eff);
6907 push @plot_corr_full_vals, \@altered_effect_vals_3;
6908
6909 my $altered_effect_stat_3 = Statistics::Descriptive::Full->new();
6910 $altered_effect_stat_3->add_data(@altered_effect_vals_3);
6911 my $sig_altered_effect_3 = $altered_effect_stat_3->variance();
6912
6913 # SIM ENV 1: ALTERED PHENO + EFFECT
6914
6915 my ($phenotypes_env_heatmap_tempfile_fh, $phenotypes_env_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6916 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile) || die "Can't open file ".$phenotypes_env_heatmap_tempfile;
6917 print $F_pheno "trait_type,row,col,value\n";
6918 foreach my $p (@unique_plot_names) {
6919 foreach my $t (@sorted_trait_names_2) {
6920 my @row = ("sim_env1_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_1_3->{$p}->{$t});
6921 my $line = join ',', @row;
6922 print $F_pheno "$line\n";
6923 }
6924 }
6925 close($F_pheno);
6926
6927 my @sim_pheno1_vals_3;
6928 my ($phenotypes_pheno_sim_heatmap_tempfile_fh_3, $phenotypes_pheno_sim_heatmap_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6929 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile_3;
6930 print $F_pheno "trait_type,row,col,value\n";
6931 foreach my $p (@unique_plot_names) {
6932 foreach my $t (@sorted_trait_names_2) {
6933 my $val = $phenotype_data_altered_env_hash_1_3->{$p}->{$t};
6934 my @row = ("simm3_pheno1_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6935 my $line = join ',', @row;
6936 print $F_pheno "$line\n";
6937 push @sim_pheno1_vals_3, $val;
6938 }
6939 }
6940 close($F_pheno);
6941 push @plot_corr_full_vals, \@sim_pheno1_vals_3;
6942
6943 my $sim_pheno1_stat_3 = Statistics::Descriptive::Full->new();
6944 $sim_pheno1_stat_3->add_data(@sim_pheno1_vals_3);
6945 my $sig_sim3_pheno1 = $sim_pheno1_stat_3->variance();
6946
6947 my @sim_effect1_vals_3;
6948 my ($effects_sim_heatmap_tempfile_fh_3, $effects_sim_heatmap_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6949 open($F_eff, ">", $effects_sim_heatmap_tempfile_3) || die "Can't open file ".$effects_sim_heatmap_tempfile_3;
6950 print $F_eff "trait_type,row,col,value\n";
6951 foreach my $p (@unique_plot_names) {
6952 foreach my $t (@sorted_trait_names_2) {
6953 my $val = $result_blup_spatial_data_altered_env_1_3->{$p}->{$t}->[0];
6954 my @row = ("effm3_sim1_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6955 my $line = join ',', @row;
6956 print $F_eff "$line\n";
6957 push @sim_effect1_vals_3, $val;
6958 }
6959 }
6960 close($F_eff);
6961 push @plot_corr_full_vals, \@sim_effect1_vals_3;
6962
6963 my $sim_effect1_stat_3 = Statistics::Descriptive::Full->new();
6964 $sim_effect1_stat_3->add_data(@sim_effect1_vals_3);
6965 my $sig_sim3_effect1 = $sim_effect1_stat_3->variance();
6966
6967 # SIM ENV 2: ALTERED PHENO + EFFECT
6968
6969 my ($phenotypes_env_heatmap_tempfile2_fh, $phenotypes_env_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6970 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile2) || die "Can't open file ".$phenotypes_env_heatmap_tempfile2;
6971 print $F_pheno "trait_type,row,col,value\n";
6972 foreach my $p (@unique_plot_names) {
6973 foreach my $t (@sorted_trait_names_2) {
6974 my @row = ("sim_env2_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_2_3->{$p}->{$t});
6975 my $line = join ',', @row;
6976 print $F_pheno "$line\n";
6977 }
6978 }
6979 close($F_pheno);
6980
6981 my @sim_pheno2_vals_3;
6982 my ($phenotypes_pheno_sim_heatmap_tempfile2_fh_3, $phenotypes_pheno_sim_heatmap_tempfile2_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
6983 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile2_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile2_3;
6984 print $F_pheno "trait_type,row,col,value\n";
6985 foreach my $p (@unique_plot_names) {
6986 foreach my $t (@sorted_trait_names_2) {
6987 my $val = $phenotype_data_altered_env_hash_2_3->{$p}->{$t};
6988 my @row = ("simm3_pheno2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
6989 my $line = join ',', @row;
6990 print $F_pheno "$line\n";
6991 push @sim_pheno2_vals_3, $val;
6992 }
6993 }
6994 close($F_pheno);
6995 push @plot_corr_full_vals, \@sim_pheno2_vals_3;
6996
6997 my $sim_pheno2_stat_3 = Statistics::Descriptive::Full->new();
6998 $sim_pheno2_stat_3->add_data(@sim_pheno2_vals_3);
6999 my $sig_sim_pheno2_3 = $sim_pheno2_stat_3->variance();
7000
7001 my @sim_effect2_vals_3;
7002 my ($effects_sim_heatmap_tempfile2_fh_3, $effects_sim_heatmap_tempfile2_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7003 open($F_eff, ">", $effects_sim_heatmap_tempfile2_3) || die "Can't open file ".$effects_sim_heatmap_tempfile2_3;
7004 print $F_eff "trait_type,row,col,value\n";
7005 foreach my $p (@unique_plot_names) {
7006 foreach my $t (@sorted_trait_names_2) {
7007 my $val = $result_blup_spatial_data_altered_env_2_3->{$p}->{$t}->[0];
7008 my @row = ("effm3_sim2_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7009 my $line = join ',', @row;
7010 print $F_eff "$line\n";
7011 push @sim_effect2_vals_3, $val;
7012 }
7013 }
7014 close($F_eff);
7015 push @plot_corr_full_vals, \@sim_effect2_vals_3;
7016
7017 my $sim_effect2_stat_3 = Statistics::Descriptive::Full->new();
7018 $sim_effect2_stat_3->add_data(@sim_effect2_vals_3);
7019 my $sig_sim_effect2_3 = $sim_effect2_stat_3->variance();
7020
7021 # SIM ENV 3: ALTERED PHENO + EFFECT
7022
7023 my ($phenotypes_env_heatmap_tempfile3_fh, $phenotypes_env_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7024 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile3) || die "Can't open file ".$phenotypes_env_heatmap_tempfile3;
7025 print $F_pheno "trait_type,row,col,value\n";
7026 foreach my $p (@unique_plot_names) {
7027 foreach my $t (@sorted_trait_names_2) {
7028 my @row = ("sim_env3_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_3_3->{$p}->{$t});
7029 my $line = join ',', @row;
7030 print $F_pheno "$line\n";
7031 }
7032 }
7033 close($F_pheno);
7034
7035 my @sim_pheno3_vals_3;
7036 my ($phenotypes_pheno_sim_heatmap_tempfile3_fh_3, $phenotypes_pheno_sim_heatmap_tempfile3_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7037 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile3_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile3_3;
7038 print $F_pheno "trait_type,row,col,value\n";
7039 foreach my $p (@unique_plot_names) {
7040 foreach my $t (@sorted_trait_names_2) {
7041 my $val = $phenotype_data_altered_env_hash_3_3->{$p}->{$t};
7042 my @row = ("simm3_pheno3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7043 my $line = join ',', @row;
7044 print $F_pheno "$line\n";
7045 push @sim_pheno3_vals_3, $val;
7046 }
7047 }
7048 close($F_pheno);
7049 push @plot_corr_full_vals, \@sim_pheno3_vals_3;
7050
7051 my $sim_pheno3_stat_3 = Statistics::Descriptive::Full->new();
7052 $sim_pheno3_stat_3->add_data(@sim_pheno3_vals_3);
7053 my $sig_sim_pheno3_3 = $sim_pheno3_stat_3->variance();
7054
7055 my @sim_effect3_vals_3;
7056 my ($effects_sim_heatmap_tempfile3_fh_3, $effects_sim_heatmap_tempfile3_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7057 open($F_eff, ">", $effects_sim_heatmap_tempfile3_3) || die "Can't open file ".$effects_sim_heatmap_tempfile3_3;
7058 print $F_eff "trait_type,row,col,value\n";
7059 foreach my $p (@unique_plot_names) {
7060 foreach my $t (@sorted_trait_names_2) {
7061 my $val = $result_blup_spatial_data_altered_env_3_3->{$p}->{$t}->[0];
7062 my @row = ("effm3_sim3_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7063 my $line = join ',', @row;
7064 print $F_eff "$line\n";
7065 push @sim_effect3_vals_3, $val;
7066 }
7067 }
7068 close($F_eff);
7069 push @plot_corr_full_vals, \@sim_effect3_vals_3;
7070
7071 my $sim_effect3_stat_3 = Statistics::Descriptive::Full->new();
7072 $sim_effect3_stat_3->add_data(@sim_effect3_vals_3);
7073 my $sig_sim_effect3_3 = $sim_effect3_stat_3->variance();
7074
7075 # SIM ENV 4: ALTERED PHENO + EFFECT
7076
7077 my ($phenotypes_env_heatmap_tempfile4_fh, $phenotypes_env_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7078 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile4) || die "Can't open file ".$phenotypes_env_heatmap_tempfile4;
7079 print $F_pheno "trait_type,row,col,value\n";
7080 foreach my $p (@unique_plot_names) {
7081 foreach my $t (@sorted_trait_names_2) {
7082 my @row = ("sim_env4_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_4_3->{$p}->{$t});
7083 my $line = join ',', @row;
7084 print $F_pheno "$line\n";
7085 }
7086 }
7087 close($F_pheno);
7088
7089 my @sim_pheno4_vals_3;
7090 my ($phenotypes_pheno_sim_heatmap_tempfile4_fh_3, $phenotypes_pheno_sim_heatmap_tempfile4_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7091 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile4_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile4_3;
7092 print $F_pheno "trait_type,row,col,value\n";
7093 foreach my $p (@unique_plot_names) {
7094 foreach my $t (@sorted_trait_names_2) {
7095 my $val = $phenotype_data_altered_env_hash_4_3->{$p}->{$t};
7096 my @row = ("simm3_pheno4_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7097 my $line = join ',', @row;
7098 print $F_pheno "$line\n";
7099 push @sim_pheno4_vals_3, $val;
7100 }
7101 }
7102 close($F_pheno);
7103 push @plot_corr_full_vals, \@sim_pheno4_vals_3;
7104
7105 my $sim_pheno4_stat_3 = Statistics::Descriptive::Full->new();
7106 $sim_pheno4_stat_3->add_data(@sim_pheno4_vals_3);
7107 my $sig_sim_pheno4_3 = $sim_pheno4_stat_3->variance();
7108
7109 my @sim_effect4_vals_3;
7110 my ($effects_sim_heatmap_tempfile4_fh_3, $effects_sim_heatmap_tempfile4_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7111 open($F_eff, ">", $effects_sim_heatmap_tempfile4_3) || die "Can't open file ".$effects_sim_heatmap_tempfile4_3;
7112 print $F_eff "trait_type,row,col,value\n";
7113 foreach my $p (@unique_plot_names) {
7114 foreach my $t (@sorted_trait_names_2) {
7115 my $val = $result_blup_spatial_data_altered_env_4_3->{$p}->{$t}->[0];
7116 my @row = ("effm3_sim4_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7117 my $line = join ',', @row;
7118 print $F_eff "$line\n";
7119 push @sim_effect4_vals_3, $val;
7120 }
7121 }
7122 close($F_eff);
7123 push @plot_corr_full_vals, \@sim_effect4_vals_3;
7124
7125 my $sim_effect4_stat_3 = Statistics::Descriptive::Full->new();
7126 $sim_effect4_stat_3->add_data(@sim_effect4_vals_3);
7127 my $sig_sim_effect4_3 = $sim_effect4_stat_3->variance();
7128
7129 # SIM ENV 5: ALTERED PHENO + EFFECT
7130
7131 my ($phenotypes_env_heatmap_tempfile5_fh, $phenotypes_env_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7132 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile5) || die "Can't open file ".$phenotypes_env_heatmap_tempfile5;
7133 print $F_pheno "trait_type,row,col,value\n";
7134 foreach my $p (@unique_plot_names) {
7135 foreach my $t (@sorted_trait_names_2) {
7136 my @row = ("sim_env5_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_5_3->{$p}->{$t});
7137 my $line = join ',', @row;
7138 print $F_pheno "$line\n";
7139 }
7140 }
7141 close($F_pheno);
7142
7143 my @sim_pheno5_vals_3;
7144 my ($phenotypes_pheno_sim_heatmap_tempfile5_fh_3, $phenotypes_pheno_sim_heatmap_tempfile5_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7145 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile5_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile5_3;
7146 print $F_pheno "trait_type,row,col,value\n";
7147 foreach my $p (@unique_plot_names) {
7148 foreach my $t (@sorted_trait_names_2) {
7149 my $val = $phenotype_data_altered_env_hash_5_3->{$p}->{$t};
7150 my @row = ("simm3_pheno5_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7151 my $line = join ',', @row;
7152 print $F_pheno "$line\n";
7153 push @sim_pheno5_vals_3, $val;
7154 }
7155 }
7156 close($F_pheno);
7157 push @plot_corr_full_vals, \@sim_pheno5_vals_3;
7158
7159 my $sim_pheno5_stat_3 = Statistics::Descriptive::Full->new();
7160 $sim_pheno5_stat_3->add_data(@sim_pheno5_vals_3);
7161 my $sig_sim_pheno5_3 = $sim_pheno5_stat_3->variance();
7162
7163 my @sim_effect5_vals_3;
7164 my ($effects_sim_heatmap_tempfile5_fh_3, $effects_sim_heatmap_tempfile5_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7165 open($F_eff, ">", $effects_sim_heatmap_tempfile5_3) || die "Can't open file ".$effects_sim_heatmap_tempfile5_3;
7166 print $F_eff "trait_type,row,col,value\n";
7167 foreach my $p (@unique_plot_names) {
7168 foreach my $t (@sorted_trait_names_2) {
7169 my $val = $result_blup_spatial_data_altered_env_5_3->{$p}->{$t}->[0];
7170 my @row = ("effm3_sim5_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7171 my $line = join ',', @row;
7172 print $F_eff "$line\n";
7173 push @sim_effect5_vals_3, $val;
7174 }
7175 }
7176 close($F_eff);
7177 push @plot_corr_full_vals, \@sim_effect5_vals_3;
7178
7179 my $sim_effect5_stat_3 = Statistics::Descriptive::Full->new();
7180 $sim_effect5_stat_3->add_data(@sim_effect5_vals_3);
7181 my $sig_sim_effect5_3 = $sim_effect5_stat_3->variance();
7182
7183 # SIM ENV 6: ALTERED PHENO + EFFECT
7184
7185 my ($phenotypes_env_heatmap_tempfile6_fh, $phenotypes_env_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7186 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile6) || die "Can't open file ".$phenotypes_env_heatmap_tempfile6;
7187 print $F_pheno "trait_type,row,col,value\n";
7188 foreach my $p (@unique_plot_names) {
7189 foreach my $t (@sorted_trait_names_2) {
7190 my @row = ("sim_env6_".$t, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_6_3->{$p}->{$t});
7191 my $line = join ',', @row;
7192 print $F_pheno "$line\n";
7193 }
7194 }
7195 close($F_pheno);
7196
7197 my @sim_pheno6_vals_3;
7198 my ($phenotypes_pheno_sim_heatmap_tempfile6_fh_3, $phenotypes_pheno_sim_heatmap_tempfile6_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7199 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile6_3) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile6_3;
7200 print $F_pheno "trait_type,row,col,value\n";
7201 foreach my $p (@unique_plot_names) {
7202 foreach my $t (@sorted_trait_names_2) {
7203 my $val = $phenotype_data_altered_env_hash_6_3->{$p}->{$t};
7204 my @row = ("simm3_pheno6_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7205 my $line = join ',', @row;
7206 print $F_pheno "$line\n";
7207 push @sim_pheno6_vals_3, $val;
7208 }
7209 }
7210 close($F_pheno);
7211 push @plot_corr_full_vals, \@sim_pheno6_vals_3;
7212
7213 my $sim_pheno6_stat_3 = Statistics::Descriptive::Full->new();
7214 $sim_pheno6_stat_3->add_data(@sim_pheno6_vals_3);
7215 my $sig_sim_pheno6_3 = $sim_pheno6_stat_3->variance();
7216
7217 my @sim_effect6_vals_3;
7218 my ($effects_sim_heatmap_tempfile6_fh_3, $effects_sim_heatmap_tempfile6_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7219 open($F_eff, ">", $effects_sim_heatmap_tempfile6_3) || die "Can't open file ".$effects_sim_heatmap_tempfile6_3;
7220 print $F_eff "trait_type,row,col,value\n";
7221 foreach my $p (@unique_plot_names) {
7222 foreach my $t (@sorted_trait_names_2) {
7223 my $val = $result_blup_spatial_data_altered_env_6_3->{$p}->{$t}->[0];
7224 my @row = ("effm3_sim6_".$trait_name_encoder_2{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
7225 my $line = join ',', @row;
7226 print $F_eff "$line\n";
7227 push @sim_effect6_vals_3, $val;
7228 }
7229 }
7230 close($F_eff);
7231 push @plot_corr_full_vals, \@sim_effect6_vals_3;
7232
7233 my $sim_effect6_stat_3 = Statistics::Descriptive::Full->new();
7234 $sim_effect6_stat_3->add_data(@sim_effect6_vals_3);
7235 my $sig_sim_effect6_3 = $sim_effect6_stat_3->variance();
7236
7237 my $plot_corr_summary_figure_inputfile_tempfile = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'tmp_drone_statistics/fileXXXX');
7238 open($F_eff, ">", $plot_corr_summary_figure_inputfile_tempfile) || die "Can't open file ".$plot_corr_summary_figure_inputfile_tempfile;
7239 foreach (@plot_corr_full_vals) {
7240 my $line = join ',', @$_;
7241 print $F_eff $line."\n";
7242 }
7243 close($F_eff);
7244
7245 my $plot_corr_summary_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7246 $plot_corr_summary_figure_tempfile_string .= '.png';
7247 my $plot_corr_summary_figure_tempfile = $c->config->{basepath}."/".$plot_corr_summary_figure_tempfile_string;
7248
7249 my $cmd_plotcorrsum_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
7250 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
7251 mat_full <- data.frame(t(mat_full_t));
7252 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_3\', \'mat_eff_3\', \'mat_eff_altered_3\',
7253 \'mat_p_sim1_3\', \'mat_eff_sim1_3\',
7254 \'mat_p_sim2_3\', \'mat_eff_sim2_3\',
7255 \'mat_p_sim3_3\', \'mat_eff_sim3_3\',
7256 \'mat_p_sim4_3\', \'mat_eff_sim4_3\',
7257 \'mat_p_sim5_3\', \'mat_eff_sim5_3\',
7258 \'mat_p_sim6_3\', \'mat_eff_sim6_3\');
7259 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
7260 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
7261 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
7262 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
7263 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
7264 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
7265 mat <- data.frame(pheno_orig = mat_full\$mat_orig, pheno_altm3 = mat_full\$mat_altered_3, eff_origm3 = mat_full\$mat_eff_3, eff_altm3 = mat_full\$mat_eff_altered_3, env_lin = mat_env\$value, pheno_linm3 = mat_full\$mat_p_sim1_3, lin_effm3 = mat_full\$mat_eff_sim1_3, env_n1d = mat_env2\$value, pheno_n1dm3 = mat_full\$mat_p_sim2_3, n1d_effm3 = mat_full\$mat_eff_sim2_3, env_n2d = mat_env3\$value, pheno_n2dm3 = mat_full\$mat_p_sim3_3, n2d_effm3 = mat_full\$mat_eff_sim3_3, env_rand = mat_env4\$value, pheno_randm3 = mat_full\$mat_p_sim4_3, rand_effm3 = mat_full\$mat_eff_sim4_3, env_ar1 = mat_env5\$value, pheno_ar1m3 = mat_full\$mat_p_sim5_3, ar1_effm3 = mat_full\$mat_eff_sim5_3, env_realdata = mat_env6\$value, pheno_realdatam3 = mat_full\$mat_p_sim6_3, realdata_effm3 = mat_full\$mat_eff_sim6_3);
7266 gg <- ggcorr(data=mat, hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE, label_round = 2);
7267 ggsave(\''.$plot_corr_summary_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, units=\'in\');
7268 "';
7269 # print STDERR Dumper $cmd_plotcorrsum_plot;
7270
7271 my $status_plotcorrsum_plot = system($cmd_plotcorrsum_plot);
7272 push @$spatial_effects_plots, [$plot_corr_summary_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_envsimscorr_"."envvar_".$env_variance_percent."_".$iterations];
7273
7274 my $env_effects_first_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7275 $env_effects_first_figure_tempfile_string .= '.png';
7276 my $env_effects_first_figure_tempfile = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string;
7277
7278 my $env_effects_first_figure_tempfile_string_2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7279 $env_effects_first_figure_tempfile_string_2 .= '.png';
7280 my $env_effects_first_figure_tempfile_2 = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string_2;
7281
7282 my $output_plot_row = 'row';
7283 my $output_plot_col = 'col';
7284 if ($max_col > $max_row) {
7285 $output_plot_row = 'col';
7286 $output_plot_col = 'row';
7287 }
7288
7289 my $cmd_spatialfirst_plot_2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7290 mat_orig <- fread(\''.$phenotypes_original_heatmap_tempfile.'\', header=TRUE, sep=\',\');
7291 mat_altered_3 <- fread(\''.$phenotypes_post_heatmap_tempfile_3.'\', header=TRUE, sep=\',\');
7292 pheno_mat <- rbind(mat_orig, mat_altered_3);
7293 options(device=\'png\');
7294 par();
7295 gg <- ggplot(pheno_mat, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7296 geom_tile() +
7297 scale_fill_viridis(discrete=FALSE) +
7298 coord_equal() +
7299 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7300 ggsave(\''.$env_effects_first_figure_tempfile_2.'\', gg, device=\'png\', width=20, height=20, units=\'in\');
7301 "';
7302 # print STDERR Dumper $cmd;
7303 my $status_spatialfirst_plot_2 = system($cmd_spatialfirst_plot_2);
7304 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string_2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_origheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7305
7306 my ($sim_effects_corr_results_fh, $sim_effects_corr_results) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7307
7308 my $cmd_spatialfirst_plot = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7309 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
7310 mat_full <- data.frame(t(mat_full_t));
7311 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_3\', \'mat_eff_3\', \'mat_eff_altered_3\',
7312 \'mat_p_sim1_3\', \'mat_eff_sim1_3\',
7313 \'mat_p_sim2_3\', \'mat_eff_sim2_3\',
7314 \'mat_p_sim3_3\', \'mat_eff_sim3_3\',
7315 \'mat_p_sim4_3\', \'mat_eff_sim4_3\',
7316 \'mat_p_sim5_3\', \'mat_eff_sim5_3\',
7317 \'mat_p_sim6_3\', \'mat_eff_sim6_3\');
7318 mat_eff_3 <- fread(\''.$effects_heatmap_tempfile_3.'\', header=TRUE, sep=\',\');
7319 mat_eff_altered_3 <- fread(\''.$effects_post_heatmap_tempfile_3.'\', header=TRUE, sep=\',\');
7320 effect_mat_3 <- rbind(mat_eff_3, mat_eff_altered_3);
7321 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
7322 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
7323 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
7324 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
7325 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
7326 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
7327 options(device=\'png\');
7328 par();
7329 gg_eff_3 <- ggplot(effect_mat_3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7330 geom_tile() +
7331 scale_fill_viridis(discrete=FALSE) +
7332 coord_equal() +
7333 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7334 ggsave(\''.$env_effects_first_figure_tempfile.'\', arrangeGrob(gg_eff_3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7335 write.table(data.frame(env_linear = c(cor(mat_env\$value, mat_full\$mat_eff_sim1_3)), env_1DN = c(cor(mat_env2\$value, mat_full\$mat_eff_sim2_3)), env_2DN = c(cor(mat_env3\$value, mat_full\$mat_eff_sim3_3)), env_random = c(cor(mat_env4\$value, mat_full\$mat_eff_sim4_3)), env_ar1xar1 = c(cor(mat_env5\$value, mat_full\$mat_eff_sim5_3)), env_realdata = c(cor(mat_env6\$value, mat_full\$mat_eff_sim6_3)) ), file=\''.$sim_effects_corr_results.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
7336 "';
7337 # print STDERR Dumper $cmd;
7338 my $status_spatialfirst_plot = system($cmd_spatialfirst_plot);
7339 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_originaleffheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7340
7341 open(my $fh_corr_result, '<', $sim_effects_corr_results) or die "Could not open file '$sim_effects_corr_results' $!";
7342 print STDERR "Opened $sim_effects_corr_results\n";
7343
7344 my $header = <$fh_corr_result>;
7345 my @header;
7346 if ($csv->parse($header)) {
7347 @header = $csv->fields();
7348 }
7349
7350 while (my $row = <$fh_corr_result>) {
7351 my @columns;
7352 my $counter = 0;
7353 if ($csv->parse($row)) {
7354 @columns = $csv->fields();
7355 }
7356 foreach (@columns) {
7357 push @{$env_corr_res->{$statistics_select."_".$header[$counter]."_corrtime_".$sim_env_change_over_time.$correlation_between_times."_envvar_".$env_variance_percent}->{values}}, $_;
7358 $counter++;
7359 }
7360 }
7361 close($fh_corr_result);
7362
7363 my $env_effects_sim_figure_tempfile_string_3_env1 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7364 $env_effects_sim_figure_tempfile_string_3_env1 .= '.png';
7365 my $env_effects_sim_figure_tempfile_3_env1 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env1;
7366
7367 my $cmd_spatialenvsim_plot_3_env1 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7368 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
7369 mat_p_sim <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile_3.'\', header=TRUE, sep=\',\');
7370 mat_eff_sim <- fread(\''.$effects_sim_heatmap_tempfile_3.'\', header=TRUE, sep=\',\');
7371 options(device=\'png\');
7372 par();
7373 gg_env <- ggplot(mat_env, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7374 geom_tile() +
7375 scale_fill_viridis(discrete=FALSE) +
7376 coord_equal() +
7377 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7378 gg_p_sim <- ggplot(mat_p_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7379 geom_tile() +
7380 scale_fill_viridis(discrete=FALSE) +
7381 coord_equal() +
7382 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7383 gg_eff_sim <- ggplot(mat_eff_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7384 geom_tile() +
7385 scale_fill_viridis(discrete=FALSE) +
7386 coord_equal() +
7387 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7388 ggsave(\''.$env_effects_sim_figure_tempfile_3_env1.'\', arrangeGrob(gg_env, gg_p_sim, gg_eff_sim, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7389 "';
7390 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env1;
7391 my $status_spatialenvsim_plot_3_env1 = system($cmd_spatialenvsim_plot_3_env1);
7392 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env1, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env1effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7393
7394 my $env_effects_sim_figure_tempfile_string_3_env2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7395 $env_effects_sim_figure_tempfile_string_3_env2 .= '.png';
7396 my $env_effects_sim_figure_tempfile_3_env2 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env2;
7397
7398 my $cmd_spatialenvsim_plot_3_env2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7399 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
7400 mat_p_sim2 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile2_3.'\', header=TRUE, sep=\',\');
7401 mat_eff_sim2 <- fread(\''.$effects_sim_heatmap_tempfile2_3.'\', header=TRUE, sep=\',\');
7402 options(device=\'png\');
7403 par();
7404 gg_env2 <- ggplot(mat_env2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7405 geom_tile() +
7406 scale_fill_viridis(discrete=FALSE) +
7407 coord_equal() +
7408 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7409 gg_p_sim2 <- ggplot(mat_p_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7410 geom_tile() +
7411 scale_fill_viridis(discrete=FALSE) +
7412 coord_equal() +
7413 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7414 gg_eff_sim2 <- ggplot(mat_eff_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7415 geom_tile() +
7416 scale_fill_viridis(discrete=FALSE) +
7417 coord_equal() +
7418 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7419 ggsave(\''.$env_effects_sim_figure_tempfile_3_env2.'\', arrangeGrob(gg_env2, gg_p_sim2, gg_eff_sim2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7420 "';
7421 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env2;
7422 my $status_spatialenvsim_plot_3_env2 = system($cmd_spatialenvsim_plot_3_env2);
7423 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env2effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7424
7425 my $env_effects_sim_figure_tempfile_string_3_env3 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7426 $env_effects_sim_figure_tempfile_string_3_env3 .= '.png';
7427 my $env_effects_sim_figure_tempfile_3_env3 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env3;
7428
7429 my $cmd_spatialenvsim_plot_3_env3 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7430 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
7431 mat_p_sim3 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile3_3.'\', header=TRUE, sep=\',\');
7432 mat_eff_sim3 <- fread(\''.$effects_sim_heatmap_tempfile3_3.'\', header=TRUE, sep=\',\');
7433 options(device=\'png\');
7434 par();
7435 gg_env3 <- ggplot(mat_env3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7436 geom_tile() +
7437 scale_fill_viridis(discrete=FALSE) +
7438 coord_equal() +
7439 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7440 gg_p_sim3 <- ggplot(mat_p_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7441 geom_tile() +
7442 scale_fill_viridis(discrete=FALSE) +
7443 coord_equal() +
7444 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7445 gg_eff_sim3 <- ggplot(mat_eff_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7446 geom_tile() +
7447 scale_fill_viridis(discrete=FALSE) +
7448 coord_equal() +
7449 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7450 ggsave(\''.$env_effects_sim_figure_tempfile_3_env3.'\', arrangeGrob(gg_env3, gg_p_sim3, gg_eff_sim3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7451 "';
7452 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env3;
7453 my $status_spatialenvsim_plot_3_env3 = system($cmd_spatialenvsim_plot_3_env3);
7454 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env3, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env3effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7455
7456 my $env_effects_sim_figure_tempfile_string_3_env4 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7457 $env_effects_sim_figure_tempfile_string_3_env4 .= '.png';
7458 my $env_effects_sim_figure_tempfile_3_env4 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env4;
7459
7460 my $cmd_spatialenvsim_plot_3_env4 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7461 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
7462 mat_p_sim4 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile4_3.'\', header=TRUE, sep=\',\');
7463 mat_eff_sim4 <- fread(\''.$effects_sim_heatmap_tempfile4_3.'\', header=TRUE, sep=\',\');
7464 options(device=\'png\');
7465 par();
7466 gg_env4 <- ggplot(mat_env4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7467 geom_tile() +
7468 scale_fill_viridis(discrete=FALSE) +
7469 coord_equal() +
7470 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7471 gg_p_sim4 <- ggplot(mat_p_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7472 geom_tile() +
7473 scale_fill_viridis(discrete=FALSE) +
7474 coord_equal() +
7475 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7476 gg_eff_sim4 <- ggplot(mat_eff_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7477 geom_tile() +
7478 scale_fill_viridis(discrete=FALSE) +
7479 coord_equal() +
7480 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7481 ggsave(\''.$env_effects_sim_figure_tempfile_3_env4.'\', arrangeGrob(gg_env4, gg_p_sim4, gg_eff_sim4, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7482 "';
7483 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env4;
7484 my $status_spatialenvsim_plot_3_env4 = system($cmd_spatialenvsim_plot_3_env4);
7485 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env4, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env4effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7486
7487 my $env_effects_sim_figure_tempfile_string_3_env5 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7488 $env_effects_sim_figure_tempfile_string_3_env5 .= '.png';
7489 my $env_effects_sim_figure_tempfile_3_env5 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env5;
7490
7491 my $cmd_spatialenvsim_plot_3_env5 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7492 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
7493 mat_p_sim5 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile5_3.'\', header=TRUE, sep=\',\');
7494 mat_eff_sim5 <- fread(\''.$effects_sim_heatmap_tempfile5_3.'\', header=TRUE, sep=\',\');
7495 options(device=\'png\');
7496 par();
7497 gg_env5 <- ggplot(mat_env5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7498 geom_tile() +
7499 scale_fill_viridis(discrete=FALSE) +
7500 coord_equal() +
7501 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7502 gg_p_sim5 <- ggplot(mat_p_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7503 geom_tile() +
7504 scale_fill_viridis(discrete=FALSE) +
7505 coord_equal() +
7506 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7507 gg_eff_sim5 <- ggplot(mat_eff_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7508 geom_tile() +
7509 scale_fill_viridis(discrete=FALSE) +
7510 coord_equal() +
7511 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7512 ggsave(\''.$env_effects_sim_figure_tempfile_3_env5.'\', arrangeGrob(gg_env5, gg_p_sim5, gg_eff_sim5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7513 "';
7514 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env5;
7515 my $status_spatialenvsim_plot_3_env5 = system($cmd_spatialenvsim_plot_3_env5);
7516 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env5, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env5effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7517
7518 my $env_effects_sim_figure_tempfile_string_3_env6 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7519 $env_effects_sim_figure_tempfile_string_3_env6 .= '.png';
7520 my $env_effects_sim_figure_tempfile_3_env6 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_3_env6;
7521
7522 my $cmd_spatialenvsim_plot_3_env6 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
7523 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
7524 mat_p_sim6 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile6_3.'\', header=TRUE, sep=\',\');
7525 mat_eff_sim6 <- fread(\''.$effects_sim_heatmap_tempfile6_3.'\', header=TRUE, sep=\',\');
7526 options(device=\'png\');
7527 par();
7528 gg_env6 <- ggplot(mat_env6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7529 geom_tile() +
7530 scale_fill_viridis(discrete=FALSE) +
7531 coord_equal() +
7532 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7533 gg_p_sim6 <- ggplot(mat_p_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7534 geom_tile() +
7535 scale_fill_viridis(discrete=FALSE) +
7536 coord_equal() +
7537 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7538 gg_eff_sim6 <- ggplot(mat_eff_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
7539 geom_tile() +
7540 scale_fill_viridis(discrete=FALSE) +
7541 coord_equal() +
7542 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_2).');
7543 ggsave(\''.$env_effects_sim_figure_tempfile_3_env6.'\', arrangeGrob(gg_env6, gg_p_sim6, gg_eff_sim6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
7544 "';
7545 # print STDERR Dumper $cmd_spatialenvsim_plot_3_env6;
7546 my $status_spatialenvsim_plot_3_env6 = system($cmd_spatialenvsim_plot_3_env6);
7547 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_3_env6, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env6effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
7548 };
7549
7550 eval {
7551 my @sorted_germplasm_names = sort keys %unique_accessions;
7552
7553 my @original_blup_vals_3;
7554 my ($effects_original_line_chart_tempfile_fh_3, $effects_original_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7555 open(my $F_pheno, ">", $effects_original_line_chart_tempfile_3) || die "Can't open file ".$effects_original_line_chart_tempfile_3;
7556 print $F_pheno "germplasmName,time,value\n";
7557 foreach my $p (@sorted_germplasm_names) {
7558 foreach my $t (@sorted_trait_names_2) {
7559 my $val = $result_blup_data_original_3->{$p}->{$t}->[0];
7560 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7561 push @original_blup_vals_3, $val;
7562 my $line = join ',', @row;
7563 print $F_pheno "$line\n";
7564 }
7565 }
7566 close($F_pheno);
7567
7568 my $original_blup_stat_3 = Statistics::Descriptive::Full->new();
7569 $original_blup_stat_3->add_data(@original_blup_vals_3);
7570 my $sig_original_blup_3 = $original_blup_stat_3->variance();
7571
7572 my @altered_blups_vals_3;
7573 my ($effects_altered_line_chart_tempfile_fh_3, $effects_altered_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7574 open($F_pheno, ">", $effects_altered_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_line_chart_tempfile_3;
7575 print $F_pheno "germplasmName,time,value\n";
7576 foreach my $p (@sorted_germplasm_names) {
7577 foreach my $t (@sorted_trait_names_2) {
7578 my $val = $result_blup_data_altered_3->{$p}->{$t}->[0];
7579 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7580 my $line = join ',', @row;
7581 print $F_pheno "$line\n";
7582 push @altered_blups_vals_3, $val;
7583 }
7584 }
7585 close($F_pheno);
7586
7587 my $altered_blup_stat_3 = Statistics::Descriptive::Full->new();
7588 $altered_blup_stat_3->add_data(@altered_blups_vals_3);
7589 my $sig_altered_blup_3 = $altered_blup_stat_3->variance();
7590
7591 my @sim1_blup_vals_3;
7592 my ($effects_altered_env1_line_chart_tempfile_fh_3, $effects_altered_env1_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7593 open($F_pheno, ">", $effects_altered_env1_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env1_line_chart_tempfile_3;
7594 print $F_pheno "germplasmName,time,value\n";
7595 foreach my $p (@sorted_germplasm_names) {
7596 foreach my $t (@sorted_trait_names_2) {
7597 my $val = $result_blup_data_altered_env_1_3->{$p}->{$t}->[0];
7598 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7599 my $line = join ',', @row;
7600 print $F_pheno "$line\n";
7601 push @sim1_blup_vals_3, $val;
7602 }
7603 }
7604 close($F_pheno);
7605
7606 my $sim1_blup_stat_3 = Statistics::Descriptive::Full->new();
7607 $sim1_blup_stat_3->add_data(@sim1_blup_vals_3);
7608 my $sig_sim1_blup_3 = $sim1_blup_stat_3->variance();
7609
7610 my @sim2_blup_vals_3;
7611 my ($effects_altered_env2_line_chart_tempfile_fh_3, $effects_altered_env2_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7612 open($F_pheno, ">", $effects_altered_env2_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env2_line_chart_tempfile_3;
7613 print $F_pheno "germplasmName,time,value\n";
7614 foreach my $p (@sorted_germplasm_names) {
7615 foreach my $t (@sorted_trait_names_2) {
7616 my $val = $result_blup_data_altered_env_2_3->{$p}->{$t}->[0];
7617 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7618 my $line = join ',', @row;
7619 print $F_pheno "$line\n";
7620 push @sim2_blup_vals_3, $val;
7621 }
7622 }
7623 close($F_pheno);
7624
7625 my $sim2_blup_stat_3 = Statistics::Descriptive::Full->new();
7626 $sim2_blup_stat_3->add_data(@sim2_blup_vals_3);
7627 my $sig_sim2_blup_3 = $sim2_blup_stat_3->variance();
7628
7629 my @sim3_blup_vals_3;
7630 my ($effects_altered_env3_line_chart_tempfile_fh_3, $effects_altered_env3_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7631 open($F_pheno, ">", $effects_altered_env3_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env3_line_chart_tempfile_3;
7632 print $F_pheno "germplasmName,time,value\n";
7633 foreach my $p (@sorted_germplasm_names) {
7634 foreach my $t (@sorted_trait_names_2) {
7635 my $val = $result_blup_data_altered_env_3_3->{$p}->{$t}->[0];
7636 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7637 my $line = join ',', @row;
7638 print $F_pheno "$line\n";
7639 push @sim3_blup_vals_3, $val;
7640 }
7641 }
7642 close($F_pheno);
7643
7644 my $sim3_blup_stat_3 = Statistics::Descriptive::Full->new();
7645 $sim3_blup_stat_3->add_data(@sim3_blup_vals_3);
7646 my $sig_sim3_blup_3 = $sim3_blup_stat_3->variance();
7647
7648 my @sim4_blup_vals_3;
7649 my ($effects_altered_env4_line_chart_tempfile_fh_3, $effects_altered_env4_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7650 open($F_pheno, ">", $effects_altered_env4_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env4_line_chart_tempfile_3;
7651 print $F_pheno "germplasmName,time,value\n";
7652 foreach my $p (@sorted_germplasm_names) {
7653 foreach my $t (@sorted_trait_names_2) {
7654 my $val = $result_blup_data_altered_env_4_3->{$p}->{$t}->[0];
7655 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7656 my $line = join ',', @row;
7657 print $F_pheno "$line\n";
7658 push @sim4_blup_vals_3, $val;
7659 }
7660 }
7661 close($F_pheno);
7662
7663 my $sim4_blup_stat_3 = Statistics::Descriptive::Full->new();
7664 $sim4_blup_stat_3->add_data(@sim4_blup_vals_3);
7665 my $sig_sim4_blup_3 = $sim4_blup_stat_3->variance();
7666
7667 my @sim5_blup_vals_3;
7668 my ($effects_altered_env5_line_chart_tempfile_fh_3, $effects_altered_env5_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7669 open($F_pheno, ">", $effects_altered_env5_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env5_line_chart_tempfile_3;
7670 print $F_pheno "germplasmName,time,value\n";
7671 foreach my $p (@sorted_germplasm_names) {
7672 foreach my $t (@sorted_trait_names_2) {
7673 my $val = $result_blup_data_altered_env_5_3->{$p}->{$t}->[0];
7674 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7675 my $line = join ',', @row;
7676 print $F_pheno "$line\n";
7677 push @sim5_blup_vals_3, $val;
7678 }
7679 }
7680 close($F_pheno);
7681
7682 my $sim5_blup_stat_3 = Statistics::Descriptive::Full->new();
7683 $sim5_blup_stat_3->add_data(@sim5_blup_vals_3);
7684 my $sig_sim5_blup_3 = $sim5_blup_stat_3->variance();
7685
7686 my @sim6_blup_vals_3;
7687 my ($effects_altered_env6_line_chart_tempfile_fh_3, $effects_altered_env6_line_chart_tempfile_3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
7688 open($F_pheno, ">", $effects_altered_env6_line_chart_tempfile_3) || die "Can't open file ".$effects_altered_env6_line_chart_tempfile_3;
7689 print $F_pheno "germplasmName,time,value\n";
7690 foreach my $p (@sorted_germplasm_names) {
7691 foreach my $t (@sorted_trait_names_2) {
7692 my $val = $result_blup_data_altered_env_6_3->{$p}->{$t}->[0];
7693 my @row = ($p, $trait_to_time_map_2{$t}, $val);
7694 my $line = join ',', @row;
7695 print $F_pheno "$line\n";
7696 push @sim6_blup_vals_3, $val;
7697 }
7698 }
7699 close($F_pheno);
7700
7701 my $sim6_blup_stat_3 = Statistics::Descriptive::Full->new();
7702 $sim6_blup_stat_3->add_data(@sim6_blup_vals_3);
7703 my $sig_sim6_blup_3 = $sim6_blup_stat_3->variance();
7704
7705 my @set = ('0' ..'9', 'A' .. 'F');
7706 my @colors;
7707 for (1..scalar(@sorted_germplasm_names)) {
7708 my $str = join '' => map $set[rand @set], 1 .. 6;
7709 push @colors, '#'.$str;
7710 }
7711 my $color_string = join '\',\'', @colors;
7712
7713 my $genetic_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7714 $genetic_effects_figure_tempfile_string .= '.png';
7715 my $genetic_effects_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_figure_tempfile_string;
7716
7717 my $genetic_effects_alt_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7718 $genetic_effects_alt_figure_tempfile_string .= '.png';
7719 my $genetic_effects_alt_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_figure_tempfile_string;
7720
7721 my $genetic_effects_alt_env1_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7722 $genetic_effects_alt_env1_figure_tempfile_string .= '.png';
7723 my $genetic_effects_alt_env1_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env1_figure_tempfile_string;
7724
7725 my $genetic_effects_alt_env2_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7726 $genetic_effects_alt_env2_figure_tempfile_string .= '.png';
7727 my $genetic_effects_alt_env2_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env2_figure_tempfile_string;
7728
7729 my $genetic_effects_alt_env3_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7730 $genetic_effects_alt_env3_figure_tempfile_string .= '.png';
7731 my $genetic_effects_alt_env3_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env3_figure_tempfile_string;
7732
7733 my $genetic_effects_alt_env4_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7734 $genetic_effects_alt_env4_figure_tempfile_string .= '.png';
7735 my $genetic_effects_alt_env4_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env4_figure_tempfile_string;
7736
7737 my $genetic_effects_alt_env5_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7738 $genetic_effects_alt_env5_figure_tempfile_string .= '.png';
7739 my $genetic_effects_alt_env5_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env5_figure_tempfile_string;
7740
7741 my $genetic_effects_alt_env6_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
7742 $genetic_effects_alt_env6_figure_tempfile_string .= '.png';
7743 my $genetic_effects_alt_env6_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env6_figure_tempfile_string;
7744
7745 my $cmd_gen_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7746 mat <- fread(\''.$effects_original_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7747 mat\$time <- as.numeric(as.character(mat\$time));
7748 options(device=\'png\');
7749 par();
7750 sp <- ggplot(mat, aes(x = time, y = value)) +
7751 geom_line(aes(color = germplasmName), size = 1) +
7752 scale_fill_manual(values = c(\''.$color_string.'\')) +
7753 theme_minimal();
7754 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7755 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7756 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7757 sp <- sp + labs(title = \'Original Genetic Effects\');';
7758 if (scalar(@sorted_germplasm_names) > 100) {
7759 $cmd_gen_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7760 }
7761 $cmd_gen_plot .= 'ggsave(\''.$genetic_effects_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7762 "';
7763 print STDERR Dumper $cmd_gen_plot;
7764 my $status_gen_plot = system($cmd_gen_plot);
7765 push @$spatial_effects_plots, [$genetic_effects_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_efforigline_"."envvar_".$env_variance_percent."_".$iterations];
7766
7767 my $cmd_gen_alt_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7768 mat <- fread(\''.$effects_altered_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7769 mat\$time <- as.numeric(as.character(mat\$time));
7770 options(device=\'png\');
7771 par();
7772 sp <- ggplot(mat, aes(x = time, y = value)) +
7773 geom_line(aes(color = germplasmName), size = 1) +
7774 scale_fill_manual(values = c(\''.$color_string.'\')) +
7775 theme_minimal();
7776 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7777 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7778 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7779 sp <- sp + labs(title = \'Altered Genetic Effects\');';
7780 if (scalar(@sorted_germplasm_names) > 100) {
7781 $cmd_gen_alt_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7782 }
7783 $cmd_gen_alt_plot .= 'ggsave(\''.$genetic_effects_alt_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7784 "';
7785 print STDERR Dumper $cmd_gen_alt_plot;
7786 my $status_gen_alt_plot = system($cmd_gen_alt_plot);
7787 push @$spatial_effects_plots, [$genetic_effects_alt_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltline_"."envvar_".$env_variance_percent."_".$iterations];
7788
7789 my $cmd_gen_env1_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7790 mat <- fread(\''.$effects_altered_env1_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7791 mat\$time <- as.numeric(as.character(mat\$time));
7792 options(device=\'png\');
7793 par();
7794 sp <- ggplot(mat, aes(x = time, y = value)) +
7795 geom_line(aes(color = germplasmName), size = 1) +
7796 scale_fill_manual(values = c(\''.$color_string.'\')) +
7797 theme_minimal();
7798 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7799 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7800 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7801 sp <- sp + labs(title = \'SimLinear Genetic Effects\');';
7802 if (scalar(@sorted_germplasm_names) > 100) {
7803 $cmd_gen_env1_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7804 }
7805 $cmd_gen_env1_plot .= 'ggsave(\''.$genetic_effects_alt_env1_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7806 "';
7807 print STDERR Dumper $cmd_gen_env1_plot;
7808 my $status_gen_env1_plot = system($cmd_gen_env1_plot);
7809 push @$spatial_effects_plots, [$genetic_effects_alt_env1_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv1line_"."envvar_".$env_variance_percent."_".$iterations];
7810
7811 my $cmd_gen_env2_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7812 mat <- fread(\''.$effects_altered_env2_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7813 mat\$time <- as.numeric(as.character(mat\$time));
7814 options(device=\'png\');
7815 par();
7816 sp <- ggplot(mat, aes(x = time, y = value)) +
7817 geom_line(aes(color = germplasmName), size = 1) +
7818 scale_fill_manual(values = c(\''.$color_string.'\')) +
7819 theme_minimal();
7820 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7821 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7822 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7823 sp <- sp + labs(title = \'Sim1DN Genetic Effects\');';
7824 if (scalar(@sorted_germplasm_names) > 100) {
7825 $cmd_gen_env2_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7826 }
7827 $cmd_gen_env2_plot .= 'ggsave(\''.$genetic_effects_alt_env2_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7828 "';
7829 print STDERR Dumper $cmd_gen_env2_plot;
7830 my $status_gen_env2_plot = system($cmd_gen_env2_plot);
7831 push @$spatial_effects_plots, [$genetic_effects_alt_env2_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv2line_"."envvar_".$env_variance_percent."_".$iterations];
7832
7833 my $cmd_gen_env3_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7834 mat <- fread(\''.$effects_altered_env3_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7835 mat\$time <- as.numeric(as.character(mat\$time));
7836 options(device=\'png\');
7837 par();
7838 sp <- ggplot(mat, aes(x = time, y = value)) +
7839 geom_line(aes(color = germplasmName), size = 1) +
7840 scale_fill_manual(values = c(\''.$color_string.'\')) +
7841 theme_minimal();
7842 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7843 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7844 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7845 sp <- sp + labs(title = \'Sim2DN Genetic Effects\');';
7846 if (scalar(@sorted_germplasm_names) > 100) {
7847 $cmd_gen_env3_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7848 }
7849 $cmd_gen_env3_plot .= 'ggsave(\''.$genetic_effects_alt_env3_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7850 "';
7851 print STDERR Dumper $cmd_gen_env3_plot;
7852 my $status_gen_env3_plot = system($cmd_gen_env3_plot);
7853 push @$spatial_effects_plots, [$genetic_effects_alt_env3_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv3line_"."envvar_".$env_variance_percent."_".$iterations];
7854
7855 my $cmd_gen_env4_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7856 mat <- fread(\''.$effects_altered_env4_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7857 mat\$time <- as.numeric(as.character(mat\$time));
7858 options(device=\'png\');
7859 par();
7860 sp <- ggplot(mat, aes(x = time, y = value)) +
7861 geom_line(aes(color = germplasmName), size = 1) +
7862 scale_fill_manual(values = c(\''.$color_string.'\')) +
7863 theme_minimal();
7864 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7865 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7866 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7867 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
7868 if (scalar(@sorted_germplasm_names) > 100) {
7869 $cmd_gen_env4_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7870 }
7871 $cmd_gen_env4_plot .= 'ggsave(\''.$genetic_effects_alt_env4_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7872 "';
7873 print STDERR Dumper $cmd_gen_env4_plot;
7874 my $status_gen_env4_plot = system($cmd_gen_env4_plot);
7875 push @$spatial_effects_plots, [$genetic_effects_alt_env4_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv4line_"."envvar_".$env_variance_percent."_".$iterations];
7876
7877 my $cmd_gen_env5_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7878 mat <- fread(\''.$effects_altered_env5_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7879 mat\$time <- as.numeric(as.character(mat\$time));
7880 options(device=\'png\');
7881 par();
7882 sp <- ggplot(mat, aes(x = time, y = value)) +
7883 geom_line(aes(color = germplasmName), size = 1) +
7884 scale_fill_manual(values = c(\''.$color_string.'\')) +
7885 theme_minimal();
7886 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7887 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7888 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7889 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
7890 if (scalar(@sorted_germplasm_names) > 100) {
7891 $cmd_gen_env5_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7892 }
7893 $cmd_gen_env5_plot .= 'ggsave(\''.$genetic_effects_alt_env5_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7894 "';
7895 print STDERR Dumper $cmd_gen_env5_plot;
7896 my $status_gen_env5_plot = system($cmd_gen_env5_plot);
7897 push @$spatial_effects_plots, [$genetic_effects_alt_env5_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv5line_"."envvar_".$env_variance_percent."_".$iterations];
7898
7899 my $cmd_gen_env6_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
7900 mat <- fread(\''.$effects_altered_env6_line_chart_tempfile_3.'\', header=TRUE, sep=\',\');
7901 mat\$time <- as.numeric(as.character(mat\$time));
7902 options(device=\'png\');
7903 par();
7904 sp <- ggplot(mat, aes(x = time, y = value)) +
7905 geom_line(aes(color = germplasmName), size = 1) +
7906 scale_fill_manual(values = c(\''.$color_string.'\')) +
7907 theme_minimal();
7908 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
7909 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
7910 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
7911 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
7912 if (scalar(@sorted_germplasm_names) > 100) {
7913 $cmd_gen_env6_plot .= 'sp <- sp + theme(legend.position = \'none\');';
7914 }
7915 $cmd_gen_env6_plot .= 'ggsave(\''.$genetic_effects_alt_env6_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
7916 "';
7917 print STDERR Dumper $cmd_gen_env6_plot;
7918 my $status_gen_env6_plot = system($cmd_gen_env6_plot);
7919 push @$spatial_effects_plots, [$genetic_effects_alt_env6_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv6line_"."envvar_".$env_variance_percent."_".$iterations];
7920 };
7921
7922 %trait_name_encoder = %trait_name_encoder_2;
7923 %trait_to_time_map = %trait_to_time_map_2;
7924
7925 push @$env_varcomps, {
7926 type => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times : Iteration $iterations",
7927 statistics_select => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times",
7928 correlation_between_times => $correlation_between_times,
7929 env_variance => $env_variance_percent,
7930 original => {
7931 covariance => $varcomp_original_array_3,
7932 cv_1 => $result_cv_original_3,
7933 cv_2 => $result_cv_2_original_3
7934 },
7935 altered => {
7936 covariance => $varcomp_altered_array_3,
7937 cv_1 => $result_cv_altered_3,
7938 cv_2 => $result_cv_2_altered_3
7939 },
7940 env_linear => {
7941 covariance => $varcomp_altered_array_env_1_3,
7942 cv_1 => $result_cv_altered_env_1_3,
7943 cv_2 => $result_cv_2_altered_env_1_3
7944 },
7945 env_1DN => {
7946 covariance => $varcomp_altered_array_env_2_3,
7947 cv_1 => $result_cv_altered_env_2_3,
7948 cv_2 => $result_cv_2_altered_env_2_3
7949 },
7950 env_2DN => {
7951 covariance => $varcomp_altered_array_env_3_3,
7952 cv_1 => $result_cv_altered_env_3_3,
7953 cv_2 => $result_cv_2_altered_env_3_3
7954 },
7955 env_random => {
7956 covariance => $varcomp_altered_array_env_4_3,
7957 cv_1 => $result_cv_altered_env_4_3,
7958 cv_2 => $result_cv_2_altered_env_4_3
7959 },
7960 env_ar1xar1 => {
7961 covariance => $varcomp_altered_array_env_5_3,
7962 cv_1 => $result_cv_altered_env_5_3,
7963 cv_2 => $result_cv_2_altered_env_5_3
7964 },
7965 env_realdata => {
7966 covariance => $varcomp_altered_array_env_6_3,
7967 cv_1 => $result_cv_altered_env_6_3,
7968 cv_2 => $result_cv_2_altered_env_6_3
7969 }
7970 };
7971 }
7972
7973 my $return_inverse_matrix = 0;
7974 my $ensure_positive_definite = 1;
7975
7976 my (%phenotype_data_original_5, @data_matrix_original_5, @data_matrix_phenotypes_original_5);
7977 my (%trait_name_encoder_5, %trait_name_encoder_rev_5, %seen_days_after_plantings_5, %stock_info_5, %seen_times_5, %seen_trial_ids_5, %trait_to_time_map_5, %trait_composing_info_5, @sorted_trait_names_5, %seen_trait_names_5, %unique_traits_ids_5, @phenotype_header_5, $header_string_5);
7978 my (@sorted_scaled_ln_times_5, %plot_id_factor_map_reverse_5, %plot_id_count_map_reverse_5, %accession_id_factor_map_5, %accession_id_factor_map_reverse_5, %time_count_map_reverse_5, @rep_time_factors_5, @ind_rep_factors_5, %plot_rep_time_factor_map_5, %seen_rep_times_5, %seen_ind_reps_5, @legs_header_5, %polynomial_map_5);
7979 my $time_min_5 = 100000000;
7980 my $time_max_5 = 0;
7981 my $phenotype_min_original_5 = 1000000000;
7982 my $phenotype_max_original_5 = -1000000000;
7983
7984 if ($statistics_select_original eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select_original eq 'asreml_grm_univariate_pure_spatial_genetic_blups') {
7985 $statistics_select = $statistics_select_original;
7986
7987 print STDERR "PREPARE ORIGINAL PHENOTYPE FILES 5\n";
7988 eval {
7989 my $phenotypes_search_5 = CXGN::Phenotypes::SearchFactory->instantiate(
7990 'MaterializedViewTable',
7991 {
7992 bcs_schema=>$schema,
7993 data_level=>'plot',
7994 trait_list=>$trait_id_list,
7995 trial_list=>$field_trial_id_list,
7996 include_timestamp=>0,
7997 exclude_phenotype_outlier=>0
7998 }
7999 );
8000 my ($data_5, $unique_traits_5) = $phenotypes_search_5->search();
8001 @sorted_trait_names_5 = sort keys %$unique_traits_5;
8002
8003 if (scalar(@$data_5) == 0) {
8004 $c->stash->{rest} = { error => "There are no phenotypes for the trials and traits you have selected!"};
8005 return;
8006 }
8007
8008 $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
8009 $h_time = $schema->storage->dbh()->prepare($q_time);
8010
8011 foreach my $obs_unit (@$data_5){
8012 my $germplasm_name = $obs_unit->{germplasm_uniquename};
8013 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
8014 my $replicate_number = $obs_unit->{obsunit_rep} || '';
8015 my $block_number = $obs_unit->{obsunit_block} || '';
8016 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
8017 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
8018 my $row_number = $obs_unit->{obsunit_row_number} || '';
8019 my $col_number = $obs_unit->{obsunit_col_number} || '';
8020 push @plot_ids_ordered, $obsunit_stock_id;
8021
8022 if ($row_number < $min_row) {
8023 $min_row = $row_number;
8024 }
8025 elsif ($row_number >= $max_row) {
8026 $max_row = $row_number;
8027 }
8028 if ($col_number < $min_col) {
8029 $min_col = $col_number;
8030 }
8031 elsif ($col_number >= $max_col) {
8032 $max_col = $col_number;
8033 }
8034
8035 $obsunit_row_col{$row_number}->{$col_number} = {
8036 stock_id => $obsunit_stock_id,
8037 stock_uniquename => $obsunit_stock_uniquename
8038 };
8039 $seen_rows{$row_number}++;
8040 $seen_cols{$col_number}++;
8041 $plot_id_map{$obsunit_stock_id} = $obsunit_stock_uniquename;
8042 $seen_plot_names{$obsunit_stock_uniquename}++;
8043 $seen_plots{$obsunit_stock_id} = $obsunit_stock_uniquename;
8044 $stock_row_col{$obsunit_stock_id} = {
8045 row_number => $row_number,
8046 col_number => $col_number,
8047 obsunit_stock_id => $obsunit_stock_id,
8048 obsunit_name => $obsunit_stock_uniquename,
8049 rep => $replicate_number,
8050 block => $block_number,
8051 germplasm_stock_id => $germplasm_stock_id,
8052 germplasm_name => $germplasm_name
8053 };
8054 $stock_name_row_col{$obsunit_stock_uniquename} = {
8055 row_number => $row_number,
8056 col_number => $col_number,
8057 obsunit_stock_id => $obsunit_stock_id,
8058 obsunit_name => $obsunit_stock_uniquename,
8059 rep => $replicate_number,
8060 block => $block_number,
8061 germplasm_stock_id => $germplasm_stock_id,
8062 germplasm_name => $germplasm_name
8063 };
8064 $stock_row_col_id{$row_number}->{$col_number} = $obsunit_stock_id;
8065 $unique_accessions{$germplasm_name}++;
8066 $stock_info_5{$germplasm_stock_id} = {
8067 uniquename => $germplasm_name
8068 };
8069 my $observations = $obs_unit->{observations};
8070 foreach (@$observations){
8071 if ($_->{associated_image_project_time_json}) {
8072 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
8073 my $time_days_cvterm = $related_time_terms_json->{day};
8074 my $time_term_string = $time_days_cvterm;
8075 my $time_days = (split '\|', $time_days_cvterm)[0];
8076 my $time = (split ' ', $time_days)[1] + 0;
8077 $seen_days_after_plantings_5{$time}++;
8078
8079 my $value = $_->{value};
8080 my $trait_name = $_->{trait_name};
8081 $phenotype_data_original_5{$obsunit_stock_uniquename}->{$time} = $value;
8082 $seen_times_5{$time} = $trait_name;
8083 $seen_trait_names_5{$trait_name} = $time_term_string;
8084 $trait_to_time_map_5{$trait_name} = $time;
8085
8086 if ($value < $phenotype_min_original_5) {
8087 $phenotype_min_original_5 = $value;
8088 }
8089 elsif ($value >= $phenotype_max_original_5) {
8090 $phenotype_max_original_5 = $value;
8091 }
8092 }
8093 }
8094 }
8095 if (scalar(keys %seen_times_5) == 0) {
8096 $c->stash->{rest} = { error => "There are no phenotypes with associated days after planting time associated to the traits you have selected!"};
8097 return;
8098 }
8099
8100 @unique_accession_names = sort keys %unique_accessions;
8101 @sorted_trait_names_5 = sort {$a <=> $b} keys %seen_times_5;
8102 # print STDERR Dumper \@sorted_trait_names_5;
8103
8104 my $trait_name_encoded_5 = 1;
8105 foreach my $trait_name (@sorted_trait_names_5) {
8106 if (!exists($trait_name_encoder_5{$trait_name})) {
8107 my $trait_name_e = 't'.$trait_name_encoded_5;
8108 $trait_name_encoder_5{$trait_name} = $trait_name_e;
8109 $trait_name_encoder_rev_5{$trait_name_e} = $trait_name;
8110 $trait_name_encoded_5++;
8111 }
8112 }
8113
8114 foreach (@sorted_trait_names_5) {
8115 if ($_ < $time_min_5) {
8116 $time_min_5 = $_;
8117 }
8118 if ($_ >= $time_max_5) {
8119 $time_max_5 = $_;
8120 }
8121 }
8122 print STDERR Dumper [$time_min_5, $time_max_5];
8123
8124 while ( my ($trait_name, $time_term) = each %seen_trait_names_5) {
8125 push @{$trait_composing_info_5{$trait_name}}, $time_term;
8126 }
8127
8128 @unique_plot_names = sort keys %seen_plot_names;
8129
8130 open(my $F_prep, ">", $stats_prep_tempfile) || die "Can't open file ".$stats_prep_tempfile;
8131 print $F_prep "accession_id,accession_id_factor,plot_id,plot_id_factor,replicate,time,replicate_time,ind_replicate\n";
8132 foreach my $p (@unique_plot_names) {
8133 my $replicate = $stock_name_row_col{$p}->{rep};
8134 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
8135 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
8136 foreach my $t (@sorted_trait_names_5) {
8137 print $F_prep "$germplasm_stock_id,,$obsunit_stock_id,,$replicate,$t,$replicate"."_"."$t,$germplasm_stock_id"."_"."$replicate\n";
8138 }
8139 }
8140 close($F_prep);
8141
8142 my $cmd_factor = 'R -e "library(data.table);
8143 mat <- fread(\''.$stats_prep_tempfile.'\', header=TRUE, sep=\',\');
8144 mat\$replicate_time <- as.numeric(as.factor(mat\$replicate_time));
8145 mat\$ind_replicate <- as.numeric(as.factor(mat\$ind_replicate));
8146 mat\$accession_id_factor <- as.numeric(as.factor(mat\$accession_id));
8147 mat\$plot_id_factor <- as.numeric(as.factor(mat\$plot_id));
8148 write.table(mat, file=\''.$stats_prep_factor_tempfile.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');"';
8149 my $status_factor = system($cmd_factor);
8150
8151 open(my $fh_factor, '<', $stats_prep_factor_tempfile) or die "Could not open file '$stats_prep_factor_tempfile' $!";
8152 print STDERR "Opened $stats_prep_factor_tempfile\n";
8153 my $header_d = <$fh_factor>;
8154
8155 my $line_factor_count = 0;
8156 while (my $row = <$fh_factor>) {
8157 my @columns;
8158 if ($csv->parse($row)) {
8159 @columns = $csv->fields();
8160 }
8161 my $accession_id = $columns[0];
8162 my $accession_id_factor = $columns[1];
8163 my $plot_id = $columns[2];
8164 my $plot_id_factor = $columns[3];
8165 my $rep = $columns[4];
8166 my $time = $columns[5];
8167 my $rep_time = $columns[6];
8168 my $ind_rep = $columns[7];
8169 $stock_row_col{$plot_id}->{plot_id_factor} = $plot_id_factor;
8170 $stock_name_row_col{$plot_id_map{$plot_id}}->{plot_id_factor} = $plot_id_factor;
8171 $plot_rep_time_factor_map{$plot_id}->{$rep}->{$time} = $rep_time;
8172 $seen_rep_times{$rep_time}++;
8173 $seen_ind_reps{$plot_id_factor}++;
8174 $accession_id_factor_map{$accession_id} = $accession_id_factor;
8175 $accession_id_factor_map_reverse{$accession_id_factor} = $stock_info_5{$accession_id}->{uniquename};
8176 $plot_id_factor_map_reverse{$plot_id_factor} = $seen_plots{$plot_id};
8177 $plot_id_count_map_reverse{$line_factor_count} = $seen_plots{$plot_id};
8178 $time_count_map_reverse{$line_factor_count} = $time;
8179 $line_factor_count++;
8180 }
8181 close($fh_factor);
8182 @rep_time_factors = sort keys %seen_rep_times;
8183 @ind_rep_factors = sort keys %seen_ind_reps;
8184
8185 foreach my $p (@unique_plot_names) {
8186 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
8187 if ($fixed_effect_type eq 'fixed_effect_trait') {
8188 $stock_name_row_col{$p}->{rep} = defined($fixed_effect_trait_data->{$obsunit_stock_id}) ? $fixed_effect_trait_data->{$obsunit_stock_id} : 0;
8189 }
8190 my $row_number = $stock_name_row_col{$p}->{row_number};
8191 my $col_number = $stock_name_row_col{$p}->{col_number};
8192 my $replicate = $stock_name_row_col{$p}->{rep};
8193 my $block = $stock_name_row_col{$p}->{block};
8194 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
8195 my $germplasm_name = $stock_name_row_col{$p}->{germplasm_name};
8196
8197 my $current_trait_index = 0;
8198 my @row = (
8199 $germplasm_stock_id,
8200 $obsunit_stock_id,
8201 $replicate,
8202 $row_number,
8203 $col_number,
8204 $accession_id_factor_map{$germplasm_stock_id},
8205 $stock_row_col{$obsunit_stock_id}->{plot_id_factor}
8206 );
8207
8208 foreach my $t (@sorted_trait_names_5) {
8209 if (defined($phenotype_data_original_5{$p}->{$t})) {
8210 push @row, $phenotype_data_original_5{$p}->{$t} + 0;
8211 } else {
8212 print STDERR $p." : $t : $germplasm_name : NA \n";
8213 push @row, '';
8214 }
8215
8216 $current_trait_index++;
8217 }
8218 push @data_matrix_original_5, \@row;
8219 }
8220
8221 @phenotype_header_5 = ("id", "plot_id", "replicate", "rowNumber", "colNumber", "id_factor", "plot_id_factor");
8222 foreach (@sorted_trait_names_5) {
8223 push @phenotype_header_5, "t$_";
8224 }
8225 $header_string_5 = join ',', @phenotype_header_5;
8226
8227 open($F, ">", $stats_tempfile_2) || die "Can't open file ".$stats_tempfile_2;
8228 print $F $header_string_5."\n";
8229 foreach (@data_matrix_original_5) {
8230 my $line = join ',', @$_;
8231 print $F "$line\n";
8232 }
8233 close($F);
8234 };
8235
8236 @seen_rows_array = keys %seen_rows;
8237 @seen_cols_array = keys %seen_cols;
8238 $row_stat = Statistics::Descriptive::Full->new();
8239 $row_stat->add_data(@seen_rows_array);
8240 $mean_row = $row_stat->mean();
8241 $sig_row = $row_stat->variance();
8242 $col_stat = Statistics::Descriptive::Full->new();
8243 $col_stat->add_data(@seen_cols_array);
8244 $mean_col = $col_stat->mean();
8245 $sig_col = $col_stat->variance();
8246
8247 print STDERR "PREPARE RELATIONSHIP MATRIX\n";
8248 eval {
8249 my %seen_accession_stock_ids;
8250 foreach my $trial_id (@$field_trial_id_list) {
8251 my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id });
8252 my $accessions = $trial->get_accessions();
8253 foreach (@$accessions) {
8254 $seen_accession_stock_ids{$_->{stock_id}}++;
8255 }
8256 }
8257 my @accession_ids = keys %seen_accession_stock_ids;
8258
8259 if ($compute_relationship_matrix_from_htp_phenotypes eq 'genotypes') {
8260
8261 if ($include_pedgiree_info_if_compute_from_parents) {
8262 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
8263 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
8264 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
8265 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
8266 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
8267 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
8268 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
8269 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
8270
8271 if (!$protocol_id) {
8272 $protocol_id = undef;
8273 }
8274
8275 my $pedigree_arm = CXGN::Pedigree::ARM->new({
8276 bcs_schema=>$schema,
8277 arm_temp_file=>$arm_tempfile,
8278 people_schema=>$people_schema,
8279 accession_id_list=>\@accession_ids,
8280 # plot_id_list=>\@plot_id_list,
8281 cache_root=>$c->config->{cache_file_path},
8282 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
8283 });
8284 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
8285 $shared_cluster_dir_config,
8286 $c->config->{backend},
8287 $c->config->{cluster_host},
8288 $c->config->{'web_cluster_queue'},
8289 $c->config->{basepath}
8290 );
8291 # print STDERR Dumper $parent_hash;
8292
8293 my $female_geno = CXGN::Genotype::GRM->new({
8294 bcs_schema=>$schema,
8295 grm_temp_file=>$grm1_tempfile,
8296 people_schema=>$people_schema,
8297 cache_root=>$c->config->{cache_file_path},
8298 accession_id_list=>$female_stock_ids,
8299 protocol_id=>$protocol_id,
8300 get_grm_for_parental_accessions=>0,
8301 download_format=>'three_column_reciprocal'
8302 # minor_allele_frequency=>$minor_allele_frequency,
8303 # marker_filter=>$marker_filter,
8304 # individuals_filter=>$individuals_filter
8305 });
8306 my $female_grm_data = $female_geno->download_grm(
8307 'data',
8308 $shared_cluster_dir_config,
8309 $c->config->{backend},
8310 $c->config->{cluster_host},
8311 $c->config->{'web_cluster_queue'},
8312 $c->config->{basepath}
8313 );
8314 my @fl = split '\n', $female_grm_data;
8315 my %female_parent_grm;
8316 foreach (@fl) {
8317 my @l = split '\t', $_;
8318 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
8319 }
8320 # print STDERR Dumper \%female_parent_grm;
8321
8322 my $male_geno = CXGN::Genotype::GRM->new({
8323 bcs_schema=>$schema,
8324 grm_temp_file=>$grm1_tempfile,
8325 people_schema=>$people_schema,
8326 cache_root=>$c->config->{cache_file_path},
8327 accession_id_list=>$male_stock_ids,
8328 protocol_id=>$protocol_id,
8329 get_grm_for_parental_accessions=>0,
8330 download_format=>'three_column_reciprocal'
8331 # minor_allele_frequency=>$minor_allele_frequency,
8332 # marker_filter=>$marker_filter,
8333 # individuals_filter=>$individuals_filter
8334 });
8335 my $male_grm_data = $male_geno->download_grm(
8336 'data',
8337 $shared_cluster_dir_config,
8338 $c->config->{backend},
8339 $c->config->{cluster_host},
8340 $c->config->{'web_cluster_queue'},
8341 $c->config->{basepath}
8342 );
8343 my @ml = split '\n', $male_grm_data;
8344 my %male_parent_grm;
8345 foreach (@ml) {
8346 my @l = split '\t', $_;
8347 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
8348 }
8349 # print STDERR Dumper \%male_parent_grm;
8350
8351 my %rel_result_hash;
8352 foreach my $a1 (@accession_ids) {
8353 foreach my $a2 (@accession_ids) {
8354 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
8355 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
8356 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
8357 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
8358
8359 my $female_rel = 0;
8360 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
8361 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
8362 }
8363 elsif ($female_parent1 && $female_parent2 && $female_parent1 == $female_parent2) {
8364 $female_rel = 1;
8365 }
8366 elsif ($a1 == $a2) {
8367 $female_rel = 1;
8368 }
8369
8370 my $male_rel = 0;
8371 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
8372 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
8373 }
8374 elsif ($male_parent1 && $male_parent2 && $male_parent1 == $male_parent2) {
8375 $male_rel = 1;
8376 }
8377 elsif ($a1 == $a2) {
8378 $male_rel = 1;
8379 }
8380 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
8381
8382 my $rel = 0.5*($female_rel + $male_rel);
8383 $rel_result_hash{$a1}->{$a2} = $rel;
8384 }
8385 }
8386 # print STDERR Dumper \%rel_result_hash;
8387
8388 my $data = '';
8389 my %result_hash;
8390 foreach my $s (sort @accession_ids) {
8391 foreach my $c (sort @accession_ids) {
8392 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8393 my $val = $rel_result_hash{$s}->{$c};
8394 if (defined $val and length $val) {
8395 $result_hash{$s}->{$c} = $val;
8396 $data .= "S$s\tS$c\t$val\n";
8397 }
8398 }
8399 }
8400 }
8401
8402 # print STDERR Dumper $data;
8403 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
8404 print $F2 $data;
8405 close($F2);
8406
8407 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
8408 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
8409 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
8410 A_1 <- A_wide[,-1];
8411 A_1[is.na(A_1)] <- 0;
8412 A <- A_1 + t(A_1);
8413 diag(A) <- diag(as.matrix(A_1));
8414 E = eigen(A);
8415 ev = E\$values;
8416 U = E\$vectors;
8417 no = dim(A)[1];
8418 nev = which(ev < 0);
8419 wr = 0;
8420 k=length(nev);
8421 if(k > 0){
8422 p = ev[no - k];
8423 B = sum(ev[nev])*2.0;
8424 wr = (B*B*100.0)+1;
8425 val = ev[nev];
8426 ev[nev] = p*(B-val)*(B-val)/wr;
8427 A = U%*%diag(ev)%*%t(U);
8428 }
8429 ';
8430 if ($return_inverse_matrix) {
8431 $cmd .= 'A <- solve(A);
8432 ';
8433 }
8434 $cmd .= 'A <- as.data.frame(A);
8435 colnames(A) <- A_wide[,1];
8436 A\$stock_id <- A_wide[,1];
8437 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
8438 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
8439 A_threecol\$variable <- substring(A_threecol\$variable, 2);
8440 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
8441 print STDERR $cmd."\n";
8442 my $status = system($cmd);
8443
8444 my %rel_pos_def_result_hash;
8445 open(my $F3, '<', $grm_out_tempfile)
8446 or die "Could not open file '$grm_out_tempfile' $!";
8447
8448 print STDERR "Opened $grm_out_tempfile\n";
8449
8450 while (my $row = <$F3>) {
8451 my @columns;
8452 if ($csv->parse($row)) {
8453 @columns = $csv->fields();
8454 }
8455 my $stock_id1 = $columns[0];
8456 my $stock_id2 = $columns[1];
8457 my $val = $columns[2];
8458 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
8459 }
8460 close($F3);
8461
8462 my $data_pos_def = '';
8463 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_univariate_pure_spatial_genetic_blups') {
8464 my %result_hash;
8465 foreach my $s (sort @accession_ids) {
8466 foreach my $c (sort @accession_ids) {
8467 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8468 my $val = $rel_pos_def_result_hash{$s}->{$c};
8469 if (defined $val and length $val) {
8470 $result_hash{$s}->{$c} = $val;
8471 $data_pos_def .= "$s\t$c\t$val\n";
8472 }
8473 }
8474 }
8475 }
8476 }
8477 else {
8478 my %result_hash;
8479 foreach my $s (sort @accession_ids) {
8480 foreach my $c (sort @accession_ids) {
8481 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8482 my $val = $rel_pos_def_result_hash{$s}->{$c};
8483 if (defined $val and length $val) {
8484 $result_hash{$s}->{$c} = $val;
8485 $result_hash{$c}->{$s} = $val;
8486 $data_pos_def .= "S$s\tS$c\t$val\n";
8487 if ($s != $c) {
8488 $data_pos_def .= "S$c\tS$s\t$val\n";
8489 }
8490 }
8491 }
8492 }
8493 }
8494 }
8495
8496 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
8497 print $F4 $data_pos_def;
8498 close($F4);
8499
8500 $grm_file = $grm_out_posdef_tempfile;
8501 }
8502 elsif ($use_parental_grms_if_compute_from_parents) {
8503 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
8504 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
8505 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
8506 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
8507 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
8508 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
8509 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
8510 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
8511
8512 if (!$protocol_id) {
8513 $protocol_id = undef;
8514 }
8515
8516 my $pedigree_arm = CXGN::Pedigree::ARM->new({
8517 bcs_schema=>$schema,
8518 arm_temp_file=>$arm_tempfile,
8519 people_schema=>$people_schema,
8520 accession_id_list=>\@accession_ids,
8521 # plot_id_list=>\@plot_id_list,
8522 cache_root=>$c->config->{cache_file_path},
8523 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
8524 });
8525 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
8526 $shared_cluster_dir_config,
8527 $c->config->{backend},
8528 $c->config->{cluster_host},
8529 $c->config->{'web_cluster_queue'},
8530 $c->config->{basepath}
8531 );
8532 # print STDERR Dumper $parent_hash;
8533
8534 my $female_geno = CXGN::Genotype::GRM->new({
8535 bcs_schema=>$schema,
8536 grm_temp_file=>$grm1_tempfile,
8537 people_schema=>$people_schema,
8538 cache_root=>$c->config->{cache_file_path},
8539 accession_id_list=>$female_stock_ids,
8540 protocol_id=>$protocol_id,
8541 get_grm_for_parental_accessions=>0,
8542 download_format=>'three_column_reciprocal'
8543 # minor_allele_frequency=>$minor_allele_frequency,
8544 # marker_filter=>$marker_filter,
8545 # individuals_filter=>$individuals_filter
8546 });
8547 my $female_grm_data = $female_geno->download_grm(
8548 'data',
8549 $shared_cluster_dir_config,
8550 $c->config->{backend},
8551 $c->config->{cluster_host},
8552 $c->config->{'web_cluster_queue'},
8553 $c->config->{basepath}
8554 );
8555 my @fl = split '\n', $female_grm_data;
8556 my %female_parent_grm;
8557 foreach (@fl) {
8558 my @l = split '\t', $_;
8559 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
8560 }
8561 # print STDERR Dumper \%female_parent_grm;
8562
8563 my $male_geno = CXGN::Genotype::GRM->new({
8564 bcs_schema=>$schema,
8565 grm_temp_file=>$grm1_tempfile,
8566 people_schema=>$people_schema,
8567 cache_root=>$c->config->{cache_file_path},
8568 accession_id_list=>$male_stock_ids,
8569 protocol_id=>$protocol_id,
8570 get_grm_for_parental_accessions=>0,
8571 download_format=>'three_column_reciprocal'
8572 # minor_allele_frequency=>$minor_allele_frequency,
8573 # marker_filter=>$marker_filter,
8574 # individuals_filter=>$individuals_filter
8575 });
8576 my $male_grm_data = $male_geno->download_grm(
8577 'data',
8578 $shared_cluster_dir_config,
8579 $c->config->{backend},
8580 $c->config->{cluster_host},
8581 $c->config->{'web_cluster_queue'},
8582 $c->config->{basepath}
8583 );
8584 my @ml = split '\n', $male_grm_data;
8585 my %male_parent_grm;
8586 foreach (@ml) {
8587 my @l = split '\t', $_;
8588 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
8589 }
8590 # print STDERR Dumper \%male_parent_grm;
8591
8592 my %rel_result_hash;
8593 foreach my $a1 (@accession_ids) {
8594 foreach my $a2 (@accession_ids) {
8595 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
8596 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
8597 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
8598 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
8599
8600 my $female_rel = 0;
8601 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
8602 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
8603 }
8604 elsif ($a1 == $a2) {
8605 $female_rel = 1;
8606 }
8607
8608 my $male_rel = 0;
8609 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
8610 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
8611 }
8612 elsif ($a1 == $a2) {
8613 $male_rel = 1;
8614 }
8615 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
8616
8617 my $rel = 0.5*($female_rel + $male_rel);
8618 $rel_result_hash{$a1}->{$a2} = $rel;
8619 }
8620 }
8621 # print STDERR Dumper \%rel_result_hash;
8622
8623 my $data = '';
8624 my %result_hash;
8625 foreach my $s (sort @accession_ids) {
8626 foreach my $c (sort @accession_ids) {
8627 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8628 my $val = $rel_result_hash{$s}->{$c};
8629 if (defined $val and length $val) {
8630 $result_hash{$s}->{$c} = $val;
8631 $data .= "S$s\tS$c\t$val\n";
8632 }
8633 }
8634 }
8635 }
8636
8637 # print STDERR Dumper $data;
8638 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
8639 print $F2 $data;
8640 close($F2);
8641
8642 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
8643 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
8644 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
8645 A_1 <- A_wide[,-1];
8646 A_1[is.na(A_1)] <- 0;
8647 A <- A_1 + t(A_1);
8648 diag(A) <- diag(as.matrix(A_1));
8649 E = eigen(A);
8650 ev = E\$values;
8651 U = E\$vectors;
8652 no = dim(A)[1];
8653 nev = which(ev < 0);
8654 wr = 0;
8655 k=length(nev);
8656 if(k > 0){
8657 p = ev[no - k];
8658 B = sum(ev[nev])*2.0;
8659 wr = (B*B*100.0)+1;
8660 val = ev[nev];
8661 ev[nev] = p*(B-val)*(B-val)/wr;
8662 A = U%*%diag(ev)%*%t(U);
8663 }
8664 ';
8665 if ($return_inverse_matrix) {
8666 $cmd .= 'A <- solve(A);
8667 ';
8668 }
8669 $cmd .= 'A <- as.data.frame(A);
8670 colnames(A) <- A_wide[,1];
8671 A\$stock_id <- A_wide[,1];
8672 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
8673 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
8674 A_threecol\$variable <- substring(A_threecol\$variable, 2);
8675 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
8676 print STDERR $cmd."\n";
8677 my $status = system($cmd);
8678
8679 my %rel_pos_def_result_hash;
8680 open(my $F3, '<', $grm_out_tempfile)
8681 or die "Could not open file '$grm_out_tempfile' $!";
8682
8683 print STDERR "Opened $grm_out_tempfile\n";
8684
8685 while (my $row = <$F3>) {
8686 my @columns;
8687 if ($csv->parse($row)) {
8688 @columns = $csv->fields();
8689 }
8690 my $stock_id1 = $columns[0];
8691 my $stock_id2 = $columns[1];
8692 my $val = $columns[2];
8693 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
8694 }
8695 close($F3);
8696
8697 my $data_pos_def = '';
8698 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_univariate_pure_spatial_genetic_blups') {
8699 my %result_hash;
8700 foreach my $s (sort @accession_ids) {
8701 foreach my $c (sort @accession_ids) {
8702 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8703 my $val = $rel_pos_def_result_hash{$s}->{$c};
8704 if (defined $val and length $val) {
8705 $result_hash{$s}->{$c} = $val;
8706 $data_pos_def .= "$s\t$c\t$val\n";
8707 }
8708 }
8709 }
8710 }
8711 }
8712 else {
8713 my %result_hash;
8714 foreach my $s (sort @accession_ids) {
8715 foreach my $c (sort @accession_ids) {
8716 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
8717 my $val = $rel_pos_def_result_hash{$s}->{$c};
8718 if (defined $val and length $val) {
8719 $result_hash{$s}->{$c} = $val;
8720 $result_hash{$c}->{$s} = $val;
8721 $data_pos_def .= "S$s\tS$c\t$val\n";
8722 if ($s != $c) {
8723 $data_pos_def .= "S$c\tS$s\t$val\n";
8724 }
8725 }
8726 }
8727 }
8728 }
8729 }
8730
8731 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
8732 print $F4 $data_pos_def;
8733 close($F4);
8734
8735 $grm_file = $grm_out_posdef_tempfile;
8736 }
8737 else {
8738 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
8739 my $tmp_grm_dir = $shared_cluster_dir_config."/tmp_genotype_download_grm";
8740 mkdir $tmp_grm_dir if ! -d $tmp_grm_dir;
8741 my ($grm_tempfile_fh, $grm_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
8742 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
8743
8744 if (!$protocol_id) {
8745 $protocol_id = undef;
8746 }
8747
8748 my $grm_search_params = {
8749 bcs_schema=>$schema,
8750 grm_temp_file=>$grm_tempfile,
8751 people_schema=>$people_schema,
8752 cache_root=>$c->config->{cache_file_path},
8753 accession_id_list=>\@accession_ids,
8754 protocol_id=>$protocol_id,
8755 get_grm_for_parental_accessions=>$compute_from_parents,
8756 return_inverse=>$return_inverse_matrix,
8757 ensure_positive_definite=>$ensure_positive_definite
8758 # minor_allele_frequency=>$minor_allele_frequency,
8759 # marker_filter=>$marker_filter,
8760 # individuals_filter=>$individuals_filter
8761 };
8762
8763 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_univariate_pure_spatial_genetic_blups') {
8764 $grm_search_params->{download_format} = 'three_column_stock_id_integer';
8765 }
8766 else {
8767 $grm_search_params->{download_format} = 'three_column_reciprocal';
8768 }
8769
8770 my $geno = CXGN::Genotype::GRM->new($grm_search_params);
8771 my $grm_data = $geno->download_grm(
8772 'data',
8773 $shared_cluster_dir_config,
8774 $c->config->{backend},
8775 $c->config->{cluster_host},
8776 $c->config->{'web_cluster_queue'},
8777 $c->config->{basepath}
8778 );
8779
8780 open(my $F2, ">", $grm_out_tempfile) || die "Can't open file ".$grm_out_tempfile;
8781 print $F2 $grm_data;
8782 close($F2);
8783 $grm_file = $grm_out_tempfile;
8784 }
8785
8786 }
8787 elsif ($compute_relationship_matrix_from_htp_phenotypes eq 'htp_phenotypes') {
8788
8789 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
8790 'MaterializedViewTable',
8791 {
8792 bcs_schema=>$schema,
8793 data_level=>'plot',
8794 trial_list=>$field_trial_id_list,
8795 include_timestamp=>0,
8796 exclude_phenotype_outlier=>0
8797 }
8798 );
8799 my ($data, $unique_traits) = $phenotypes_search->search();
8800
8801 if (scalar(@$data) == 0) {
8802 $c->stash->{rest} = { error => "There are no phenotypes for the trial you have selected!"};
8803 return;
8804 }
8805
8806 my $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
8807 my $h_time = $schema->storage->dbh()->prepare($q_time);
8808
8809 my %seen_plot_names_htp_rel;
8810 my %phenotype_data_htp_rel;
8811 my %seen_times_htp_rel;
8812 foreach my $obs_unit (@$data){
8813 my $germplasm_name = $obs_unit->{germplasm_uniquename};
8814 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
8815 my $row_number = $obs_unit->{obsunit_row_number} || '';
8816 my $col_number = $obs_unit->{obsunit_col_number} || '';
8817 my $rep = $obs_unit->{obsunit_rep};
8818 my $block = $obs_unit->{obsunit_block};
8819 $seen_plot_names_htp_rel{$obs_unit->{observationunit_uniquename}} = $obs_unit;
8820 my $observations = $obs_unit->{observations};
8821 foreach (@$observations){
8822 if ($_->{associated_image_project_time_json}) {
8823 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
8824
8825 my $time_days_cvterm = $related_time_terms_json->{day};
8826 my $time_days_term_string = $time_days_cvterm;
8827 my $time_days = (split '\|', $time_days_cvterm)[0];
8828 my $time_days_value = (split ' ', $time_days)[1];
8829
8830 my $time_gdd_value = $related_time_terms_json->{gdd_average_temp} + 0;
8831 my $gdd_term_string = "GDD $time_gdd_value";
8832 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
8833 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
8834 if (!$gdd_cvterm_id) {
8835 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
8836 name => $gdd_term_string,
8837 cv => 'cxgn_time_ontology'
8838 });
8839 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
8840 }
8841 my $time_gdd_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
8842
8843 $phenotype_data_htp_rel{$obs_unit->{observationunit_uniquename}}->{$_->{trait_name}} = $_->{value};
8844 $seen_times_htp_rel{$_->{trait_name}} = [$time_days_value, $time_days_term_string, $time_gdd_value, $time_gdd_term_string];
8845 }
8846 }
8847 }
8848
8849 my @allowed_standard_htp_values = ('Nonzero Pixel Count', 'Total Pixel Sum', 'Mean Pixel Value', 'Harmonic Mean Pixel Value', 'Median Pixel Value', 'Pixel Variance', 'Pixel Standard Deviation', 'Pixel Population Standard Deviation', 'Minimum Pixel Value', 'Maximum Pixel Value', 'Minority Pixel Value', 'Minority Pixel Count', 'Majority Pixel Value', 'Majority Pixel Count', 'Pixel Group Count');
8850 my %filtered_seen_times_htp_rel;
8851 while (my ($t, $time) = each %seen_times_htp_rel) {
8852 my $allowed = 0;
8853 foreach (@allowed_standard_htp_values) {
8854 if (index($t, $_) != -1) {
8855 $allowed = 1;
8856 last;
8857 }
8858 }
8859 if ($allowed) {
8860 $filtered_seen_times_htp_rel{$t} = $time;
8861 }
8862 }
8863
8864 my @seen_plot_names_htp_rel_sorted = sort keys %seen_plot_names_htp_rel;
8865 my @filtered_seen_times_htp_rel_sorted = sort keys %filtered_seen_times_htp_rel;
8866
8867 my @header_htp = ('plot_id', 'plot_name', 'accession_id', 'accession_name', 'rep', 'block');
8868
8869 my %trait_name_encoder_htp;
8870 my %trait_name_encoder_rev_htp;
8871 my $trait_name_encoded_htp = 1;
8872 my @header_traits_htp;
8873 foreach my $trait_name (@filtered_seen_times_htp_rel_sorted) {
8874 if (!exists($trait_name_encoder_htp{$trait_name})) {
8875 my $trait_name_e = 't'.$trait_name_encoded_htp;
8876 $trait_name_encoder_htp{$trait_name} = $trait_name_e;
8877 $trait_name_encoder_rev_htp{$trait_name_e} = $trait_name;
8878 push @header_traits_htp, $trait_name_e;
8879 $trait_name_encoded_htp++;
8880 }
8881 }
8882
8883 my @htp_pheno_matrix;
8884 if ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'all') {
8885 push @header_htp, @header_traits_htp;
8886 push @htp_pheno_matrix, \@header_htp;
8887
8888 foreach my $p (@seen_plot_names_htp_rel_sorted) {
8889 my $obj = $seen_plot_names_htp_rel{$p};
8890 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
8891 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
8892 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
8893 push @row, $val;
8894 }
8895 push @htp_pheno_matrix, \@row;
8896 }
8897 }
8898 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'latest_trait') {
8899 my $max_day = 0;
8900 foreach (keys %seen_days_after_plantings) {
8901 if ($_ + 0 > $max_day) {
8902 $max_day = $_;
8903 }
8904 }
8905
8906 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
8907 my $day = $filtered_seen_times_htp_rel{$t}->[0];
8908 if ($day <= $max_day) {
8909 push @header_htp, $t;
8910 }
8911 }
8912 push @htp_pheno_matrix, \@header_htp;
8913
8914 foreach my $p (@seen_plot_names_htp_rel_sorted) {
8915 my $obj = $seen_plot_names_htp_rel{$p};
8916 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
8917 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
8918 my $day = $filtered_seen_times_htp_rel{$t}->[0];
8919 if ($day <= $max_day) {
8920 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
8921 push @row, $val;
8922 }
8923 }
8924 push @htp_pheno_matrix, \@row;
8925 }
8926 }
8927 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'vegetative') {
8928
8929 }
8930 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'reproductive') {
8931
8932 }
8933 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'mature') {
8934
8935 }
8936 else {
8937 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_time_points htp_pheno_rel_matrix_time_points is not valid!" };
8938 return;
8939 }
8940
8941 open(my $htp_pheno_f, ">", $stats_out_htp_rel_tempfile_input) || die "Can't open file ".$stats_out_htp_rel_tempfile_input;
8942 foreach (@htp_pheno_matrix) {
8943 my $line = join "\t", @$_;
8944 print $htp_pheno_f $line."\n";
8945 }
8946 close($htp_pheno_f);
8947
8948 my %rel_htp_result_hash;
8949 if ($compute_relationship_matrix_from_htp_phenotypes_type eq 'correlations') {
8950 my $htp_cmd = 'R -e "library(lme4); library(data.table);
8951 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
8952 mat_agg <- aggregate(mat[, 7:ncol(mat)], list(mat\$accession_id), mean);
8953 mat_pheno <- mat_agg[,2:ncol(mat_agg)];
8954 cor_mat <- cor(t(mat_pheno));
8955 rownames(cor_mat) <- mat_agg[,1];
8956 colnames(cor_mat) <- mat_agg[,1];
8957 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
8958 cor_mat <- range01(cor_mat);
8959 ';
8960 if ($return_inverse_matrix) {
8961 $htp_cmd .= 'cor_mat <- solve(cor_mat);
8962 ';
8963 }
8964 $htp_cmd .= 'write.table(cor_mat, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
8965 print STDERR Dumper $htp_cmd;
8966 my $status = system($htp_cmd);
8967 }
8968 elsif ($compute_relationship_matrix_from_htp_phenotypes_type eq 'blues') {
8969 my $htp_cmd = 'R -e "library(lme4); library(data.table);
8970 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
8971 blues <- data.frame(id = seq(1,length(unique(mat\$accession_id))));
8972 varlist <- names(mat)[7:ncol(mat)];
8973 blues.models <- lapply(varlist, function(x) {
8974 tryCatch(
8975 lmer(substitute(i ~ 1 + (1|accession_id), list(i = as.name(x))), data = mat, REML = FALSE, control = lmerControl(optimizer =\'Nelder_Mead\', boundary.tol='.$compute_relationship_matrix_from_htp_phenotypes_blues_inversion.' ) ), error=function(e) {}
8976 )
8977 });
8978 counter = 1;
8979 for (m in blues.models) {
8980 if (!is.null(m)) {
8981 blues\$accession_id <- row.names(ranef(m)\$accession_id);
8982 blues[,ncol(blues) + 1] <- ranef(m)\$accession_id\$\`(Intercept)\`;
8983 colnames(blues)[ncol(blues)] <- varlist[counter];
8984 }
8985 counter = counter + 1;
8986 }
8987 blues_vals <- as.matrix(blues[,3:ncol(blues)]);
8988 blues_vals <- apply(blues_vals, 2, function(y) (y - mean(y)) / sd(y) ^ as.logical(sd(y)));
8989 rel <- (1/ncol(blues_vals)) * (blues_vals %*% t(blues_vals));
8990 ';
8991 if ($return_inverse_matrix) {
8992 $htp_cmd .= 'rel <- solve(rel);
8993 ';
8994 }
8995 $htp_cmd .= 'rownames(rel) <- blues[,2];
8996 colnames(rel) <- blues[,2];
8997 write.table(rel, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
8998 print STDERR Dumper $htp_cmd;
8999 my $status = system($htp_cmd);
9000 }
9001 else {
9002 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_type htp_pheno_rel_matrix_type is not valid!" };
9003 return;
9004 }
9005
9006 open(my $htp_rel_res, '<', $stats_out_htp_rel_tempfile) or die "Could not open file '$stats_out_htp_rel_tempfile' $!";
9007 print STDERR "Opened $stats_out_htp_rel_tempfile\n";
9008 my $header_row = <$htp_rel_res>;
9009 my @header;
9010 if ($csv->parse($header_row)) {
9011 @header = $csv->fields();
9012 }
9013
9014 while (my $row = <$htp_rel_res>) {
9015 my @columns;
9016 if ($csv->parse($row)) {
9017 @columns = $csv->fields();
9018 }
9019 my $stock_id1 = $columns[0];
9020 my $counter = 1;
9021 foreach my $stock_id2 (@header) {
9022 my $val = $columns[$counter];
9023 $rel_htp_result_hash{$stock_id1}->{$stock_id2} = $val;
9024 $counter++;
9025 }
9026 }
9027 close($htp_rel_res);
9028
9029 my $data_rel_htp = '';
9030 my %result_hash;
9031 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_univariate_pure_spatial_genetic_blups') {
9032 foreach my $s (sort @accession_ids) {
9033 foreach my $c (sort @accession_ids) {
9034 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
9035 my $val = $rel_htp_result_hash{$s}->{$c};
9036 if (defined $val and length $val) {
9037 $result_hash{$s}->{$c} = $val;
9038 $data_rel_htp .= "$s\t$c\t$val\n";
9039 }
9040 }
9041 }
9042 }
9043 }
9044 else {
9045 foreach my $s (sort @accession_ids) {
9046 foreach my $c (sort @accession_ids) {
9047 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
9048 my $val = $rel_htp_result_hash{$s}->{$c};
9049 if (defined $val and length $val) {
9050 $result_hash{$s}->{$c} = $val;
9051 $result_hash{$c}->{$s} = $val;
9052 $data_rel_htp .= "S$s\tS$c\t$val\n";
9053 if ($s != $c) {
9054 $data_rel_htp .= "S$c\tS$s\t$val\n";
9055 }
9056 }
9057 }
9058 }
9059 }
9060 }
9061
9062 open(my $htp_rel_out, ">", $stats_out_htp_rel_tempfile_out) || die "Can't open file ".$stats_out_htp_rel_tempfile_out;
9063 print $htp_rel_out $data_rel_htp;
9064 close($htp_rel_out);
9065
9066 $grm_file = $stats_out_htp_rel_tempfile_out;
9067 }
9068 else {
9069 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes is not valid!" };
9070 return;
9071 }
9072 };
9073
9074 my $result_5 = CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation::perform_drone_imagery_analytics($schema, $a_env, $b_env, $ro_env, $row_ro_env, $env_variance_percent, $protocol_id, $statistics_select, $analytics_select, $tolparinv, $use_area_under_curve, $legendre_order_number, $permanent_environment_structure, \@legendre_coeff_exec, \%trait_name_encoder_5, \%trait_name_encoder_rev_5, \%stock_info_5, \%plot_id_map, \@sorted_trait_names_5, \%accession_id_factor_map, \@rep_time_factors, \@ind_rep_factors, \@unique_accession_names, \%plot_id_count_map_reverse, \@sorted_scaled_ln_times, \%time_count_map_reverse, \%accession_id_factor_map_reverse, \%seen_times, \%plot_id_factor_map_reverse, \%trait_to_time_map_5, \@unique_plot_names, \%stock_name_row_col, \%phenotype_data_original_5, \%plot_rep_time_factor_map, \%stock_row_col, \%stock_row_col_id, \%polynomial_map, \@plot_ids_ordered, $csv, $timestamp, $user_name, $stats_tempfile, $grm_file, $grm_rename_tempfile, $tmp_stats_dir, $stats_out_tempfile, $stats_out_tempfile_row, $stats_out_tempfile_col, $stats_out_tempfile_residual, $stats_out_tempfile_2dspl, $stats_prep2_tempfile, $stats_out_param_tempfile, $parameter_tempfile, $parameter_asreml_tempfile, $stats_tempfile_2, $permanent_environment_structure_tempfile, $permanent_environment_structure_env_tempfile, $permanent_environment_structure_env_tempfile2, $permanent_environment_structure_env_tempfile_mat, $sim_env_changing_mat_tempfile, $sim_env_changing_mat_full_tempfile, $yhat_residual_tempfile, $blupf90_solutions_tempfile, $coeff_genetic_tempfile, $coeff_pe_tempfile, $stats_out_tempfile_varcomp, $time_min, $time_max, $header_string_5, $env_sim_exec, $min_row, $max_row, $min_col, $max_col, $mean_row, $sig_row, $mean_col, $sig_col, $sim_env_change_over_time, $correlation_between_times, $field_trial_id_list, $simulated_environment_real_data_trait_id, $fixed_effect_type, $perform_cv);
9075 if (ref($result_5) eq 'HASH') {
9076 $c->stash->{rest} = $result_5;
9077 $c->detach();
9078 }
9079 my ($statistical_ontology_term_5, $analysis_model_training_data_file_type_5, $analysis_model_language_5, $sorted_residual_trait_names_array_5, $rr_unique_traits_hash_5, $rr_residual_unique_traits_hash_5, $statistics_cmd_5, $cmd_f90_5, $number_traits_5, $trait_to_time_map_hash_5,
9080
9081 $result_blup_data_original_5, $result_blup_data_delta_original_5, $result_blup_spatial_data_original_5, $result_blup_pe_data_original_5, $result_blup_pe_data_delta_original_5, $result_residual_data_original_5, $result_fitted_data_original_5, $fixed_effects_original_hash_5,
9082 $rr_genetic_coefficients_original_hash_5, $rr_temporal_coefficients_original_hash_5,
9083 $rr_coeff_genetic_covariance_original_array_5, $rr_coeff_env_covariance_original_array_5, $rr_coeff_genetic_correlation_original_array_5, $rr_coeff_env_correlation_original_array_5, $rr_residual_variance_original_5, $varcomp_original_array_5,
9084 $model_sum_square_residual_original_5, $genetic_effect_min_original_5, $genetic_effect_max_original_5, $env_effect_min_original_5, $env_effect_max_original_5, $genetic_effect_sum_square_original_5, $genetic_effect_sum_original_5, $env_effect_sum_square_original_5, $env_effect_sum_original_5, $residual_sum_square_original_5, $residual_sum_original_5, $result_cv_original_5, $result_cv_2_original_5,
9085
9086 $phenotype_data_altered_hash_5, $data_matrix_altered_array_5, $data_matrix_phenotypes_altered_array_5, $phenotype_min_altered_5, $phenotype_max_altered_5,
9087 $result_blup_data_altered_5, $result_blup_data_delta_altered_5, $result_blup_spatial_data_altered_5, $result_blup_pe_data_altered_5, $result_blup_pe_data_delta_altered_5, $result_residual_data_altered_5, $result_fitted_data_altered_5, $fixed_effects_altered_hash_5,
9088 $rr_genetic_coefficients_altered_hash_5, $rr_temporal_coefficients_altered_hash_5,
9089 $rr_coeff_genetic_covariance_altered_array_5, $rr_coeff_env_covariance_altered_array_5, $rr_coeff_genetic_correlation_altered_array_5, $rr_coeff_env_correlation_altered_array_5, $rr_residual_variance_altered_5, $varcomp_altered_array_5,
9090 $model_sum_square_residual_altered_5, $genetic_effect_min_altered_5, $genetic_effect_max_altered_5, $env_effect_min_altered_5, $env_effect_max_altered_5, $genetic_effect_sum_square_altered_5, $genetic_effect_sum_altered_5, $env_effect_sum_square_altered_5, $env_effect_sum_altered_5, $residual_sum_square_altered_5, $residual_sum_altered_5, $result_cv_altered_5, $result_cv_2_altered_5,
9091
9092 $phenotype_data_altered_env_hash_1_5, $data_matrix_altered_env_array_1_5, $data_matrix_phenotypes_altered_env_array_1_5, $phenotype_min_altered_env_1_5, $phenotype_max_altered_env_1_5, $env_sim_min_1_5, $env_sim_max_1_5, $sim_data_hash_1_5,
9093 $result_blup_data_altered_env_1_5, $result_blup_data_delta_altered_env_1_5, $result_blup_spatial_data_altered_env_1_5, $result_blup_pe_data_altered_env_1_5, $result_blup_pe_data_delta_altered_env_1_5, $result_residual_data_altered_env_1_5, $result_fitted_data_altered_env_1_5, $fixed_effects_altered_env_hash_1_5, $rr_genetic_coefficients_altered_env_hash_1_5, $rr_temporal_coefficients_altered_env_hash_1_5,
9094 $rr_coeff_genetic_covariance_altered_env_array_1_5, $rr_coeff_env_covariance_altered_env_array_1_5, $rr_coeff_genetic_correlation_altered_env_array_1_5, $rr_coeff_env_correlation_altered_env_array_1_5, $rr_residual_variance_altered_env_1_5, $varcomp_altered_array_env_1_5,
9095 $model_sum_square_residual_altered_env_1_5, $genetic_effect_min_altered_env_1_5, $genetic_effect_max_altered_env_1_5, $env_effect_min_altered_env_1_5, $env_effect_max_altered_env_1_5, $genetic_effect_sum_square_altered_env_1_5, $genetic_effect_sum_altered_env_1_5, $env_effect_sum_square_altered_env_1_5, $env_effect_sum_altered_env_1_5, $residual_sum_square_altered_env_1_5, $residual_sum_altered_env_1_5, $result_cv_altered_env_1_5, $result_cv_2_altered_env_1_5,
9096
9097 $phenotype_data_altered_env_hash_2_5, $data_matrix_altered_env_array_2_5, $data_matrix_phenotypes_altered_env_array_2_5, $phenotype_min_altered_env_2_5, $phenotype_max_altered_env_2_5, $env_sim_min_2_5, $env_sim_max_2_5, $sim_data_hash_2_5,
9098 $result_blup_data_altered_env_2_5, $result_blup_data_delta_altered_env_2_5, $result_blup_spatial_data_altered_env_2_5, $result_blup_pe_data_altered_env_2_5, $result_blup_pe_data_delta_altered_env_2_5, $result_residual_data_altered_env_2_5, $result_fitted_data_altered_env_2_5, $fixed_effects_altered_env_hash_2_5, $rr_genetic_coefficients_altered_env_hash_2_5, $rr_temporal_coefficients_altered_env_hash_2_5,
9099 $rr_coeff_genetic_covariance_altered_env_array_2_5, $rr_coeff_env_covariance_altered_env_array_2_5, $rr_coeff_genetic_correlation_altered_env_array_2_5, $rr_coeff_env_correlation_altered_env_array_2_5, $rr_residual_variance_altered_env_2_5, $varcomp_altered_array_env_2_5,
9100 $model_sum_square_residual_altered_env_2_5, $genetic_effect_min_altered_env_2_5, $genetic_effect_max_altered_env_2_5, $env_effect_min_altered_env_2_5, $env_effect_max_altered_env_2_5, $genetic_effect_sum_square_altered_env_2_5, $genetic_effect_sum_altered_env_2_5, $env_effect_sum_square_altered_env_2_5, $env_effect_sum_altered_env_2_5, $residual_sum_square_altered_env_2_5, $residual_sum_altered_env_2_5, $result_cv_altered_env_2_5, $result_cv_2_altered_env_2_5,
9101
9102 $phenotype_data_altered_env_hash_3_5, $data_matrix_altered_env_array_3_5, $data_matrix_phenotypes_altered_env_array_3_5, $phenotype_min_altered_env_3_5, $phenotype_max_altered_env_3_5, $env_sim_min_3_5, $env_sim_max_3_5, $sim_data_hash_3_5,
9103 $result_blup_data_altered_env_3_5, $result_blup_data_delta_altered_env_3_5, $result_blup_spatial_data_altered_env_3_5, $result_blup_pe_data_altered_env_3_5, $result_blup_pe_data_delta_altered_env_3_5, $result_residual_data_altered_env_3_5, $result_fitted_data_altered_env_3_5, $fixed_effects_altered_env_hash_3_5, $rr_genetic_coefficients_altered_env_hash_3_5, $rr_temporal_coefficients_altered_env_hash_3_5,
9104 $rr_coeff_genetic_covariance_altered_env_array_3_5, $rr_coeff_env_covariance_altered_env_array_3_5, $rr_coeff_genetic_correlation_altered_env_array_3_5, $rr_coeff_env_correlation_altered_env_array_3_5, $rr_residual_variance_altered_env_3_5, $varcomp_altered_array_env_3_5,
9105 $model_sum_square_residual_altered_env_3_5, $genetic_effect_min_altered_env_3_5, $genetic_effect_max_altered_env_3_5, $env_effect_min_altered_env_3_5, $env_effect_max_altered_env_3_5, $genetic_effect_sum_square_altered_env_3_5, $genetic_effect_sum_altered_env_3_5, $env_effect_sum_square_altered_env_3_5, $env_effect_sum_altered_env_3_5, $residual_sum_square_altered_env_3_5, $residual_sum_altered_env_3_5, $result_cv_altered_env_3_5, $result_cv_2_altered_env_3_5,
9106
9107 $phenotype_data_altered_env_hash_4_5, $data_matrix_altered_env_array_4_5, $data_matrix_phenotypes_altered_env_array_4_5, $phenotype_min_altered_env_4_5, $phenotype_max_altered_env_4_5, $env_sim_min_4_5, $env_sim_max_4_5, $sim_data_hash_4_5,
9108 $result_blup_data_altered_env_4_5, $result_blup_data_delta_altered_env_4_5, $result_blup_spatial_data_altered_env_4_5, $result_blup_pe_data_altered_env_4_5, $result_blup_pe_data_delta_altered_env_4_5, $result_residual_data_altered_env_4_5, $result_fitted_data_altered_env_4_5, $fixed_effects_altered_env_hash_4_5, $rr_genetic_coefficients_altered_env_hash_4_5, $rr_temporal_coefficients_altered_env_hash_4_5,
9109 $rr_coeff_genetic_covariance_altered_env_array_4_5, $rr_coeff_env_covariance_altered_env_array_4_5, $rr_coeff_genetic_correlation_altered_env_array_4_5, $rr_coeff_env_correlation_altered_env_array_4_5, $rr_residual_variance_altered_env_4_5, $varcomp_altered_array_env_4_5,
9110 $model_sum_square_residual_altered_env_4_5, $genetic_effect_min_altered_env_4_5, $genetic_effect_max_altered_env_4_5, $env_effect_min_altered_env_4_5, $env_effect_max_altered_env_4_5, $genetic_effect_sum_square_altered_env_4_5, $genetic_effect_sum_altered_env_4_5, $env_effect_sum_square_altered_env_4_5, $env_effect_sum_altered_env_4_5, $residual_sum_square_altered_env_4_5, $residual_sum_altered_env_4_5, $result_cv_altered_env_4_5, $result_cv_2_altered_env_4_5,
9111
9112 $phenotype_data_altered_env_hash_5_5, $data_matrix_altered_env_array_5_5, $data_matrix_phenotypes_altered_env_array_5_5, $phenotype_min_altered_env_5_5, $phenotype_max_altered_env_5_5, $env_sim_min_5_5, $env_sim_max_5_5, $sim_data_hash_5_5,
9113 $result_blup_data_altered_env_5_5, $result_blup_data_delta_altered_env_5_5, $result_blup_spatial_data_altered_env_5_5, $result_blup_pe_data_altered_env_5_5, $result_blup_pe_data_delta_altered_env_5_5, $result_residual_data_altered_env_5_5, $result_fitted_data_altered_env_5_5, $fixed_effects_altered_env_hash_5_5, $rr_genetic_coefficients_altered_env_hash_5_5, $rr_temporal_coefficients_altered_env_hash_5_5,
9114 $rr_coeff_genetic_covariance_altered_env_array_5_5, $rr_coeff_env_covariance_altered_env_array_5_5, $rr_coeff_genetic_correlation_altered_env_array_5_5, $rr_coeff_env_correlation_altered_env_array_5_5, $rr_residual_variance_altered_env_5_5, $varcomp_altered_array_env_5_5,
9115 $model_sum_square_residual_altered_env_5_5, $genetic_effect_min_altered_env_5_5, $genetic_effect_max_altered_env_5_5, $env_effect_min_altered_env_5_5, $env_effect_max_altered_env_5_5, $genetic_effect_sum_square_altered_env_5_5, $genetic_effect_sum_altered_env_5_5, $env_effect_sum_square_altered_env_5_5, $env_effect_sum_altered_env_5_5, $residual_sum_square_altered_env_5_5, $residual_sum_altered_env_5_5, $result_cv_altered_env_5_5, $result_cv_2_altered_env_5_5,
9116
9117 $phenotype_data_altered_env_hash_6_5, $data_matrix_altered_env_array_6_5, $data_matrix_phenotypes_altered_env_array_6_5, $phenotype_min_altered_env_6_5, $phenotype_max_altered_env_6_5, $env_sim_min_6_5, $env_sim_max_6_5, $sim_data_hash_6_5,
9118 $result_blup_data_altered_env_6_5, $result_blup_data_delta_altered_env_6_5, $result_blup_spatial_data_altered_env_6_5, $result_blup_pe_data_altered_env_6_5, $result_blup_pe_data_delta_altered_env_6_5, $result_residual_data_altered_env_6_5, $result_fitted_data_altered_env_6_5, $fixed_effects_altered_env_hash_6_5, $rr_genetic_coefficients_altered_env_hash_6_5, $rr_temporal_coefficients_altered_env_hash_6_5,
9119 $rr_coeff_genetic_covariance_altered_env_array_6_5, $rr_coeff_env_covariance_altered_env_array_6_5, $rr_coeff_genetic_correlation_altered_env_array_6_5, $rr_coeff_env_correlation_altered_env_array_6_5, $rr_residual_variance_altered_env_6_5, $varcomp_altered_array_env_6_5,
9120 $model_sum_square_residual_altered_env_6_5, $genetic_effect_min_altered_env_6_5, $genetic_effect_max_altered_env_6_5, $env_effect_min_altered_env_6_5, $env_effect_max_altered_env_6_5, $genetic_effect_sum_square_altered_env_6_5, $genetic_effect_sum_altered_env_6_5, $env_effect_sum_square_altered_env_6_5, $env_effect_sum_altered_env_6_5, $residual_sum_square_altered_env_6_5, $residual_sum_altered_env_6_5, $result_cv_altered_env_6_5, $result_cv_2_altered_env_6_5
9121 ) = @$result_5;
9122
9123 eval {
9124 print STDERR "PLOTTING CORRELATION\n";
9125 my ($full_plot_level_correlation_tempfile_fh, $full_plot_level_correlation_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9126 open(my $F_fullplot, ">", $full_plot_level_correlation_tempfile) || die "Can't open file ".$full_plot_level_correlation_tempfile;
9127 print STDERR "OPENED PLOTCORR FILE $full_plot_level_correlation_tempfile\n";
9128
9129 my @header_full_plot_corr = ('plot_name, plot_id, row_number, col_number, rep, block, germplasm_name, germplasm_id');
9130 my @types_full_plot_corr = ('pheno_orig_', 'pheno_postm5_', 'eff_origm5_', 'eff_postm5_',
9131 'sim_env1_', 'simm5_pheno1_', 'effm5_sim1_',
9132 'sim_env2_', 'simm5_pheno2_', 'effm5_sim2_',
9133 'sim_env3_', 'simm5_pheno3_', 'effm5_sim3_',
9134 'sim_env4_', 'simm5_pheno4_', 'effm5_sim4_',
9135 'sim_env5_', 'simm5_pheno5_', 'effm5_sim5_',
9136 'sim_env6_', 'simm5_pheno6_', 'effm5_sim6_');
9137 foreach my $t (@sorted_trait_names_5) {
9138 foreach my $type (@types_full_plot_corr) {
9139 push @header_full_plot_corr, $type.$trait_name_encoder_5{$t};
9140 }
9141 }
9142 my $header_string_full_plot_corr = join ',', @header_full_plot_corr;
9143 print $F_fullplot "$header_string_full_plot_corr\n";
9144 foreach my $p (@unique_plot_names) {
9145 my @row = ($p, $stock_name_row_col{$p}->{obsunit_stock_id}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $stock_name_row_col{$p}->{rep}, $stock_name_row_col{$p}->{block}, $stock_name_row_col{$p}->{germplasm_name}, $stock_name_row_col{$p}->{germplasm_stock_id});
9146 foreach my $t (@sorted_trait_names_5) {
9147 my $phenotype_original = $phenotype_data_original_5{$p}->{$t};
9148 my $phenotype_post_5 = $phenotype_data_altered_hash_5->{$p}->{$t};
9149 my $effect_original_5 = $result_blup_spatial_data_original_5->{$p}->{$t}->[0];
9150 my $effect_post_5 = $result_blup_spatial_data_altered_5->{$p}->{$t}->[0];
9151 push @row, ($phenotype_original, $phenotype_post_5, $effect_original_5, $effect_post_5);
9152
9153 my $sim_env = $sim_data_hash_1_5->{$p}->{$t};
9154 my $pheno_sim_5 = $phenotype_data_altered_env_hash_1_5->{$p}->{$t};
9155 my $effect_sim_5 = $result_blup_spatial_data_altered_env_1_5->{$p}->{$t}->[0];
9156 push @row, ($sim_env, $pheno_sim_5, $effect_sim_5);
9157
9158 my $sim_env2 = $sim_data_hash_2_5->{$p}->{$t};
9159 my $pheno_sim2_5 = $phenotype_data_altered_env_hash_2_5->{$p}->{$t};
9160 my $effect_sim2_5 = $result_blup_spatial_data_altered_env_2_5->{$p}->{$t}->[0];
9161 push @row, ($sim_env2, $pheno_sim2_5, $effect_sim2_5);
9162
9163 my $sim_env3 = $sim_data_hash_3_5->{$p}->{$t};
9164 my $pheno_sim3_5 = $phenotype_data_altered_env_hash_3_5->{$p}->{$t};
9165 my $effect_sim3_5 = $result_blup_spatial_data_altered_env_3_5->{$p}->{$t}->[0];
9166 push @row, ($sim_env3, $pheno_sim3_5, $effect_sim3_5);
9167
9168 my $sim_env4 = $sim_data_hash_4_5->{$p}->{$t};
9169 my $pheno_sim4_5 = $phenotype_data_altered_env_hash_4_5->{$p}->{$t};
9170 my $effect_sim4_5 = $result_blup_spatial_data_altered_env_4_5->{$p}->{$t}->[0];
9171 push @row, ($sim_env4, $pheno_sim4_5, $effect_sim4_5);
9172
9173 my $sim_env5 = $sim_data_hash_5_5->{$p}->{$t};
9174 my $pheno_sim5_5 = $phenotype_data_altered_env_hash_5_5->{$p}->{$t};
9175 my $effect_sim5_5 = $result_blup_spatial_data_altered_env_5_5->{$p}->{$t}->[0];
9176 push @row, ($sim_env5, $pheno_sim5_5, $effect_sim5_5);
9177
9178 my $sim_env6 = $sim_data_hash_6_5->{$p}->{$t};
9179 my $pheno_sim6_5 = $phenotype_data_altered_env_hash_6_5->{$p}->{$t};
9180 my $effect_sim6_5 = $result_blup_spatial_data_altered_env_6_5->{$p}->{$t}->[0];
9181 push @row, ($sim_env6, $pheno_sim6_5, $effect_sim6_5);
9182 }
9183 my $line = join ',', @row;
9184 print $F_fullplot "$line\n";
9185 }
9186 close($F_fullplot);
9187
9188 my $plot_corr_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9189 $plot_corr_figure_tempfile_string .= '.png';
9190 my $plot_corr_figure_tempfile = $c->config->{basepath}."/".$plot_corr_figure_tempfile_string;
9191
9192 my $cmd_plotcorr_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
9193 mat_orig <- fread(\''.$full_plot_level_correlation_tempfile.'\', header=TRUE, sep=\',\');
9194 gg <- ggcorr(data=mat_orig[,-seq(1,8)], hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE);
9195 ggsave(\''.$plot_corr_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, limitsize = FALSE, units=\'in\');
9196 "';
9197 # print STDERR Dumper $cmd;
9198 my $status_plotcorr_plot = system($cmd_plotcorr_plot);
9199 push @$spatial_effects_plots, [$plot_corr_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
9200 push @$spatial_effects_files_store, [$full_plot_level_correlation_tempfile, "datafile_".$statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
9201 };
9202
9203 eval {
9204 my @plot_corr_full_vals;
9205
9206 my @original_pheno_vals;
9207 my ($phenotypes_original_heatmap_tempfile_fh, $phenotypes_original_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9208 open(my $F_pheno, ">", $phenotypes_original_heatmap_tempfile) || die "Can't open file ".$phenotypes_original_heatmap_tempfile;
9209 print $F_pheno "trait_type,row,col,value\n";
9210 foreach my $p (@unique_plot_names) {
9211 foreach my $t (@sorted_trait_names_5) {
9212 my $val = $phenotype_data_original_5{$p}->{$t};
9213 my @row = ("pheno_orig_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9214 push @original_pheno_vals, $val;
9215 my $line = join ',', @row;
9216 print $F_pheno "$line\n";
9217 }
9218 }
9219 close($F_pheno);
9220 push @plot_corr_full_vals, \@original_pheno_vals;
9221
9222 my $original_pheno_stat = Statistics::Descriptive::Full->new();
9223 $original_pheno_stat->add_data(@original_pheno_vals);
9224 my $sig_original_pheno = $original_pheno_stat->variance();
9225
9226 #PHENO POST M START
9227
9228 my @altered_pheno_vals_5;
9229 my ($phenotypes_post_heatmap_tempfile_fh_5, $phenotypes_post_heatmap_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9230 open($F_pheno, ">", $phenotypes_post_heatmap_tempfile_5) || die "Can't open file ".$phenotypes_post_heatmap_tempfile_5;
9231 print $F_pheno "trait_type,row,col,value\n";
9232 foreach my $p (@unique_plot_names) {
9233 foreach my $t (@sorted_trait_names_5) {
9234 my $val = $phenotype_data_altered_hash_5->{$p}->{$t};
9235 my @row = ("pheno_postm5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9236 push @altered_pheno_vals_5, $val;
9237 my $line = join ',', @row;
9238 print $F_pheno "$line\n";
9239 }
9240 }
9241 close($F_pheno);
9242 push @plot_corr_full_vals, \@altered_pheno_vals_5;
9243
9244 my $altered_pheno_stat_5 = Statistics::Descriptive::Full->new();
9245 $altered_pheno_stat_5->add_data(@altered_pheno_vals_5);
9246 my $sig_altered_pheno_5 = $altered_pheno_stat_5->variance();
9247
9248 # EFFECT ORIGINAL M
9249
9250 my @original_effect_vals_5;
9251 my ($effects_heatmap_tempfile_fh_5, $effects_heatmap_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9252 open(my $F_eff, ">", $effects_heatmap_tempfile_5) || die "Can't open file ".$effects_heatmap_tempfile_5;
9253 print $F_eff "trait_type,row,col,value\n";
9254 foreach my $p (@unique_plot_names) {
9255 foreach my $t (@sorted_trait_names_5) {
9256 my $val = $result_blup_spatial_data_original_5->{$p}->{$t}->[0];
9257 my @row = ("eff_origm5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9258 my $line = join ',', @row;
9259 print $F_eff "$line\n";
9260 push @original_effect_vals_5, $val;
9261 }
9262 }
9263 close($F_eff);
9264 push @plot_corr_full_vals, \@original_effect_vals_5;
9265
9266 my $original_effect_stat_5 = Statistics::Descriptive::Full->new();
9267 $original_effect_stat_5->add_data(@original_effect_vals_5);
9268 my $sig_original_effect_5 = $original_effect_stat_5->variance();
9269
9270 # EFFECT POST M MIN
9271
9272 my @altered_effect_vals_5;
9273 my ($effects_post_heatmap_tempfile_fh_5, $effects_post_heatmap_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9274 open($F_eff, ">", $effects_post_heatmap_tempfile_5) || die "Can't open file ".$effects_post_heatmap_tempfile_5;
9275 print $F_eff "trait_type,row,col,value\n";
9276 foreach my $p (@unique_plot_names) {
9277 foreach my $t (@sorted_trait_names_5) {
9278 my $val = $result_blup_spatial_data_altered_5->{$p}->{$t}->[0];
9279 my @row = ("eff_postm5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9280 my $line = join ',', @row;
9281 print $F_eff "$line\n";
9282 push @altered_effect_vals_5, $val;
9283 }
9284 }
9285 close($F_eff);
9286 push @plot_corr_full_vals, \@altered_effect_vals_5;
9287
9288 my $altered_effect_stat_5 = Statistics::Descriptive::Full->new();
9289 $altered_effect_stat_5->add_data(@altered_effect_vals_5);
9290 my $sig_altered_effect_5 = $altered_effect_stat_5->variance();
9291
9292 # SIM ENV 1: ALTERED PHENO + EFFECT
9293
9294 my ($phenotypes_env_heatmap_tempfile_fh, $phenotypes_env_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9295 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile) || die "Can't open file ".$phenotypes_env_heatmap_tempfile;
9296 print $F_pheno "trait_type,row,col,value\n";
9297 foreach my $p (@unique_plot_names) {
9298 foreach my $t (@sorted_trait_names_5) {
9299 my @row = ("sim_env1_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_1_5->{$p}->{$t});
9300 my $line = join ',', @row;
9301 print $F_pheno "$line\n";
9302 }
9303 }
9304 close($F_pheno);
9305
9306 my @sim_pheno1_vals_5;
9307 my ($phenotypes_pheno_sim_heatmap_tempfile_fh_5, $phenotypes_pheno_sim_heatmap_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9308 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile_5;
9309 print $F_pheno "trait_type,row,col,value\n";
9310 foreach my $p (@unique_plot_names) {
9311 foreach my $t (@sorted_trait_names_5) {
9312 my $val = $phenotype_data_altered_env_hash_1_5->{$p}->{$t};
9313 my @row = ("simm5_pheno1_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9314 my $line = join ',', @row;
9315 print $F_pheno "$line\n";
9316 push @sim_pheno1_vals_5, $val;
9317 }
9318 }
9319 close($F_pheno);
9320 push @plot_corr_full_vals, \@sim_pheno1_vals_5;
9321
9322 my $sim_pheno1_stat_5 = Statistics::Descriptive::Full->new();
9323 $sim_pheno1_stat_5->add_data(@sim_pheno1_vals_5);
9324 my $sig_sim5_pheno1 = $sim_pheno1_stat_5->variance();
9325
9326 my @sim_effect1_vals_5;
9327 my ($effects_sim_heatmap_tempfile_fh_5, $effects_sim_heatmap_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9328 open($F_eff, ">", $effects_sim_heatmap_tempfile_5) || die "Can't open file ".$effects_sim_heatmap_tempfile_5;
9329 print $F_eff "trait_type,row,col,value\n";
9330 foreach my $p (@unique_plot_names) {
9331 foreach my $t (@sorted_trait_names_5) {
9332 my $val = $result_blup_spatial_data_altered_env_1_5->{$p}->{$t}->[0];
9333 my @row = ("effm5_sim1_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9334 my $line = join ',', @row;
9335 print $F_eff "$line\n";
9336 push @sim_effect1_vals_5, $val;
9337 }
9338 }
9339 close($F_eff);
9340 push @plot_corr_full_vals, \@sim_effect1_vals_5;
9341
9342 my $sim_effect1_stat_5 = Statistics::Descriptive::Full->new();
9343 $sim_effect1_stat_5->add_data(@sim_effect1_vals_5);
9344 my $sig_sim5_effect1 = $sim_effect1_stat_5->variance();
9345
9346 # SIM ENV 2: ALTERED PHENO + EFFECT
9347
9348 my ($phenotypes_env_heatmap_tempfile2_fh, $phenotypes_env_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9349 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile2) || die "Can't open file ".$phenotypes_env_heatmap_tempfile2;
9350 print $F_pheno "trait_type,row,col,value\n";
9351 foreach my $p (@unique_plot_names) {
9352 foreach my $t (@sorted_trait_names_5) {
9353 my @row = ("sim_env2_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_2_5->{$p}->{$t});
9354 my $line = join ',', @row;
9355 print $F_pheno "$line\n";
9356 }
9357 }
9358 close($F_pheno);
9359
9360 my @sim_pheno2_vals_5;
9361 my ($phenotypes_pheno_sim_heatmap_tempfile2_fh_5, $phenotypes_pheno_sim_heatmap_tempfile2_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9362 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile2_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile2_5;
9363 print $F_pheno "trait_type,row,col,value\n";
9364 foreach my $p (@unique_plot_names) {
9365 foreach my $t (@sorted_trait_names_5) {
9366 my $val = $phenotype_data_altered_env_hash_2_5->{$p}->{$t};
9367 my @row = ("simm5_pheno2_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9368 my $line = join ',', @row;
9369 print $F_pheno "$line\n";
9370 push @sim_pheno2_vals_5, $val;
9371 }
9372 }
9373 close($F_pheno);
9374 push @plot_corr_full_vals, \@sim_pheno2_vals_5;
9375
9376 my $sim_pheno2_stat_5 = Statistics::Descriptive::Full->new();
9377 $sim_pheno2_stat_5->add_data(@sim_pheno2_vals_5);
9378 my $sig_sim_pheno2_5 = $sim_pheno2_stat_5->variance();
9379
9380 my @sim_effect2_vals_5;
9381 my ($effects_sim_heatmap_tempfile2_fh_5, $effects_sim_heatmap_tempfile2_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9382 open($F_eff, ">", $effects_sim_heatmap_tempfile2_5) || die "Can't open file ".$effects_sim_heatmap_tempfile2_5;
9383 print $F_eff "trait_type,row,col,value\n";
9384 foreach my $p (@unique_plot_names) {
9385 foreach my $t (@sorted_trait_names_5) {
9386 my $val = $result_blup_spatial_data_altered_env_2_5->{$p}->{$t}->[0];
9387 my @row = ("effm5_sim2_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9388 my $line = join ',', @row;
9389 print $F_eff "$line\n";
9390 push @sim_effect2_vals_5, $val;
9391 }
9392 }
9393 close($F_eff);
9394 push @plot_corr_full_vals, \@sim_effect2_vals_5;
9395
9396 my $sim_effect2_stat_5 = Statistics::Descriptive::Full->new();
9397 $sim_effect2_stat_5->add_data(@sim_effect2_vals_5);
9398 my $sig_sim_effect2_5 = $sim_effect2_stat_5->variance();
9399
9400 # SIM ENV 3: ALTERED PHENO + EFFECT
9401
9402 my ($phenotypes_env_heatmap_tempfile3_fh, $phenotypes_env_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9403 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile3) || die "Can't open file ".$phenotypes_env_heatmap_tempfile3;
9404 print $F_pheno "trait_type,row,col,value\n";
9405 foreach my $p (@unique_plot_names) {
9406 foreach my $t (@sorted_trait_names_5) {
9407 my @row = ("sim_env3_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_3_5->{$p}->{$t});
9408 my $line = join ',', @row;
9409 print $F_pheno "$line\n";
9410 }
9411 }
9412 close($F_pheno);
9413
9414 my @sim_pheno3_vals_5;
9415 my ($phenotypes_pheno_sim_heatmap_tempfile3_fh_5, $phenotypes_pheno_sim_heatmap_tempfile3_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9416 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile3_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile3_5;
9417 print $F_pheno "trait_type,row,col,value\n";
9418 foreach my $p (@unique_plot_names) {
9419 foreach my $t (@sorted_trait_names_5) {
9420 my $val = $phenotype_data_altered_env_hash_3_5->{$p}->{$t};
9421 my @row = ("simm5_pheno3_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9422 my $line = join ',', @row;
9423 print $F_pheno "$line\n";
9424 push @sim_pheno3_vals_5, $val;
9425 }
9426 }
9427 close($F_pheno);
9428 push @plot_corr_full_vals, \@sim_pheno3_vals_5;
9429
9430 my $sim_pheno3_stat_5 = Statistics::Descriptive::Full->new();
9431 $sim_pheno3_stat_5->add_data(@sim_pheno3_vals_5);
9432 my $sig_sim_pheno3_5 = $sim_pheno3_stat_5->variance();
9433
9434 my @sim_effect3_vals_5;
9435 my ($effects_sim_heatmap_tempfile3_fh_5, $effects_sim_heatmap_tempfile3_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9436 open($F_eff, ">", $effects_sim_heatmap_tempfile3_5) || die "Can't open file ".$effects_sim_heatmap_tempfile3_5;
9437 print $F_eff "trait_type,row,col,value\n";
9438 foreach my $p (@unique_plot_names) {
9439 foreach my $t (@sorted_trait_names_5) {
9440 my $val = $result_blup_spatial_data_altered_env_3_5->{$p}->{$t}->[0];
9441 my @row = ("effm5_sim3_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9442 my $line = join ',', @row;
9443 print $F_eff "$line\n";
9444 push @sim_effect3_vals_5, $val;
9445 }
9446 }
9447 close($F_eff);
9448 push @plot_corr_full_vals, \@sim_effect3_vals_5;
9449
9450 my $sim_effect3_stat_5 = Statistics::Descriptive::Full->new();
9451 $sim_effect3_stat_5->add_data(@sim_effect3_vals_5);
9452 my $sig_sim_effect3_5 = $sim_effect3_stat_5->variance();
9453
9454 # SIM ENV 4: ALTERED PHENO + EFFECT
9455
9456 my ($phenotypes_env_heatmap_tempfile4_fh, $phenotypes_env_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9457 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile4) || die "Can't open file ".$phenotypes_env_heatmap_tempfile4;
9458 print $F_pheno "trait_type,row,col,value\n";
9459 foreach my $p (@unique_plot_names) {
9460 foreach my $t (@sorted_trait_names_5) {
9461 my @row = ("sim_env4_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_4_5->{$p}->{$t});
9462 my $line = join ',', @row;
9463 print $F_pheno "$line\n";
9464 }
9465 }
9466 close($F_pheno);
9467
9468 my @sim_pheno4_vals_5;
9469 my ($phenotypes_pheno_sim_heatmap_tempfile4_fh_5, $phenotypes_pheno_sim_heatmap_tempfile4_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9470 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile4_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile4_5;
9471 print $F_pheno "trait_type,row,col,value\n";
9472 foreach my $p (@unique_plot_names) {
9473 foreach my $t (@sorted_trait_names_5) {
9474 my $val = $phenotype_data_altered_env_hash_4_5->{$p}->{$t};
9475 my @row = ("simm5_pheno4_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9476 my $line = join ',', @row;
9477 print $F_pheno "$line\n";
9478 push @sim_pheno4_vals_5, $val;
9479 }
9480 }
9481 close($F_pheno);
9482 push @plot_corr_full_vals, \@sim_pheno4_vals_5;
9483
9484 my $sim_pheno4_stat_5 = Statistics::Descriptive::Full->new();
9485 $sim_pheno4_stat_5->add_data(@sim_pheno4_vals_5);
9486 my $sig_sim_pheno4_5 = $sim_pheno4_stat_5->variance();
9487
9488 my @sim_effect4_vals_5;
9489 my ($effects_sim_heatmap_tempfile4_fh_5, $effects_sim_heatmap_tempfile4_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9490 open($F_eff, ">", $effects_sim_heatmap_tempfile4_5) || die "Can't open file ".$effects_sim_heatmap_tempfile4_5;
9491 print $F_eff "trait_type,row,col,value\n";
9492 foreach my $p (@unique_plot_names) {
9493 foreach my $t (@sorted_trait_names_5) {
9494 my $val = $result_blup_spatial_data_altered_env_4_5->{$p}->{$t}->[0];
9495 my @row = ("effm5_sim4_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9496 my $line = join ',', @row;
9497 print $F_eff "$line\n";
9498 push @sim_effect4_vals_5, $val;
9499 }
9500 }
9501 close($F_eff);
9502 push @plot_corr_full_vals, \@sim_effect4_vals_5;
9503
9504 my $sim_effect4_stat_5 = Statistics::Descriptive::Full->new();
9505 $sim_effect4_stat_5->add_data(@sim_effect4_vals_5);
9506 my $sig_sim_effect4_5 = $sim_effect4_stat_5->variance();
9507
9508 # SIM ENV 5: ALTERED PHENO + EFFECT
9509
9510 my ($phenotypes_env_heatmap_tempfile5_fh, $phenotypes_env_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9511 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile5) || die "Can't open file ".$phenotypes_env_heatmap_tempfile5;
9512 print $F_pheno "trait_type,row,col,value\n";
9513 foreach my $p (@unique_plot_names) {
9514 foreach my $t (@sorted_trait_names_5) {
9515 my @row = ("sim_env5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_5_5->{$p}->{$t});
9516 my $line = join ',', @row;
9517 print $F_pheno "$line\n";
9518 }
9519 }
9520 close($F_pheno);
9521
9522 my @sim_pheno5_vals_5;
9523 my ($phenotypes_pheno_sim_heatmap_tempfile5_fh_5, $phenotypes_pheno_sim_heatmap_tempfile5_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9524 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile5_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile5_5;
9525 print $F_pheno "trait_type,row,col,value\n";
9526 foreach my $p (@unique_plot_names) {
9527 foreach my $t (@sorted_trait_names_5) {
9528 my $val = $phenotype_data_altered_env_hash_5_5->{$p}->{$t};
9529 my @row = ("simm5_pheno5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9530 my $line = join ',', @row;
9531 print $F_pheno "$line\n";
9532 push @sim_pheno5_vals_5, $val;
9533 }
9534 }
9535 close($F_pheno);
9536 push @plot_corr_full_vals, \@sim_pheno5_vals_5;
9537
9538 my $sim_pheno5_stat_5 = Statistics::Descriptive::Full->new();
9539 $sim_pheno5_stat_5->add_data(@sim_pheno5_vals_5);
9540 my $sig_sim_pheno5_5 = $sim_pheno5_stat_5->variance();
9541
9542 my @sim_effect5_vals_5;
9543 my ($effects_sim_heatmap_tempfile5_fh_5, $effects_sim_heatmap_tempfile5_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9544 open($F_eff, ">", $effects_sim_heatmap_tempfile5_5) || die "Can't open file ".$effects_sim_heatmap_tempfile5_5;
9545 print $F_eff "trait_type,row,col,value\n";
9546 foreach my $p (@unique_plot_names) {
9547 foreach my $t (@sorted_trait_names_5) {
9548 my $val = $result_blup_spatial_data_altered_env_5_5->{$p}->{$t}->[0];
9549 my @row = ("effm5_sim5_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9550 my $line = join ',', @row;
9551 print $F_eff "$line\n";
9552 push @sim_effect5_vals_5, $val;
9553 }
9554 }
9555 close($F_eff);
9556 push @plot_corr_full_vals, \@sim_effect5_vals_5;
9557
9558 my $sim_effect5_stat_5 = Statistics::Descriptive::Full->new();
9559 $sim_effect5_stat_5->add_data(@sim_effect5_vals_5);
9560 my $sig_sim_effect5_5 = $sim_effect5_stat_5->variance();
9561
9562 # SIM ENV 6: ALTERED PHENO + EFFECT
9563
9564 my ($phenotypes_env_heatmap_tempfile6_fh, $phenotypes_env_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9565 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile6) || die "Can't open file ".$phenotypes_env_heatmap_tempfile6;
9566 print $F_pheno "trait_type,row,col,value\n";
9567 foreach my $p (@unique_plot_names) {
9568 foreach my $t (@sorted_trait_names_5) {
9569 my @row = ("sim_env6_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_6_5->{$p}->{$t});
9570 my $line = join ',', @row;
9571 print $F_pheno "$line\n";
9572 }
9573 }
9574 close($F_pheno);
9575
9576 my @sim_pheno6_vals_5;
9577 my ($phenotypes_pheno_sim_heatmap_tempfile6_fh_5, $phenotypes_pheno_sim_heatmap_tempfile6_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9578 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile6_5) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile6_5;
9579 print $F_pheno "trait_type,row,col,value\n";
9580 foreach my $p (@unique_plot_names) {
9581 foreach my $t (@sorted_trait_names_5) {
9582 my $val = $phenotype_data_altered_env_hash_6_5->{$p}->{$t};
9583 my @row = ("simm5_pheno6_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9584 my $line = join ',', @row;
9585 print $F_pheno "$line\n";
9586 push @sim_pheno6_vals_5, $val;
9587 }
9588 }
9589 close($F_pheno);
9590 push @plot_corr_full_vals, \@sim_pheno6_vals_5;
9591
9592 my $sim_pheno6_stat_5 = Statistics::Descriptive::Full->new();
9593 $sim_pheno6_stat_5->add_data(@sim_pheno6_vals_5);
9594 my $sig_sim_pheno6_5 = $sim_pheno6_stat_5->variance();
9595
9596 my @sim_effect6_vals_5;
9597 my ($effects_sim_heatmap_tempfile6_fh_5, $effects_sim_heatmap_tempfile6_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9598 open($F_eff, ">", $effects_sim_heatmap_tempfile6_5) || die "Can't open file ".$effects_sim_heatmap_tempfile6_5;
9599 print $F_eff "trait_type,row,col,value\n";
9600 foreach my $p (@unique_plot_names) {
9601 foreach my $t (@sorted_trait_names_5) {
9602 my $val = $result_blup_spatial_data_altered_env_6_5->{$p}->{$t}->[0];
9603 my @row = ("effm5_sim6_".$trait_name_encoder_5{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
9604 my $line = join ',', @row;
9605 print $F_eff "$line\n";
9606 push @sim_effect6_vals_5, $val;
9607 }
9608 }
9609 close($F_eff);
9610 push @plot_corr_full_vals, \@sim_effect6_vals_5;
9611
9612 my $sim_effect6_stat_5 = Statistics::Descriptive::Full->new();
9613 $sim_effect6_stat_5->add_data(@sim_effect6_vals_5);
9614 my $sig_sim_effect6_5 = $sim_effect6_stat_5->variance();
9615
9616 my $plot_corr_summary_figure_inputfile_tempfile = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'tmp_drone_statistics/fileXXXX');
9617 open($F_eff, ">", $plot_corr_summary_figure_inputfile_tempfile) || die "Can't open file ".$plot_corr_summary_figure_inputfile_tempfile;
9618 foreach (@plot_corr_full_vals) {
9619 my $line = join ',', @$_;
9620 print $F_eff $line."\n";
9621 }
9622 close($F_eff);
9623
9624 my $plot_corr_summary_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9625 $plot_corr_summary_figure_tempfile_string .= '.png';
9626 my $plot_corr_summary_figure_tempfile = $c->config->{basepath}."/".$plot_corr_summary_figure_tempfile_string;
9627
9628 my $cmd_plotcorrsum_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
9629 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
9630 mat_full <- data.frame(t(mat_full_t));
9631 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_5\', \'mat_eff_5\', \'mat_eff_altered_5\',
9632 \'mat_p_sim1_5\', \'mat_eff_sim1_5\',
9633 \'mat_p_sim2_5\', \'mat_eff_sim2_5\',
9634 \'mat_p_sim3_5\', \'mat_eff_sim3_5\',
9635 \'mat_p_sim4_5\', \'mat_eff_sim4_5\',
9636 \'mat_p_sim5_5\', \'mat_eff_sim5_5\',
9637 \'mat_p_sim6_5\', \'mat_eff_sim6_5\');
9638 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
9639 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
9640 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
9641 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
9642 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
9643 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
9644 mat <- data.frame(pheno_orig = mat_full\$mat_orig, pheno_altm5 = mat_full\$mat_altered_5, eff_origm5 = mat_full\$mat_eff_5, eff_altm5 = mat_full\$mat_eff_altered_5, env_lin = mat_env\$value, pheno_linm5 = mat_full\$mat_p_sim1_5, lin_effm5 = mat_full\$mat_eff_sim1_5, env_n1d = mat_env2\$value, pheno_n1dm5 = mat_full\$mat_p_sim2_5, n1d_effm5 = mat_full\$mat_eff_sim2_5, env_n2d = mat_env3\$value, pheno_n2dm5 = mat_full\$mat_p_sim3_5, env_rand = mat_env4\$value, pheno_randm5 = mat_full\$mat_p_sim4_5, rand_effm5 = mat_full\$mat_eff_sim4_5, env_ar1 = mat_env5\$value, pheno_ar1m5 = mat_full\$mat_p_sim5_5, ar1_effm5 = mat_full\$mat_eff_sim5_5, env_realdata = mat_env6\$value, pheno_realdatam5 = mat_full\$mat_p_sim6_5, realdata_effm5 = mat_full\$mat_eff_sim6_5);
9645 gg <- ggcorr(data=mat, hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE, label_round = 2);
9646 ggsave(\''.$plot_corr_summary_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, units=\'in\');
9647 "';
9648 # print STDERR Dumper $cmd_plotcorrsum_plot;
9649
9650 my $status_plotcorrsum_plot = system($cmd_plotcorrsum_plot);
9651 push @$spatial_effects_plots, [$plot_corr_summary_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_envsimscorr_"."envvar_".$env_variance_percent."_".$iterations];
9652
9653 my $env_effects_first_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9654 $env_effects_first_figure_tempfile_string .= '.png';
9655 my $env_effects_first_figure_tempfile = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string;
9656
9657 my $env_effects_first_figure_tempfile_string_2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9658 $env_effects_first_figure_tempfile_string_2 .= '.png';
9659 my $env_effects_first_figure_tempfile_2 = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string_2;
9660
9661 my $output_plot_row = 'row';
9662 my $output_plot_col = 'col';
9663 if ($max_col > $max_row) {
9664 $output_plot_row = 'col';
9665 $output_plot_col = 'row';
9666 }
9667
9668 my $cmd_spatialfirst_plot_2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9669 mat_orig <- fread(\''.$phenotypes_original_heatmap_tempfile.'\', header=TRUE, sep=\',\');
9670 mat_altered_5 <- fread(\''.$phenotypes_post_heatmap_tempfile_5.'\', header=TRUE, sep=\',\');
9671 pheno_mat <- rbind(mat_orig, mat_altered_5);
9672 options(device=\'png\');
9673 par();
9674 gg <- ggplot(pheno_mat, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9675 geom_tile() +
9676 scale_fill_viridis(discrete=FALSE) +
9677 coord_equal() +
9678 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9679 ggsave(\''.$env_effects_first_figure_tempfile_2.'\', gg, device=\'png\', width=20, height=20, units=\'in\');
9680 "';
9681 # print STDERR Dumper $cmd;
9682 my $status_spatialfirst_plot_2 = system($cmd_spatialfirst_plot_2);
9683 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string_2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_origheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9684
9685 my ($sim_effects_corr_results_fh, $sim_effects_corr_results) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9686
9687 my $cmd_spatialfirst_plot = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9688 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
9689 mat_full <- data.frame(t(mat_full_t));
9690 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_5\', \'mat_eff_5\', \'mat_eff_altered_5\',
9691 \'mat_p_sim1_5\', \'mat_eff_sim1_5\',
9692 \'mat_p_sim2_5\', \'mat_eff_sim2_5\',
9693 \'mat_p_sim3_5\', \'mat_eff_sim3_5\',
9694 \'mat_p_sim4_5\', \'mat_eff_sim4_5\',
9695 \'mat_p_sim5_5\', \'mat_eff_sim5_5\',
9696 \'mat_p_sim6_5\', \'mat_eff_sim6_5\');
9697 mat_eff_5 <- fread(\''.$effects_heatmap_tempfile_5.'\', header=TRUE, sep=\',\');
9698 mat_eff_altered_5 <- fread(\''.$effects_post_heatmap_tempfile_5.'\', header=TRUE, sep=\',\');
9699 effect_mat_5 <- rbind(mat_eff_5, mat_eff_altered_5);
9700 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
9701 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
9702 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
9703 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
9704 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
9705 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
9706 options(device=\'png\');
9707 par();
9708 gg_eff_5 <- ggplot(effect_mat_5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9709 geom_tile() +
9710 scale_fill_viridis(discrete=FALSE) +
9711 coord_equal() +
9712 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9713 ggsave(\''.$env_effects_first_figure_tempfile.'\', arrangeGrob(gg_eff_5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9714 write.table(data.frame(asreml_grm_univariate_spatial_genetic_blups_env_linear = c(cor(mat_env\$value, mat_full\$mat_eff_sim1_5)), asreml_grm_univariate_spatial_genetic_blups_env_1DN = c(cor(mat_env2\$value, mat_full\$mat_eff_sim2_5)), asreml_grm_univariate_spatial_genetic_blups_env_2DN = c(cor(mat_env3\$value, mat_full\$mat_eff_sim3_5)), asreml_grm_univariate_spatial_genetic_blups_env_random = c(cor(mat_env4\$value, mat_full\$mat_eff_sim4_5)), asreml_grm_univariate_spatial_genetic_blups_env_ar1xar1 = c(cor(mat_env5\$value, mat_full\$mat_eff_sim5_5)), asreml_grm_univariate_spatial_genetic_blups_env_realdata = c(cor(mat_env6\$value, mat_full\$mat_eff_sim6_5)) ), file=\''.$sim_effects_corr_results.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
9715 "';
9716 # print STDERR Dumper $cmd;
9717 my $status_spatialfirst_plot = system($cmd_spatialfirst_plot);
9718 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_originaleffheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9719
9720 open(my $fh_corr_result, '<', $sim_effects_corr_results) or die "Could not open file '$sim_effects_corr_results' $!";
9721 print STDERR "Opened $sim_effects_corr_results\n";
9722
9723 my $header = <$fh_corr_result>;
9724 my @header;
9725 if ($csv->parse($header)) {
9726 @header = $csv->fields();
9727 }
9728
9729 while (my $row = <$fh_corr_result>) {
9730 my @columns;
9731 my $counter = 0;
9732 if ($csv->parse($row)) {
9733 @columns = $csv->fields();
9734 }
9735 foreach (@columns) {
9736 push @{$env_corr_res->{$header[$counter]."_corrtime_".$sim_env_change_over_time.$correlation_between_times."_envvar_".$env_variance_percent}->{values}}, $_;
9737 $counter++;
9738 }
9739 }
9740 close($fh_corr_result);
9741
9742 my $env_effects_sim_figure_tempfile_string_5_env1 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9743 $env_effects_sim_figure_tempfile_string_5_env1 .= '.png';
9744 my $env_effects_sim_figure_tempfile_5_env1 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env1;
9745
9746 my $cmd_spatialenvsim_plot_5_env1 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9747 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
9748 mat_p_sim <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile_5.'\', header=TRUE, sep=\',\');
9749 mat_eff_sim <- fread(\''.$effects_sim_heatmap_tempfile_5.'\', header=TRUE, sep=\',\');
9750 options(device=\'png\');
9751 par();
9752 gg_env <- ggplot(mat_env, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9753 geom_tile() +
9754 scale_fill_viridis(discrete=FALSE) +
9755 coord_equal() +
9756 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9757 gg_p_sim <- ggplot(mat_p_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9758 geom_tile() +
9759 scale_fill_viridis(discrete=FALSE) +
9760 coord_equal() +
9761 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9762 gg_eff_sim <- ggplot(mat_eff_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9763 geom_tile() +
9764 scale_fill_viridis(discrete=FALSE) +
9765 coord_equal() +
9766 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9767 ggsave(\''.$env_effects_sim_figure_tempfile_5_env1.'\', arrangeGrob(gg_env, gg_p_sim, gg_eff_sim, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9768 "';
9769 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env1;
9770 my $status_spatialenvsim_plot_5_env1 = system($cmd_spatialenvsim_plot_5_env1);
9771 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env1, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env1effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9772
9773 my $env_effects_sim_figure_tempfile_string_5_env2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9774 $env_effects_sim_figure_tempfile_string_5_env2 .= '.png';
9775 my $env_effects_sim_figure_tempfile_5_env2 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env2;
9776
9777 my $cmd_spatialenvsim_plot_5_env2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9778 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
9779 mat_p_sim2 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile2_5.'\', header=TRUE, sep=\',\');
9780 mat_eff_sim2 <- fread(\''.$effects_sim_heatmap_tempfile2_5.'\', header=TRUE, sep=\',\');
9781 options(device=\'png\');
9782 par();
9783 gg_env2 <- ggplot(mat_env2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9784 geom_tile() +
9785 scale_fill_viridis(discrete=FALSE) +
9786 coord_equal() +
9787 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9788 gg_p_sim2 <- ggplot(mat_p_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9789 geom_tile() +
9790 scale_fill_viridis(discrete=FALSE) +
9791 coord_equal() +
9792 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9793 gg_eff_sim2 <- ggplot(mat_eff_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9794 geom_tile() +
9795 scale_fill_viridis(discrete=FALSE) +
9796 coord_equal() +
9797 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9798 ggsave(\''.$env_effects_sim_figure_tempfile_5_env2.'\', arrangeGrob(gg_env2, gg_p_sim2, gg_eff_sim2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9799 "';
9800 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env2;
9801 my $status_spatialenvsim_plot_5_env2 = system($cmd_spatialenvsim_plot_5_env2);
9802 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env2effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9803
9804 my $env_effects_sim_figure_tempfile_string_5_env3 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9805 $env_effects_sim_figure_tempfile_string_5_env3 .= '.png';
9806 my $env_effects_sim_figure_tempfile_5_env3 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env3;
9807
9808 my $cmd_spatialenvsim_plot_5_env3 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9809 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
9810 mat_p_sim3 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile3_5.'\', header=TRUE, sep=\',\');
9811 mat_eff_sim3 <- fread(\''.$effects_sim_heatmap_tempfile3_5.'\', header=TRUE, sep=\',\');
9812 options(device=\'png\');
9813 par();
9814 gg_env3 <- ggplot(mat_env3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9815 geom_tile() +
9816 scale_fill_viridis(discrete=FALSE) +
9817 coord_equal() +
9818 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9819 gg_p_sim3 <- ggplot(mat_p_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9820 geom_tile() +
9821 scale_fill_viridis(discrete=FALSE) +
9822 coord_equal() +
9823 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9824 gg_eff_sim3 <- ggplot(mat_eff_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9825 geom_tile() +
9826 scale_fill_viridis(discrete=FALSE) +
9827 coord_equal() +
9828 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9829 ggsave(\''.$env_effects_sim_figure_tempfile_5_env3.'\', arrangeGrob(gg_env3, gg_p_sim3, gg_eff_sim3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9830 "';
9831 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env3;
9832 my $status_spatialenvsim_plot_5_env3 = system($cmd_spatialenvsim_plot_5_env3);
9833 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env3, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env3effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9834
9835 my $env_effects_sim_figure_tempfile_string_5_env4 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9836 $env_effects_sim_figure_tempfile_string_5_env4 .= '.png';
9837 my $env_effects_sim_figure_tempfile_5_env4 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env4;
9838
9839 my $cmd_spatialenvsim_plot_5_env4 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9840 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
9841 mat_p_sim4 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile4_5.'\', header=TRUE, sep=\',\');
9842 mat_eff_sim4 <- fread(\''.$effects_sim_heatmap_tempfile4_5.'\', header=TRUE, sep=\',\');
9843 options(device=\'png\');
9844 par();
9845 gg_env4 <- ggplot(mat_env4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9846 geom_tile() +
9847 scale_fill_viridis(discrete=FALSE) +
9848 coord_equal() +
9849 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9850 gg_p_sim4 <- ggplot(mat_p_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9851 geom_tile() +
9852 scale_fill_viridis(discrete=FALSE) +
9853 coord_equal() +
9854 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9855 gg_eff_sim4 <- ggplot(mat_eff_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9856 geom_tile() +
9857 scale_fill_viridis(discrete=FALSE) +
9858 coord_equal() +
9859 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9860 ggsave(\''.$env_effects_sim_figure_tempfile_5_env4.'\', arrangeGrob(gg_env4, gg_p_sim4, gg_eff_sim4, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9861 "';
9862 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env4;
9863 my $status_spatialenvsim_plot_5_env4 = system($cmd_spatialenvsim_plot_5_env4);
9864 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env4, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env4effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9865
9866 my $env_effects_sim_figure_tempfile_string_5_env5 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9867 $env_effects_sim_figure_tempfile_string_5_env5 .= '.png';
9868 my $env_effects_sim_figure_tempfile_5_env5 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env5;
9869
9870 my $cmd_spatialenvsim_plot_5_env5 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9871 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
9872 mat_p_sim5 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile5_5.'\', header=TRUE, sep=\',\');
9873 mat_eff_sim5 <- fread(\''.$effects_sim_heatmap_tempfile5_5.'\', header=TRUE, sep=\',\');
9874 options(device=\'png\');
9875 par();
9876 gg_env5 <- ggplot(mat_env5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9877 geom_tile() +
9878 scale_fill_viridis(discrete=FALSE) +
9879 coord_equal() +
9880 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9881 gg_p_sim5 <- ggplot(mat_p_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9882 geom_tile() +
9883 scale_fill_viridis(discrete=FALSE) +
9884 coord_equal() +
9885 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9886 gg_eff_sim5 <- ggplot(mat_eff_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9887 geom_tile() +
9888 scale_fill_viridis(discrete=FALSE) +
9889 coord_equal() +
9890 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9891 ggsave(\''.$env_effects_sim_figure_tempfile_5_env5.'\', arrangeGrob(gg_env5, gg_p_sim5, gg_eff_sim5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9892 "';
9893 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env5;
9894 my $status_spatialenvsim_plot_5_env5 = system($cmd_spatialenvsim_plot_5_env5);
9895 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env5, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env5effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9896
9897 my $env_effects_sim_figure_tempfile_string_5_env6 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
9898 $env_effects_sim_figure_tempfile_string_5_env6 .= '.png';
9899 my $env_effects_sim_figure_tempfile_5_env6 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_5_env6;
9900
9901 my $cmd_spatialenvsim_plot_5_env6 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
9902 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
9903 mat_p_sim6 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile6_5.'\', header=TRUE, sep=\',\');
9904 mat_eff_sim6 <- fread(\''.$effects_sim_heatmap_tempfile6_5.'\', header=TRUE, sep=\',\');
9905 options(device=\'png\');
9906 par();
9907 gg_env6 <- ggplot(mat_env6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9908 geom_tile() +
9909 scale_fill_viridis(discrete=FALSE) +
9910 coord_equal() +
9911 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9912 gg_p_sim6 <- ggplot(mat_p_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9913 geom_tile() +
9914 scale_fill_viridis(discrete=FALSE) +
9915 coord_equal() +
9916 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9917 gg_eff_sim6 <- ggplot(mat_eff_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
9918 geom_tile() +
9919 scale_fill_viridis(discrete=FALSE) +
9920 coord_equal() +
9921 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_5).');
9922 ggsave(\''.$env_effects_sim_figure_tempfile_5_env6.'\', arrangeGrob(gg_env6, gg_p_sim6, gg_eff_sim6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
9923 "';
9924 # print STDERR Dumper $cmd_spatialenvsim_plot_5_env6;
9925 my $status_spatialenvsim_plot_5_env6 = system($cmd_spatialenvsim_plot_5_env6);
9926 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_5_env6, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env6effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
9927 };
9928
9929 eval {
9930 my @sorted_germplasm_names = sort keys %unique_accessions;
9931
9932 my @original_blup_vals_5;
9933 my ($effects_original_line_chart_tempfile_fh_5, $effects_original_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9934 open(my $F_pheno, ">", $effects_original_line_chart_tempfile_5) || die "Can't open file ".$effects_original_line_chart_tempfile_5;
9935 print $F_pheno "germplasmName,time,value\n";
9936 foreach my $p (@sorted_germplasm_names) {
9937 foreach my $t (@sorted_trait_names_5) {
9938 my $val = $result_blup_data_original_5->{$p}->{$t}->[0];
9939 my @row = ($p, $t, $val);
9940 push @original_blup_vals_5, $val;
9941 my $line = join ',', @row;
9942 print $F_pheno "$line\n";
9943 }
9944 }
9945 close($F_pheno);
9946
9947 my $original_blup_stat_5 = Statistics::Descriptive::Full->new();
9948 $original_blup_stat_5->add_data(@original_blup_vals_5);
9949 my $sig_original_blup_5 = $original_blup_stat_5->variance();
9950
9951 my @altered_blups_vals_5;
9952 my ($effects_altered_line_chart_tempfile_fh_5, $effects_altered_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9953 open($F_pheno, ">", $effects_altered_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_line_chart_tempfile_5;
9954 print $F_pheno "germplasmName,time,value\n";
9955 foreach my $p (@sorted_germplasm_names) {
9956 foreach my $t (@sorted_trait_names_5) {
9957 my $val = $result_blup_data_altered_5->{$p}->{$t}->[0];
9958 my @row = ($p, $t, $val);
9959 my $line = join ',', @row;
9960 print $F_pheno "$line\n";
9961 push @altered_blups_vals_5, $val;
9962 }
9963 }
9964 close($F_pheno);
9965
9966 my $altered_blup_stat_5 = Statistics::Descriptive::Full->new();
9967 $altered_blup_stat_5->add_data(@altered_blups_vals_5);
9968 my $sig_altered_blup_5 = $altered_blup_stat_5->variance();
9969
9970 my @sim1_blup_vals_5;
9971 my ($effects_altered_env1_line_chart_tempfile_fh_5, $effects_altered_env1_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9972 open($F_pheno, ">", $effects_altered_env1_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env1_line_chart_tempfile_5;
9973 print $F_pheno "germplasmName,time,value\n";
9974 foreach my $p (@sorted_germplasm_names) {
9975 foreach my $t (@sorted_trait_names_5) {
9976 my $val = $result_blup_data_altered_env_1_5->{$p}->{$t}->[0];
9977 my @row = ($p, $t, $val);
9978 my $line = join ',', @row;
9979 print $F_pheno "$line\n";
9980 push @sim1_blup_vals_5, $val;
9981 }
9982 }
9983 close($F_pheno);
9984
9985 my $sim1_blup_stat_5 = Statistics::Descriptive::Full->new();
9986 $sim1_blup_stat_5->add_data(@sim1_blup_vals_5);
9987 my $sig_sim1_blup_5 = $sim1_blup_stat_5->variance();
9988
9989 my @sim2_blup_vals_5;
9990 my ($effects_altered_env2_line_chart_tempfile_fh_5, $effects_altered_env2_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
9991 open($F_pheno, ">", $effects_altered_env2_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env2_line_chart_tempfile_5;
9992 print $F_pheno "germplasmName,time,value\n";
9993 foreach my $p (@sorted_germplasm_names) {
9994 foreach my $t (@sorted_trait_names_5) {
9995 my $val = $result_blup_data_altered_env_2_5->{$p}->{$t}->[0];
9996 my @row = ($p, $t, $val);
9997 my $line = join ',', @row;
9998 print $F_pheno "$line\n";
9999 push @sim2_blup_vals_5, $val;
10000 }
10001 }
10002 close($F_pheno);
10003
10004 my $sim2_blup_stat_5 = Statistics::Descriptive::Full->new();
10005 $sim2_blup_stat_5->add_data(@sim2_blup_vals_5);
10006 my $sig_sim2_blup_5 = $sim2_blup_stat_5->variance();
10007
10008 my @sim3_blup_vals_5;
10009 my ($effects_altered_env3_line_chart_tempfile_fh_5, $effects_altered_env3_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
10010 open($F_pheno, ">", $effects_altered_env3_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env3_line_chart_tempfile_5;
10011 print $F_pheno "germplasmName,time,value\n";
10012 foreach my $p (@sorted_germplasm_names) {
10013 foreach my $t (@sorted_trait_names_5) {
10014 my $val = $result_blup_data_altered_env_3_5->{$p}->{$t}->[0];
10015 my @row = ($p, $t, $val);
10016 my $line = join ',', @row;
10017 print $F_pheno "$line\n";
10018 push @sim3_blup_vals_5, $val;
10019 }
10020 }
10021 close($F_pheno);
10022
10023 my $sim3_blup_stat_5 = Statistics::Descriptive::Full->new();
10024 $sim3_blup_stat_5->add_data(@sim3_blup_vals_5);
10025 my $sig_sim3_blup_5 = $sim3_blup_stat_5->variance();
10026
10027 my @sim4_blup_vals_5;
10028 my ($effects_altered_env4_line_chart_tempfile_fh_5, $effects_altered_env4_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
10029 open($F_pheno, ">", $effects_altered_env4_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env4_line_chart_tempfile_5;
10030 print $F_pheno "germplasmName,time,value\n";
10031 foreach my $p (@sorted_germplasm_names) {
10032 foreach my $t (@sorted_trait_names_5) {
10033 my $val = $result_blup_data_altered_env_4_5->{$p}->{$t}->[0];
10034 my @row = ($p, $t, $val);
10035 my $line = join ',', @row;
10036 print $F_pheno "$line\n";
10037 push @sim4_blup_vals_5, $val;
10038 }
10039 }
10040 close($F_pheno);
10041
10042 my $sim4_blup_stat_5 = Statistics::Descriptive::Full->new();
10043 $sim4_blup_stat_5->add_data(@sim4_blup_vals_5);
10044 my $sig_sim4_blup_5 = $sim4_blup_stat_5->variance();
10045
10046 my @sim5_blup_vals_5;
10047 my ($effects_altered_env5_line_chart_tempfile_fh_5, $effects_altered_env5_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
10048 open($F_pheno, ">", $effects_altered_env5_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env5_line_chart_tempfile_5;
10049 print $F_pheno "germplasmName,time,value\n";
10050 foreach my $p (@sorted_germplasm_names) {
10051 foreach my $t (@sorted_trait_names_5) {
10052 my $val = $result_blup_data_altered_env_5_5->{$p}->{$t}->[0];
10053 my @row = ($p, $t, $val);
10054 my $line = join ',', @row;
10055 print $F_pheno "$line\n";
10056 push @sim5_blup_vals_5, $val;
10057 }
10058 }
10059 close($F_pheno);
10060
10061 my $sim5_blup_stat_5 = Statistics::Descriptive::Full->new();
10062 $sim5_blup_stat_5->add_data(@sim5_blup_vals_5);
10063 my $sig_sim5_blup_5 = $sim5_blup_stat_5->variance();
10064
10065 my @sim6_blup_vals_5;
10066 my ($effects_altered_env6_line_chart_tempfile_fh_5, $effects_altered_env6_line_chart_tempfile_5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
10067 open($F_pheno, ">", $effects_altered_env6_line_chart_tempfile_5) || die "Can't open file ".$effects_altered_env6_line_chart_tempfile_5;
10068 print $F_pheno "germplasmName,time,value\n";
10069 foreach my $p (@sorted_germplasm_names) {
10070 foreach my $t (@sorted_trait_names_5) {
10071 my $val = $result_blup_data_altered_env_6_5->{$p}->{$t}->[0];
10072 my @row = ($p, $t, $val);
10073 my $line = join ',', @row;
10074 print $F_pheno "$line\n";
10075 push @sim6_blup_vals_5, $val;
10076 }
10077 }
10078 close($F_pheno);
10079
10080 my $sim6_blup_stat_5 = Statistics::Descriptive::Full->new();
10081 $sim6_blup_stat_5->add_data(@sim6_blup_vals_5);
10082 my $sig_sim6_blup_5 = $sim6_blup_stat_5->variance();
10083
10084 my @set = ('0' ..'9', 'A' .. 'F');
10085 my @colors;
10086 for (1..scalar(@sorted_germplasm_names)) {
10087 my $str = join '' => map $set[rand @set], 1 .. 6;
10088 push @colors, '#'.$str;
10089 }
10090 my $color_string = join '\',\'', @colors;
10091
10092 my $genetic_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10093 $genetic_effects_figure_tempfile_string .= '.png';
10094 my $genetic_effects_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_figure_tempfile_string;
10095
10096 my $genetic_effects_alt_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10097 $genetic_effects_alt_figure_tempfile_string .= '.png';
10098 my $genetic_effects_alt_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_figure_tempfile_string;
10099
10100 my $genetic_effects_alt_env1_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10101 $genetic_effects_alt_env1_figure_tempfile_string .= '.png';
10102 my $genetic_effects_alt_env1_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env1_figure_tempfile_string;
10103
10104 my $genetic_effects_alt_env2_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10105 $genetic_effects_alt_env2_figure_tempfile_string .= '.png';
10106 my $genetic_effects_alt_env2_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env2_figure_tempfile_string;
10107
10108 my $genetic_effects_alt_env3_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10109 $genetic_effects_alt_env3_figure_tempfile_string .= '.png';
10110 my $genetic_effects_alt_env3_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env3_figure_tempfile_string;
10111
10112 my $genetic_effects_alt_env4_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10113 $genetic_effects_alt_env4_figure_tempfile_string .= '.png';
10114 my $genetic_effects_alt_env4_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env4_figure_tempfile_string;
10115
10116 my $genetic_effects_alt_env5_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10117 $genetic_effects_alt_env5_figure_tempfile_string .= '.png';
10118 my $genetic_effects_alt_env5_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env5_figure_tempfile_string;
10119
10120 my $genetic_effects_alt_env6_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
10121 $genetic_effects_alt_env6_figure_tempfile_string .= '.png';
10122 my $genetic_effects_alt_env6_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env6_figure_tempfile_string;
10123
10124 my $cmd_gen_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10125 mat <- fread(\''.$effects_original_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10126 mat\$time <- as.numeric(as.character(mat\$time));
10127 options(device=\'png\');
10128 par();
10129 sp <- ggplot(mat, aes(x = time, y = value)) +
10130 geom_line(aes(color = germplasmName), size = 1) +
10131 scale_fill_manual(values = c(\''.$color_string.'\')) +
10132 theme_minimal();
10133 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10134 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10135 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10136 sp <- sp + labs(title = \'Original Genetic Effects\');';
10137 if (scalar(@sorted_germplasm_names) > 100) {
10138 $cmd_gen_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10139 }
10140 $cmd_gen_plot .= 'ggsave(\''.$genetic_effects_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10141 "';
10142 print STDERR Dumper $cmd_gen_plot;
10143 my $status_gen_plot = system($cmd_gen_plot);
10144 push @$spatial_effects_plots, [$genetic_effects_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_efforigline_"."envvar_".$env_variance_percent."_".$iterations];
10145
10146 my $cmd_gen_alt_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10147 mat <- fread(\''.$effects_altered_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10148 mat\$time <- as.numeric(as.character(mat\$time));
10149 options(device=\'png\');
10150 par();
10151 sp <- ggplot(mat, aes(x = time, y = value)) +
10152 geom_line(aes(color = germplasmName), size = 1) +
10153 scale_fill_manual(values = c(\''.$color_string.'\')) +
10154 theme_minimal();
10155 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10156 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10157 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10158 sp <- sp + labs(title = \'Altered Genetic Effects\');';
10159 if (scalar(@sorted_germplasm_names) > 100) {
10160 $cmd_gen_alt_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10161 }
10162 $cmd_gen_alt_plot .= 'ggsave(\''.$genetic_effects_alt_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10163 "';
10164 print STDERR Dumper $cmd_gen_alt_plot;
10165 my $status_gen_alt_plot = system($cmd_gen_alt_plot);
10166 push @$spatial_effects_plots, [$genetic_effects_alt_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltline_"."envvar_".$env_variance_percent."_".$iterations];
10167
10168 my $cmd_gen_env1_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10169 mat <- fread(\''.$effects_altered_env1_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10170 mat\$time <- as.numeric(as.character(mat\$time));
10171 options(device=\'png\');
10172 par();
10173 sp <- ggplot(mat, aes(x = time, y = value)) +
10174 geom_line(aes(color = germplasmName), size = 1) +
10175 scale_fill_manual(values = c(\''.$color_string.'\')) +
10176 theme_minimal();
10177 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10178 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10179 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10180 sp <- sp + labs(title = \'SimLinear Genetic Effects\');';
10181 if (scalar(@sorted_germplasm_names) > 100) {
10182 $cmd_gen_env1_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10183 }
10184 $cmd_gen_env1_plot .= 'ggsave(\''.$genetic_effects_alt_env1_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10185 "';
10186 print STDERR Dumper $cmd_gen_env1_plot;
10187 my $status_gen_env1_plot = system($cmd_gen_env1_plot);
10188 push @$spatial_effects_plots, [$genetic_effects_alt_env1_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv1line_"."envvar_".$env_variance_percent."_".$iterations];
10189
10190 my $cmd_gen_env2_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10191 mat <- fread(\''.$effects_altered_env2_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10192 mat\$time <- as.numeric(as.character(mat\$time));
10193 options(device=\'png\');
10194 par();
10195 sp <- ggplot(mat, aes(x = time, y = value)) +
10196 geom_line(aes(color = germplasmName), size = 1) +
10197 scale_fill_manual(values = c(\''.$color_string.'\')) +
10198 theme_minimal();
10199 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10200 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10201 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10202 sp <- sp + labs(title = \'Sim1DN Genetic Effects\');';
10203 if (scalar(@sorted_germplasm_names) > 100) {
10204 $cmd_gen_env2_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10205 }
10206 $cmd_gen_env2_plot .= 'ggsave(\''.$genetic_effects_alt_env2_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10207 "';
10208 print STDERR Dumper $cmd_gen_env2_plot;
10209 my $status_gen_env2_plot = system($cmd_gen_env2_plot);
10210 push @$spatial_effects_plots, [$genetic_effects_alt_env2_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv2line_"."envvar_".$env_variance_percent."_".$iterations];
10211
10212 my $cmd_gen_env3_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10213 mat <- fread(\''.$effects_altered_env3_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10214 mat\$time <- as.numeric(as.character(mat\$time));
10215 options(device=\'png\');
10216 par();
10217 sp <- ggplot(mat, aes(x = time, y = value)) +
10218 geom_line(aes(color = germplasmName), size = 1) +
10219 scale_fill_manual(values = c(\''.$color_string.'\')) +
10220 theme_minimal();
10221 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10222 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10223 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10224 sp <- sp + labs(title = \'Sim2DN Genetic Effects\');';
10225 if (scalar(@sorted_germplasm_names) > 100) {
10226 $cmd_gen_env3_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10227 }
10228 $cmd_gen_env3_plot .= 'ggsave(\''.$genetic_effects_alt_env3_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10229 "';
10230 print STDERR Dumper $cmd_gen_env3_plot;
10231 my $status_gen_env3_plot = system($cmd_gen_env3_plot);
10232 push @$spatial_effects_plots, [$genetic_effects_alt_env3_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv3line_"."envvar_".$env_variance_percent."_".$iterations];
10233
10234 my $cmd_gen_env4_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10235 mat <- fread(\''.$effects_altered_env4_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10236 mat\$time <- as.numeric(as.character(mat\$time));
10237 options(device=\'png\');
10238 par();
10239 sp <- ggplot(mat, aes(x = time, y = value)) +
10240 geom_line(aes(color = germplasmName), size = 1) +
10241 scale_fill_manual(values = c(\''.$color_string.'\')) +
10242 theme_minimal();
10243 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10244 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10245 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10246 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
10247 if (scalar(@sorted_germplasm_names) > 100) {
10248 $cmd_gen_env4_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10249 }
10250 $cmd_gen_env4_plot .= 'ggsave(\''.$genetic_effects_alt_env4_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10251 "';
10252 print STDERR Dumper $cmd_gen_env4_plot;
10253 my $status_gen_env4_plot = system($cmd_gen_env4_plot);
10254 push @$spatial_effects_plots, [$genetic_effects_alt_env4_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv4line_"."envvar_".$env_variance_percent."_".$iterations];
10255
10256 my $cmd_gen_env5_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10257 mat <- fread(\''.$effects_altered_env5_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10258 mat\$time <- as.numeric(as.character(mat\$time));
10259 options(device=\'png\');
10260 par();
10261 sp <- ggplot(mat, aes(x = time, y = value)) +
10262 geom_line(aes(color = germplasmName), size = 1) +
10263 scale_fill_manual(values = c(\''.$color_string.'\')) +
10264 theme_minimal();
10265 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10266 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10267 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10268 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
10269 if (scalar(@sorted_germplasm_names) > 100) {
10270 $cmd_gen_env5_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10271 }
10272 $cmd_gen_env5_plot .= 'ggsave(\''.$genetic_effects_alt_env5_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10273 "';
10274 print STDERR Dumper $cmd_gen_env5_plot;
10275 my $status_gen_env5_plot = system($cmd_gen_env5_plot);
10276 push @$spatial_effects_plots, [$genetic_effects_alt_env5_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv5line_"."envvar_".$env_variance_percent."_".$iterations];
10277
10278 my $cmd_gen_env6_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
10279 mat <- fread(\''.$effects_altered_env6_line_chart_tempfile_5.'\', header=TRUE, sep=\',\');
10280 mat\$time <- as.numeric(as.character(mat\$time));
10281 options(device=\'png\');
10282 par();
10283 sp <- ggplot(mat, aes(x = time, y = value)) +
10284 geom_line(aes(color = germplasmName), size = 1) +
10285 scale_fill_manual(values = c(\''.$color_string.'\')) +
10286 theme_minimal();
10287 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
10288 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
10289 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
10290 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
10291 if (scalar(@sorted_germplasm_names) > 100) {
10292 $cmd_gen_env6_plot .= 'sp <- sp + theme(legend.position = \'none\');';
10293 }
10294 $cmd_gen_env6_plot .= 'ggsave(\''.$genetic_effects_alt_env6_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
10295 "';
10296 print STDERR Dumper $cmd_gen_env6_plot;
10297 my $status_gen_env6_plot = system($cmd_gen_env6_plot);
10298 push @$spatial_effects_plots, [$genetic_effects_alt_env6_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv6line_"."envvar_".$env_variance_percent."_".$iterations];
10299 };
10300
10301 %trait_name_encoder = %trait_name_encoder_5;
10302 %trait_to_time_map = %trait_to_time_map_5;
10303
10304 push @$env_varcomps, {
10305 type => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times : Iteration $iterations",
10306 statistics_select => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times",
10307 correlation_between_times => $correlation_between_times,
10308 env_variance => $env_variance_percent,
10309 original => {
10310 covariance => $varcomp_original_array_5,
10311 cv_1 => $result_cv_original_5,
10312 cv_2 => $result_cv_2_original_5
10313 },
10314 altered => {
10315 covariance => $varcomp_altered_array_5,
10316 cv_1 => $result_cv_altered_5,
10317 cv_2 => $result_cv_2_altered_5
10318 },
10319 env_linear => {
10320 covariance => $varcomp_altered_array_env_1_5,
10321 cv_1 => $result_cv_altered_env_1_5,
10322 cv_2 => $result_cv_2_altered_env_1_5
10323 },
10324 env_1DN => {
10325 covariance => $varcomp_altered_array_env_2_5,
10326 cv_1 => $result_cv_altered_env_2_5,
10327 cv_2 => $result_cv_2_altered_env_2_5
10328 },
10329 env_2DN => {
10330 covariance => $varcomp_altered_array_env_3_5,
10331 cv_1 => $result_cv_altered_env_3_5,
10332 cv_2 => $result_cv_2_altered_env_3_5
10333 },
10334 env_random => {
10335 covariance => $varcomp_altered_array_env_4_5,
10336 cv_1 => $result_cv_altered_env_4_5,
10337 cv_2 => $result_cv_2_altered_env_4_5
10338 },
10339 env_ar1xar1 => {
10340 covariance => $varcomp_altered_array_env_5_5,
10341 cv_1 => $result_cv_altered_env_5_5,
10342 cv_2 => $result_cv_2_altered_env_5_5
10343 },
10344 env_realdata => {
10345 covariance => $varcomp_altered_array_env_6_5,
10346 cv_1 => $result_cv_altered_env_6_5,
10347 cv_2 => $result_cv_2_altered_env_6_5
10348 }
10349 };
10350 }
10351
10352 my (%phenotype_data_original_6, @data_matrix_original_6, @data_matrix_phenotypes_original_6);
10353 my (%trait_name_encoder_6, %trait_name_encoder_rev_6, %seen_days_after_plantings_6, %stock_info_6, %seen_times_6, %seen_trial_ids_6, %trait_to_time_map_6, %trait_composing_info_6, @sorted_trait_names_6, %seen_trait_names_6, %unique_traits_ids_6, @phenotype_header_6, $header_string_6);
10354 my (@sorted_scaled_ln_times_6, %plot_id_factor_map_reverse_6, %plot_id_count_map_reverse_6, %accession_id_factor_map_6, %accession_id_factor_map_reverse_6, %time_count_map_reverse_6, @rep_time_factors_6, @ind_rep_factors_6, %plot_rep_time_factor_map_6, %seen_rep_times_6, %seen_ind_reps_6, @legs_header_6, %polynomial_map_6);
10355 my $time_min_6 = 100000000;
10356 my $time_max_6 = 0;
10357 my $phenotype_min_original_6 = 1000000000;
10358 my $phenotype_max_original_6 = -1000000000;
10359
10360 if ($statistics_select_original eq 'asreml_grm_multivariate_spatial_genetic_blups') {
10361 $statistics_select = 'asreml_grm_multivariate_spatial_genetic_blups';
10362
10363 print STDERR "PREPARE ORIGINAL PHENOTYPE FILES 6\n";
10364 eval {
10365 my $phenotypes_search_6 = CXGN::Phenotypes::SearchFactory->instantiate(
10366 'MaterializedViewTable',
10367 {
10368 bcs_schema=>$schema,
10369 data_level=>'plot',
10370 trait_list=>$trait_id_list,
10371 trial_list=>$field_trial_id_list,
10372 include_timestamp=>0,
10373 exclude_phenotype_outlier=>0
10374 }
10375 );
10376 my ($data_6, $unique_traits_6) = $phenotypes_search_6->search();
10377 @sorted_trait_names_6 = sort keys %$unique_traits_6;
10378
10379 if (scalar(@$data_6) == 0) {
10380 $c->stash->{rest} = { error => "There are no phenotypes for the trials and traits you have selected!"};
10381 return;
10382 }
10383
10384 $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
10385 $h_time = $schema->storage->dbh()->prepare($q_time);
10386
10387 foreach my $obs_unit (@$data_6){
10388 my $germplasm_name = $obs_unit->{germplasm_uniquename};
10389 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
10390 my $replicate_number = $obs_unit->{obsunit_rep} || '';
10391 my $block_number = $obs_unit->{obsunit_block} || '';
10392 my $obsunit_stock_id = $obs_unit->{observationunit_stock_id};
10393 my $obsunit_stock_uniquename = $obs_unit->{observationunit_uniquename};
10394 my $row_number = $obs_unit->{obsunit_row_number} || '';
10395 my $col_number = $obs_unit->{obsunit_col_number} || '';
10396 push @plot_ids_ordered, $obsunit_stock_id;
10397
10398 if ($row_number < $min_row) {
10399 $min_row = $row_number;
10400 }
10401 elsif ($row_number >= $max_row) {
10402 $max_row = $row_number;
10403 }
10404 if ($col_number < $min_col) {
10405 $min_col = $col_number;
10406 }
10407 elsif ($col_number >= $max_col) {
10408 $max_col = $col_number;
10409 }
10410
10411 $obsunit_row_col{$row_number}->{$col_number} = {
10412 stock_id => $obsunit_stock_id,
10413 stock_uniquename => $obsunit_stock_uniquename
10414 };
10415 $seen_rows{$row_number}++;
10416 $seen_cols{$col_number}++;
10417 $plot_id_map{$obsunit_stock_id} = $obsunit_stock_uniquename;
10418 $seen_plot_names{$obsunit_stock_uniquename}++;
10419 $seen_plots{$obsunit_stock_id} = $obsunit_stock_uniquename;
10420 $stock_row_col{$obsunit_stock_id} = {
10421 row_number => $row_number,
10422 col_number => $col_number,
10423 obsunit_stock_id => $obsunit_stock_id,
10424 obsunit_name => $obsunit_stock_uniquename,
10425 rep => $replicate_number,
10426 block => $block_number,
10427 germplasm_stock_id => $germplasm_stock_id,
10428 germplasm_name => $germplasm_name
10429 };
10430 $stock_name_row_col{$obsunit_stock_uniquename} = {
10431 row_number => $row_number,
10432 col_number => $col_number,
10433 obsunit_stock_id => $obsunit_stock_id,
10434 obsunit_name => $obsunit_stock_uniquename,
10435 rep => $replicate_number,
10436 block => $block_number,
10437 germplasm_stock_id => $germplasm_stock_id,
10438 germplasm_name => $germplasm_name
10439 };
10440 $stock_row_col_id{$row_number}->{$col_number} = $obsunit_stock_id;
10441 $unique_accessions{$germplasm_name}++;
10442 $stock_info_6{$germplasm_stock_id} = {
10443 uniquename => $germplasm_name
10444 };
10445 my $observations = $obs_unit->{observations};
10446 foreach (@$observations){
10447 if ($_->{associated_image_project_time_json}) {
10448 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
10449 my $time_days_cvterm = $related_time_terms_json->{day};
10450 my $time_term_string = $time_days_cvterm;
10451 my $time_days = (split '\|', $time_days_cvterm)[0];
10452 my $time = (split ' ', $time_days)[1] + 0;
10453 $seen_days_after_plantings_6{$time}++;
10454
10455 my $value = $_->{value};
10456 my $trait_name = $_->{trait_name};
10457 $phenotype_data_original_6{$obsunit_stock_uniquename}->{$time} = $value;
10458 $seen_times_6{$time} = $trait_name;
10459 $seen_trait_names_6{$trait_name} = $time_term_string;
10460 $trait_to_time_map_6{$trait_name} = $time;
10461
10462 if ($value < $phenotype_min_original_6) {
10463 $phenotype_min_original_6 = $value;
10464 }
10465 elsif ($value >= $phenotype_max_original_6) {
10466 $phenotype_max_original_6 = $value;
10467 }
10468 }
10469 }
10470 }
10471 if (scalar(keys %seen_times_6) == 0) {
10472 $c->stash->{rest} = { error => "There are no phenotypes with associated days after planting time associated to the traits you have selected!"};
10473 return;
10474 }
10475
10476 @unique_accession_names = sort keys %unique_accessions;
10477 @sorted_trait_names_6 = sort {$a <=> $b} keys %seen_times_6;
10478 # print STDERR Dumper \@sorted_trait_names_6;
10479
10480 my $trait_name_encoded_6 = 1;
10481 foreach my $trait_name (@sorted_trait_names_6) {
10482 if (!exists($trait_name_encoder_6{$trait_name})) {
10483 my $trait_name_e = 't'.$trait_name_encoded_6;
10484 $trait_name_encoder_6{$trait_name} = $trait_name_e;
10485 $trait_name_encoder_rev_6{$trait_name_e} = $trait_name;
10486 $trait_name_encoded_6++;
10487 }
10488 }
10489
10490 foreach (@sorted_trait_names_6) {
10491 if ($_ < $time_min_6) {
10492 $time_min_6 = $_;
10493 }
10494 if ($_ >= $time_max_6) {
10495 $time_max_6 = $_;
10496 }
10497 }
10498 print STDERR Dumper [$time_min_6, $time_max_6];
10499
10500 while ( my ($trait_name, $time_term) = each %seen_trait_names_6) {
10501 push @{$trait_composing_info_6{$trait_name}}, $time_term;
10502 }
10503
10504 @unique_plot_names = sort keys %seen_plot_names;
10505
10506 open(my $F_prep, ">", $stats_prep_tempfile) || die "Can't open file ".$stats_prep_tempfile;
10507 print $F_prep "accession_id,accession_id_factor,plot_id,plot_id_factor,replicate,time,replicate_time,ind_replicate\n";
10508 foreach my $p (@unique_plot_names) {
10509 my $replicate = $stock_name_row_col{$p}->{rep};
10510 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
10511 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
10512 foreach my $t (@sorted_trait_names_6) {
10513 print $F_prep "$germplasm_stock_id,,$obsunit_stock_id,,$replicate,$t,$replicate"."_"."$t,$germplasm_stock_id"."_"."$replicate\n";
10514 }
10515 }
10516 close($F_prep);
10517
10518 my $cmd_factor = 'R -e "library(data.table);
10519 mat <- fread(\''.$stats_prep_tempfile.'\', header=TRUE, sep=\',\');
10520 mat\$replicate_time <- as.numeric(as.factor(mat\$replicate_time));
10521 mat\$ind_replicate <- as.numeric(as.factor(mat\$ind_replicate));
10522 mat\$accession_id_factor <- as.numeric(as.factor(mat\$accession_id));
10523 mat\$plot_id_factor <- as.numeric(as.factor(mat\$plot_id));
10524 write.table(mat, file=\''.$stats_prep_factor_tempfile.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');"';
10525 my $status_factor = system($cmd_factor);
10526
10527 open(my $fh_factor, '<', $stats_prep_factor_tempfile) or die "Could not open file '$stats_prep_factor_tempfile' $!";
10528 print STDERR "Opened $stats_prep_factor_tempfile\n";
10529 my $header_d = <$fh_factor>;
10530
10531 my $line_factor_count = 0;
10532 while (my $row = <$fh_factor>) {
10533 my @columns;
10534 if ($csv->parse($row)) {
10535 @columns = $csv->fields();
10536 }
10537 my $accession_id = $columns[0];
10538 my $accession_id_factor = $columns[1];
10539 my $plot_id = $columns[2];
10540 my $plot_id_factor = $columns[3];
10541 my $rep = $columns[4];
10542 my $time = $columns[5];
10543 my $rep_time = $columns[6];
10544 my $ind_rep = $columns[7];
10545 $stock_row_col{$plot_id}->{plot_id_factor} = $plot_id_factor;
10546 $stock_name_row_col{$plot_id_map{$plot_id}}->{plot_id_factor} = $plot_id_factor;
10547 $plot_rep_time_factor_map{$plot_id}->{$rep}->{$time} = $rep_time;
10548 $seen_rep_times{$rep_time}++;
10549 $seen_ind_reps{$plot_id_factor}++;
10550 $accession_id_factor_map{$accession_id} = $accession_id_factor;
10551 $accession_id_factor_map_reverse{$accession_id_factor} = $stock_info_6{$accession_id}->{uniquename};
10552 $plot_id_factor_map_reverse{$plot_id_factor} = $seen_plots{$plot_id};
10553 $plot_id_count_map_reverse{$line_factor_count} = $seen_plots{$plot_id};
10554 $time_count_map_reverse{$line_factor_count} = $time;
10555 $line_factor_count++;
10556 }
10557 close($fh_factor);
10558 @rep_time_factors = sort keys %seen_rep_times;
10559 @ind_rep_factors = sort keys %seen_ind_reps;
10560
10561 foreach my $p (@unique_plot_names) {
10562 my $obsunit_stock_id = $stock_name_row_col{$p}->{obsunit_stock_id};
10563 if ($fixed_effect_type eq 'fixed_effect_trait') {
10564 $stock_name_row_col{$p}->{rep} = defined($fixed_effect_trait_data->{$obsunit_stock_id}) ? $fixed_effect_trait_data->{$obsunit_stock_id} : 0;
10565 }
10566 my $row_number = $stock_name_row_col{$p}->{row_number};
10567 my $col_number = $stock_name_row_col{$p}->{col_number};
10568 my $replicate = $stock_name_row_col{$p}->{rep};
10569 my $block = $stock_name_row_col{$p}->{block};
10570 my $germplasm_stock_id = $stock_name_row_col{$p}->{germplasm_stock_id};
10571 my $germplasm_name = $stock_name_row_col{$p}->{germplasm_name};
10572
10573 my $current_trait_index = 0;
10574 my @row = (
10575 $germplasm_stock_id,
10576 $obsunit_stock_id,
10577 $replicate,
10578 $row_number,
10579 $col_number,
10580 $accession_id_factor_map{$germplasm_stock_id},
10581 $stock_row_col{$obsunit_stock_id}->{plot_id_factor}
10582 );
10583
10584 foreach my $t (@sorted_trait_names_6) {
10585 if (defined($phenotype_data_original_6{$p}->{$t})) {
10586 push @row, $phenotype_data_original_6{$p}->{$t} + 0;
10587 } else {
10588 print STDERR $p." : $t : $germplasm_name : NA \n";
10589 push @row, '';
10590 }
10591
10592 $current_trait_index++;
10593 }
10594 push @data_matrix_original_6, \@row;
10595 }
10596
10597 @phenotype_header_6 = ("id", "plot_id", "replicate", "rowNumber", "colNumber", "id_factor", "plot_id_factor");
10598 foreach (@sorted_trait_names_6) {
10599 push @phenotype_header_6, "t$_";
10600 }
10601 $header_string_6 = join ',', @phenotype_header_6;
10602
10603 open($F, ">", $stats_tempfile_2) || die "Can't open file ".$stats_tempfile_2;
10604 print $F $header_string_6."\n";
10605 foreach (@data_matrix_original_6) {
10606 my $line = join ',', @$_;
10607 print $F "$line\n";
10608 }
10609 close($F);
10610 };
10611
10612 @seen_rows_array = keys %seen_rows;
10613 @seen_cols_array = keys %seen_cols;
10614 $row_stat = Statistics::Descriptive::Full->new();
10615 $row_stat->add_data(@seen_rows_array);
10616 $mean_row = $row_stat->mean();
10617 $sig_row = $row_stat->variance();
10618 $col_stat = Statistics::Descriptive::Full->new();
10619 $col_stat->add_data(@seen_cols_array);
10620 $mean_col = $col_stat->mean();
10621 $sig_col = $col_stat->variance();
10622
10623 print STDERR "PREPARE RELATIONSHIP MATRIX\n";
10624 eval {
10625 my %seen_accession_stock_ids;
10626 foreach my $trial_id (@$field_trial_id_list) {
10627 my $trial = CXGN::Trial->new({ bcs_schema => $schema, trial_id => $trial_id });
10628 my $accessions = $trial->get_accessions();
10629 foreach (@$accessions) {
10630 $seen_accession_stock_ids{$_->{stock_id}}++;
10631 }
10632 }
10633 my @accession_ids = keys %seen_accession_stock_ids;
10634
10635 if ($compute_relationship_matrix_from_htp_phenotypes eq 'genotypes') {
10636
10637 if ($include_pedgiree_info_if_compute_from_parents) {
10638 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
10639 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
10640 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
10641 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
10642 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
10643 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
10644 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
10645 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
10646
10647 if (!$protocol_id) {
10648 $protocol_id = undef;
10649 }
10650
10651 my $pedigree_arm = CXGN::Pedigree::ARM->new({
10652 bcs_schema=>$schema,
10653 arm_temp_file=>$arm_tempfile,
10654 people_schema=>$people_schema,
10655 accession_id_list=>\@accession_ids,
10656 # plot_id_list=>\@plot_id_list,
10657 cache_root=>$c->config->{cache_file_path},
10658 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
10659 });
10660 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
10661 $shared_cluster_dir_config,
10662 $c->config->{backend},
10663 $c->config->{cluster_host},
10664 $c->config->{'web_cluster_queue'},
10665 $c->config->{basepath}
10666 );
10667 # print STDERR Dumper $parent_hash;
10668
10669 my $female_geno = CXGN::Genotype::GRM->new({
10670 bcs_schema=>$schema,
10671 grm_temp_file=>$grm1_tempfile,
10672 people_schema=>$people_schema,
10673 cache_root=>$c->config->{cache_file_path},
10674 accession_id_list=>$female_stock_ids,
10675 protocol_id=>$protocol_id,
10676 get_grm_for_parental_accessions=>0,
10677 download_format=>'three_column_reciprocal'
10678 # minor_allele_frequency=>$minor_allele_frequency,
10679 # marker_filter=>$marker_filter,
10680 # individuals_filter=>$individuals_filter
10681 });
10682 my $female_grm_data = $female_geno->download_grm(
10683 'data',
10684 $shared_cluster_dir_config,
10685 $c->config->{backend},
10686 $c->config->{cluster_host},
10687 $c->config->{'web_cluster_queue'},
10688 $c->config->{basepath}
10689 );
10690 my @fl = split '\n', $female_grm_data;
10691 my %female_parent_grm;
10692 foreach (@fl) {
10693 my @l = split '\t', $_;
10694 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
10695 }
10696 # print STDERR Dumper \%female_parent_grm;
10697
10698 my $male_geno = CXGN::Genotype::GRM->new({
10699 bcs_schema=>$schema,
10700 grm_temp_file=>$grm1_tempfile,
10701 people_schema=>$people_schema,
10702 cache_root=>$c->config->{cache_file_path},
10703 accession_id_list=>$male_stock_ids,
10704 protocol_id=>$protocol_id,
10705 get_grm_for_parental_accessions=>0,
10706 download_format=>'three_column_reciprocal'
10707 # minor_allele_frequency=>$minor_allele_frequency,
10708 # marker_filter=>$marker_filter,
10709 # individuals_filter=>$individuals_filter
10710 });
10711 my $male_grm_data = $male_geno->download_grm(
10712 'data',
10713 $shared_cluster_dir_config,
10714 $c->config->{backend},
10715 $c->config->{cluster_host},
10716 $c->config->{'web_cluster_queue'},
10717 $c->config->{basepath}
10718 );
10719 my @ml = split '\n', $male_grm_data;
10720 my %male_parent_grm;
10721 foreach (@ml) {
10722 my @l = split '\t', $_;
10723 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
10724 }
10725 # print STDERR Dumper \%male_parent_grm;
10726
10727 my %rel_result_hash;
10728 foreach my $a1 (@accession_ids) {
10729 foreach my $a2 (@accession_ids) {
10730 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
10731 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
10732 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
10733 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
10734
10735 my $female_rel = 0;
10736 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
10737 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
10738 }
10739 elsif ($female_parent1 && $female_parent2 && $female_parent1 == $female_parent2) {
10740 $female_rel = 1;
10741 }
10742 elsif ($a1 == $a2) {
10743 $female_rel = 1;
10744 }
10745
10746 my $male_rel = 0;
10747 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
10748 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
10749 }
10750 elsif ($male_parent1 && $male_parent2 && $male_parent1 == $male_parent2) {
10751 $male_rel = 1;
10752 }
10753 elsif ($a1 == $a2) {
10754 $male_rel = 1;
10755 }
10756 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
10757
10758 my $rel = 0.5*($female_rel + $male_rel);
10759 $rel_result_hash{$a1}->{$a2} = $rel;
10760 }
10761 }
10762 # print STDERR Dumper \%rel_result_hash;
10763
10764 my $data = '';
10765 my %result_hash;
10766 foreach my $s (sort @accession_ids) {
10767 foreach my $c (sort @accession_ids) {
10768 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
10769 my $val = $rel_result_hash{$s}->{$c};
10770 if (defined $val and length $val) {
10771 $result_hash{$s}->{$c} = $val;
10772 $data .= "S$s\tS$c\t$val\n";
10773 }
10774 }
10775 }
10776 }
10777
10778 # print STDERR Dumper $data;
10779 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
10780 print $F2 $data;
10781 close($F2);
10782
10783 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
10784 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
10785 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
10786 A_1 <- A_wide[,-1];
10787 A_1[is.na(A_1)] <- 0;
10788 A <- A_1 + t(A_1);
10789 diag(A) <- diag(as.matrix(A_1));
10790 E = eigen(A);
10791 ev = E\$values;
10792 U = E\$vectors;
10793 no = dim(A)[1];
10794 nev = which(ev < 0);
10795 wr = 0;
10796 k=length(nev);
10797 if(k > 0){
10798 p = ev[no - k];
10799 B = sum(ev[nev])*2.0;
10800 wr = (B*B*100.0)+1;
10801 val = ev[nev];
10802 ev[nev] = p*(B-val)*(B-val)/wr;
10803 A = U%*%diag(ev)%*%t(U);
10804 }
10805 ';
10806 if ($return_inverse_matrix) {
10807 $cmd .= 'A <- solve(A);
10808 ';
10809 }
10810 $cmd .= 'A <- as.data.frame(A);
10811 colnames(A) <- A_wide[,1];
10812 A\$stock_id <- A_wide[,1];
10813 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
10814 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
10815 A_threecol\$variable <- substring(A_threecol\$variable, 2);
10816 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
10817 print STDERR $cmd."\n";
10818 my $status = system($cmd);
10819
10820 my %rel_pos_def_result_hash;
10821 open(my $F3, '<', $grm_out_tempfile)
10822 or die "Could not open file '$grm_out_tempfile' $!";
10823
10824 print STDERR "Opened $grm_out_tempfile\n";
10825
10826 while (my $row = <$F3>) {
10827 my @columns;
10828 if ($csv->parse($row)) {
10829 @columns = $csv->fields();
10830 }
10831 my $stock_id1 = $columns[0];
10832 my $stock_id2 = $columns[1];
10833 my $val = $columns[2];
10834 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
10835 }
10836 close($F3);
10837
10838 my $data_pos_def = '';
10839 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_multivariate_spatial_genetic_blups') {
10840 my %result_hash;
10841 foreach my $s (sort @accession_ids) {
10842 foreach my $c (sort @accession_ids) {
10843 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
10844 my $val = $rel_pos_def_result_hash{$s}->{$c};
10845 if (defined $val and length $val) {
10846 $result_hash{$s}->{$c} = $val;
10847 $data_pos_def .= "$s\t$c\t$val\n";
10848 }
10849 }
10850 }
10851 }
10852 }
10853 else {
10854 my %result_hash;
10855 foreach my $s (sort @accession_ids) {
10856 foreach my $c (sort @accession_ids) {
10857 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
10858 my $val = $rel_pos_def_result_hash{$s}->{$c};
10859 if (defined $val and length $val) {
10860 $result_hash{$s}->{$c} = $val;
10861 $result_hash{$c}->{$s} = $val;
10862 $data_pos_def .= "S$s\tS$c\t$val\n";
10863 if ($s != $c) {
10864 $data_pos_def .= "S$c\tS$s\t$val\n";
10865 }
10866 }
10867 }
10868 }
10869 }
10870 }
10871
10872 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
10873 print $F4 $data_pos_def;
10874 close($F4);
10875
10876 $grm_file = $grm_out_posdef_tempfile;
10877 }
10878 elsif ($use_parental_grms_if_compute_from_parents) {
10879 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
10880 my $tmp_arm_dir = $shared_cluster_dir_config."/tmp_download_arm";
10881 mkdir $tmp_arm_dir if ! -d $tmp_arm_dir;
10882 my ($arm_tempfile_fh, $arm_tempfile) = tempfile("drone_stats_download_arm_XXXXX", DIR=> $tmp_arm_dir);
10883 my ($grm1_tempfile_fh, $grm1_tempfile) = tempfile("drone_stats_download_grm1_XXXXX", DIR=> $tmp_arm_dir);
10884 my ($grm_out_temp_tempfile_fh, $grm_out_temp_tempfile) = tempfile("drone_stats_download_grm_temp_out_XXXXX", DIR=> $tmp_arm_dir);
10885 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
10886 my ($grm_out_posdef_tempfile_fh, $grm_out_posdef_tempfile) = tempfile("drone_stats_download_grm_out_XXXXX", DIR=> $tmp_arm_dir);
10887
10888 if (!$protocol_id) {
10889 $protocol_id = undef;
10890 }
10891
10892 my $pedigree_arm = CXGN::Pedigree::ARM->new({
10893 bcs_schema=>$schema,
10894 arm_temp_file=>$arm_tempfile,
10895 people_schema=>$people_schema,
10896 accession_id_list=>\@accession_ids,
10897 # plot_id_list=>\@plot_id_list,
10898 cache_root=>$c->config->{cache_file_path},
10899 download_format=>'matrix', #either 'matrix', 'three_column', or 'heatmap'
10900 });
10901 my ($parent_hash, $stock_ids, $all_accession_stock_ids, $female_stock_ids, $male_stock_ids) = $pedigree_arm->get_arm(
10902 $shared_cluster_dir_config,
10903 $c->config->{backend},
10904 $c->config->{cluster_host},
10905 $c->config->{'web_cluster_queue'},
10906 $c->config->{basepath}
10907 );
10908 # print STDERR Dumper $parent_hash;
10909
10910 my $female_geno = CXGN::Genotype::GRM->new({
10911 bcs_schema=>$schema,
10912 grm_temp_file=>$grm1_tempfile,
10913 people_schema=>$people_schema,
10914 cache_root=>$c->config->{cache_file_path},
10915 accession_id_list=>$female_stock_ids,
10916 protocol_id=>$protocol_id,
10917 get_grm_for_parental_accessions=>0,
10918 download_format=>'three_column_reciprocal'
10919 # minor_allele_frequency=>$minor_allele_frequency,
10920 # marker_filter=>$marker_filter,
10921 # individuals_filter=>$individuals_filter
10922 });
10923 my $female_grm_data = $female_geno->download_grm(
10924 'data',
10925 $shared_cluster_dir_config,
10926 $c->config->{backend},
10927 $c->config->{cluster_host},
10928 $c->config->{'web_cluster_queue'},
10929 $c->config->{basepath}
10930 );
10931 my @fl = split '\n', $female_grm_data;
10932 my %female_parent_grm;
10933 foreach (@fl) {
10934 my @l = split '\t', $_;
10935 $female_parent_grm{$l[0]}->{$l[1]} = $l[2];
10936 }
10937 # print STDERR Dumper \%female_parent_grm;
10938
10939 my $male_geno = CXGN::Genotype::GRM->new({
10940 bcs_schema=>$schema,
10941 grm_temp_file=>$grm1_tempfile,
10942 people_schema=>$people_schema,
10943 cache_root=>$c->config->{cache_file_path},
10944 accession_id_list=>$male_stock_ids,
10945 protocol_id=>$protocol_id,
10946 get_grm_for_parental_accessions=>0,
10947 download_format=>'three_column_reciprocal'
10948 # minor_allele_frequency=>$minor_allele_frequency,
10949 # marker_filter=>$marker_filter,
10950 # individuals_filter=>$individuals_filter
10951 });
10952 my $male_grm_data = $male_geno->download_grm(
10953 'data',
10954 $shared_cluster_dir_config,
10955 $c->config->{backend},
10956 $c->config->{cluster_host},
10957 $c->config->{'web_cluster_queue'},
10958 $c->config->{basepath}
10959 );
10960 my @ml = split '\n', $male_grm_data;
10961 my %male_parent_grm;
10962 foreach (@ml) {
10963 my @l = split '\t', $_;
10964 $male_parent_grm{$l[0]}->{$l[1]} = $l[2];
10965 }
10966 # print STDERR Dumper \%male_parent_grm;
10967
10968 my %rel_result_hash;
10969 foreach my $a1 (@accession_ids) {
10970 foreach my $a2 (@accession_ids) {
10971 my $female_parent1 = $parent_hash->{$a1}->{female_stock_id};
10972 my $male_parent1 = $parent_hash->{$a1}->{male_stock_id};
10973 my $female_parent2 = $parent_hash->{$a2}->{female_stock_id};
10974 my $male_parent2 = $parent_hash->{$a2}->{male_stock_id};
10975
10976 my $female_rel = 0;
10977 if ($female_parent1 && $female_parent2 && $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2}) {
10978 $female_rel = $female_parent_grm{'S'.$female_parent1}->{'S'.$female_parent2};
10979 }
10980 elsif ($a1 == $a2) {
10981 $female_rel = 1;
10982 }
10983
10984 my $male_rel = 0;
10985 if ($male_parent1 && $male_parent2 && $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2}) {
10986 $male_rel = $male_parent_grm{'S'.$male_parent1}->{'S'.$male_parent2};
10987 }
10988 elsif ($a1 == $a2) {
10989 $male_rel = 1;
10990 }
10991 # print STDERR "$a1 $a2 $female_rel $male_rel\n";
10992
10993 my $rel = 0.5*($female_rel + $male_rel);
10994 $rel_result_hash{$a1}->{$a2} = $rel;
10995 }
10996 }
10997 # print STDERR Dumper \%rel_result_hash;
10998
10999 my $data = '';
11000 my %result_hash;
11001 foreach my $s (sort @accession_ids) {
11002 foreach my $c (sort @accession_ids) {
11003 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
11004 my $val = $rel_result_hash{$s}->{$c};
11005 if (defined $val and length $val) {
11006 $result_hash{$s}->{$c} = $val;
11007 $data .= "S$s\tS$c\t$val\n";
11008 }
11009 }
11010 }
11011 }
11012
11013 # print STDERR Dumper $data;
11014 open(my $F2, ">", $grm_out_temp_tempfile) || die "Can't open file ".$grm_out_temp_tempfile;
11015 print $F2 $data;
11016 close($F2);
11017
11018 my $cmd = 'R -e "library(data.table); library(scales); library(tidyr); library(reshape2);
11019 three_col <- fread(\''.$grm_out_temp_tempfile.'\', header=FALSE, sep=\'\t\');
11020 A_wide <- dcast(three_col, V1~V2, value.var=\'V3\');
11021 A_1 <- A_wide[,-1];
11022 A_1[is.na(A_1)] <- 0;
11023 A <- A_1 + t(A_1);
11024 diag(A) <- diag(as.matrix(A_1));
11025 E = eigen(A);
11026 ev = E\$values;
11027 U = E\$vectors;
11028 no = dim(A)[1];
11029 nev = which(ev < 0);
11030 wr = 0;
11031 k=length(nev);
11032 if(k > 0){
11033 p = ev[no - k];
11034 B = sum(ev[nev])*2.0;
11035 wr = (B*B*100.0)+1;
11036 val = ev[nev];
11037 ev[nev] = p*(B-val)*(B-val)/wr;
11038 A = U%*%diag(ev)%*%t(U);
11039 }
11040 ';
11041 if ($return_inverse_matrix) {
11042 $cmd .= 'A <- solve(A);
11043 ';
11044 }
11045 $cmd .= 'A <- as.data.frame(A);
11046 colnames(A) <- A_wide[,1];
11047 A\$stock_id <- A_wide[,1];
11048 A_threecol <- melt(A, id.vars = c(\'stock_id\'), measure.vars = A_wide[,1]);
11049 A_threecol\$stock_id <- substring(A_threecol\$stock_id, 2);
11050 A_threecol\$variable <- substring(A_threecol\$variable, 2);
11051 write.table(data.frame(variable = A_threecol\$variable, stock_id = A_threecol\$stock_id, value = A_threecol\$value), file=\''.$grm_out_tempfile.'\', row.names=FALSE, col.names=FALSE, sep=\'\t\');"';
11052 print STDERR $cmd."\n";
11053 my $status = system($cmd);
11054
11055 my %rel_pos_def_result_hash;
11056 open(my $F3, '<', $grm_out_tempfile)
11057 or die "Could not open file '$grm_out_tempfile' $!";
11058
11059 print STDERR "Opened $grm_out_tempfile\n";
11060
11061 while (my $row = <$F3>) {
11062 my @columns;
11063 if ($csv->parse($row)) {
11064 @columns = $csv->fields();
11065 }
11066 my $stock_id1 = $columns[0];
11067 my $stock_id2 = $columns[1];
11068 my $val = $columns[2];
11069 $rel_pos_def_result_hash{$stock_id1}->{$stock_id2} = $val;
11070 }
11071 close($F3);
11072
11073 my $data_pos_def = '';
11074 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_multivariate_spatial_genetic_blups') {
11075 my %result_hash;
11076 foreach my $s (sort @accession_ids) {
11077 foreach my $c (sort @accession_ids) {
11078 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
11079 my $val = $rel_pos_def_result_hash{$s}->{$c};
11080 if (defined $val and length $val) {
11081 $result_hash{$s}->{$c} = $val;
11082 $data_pos_def .= "$s\t$c\t$val\n";
11083 }
11084 }
11085 }
11086 }
11087 }
11088 else {
11089 my %result_hash;
11090 foreach my $s (sort @accession_ids) {
11091 foreach my $c (sort @accession_ids) {
11092 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
11093 my $val = $rel_pos_def_result_hash{$s}->{$c};
11094 if (defined $val and length $val) {
11095 $result_hash{$s}->{$c} = $val;
11096 $result_hash{$c}->{$s} = $val;
11097 $data_pos_def .= "S$s\tS$c\t$val\n";
11098 if ($s != $c) {
11099 $data_pos_def .= "S$c\tS$s\t$val\n";
11100 }
11101 }
11102 }
11103 }
11104 }
11105 }
11106
11107 open(my $F4, ">", $grm_out_posdef_tempfile) || die "Can't open file ".$grm_out_posdef_tempfile;
11108 print $F4 $data_pos_def;
11109 close($F4);
11110
11111 $grm_file = $grm_out_posdef_tempfile;
11112 }
11113 else {
11114 my $shared_cluster_dir_config = $c->config->{cluster_shared_tempdir};
11115 my $tmp_grm_dir = $shared_cluster_dir_config."/tmp_genotype_download_grm";
11116 mkdir $tmp_grm_dir if ! -d $tmp_grm_dir;
11117 my ($grm_tempfile_fh, $grm_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
11118 my ($grm_out_tempfile_fh, $grm_out_tempfile) = tempfile("drone_stats_download_grm_XXXXX", DIR=> $tmp_grm_dir);
11119
11120 if (!$protocol_id) {
11121 $protocol_id = undef;
11122 }
11123
11124 my $grm_search_params = {
11125 bcs_schema=>$schema,
11126 grm_temp_file=>$grm_tempfile,
11127 people_schema=>$people_schema,
11128 cache_root=>$c->config->{cache_file_path},
11129 accession_id_list=>\@accession_ids,
11130 protocol_id=>$protocol_id,
11131 get_grm_for_parental_accessions=>$compute_from_parents,
11132 return_inverse=>$return_inverse_matrix,
11133 ensure_positive_definite=>$ensure_positive_definite
11134 # minor_allele_frequency=>$minor_allele_frequency,
11135 # marker_filter=>$marker_filter,
11136 # individuals_filter=>$individuals_filter
11137 };
11138
11139 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_multivariate_spatial_genetic_blups') {
11140 $grm_search_params->{download_format} = 'three_column_stock_id_integer';
11141 }
11142 else {
11143 $grm_search_params->{download_format} = 'three_column_reciprocal';
11144 }
11145
11146 my $geno = CXGN::Genotype::GRM->new($grm_search_params);
11147 my $grm_data = $geno->download_grm(
11148 'data',
11149 $shared_cluster_dir_config,
11150 $c->config->{backend},
11151 $c->config->{cluster_host},
11152 $c->config->{'web_cluster_queue'},
11153 $c->config->{basepath}
11154 );
11155
11156 open(my $F2, ">", $grm_out_tempfile) || die "Can't open file ".$grm_out_tempfile;
11157 print $F2 $grm_data;
11158 close($F2);
11159 $grm_file = $grm_out_tempfile;
11160 }
11161
11162 }
11163 elsif ($compute_relationship_matrix_from_htp_phenotypes eq 'htp_phenotypes') {
11164
11165 my $phenotypes_search = CXGN::Phenotypes::SearchFactory->instantiate(
11166 'MaterializedViewTable',
11167 {
11168 bcs_schema=>$schema,
11169 data_level=>'plot',
11170 trial_list=>$field_trial_id_list,
11171 include_timestamp=>0,
11172 exclude_phenotype_outlier=>0
11173 }
11174 );
11175 my ($data, $unique_traits) = $phenotypes_search->search();
11176
11177 if (scalar(@$data) == 0) {
11178 $c->stash->{rest} = { error => "There are no phenotypes for the trial you have selected!"};
11179 return;
11180 }
11181
11182 my $q_time = "SELECT t.cvterm_id FROM cvterm as t JOIN cv ON(t.cv_id=cv.cv_id) WHERE t.name=? and cv.name=?;";
11183 my $h_time = $schema->storage->dbh()->prepare($q_time);
11184
11185 my %seen_plot_names_htp_rel;
11186 my %phenotype_data_htp_rel;
11187 my %seen_times_htp_rel;
11188 foreach my $obs_unit (@$data){
11189 my $germplasm_name = $obs_unit->{germplasm_uniquename};
11190 my $germplasm_stock_id = $obs_unit->{germplasm_stock_id};
11191 my $row_number = $obs_unit->{obsunit_row_number} || '';
11192 my $col_number = $obs_unit->{obsunit_col_number} || '';
11193 my $rep = $obs_unit->{obsunit_rep};
11194 my $block = $obs_unit->{obsunit_block};
11195 $seen_plot_names_htp_rel{$obs_unit->{observationunit_uniquename}} = $obs_unit;
11196 my $observations = $obs_unit->{observations};
11197 foreach (@$observations){
11198 if ($_->{associated_image_project_time_json}) {
11199 my $related_time_terms_json = decode_json $_->{associated_image_project_time_json};
11200
11201 my $time_days_cvterm = $related_time_terms_json->{day};
11202 my $time_days_term_string = $time_days_cvterm;
11203 my $time_days = (split '\|', $time_days_cvterm)[0];
11204 my $time_days_value = (split ' ', $time_days)[1];
11205
11206 my $time_gdd_value = $related_time_terms_json->{gdd_average_temp} + 0;
11207 my $gdd_term_string = "GDD $time_gdd_value";
11208 $h_time->execute($gdd_term_string, 'cxgn_time_ontology');
11209 my ($gdd_cvterm_id) = $h_time->fetchrow_array();
11210 if (!$gdd_cvterm_id) {
11211 my $new_gdd_term = $schema->resultset("Cv::Cvterm")->create_with({
11212 name => $gdd_term_string,
11213 cv => 'cxgn_time_ontology'
11214 });
11215 $gdd_cvterm_id = $new_gdd_term->cvterm_id();
11216 }
11217 my $time_gdd_term_string = SGN::Model::Cvterm::get_trait_from_cvterm_id($schema, $gdd_cvterm_id, 'extended');
11218
11219 $phenotype_data_htp_rel{$obs_unit->{observationunit_uniquename}}->{$_->{trait_name}} = $_->{value};
11220 $seen_times_htp_rel{$_->{trait_name}} = [$time_days_value, $time_days_term_string, $time_gdd_value, $time_gdd_term_string];
11221 }
11222 }
11223 }
11224
11225 my @allowed_standard_htp_values = ('Nonzero Pixel Count', 'Total Pixel Sum', 'Mean Pixel Value', 'Harmonic Mean Pixel Value', 'Median Pixel Value', 'Pixel Variance', 'Pixel Standard Deviation', 'Pixel Population Standard Deviation', 'Minimum Pixel Value', 'Maximum Pixel Value', 'Minority Pixel Value', 'Minority Pixel Count', 'Majority Pixel Value', 'Majority Pixel Count', 'Pixel Group Count');
11226 my %filtered_seen_times_htp_rel;
11227 while (my ($t, $time) = each %seen_times_htp_rel) {
11228 my $allowed = 0;
11229 foreach (@allowed_standard_htp_values) {
11230 if (index($t, $_) != -1) {
11231 $allowed = 1;
11232 last;
11233 }
11234 }
11235 if ($allowed) {
11236 $filtered_seen_times_htp_rel{$t} = $time;
11237 }
11238 }
11239
11240 my @seen_plot_names_htp_rel_sorted = sort keys %seen_plot_names_htp_rel;
11241 my @filtered_seen_times_htp_rel_sorted = sort keys %filtered_seen_times_htp_rel;
11242
11243 my @header_htp = ('plot_id', 'plot_name', 'accession_id', 'accession_name', 'rep', 'block');
11244
11245 my %trait_name_encoder_htp;
11246 my %trait_name_encoder_rev_htp;
11247 my $trait_name_encoded_htp = 1;
11248 my @header_traits_htp;
11249 foreach my $trait_name (@filtered_seen_times_htp_rel_sorted) {
11250 if (!exists($trait_name_encoder_htp{$trait_name})) {
11251 my $trait_name_e = 't'.$trait_name_encoded_htp;
11252 $trait_name_encoder_htp{$trait_name} = $trait_name_e;
11253 $trait_name_encoder_rev_htp{$trait_name_e} = $trait_name;
11254 push @header_traits_htp, $trait_name_e;
11255 $trait_name_encoded_htp++;
11256 }
11257 }
11258
11259 my @htp_pheno_matrix;
11260 if ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'all') {
11261 push @header_htp, @header_traits_htp;
11262 push @htp_pheno_matrix, \@header_htp;
11263
11264 foreach my $p (@seen_plot_names_htp_rel_sorted) {
11265 my $obj = $seen_plot_names_htp_rel{$p};
11266 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
11267 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
11268 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
11269 push @row, $val;
11270 }
11271 push @htp_pheno_matrix, \@row;
11272 }
11273 }
11274 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'latest_trait') {
11275 my $max_day = 0;
11276 foreach (keys %seen_days_after_plantings) {
11277 if ($_ + 0 > $max_day) {
11278 $max_day = $_;
11279 }
11280 }
11281
11282 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
11283 my $day = $filtered_seen_times_htp_rel{$t}->[0];
11284 if ($day <= $max_day) {
11285 push @header_htp, $t;
11286 }
11287 }
11288 push @htp_pheno_matrix, \@header_htp;
11289
11290 foreach my $p (@seen_plot_names_htp_rel_sorted) {
11291 my $obj = $seen_plot_names_htp_rel{$p};
11292 my @row = ($obj->{observationunit_stock_id}, $obj->{observationunit_uniquename}, $obj->{germplasm_stock_id}, $obj->{germplasm_uniquename}, $obj->{obsunit_rep}, $obj->{obsunit_block});
11293 foreach my $t (@filtered_seen_times_htp_rel_sorted) {
11294 my $day = $filtered_seen_times_htp_rel{$t}->[0];
11295 if ($day <= $max_day) {
11296 my $val = $phenotype_data_htp_rel{$p}->{$t} + 0;
11297 push @row, $val;
11298 }
11299 }
11300 push @htp_pheno_matrix, \@row;
11301 }
11302 }
11303 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'vegetative') {
11304
11305 }
11306 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'reproductive') {
11307
11308 }
11309 elsif ($compute_relationship_matrix_from_htp_phenotypes_time_points eq 'mature') {
11310
11311 }
11312 else {
11313 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_time_points htp_pheno_rel_matrix_time_points is not valid!" };
11314 return;
11315 }
11316
11317 open(my $htp_pheno_f, ">", $stats_out_htp_rel_tempfile_input) || die "Can't open file ".$stats_out_htp_rel_tempfile_input;
11318 foreach (@htp_pheno_matrix) {
11319 my $line = join "\t", @$_;
11320 print $htp_pheno_f $line."\n";
11321 }
11322 close($htp_pheno_f);
11323
11324 my %rel_htp_result_hash;
11325 if ($compute_relationship_matrix_from_htp_phenotypes_type eq 'correlations') {
11326 my $htp_cmd = 'R -e "library(lme4); library(data.table);
11327 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
11328 mat_agg <- aggregate(mat[, 7:ncol(mat)], list(mat\$accession_id), mean);
11329 mat_pheno <- mat_agg[,2:ncol(mat_agg)];
11330 cor_mat <- cor(t(mat_pheno));
11331 rownames(cor_mat) <- mat_agg[,1];
11332 colnames(cor_mat) <- mat_agg[,1];
11333 range01 <- function(x){(x-min(x))/(max(x)-min(x))};
11334 cor_mat <- range01(cor_mat);
11335 ';
11336 if ($return_inverse_matrix) {
11337 $htp_cmd .= 'cor_mat <- solve(cor_mat);
11338 ';
11339 }
11340 $htp_cmd .= 'write.table(cor_mat, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
11341 print STDERR Dumper $htp_cmd;
11342 my $status = system($htp_cmd);
11343 }
11344 elsif ($compute_relationship_matrix_from_htp_phenotypes_type eq 'blues') {
11345 my $htp_cmd = 'R -e "library(lme4); library(data.table);
11346 mat <- fread(\''.$stats_out_htp_rel_tempfile_input.'\', header=TRUE, sep=\'\t\');
11347 blues <- data.frame(id = seq(1,length(unique(mat\$accession_id))));
11348 varlist <- names(mat)[7:ncol(mat)];
11349 blues.models <- lapply(varlist, function(x) {
11350 tryCatch(
11351 lmer(substitute(i ~ 1 + (1|accession_id), list(i = as.name(x))), data = mat, REML = FALSE, control = lmerControl(optimizer =\'Nelder_Mead\', boundary.tol='.$compute_relationship_matrix_from_htp_phenotypes_blues_inversion.' ) ), error=function(e) {}
11352 )
11353 });
11354 counter = 1;
11355 for (m in blues.models) {
11356 if (!is.null(m)) {
11357 blues\$accession_id <- row.names(ranef(m)\$accession_id);
11358 blues[,ncol(blues) + 1] <- ranef(m)\$accession_id\$\`(Intercept)\`;
11359 colnames(blues)[ncol(blues)] <- varlist[counter];
11360 }
11361 counter = counter + 1;
11362 }
11363 blues_vals <- as.matrix(blues[,3:ncol(blues)]);
11364 blues_vals <- apply(blues_vals, 2, function(y) (y - mean(y)) / sd(y) ^ as.logical(sd(y)));
11365 rel <- (1/ncol(blues_vals)) * (blues_vals %*% t(blues_vals));
11366 ';
11367 if ($return_inverse_matrix) {
11368 $htp_cmd .= 'rel <- solve(rel);
11369 ';
11370 }
11371 $htp_cmd .= 'rownames(rel) <- blues[,2];
11372 colnames(rel) <- blues[,2];
11373 write.table(rel, file=\''.$stats_out_htp_rel_tempfile.'\', row.names=TRUE, col.names=TRUE, sep=\'\t\');"';
11374 print STDERR Dumper $htp_cmd;
11375 my $status = system($htp_cmd);
11376 }
11377 else {
11378 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes_type htp_pheno_rel_matrix_type is not valid!" };
11379 return;
11380 }
11381
11382 open(my $htp_rel_res, '<', $stats_out_htp_rel_tempfile)
11383 or die "Could not open file '$stats_out_htp_rel_tempfile' $!";
11384
11385 print STDERR "Opened $stats_out_htp_rel_tempfile\n";
11386 my $header_row = <$htp_rel_res>;
11387 my @header;
11388 if ($csv->parse($header_row)) {
11389 @header = $csv->fields();
11390 }
11391
11392 while (my $row = <$htp_rel_res>) {
11393 my @columns;
11394 if ($csv->parse($row)) {
11395 @columns = $csv->fields();
11396 }
11397 my $stock_id1 = $columns[0];
11398 my $counter = 1;
11399 foreach my $stock_id2 (@header) {
11400 my $val = $columns[$counter];
11401 $rel_htp_result_hash{$stock_id1}->{$stock_id2} = $val;
11402 $counter++;
11403 }
11404 }
11405 close($htp_rel_res);
11406
11407 my $data_rel_htp = '';
11408 my %result_hash;
11409 if ($statistics_select eq 'blupf90_grm_random_regression_gdd_blups' || $statistics_select eq 'blupf90_grm_random_regression_dap_blups' || $statistics_select eq 'airemlf90_grm_random_regression_gdd_blups' || $statistics_select eq 'airemlf90_grm_random_regression_dap_blups' || $statistics_select eq 'asreml_grm_univariate_spatial_genetic_blups' || $statistics_select eq 'asreml_grm_multivariate_spatial_genetic_blups') {
11410 foreach my $s (sort @accession_ids) {
11411 foreach my $c (sort @accession_ids) {
11412 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
11413 my $val = $rel_htp_result_hash{$s}->{$c};
11414 if (defined $val and length $val) {
11415 $result_hash{$s}->{$c} = $val;
11416 $data_rel_htp .= "$s\t$c\t$val\n";
11417 }
11418 }
11419 }
11420 }
11421 }
11422 else {
11423 foreach my $s (sort @accession_ids) {
11424 foreach my $c (sort @accession_ids) {
11425 if (!exists($result_hash{$s}->{$c}) && !exists($result_hash{$c}->{$s})) {
11426 my $val = $rel_htp_result_hash{$s}->{$c};
11427 if (defined $val and length $val) {
11428 $result_hash{$s}->{$c} = $val;
11429 $result_hash{$c}->{$s} = $val;
11430 $data_rel_htp .= "S$s\tS$c\t$val\n";
11431 if ($s != $c) {
11432 $data_rel_htp .= "S$c\tS$s\t$val\n";
11433 }
11434 }
11435 }
11436 }
11437 }
11438 }
11439
11440 open(my $htp_rel_out, ">", $stats_out_htp_rel_tempfile_out) || die "Can't open file ".$stats_out_htp_rel_tempfile_out;
11441 print $htp_rel_out $data_rel_htp;
11442 close($htp_rel_out);
11443
11444 $grm_file = $stats_out_htp_rel_tempfile_out;
11445 }
11446 else {
11447 $c->stash->{rest} = { error => "The value of $compute_relationship_matrix_from_htp_phenotypes is not valid!" };
11448 return;
11449 }
11450 };
11451
11452 my $result_6 = CXGN::DroneImagery::DroneImageryAnalyticsRunSimulation::perform_drone_imagery_analytics($schema, $a_env, $b_env, $ro_env, $row_ro_env, $env_variance_percent, $protocol_id, $statistics_select, $analytics_select, $tolparinv, $use_area_under_curve, $legendre_order_number, $permanent_environment_structure, \@legendre_coeff_exec, \%trait_name_encoder_6, \%trait_name_encoder_rev_6, \%stock_info_6, \%plot_id_map, \@sorted_trait_names_6, \%accession_id_factor_map, \@rep_time_factors, \@ind_rep_factors, \@unique_accession_names, \%plot_id_count_map_reverse, \@sorted_scaled_ln_times, \%time_count_map_reverse, \%accession_id_factor_map_reverse, \%seen_times, \%plot_id_factor_map_reverse, \%trait_to_time_map_6, \@unique_plot_names, \%stock_name_row_col, \%phenotype_data_original_6, \%plot_rep_time_factor_map, \%stock_row_col, \%stock_row_col_id, \%polynomial_map, \@plot_ids_ordered, $csv, $timestamp, $user_name, $stats_tempfile, $grm_file, $grm_rename_tempfile, $tmp_stats_dir, $stats_out_tempfile, $stats_out_tempfile_row, $stats_out_tempfile_col, $stats_out_tempfile_residual, $stats_out_tempfile_2dspl, $stats_prep2_tempfile, $stats_out_param_tempfile, $parameter_tempfile, $parameter_asreml_tempfile, $stats_tempfile_2, $permanent_environment_structure_tempfile, $permanent_environment_structure_env_tempfile, $permanent_environment_structure_env_tempfile2, $permanent_environment_structure_env_tempfile_mat, $sim_env_changing_mat_tempfile, $sim_env_changing_mat_full_tempfile, $yhat_residual_tempfile, $blupf90_solutions_tempfile, $coeff_genetic_tempfile, $coeff_pe_tempfile, $stats_out_tempfile_varcomp, $time_min, $time_max, $header_string_6, $env_sim_exec, $min_row, $max_row, $min_col, $max_col, $mean_row, $sig_row, $mean_col, $sig_col, $sim_env_change_over_time, $correlation_between_times, $field_trial_id_list, $simulated_environment_real_data_trait_id, $fixed_effect_type, $perform_cv);
11453 if (ref($result_6) eq 'HASH') {
11454 $c->stash->{rest} = $result_6;
11455 $c->detach();
11456 }
11457 my ($statistical_ontology_term_6, $analysis_model_training_data_file_type_6, $analysis_model_language_6, $sorted_residual_trait_names_array_6, $rr_unique_traits_hash_6, $rr_residual_unique_traits_hash_6, $statistics_cmd_6, $cmd_f90_6, $number_traits_6, $trait_to_time_map_hash_6,
11458
11459 $result_blup_data_original_6, $result_blup_data_delta_original_6, $result_blup_spatial_data_original_6, $result_blup_pe_data_original_6, $result_blup_pe_data_delta_original_6, $result_residual_data_original_6, $result_fitted_data_original_6, $fixed_effects_original_hash_6,
11460 $rr_genetic_coefficients_original_hash_6, $rr_temporal_coefficients_original_hash_6,
11461 $rr_coeff_genetic_covariance_original_array_6, $rr_coeff_env_covariance_original_array_6, $rr_coeff_genetic_correlation_original_array_6, $rr_coeff_env_correlation_original_array_6, $rr_residual_variance_original_6, $varcomp_original_array_6,
11462 $model_sum_square_residual_original_6, $genetic_effect_min_original_6, $genetic_effect_max_original_6, $env_effect_min_original_6, $env_effect_max_original_6, $genetic_effect_sum_square_original_6, $genetic_effect_sum_original_6, $env_effect_sum_square_original_6, $env_effect_sum_original_6, $residual_sum_square_original_6, $residual_sum_original_6, $result_cv_original_6, $result_cv_2_original_6,
11463
11464 $phenotype_data_altered_hash_6, $data_matrix_altered_array_6, $data_matrix_phenotypes_altered_array_6, $phenotype_min_altered_6, $phenotype_max_altered_6,
11465 $result_blup_data_altered_6, $result_blup_data_delta_altered_6, $result_blup_spatial_data_altered_6, $result_blup_pe_data_altered_6, $result_blup_pe_data_delta_altered_6, $result_residual_data_altered_6, $result_fitted_data_altered_6, $fixed_effects_altered_hash_6,
11466 $rr_genetic_coefficients_altered_hash_6, $rr_temporal_coefficients_altered_hash_6,
11467 $rr_coeff_genetic_covariance_altered_array_6, $rr_coeff_env_covariance_altered_array_6, $rr_coeff_genetic_correlation_altered_array_6, $rr_coeff_env_correlation_altered_array_6, $rr_residual_variance_altered_6, $varcomp_altered_array_6,
11468 $model_sum_square_residual_altered_6, $genetic_effect_min_altered_6, $genetic_effect_max_altered_6, $env_effect_min_altered_6, $env_effect_max_altered_6, $genetic_effect_sum_square_altered_6, $genetic_effect_sum_altered_6, $env_effect_sum_square_altered_6, $env_effect_sum_altered_6, $residual_sum_square_altered_6, $residual_sum_altered_6, $result_cv_altered_6, $result_cv_2_altered_6,
11469
11470 $phenotype_data_altered_env_hash_1_6, $data_matrix_altered_env_array_1_6, $data_matrix_phenotypes_altered_env_array_1_6, $phenotype_min_altered_env_1_6, $phenotype_max_altered_env_1_6, $env_sim_min_1_6, $env_sim_max_1_6, $sim_data_hash_1_6,
11471 $result_blup_data_altered_env_1_6, $result_blup_data_delta_altered_env_1_6, $result_blup_spatial_data_altered_env_1_6, $result_blup_pe_data_altered_env_1_6, $result_blup_pe_data_delta_altered_env_1_6, $result_residual_data_altered_env_1_6, $result_fitted_data_altered_env_1_6, $fixed_effects_altered_env_hash_1_6, $rr_genetic_coefficients_altered_env_hash_1_6, $rr_temporal_coefficients_altered_env_hash_1_6,
11472 $rr_coeff_genetic_covariance_altered_env_array_1_6, $rr_coeff_env_covariance_altered_env_array_1_6, $rr_coeff_genetic_correlation_altered_env_array_1_6, $rr_coeff_env_correlation_altered_env_array_1_6, $rr_residual_variance_altered_env_1_6, $varcomp_altered_array_env_1_6,
11473 $model_sum_square_residual_altered_env_1_6, $genetic_effect_min_altered_env_1_6, $genetic_effect_max_altered_env_1_6, $env_effect_min_altered_env_1_6, $env_effect_max_altered_env_1_6, $genetic_effect_sum_square_altered_env_1_6, $genetic_effect_sum_altered_env_1_6, $env_effect_sum_square_altered_env_1_6, $env_effect_sum_altered_env_1_6, $residual_sum_square_altered_env_1_6, $residual_sum_altered_env_1_6, $result_cv_altered_env_1_6, $result_cv_2_altered_env_1_6,
11474
11475 $phenotype_data_altered_env_hash_2_6, $data_matrix_altered_env_array_2_6, $data_matrix_phenotypes_altered_env_array_2_6, $phenotype_min_altered_env_2_6, $phenotype_max_altered_env_2_6, $env_sim_min_2_6, $env_sim_max_2_6, $sim_data_hash_2_6,
11476 $result_blup_data_altered_env_2_6, $result_blup_data_delta_altered_env_2_6, $result_blup_spatial_data_altered_env_2_6, $result_blup_pe_data_altered_env_2_6, $result_blup_pe_data_delta_altered_env_2_6, $result_residual_data_altered_env_2_6, $result_fitted_data_altered_env_2_6, $fixed_effects_altered_env_hash_2_6, $rr_genetic_coefficients_altered_env_hash_2_6, $rr_temporal_coefficients_altered_env_hash_2_6,
11477 $rr_coeff_genetic_covariance_altered_env_array_2_6, $rr_coeff_env_covariance_altered_env_array_2_6, $rr_coeff_genetic_correlation_altered_env_array_2_6, $rr_coeff_env_correlation_altered_env_array_2_6, $rr_residual_variance_altered_env_2_6, $varcomp_altered_array_env_2_6,
11478 $model_sum_square_residual_altered_env_2_6, $genetic_effect_min_altered_env_2_6, $genetic_effect_max_altered_env_2_6, $env_effect_min_altered_env_2_6, $env_effect_max_altered_env_2_6, $genetic_effect_sum_square_altered_env_2_6, $genetic_effect_sum_altered_env_2_6, $env_effect_sum_square_altered_env_2_6, $env_effect_sum_altered_env_2_6, $residual_sum_square_altered_env_2_6, $residual_sum_altered_env_2_6, $result_cv_altered_env_2_6, $result_cv_2_altered_env_2_6,
11479
11480 $phenotype_data_altered_env_hash_3_6, $data_matrix_altered_env_array_3_6, $data_matrix_phenotypes_altered_env_array_3_6, $phenotype_min_altered_env_3_6, $phenotype_max_altered_env_3_6, $env_sim_min_3_6, $env_sim_max_3_6, $sim_data_hash_3_6,
11481 $result_blup_data_altered_env_3_6, $result_blup_data_delta_altered_env_3_6, $result_blup_spatial_data_altered_env_3_6, $result_blup_pe_data_altered_env_3_6, $result_blup_pe_data_delta_altered_env_3_6, $result_residual_data_altered_env_3_6, $result_fitted_data_altered_env_3_6, $fixed_effects_altered_env_hash_3_6, $rr_genetic_coefficients_altered_env_hash_3_6, $rr_temporal_coefficients_altered_env_hash_3_6,
11482 $rr_coeff_genetic_covariance_altered_env_array_3_6, $rr_coeff_env_covariance_altered_env_array_3_6, $rr_coeff_genetic_correlation_altered_env_array_3_6, $rr_coeff_env_correlation_altered_env_array_3_6, $rr_residual_variance_altered_env_3_6, $varcomp_altered_array_env_3_6,
11483 $model_sum_square_residual_altered_env_3_6, $genetic_effect_min_altered_env_3_6, $genetic_effect_max_altered_env_3_6, $env_effect_min_altered_env_3_6, $env_effect_max_altered_env_3_6, $genetic_effect_sum_square_altered_env_3_6, $genetic_effect_sum_altered_env_3_6, $env_effect_sum_square_altered_env_3_6, $env_effect_sum_altered_env_3_6, $residual_sum_square_altered_env_3_6, $residual_sum_altered_env_3_6, $result_cv_altered_env_3_6, $result_cv_2_altered_env_3_6,
11484
11485 $phenotype_data_altered_env_hash_4_6, $data_matrix_altered_env_array_4_6, $data_matrix_phenotypes_altered_env_array_4_6, $phenotype_min_altered_env_4_6, $phenotype_max_altered_env_4_6, $env_sim_min_4_6, $env_sim_max_4_6, $sim_data_hash_4_6,
11486 $result_blup_data_altered_env_4_6, $result_blup_data_delta_altered_env_4_6, $result_blup_spatial_data_altered_env_4_6, $result_blup_pe_data_altered_env_4_6, $result_blup_pe_data_delta_altered_env_4_6, $result_residual_data_altered_env_4_6, $result_fitted_data_altered_env_4_6, $fixed_effects_altered_env_hash_4_6, $rr_genetic_coefficients_altered_env_hash_4_6, $rr_temporal_coefficients_altered_env_hash_4_6,
11487 $rr_coeff_genetic_covariance_altered_env_array_4_6, $rr_coeff_env_covariance_altered_env_array_4_6, $rr_coeff_genetic_correlation_altered_env_array_4_6, $rr_coeff_env_correlation_altered_env_array_4_6, $rr_residual_variance_altered_env_4_6, $varcomp_altered_array_env_4_6,
11488 $model_sum_square_residual_altered_env_4_6, $genetic_effect_min_altered_env_4_6, $genetic_effect_max_altered_env_4_6, $env_effect_min_altered_env_4_6, $env_effect_max_altered_env_4_6, $genetic_effect_sum_square_altered_env_4_6, $genetic_effect_sum_altered_env_4_6, $env_effect_sum_square_altered_env_4_6, $env_effect_sum_altered_env_4_6, $residual_sum_square_altered_env_4_6, $residual_sum_altered_env_4_6, $result_cv_altered_env_4_6, $result_cv_2_altered_env_4_6,
11489
11490 $phenotype_data_altered_env_hash_5_6, $data_matrix_altered_env_array_5_6, $data_matrix_phenotypes_altered_env_array_5_6, $phenotype_min_altered_env_5_6, $phenotype_max_altered_env_5_6, $env_sim_min_5_6, $env_sim_max_5_6, $sim_data_hash_5_6,
11491 $result_blup_data_altered_env_5_6, $result_blup_data_delta_altered_env_5_6, $result_blup_spatial_data_altered_env_5_6, $result_blup_pe_data_altered_env_5_6, $result_blup_pe_data_delta_altered_env_5_6, $result_residual_data_altered_env_5_6, $result_fitted_data_altered_env_5_6, $fixed_effects_altered_env_hash_5_6, $rr_genetic_coefficients_altered_env_hash_5_6, $rr_temporal_coefficients_altered_env_hash_5_6,
11492 $rr_coeff_genetic_covariance_altered_env_array_5_6, $rr_coeff_env_covariance_altered_env_array_5_6, $rr_coeff_genetic_correlation_altered_env_array_5_6, $rr_coeff_env_correlation_altered_env_array_5_6, $rr_residual_variance_altered_env_5_6, $varcomp_altered_array_env_5_6,
11493 $model_sum_square_residual_altered_env_5_6, $genetic_effect_min_altered_env_5_6, $genetic_effect_max_altered_env_5_6, $env_effect_min_altered_env_5_6, $env_effect_max_altered_env_5_6, $genetic_effect_sum_square_altered_env_5_6, $genetic_effect_sum_altered_env_5_6, $env_effect_sum_square_altered_env_5_6, $env_effect_sum_altered_env_5_6, $residual_sum_square_altered_env_5_6, $residual_sum_altered_env_5_6, $result_cv_altered_env_5_6, $result_cv_2_altered_env_5_6,
11494
11495 $phenotype_data_altered_env_hash_6_6, $data_matrix_altered_env_array_6_6, $data_matrix_phenotypes_altered_env_array_6_6, $phenotype_min_altered_env_6_6, $phenotype_max_altered_env_6_6, $env_sim_min_6_6, $env_sim_max_6_6, $sim_data_hash_6_6,
11496 $result_blup_data_altered_env_6_6, $result_blup_data_delta_altered_env_6_6, $result_blup_spatial_data_altered_env_6_6, $result_blup_pe_data_altered_env_6_6, $result_blup_pe_data_delta_altered_env_6_6, $result_residual_data_altered_env_6_6, $result_fitted_data_altered_env_6_6, $fixed_effects_altered_env_hash_6_6, $rr_genetic_coefficients_altered_env_hash_6_6, $rr_temporal_coefficients_altered_env_hash_6_6,
11497 $rr_coeff_genetic_covariance_altered_env_array_6_6, $rr_coeff_env_covariance_altered_env_array_6_6, $rr_coeff_genetic_correlation_altered_env_array_6_6, $rr_coeff_env_correlation_altered_env_array_6_6, $rr_residual_variance_altered_env_6_6, $varcomp_altered_array_env_6_6,
11498 $model_sum_square_residual_altered_env_6_6, $genetic_effect_min_altered_env_6_6, $genetic_effect_max_altered_env_6_6, $env_effect_min_altered_env_6_6, $env_effect_max_altered_env_6_6, $genetic_effect_sum_square_altered_env_6_6, $genetic_effect_sum_altered_env_6_6, $env_effect_sum_square_altered_env_6_6, $env_effect_sum_altered_env_6_6, $residual_sum_square_altered_env_6_6, $residual_sum_altered_env_6_6, $result_cv_altered_env_6_6, $result_cv_2_altered_env_6_6
11499 ) = @$result_6;
11500
11501 eval {
11502 print STDERR "PLOTTING CORRELATION\n";
11503 my ($full_plot_level_correlation_tempfile_fh, $full_plot_level_correlation_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11504 open(my $F_fullplot, ">", $full_plot_level_correlation_tempfile) || die "Can't open file ".$full_plot_level_correlation_tempfile;
11505 print STDERR "OPENED PLOTCORR FILE $full_plot_level_correlation_tempfile\n";
11506
11507 my @header_full_plot_corr = ('plot_name, plot_id, row_number, col_number, rep, block, germplasm_name, germplasm_id');
11508 my @types_full_plot_corr = ('pheno_orig_', 'pheno_postm6_', 'eff_origm6_', 'eff_postm6_',
11509 'sim_env1_', 'simm6_pheno1_', 'effm6_sim1_',
11510 'sim_env2_', 'simm6_pheno2_', 'effm6_sim2_',
11511 'sim_env3_', 'simm6_pheno3_', 'effm6_sim3_',
11512 'sim_env4_', 'simm6_pheno4_', 'effm6_sim4_',
11513 'sim_env5_', 'simm6_pheno5_', 'effm6_sim5_',
11514 'sim_env6_', 'simm6_pheno6_', 'effm6_sim6_');
11515 foreach my $t (@sorted_trait_names_6) {
11516 foreach my $type (@types_full_plot_corr) {
11517 push @header_full_plot_corr, $type.$trait_name_encoder_6{$t};
11518 }
11519 }
11520 my $header_string_full_plot_corr = join ',', @header_full_plot_corr;
11521 print $F_fullplot "$header_string_full_plot_corr\n";
11522 foreach my $p (@unique_plot_names) {
11523 my @row = ($p, $stock_name_row_col{$p}->{obsunit_stock_id}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $stock_name_row_col{$p}->{rep}, $stock_name_row_col{$p}->{block}, $stock_name_row_col{$p}->{germplasm_name}, $stock_name_row_col{$p}->{germplasm_stock_id});
11524 foreach my $t (@sorted_trait_names_6) {
11525 my $phenotype_original = $phenotype_data_original_6{$p}->{$t};
11526 my $phenotype_post_6 = $phenotype_data_altered_hash_6->{$p}->{$t};
11527 my $effect_original_6 = $result_blup_spatial_data_original_6->{$p}->{$t}->[0];
11528 my $effect_post_6 = $result_blup_spatial_data_altered_6->{$p}->{$t}->[0];
11529 push @row, ($phenotype_original, $phenotype_post_6, $effect_original_6, $effect_post_6);
11530
11531 my $sim_env = $sim_data_hash_1_6->{$p}->{$t};
11532 my $pheno_sim_6 = $phenotype_data_altered_env_hash_1_6->{$p}->{$t};
11533 my $effect_sim_6 = $result_blup_spatial_data_altered_env_1_6->{$p}->{$t}->[0];
11534 push @row, ($sim_env, $pheno_sim_6, $effect_sim_6);
11535
11536 my $sim_env2 = $sim_data_hash_2_6->{$p}->{$t};
11537 my $pheno_sim2_6 = $phenotype_data_altered_env_hash_2_6->{$p}->{$t};
11538 my $effect_sim2_6 = $result_blup_spatial_data_altered_env_2_6->{$p}->{$t}->[0];
11539 push @row, ($sim_env2, $pheno_sim2_6, $effect_sim2_6);
11540
11541 my $sim_env3 = $sim_data_hash_3_6->{$p}->{$t};
11542 my $pheno_sim3_6 = $phenotype_data_altered_env_hash_3_6->{$p}->{$t};
11543 my $effect_sim3_6 = $result_blup_spatial_data_altered_env_3_6->{$p}->{$t}->[0];
11544 push @row, ($sim_env3, $pheno_sim3_6, $effect_sim3_6);
11545
11546 my $sim_env4 = $sim_data_hash_4_6->{$p}->{$t};
11547 my $pheno_sim4_6 = $phenotype_data_altered_env_hash_4_6->{$p}->{$t};
11548 my $effect_sim4_6 = $result_blup_spatial_data_altered_env_4_6->{$p}->{$t}->[0];
11549 push @row, ($sim_env4, $pheno_sim4_6, $effect_sim4_6);
11550
11551 my $sim_env5 = $sim_data_hash_5_6->{$p}->{$t};
11552 my $pheno_sim5_6 = $phenotype_data_altered_env_hash_5_6->{$p}->{$t};
11553 my $effect_sim5_6 = $result_blup_spatial_data_altered_env_5_6->{$p}->{$t}->[0];
11554 push @row, ($sim_env5, $pheno_sim5_6, $effect_sim5_6);
11555
11556 my $sim_env6 = $sim_data_hash_6_6->{$p}->{$t};
11557 my $pheno_sim6_6 = $phenotype_data_altered_env_hash_6_6->{$p}->{$t};
11558 my $effect_sim6_6 = $result_blup_spatial_data_altered_env_6_6->{$p}->{$t}->[0];
11559 push @row, ($sim_env6, $pheno_sim6_6, $effect_sim6_6);
11560 }
11561 my $line = join ',', @row;
11562 print $F_fullplot "$line\n";
11563 }
11564 close($F_fullplot);
11565
11566 my $plot_corr_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
11567 $plot_corr_figure_tempfile_string .= '.png';
11568 my $plot_corr_figure_tempfile = $c->config->{basepath}."/".$plot_corr_figure_tempfile_string;
11569
11570 my $cmd_plotcorr_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
11571 mat_orig <- fread(\''.$full_plot_level_correlation_tempfile.'\', header=TRUE, sep=\',\');
11572 gg <- ggcorr(data=mat_orig[,-seq(1,8)], hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE);
11573 ggsave(\''.$plot_corr_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, limitsize = FALSE, units=\'in\');
11574 "';
11575 # print STDERR Dumper $cmd;
11576 my $status_plotcorr_plot = system($cmd_plotcorr_plot);
11577 push @$spatial_effects_plots, [$plot_corr_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
11578 push @$spatial_effects_files_store, [$full_plot_level_correlation_tempfile, "datafile_".$statistics_select.$sim_env_change_over_time.$correlation_between_times."_fullcorr_"."envvar_".$env_variance_percent."_".$iterations];
11579 };
11580
11581 eval {
11582 my @plot_corr_full_vals;
11583
11584 my @original_pheno_vals;
11585 my ($phenotypes_original_heatmap_tempfile_fh, $phenotypes_original_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11586 open(my $F_pheno, ">", $phenotypes_original_heatmap_tempfile) || die "Can't open file ".$phenotypes_original_heatmap_tempfile;
11587 print $F_pheno "trait_type,row,col,value\n";
11588 foreach my $p (@unique_plot_names) {
11589 foreach my $t (@sorted_trait_names_6) {
11590 my $val = $phenotype_data_original_6{$p}->{$t};
11591 my @row = ("pheno_orig_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11592 push @original_pheno_vals, $val;
11593 my $line = join ',', @row;
11594 print $F_pheno "$line\n";
11595 }
11596 }
11597 close($F_pheno);
11598 push @plot_corr_full_vals, \@original_pheno_vals;
11599
11600 my $original_pheno_stat = Statistics::Descriptive::Full->new();
11601 $original_pheno_stat->add_data(@original_pheno_vals);
11602 my $sig_original_pheno = $original_pheno_stat->variance();
11603
11604 #PHENO POST M START
11605
11606 my @altered_pheno_vals_6;
11607 my ($phenotypes_post_heatmap_tempfile_fh_6, $phenotypes_post_heatmap_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11608 open($F_pheno, ">", $phenotypes_post_heatmap_tempfile_6) || die "Can't open file ".$phenotypes_post_heatmap_tempfile_6;
11609 print $F_pheno "trait_type,row,col,value\n";
11610 foreach my $p (@unique_plot_names) {
11611 foreach my $t (@sorted_trait_names_6) {
11612 my $val = $phenotype_data_altered_hash_6->{$p}->{$t};
11613 my @row = ("pheno_postm6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11614 push @altered_pheno_vals_6, $val;
11615 my $line = join ',', @row;
11616 print $F_pheno "$line\n";
11617 }
11618 }
11619 close($F_pheno);
11620 push @plot_corr_full_vals, \@altered_pheno_vals_6;
11621
11622 my $altered_pheno_stat_6 = Statistics::Descriptive::Full->new();
11623 $altered_pheno_stat_6->add_data(@altered_pheno_vals_6);
11624 my $sig_altered_pheno_6 = $altered_pheno_stat_6->variance();
11625
11626 # EFFECT ORIGINAL M
11627
11628 my @original_effect_vals_6;
11629 my ($effects_heatmap_tempfile_fh_6, $effects_heatmap_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11630 open(my $F_eff, ">", $effects_heatmap_tempfile_6) || die "Can't open file ".$effects_heatmap_tempfile_6;
11631 print $F_eff "trait_type,row,col,value\n";
11632 foreach my $p (@unique_plot_names) {
11633 foreach my $t (@sorted_trait_names_6) {
11634 my $val = $result_blup_spatial_data_original_6->{$p}->{$t}->[0];
11635 my @row = ("eff_origm6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11636 my $line = join ',', @row;
11637 print $F_eff "$line\n";
11638 push @original_effect_vals_6, $val;
11639 }
11640 }
11641 close($F_eff);
11642 push @plot_corr_full_vals, \@original_effect_vals_6;
11643
11644 my $original_effect_stat_6 = Statistics::Descriptive::Full->new();
11645 $original_effect_stat_6->add_data(@original_effect_vals_6);
11646 my $sig_original_effect_6 = $original_effect_stat_6->variance();
11647
11648 # EFFECT POST M MIN
11649
11650 my @altered_effect_vals_6;
11651 my ($effects_post_heatmap_tempfile_fh_6, $effects_post_heatmap_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11652 open($F_eff, ">", $effects_post_heatmap_tempfile_6) || die "Can't open file ".$effects_post_heatmap_tempfile_6;
11653 print $F_eff "trait_type,row,col,value\n";
11654 foreach my $p (@unique_plot_names) {
11655 foreach my $t (@sorted_trait_names_6) {
11656 my $val = $result_blup_spatial_data_altered_6->{$p}->{$t}->[0];
11657 my @row = ("eff_postm6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11658 my $line = join ',', @row;
11659 print $F_eff "$line\n";
11660 push @altered_effect_vals_6, $val;
11661 }
11662 }
11663 close($F_eff);
11664 push @plot_corr_full_vals, \@altered_effect_vals_6;
11665
11666 my $altered_effect_stat_6 = Statistics::Descriptive::Full->new();
11667 $altered_effect_stat_6->add_data(@altered_effect_vals_6);
11668 my $sig_altered_effect_6 = $altered_effect_stat_6->variance();
11669
11670 # SIM ENV 1: ALTERED PHENO + EFFECT
11671
11672 my ($phenotypes_env_heatmap_tempfile_fh, $phenotypes_env_heatmap_tempfile) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11673 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile) || die "Can't open file ".$phenotypes_env_heatmap_tempfile;
11674 print $F_pheno "trait_type,row,col,value\n";
11675 foreach my $p (@unique_plot_names) {
11676 foreach my $t (@sorted_trait_names_6) {
11677 my @row = ("sim_env1_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_1_6->{$p}->{$t});
11678 my $line = join ',', @row;
11679 print $F_pheno "$line\n";
11680 }
11681 }
11682 close($F_pheno);
11683
11684 my @sim_pheno1_vals_6;
11685 my ($phenotypes_pheno_sim_heatmap_tempfile_fh_6, $phenotypes_pheno_sim_heatmap_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11686 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile_6;
11687 print $F_pheno "trait_type,row,col,value\n";
11688 foreach my $p (@unique_plot_names) {
11689 foreach my $t (@sorted_trait_names_6) {
11690 my $val = $phenotype_data_altered_env_hash_1_6->{$p}->{$t};
11691 my @row = ("simm6_pheno1_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11692 my $line = join ',', @row;
11693 print $F_pheno "$line\n";
11694 push @sim_pheno1_vals_6, $val;
11695 }
11696 }
11697 close($F_pheno);
11698 push @plot_corr_full_vals, \@sim_pheno1_vals_6;
11699
11700 my $sim_pheno1_stat_6 = Statistics::Descriptive::Full->new();
11701 $sim_pheno1_stat_6->add_data(@sim_pheno1_vals_6);
11702 my $sig_sim6_pheno1 = $sim_pheno1_stat_6->variance();
11703
11704 my @sim_effect1_vals_6;
11705 my ($effects_sim_heatmap_tempfile_fh_6, $effects_sim_heatmap_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11706 open($F_eff, ">", $effects_sim_heatmap_tempfile_6) || die "Can't open file ".$effects_sim_heatmap_tempfile_6;
11707 print $F_eff "trait_type,row,col,value\n";
11708 foreach my $p (@unique_plot_names) {
11709 foreach my $t (@sorted_trait_names_6) {
11710 my $val = $result_blup_spatial_data_altered_env_1_6->{$p}->{$t}->[0];
11711 my @row = ("effm6_sim1_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11712 my $line = join ',', @row;
11713 print $F_eff "$line\n";
11714 push @sim_effect1_vals_6, $val;
11715 }
11716 }
11717 close($F_eff);
11718 push @plot_corr_full_vals, \@sim_effect1_vals_6;
11719
11720 my $sim_effect1_stat_6 = Statistics::Descriptive::Full->new();
11721 $sim_effect1_stat_6->add_data(@sim_effect1_vals_6);
11722 my $sig_sim6_effect1 = $sim_effect1_stat_6->variance();
11723
11724 # SIM ENV 2: ALTERED PHENO + EFFECT
11725
11726 my ($phenotypes_env_heatmap_tempfile2_fh, $phenotypes_env_heatmap_tempfile2) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11727 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile2) || die "Can't open file ".$phenotypes_env_heatmap_tempfile2;
11728 print $F_pheno "trait_type,row,col,value\n";
11729 foreach my $p (@unique_plot_names) {
11730 foreach my $t (@sorted_trait_names_6) {
11731 my @row = ("sim_env2_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_2_6->{$p}->{$t});
11732 my $line = join ',', @row;
11733 print $F_pheno "$line\n";
11734 }
11735 }
11736 close($F_pheno);
11737
11738 my @sim_pheno2_vals_6;
11739 my ($phenotypes_pheno_sim_heatmap_tempfile2_fh_6, $phenotypes_pheno_sim_heatmap_tempfile2_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11740 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile2_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile2_6;
11741 print $F_pheno "trait_type,row,col,value\n";
11742 foreach my $p (@unique_plot_names) {
11743 foreach my $t (@sorted_trait_names_6) {
11744 my $val = $phenotype_data_altered_env_hash_2_6->{$p}->{$t};
11745 my @row = ("simm6_pheno2_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11746 my $line = join ',', @row;
11747 print $F_pheno "$line\n";
11748 push @sim_pheno2_vals_6, $val;
11749 }
11750 }
11751 close($F_pheno);
11752 push @plot_corr_full_vals, \@sim_pheno2_vals_6;
11753
11754 my $sim_pheno2_stat_6 = Statistics::Descriptive::Full->new();
11755 $sim_pheno2_stat_6->add_data(@sim_pheno2_vals_6);
11756 my $sig_sim_pheno2_6 = $sim_pheno2_stat_6->variance();
11757
11758 my @sim_effect2_vals_6;
11759 my ($effects_sim_heatmap_tempfile2_fh_6, $effects_sim_heatmap_tempfile2_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11760 open($F_eff, ">", $effects_sim_heatmap_tempfile2_6) || die "Can't open file ".$effects_sim_heatmap_tempfile2_6;
11761 print $F_eff "trait_type,row,col,value\n";
11762 foreach my $p (@unique_plot_names) {
11763 foreach my $t (@sorted_trait_names_6) {
11764 my $val = $result_blup_spatial_data_altered_env_2_6->{$p}->{$t}->[0];
11765 my @row = ("effm6_sim2_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11766 my $line = join ',', @row;
11767 print $F_eff "$line\n";
11768 push @sim_effect2_vals_6, $val;
11769 }
11770 }
11771 close($F_eff);
11772 push @plot_corr_full_vals, \@sim_effect2_vals_6;
11773
11774 my $sim_effect2_stat_6 = Statistics::Descriptive::Full->new();
11775 $sim_effect2_stat_6->add_data(@sim_effect2_vals_6);
11776 my $sig_sim_effect2_6 = $sim_effect2_stat_6->variance();
11777
11778 # SIM ENV 3: ALTERED PHENO + EFFECT
11779
11780 my ($phenotypes_env_heatmap_tempfile3_fh, $phenotypes_env_heatmap_tempfile3) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11781 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile3) || die "Can't open file ".$phenotypes_env_heatmap_tempfile3;
11782 print $F_pheno "trait_type,row,col,value\n";
11783 foreach my $p (@unique_plot_names) {
11784 foreach my $t (@sorted_trait_names_6) {
11785 my @row = ("sim_env3_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_3_6->{$p}->{$t});
11786 my $line = join ',', @row;
11787 print $F_pheno "$line\n";
11788 }
11789 }
11790 close($F_pheno);
11791
11792 my @sim_pheno3_vals_6;
11793 my ($phenotypes_pheno_sim_heatmap_tempfile3_fh_6, $phenotypes_pheno_sim_heatmap_tempfile3_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11794 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile3_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile3_6;
11795 print $F_pheno "trait_type,row,col,value\n";
11796 foreach my $p (@unique_plot_names) {
11797 foreach my $t (@sorted_trait_names_6) {
11798 my $val = $phenotype_data_altered_env_hash_3_6->{$p}->{$t};
11799 my @row = ("simm6_pheno3_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11800 my $line = join ',', @row;
11801 print $F_pheno "$line\n";
11802 push @sim_pheno3_vals_6, $val;
11803 }
11804 }
11805 close($F_pheno);
11806 push @plot_corr_full_vals, \@sim_pheno3_vals_6;
11807
11808 my $sim_pheno3_stat_6 = Statistics::Descriptive::Full->new();
11809 $sim_pheno3_stat_6->add_data(@sim_pheno3_vals_6);
11810 my $sig_sim_pheno3_6 = $sim_pheno3_stat_6->variance();
11811
11812 my @sim_effect3_vals_6;
11813 my ($effects_sim_heatmap_tempfile3_fh_6, $effects_sim_heatmap_tempfile3_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11814 open($F_eff, ">", $effects_sim_heatmap_tempfile3_6) || die "Can't open file ".$effects_sim_heatmap_tempfile3_6;
11815 print $F_eff "trait_type,row,col,value\n";
11816 foreach my $p (@unique_plot_names) {
11817 foreach my $t (@sorted_trait_names_6) {
11818 my $val = $result_blup_spatial_data_altered_env_3_6->{$p}->{$t}->[0];
11819 my @row = ("effm6_sim3_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11820 my $line = join ',', @row;
11821 print $F_eff "$line\n";
11822 push @sim_effect3_vals_6, $val;
11823 }
11824 }
11825 close($F_eff);
11826 push @plot_corr_full_vals, \@sim_effect3_vals_6;
11827
11828 my $sim_effect3_stat_6 = Statistics::Descriptive::Full->new();
11829 $sim_effect3_stat_6->add_data(@sim_effect3_vals_6);
11830 my $sig_sim_effect3_6 = $sim_effect3_stat_6->variance();
11831
11832 # SIM ENV 4: ALTERED PHENO + EFFECT
11833
11834 my ($phenotypes_env_heatmap_tempfile4_fh, $phenotypes_env_heatmap_tempfile4) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11835 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile4) || die "Can't open file ".$phenotypes_env_heatmap_tempfile4;
11836 print $F_pheno "trait_type,row,col,value\n";
11837 foreach my $p (@unique_plot_names) {
11838 foreach my $t (@sorted_trait_names_6) {
11839 my @row = ("sim_env4_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_4_6->{$p}->{$t});
11840 my $line = join ',', @row;
11841 print $F_pheno "$line\n";
11842 }
11843 }
11844 close($F_pheno);
11845
11846 my @sim_pheno4_vals_6;
11847 my ($phenotypes_pheno_sim_heatmap_tempfile4_fh_6, $phenotypes_pheno_sim_heatmap_tempfile4_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11848 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile4_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile4_6;
11849 print $F_pheno "trait_type,row,col,value\n";
11850 foreach my $p (@unique_plot_names) {
11851 foreach my $t (@sorted_trait_names_6) {
11852 my $val = $phenotype_data_altered_env_hash_4_6->{$p}->{$t};
11853 my @row = ("simm6_pheno4_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11854 my $line = join ',', @row;
11855 print $F_pheno "$line\n";
11856 push @sim_pheno4_vals_6, $val;
11857 }
11858 }
11859 close($F_pheno);
11860 push @plot_corr_full_vals, \@sim_pheno4_vals_6;
11861
11862 my $sim_pheno4_stat_6 = Statistics::Descriptive::Full->new();
11863 $sim_pheno4_stat_6->add_data(@sim_pheno4_vals_6);
11864 my $sig_sim_pheno4_6 = $sim_pheno4_stat_6->variance();
11865
11866 my @sim_effect4_vals_6;
11867 my ($effects_sim_heatmap_tempfile4_fh_6, $effects_sim_heatmap_tempfile4_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11868 open($F_eff, ">", $effects_sim_heatmap_tempfile4_6) || die "Can't open file ".$effects_sim_heatmap_tempfile4_6;
11869 print $F_eff "trait_type,row,col,value\n";
11870 foreach my $p (@unique_plot_names) {
11871 foreach my $t (@sorted_trait_names_6) {
11872 my $val = $result_blup_spatial_data_altered_env_4_6->{$p}->{$t}->[0];
11873 my @row = ("effm6_sim4_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11874 my $line = join ',', @row;
11875 print $F_eff "$line\n";
11876 push @sim_effect4_vals_6, $val;
11877 }
11878 }
11879 close($F_eff);
11880 push @plot_corr_full_vals, \@sim_effect4_vals_6;
11881
11882 my $sim_effect4_stat_6 = Statistics::Descriptive::Full->new();
11883 $sim_effect4_stat_6->add_data(@sim_effect4_vals_6);
11884 my $sig_sim_effect4_6 = $sim_effect4_stat_6->variance();
11885
11886 # SIM ENV 5: ALTERED PHENO + EFFECT
11887
11888 my ($phenotypes_env_heatmap_tempfile5_fh, $phenotypes_env_heatmap_tempfile5) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11889 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile5) || die "Can't open file ".$phenotypes_env_heatmap_tempfile5;
11890 print $F_pheno "trait_type,row,col,value\n";
11891 foreach my $p (@unique_plot_names) {
11892 foreach my $t (@sorted_trait_names_6) {
11893 my @row = ("sim_env5_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_5_6->{$p}->{$t});
11894 my $line = join ',', @row;
11895 print $F_pheno "$line\n";
11896 }
11897 }
11898 close($F_pheno);
11899
11900 my @sim_pheno5_vals_6;
11901 my ($phenotypes_pheno_sim_heatmap_tempfile5_fh_6, $phenotypes_pheno_sim_heatmap_tempfile5_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11902 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile5_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile5_6;
11903 print $F_pheno "trait_type,row,col,value\n";
11904 foreach my $p (@unique_plot_names) {
11905 foreach my $t (@sorted_trait_names_6) {
11906 my $val = $phenotype_data_altered_env_hash_5_6->{$p}->{$t};
11907 my @row = ("simm6_pheno5_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11908 my $line = join ',', @row;
11909 print $F_pheno "$line\n";
11910 push @sim_pheno5_vals_6, $val;
11911 }
11912 }
11913 close($F_pheno);
11914 push @plot_corr_full_vals, \@sim_pheno5_vals_6;
11915
11916 my $sim_pheno5_stat_6 = Statistics::Descriptive::Full->new();
11917 $sim_pheno5_stat_6->add_data(@sim_pheno5_vals_6);
11918 my $sig_sim_pheno5_6 = $sim_pheno5_stat_6->variance();
11919
11920 my @sim_effect5_vals_6;
11921 my ($effects_sim_heatmap_tempfile5_fh_6, $effects_sim_heatmap_tempfile5_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11922 open($F_eff, ">", $effects_sim_heatmap_tempfile5_6) || die "Can't open file ".$effects_sim_heatmap_tempfile5_6;
11923 print $F_eff "trait_type,row,col,value\n";
11924 foreach my $p (@unique_plot_names) {
11925 foreach my $t (@sorted_trait_names_6) {
11926 my $val = $result_blup_spatial_data_altered_env_5_6->{$p}->{$t}->[0];
11927 my @row = ("effm6_sim5_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11928 my $line = join ',', @row;
11929 print $F_eff "$line\n";
11930 push @sim_effect5_vals_6, $val;
11931 }
11932 }
11933 close($F_eff);
11934 push @plot_corr_full_vals, \@sim_effect5_vals_6;
11935
11936 my $sim_effect5_stat_6 = Statistics::Descriptive::Full->new();
11937 $sim_effect5_stat_6->add_data(@sim_effect5_vals_6);
11938 my $sig_sim_effect5_6 = $sim_effect5_stat_6->variance();
11939
11940 # SIM ENV 6: ALTERED PHENO + EFFECT
11941
11942 my ($phenotypes_env_heatmap_tempfile6_fh, $phenotypes_env_heatmap_tempfile6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11943 open($F_pheno, ">", $phenotypes_env_heatmap_tempfile6) || die "Can't open file ".$phenotypes_env_heatmap_tempfile6;
11944 print $F_pheno "trait_type,row,col,value\n";
11945 foreach my $p (@unique_plot_names) {
11946 foreach my $t (@sorted_trait_names_6) {
11947 my @row = ("sim_env6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $sim_data_hash_6_6->{$p}->{$t});
11948 my $line = join ',', @row;
11949 print $F_pheno "$line\n";
11950 }
11951 }
11952 close($F_pheno);
11953
11954 my @sim_pheno6_vals_6;
11955 my ($phenotypes_pheno_sim_heatmap_tempfile6_fh_6, $phenotypes_pheno_sim_heatmap_tempfile6_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11956 open($F_pheno, ">", $phenotypes_pheno_sim_heatmap_tempfile6_6) || die "Can't open file ".$phenotypes_pheno_sim_heatmap_tempfile6_6;
11957 print $F_pheno "trait_type,row,col,value\n";
11958 foreach my $p (@unique_plot_names) {
11959 foreach my $t (@sorted_trait_names_6) {
11960 my $val = $phenotype_data_altered_env_hash_6_6->{$p}->{$t};
11961 my @row = ("simm6_pheno6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11962 my $line = join ',', @row;
11963 print $F_pheno "$line\n";
11964 push @sim_pheno6_vals_6, $val;
11965 }
11966 }
11967 close($F_pheno);
11968 push @plot_corr_full_vals, \@sim_pheno6_vals_6;
11969
11970 my $sim_pheno6_stat_6 = Statistics::Descriptive::Full->new();
11971 $sim_pheno6_stat_6->add_data(@sim_pheno6_vals_6);
11972 my $sig_sim_pheno6_6 = $sim_pheno6_stat_6->variance();
11973
11974 my @sim_effect6_vals_6;
11975 my ($effects_sim_heatmap_tempfile6_fh_6, $effects_sim_heatmap_tempfile6_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
11976 open($F_eff, ">", $effects_sim_heatmap_tempfile6_6) || die "Can't open file ".$effects_sim_heatmap_tempfile6_6;
11977 print $F_eff "trait_type,row,col,value\n";
11978 foreach my $p (@unique_plot_names) {
11979 foreach my $t (@sorted_trait_names_6) {
11980 my $val = $result_blup_spatial_data_altered_env_6_6->{$p}->{$t}->[0];
11981 my @row = ("effm6_sim6_".$trait_name_encoder_6{$t}, $stock_name_row_col{$p}->{row_number}, $stock_name_row_col{$p}->{col_number}, $val);
11982 my $line = join ',', @row;
11983 print $F_eff "$line\n";
11984 push @sim_effect6_vals_6, $val;
11985 }
11986 }
11987 close($F_eff);
11988 push @plot_corr_full_vals, \@sim_effect6_vals_6;
11989
11990 my $sim_effect6_stat_6 = Statistics::Descriptive::Full->new();
11991 $sim_effect6_stat_6->add_data(@sim_effect6_vals_6);
11992 my $sig_sim_effect6_6 = $sim_effect6_stat_6->variance();
11993
11994 my $plot_corr_summary_figure_inputfile_tempfile = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => 'tmp_drone_statistics/fileXXXX');
11995 open($F_eff, ">", $plot_corr_summary_figure_inputfile_tempfile) || die "Can't open file ".$plot_corr_summary_figure_inputfile_tempfile;
11996 foreach (@plot_corr_full_vals) {
11997 my $line = join ',', @$_;
11998 print $F_eff $line."\n";
11999 }
12000 close($F_eff);
12001
12002 my $plot_corr_summary_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12003 $plot_corr_summary_figure_tempfile_string .= '.png';
12004 my $plot_corr_summary_figure_tempfile = $c->config->{basepath}."/".$plot_corr_summary_figure_tempfile_string;
12005
12006 my $cmd_plotcorrsum_plot = 'R -e "library(data.table); library(ggplot2); library(GGally);
12007 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
12008 mat_full <- data.frame(t(mat_full_t));
12009 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_6\', \'mat_eff_6\', \'mat_eff_altered_6\',
12010 \'mat_p_sim1_6\', \'mat_eff_sim1_6\',
12011 \'mat_p_sim2_6\', \'mat_eff_sim2_6\',
12012 \'mat_p_sim3_6\', \'mat_eff_sim3_6\',
12013 \'mat_p_sim4_6\', \'mat_eff_sim4_6\',
12014 \'mat_p_sim5_6\', \'mat_eff_sim5_6\',
12015 \'mat_p_sim6_6\', \'mat_eff_sim6_6\');
12016 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
12017 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
12018 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
12019 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
12020 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
12021 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
12022 mat <- data.frame(pheno_orig = mat_full\$mat_orig, pheno_altm6 = mat_full\$mat_altered_6, eff_origm6 = mat_full\$mat_eff_6, eff_altm6 = mat_full\$mat_eff_altered_6, env_lin = mat_env\$value, pheno_linm6 = mat_full\$mat_p_sim1_6, lin_effm6 = mat_full\$mat_eff_sim1_6, env_n1d = mat_env2\$value, pheno_n1dm6 = mat_full\$mat_p_sim2_6, n1d_effm6 = mat_full\$mat_eff_sim2_6, env_n2d = mat_env3\$value, pheno_n2dm6 = mat_full\$mat_p_sim3_6, env_rand = mat_env4\$value, pheno_randm6 = mat_full\$mat_p_sim4_6, rand_effm6 = mat_full\$mat_eff_sim4_6, env_ar1 = mat_env5\$value, pheno_ar1m6 = mat_full\$mat_p_sim5_6, ar1_effm6 = mat_full\$mat_eff_sim5_6, env_realdata = mat_env6\$value, pheno_realdatam6 = mat_full\$mat_p_sim6_6, realdata_effm6 = mat_full\$mat_eff_sim6_6);
12023 gg <- ggcorr(data=mat, hjust = 1, size = 2, color = \'grey50\', layout.exp = 1, label = TRUE, label_round = 2);
12024 ggsave(\''.$plot_corr_summary_figure_tempfile.'\', gg, device=\'png\', width=30, height=30, units=\'in\');
12025 "';
12026 # print STDERR Dumper $cmd_plotcorrsum_plot;
12027
12028 my $status_plotcorrsum_plot = system($cmd_plotcorrsum_plot);
12029 push @$spatial_effects_plots, [$plot_corr_summary_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_envsimscorr_"."envvar_".$env_variance_percent."_".$iterations];
12030
12031 my $env_effects_first_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12032 $env_effects_first_figure_tempfile_string .= '.png';
12033 my $env_effects_first_figure_tempfile = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string;
12034
12035 my $env_effects_first_figure_tempfile_string_2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12036 $env_effects_first_figure_tempfile_string_2 .= '.png';
12037 my $env_effects_first_figure_tempfile_2 = $c->config->{basepath}."/".$env_effects_first_figure_tempfile_string_2;
12038
12039 my $output_plot_row = 'row';
12040 my $output_plot_col = 'col';
12041 if ($max_col > $max_row) {
12042 $output_plot_row = 'col';
12043 $output_plot_col = 'row';
12044 }
12045
12046 my $cmd_spatialfirst_plot_2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12047 mat_orig <- fread(\''.$phenotypes_original_heatmap_tempfile.'\', header=TRUE, sep=\',\');
12048 mat_altered_6 <- fread(\''.$phenotypes_post_heatmap_tempfile_6.'\', header=TRUE, sep=\',\');
12049 pheno_mat <- rbind(mat_orig, mat_altered_6);
12050 options(device=\'png\');
12051 par();
12052 gg <- ggplot(pheno_mat, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12053 geom_tile() +
12054 scale_fill_viridis(discrete=FALSE) +
12055 coord_equal() +
12056 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12057 ggsave(\''.$env_effects_first_figure_tempfile_2.'\', gg, device=\'png\', width=20, height=20, units=\'in\');
12058 "';
12059 # print STDERR Dumper $cmd;
12060 my $status_spatialfirst_plot_2 = system($cmd_spatialfirst_plot_2);
12061 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string_2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_origheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12062
12063 my ($sim_effects_corr_results_fh, $sim_effects_corr_results) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12064
12065 my $cmd_spatialfirst_plot = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12066 mat_full_t <- fread(\''.$plot_corr_summary_figure_inputfile_tempfile.'\', header=FALSE, sep=\',\');
12067 mat_full <- data.frame(t(mat_full_t));
12068 colnames(mat_full) <- c(\'mat_orig\', \'mat_altered_6\', \'mat_eff_6\', \'mat_eff_altered_6\',
12069 \'mat_p_sim1_6\', \'mat_eff_sim1_6\',
12070 \'mat_p_sim2_6\', \'mat_eff_sim2_6\',
12071 \'mat_p_sim3_6\', \'mat_eff_sim3_6\',
12072 \'mat_p_sim4_6\', \'mat_eff_sim4_6\',
12073 \'mat_p_sim5_6\', \'mat_eff_sim5_6\',
12074 \'mat_p_sim6_6\', \'mat_eff_sim6_6\');
12075 mat_eff_6 <- fread(\''.$effects_heatmap_tempfile_6.'\', header=TRUE, sep=\',\');
12076 mat_eff_altered_6 <- fread(\''.$effects_post_heatmap_tempfile_6.'\', header=TRUE, sep=\',\');
12077 effect_mat_6 <- rbind(mat_eff_6, mat_eff_altered_6);
12078 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
12079 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
12080 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
12081 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
12082 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
12083 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
12084 options(device=\'png\');
12085 par();
12086 gg_eff_6 <- ggplot(effect_mat_6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12087 geom_tile() +
12088 scale_fill_viridis(discrete=FALSE) +
12089 coord_equal() +
12090 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12091 ggsave(\''.$env_effects_first_figure_tempfile.'\', arrangeGrob(gg_eff_6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12092 write.table(data.frame(asreml_grm_univariate_spatial_genetic_blups_env_linear = c(cor(mat_env\$value, mat_full\$mat_eff_sim1_6)), asreml_grm_univariate_spatial_genetic_blups_env_1DN = c(cor(mat_env2\$value, mat_full\$mat_eff_sim2_6)), asreml_grm_univariate_spatial_genetic_blups_env_2DN = c(cor(mat_env3\$value, mat_full\$mat_eff_sim3_6)), asreml_grm_univariate_spatial_genetic_blups_env_random = c(cor(mat_env4\$value, mat_full\$mat_eff_sim4_6)), asreml_grm_univariate_spatial_genetic_blups_env_ar1xar1 = c(cor(mat_env5\$value, mat_full\$mat_eff_sim5_6)), asreml_grm_univariate_spatial_genetic_blups_env_realdata = c(cor(mat_env6\$value, mat_full\$mat_eff_sim6_6)) ), file=\''.$sim_effects_corr_results.'\', row.names=FALSE, col.names=TRUE, sep=\'\t\');
12093 "';
12094 # print STDERR Dumper $cmd;
12095 my $status_spatialfirst_plot = system($cmd_spatialfirst_plot);
12096 push @$spatial_effects_plots, [$env_effects_first_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_originaleffheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12097
12098 open(my $fh_corr_result, '<', $sim_effects_corr_results) or die "Could not open file '$sim_effects_corr_results' $!";
12099 print STDERR "Opened $sim_effects_corr_results\n";
12100
12101 my $header = <$fh_corr_result>;
12102 my @header;
12103 if ($csv->parse($header)) {
12104 @header = $csv->fields();
12105 }
12106
12107 while (my $row = <$fh_corr_result>) {
12108 my @columns;
12109 my $counter = 0;
12110 if ($csv->parse($row)) {
12111 @columns = $csv->fields();
12112 }
12113 foreach (@columns) {
12114 push @{$env_corr_res->{$header[$counter]."_corrtime_".$sim_env_change_over_time.$correlation_between_times."_envvar_".$env_variance_percent}->{values}}, $_;
12115 $counter++;
12116 }
12117 }
12118 close($fh_corr_result);
12119
12120 my $env_effects_sim_figure_tempfile_string_6_env1 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12121 $env_effects_sim_figure_tempfile_string_6_env1 .= '.png';
12122 my $env_effects_sim_figure_tempfile_6_env1 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env1;
12123
12124 my $cmd_spatialenvsim_plot_6_env1 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12125 mat_env <- fread(\''.$phenotypes_env_heatmap_tempfile.'\', header=TRUE, sep=\',\');
12126 mat_p_sim <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile_6.'\', header=TRUE, sep=\',\');
12127 mat_eff_sim <- fread(\''.$effects_sim_heatmap_tempfile_6.'\', header=TRUE, sep=\',\');
12128 options(device=\'png\');
12129 par();
12130 gg_env <- ggplot(mat_env, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12131 geom_tile() +
12132 scale_fill_viridis(discrete=FALSE) +
12133 coord_equal() +
12134 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12135 gg_p_sim <- ggplot(mat_p_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12136 geom_tile() +
12137 scale_fill_viridis(discrete=FALSE) +
12138 coord_equal() +
12139 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12140 gg_eff_sim <- ggplot(mat_eff_sim, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12141 geom_tile() +
12142 scale_fill_viridis(discrete=FALSE) +
12143 coord_equal() +
12144 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12145 ggsave(\''.$env_effects_sim_figure_tempfile_6_env1.'\', arrangeGrob(gg_env, gg_p_sim, gg_eff_sim, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12146 "';
12147 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env1;
12148 my $status_spatialenvsim_plot_6_env1 = system($cmd_spatialenvsim_plot_6_env1);
12149 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env1, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env1effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12150
12151 my $env_effects_sim_figure_tempfile_string_6_env2 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12152 $env_effects_sim_figure_tempfile_string_6_env2 .= '.png';
12153 my $env_effects_sim_figure_tempfile_6_env2 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env2;
12154
12155 my $cmd_spatialenvsim_plot_6_env2 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12156 mat_env2 <- fread(\''.$phenotypes_env_heatmap_tempfile2.'\', header=TRUE, sep=\',\');
12157 mat_p_sim2 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile2_6.'\', header=TRUE, sep=\',\');
12158 mat_eff_sim2 <- fread(\''.$effects_sim_heatmap_tempfile2_6.'\', header=TRUE, sep=\',\');
12159 options(device=\'png\');
12160 par();
12161 gg_env2 <- ggplot(mat_env2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12162 geom_tile() +
12163 scale_fill_viridis(discrete=FALSE) +
12164 coord_equal() +
12165 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12166 gg_p_sim2 <- ggplot(mat_p_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12167 geom_tile() +
12168 scale_fill_viridis(discrete=FALSE) +
12169 coord_equal() +
12170 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12171 gg_eff_sim2 <- ggplot(mat_eff_sim2, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12172 geom_tile() +
12173 scale_fill_viridis(discrete=FALSE) +
12174 coord_equal() +
12175 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12176 ggsave(\''.$env_effects_sim_figure_tempfile_6_env2.'\', arrangeGrob(gg_env2, gg_p_sim2, gg_eff_sim2, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12177 "';
12178 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env2;
12179 my $status_spatialenvsim_plot_6_env2 = system($cmd_spatialenvsim_plot_6_env2);
12180 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env2, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env2effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12181
12182 my $env_effects_sim_figure_tempfile_string_6_env3 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12183 $env_effects_sim_figure_tempfile_string_6_env3 .= '.png';
12184 my $env_effects_sim_figure_tempfile_6_env3 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env3;
12185
12186 my $cmd_spatialenvsim_plot_6_env3 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12187 mat_env3 <- fread(\''.$phenotypes_env_heatmap_tempfile3.'\', header=TRUE, sep=\',\');
12188 mat_p_sim3 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile3_6.'\', header=TRUE, sep=\',\');
12189 mat_eff_sim3 <- fread(\''.$effects_sim_heatmap_tempfile3_6.'\', header=TRUE, sep=\',\');
12190 options(device=\'png\');
12191 par();
12192 gg_env3 <- ggplot(mat_env3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12193 geom_tile() +
12194 scale_fill_viridis(discrete=FALSE) +
12195 coord_equal() +
12196 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12197 gg_p_sim3 <- ggplot(mat_p_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12198 geom_tile() +
12199 scale_fill_viridis(discrete=FALSE) +
12200 coord_equal() +
12201 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12202 gg_eff_sim3 <- ggplot(mat_eff_sim3, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12203 geom_tile() +
12204 scale_fill_viridis(discrete=FALSE) +
12205 coord_equal() +
12206 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12207 ggsave(\''.$env_effects_sim_figure_tempfile_6_env3.'\', arrangeGrob(gg_env3, gg_p_sim3, gg_eff_sim3, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12208 "';
12209 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env3;
12210 my $status_spatialenvsim_plot_6_env3 = system($cmd_spatialenvsim_plot_6_env3);
12211 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env3, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env3effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12212
12213 my $env_effects_sim_figure_tempfile_string_6_env4 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12214 $env_effects_sim_figure_tempfile_string_6_env4 .= '.png';
12215 my $env_effects_sim_figure_tempfile_6_env4 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env4;
12216
12217 my $cmd_spatialenvsim_plot_6_env4 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12218 mat_env4 <- fread(\''.$phenotypes_env_heatmap_tempfile4.'\', header=TRUE, sep=\',\');
12219 mat_p_sim4 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile4_6.'\', header=TRUE, sep=\',\');
12220 mat_eff_sim4 <- fread(\''.$effects_sim_heatmap_tempfile4_6.'\', header=TRUE, sep=\',\');
12221 options(device=\'png\');
12222 par();
12223 gg_env4 <- ggplot(mat_env4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12224 geom_tile() +
12225 scale_fill_viridis(discrete=FALSE) +
12226 coord_equal() +
12227 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12228 gg_p_sim4 <- ggplot(mat_p_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12229 geom_tile() +
12230 scale_fill_viridis(discrete=FALSE) +
12231 coord_equal() +
12232 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12233 gg_eff_sim4 <- ggplot(mat_eff_sim4, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12234 geom_tile() +
12235 scale_fill_viridis(discrete=FALSE) +
12236 coord_equal() +
12237 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12238 ggsave(\''.$env_effects_sim_figure_tempfile_6_env4.'\', arrangeGrob(gg_env4, gg_p_sim4, gg_eff_sim4, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12239 "';
12240 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env4;
12241 my $status_spatialenvsim_plot_6_env4 = system($cmd_spatialenvsim_plot_6_env4);
12242 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env4, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env4effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12243
12244 my $env_effects_sim_figure_tempfile_string_6_env5 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12245 $env_effects_sim_figure_tempfile_string_6_env5 .= '.png';
12246 my $env_effects_sim_figure_tempfile_6_env5 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env5;
12247
12248 my $cmd_spatialenvsim_plot_6_env5 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12249 mat_env5 <- fread(\''.$phenotypes_env_heatmap_tempfile5.'\', header=TRUE, sep=\',\');
12250 mat_p_sim5 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile5_6.'\', header=TRUE, sep=\',\');
12251 mat_eff_sim5 <- fread(\''.$effects_sim_heatmap_tempfile5_6.'\', header=TRUE, sep=\',\');
12252 options(device=\'png\');
12253 par();
12254 gg_env5 <- ggplot(mat_env5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12255 geom_tile() +
12256 scale_fill_viridis(discrete=FALSE) +
12257 coord_equal() +
12258 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12259 gg_p_sim5 <- ggplot(mat_p_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12260 geom_tile() +
12261 scale_fill_viridis(discrete=FALSE) +
12262 coord_equal() +
12263 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12264 gg_eff_sim5 <- ggplot(mat_eff_sim5, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12265 geom_tile() +
12266 scale_fill_viridis(discrete=FALSE) +
12267 coord_equal() +
12268 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12269 ggsave(\''.$env_effects_sim_figure_tempfile_6_env5.'\', arrangeGrob(gg_env5, gg_p_sim5, gg_eff_sim5, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12270 "';
12271 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env5;
12272 my $status_spatialenvsim_plot_6_env5 = system($cmd_spatialenvsim_plot_6_env5);
12273 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env5, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env5effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12274
12275 my $env_effects_sim_figure_tempfile_string_6_env6 = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12276 $env_effects_sim_figure_tempfile_string_6_env6 .= '.png';
12277 my $env_effects_sim_figure_tempfile_6_env6 = $c->config->{basepath}."/".$env_effects_sim_figure_tempfile_string_6_env6;
12278
12279 my $cmd_spatialenvsim_plot_6_env6 = 'R -e "library(data.table); library(ggplot2); library(dplyr); library(viridis); library(GGally); library(gridExtra);
12280 mat_env6 <- fread(\''.$phenotypes_env_heatmap_tempfile6.'\', header=TRUE, sep=\',\');
12281 mat_p_sim6 <- fread(\''.$phenotypes_pheno_sim_heatmap_tempfile6_6.'\', header=TRUE, sep=\',\');
12282 mat_eff_sim6 <- fread(\''.$effects_sim_heatmap_tempfile6_6.'\', header=TRUE, sep=\',\');
12283 options(device=\'png\');
12284 par();
12285 gg_env6 <- ggplot(mat_env6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12286 geom_tile() +
12287 scale_fill_viridis(discrete=FALSE) +
12288 coord_equal() +
12289 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12290 gg_p_sim6 <- ggplot(mat_p_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12291 geom_tile() +
12292 scale_fill_viridis(discrete=FALSE) +
12293 coord_equal() +
12294 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12295 gg_eff_sim6 <- ggplot(mat_eff_sim6, aes('.$output_plot_col.', '.$output_plot_row.', fill=value)) +
12296 geom_tile() +
12297 scale_fill_viridis(discrete=FALSE) +
12298 coord_equal() +
12299 facet_wrap(~trait_type, ncol='.scalar(@sorted_trait_names_6).');
12300 ggsave(\''.$env_effects_sim_figure_tempfile_6_env6.'\', arrangeGrob(gg_env6, gg_p_sim6, gg_eff_sim6, nrow=1), device=\'png\', width=25, height=25, units=\'in\');
12301 "';
12302 # print STDERR Dumper $cmd_spatialenvsim_plot_6_env6;
12303 my $status_spatialenvsim_plot_6_env6 = system($cmd_spatialenvsim_plot_6_env6);
12304 push @$spatial_effects_plots, [$env_effects_sim_figure_tempfile_string_6_env6, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_env6effheatmap_"."envvar_".$env_variance_percent."_".$iterations];
12305 };
12306
12307 eval {
12308 my @sorted_germplasm_names = sort keys %unique_accessions;
12309
12310 my @original_blup_vals_6;
12311 my ($effects_original_line_chart_tempfile_fh_6, $effects_original_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12312 open(my $F_pheno, ">", $effects_original_line_chart_tempfile_6) || die "Can't open file ".$effects_original_line_chart_tempfile_6;
12313 print $F_pheno "germplasmName,time,value\n";
12314 foreach my $p (@sorted_germplasm_names) {
12315 foreach my $t (@sorted_trait_names_6) {
12316 my $val = $result_blup_data_original_6->{$p}->{$t}->[0];
12317 my @row = ($p, $t, $val);
12318 push @original_blup_vals_6, $val;
12319 my $line = join ',', @row;
12320 print $F_pheno "$line\n";
12321 }
12322 }
12323 close($F_pheno);
12324
12325 my $original_blup_stat_6 = Statistics::Descriptive::Full->new();
12326 $original_blup_stat_6->add_data(@original_blup_vals_6);
12327 my $sig_original_blup_6 = $original_blup_stat_6->variance();
12328
12329 my @altered_blups_vals_6;
12330 my ($effects_altered_line_chart_tempfile_fh_6, $effects_altered_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12331 open($F_pheno, ">", $effects_altered_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_line_chart_tempfile_6;
12332 print $F_pheno "germplasmName,time,value\n";
12333 foreach my $p (@sorted_germplasm_names) {
12334 foreach my $t (@sorted_trait_names_6) {
12335 my $val = $result_blup_data_altered_6->{$p}->{$t}->[0];
12336 my @row = ($p, $t, $val);
12337 my $line = join ',', @row;
12338 print $F_pheno "$line\n";
12339 push @altered_blups_vals_6, $val;
12340 }
12341 }
12342 close($F_pheno);
12343
12344 my $altered_blup_stat_6 = Statistics::Descriptive::Full->new();
12345 $altered_blup_stat_6->add_data(@altered_blups_vals_6);
12346 my $sig_altered_blup_6 = $altered_blup_stat_6->variance();
12347
12348 my @sim1_blup_vals_6;
12349 my ($effects_altered_env1_line_chart_tempfile_fh_6, $effects_altered_env1_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12350 open($F_pheno, ">", $effects_altered_env1_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env1_line_chart_tempfile_6;
12351 print $F_pheno "germplasmName,time,value\n";
12352 foreach my $p (@sorted_germplasm_names) {
12353 foreach my $t (@sorted_trait_names_6) {
12354 my $val = $result_blup_data_altered_env_1_6->{$p}->{$t}->[0];
12355 my @row = ($p, $t, $val);
12356 my $line = join ',', @row;
12357 print $F_pheno "$line\n";
12358 push @sim1_blup_vals_6, $val;
12359 }
12360 }
12361 close($F_pheno);
12362
12363 my $sim1_blup_stat_6 = Statistics::Descriptive::Full->new();
12364 $sim1_blup_stat_6->add_data(@sim1_blup_vals_6);
12365 my $sig_sim1_blup_6 = $sim1_blup_stat_6->variance();
12366
12367 my @sim2_blup_vals_6;
12368 my ($effects_altered_env2_line_chart_tempfile_fh_6, $effects_altered_env2_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12369 open($F_pheno, ">", $effects_altered_env2_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env2_line_chart_tempfile_6;
12370 print $F_pheno "germplasmName,time,value\n";
12371 foreach my $p (@sorted_germplasm_names) {
12372 foreach my $t (@sorted_trait_names_6) {
12373 my $val = $result_blup_data_altered_env_2_6->{$p}->{$t}->[0];
12374 my @row = ($p, $t, $val);
12375 my $line = join ',', @row;
12376 print $F_pheno "$line\n";
12377 push @sim2_blup_vals_6, $val;
12378 }
12379 }
12380 close($F_pheno);
12381
12382 my $sim2_blup_stat_6 = Statistics::Descriptive::Full->new();
12383 $sim2_blup_stat_6->add_data(@sim2_blup_vals_6);
12384 my $sig_sim2_blup_6 = $sim2_blup_stat_6->variance();
12385
12386 my @sim3_blup_vals_6;
12387 my ($effects_altered_env3_line_chart_tempfile_fh_6, $effects_altered_env3_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12388 open($F_pheno, ">", $effects_altered_env3_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env3_line_chart_tempfile_6;
12389 print $F_pheno "germplasmName,time,value\n";
12390 foreach my $p (@sorted_germplasm_names) {
12391 foreach my $t (@sorted_trait_names_6) {
12392 my $val = $result_blup_data_altered_env_3_6->{$p}->{$t}->[0];
12393 my @row = ($p, $t, $val);
12394 my $line = join ',', @row;
12395 print $F_pheno "$line\n";
12396 push @sim3_blup_vals_6, $val;
12397 }
12398 }
12399 close($F_pheno);
12400
12401 my $sim3_blup_stat_6 = Statistics::Descriptive::Full->new();
12402 $sim3_blup_stat_6->add_data(@sim3_blup_vals_6);
12403 my $sig_sim3_blup_6 = $sim3_blup_stat_6->variance();
12404
12405 my @sim4_blup_vals_6;
12406 my ($effects_altered_env4_line_chart_tempfile_fh_6, $effects_altered_env4_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12407 open($F_pheno, ">", $effects_altered_env4_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env4_line_chart_tempfile_6;
12408 print $F_pheno "germplasmName,time,value\n";
12409 foreach my $p (@sorted_germplasm_names) {
12410 foreach my $t (@sorted_trait_names_6) {
12411 my $val = $result_blup_data_altered_env_4_6->{$p}->{$t}->[0];
12412 my @row = ($p, $t, $val);
12413 my $line = join ',', @row;
12414 print $F_pheno "$line\n";
12415 push @sim4_blup_vals_6, $val;
12416 }
12417 }
12418 close($F_pheno);
12419
12420 my $sim4_blup_stat_6 = Statistics::Descriptive::Full->new();
12421 $sim4_blup_stat_6->add_data(@sim4_blup_vals_6);
12422 my $sig_sim4_blup_6 = $sim4_blup_stat_6->variance();
12423
12424 my @sim5_blup_vals_6;
12425 my ($effects_altered_env5_line_chart_tempfile_fh_6, $effects_altered_env5_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12426 open($F_pheno, ">", $effects_altered_env5_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env5_line_chart_tempfile_6;
12427 print $F_pheno "germplasmName,time,value\n";
12428 foreach my $p (@sorted_germplasm_names) {
12429 foreach my $t (@sorted_trait_names_6) {
12430 my $val = $result_blup_data_altered_env_5_6->{$p}->{$t}->[0];
12431 my @row = ($p, $t, $val);
12432 my $line = join ',', @row;
12433 print $F_pheno "$line\n";
12434 push @sim5_blup_vals_6, $val;
12435 }
12436 }
12437 close($F_pheno);
12438
12439 my $sim5_blup_stat_6 = Statistics::Descriptive::Full->new();
12440 $sim5_blup_stat_6->add_data(@sim5_blup_vals_6);
12441 my $sig_sim5_blup_6 = $sim5_blup_stat_6->variance();
12442
12443 my @sim6_blup_vals_6;
12444 my ($effects_altered_env6_line_chart_tempfile_fh_6, $effects_altered_env6_line_chart_tempfile_6) = tempfile("drone_stats_XXXXX", DIR=> $tmp_stats_dir);
12445 open($F_pheno, ">", $effects_altered_env6_line_chart_tempfile_6) || die "Can't open file ".$effects_altered_env6_line_chart_tempfile_6;
12446 print $F_pheno "germplasmName,time,value\n";
12447 foreach my $p (@sorted_germplasm_names) {
12448 foreach my $t (@sorted_trait_names_6) {
12449 my $val = $result_blup_data_altered_env_6_6->{$p}->{$t}->[0];
12450 my @row = ($p, $t, $val);
12451 my $line = join ',', @row;
12452 print $F_pheno "$line\n";
12453 push @sim6_blup_vals_6, $val;
12454 }
12455 }
12456 close($F_pheno);
12457
12458 my $sim6_blup_stat_6 = Statistics::Descriptive::Full->new();
12459 $sim6_blup_stat_6->add_data(@sim6_blup_vals_6);
12460 my $sig_sim6_blup_6 = $sim6_blup_stat_6->variance();
12461
12462 my @set = ('0' ..'9', 'A' .. 'F');
12463 my @colors;
12464 for (1..scalar(@sorted_germplasm_names)) {
12465 my $str = join '' => map $set[rand @set], 1 .. 6;
12466 push @colors, '#'.$str;
12467 }
12468 my $color_string = join '\',\'', @colors;
12469
12470 my $genetic_effects_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12471 $genetic_effects_figure_tempfile_string .= '.png';
12472 my $genetic_effects_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_figure_tempfile_string;
12473
12474 my $genetic_effects_alt_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12475 $genetic_effects_alt_figure_tempfile_string .= '.png';
12476 my $genetic_effects_alt_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_figure_tempfile_string;
12477
12478 my $genetic_effects_alt_env1_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12479 $genetic_effects_alt_env1_figure_tempfile_string .= '.png';
12480 my $genetic_effects_alt_env1_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env1_figure_tempfile_string;
12481
12482 my $genetic_effects_alt_env2_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12483 $genetic_effects_alt_env2_figure_tempfile_string .= '.png';
12484 my $genetic_effects_alt_env2_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env2_figure_tempfile_string;
12485
12486 my $genetic_effects_alt_env3_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12487 $genetic_effects_alt_env3_figure_tempfile_string .= '.png';
12488 my $genetic_effects_alt_env3_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env3_figure_tempfile_string;
12489
12490 my $genetic_effects_alt_env4_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12491 $genetic_effects_alt_env4_figure_tempfile_string .= '.png';
12492 my $genetic_effects_alt_env4_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env4_figure_tempfile_string;
12493
12494 my $genetic_effects_alt_env5_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12495 $genetic_effects_alt_env5_figure_tempfile_string .= '.png';
12496 my $genetic_effects_alt_env5_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env5_figure_tempfile_string;
12497
12498 my $genetic_effects_alt_env6_figure_tempfile_string = $c->tempfile( TEMPLATE => 'tmp_drone_statistics/figureXXXX');
12499 $genetic_effects_alt_env6_figure_tempfile_string .= '.png';
12500 my $genetic_effects_alt_env6_figure_tempfile = $c->config->{basepath}."/".$genetic_effects_alt_env6_figure_tempfile_string;
12501
12502 my $cmd_gen_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12503 mat <- fread(\''.$effects_original_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12504 mat\$time <- as.numeric(as.character(mat\$time));
12505 options(device=\'png\');
12506 par();
12507 sp <- ggplot(mat, aes(x = time, y = value)) +
12508 geom_line(aes(color = germplasmName), size = 1) +
12509 scale_fill_manual(values = c(\''.$color_string.'\')) +
12510 theme_minimal();
12511 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12512 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12513 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12514 sp <- sp + labs(title = \'Original Genetic Effects\');';
12515 if (scalar(@sorted_germplasm_names) > 100) {
12516 $cmd_gen_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12517 }
12518 $cmd_gen_plot .= 'ggsave(\''.$genetic_effects_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12519 "';
12520 print STDERR Dumper $cmd_gen_plot;
12521 my $status_gen_plot = system($cmd_gen_plot);
12522 push @$spatial_effects_plots, [$genetic_effects_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_efforigline_"."envvar_".$env_variance_percent."_".$iterations];
12523
12524 my $cmd_gen_alt_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12525 mat <- fread(\''.$effects_altered_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12526 mat\$time <- as.numeric(as.character(mat\$time));
12527 options(device=\'png\');
12528 par();
12529 sp <- ggplot(mat, aes(x = time, y = value)) +
12530 geom_line(aes(color = germplasmName), size = 1) +
12531 scale_fill_manual(values = c(\''.$color_string.'\')) +
12532 theme_minimal();
12533 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12534 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12535 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12536 sp <- sp + labs(title = \'Altered Genetic Effects\');';
12537 if (scalar(@sorted_germplasm_names) > 100) {
12538 $cmd_gen_alt_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12539 }
12540 $cmd_gen_alt_plot .= 'ggsave(\''.$genetic_effects_alt_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12541 "';
12542 print STDERR Dumper $cmd_gen_alt_plot;
12543 my $status_gen_alt_plot = system($cmd_gen_alt_plot);
12544 push @$spatial_effects_plots, [$genetic_effects_alt_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltline_"."envvar_".$env_variance_percent."_".$iterations];
12545
12546 my $cmd_gen_env1_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12547 mat <- fread(\''.$effects_altered_env1_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12548 mat\$time <- as.numeric(as.character(mat\$time));
12549 options(device=\'png\');
12550 par();
12551 sp <- ggplot(mat, aes(x = time, y = value)) +
12552 geom_line(aes(color = germplasmName), size = 1) +
12553 scale_fill_manual(values = c(\''.$color_string.'\')) +
12554 theme_minimal();
12555 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12556 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12557 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12558 sp <- sp + labs(title = \'SimLinear Genetic Effects\');';
12559 if (scalar(@sorted_germplasm_names) > 100) {
12560 $cmd_gen_env1_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12561 }
12562 $cmd_gen_env1_plot .= 'ggsave(\''.$genetic_effects_alt_env1_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12563 "';
12564 print STDERR Dumper $cmd_gen_env1_plot;
12565 my $status_gen_env1_plot = system($cmd_gen_env1_plot);
12566 push @$spatial_effects_plots, [$genetic_effects_alt_env1_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv1line_"."envvar_".$env_variance_percent."_".$iterations];
12567
12568 my $cmd_gen_env2_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12569 mat <- fread(\''.$effects_altered_env2_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12570 mat\$time <- as.numeric(as.character(mat\$time));
12571 options(device=\'png\');
12572 par();
12573 sp <- ggplot(mat, aes(x = time, y = value)) +
12574 geom_line(aes(color = germplasmName), size = 1) +
12575 scale_fill_manual(values = c(\''.$color_string.'\')) +
12576 theme_minimal();
12577 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12578 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12579 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12580 sp <- sp + labs(title = \'Sim1DN Genetic Effects\');';
12581 if (scalar(@sorted_germplasm_names) > 100) {
12582 $cmd_gen_env2_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12583 }
12584 $cmd_gen_env2_plot .= 'ggsave(\''.$genetic_effects_alt_env2_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12585 "';
12586 print STDERR Dumper $cmd_gen_env2_plot;
12587 my $status_gen_env2_plot = system($cmd_gen_env2_plot);
12588 push @$spatial_effects_plots, [$genetic_effects_alt_env2_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv2line_"."envvar_".$env_variance_percent."_".$iterations];
12589
12590 my $cmd_gen_env3_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12591 mat <- fread(\''.$effects_altered_env3_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12592 mat\$time <- as.numeric(as.character(mat\$time));
12593 options(device=\'png\');
12594 par();
12595 sp <- ggplot(mat, aes(x = time, y = value)) +
12596 geom_line(aes(color = germplasmName), size = 1) +
12597 scale_fill_manual(values = c(\''.$color_string.'\')) +
12598 theme_minimal();
12599 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12600 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12601 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12602 sp <- sp + labs(title = \'Sim2DN Genetic Effects\');';
12603 if (scalar(@sorted_germplasm_names) > 100) {
12604 $cmd_gen_env3_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12605 }
12606 $cmd_gen_env3_plot .= 'ggsave(\''.$genetic_effects_alt_env3_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12607 "';
12608 print STDERR Dumper $cmd_gen_env3_plot;
12609 my $status_gen_env3_plot = system($cmd_gen_env3_plot);
12610 push @$spatial_effects_plots, [$genetic_effects_alt_env3_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv3line_"."envvar_".$env_variance_percent."_".$iterations];
12611
12612 my $cmd_gen_env4_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12613 mat <- fread(\''.$effects_altered_env4_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12614 mat\$time <- as.numeric(as.character(mat\$time));
12615 options(device=\'png\');
12616 par();
12617 sp <- ggplot(mat, aes(x = time, y = value)) +
12618 geom_line(aes(color = germplasmName), size = 1) +
12619 scale_fill_manual(values = c(\''.$color_string.'\')) +
12620 theme_minimal();
12621 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12622 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12623 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12624 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
12625 if (scalar(@sorted_germplasm_names) > 100) {
12626 $cmd_gen_env4_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12627 }
12628 $cmd_gen_env4_plot .= 'ggsave(\''.$genetic_effects_alt_env4_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12629 "';
12630 print STDERR Dumper $cmd_gen_env4_plot;
12631 my $status_gen_env4_plot = system($cmd_gen_env4_plot);
12632 push @$spatial_effects_plots, [$genetic_effects_alt_env4_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv4line_"."envvar_".$env_variance_percent."_".$iterations];
12633
12634 my $cmd_gen_env5_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12635 mat <- fread(\''.$effects_altered_env5_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12636 mat\$time <- as.numeric(as.character(mat\$time));
12637 options(device=\'png\');
12638 par();
12639 sp <- ggplot(mat, aes(x = time, y = value)) +
12640 geom_line(aes(color = germplasmName), size = 1) +
12641 scale_fill_manual(values = c(\''.$color_string.'\')) +
12642 theme_minimal();
12643 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12644 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12645 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12646 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
12647 if (scalar(@sorted_germplasm_names) > 100) {
12648 $cmd_gen_env5_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12649 }
12650 $cmd_gen_env5_plot .= 'ggsave(\''.$genetic_effects_alt_env5_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12651 "';
12652 print STDERR Dumper $cmd_gen_env5_plot;
12653 my $status_gen_env5_plot = system($cmd_gen_env5_plot);
12654 push @$spatial_effects_plots, [$genetic_effects_alt_env5_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv5line_"."envvar_".$env_variance_percent."_".$iterations];
12655
12656 my $cmd_gen_env6_plot = 'R -e "library(data.table); library(ggplot2); library(GGally); library(gridExtra);
12657 mat <- fread(\''.$effects_altered_env6_line_chart_tempfile_6.'\', header=TRUE, sep=\',\');
12658 mat\$time <- as.numeric(as.character(mat\$time));
12659 options(device=\'png\');
12660 par();
12661 sp <- ggplot(mat, aes(x = time, y = value)) +
12662 geom_line(aes(color = germplasmName), size = 1) +
12663 scale_fill_manual(values = c(\''.$color_string.'\')) +
12664 theme_minimal();
12665 sp <- sp + guides(shape = guide_legend(override.aes = list(size = 0.5)));
12666 sp <- sp + guides(color = guide_legend(override.aes = list(size = 0.5)));
12667 sp <- sp + theme(legend.title = element_text(size = 3), legend.text = element_text(size = 3));
12668 sp <- sp + labs(title = \'SimRandom Genetic Effects\');';
12669 if (scalar(@sorted_germplasm_names) > 100) {
12670 $cmd_gen_env6_plot .= 'sp <- sp + theme(legend.position = \'none\');';
12671 }
12672 $cmd_gen_env6_plot .= 'ggsave(\''.$genetic_effects_alt_env6_figure_tempfile.'\', sp, device=\'png\', width=12, height=6, units=\'in\');
12673 "';
12674 print STDERR Dumper $cmd_gen_env6_plot;
12675 my $status_gen_env6_plot = system($cmd_gen_env6_plot);
12676 push @$spatial_effects_plots, [$genetic_effects_alt_env6_figure_tempfile_string, $statistics_select.$sim_env_change_over_time.$correlation_between_times."_effaltenv6line_"."envvar_".$env_variance_percent."_".$iterations];
12677 };
12678
12679 %trait_name_encoder = %trait_name_encoder_6;
12680 %trait_to_time_map = %trait_to_time_map_6;
12681
12682 push @$env_varcomps, {
12683 type => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times : Iteration $iterations",
12684 statistics_select => "$statistics_select: Env Variance $env_variance_percent : SimCorrelation: $correlation_between_times",
12685 correlation_between_times => $correlation_between_times,
12686 env_variance => $env_variance_percent,
12687 original => {
12688 covariance => $varcomp_original_array_6,
12689 cv_1 => $result_cv_original_6,
12690 cv_2 => $result_cv_2_original_6
12691 },
12692 altered => {
12693 covariance => $varcomp_altered_array_6,
12694 cv_1 => $result_cv_altered_6,
12695 cv_2 => $result_cv_2_altered_6
12696 },
12697 env_linear => {
12698 covariance => $varcomp_altered_array_env_1_6,
12699 cv_1 => $result_cv_altered_env_1_6,
12700 cv_2 => $result_cv_2_altered_env_1_6
12701 },
12702 env_1DN => {
12703 covariance => $varcomp_altered_array_env_2_6,
12704 cv_1 => $result_cv_altered_env_2_6,
12705 cv_2 => $result_cv_2_altered_env_2_6
12706 },
12707 env_2DN => {
12708 covariance => $varcomp_altered_array_env_3_6,
12709 cv_1 => $result_cv_altered_env_3_6,
12710 cv_2 => $result_cv_2_altered_env_3_6
12711 },
12712 env_random => {
12713 covariance => $varcomp_altered_array_env_4_6,
12714 cv_1 => $result_cv_altered_env_4_6,
12715 cv_2 => $result_cv_2_altered_env_4_6
12716 },
12717 env_ar1xar1 => {
12718 covariance => $varcomp_altered_array_env_5_6,
12719 cv_1 => $result_cv_altered_env_5_6,
12720 cv_2 => $result_cv_2_altered_env_5_6
12721 },
12722 env_realdata => {
12723 covariance => $varcomp_altered_array_env_6_6,
12724 cv_1 => $result_cv_altered_env_6_6,
12725 cv_2 => $result_cv_2_altered_env_6_6
12726 }
12727 };
12728 }
12729
12730 }
12731 }
12732
12733 foreach my $t (keys %$env_corr_res) {
12734 my $vals = $env_corr_res->{$t}->{values};
12735 my $env_corr_res_stat = Statistics::Descriptive::Full->new();
12736 $env_corr_res_stat->add_data(@$vals);
12737 $env_corr_res->{$t}->{std} = $env_corr_res_stat->standard_deviation();
12738 $env_corr_res->{$t}->{mean} = $env_corr_res_stat->mean();
12739 }
12740
12741 # print STDERR Dumper $env_corr_res;
12742 # print STDERR Dumper $env_iterations;
12743 # print STDERR Dumper \%trait_name_encoder;
12744 # print STDERR Dumper \%trait_to_time_map;
12745 # print STDERR Dumper $env_varcomps;
12746
12747 my %avg_varcomps;
12748 my @avg_varcomps_display;
12749 my @varcomp_keys = ('original', 'altered', 'env_linear', 'env_1DN', 'env_2DN', 'env_ar1xar1', 'env_random', 'env_realdata');
12750 if ($statistics_select ne 'blupf90_grm_random_regression_gdd_blups' && $statistics_select ne 'blupf90_grm_random_regression_dap_blups' && $statistics_select ne 'airemlf90_grm_random_regression_gdd_blups' && $statistics_select ne 'airemlf90_grm_random_regression_dap_blups') {
12751 foreach (@$env_varcomps) {
12752 my $type = $_->{statistics_select};
12753 foreach my $t (@varcomp_keys) {
12754 foreach my $a (@{$_->{$t}->{covariance}}) {
12755 push @{$avg_varcomps{$type}->{$t}->{$a->[0]}->{vals}}, $a->[1];
12756 }
12757 }
12758 }
12759 # print STDERR Dumper \%avg_varcomps;
12760
12761 my %avg_varcomps_save;
12762 while (my($t, $type_obj) = each %avg_varcomps) {
12763 while (my($type, $level_obj) = each %$type_obj) {
12764 while (my($level, $vals) = each %$level_obj) {
12765 my @values = @{$vals->{vals}};
12766 my $level_rec;
12767 my @level_split = split '\.', $level;
12768 if (scalar(@level_split) == 2) { #For Sommer Varcomp
12769 my ($level_type, $level_times) = @level_split;
12770 my @ar = split '-', $level_times;
12771 my ($t1, $t2) = sort @ar;
12772 # print STDERR Dumper [$t1, $t2];
12773 $level_rec = $level_type.".".$t1."-".$t2;
12774 }
12775 else { #For ASREML-R Varcomp
12776 $level_rec = $level;
12777 }
12778 my $stat = Statistics::Descriptive::Full->new();
12779 $stat->add_data(@values);
12780 my $std = $stat->standard_deviation();
12781 my $mean = $stat->mean();
12782 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{std} = $std;
12783 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{mean} = $mean;
12784 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{vals} = \@values;
12785 push @avg_varcomps_display, {
12786 type => $t,
12787 type_scenario => $type,
12788 level => $level_rec,
12789 vals => $vals->{vals},
12790 std => $std,
12791 mean => $mean
12792 };
12793 }
12794 }
12795 }
12796 %avg_varcomps = %avg_varcomps_save;
12797
12798 my @potential_times;
12799 #Sommer
12800 foreach (keys %trait_name_encoder) {
12801 push @potential_times, "t$_";
12802 }
12803 #ASREML-R
12804 foreach (values %trait_name_encoder) {
12805 push @potential_times, $_;
12806 }
12807
12808 while (my($t, $type_obj) = each %avg_varcomps) {
12809 while (my($type, $level_obj) = each %$type_obj) {
12810 my @h_values;
12811 foreach my $time (@potential_times) {
12812 #Sommer varcomps
12813 if (exists($avg_varcomps{$t}->{$type}->{"u:id.$time-$time"}->{mean}) && exists($avg_varcomps{$t}->{$type}->{"u:units.$time-$time"}->{mean})) {
12814 my $g = $avg_varcomps{$t}->{$type}->{"u:id.$time-$time"}->{mean};
12815 my $r = $avg_varcomps{$t}->{$type}->{"u:units.$time-$time"}->{mean};
12816 my $h = $g + $r == 0 ? 0 : $g/($g + $r);
12817 push @h_values, $h;
12818 push @avg_varcomps_display, {
12819 type => $t,
12820 type_scenario => $type,
12821 level => "h2-$time",
12822 vals => [$h],
12823 std => 0,
12824 mean => $h
12825 };
12826 }
12827 #ASREML-R multivariate + univariate
12828 elsif (exists($avg_varcomps{$t}->{$type}->{"trait:vm(id_factor, geno_mat_3col)!trait_$time:$time"}->{mean}) && (exists($avg_varcomps{$t}->{$type}->{"units:trait!trait_$time:$time"}->{mean}) || exists($avg_varcomps{$t}->{$type}->{"trait:units!units!trait_$time:$time"}->{mean}) ) ) {
12829 my $g = $avg_varcomps{$t}->{$type}->{"trait:vm(id_factor, geno_mat_3col)!trait_$time:$time"}->{mean};
12830 my $r = $avg_varcomps{$t}->{$type}->{"units:trait!trait_$time:$time"}->{mean} || $avg_varcomps{$t}->{$type}->{"trait:units!units!trait_$time:$time"}->{mean};
12831 my $h = $g + $r == 0 ? 0 : $g/($g + $r);
12832 push @h_values, $h;
12833 push @avg_varcomps_display, {
12834 type => $t,
12835 type_scenario => $type,
12836 level => "h2-$time",
12837 vals => [$h],
12838 std => 0,
12839 mean => $h
12840 };
12841 }
12842 }
12843 my $stat = Statistics::Descriptive::Full->new();
12844 $stat->add_data(@h_values);
12845 my $std = $stat->standard_deviation();
12846 my $mean = $stat->mean();
12847 push @avg_varcomps_display, {
12848 type => $t,
12849 type_scenario => $type,
12850 level => "h2-avg",
12851 vals => \@h_values,
12852 std => $std,
12853 mean => $mean
12854 };
12855 }
12856 }
12857 }
12858 #AIREMLF90 RR
12859 else {
12860 foreach (@$env_varcomps) {
12861 my $type = $_->{statistics_select};
12862 foreach my $t (@varcomp_keys) {
12863 my $res = $_->{$t}->{residual};
12864 my $genetic_line = 1;
12865 foreach $a (@{$_->{$t}->{genetic_covariance}}) {
12866 push @{$avg_varcomps{$type}->{$t}->{genetic_covariance}->{$genetic_line}->{vals}}, $a;
12867 $genetic_line++;
12868 }
12869 my $env_line = 1;
12870 foreach $a (@{$_->{$t}->{env_covariance}}) {
12871 push @{$avg_varcomps{$type}->{$t}->{env_covariance}->{$env_line}->{vals}}, $a;
12872 $env_line++;
12873 }
12874 my $hg_line = 1;
12875 foreach $a (@{$_->{$t}->{genetic_covariance}}) {
12876 my $hg = $a + $res == 0 ? 0 : $a/($a + $res);
12877 push @{$avg_varcomps{$type}->{$t}->{h2_coeff}->{$hg_line}->{vals}}, $hg;
12878 $hg_line++;
12879 }
12880 my $he_line = 1;
12881 foreach $a (@{$_->{$t}->{env_covariance}}) {
12882 my $he = $a + $res == 0 ? 0 : $a/($a + $res);
12883 push @{$avg_varcomps{$type}->{$t}->{env2_coeff}->{$he_line}->{vals}}, $he;
12884 $he_line++;
12885 }
12886 my $genetic_corr_line = 1;
12887 foreach $a (@{$_->{$t}->{genetic_correlation}}) {
12888 push @{$avg_varcomps{$type}->{$t}->{genetic_correlation}->{$genetic_corr_line}->{vals}}, $a;
12889 $genetic_corr_line++;
12890 }
12891 my $env_corr_line = 1;
12892 foreach $a (@{$_->{$t}->{env_correlation}}) {
12893 push @{$avg_varcomps{$type}->{$t}->{env_correlation}->{$env_corr_line}->{vals}}, $a;
12894 $env_corr_line++;
12895 }
12896 push @{$avg_varcomps{$type}->{$t}->{residual}->{1}->{vals}}, $res;
12897 }
12898 }
12899 # print STDERR Dumper \%avg_varcomps;
12900
12901 my %avg_varcomps_save;
12902 while (my($t, $type_obj) = each %avg_varcomps) {
12903 while (my($type, $line_obj) = each %$type_obj) {
12904 while (my($var_type, $level_obj) = each %$line_obj) {
12905 while (my($line_num, $vals) = each %$level_obj) {
12906 my $values = $vals->{vals};
12907 my $level_rec = $var_type."_".$line_num;
12908 my $stat = Statistics::Descriptive::Full->new();
12909 $stat->add_data(@$values);
12910 my $std = $stat->standard_deviation();
12911 my $mean = $stat->mean();
12912 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{std} = $std;
12913 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{mean} = $mean;
12914 $avg_varcomps_save{$t}->{$type}->{$level_rec}->{vals} = $values;
12915 push @avg_varcomps_display, {
12916 type => $t,
12917 type_scenario => $type,
12918 level => $level_rec,
12919 vals => $values,
12920 std => $std,
12921 mean => $mean
12922 };
12923 }
12924 }
12925 }
12926 }
12927 %avg_varcomps = %avg_varcomps_save;
12928 }
12929 # print STDERR Dumper \%avg_varcomps;
12930
12931 my %avg_cross_validation;
12932 my @avg_cross_validation_display;
12933 foreach (@$env_varcomps) {
12934 my $type = $_->{statistics_select};
12935 foreach my $t (@varcomp_keys) {
12936 my $cv1 = $_->{$t}->{cv_1};
12937 my $cv2 = $_->{$t}->{cv_2};
12938 my $cv1_values = $cv1->{values};
12939 my $cv2_values = $cv2->{values};
12940 push @{$avg_cross_validation{$type}->{$t}->{cv_1}}, @$cv1_values;
12941 push @{$avg_cross_validation{$type}->{$t}->{cv_2}}, @$cv2_values;
12942 }
12943 }
12944 while (my($t, $type_obj_m) = each %avg_cross_validation) {
12945 while (my($type_scenario, $type_obj) = each %$type_obj_m) {
12946 my $cv1_values = $type_obj->{cv_1};
12947 my $cv2_values = $type_obj->{cv_2};
12948 my $stat_cv1 = Statistics::Descriptive::Full->new();
12949 $stat_cv1->add_data(@$cv1_values);
12950 my $cv1_std = $stat_cv1->standard_deviation();
12951 my $cv1_mean = $stat_cv1->mean();
12952 my $stat_cv2 = Statistics::Descriptive::Full->new();
12953 $stat_cv2->add_data(@$cv2_values);
12954 my $cv2_std = $stat_cv2->standard_deviation();
12955 my $cv2_mean = $stat_cv2->mean();
12956 push @avg_cross_validation_display, {
12957 type => $t,
12958 type_scenario => $type_scenario,
12959 cv1_mean => $cv1_mean,
12960 cv1_std => $cv1_std,
12961 cv2_mean => $cv2_mean,
12962 cv2_std => $cv2_std
12963 };
12964 }
12965 }
12966
12967 my $q_save_res = "SELECT nd_protocolprop_id, value FROM nd_protocolprop WHERE nd_protocol_id=? AND type_id=?;";
12968 my $h_save_res = $schema->storage->dbh()->prepare($q_save_res);
12969 $h_save_res->execute($analytics_protocol_id, $protocolprop_result_type_cvterm_id);
12970 my ($protocol_result_summary_id_select, $value2) = $h_save_res->fetchrow_array();
12971 $protocol_result_summary = $value2 ? decode_json $value2 : [];
12972 $protocol_result_summary_id = $protocol_result_summary_id_select;
12973
12974 push @$protocol_result_summary, {
12975 statistics_select_original => $statistics_select_original,
12976 number_iterations => $number_iterations,
12977 env_iterations => $env_iterations,
12978 env_correlation_results => $env_corr_res,
12979 trait_name_map => \%trait_name_encoder,
12980 trait_to_time_map => \%trait_to_time_map,
12981 env_varcomps => $env_varcomps,
12982 avg_varcomps => \%avg_varcomps,
12983 avg_varcomps_display => \@avg_varcomps_display,
12984 avg_cross_validation => \%avg_cross_validation,
12985 avg_cross_validation_display => \@avg_cross_validation_display
12986 };
12987 my $q2 = "UPDATE nd_protocolprop SET value=? WHERE nd_protocolprop_id=?;";
12988 my $h2 = $schema->storage->dbh()->prepare($q2);
12989 $h2->execute(encode_json $protocol_result_summary, $protocol_result_summary_id);
12990
12991 foreach my $f (@$spatial_effects_plots) {
12992 my $auxiliary_model_file = $c->config->{basepath}.$f->[0];
12993 my $auxiliary_model_file_archive_type = "nicksmixedmodelsanalytics_v1_".$f->[1];
12994 print STDERR "$auxiliary_model_file_archive_type : $auxiliary_model_file\n";
12995
12996 my $model_aux_original_name = basename($auxiliary_model_file);
12997
12998 my $uploader_autoencoder = CXGN::UploadFile->new({
12999 tempfile => $auxiliary_model_file,
13000 subdirectory => $auxiliary_model_file_archive_type,
13001 archive_path => $c->config->{archive_path},
13002 archive_filename => $model_aux_original_name,
13003 timestamp => $timestamp,
13004 user_id => $user_id,
13005 user_role => $user_role
13006 });
13007 my $archived_aux_filename_with_path = $uploader_autoencoder->archive();
13008 my $md5_aux = $uploader_autoencoder->get_md5($archived_aux_filename_with_path);
13009 if (!$archived_aux_filename_with_path) {
13010 return { error => "Could not save file $model_aux_original_name in archive." };
13011 }
13012 print STDERR "Archived Analytics Figure File: $archived_aux_filename_with_path\n";
13013
13014 my $md_row_aux = $metadata_schema->resultset("MdMetadata")->create({create_person_id => $user_id});
13015 my $file_row_aux = $metadata_schema->resultset("MdFiles")->create({
13016 basename => basename($archived_aux_filename_with_path),
13017 dirname => dirname($archived_aux_filename_with_path),
13018 filetype => $auxiliary_model_file_archive_type,
13019 md5checksum => $md5_aux->hexdigest(),
13020 metadata_id => $md_row_aux->metadata_id()
13021 });
13022
13023 my $experiment_files_aux = $phenome_schema->resultset("NdExperimentMdFiles")->create({
13024 nd_experiment_id => $analytics_nd_experiment_id,
13025 file_id => $file_row_aux->file_id()
13026 });
13027 }
13028
13029 foreach my $f (@$spatial_effects_files_store) {
13030 my $auxiliary_model_file = $f->[0];
13031 my $auxiliary_model_file_archive_type = "nicksmixedmodelsanalytics_v1_".$f->[1];
13032 print STDERR "$auxiliary_model_file_archive_type : $auxiliary_model_file\n";
13033
13034 my $model_aux_original_name = basename($auxiliary_model_file);
13035
13036 my $uploader_autoencoder = CXGN::UploadFile->new({
13037 tempfile => $auxiliary_model_file,
13038 subdirectory => $auxiliary_model_file_archive_type,
13039 archive_path => $c->config->{archive_path},
13040 archive_filename => $model_aux_original_name,
13041 timestamp => $timestamp,
13042 user_id => $user_id,
13043 user_role => $user_role
13044 });
13045 my $archived_aux_filename_with_path = $uploader_autoencoder->archive();
13046 my $md5_aux = $uploader_autoencoder->get_md5($archived_aux_filename_with_path);
13047 if (!$archived_aux_filename_with_path) {
13048 return { error => "Could not save file $model_aux_original_name in archive." };
13049 }
13050 print STDERR "Archived Analytics Data File: $archived_aux_filename_with_path\n";
13051
13052 my $md_row_aux = $metadata_schema->resultset("MdMetadata")->create({create_person_id => $user_id});
13053 my $file_row_aux = $metadata_schema->resultset("MdFiles")->create({
13054 basename => basename($archived_aux_filename_with_path),
13055 dirname => dirname($archived_aux_filename_with_path),
13056 filetype => $auxiliary_model_file_archive_type,
13057 md5checksum => $md5_aux->hexdigest(),
13058 metadata_id => $md_row_aux->metadata_id()
13059 });
13060
13061 my $experiment_files_aux = $phenome_schema->resultset("NdExperimentMdFiles")->create({
13062 nd_experiment_id => $analytics_nd_experiment_id,
13063 file_id => $file_row_aux->file_id()
13064 });
13065 }
13066
13067 $c->stash->{rest} = {
13068 analytics_protocol_id => $analytics_protocol_id,
13069 unique_traits => \@sorted_trait_names,
13070 unique_accessions => \@unique_accession_names,
13071 unique_plots => \@unique_plot_names,
13072 statistics_select => $statistics_select,
13073 grm_file => $grm_file,
13074 stats_tempfile => $stats_tempfile,
13075 blupf90_grm_file => $grm_rename_tempfile,
13076 blupf90_param_file => $parameter_tempfile,
13077 blupf90_training_file => $stats_tempfile_2,
13078 blupf90_permanent_environment_structure_file => $permanent_environment_structure_tempfile,
13079 yhat_residual_tempfile => $yhat_residual_tempfile,
13080 rr_genetic_coefficients => $coeff_genetic_tempfile,
13081 rr_pe_coefficients => $coeff_pe_tempfile,
13082 blupf90_solutions => $blupf90_solutions_tempfile,
13083 stats_out_tempfile => $stats_out_tempfile,
13084 stats_out_tempfile_string => $stats_out_tempfile_string,
13085 stats_out_htp_rel_tempfile_out_string => $stats_out_htp_rel_tempfile_out_string,
13086 stats_out_tempfile_col => $stats_out_tempfile_col,
13087 stats_out_tempfile_row => $stats_out_tempfile_row,
13088 analysis_model_type => $statistics_select,
13089 application_name => "NickMorales Mixed Models Analytics",
13090 application_version => "V1.01",
13091 field_trial_design => $field_trial_design,
13092 spatial_effects_plots => $spatial_effects_plots,
13093 simulated_environment_to_effect_correlations => $env_corr_res,
13094 };
13095 }
13096
13097 sub _check_user_login {
13098 my $c = shift;
13099 my $role_check = shift;
13100 my $user_id;
13101 my $user_name;
13102 my $user_role;
13103 my $session_id = $c->req->param("sgn_session_id");
13104
13105 if ($session_id){
13106 my $dbh = $c->dbc->dbh;
13107 my @user_info = CXGN::Login->new($dbh)->query_from_cookie($session_id);
13108 if (!$user_info[0]){
13109 $c->stash->{rest} = {error=>'You must be logged in to do this!'};
13110 $c->detach();
13111 }
13112 $user_id = $user_info[0];
13113 $user_role = $user_info[1];
13114 my $p = CXGN::People::Person->new($dbh, $user_id);
13115 $user_name = $p->get_username;
13116 } else{
13117 if (!$c->user){
13118 $c->stash->{rest} = {error=>'You must be logged in to do this!'};
13119 $c->detach();
13120 }
13121 $user_id = $c->user()->get_object()->get_sp_person_id();
13122 $user_name = $c->user()->get_object()->get_username();
13123 $user_role = $c->user->get_object->get_user_type();
13124 }
13125 if ($role_check && $user_role ne $role_check) {
13126 $c->stash->{rest} = {error=>'You must have permission to do this! Please contact us!'};
13127 $c->detach();
13128 }
13129 return ($user_id, $user_name, $user_role);
13130 }
13131
13132 1;
Sol Genomics Network