Merge pull request #5191 from solgenomics/topic/quality_control
[sgn.git] / lib / CXGN / BrAPI / v2 / ObservationVariables.pm
blob2b824e124c5befb55ccf7e0c2a6f679e2ff65cef
1 package CXGN::BrAPI::v2::ObservationVariables;
3 use Moose;
4 use Data::Dumper;
5 use JSON;
6 use Try::Tiny;
7 use CXGN::Trait;
8 use CXGN::BrAPI::Pagination;
9 use CXGN::BrAPI::JSONResponse;
10 use SGN::Model::Cvterm;
11 use CXGN::BrAPI::v2::ExternalReferences;
12 use CXGN::BrAPI::v2::Methods;
13 use CXGN::BrAPI::v2::Scales;
14 use CXGN::BrAPI::Exceptions::NotFoundException;
16 extends 'CXGN::BrAPI::v2::Common';
18 has 'trait_ontology_cv_id' => (
19 isa => 'Int',
20 is => 'ro',
21 lazy => 1,
22 default => sub {
23 my $self = shift;
24 my $context = SGN::Context->new;
25 my $cv_name = $context->get_conf('trait_ontology_cv_name');
26 # get cv_id for external references
27 my $cv_id = $self->bcs_schema->resultset("Cv::Cv")->find(
29 name => $cv_name
31 { key => 'cv_c1' }
32 )->get_column('cv_id');
33 return $cv_id;
37 sub observation_levels {
38 my $self = shift;
39 my $page_size = $self->page_size;
40 my $page = $self->page;
41 my $status = $self->status;
43 my @data_window;
44 push @data_window, ({
45 levelName => 'rep',
46 levelOrder => 0 },
48 levelName => 'block',
49 levelOrder => 1 },
51 levelName => 'plot',
52 levelOrder => 2 },
54 levelName => 'subplot',
55 levelOrder => 3 },
57 levelName => 'plant',
58 levelOrder => 4 },
60 levelName => 'tissue_sample',
61 levelOrder => 5
62 });
64 my $total_count = 6;
66 my @data_files;
67 my %result = (data=>\@data_window);
68 my $pagination = CXGN::BrAPI::Pagination->pagination_response($total_count,$total_count,$page);
69 return CXGN::BrAPI::JSONResponse->return_success(\%result, $pagination, \@data_files, $status, 'Observation Levels result constructed');
72 sub search {
73 my $self = shift;
74 my $page_size = $self->page_size;
75 my $page = $self->page;
76 my $inputs = shift;
77 my $c = shift;
78 my $status = $self->status;
79 my @classes = $inputs->{traitClasses} ? @{$inputs->{traitClasses}} : ();
80 my @cvterm_names = $inputs->{observationVariableNames} ? @{$inputs->{observationVariableNames}} : ();
81 my @datatypes = $inputs->{datatypes} ? @{$inputs->{datatypes}} : ();
82 my @db_ids = $inputs->{ontologyDbIds} ? @{$inputs->{ontologyDbIds}} : ();
83 my @dbxref_ids = $inputs->{externalReferenceIds} ? @{$inputs->{externalReferenceIds}} : ();
84 my @dbxref_terms = $inputs->{externalReferenceSources} ? @{$inputs->{externalReferenceSources}} : ();
85 my @method_ids = $inputs->{methodDbIds} ? @{$inputs->{methodDbIds}} : ();
86 my @scale_ids = $inputs->{scaleDbIds} ? @{$inputs->{scaleDbIds}} : ();
87 my @study_ids = $inputs->{studyDbIds} ? @{$inputs->{studyDbIds}} : ();
88 my @trait_dbids = $inputs->{traitDbIds} ? @{$inputs->{traitDbIds}} : ();
89 my @trait_ids = $inputs->{observationVariableDbIds} ? @{$inputs->{observationVariableDbIds}} : ();
91 if (scalar(@classes)>0 || scalar(@method_ids)>0 || scalar(@scale_ids)>0){
92 push @$status, { 'error' => 'The following search parameters are not implemented yet: scaleDbId, traitClasses, methodDbId' };
93 my %result;
94 my @data_files;
95 my $pagination = CXGN::BrAPI::Pagination->pagination_response(0,$page_size,$page);
96 return CXGN::BrAPI::JSONResponse->return_success(\%result, $pagination, \@data_files, $status, 'Observationvariable search result constructed');
99 my $join = '';
100 my @and_wheres;
102 if (scalar(@trait_ids)>0){
103 my $trait_ids_sql = join ',', @trait_ids;
104 push @and_wheres, "cvterm.cvterm_id IN ($trait_ids_sql)";
106 if (scalar(@cvterm_names)>0){
107 my @quotedNames;
108 my $dbh = $self->bcs_schema->storage->dbh();
109 for my $name (@cvterm_names) {push @quotedNames, $dbh->quote($name);}
110 my $cvterm_names_sql = join ("," , @quotedNames);
111 push @and_wheres, "cvterm.name IN ($cvterm_names_sql)";
113 if (scalar(@trait_dbids)>0){
114 my $trait_ids_sql = join ',', @trait_dbids;
115 push @and_wheres, "cvterm.cvterm_id IN ($trait_ids_sql)";
117 if (scalar(@db_ids)>0){
118 foreach (@db_ids){
119 push @and_wheres, "db.db_id = '$_'";
123 # External reference id and reference source search
124 if (scalar(@dbxref_ids) > 0 || scalar(@dbxref_terms)>0) {
125 my @sub_and_wheres;
126 my @dbxrefid_where;
127 if (scalar(@dbxref_ids)>0){
128 foreach (@dbxref_ids) {
129 my ($db_name,$acc) = split(/:/, $_);
130 push @dbxrefid_where, "dbxref.accession = '$acc'";
131 push @dbxrefid_where, "db.name = '$db_name'";
134 if(scalar(@dbxrefid_where)>0) {
135 my $dbxref_id_where_str = '('. (join ' AND ', @dbxrefid_where) . ')';
136 push @sub_and_wheres, $dbxref_id_where_str;
139 my @dbxref_term_where;
140 if (scalar(@dbxref_terms)>0) {
141 foreach (@dbxref_terms) {
142 push @dbxref_term_where, "db.name = '$_'";
143 push @dbxref_term_where, "db.description = '$_'";
144 push @dbxref_term_where, "db.url = '$_'";
147 if(scalar(@dbxref_term_where)>0) {
148 my $dbxref_term_where_str = '('. (join ' OR ', @dbxref_term_where) . ')';
149 push @sub_and_wheres, $dbxref_term_where_str;
152 @and_wheres = @sub_and_wheres;
153 # my $sub_and_where_clause = join ' AND ', @sub_and_wheres;
155 # $join = "JOIN (" .
156 # "select cvterm_id, json_agg(json_build_object(" .
157 # "'referenceSource', references_query.reference_source, " .
158 # "'referenceID', references_query.reference_id " .
159 # ")) AS externalReferences " .
160 # "from " .
161 # "(" .
162 # "select cvterm.cvterm_id, db.name as reference_source, dbxref.accession as reference_id " .
163 # "FROM " .
164 # "cvterm " .
165 # "JOIN cvterm_relationship as rel on (rel.subject_id=cvterm.cvterm_id) " .
166 # "JOIN cvterm as reltype on (rel.type_id=reltype.cvterm_id) " .
167 # "JOIN cvterm_dbxref on cvterm.cvterm_id = cvterm_dbxref.cvterm_id " .
168 # "JOIN dbxref on cvterm_dbxref.dbxref_id = dbxref.dbxref_id " .
169 # "JOIN db on dbxref.db_id = db.db_id " .
170 # "where $sub_and_where_clause " .
171 # ") as references_query " .
172 # "group by cvterm_id " .
173 # ") as external_references on external_references.cvterm_id = cvterm.cvterm_id "
176 if (scalar(@datatypes)>0){
177 $join = 'JOIN cvtermprop on (cvterm.cvterm_id=cvtermprop.cvterm_id)';
178 foreach (@datatypes){
179 push @and_wheres, "cvtermprop.value = '$_'";
183 if (scalar(@study_ids)>0){
184 my $trait_ids_sql;
186 foreach my $study_id (@study_ids){
187 my $study_check = $self->bcs_schema->resultset('Project::Project')->find({project_id=>$study_id});
188 if ($study_check) {
189 my $t = CXGN::Trial->new({ bcs_schema => $self->bcs_schema, trial_id => $study_id });
190 my $traits_assayed = $t->get_traits_assayed();
192 foreach (@$traits_assayed){
193 $trait_ids_sql .= ',' . $_->[0] ;
198 $trait_ids_sql =~ s/^,//g;
199 if ($trait_ids_sql){
200 push @and_wheres, "cvterm.cvterm_id IN ($trait_ids_sql)";
201 } else {
202 my @data_files;
203 my $pagination = CXGN::BrAPI::Pagination->pagination_response(0,$page_size,$page);
204 return CXGN::BrAPI::JSONResponse->return_success({data => []}, $pagination, \@data_files, $status, 'Observationvariable search result constructed');
209 push @and_wheres, "reltype.name='VARIABLE_OF'";
211 my $and_where_clause = join ' AND ', @and_wheres;
213 $self->get_query($c, $and_where_clause, $join, 1);
217 sub detail {
218 my $self = shift;
219 my $trait_id = shift;
220 my $c = shift;
222 my $join = '';
223 my $and_where;
225 if ($trait_id){
226 $and_where = $and_where." cvterm.cvterm_id IN ($trait_id)";
229 $self->get_query($c, $and_where, $join, 0);
233 sub get_query {
234 my $self = shift;
235 my $c = shift;
236 my $and_where = shift;
237 my $join = shift;
238 my $array = shift;
239 my $page_size = $self->page_size;
240 my $page = $self->page;
241 my $status = $self->status;
243 my @variables;
244 my %result;
245 my $limit = $page_size;
246 my $offset = $page*$page_size;
247 my $total_count = 0;
249 my $additional_info_type_id = SGN::Model::Cvterm->get_cvterm_row($self->bcs_schema, 'cvterm_additional_info', 'trait_property')->cvterm_id();
251 #TODO: Can try a pivot to speed up this function so we retrieve all at once;
252 my $q = "SELECT cvterm.cvterm_id, cvterm.name, cvterm.definition, db.name, db.db_id, db.url, dbxref.dbxref_id, dbxref.accession, array_agg(cvtermsynonym.synonym ORDER BY CHAR_LENGTH(cvtermsynonym.synonym)) filter (where cvtermsynonym.synonym is not null), cvterm.is_obsolete, additional_info.value, count(cvterm.cvterm_id) OVER() AS full_count FROM cvterm ".
253 "JOIN dbxref USING(dbxref_id) ".
254 "JOIN db using(db_id) ".
255 "LEFT JOIN cvtermsynonym using(cvterm_id) ". # left join to include non-synoynm variables, may break field book due to bug
256 "JOIN cvterm_relationship as rel on (rel.subject_id=cvterm.cvterm_id) ".
257 "JOIN cvterm as reltype on (rel.type_id=reltype.cvterm_id) $join ".
258 "LEFT JOIN cvtermprop as additional_info on (cvterm.cvterm_id = additional_info.cvterm_id and additional_info.type_id = $additional_info_type_id) ".
259 "WHERE $and_where " .
260 "GROUP BY cvterm.cvterm_id, db.name, db.db_id, dbxref.dbxref_id, dbxref.accession, additional_info.value ".
261 "ORDER BY cvterm.name ASC LIMIT $limit OFFSET $offset; " ;
264 my $sth = $self->bcs_schema->storage->dbh->prepare($q);
265 $sth->execute();
266 while (my ($cvterm_id, $cvterm_name, $cvterm_definition, $db_name, $db_id, $db_url, $dbxref_id, $accession, $synonym, $obsolete, $additional_info_string, $count) = $sth->fetchrow_array()) {
267 $total_count = $count;
268 foreach (@$synonym){
269 $_ =~ s/ EXACT \[\]//;
270 $_ =~ s/\"//g;
273 # Get the external references
274 my @references_cvterms = ($cvterm_id);
275 my $references = CXGN::BrAPI::v2::ExternalReferences->new({
276 bcs_schema => $self->bcs_schema,
277 table_name => 'cvterm',
278 table_id_key => 'cvterm_id',
279 id => \@references_cvterms
282 my $additional_info;
283 if (defined $additional_info_string) {
284 $additional_info = decode_json $additional_info_string;
286 #TODO: This is running many queries each time, can make one big query above if need be
287 # Retrieve the trait, which retrieves its scales and methods
288 my $trait = CXGN::Trait->new({
289 bcs_schema => $self->bcs_schema,
290 cvterm_id => $cvterm_id,
291 dbxref_id => $dbxref_id,
292 db_id => $db_id,
293 db => $db_name,
294 accession => $accession,
295 name => $cvterm_name,
296 external_references => $references,
297 additional_info => $additional_info,
298 synonyms => $synonym
301 push @variables, $self->_construct_variable_response($c, $trait);
304 my $pagination;
305 my %result;
306 my @data_files;
308 if ($array) {
309 %result = (data => \@variables);
310 $pagination = CXGN::BrAPI::Pagination->pagination_response($total_count,$page_size,$page);
311 return CXGN::BrAPI::JSONResponse->return_success(\%result, $pagination, \@data_files, $status, 'Observationvariable search result constructed');
312 } else {
313 $pagination = CXGN::BrAPI::Pagination->pagination_response($total_count,$page_size,$page);
314 return CXGN::BrAPI::JSONResponse->return_success(@variables[0], $pagination, \@data_files, $status, 'Observationvariable search result constructed');
319 # TODO: Make validation errors better
320 sub store {
322 my $self = shift;
323 my $data = shift;
324 my $c = shift;
326 my $page_size = $self->page_size;
327 my $page = $self->page;
328 my $schema = $self->bcs_schema();
330 my @variable_ids;
332 my @result;
334 my $coderef = sub {
335 foreach my $params (@{$data}) {
336 my $cvterm_id = $params->{observationVariableDbId} || undef;
337 my $name = $params->{observationVariableName};
338 my $ontology_id = $params->{ontologyReference}{ontologyDbId};
339 my $description = $params->{trait}{traitDescription};
340 my $entity = $params->{trait}{entity};
341 my $attribute = $params->{trait}{attribute};
342 my $synonyms = $params->{synonyms};
343 my $active = $params->{status} ne "archived";
344 my $additional_info = $params->{additionalInfo} || undef;
346 #TODO: Parse this when it initially comes into the brapi controller
347 my $scale = CXGN::BrAPI::v2::Scales->new({
348 bcs_schema => $self->bcs_schema,
349 scale => $params->{scale}
351 my $method = CXGN::BrAPI::v2::Methods->new({
352 bcs_schema => $self->bcs_schema,
353 method => $params->{method}
355 my $external_references = CXGN::BrAPI::v2::ExternalReferences->new({
356 bcs_schema => $self->bcs_schema,
357 external_references => $params->{externalReferences} || [],
358 table_name => "cvterm",
359 table_id_key => "cvterm_id",
360 id => $cvterm_id
362 my $trait = CXGN::Trait->new({ bcs_schema => $self->bcs_schema,
363 cvterm_id => $cvterm_id,
364 name => $name,
365 ontology_id => $ontology_id,
366 definition => $description,
367 entity => $entity,
368 attribute => $attribute,
369 synonyms => $synonyms,
370 external_references => $external_references,
371 method => $method,
372 scale => $scale,
373 additional_info => $additional_info
375 $trait->{active} = $active;
377 my $variable = $trait->store();
378 push @result, $self->_construct_variable_response($c, $variable);
382 try {
383 $schema->txn_do($coderef);
384 } catch {
385 throw $_;
388 my $count = scalar @variable_ids;
389 my $pagination = CXGN::BrAPI::Pagination->pagination_response($count,$page_size,$page);
390 my %data_result = (data => \@result);
391 return CXGN::BrAPI::JSONResponse->return_success( \%data_result, $pagination, undef, $self->status(), $count . " Variables were saved.");
394 sub update {
396 my $self = shift;
397 my $data = shift;
398 my $c = shift;
400 my $schema = $self->bcs_schema();
402 # Check that cvterm that was passed in exists
403 #TODO: This can go away once trait parsed in controller
404 if ($data->{observationVariableDbId}){
405 my ($existing_cvterm) = $schema->resultset("Cv::Cvterm")->find({ cvterm_id => $data->{observationVariableDbId} });
406 if (!defined($existing_cvterm)) {
407 warn "An observationVariableId is required for variable update.";
408 CXGN::BrAPI::Exceptions::NotFoundException->throw({message => 'observationVariableId not specified.'});
412 #TODO: Add a check for these required values
413 my $cvterm_id = $data->{observationVariableDbId} || undef;
414 my $name = $data->{observationVariableName};
415 my $ontology_id = $data->{ontologyReference}{ontologyDbId};
416 my $description = $data->{trait}{traitDescription};
417 my $entity = $data->{trait}{entity};
418 my $attribute = $data->{trait}{attribute};
419 my $synonyms = $data->{synonyms};
420 my $active = $data->{status} ne "archived";
421 my $additional_info = $data->{additionalInfo} || undef;
423 my $scale = CXGN::BrAPI::v2::Scales->new({
424 bcs_schema => $self->bcs_schema,
425 scale => $data->{scale}
427 my $method = CXGN::BrAPI::v2::Methods->new({
428 bcs_schema => $self->bcs_schema,
429 method => $data->{method}
431 my $external_references = CXGN::BrAPI::v2::ExternalReferences->new({
432 bcs_schema => $self->bcs_schema,
433 external_references => $data->{externalReferences} || [],
434 table_name => "cvterm",
435 table_id_key => "cvterm_id",
436 id => $cvterm_id
438 my $trait = CXGN::Trait->new({ bcs_schema => $self->bcs_schema,
439 cvterm_id => $cvterm_id,
440 name => $name,
441 ontology_id => $ontology_id,
442 definition => $description,
443 entity => $entity,
444 attribute => $attribute,
445 synonyms => $synonyms,
446 external_references => $external_references,
447 method => $method,
448 scale => $scale,
449 additional_info => $additional_info
451 $trait->{active} = $active;
453 my $variable = $trait->update();
454 my $pagination = CXGN::BrAPI::Pagination->pagination_response(1,1,1);
455 my $response = $self->_construct_variable_response($c, $variable);
456 return CXGN::BrAPI::JSONResponse->return_success($response, $pagination, undef, $self->status(), "Variable was updated.");
459 sub observation_variable_ontologies {
460 my $self = shift;
461 my $inputs = shift;
462 my $name_spaces = $inputs->{name_spaces};
463 my $ontology_id = $inputs->{ontologyDbId};
464 my $cvprop_types = $inputs->{cvprop_type_names} || [];
465 my $page_size = $self->page_size;
466 my $page = $self->page;
467 my $status = $self->status;
468 my @available;
470 my @composable_cv_prop_types;
471 foreach (@$cvprop_types) {
472 my $composable_cv_type_cvterm_id = SGN::Model::Cvterm->get_cvterm_row($self->bcs_schema, $_, 'composable_cvtypes')->cvterm_id();
473 push @composable_cv_prop_types, $composable_cv_type_cvterm_id;
475 my $composable_cv_prop_sql = "";
476 if (scalar(@composable_cv_prop_types)>0) {
477 $composable_cv_prop_sql = join ("," , @composable_cv_prop_types);
478 $composable_cv_prop_sql = " AND cvprop.type_id IN ($composable_cv_prop_sql)";
481 #Using code pattern from SGN::Controller::AJAX::Onto->roots_GET
482 my $q = "SELECT cvterm.cvterm_id, cvterm.name, cvterm.definition, db.name, db.db_id, dbxref.accession, dbxref.version, dbxref.description, cv.cv_id, cv.name, cv.definition FROM cvterm JOIN dbxref USING(dbxref_id) JOIN db USING(db_id) JOIN cv USING(cv_id) JOIN cvprop USING(cv_id) LEFT JOIN cvterm_relationship ON (cvterm.cvterm_id=cvterm_relationship.subject_id) WHERE cvterm_relationship.subject_id IS NULL AND is_obsolete= 0 AND is_relationshiptype = 0 and db.name=? $composable_cv_prop_sql;";
483 my $sth = $self->bcs_schema->storage->dbh->prepare($q);
484 foreach (@$name_spaces){
485 $sth->execute($_);
486 while (my ($cvterm_id, $cvterm_name, $cvterm_definition, $db_name, $db_id, $dbxref_accession, $dbxref_version, $dbxref_description, $cv_id, $cv_name, $cv_definition) = $sth->fetchrow_array()) {
487 if ( $ontology_id && $ontology_id ne $db_id) { next; }
488 my $info;
489 if($dbxref_description){
490 $info = decode_json($dbxref_description);
492 push @available, {
493 additionalInfo=>{},
494 ontologyDbId=>qq|$db_id|,
495 ontologyName=>$db_name,
496 description=>$cvterm_name,
497 authors=>$info->{authors} ? $info->{authors} : '',
498 version=>$dbxref_version,
499 copyright=>$info->{copyright} ? $info->{copyright} : '',
500 licence=>$info->{licence} ? $info->{licence} : '',
501 documentationURL=>$dbxref_accession,
506 my ($data_window, $pagination) = CXGN::BrAPI::Pagination->paginate_array(\@available,$page_size,$page);
507 my %result = (data=>$data_window);
508 my @data_files;
509 return CXGN::BrAPI::JSONResponse->return_success(\%result, $pagination, \@data_files, $status, 'Ontologies result constructed');
512 sub _construct_variable_response {
513 my $self = shift;
514 my $c = shift;
515 my $variable = shift;
517 my $external_references_json;
518 if (defined($variable->external_references)) {
519 $external_references_json = $variable->external_references->search()->{$variable->cvterm_id};
521 if($c->config->{'brapi_include_CO_xref'}) {
522 push @{ $external_references_json }, {
523 #referenceId => "http://www.cropontology.org/terms/".$variable->db.":".$variable->accession . "/",
524 referenceId => $variable->db.":".$variable->accession,
525 referenceSource => "Crop Ontology"
529 my $method_json;
530 if (defined($variable->method)) { $method_json = $variable->method->method_db();}
531 my $scale_json;
532 if (defined($variable->scale)) { $scale_json = $variable->scale->scale_db();}
533 my @synonyms = $variable->synonyms;
534 my $variable_id = $variable->cvterm_id;
535 my $variable_db_id = $variable->db_id ;
537 my $documentation_links;
538 if($variable->uri){
539 push @$documentation_links, {
540 "URL" => $variable->uri,
541 "type" => "OBO"
545 return {
546 additionalInfo => $variable->additional_info || {},
547 commonCropName => $c->config->{'supportedCrop'},
548 contextOfUse => undef,
549 defaultValue => $variable->default_value,
550 documentationURL => $variable->uri,
551 externalReferences => $external_references_json,
552 growthStage => undef,
553 institution => undef,
554 language => 'eng',
555 method => $method_json,
556 observationVariableDbId => qq|$variable_id|,
557 observationVariableName => $variable->name."|".$variable->db.":".$variable->accession,
558 observationVariablePUI => $variable->db.":".$variable->accession,
559 ontologyReference => {
560 documentationLinks => $documentation_links,
561 ontologyDbId => $variable->db_id ? qq|$variable_db_id| : undef,
562 ontologyName => $variable->db ? $variable->db : undef,
563 version => undef,
565 scale => $scale_json,
566 scientist => undef,
567 status => $variable->get_active_string(),
568 submissionTimestamp => undef,
569 synonyms => @synonyms,
570 trait => {
571 additionalInfo => {},
572 alternativeAbbreviations => undef,
573 attribute => $variable->attribute ? $variable->attribute : undef,
574 attributePUI=> undef,
575 entity => $variable->entity ? $variable->entity : undef,
576 entityPUI=> undef,
577 externalReferences => $external_references_json,
578 mainAbbreviation => undef,
579 ontologyReference => {
580 documentationLinks => $documentation_links,
581 ontologyDbId => $variable->db_id ? qq|$variable_db_id| : undef,
582 ontologyName => $variable->db ? $variable->db : undef,
583 version => undef,
585 status => $variable->get_active_string(),
586 synonyms => @synonyms,
587 traitClass => undef,
588 traitDescription => $variable->definition,
589 traitDbId => qq|$variable_id|,
590 traitName => $variable->name,
591 traitPUI => undef,