src/table.c

   1 /*
   2  *  This library is free software; you can redistribute it and/or
   3  *  modify it under the terms of the GNU Lesser General Public
   4  *  License as published by the Free Software Foundation; either
   5  *  version 2 of the License, or (at your option) any later version.
   6  *
   7  *  This library is distributed in the hope that it will be useful,
   8  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  10  *  Lesser General Public License for more details.
  11  *
  12  *  You should have received a copy of the GNU General Public License
  13  *  along with this program; if not, write to the Free Software
  14  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15  *
  16  *  Copyright (C) 2008 Liam Girdwood
  17  */
  18
  19 #define _GNU_SOURCE             /* for strtof and NAN */
  20
  21 #include <stdlib.h>
  22 #include <stdio.h>
  23 #include <errno.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <math.h>
  27 #include <dirent.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/wait.h>
  31 #include <errno.h>
  32
  33 #include <libastrodb/db.h>
  34 #include <libastrodb/astro.h>
  35 #include <libastrodb/table.h>
  36 #include <libastrodb/wget.h>
  37 #include <libastrodb/gunzip.h>
  38 #include <libastrodb/adbstdio.h>
  39
  40
  41 #define FLOAT_SIZE '6'  /* CDS F string size and below for float else double */
  42 #define FILE_LEN        1024
  43
  44 struct _tile_dim {
  45         int records;            /*!< Number of records in dataset */
  46         int ra, dec, mag;       /*!< Stride size */
  47 };
  48
  49 /*
  50  * Table containing default dataset 3d tile dimensions
  51  */
  52 static struct _tile_dim tile_dim[] = {
  53         {10000, 6, 3, 20},
  54         {100000, 18, 9, 20},
  55         {500000, 36, 18, 20},
  56         {1000000, 72, 36, 20},
  57         {10000000, 180, 90, 20},
  58 };
  59
  60 /*! \fn static int init_table(astrodb_table * table, int ra_stride, int dec_stride, int mag_stride)
  61  * \brief Initialise a dataset
  62  */
  63 static int init_table(struct astrodb_table *table,
  64                         int ra_stride, int dec_stride, int mag_stride)
  65 {
  66         struct astrodb_slist **slist, **hash_object;
  67         struct astrodb_tile_status *status;
  68         int i = 0;
  69
  70         /* do we use the default tile dimensions */
  71         if (ra_stride == 0) {
  72                 for (; i < astrodb_size(tile_dim); i++) {
  73                         if (table->table_size <= tile_dim[i].records)
  74                                 break;
  75                 }
  76                 table->ra_stride = tile_dim[i].ra;
  77                 table->dec_stride = tile_dim[i].dec;
  78                 table->mag_stride = tile_dim[i].mag;
  79         } else {
  80                 table->ra_stride = ra_stride;
  81                 table->dec_stride = dec_stride;
  82                 table->mag_stride = mag_stride;
  83         }
  84
  85         table->array_size = table->ra_stride * table->dec_stride *
  86                 table->mag_stride * sizeof(struct astrodb_slist *);
  87         table->status_size = table->ra_stride * table->dec_stride *
  88                 table->mag_stride * sizeof(struct astrodb_tile_status);
  89         table->no_tiles = table->ra_stride * table->dec_stride *
  90                 table->mag_stride;
  91
  92         table->ra_stride_size =
  93                 fabs((table->db->ra_max - table->db->ra_min) / table->ra_stride);
  94         table->dec_stride_size =
  95                 fabs((table->db->dec_max - table->db->dec_min) / table->dec_stride);
  96         table->mag_stride_size =
  97                 fabs((table->db->mag_bright - table->db->mag_faint) / table->mag_stride);
  98
  99         slist = (struct astrodb_slist **) calloc(1, table->array_size);
 100         if (slist == 0) {
 101                 free(table);
 102                 return -ENOMEM;
 103         }
 104         table->objects = slist;
 105
 106         status = (struct astrodb_tile_status *) calloc(1, table->status_size);
 107         if (status == 0) {
 108                 free(slist);
 109                 free(table);
 110                 return -ENOMEM;
 111         }
 112         table->status = status;
 113
 114         for (i = 0; i < table->num_hmaps; i++) {
 115                 hash_object = (struct astrodb_slist **) calloc(1, table->array_size);
 116                 if (hash_object == NULL) {
 117                         free(slist);
 118                         free(status);
 119                         while(i--)
 120                                 free(table->hmaps[i-1].hobjects);
 121
 122                         free(table);
 123                         return -ENOMEM;
 124                 }
 125                 table->hmaps[i].hobjects = hash_object;
 126         }
 127
 128         return 0;
 129 }
 130
 131 /*! \fn static astrodb_table* create_table(astrodb_db*db, astrodb_dlist* desc, astrodb_dlist* files)
 132  * \brief Create a object dataset and populate fields
 133  */
 134 static struct astrodb_table *create_table(struct astrodb_db *db,
 135         struct astrodb_dlist *desc, struct astrodb_dlist *files)
 136 {
 137         struct astrodb_table *table;
 138
 139         table = calloc(1, sizeof(struct  astrodb_table));
 140         if (table == NULL)
 141                 return NULL;
 142
 143         bzero(table, sizeof(struct astrodb_table));
 144         table->byte_description =
 145                 ((struct astrodb_table_info *) files->data)->byte_description;
 146         table->file = ((struct astrodb_table_info *) files->data)->name;
 147         table->table_size = ((struct astrodb_table_info *) files->data)->records;
 148         table->record_size = ((struct astrodb_table_info *) files->data)->length;
 149         table->db = db;
 150         db->num_tables++;
 151         return table;
 152 }
 153
 154
 155
 156 /*! \fn astrodb_table *astrodb_table_create(astrodb_db *db, char *table_name, unsigned int flags)
 157  * \param db Catalog
 158  * \param table_name Dataset name (dataset file name in ReadMe)
 159  * \param flags Dataset creation flags.
 160  * \return A astrodb_table* for success or NULL on error
 161  *
 162  * Creates a dataset object.
 163  */
 164 struct astrodb_table *astrodb_table_create(struct astrodb_db *db,
 165                         char *table_name, unsigned int flags)
 166 {
 167         struct astrodb_dlist *files = db->info->files;
 168         struct astrodb_table *table = NULL;
 169         astrodb_info("Scanning for key fields in table: %s\n", table_name);
 170
 171         while (files) {
 172                 struct astrodb_table_info *table_info =
 173                         (struct  astrodb_table_info *) files->data;
 174                 struct astrodb_dlist *byte_desc = table_info->byte_description;
 175
 176                 /* create a tile array per file for files that have a byte desc */
 177                 if (byte_desc &&
 178                         !strncmp(table_name, table_info->name, strlen(table_name))) {
 179                         int afields, dfields;
 180
 181                         table = create_table(db, byte_desc, files);
 182                         table->init_flags = flags;
 183                         astrodb_info("  scan: %s\n", table_info->name);
 184                         afields = table_scan_afields(table, byte_desc);
 185                         dfields = table_scan_dfields(table, byte_desc);
 186
 187                         if (afields || dfields) {
 188                                 db->tables =
 189                                         astrodb_dlist_append(db->tables,
 190                                                                 (char *) table);
 191                                 return table;
 192                         } else {
 193                                 astrodb_info("failed to find key fields\n");
 194                                 free(table);
 195                                 return NULL;
 196                         }
 197                 }
 198                 files = files->tail;
 199         }
 200
 201         return NULL;
 202 }
 203
 204 /*! \fn int astrodb_table_open(astrodb_table * table, astrodb_progress progress, int ra, int dec, int mag)
 205  * \param table dataset
 206  * \param progress Progress callback
 207  * \param ra RA stride
 208  * \param dec Dec stride
 209  * \param mag Magnitude stride
 210  *
 211  * Initialise a dataset. This will download and import the raw data if necessary.
 212  */
 213 int astrodb_table_open(struct astrodb_table *table, int ra, int dec, int mag)
 214 {
 215         struct stat stat_info;
 216         char file[FILE_LEN];
 217         char pstr[FILE_LEN];
 218         int ret;
 219
 220         sprintf(pstr, "Initialising table %s", table->file);
 221
 222         /* do we have to delete existing dataset */
 223         //if (flags & ADB_REFRESH)
 224         //      delete_table(astrodb_table* table);
 225         table_get_object_insert(table);
 226         ret = init_table(table, ra, dec, mag);
 227         if (ret < 0) {
 228                 astrodb_error("failed to initialise table %d\n", ret);
 229                 return ret;
 230         }
 231
 232         /* check for local binary copy */
 233         sprintf(file, "%s%s%s", table->db->local_path, table->file, ".idx");
 234         ret = stat(file, &stat_info);
 235         if (ret == 0) {
 236                 ret = table_load_hdr(table);
 237                 if (ret < 0) {
 238                         astrodb_error("failed to load binary header %s %d\n",
 239                                 file, ret);
 240                         return ret;
 241                 }
 242                 ret = table_uncache_tiles(table);
 243                 if (ret < 0)
 244                         astrodb_error("failed to uncache table tiles %d\n",
 245                                 ret);
 246                 return ret;
 247         }
 248
 249         /* check for local ASCII copy */
 250         sprintf(file, "%s%s", table->db->local_path, table->file);
 251         ret = stat(file, &stat_info);
 252         if (ret == 0)
 253                 goto import;
 254
 255         /* check for local */
 256
 257         /* local binary or ASCII not available, so download ASCII */
 258         if (!(strncmp("http://", table->db->lib->remote, 7)) ||
 259                 !(strncmp("ftp://", table->db->lib->remote, 6))) {
 260                 ret = table_get_remote_dataset(table);
 261                 if (ret < 0) {
 262                         astrodb_error("failed to get remote table %d\n", ret);
 263                         return ret;
 264                 }
 265         }
 266
 267 import:
 268         if (table->init_flags & ADB_MEM) {
 269                 ret = table_import(table, file);
 270                 if (ret < 0) {
 271                         astrodb_error("failed to import table %s %d\n", file,
 272                                 ret);
 273                         return ret;
 274                 }
 275         }
 276
 277         if (table->init_flags & ADB_FILE) {
 278                 ret = table_save_hdr(table);
 279                 if (ret < 0) {
 280                         astrodb_error("failed to save table schema %d\n", ret);
 281                         return ret;
 282                 }
 283                 ret = table_cache_tiles(table);
 284         }
 285
 286         return ret;
 287 }
 288
 289 /*! \fn void astrodb_table_close(astrodb_table* table)
 290  * \param table dataset
 291  *
 292  * Free dataset resources
 293  */
 294 void astrodb_table_close(struct astrodb_table *table)
 295 {
 296         int i, j;
 297         struct astrodb_db *db = table->db;
 298
 299         /* free hash maps */
 300         for (i = 0; i < table->num_hmaps; i++) {
 301                 for (j = 0; j < table->no_tiles; j++) {
 302
 303                         struct astrodb_slist *slist, *slist_;
 304                         slist_ = *(table->hmaps[i].hobjects + j);
 305
 306                         while (slist_) {
 307                                 slist = slist_;
 308                                 slist_ = slist_->tail;
 309                                 free(slist);
 310                         }
 311                 }
 312                 /* free array */
 313                 free(table->hmaps[i].hobjects);
 314         }
 315
 316         if (table->objects) {
 317                 /* free objects in array */
 318                 for (i = 0; i < table->no_tiles; i++)
 319                         astrodb_slist_foreach_free(*(table->objects + i), NULL, 0);
 320                 /* free array */
 321                 free(table->objects);
 322         }
 323
 324         if (table->status)
 325                 free(table->status);
 326
 327         db->tables = astrodb_dlist_free_object(db->tables, table);
 328 }
 329
 330 /*! \fn int astrodb_table_add_custom_field(astrodb_table* table, char* field)
 331  * \param table dataset
 332  * \param field Field name to add
 333  * \return 0 on success
 334  *
 335  * Add a custom field to a dataset for import.
 336  */
 337 int astrodb_table_add_custom_field(struct astrodb_table *table, char *field)
 338 {
 339         struct astrodb_table_column_info *byte = NULL;
 340         struct astrodb_schema_object *idx = NULL;
 341         struct astrodb_dlist *desc = table->byte_description;
 342         int full = 0;
 343
 344         /* do we add all the fields */
 345         if (!strncmp(field, "*", 1))
 346                 full = 1;
 347
 348         while (desc != NULL) {
 349
 350                 byte = (struct astrodb_table_column_info *) desc->data;
 351                 idx = &table->idx[table->object_fields];
 352
 353                 if ((!strncmp(field, byte->label, strlen(byte->label)) || full) &&
 354                         !table_is_field(table, byte->label)) {
 355
 356                         idx->s_offset = table->object_size;
 357                         idx->l_offset = byte->start;
 358                         idx->l_size = byte->end - byte->start + 1;
 359                         strncpy(idx->name, byte->explanation, 32);
 360                         strncpy(idx->units, byte->units, 16);
 361                         strncpy(idx->symbol, byte->label, 8);
 362                         idx->type = table_get_ctype(byte->type);
 363                         idx->s_size = table_get_csize(byte->type);
 364
 365                         if (table->idx[table->object_fields].insert == NULL)
 366                                 table->idx[table->object_fields].insert =
 367                                         table_get_custom_insert(byte->type);
 368
 369                         table->object_fields++;
 370                         table->object_size += idx->s_size;
 371
 372                         if (!full)
 373                                 return 0;
 374                 }
 375                 desc = desc->tail;
 376         }
 377         if (full)
 378                 return 0;
 379
 380         astrodb_error("failed to add custom field %s\n", field);
 381         return -EINVAL;
 382 }
 383
 384 /*! \fn void astrodb_table_put_objects(astrodb_slist *results)
 385  * \param results Results from astrodb_table_get_objects()
 386  *
 387  * Releases resources held by the results from astrodb_table_get_objects().
 388  */
 389 void astrodb_table_put_objects(struct astrodb_slist *results)
 390 {
 391         struct astrodb_slist *slist;
 392
 393         while (results) {
 394                 slist = results;
 395                 results = results->tail;
 396                 free(slist);
 397         }
 398 }
 399
 400 /*! \fn int astrodb_table_alt_column(astrodb_table* table, char* field, char* alt, int flags)
 401  * \param table dataset
 402  * \param field Primary field
 403  * \param alt Alternative field
 404  * \param flags flags
 405  *
 406  * Set an alternative import field if the primary field is blank.
 407  */
 408 int astrodb_table_alt_column(struct astrodb_table *table, char *field,
 409                                 char *alt, int flags)
 410 {
 411         int idx, err;
 412
 413         if (table->alt_fields >= TABLE_MAX_AIDX) {
 414                 astrodb_error("too many alt fields %s\n", field);
 415                 return -EINVAL;
 416         }
 417
 418         idx = table_find_field(table, field);
 419         if (idx < 0) {
 420                 astrodb_error("field not found %s\n", field);
 421                 return -EINVAL;
 422         }
 423
 424         err = table_add_alt_field(table, alt, idx);
 425         if (err < 0) {
 426                 astrodb_error("failed to add field %s at idx %d\n", field, idx);
 427                 return -EINVAL;
 428         }
 429
 430         return 0;
 431 }
 432
 433 /*! \fn void astrodb_table_clip_on_position (astrodb_table* table, double min_ra, double min_dec, double max_ra, double max_dec, double faint_mag,double bright_mag)
 434  * \param table dataset
 435  * \param min_ra Min RA
 436  * \param min_dec Min Dec
 437  * \param max_ra Max RA
 438  * \param max_dec Max Dec
 439  * \param faint_mag Faint Magnitude
 440  * \param bright_mag Bright Magnitude
 441  *
 442  * Clips the dataset to the within the given position range. Note: Clipping
 443  * boundaries are not exact and depend on the granularity of the
 444  * dataset position strides.
 445  */
 446 void astrodb_table_clip_on_position(struct astrodb_table *table,
 447                                         double min_ra, double min_dec,
 448                                         double max_ra, double max_dec,
 449                                         double faint_mag, double bright_mag)
 450 {
 451         /* check ra,deg & mag boundaries */
 452         if (min_ra < table->db->ra_min)
 453                 min_ra = table->db->ra_min;
 454         if (max_ra > table->db->ra_max)
 455                 max_ra = table->db->ra_max;
 456         if (min_dec < table->db->dec_min)
 457                 min_dec = table->db->dec_min;
 458         if (max_dec > table->db->dec_max)
 459                 max_dec = table->db->dec_max;
 460         if (faint_mag > table->db->mag_faint)
 461                 faint_mag = table->db->mag_faint;
 462         if (bright_mag < table->db->mag_bright)
 463                 bright_mag = table->db->mag_bright;
 464
 465         /* normalise boundaries */
 466         min_ra -= table->db->ra_min;
 467         max_ra -= table->db->ra_min;
 468         min_dec -= table->db->dec_min;
 469         max_dec -= table->db->dec_min;
 470         faint_mag -= table->db->mag_bright;
 471         bright_mag -= table->db->mag_bright;
 472
 473         table->clip_radius = 0;
 474
 475         /* convert to stride boundaries */
 476         table->clip_min_ra = floor(min_ra / table->ra_stride_size);
 477         table->clip_max_ra = floor(max_ra / table->ra_stride_size);
 478         table->clip_min_dec = floor(min_dec / table->dec_stride_size);
 479         table->clip_max_dec = floor(max_dec / table->dec_stride_size);
 480
 481         table->clip_bright_mag = floor(bright_mag / table->mag_stride_size);
 482         table->clip_faint_mag = floor(faint_mag / table->mag_stride_size);
 483 }
 484
 485 /*! \fn void astrodb_table_clip_on_distance (astrodb_table* table, double min_AU, double max_AU)
 486  * \param table dataset
 487  * \param min_AU Min AU
 488  * \param max_AU Max AU
 489  *
 490  * Clips the dataset to the within the given distance range. Note: Clipping
 491  * boundaries are not exact and depend on the granularity of the
 492  * dataset distance stride.
 493  */
 494 void astrodb_table_clip_on_distance(struct astrodb_table *table,
 495                                         double min_AU, double max_AU)
 496 {
 497         /* check AU boundaries */
 498         if (min_AU < table->db->ra_min)
 499                 min_AU = table->db->ra_min;
 500         if (max_AU > table->db->ra_max)
 501                 max_AU = table->db->ra_max;
 502
 503         /* normalise boundaries */
 504         min_AU -= table->db->ra_min;
 505         max_AU -= table->db->ra_min;
 506
 507         table->clip_radius = 0;
 508
 509         /* convert to stride boundaries */
 510         table->clip_min_ra = (int) min_AU / table->ra_stride_size;
 511         table->clip_max_ra = ((int) max_AU / table->ra_stride_size) + 1;
 512 }
 513
 514 /*! \fn void astrodb_table_clip_on_fov (astrodb_table* table, double ra, double dec, double radius,
 515  double faint_mag, double bright_mag);
 516  * \brief Set dataset clipping area based on field of view
 517  * \ingroup dataset
 518  */
 519 void astrodb_table_clip_on_fov (struct astrodb_table *table,
 520                                 double ra, double dec, double radius,
 521                                 double faint_mag, double bright_mag)
 522 {
 523         /* check boundaries */
 524         if (radius > 90)
 525                 radius = 90;
 526         table->clip_radius = radius;
 527         if (ra < table->db->ra_min)
 528                 ra = table->db->ra_min;
 529         if (ra > table->db->ra_max)
 530                 ra = table->db->ra_max;
 531         if (dec < table->db->dec_min)
 532                 dec = table->db->dec_min;
 533         if (dec > table->db->dec_max)
 534                 dec = table->db->dec_max;
 535         if (faint_mag > table->db->mag_faint)
 536                 faint_mag = table->db->mag_faint;
 537         if (bright_mag < table->db->mag_bright)
 538                 bright_mag = table->db->mag_bright;
 539
 540         table->clip_centre_ra = ra;
 541         table->clip_centre_dec = dec;
 542
 543         /* normalise boundaries */
 544         faint_mag -= table->db->mag_bright;
 545         bright_mag -= table->db->mag_bright;
 546
 547         /* convert to stride boundaries */
 548         table->clip_bright_mag = (int) bright_mag / table->mag_stride_size;
 549         table->clip_faint_mag = ((int) faint_mag / table->mag_stride_size) + 1;
 550 }
 551
 552 /*! \fn void astrodb_table_unclip (astrodb_table* table)
 553  * \param table dataset
 554  *
 555  * Unclip the dataset to it's full boundaries
 556  */
 557 void astrodb_table_unclip(struct astrodb_table *table)
 558 {
 559         int i,j,k, offset;
 560
 561         /* in stride */
 562         table->clip_min_ra = 0;
 563         table->clip_min_dec = 0;
 564         table->clip_max_ra = table->ra_stride - 1;
 565         table->clip_max_dec = table->dec_stride - 1;
 566         table->clip_faint_mag = table->mag_stride - 1;
 567         table->clip_bright_mag = 0;
 568         table->clip_radius = 0;
 569
 570         for (i = 0; i <= table->clip_max_ra; i++) {
 571                 for (j = 0; j <= table->clip_max_dec; j++) {
 572                         for (k = 0; k <= table->clip_faint_mag; k++) {
 573                                 offset = table_calc_deep_offset(table, i, j, k);
 574                                 (table->status + offset)->flags &= ~ADB_SUSED;
 575                         }
 576                 }
 577         }
 578 }
 579
 580 /*! int astrodb_table_get_objects (astrodb_table* table, astrodb_progress progress, astrodb_slist **result, unsigned int src)
 581  * \param table dataset
 582  * \param progress Progress callback
 583  * \param results dataset objects
 584  * \param src Get objects flags
 585  * \return number of objects, or negative for failed
 586  *
 587  * Get objects from dataset based on clipping area. Results must be released
 588  * with astrodb_table_put_objects() after use.
 589  */
 590 int astrodb_table_get_objects(struct astrodb_table *table,
 591                                 struct astrodb_slist **result,
 592                                 unsigned int src)
 593 {
 594         if (table->clip_radius) {
 595                 switch (src) {
 596                 case ADB_SMEM:
 597                         return table_get_objects_memc(table, result);
 598                 case ADB_SCACHE:
 599                         return table_get_objects_cachec(table, result);
 600                 case ADB_SLOCAL:
 601                         return table_get_objects_rawc(table, result);
 602                 case ADB_SONLINE:
 603                         return table_get_objects_onlinec(table, result);
 604                 default:
 605                         return ADB_ENOGET;
 606                 }
 607         } else {
 608                 switch (src) {
 609                 case ADB_SMEM:
 610                         return table_get_objects_mem(table, result);
 611                 case ADB_SCACHE:
 612                         return table_get_objects_cache(table, result);
 613                 case ADB_SLOCAL:
 614                         return table_get_objects_raw(table, result);
 615                 case ADB_SONLINE:
 616                         return table_get_objects_online(table, result);
 617                 default:
 618                         return ADB_ENOGET;
 619                 }
 620         }
 621 }
 622
 623 /*! \fn int astrodb_table_prune(astrodb_table* table)
 624  * \param table dataset
 625  * \return bytes free'd
 626  *
 627  * Free's oldest unused object tiles.
 628  */
 629 int astrodb_table_prune(struct astrodb_table *table)
 630 {
 631         return 0;
 632 }
 633
 634 /*! \fn int astrodb_table_get_size(astrodb_table* table)
 635  * \param table dataset
 636  * \return size in bytes
 637  *
 638  * Get dataset memory usage.
 639  */
 640 int astrodb_table_get_size(struct astrodb_table *table)
 641 {
 642         return table->object_size * table->db->db_size;
 643 }
 644
 645 /*! \fn void astrodb_table_set_status_posn (astrodb_table* table, double min_ra, double min_dec, double max_ra, double max_dec, double faint_mag,double bright_mag, unsigned int status)
 646  * \param table dataset
 647  * \param min_ra Min RA
 648  * \param min_dec Min Dec
 649  * \param max_ra Max RA
 650  * \param max_dec Max Dec
 651  * \param faint_mag Faint Magnitude
 652  * \param bright_mag Bright Magnitude
 653  * \param status Object status
 654  *
 655  * Set the status of objects within a given position range.
 656  */
 657 void astrodb_table_set_status_posn(struct astrodb_table *table,
 658                                         double min_ra, double min_dec,
 659                                         double max_ra, double max_dec,
 660                                         double faint_mag, double bright_mag,
 661                                         unsigned int status)
 662 {
 663         int i, j, k;
 664         int clip_min_ra, clip_max_ra, clip_min_dec, clip_max_dec,
 665         clip_bright_mag, clip_faint_mag;
 666
 667         clip_min_ra = table->clip_min_ra;
 668         clip_max_ra = table->clip_max_ra;
 669         clip_min_dec = table->clip_min_dec;
 670         clip_max_dec = table->clip_max_dec;
 671         clip_bright_mag = table->clip_bright_mag;
 672         clip_faint_mag = table->clip_faint_mag;
 673
 674         astrodb_table_clip_on_position(table, min_ra, min_dec,
 675                                         max_ra, max_dec, bright_mag, faint_mag);
 676
 677         for (i = table->clip_min_ra; i < table->clip_max_ra; i++) {
 678                 for (j = table->clip_min_dec; j < table->clip_max_dec; j++) {
 679                         for (k = table->clip_bright_mag;
 680                                 k < table->clip_faint_mag; k++) {
 681
 682                         }
 683                 }
 684         }
 685 }
 686
 687 /*! \fn void astrodb_table_set_status_dist (astrodb_table* table, double min_AU, double max_AU, unsigned int status)
 688  * \param table Dataset
 689  * \param min_AU Minimum distance in AU
 690  * \param max_AU Maximum distance in AU
 691  * \param status Status flags to set
 692  *
 693  * Set the status of objects within a given distance range.
 694  */
 695 void astrodb_table_set_status_dist(struct astrodb_table *table,
 696                                         double min_AU, double max_AU,
 697                                         unsigned int status)
 698 {
 699         int min_offset = (min_AU - table->db->ra_min) / table->ra_stride_size;
 700         int max_offset = (max_AU - table->db->ra_min) / table->ra_stride_size;
 701
 702         for (; min_offset < max_offset; min_offset++) {
 703                 ((struct astrodb_tile_status*)
 704                         (table->status + min_offset))->flags = status;
 705         }
 706 }
 707
 708 /*! \fn astrodb_tile_status astrodb_table_get_status_posn (astrodb_table* table, double ra, double dec, double mag)
 709  * \param table Dataset
 710  * \param ra RA in degrees
 711  * \param dec DEC in dergrees
 712  * \param mag Magnitude
 713  *
 714  * Get the status of objects within a tile at a give (ra,dec,mag) offset.
 715  */
 716 struct astrodb_tile_status
 717 astrodb_table_get_status_posn(struct astrodb_table *table,
 718                                 double ra, double dec, double mag)
 719 {
 720         int x, y, z, offset;
 721
 722         x = floor((ra - table->db->ra_min) / table->ra_stride_size);
 723         y = floor((dec - table->db->dec_min) / table->dec_stride_size);
 724         z = floor((mag - table->db->mag_bright) / table->mag_stride_size);
 725         offset = table_calc_deep_offset(table, x, y, z);
 726
 727         return *(table->status + offset);
 728 }
 729
 730 /*! \fn astrodb_tile_status astrodb_table_get_status_dist (astrodb_table* table, double AU)
 731  * \param table Dataset
 732  * \param AU Distance in AU
 733  * \returns Status
 734  *
 735  * Get the status of objects within a tile at a give distance offset.
 736  */
 737 struct astrodb_tile_status
 738 astrodb_table_get_status_dist(struct astrodb_table *table, double AU)
 739 {
 740         int offset = floor((AU - table->db->ra_min) / table->ra_stride_size);
 741         return *(table->status + offset);
 742 }
 743
 744 /*! \fn astrodb_ctype astrodb_table_get_column_type(astrodb_table* table, char* field)
 745  * \param table Dataset
 746  * \param field Dataset field name
 747  * \return C type
 748  *
 749  * Get the C type for a field within a dataset.
 750  */
 751 astrodb_ctype astrodb_table_get_column_type(struct astrodb_table *table,
 752                                                 char *field)
 753 {
 754         int i;
 755
 756         for (i = 0; i < table->object_fields; i++) {
 757                 if (!strcmp(table->idx[i].symbol, field))
 758                         return table->idx[i].type;
 759         }
 760
 761         return CT_NULL;
 762 }
 763
 764 /*! \fn int astrodb_table_get_column_offset(astrodb_table* table, char* field)
 765  * \param table Dataset
 766  * \param field Dataset field name
 767  * \return Field offset in bytes
 768  *
 769  * Gets the offset in bytes for a field within a dataset.
 770  */
 771 int astrodb_table_get_column_offset(struct astrodb_table *table, char *field)
 772 {
 773         int i, ret;
 774
 775         ret = get_astro_field_offset(field);
 776         if (ret >= 0)
 777                 return ret;
 778
 779         ret = get_deep_field_offset(field);
 780         if (ret >= 0)
 781                 return ret;
 782
 783         /* check custom fields */
 784         for (i = 0; i < table->object_fields; i++) {
 785                 if (!strcmp(table->idx[i].symbol, field)) {
 786                         return table->idx[i].s_offset;
 787                 }
 788         }
 789
 790         astrodb_error("failed to find field %s\n", field);
 791         return -EINVAL;
 792 }
 793
 794 /*! \fn int astrodb_table_for_search_results_do(astrodb_slist* res, void(*func)(void* object, void* data), void* data)
 795  * \param res Results from get_objects
 796  * \param func Function to call per object in res
 797  * \param data Pointer to pass to func
 798  * \return 0 for success, negative for error
 799  *
 800  * Calls func for every object in res.
 801  */
 802
 803 int astrodb_table_for_search_results_do(struct astrodb_slist *results,
 804                                         void (*func) (void *object, void *data),
 805                                         void *data)
 806 {
 807         if (func == NULL)
 808                 return -EINVAL;
 809
 810         while (results) {
 811                 struct astrodb_slist *slist = results->data;
 812                 while (slist) {
 813                         func(slist->data, data);
 814                         slist = slist->tail;
 815                 }
 816                 results = results->tail;
 817         }
 818
 819         return 0;
 820 }
 821
 822
 823 /*! \fn int astrodb_table_get_row_size(astrodb_table* table);
 824  */
 825 int astrodb_table_get_row_size(struct astrodb_table *table)
 826 {
 827         return table->object_size;
 828 }
 829
 830
 831 /*! \fn void* astrodb_table_get_object (astrodb_table* table, char* id, char* field);
 832  * \param table dataset
 833  * \param id object id
 834  * \param field dataset field
 835  * \return object or NULL if not found
 836  *
 837  * Get an object based on it' ID
 838  */
 839 void* astrodb_table_get_object (struct astrodb_table *table, char *id,
 840                                 char *field)
 841 {
 842         struct astrodb_slist *slist = NULL;
 843         int offset, i;
 844
 845         for (i = 0; i < table->num_hmaps; i++) {
 846                 if (!strncmp(field, table->hmaps[i].field,
 847                         strlen(table->hmaps[i].field))) {
 848                         goto get_object;
 849                 }
 850         }
 851         astrodb_error("failed to find field %s\n", field);
 852         return NULL;
 853
 854 get_object:
 855         offset = table_calc_hash(id, strlen(id), table->no_tiles);
 856         slist = *(table->hmaps[i].hobjects + offset);
 857
 858         while (slist) {
 859                 char* object_id = slist->data + table->hmaps[i].offset;
 860                 if (!strstr(object_id, id))
 861                         return slist->data;
 862                 slist = slist->tail;
 863         }
 864
 865         astrodb_error("failed to find id %s\n", id);
 866         return NULL;
 867 }
 868
 869 /*! \fn int astrodb_table_hash_key(astrodb_table* table, char* field)
 870  * \param table dataset
 871  * \param field Field to be hashed.
 872  * \return 0 on success
 873  *
 874  * Add a field to be hashed for fast lookups.
 875  */
 876 int astrodb_table_hash_key(struct astrodb_table* table, char* key)
 877 {
 878         if (table->num_hmaps == HASH_MAPS) {
 879                 astrodb_error("too many hashed keys %s\n", key);
 880                 return -EINVAL;
 881         }
 882
 883         table->hmaps[table->num_hmaps].offset =
 884                 astrodb_table_get_column_offset(table, key);
 885         if (table->hmaps[table->num_hmaps].offset < 0) {
 886                 astrodb_error("invalid column offset %s\n", key);
 887                 return -EINVAL;
 888         }
 889
 890         strncpy(table->hmaps[table->num_hmaps].field, key, 8);
 891         table->num_hmaps++;
 892         return 0;
 893 }
 894
 895 /*! \fn int astrodb_table_register_schema(astrodb_table* table, astrodb_schema_object* idx, int idx_size);
 896  * \param table dataset
 897  * \param idx Object field index
 898  * \param idx_size Number of fields in index
 899  * \return 0 on success
 900  *
 901  * Register a new custom object type
 902  */
 903 int astrodb_table_register_schema(struct astrodb_table *table,
 904                                         struct astrodb_schema_object *idx,
 905                                         int idx_size)
 906 {
 907         int n, i;
 908
 909         for (i = 0, n = 0; i < idx_size; i++) {
 910                 if (!table_add_custom_struct_field(table, &idx[i]))
 911                         n++;
 912         }
 913
 914         return n;
 915 }