src/table.c

   1 /*
   2  *  This library is free software; you can redistribute it and/or
   3  *  modify it under the terms of the GNU Lesser General Public
   4  *  License as published by the Free Software Foundation; either
   5  *  version 2 of the License, or (at your option) any later version.
   6  *
   7  *  This library is distributed in the hope that it will be useful,
   8  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  10  *  Lesser General Public License for more details.
  11  *
  12  *  You should have received a copy of the GNU General Public License
  13  *  along with this program; if not, write to the Free Software
  14  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15  *
  16  *  Copyright (C) 2008 Liam Girdwood
  17  */
  18
  19 #define _GNU_SOURCE             /* for strtof and NAN */
  20
  21 #include <stdlib.h>
  22 #include <stdio.h>
  23 #include <errno.h>
  24 #include <string.h>
  25 #include <unistd.h>
  26 #include <math.h>
  27 #include <dirent.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/wait.h>
  31 #include <errno.h>
  32
  33 #include <libastrodb/db.h>
  34 #include <libastrodb/table.h>
  35 #include <libastrodb/wget.h>
  36 #include <libastrodb/gunzip.h>
  37 #include <libastrodb/adbstdio.h>
  38
  39
  40 #define FLOAT_SIZE '6'  /* CDS F string size and below for float else double */
  41 #define FILE_LEN        1024
  42
  43 struct _tile_dim {
  44         int records;            /*!< Number of records in dataset */
  45         int ra, dec, mag;       /*!< Stride size */
  46 };
  47
  48 /*
  49  * Table containing default dataset 3d tile dimensions
  50  */
  51 static struct _tile_dim tile_dim[] = {
  52         {10000, 6, 3, 20},
  53         {100000, 45, 20, 20},
  54         {500000, 90, 45, 20},
  55         {1000000, 180, 90, 20},
  56         {10000000, 360, 180, 20},
  57 };
  58
  59 /*! \fn static int init_table(astrodb_table * table, int ra_stride, int dec_stride, int mag_stride)
  60  * \brief Initialise a dataset
  61  */
  62 static int init_table(struct astrodb_table *table,
  63                         int ra_stride, int dec_stride, int mag_stride)
  64 {
  65         struct astrodb_slist **slist, **hash_object;
  66         struct astrodb_tile_status *status;
  67         int i = 0;
  68
  69         /* do we use the default tile dimensions */
  70         if (ra_stride == 0) {
  71                 for (; i < astrodb_size(tile_dim); i++) {
  72                         if (table->table_size <= tile_dim[i].records)
  73                                 break;
  74                 }
  75                 table->ra_stride = tile_dim[i].ra;
  76                 table->dec_stride = tile_dim[i].dec;
  77                 table->mag_stride = tile_dim[i].mag;
  78         } else {
  79                 table->ra_stride = ra_stride;
  80                 table->dec_stride = dec_stride;
  81                 table->mag_stride = mag_stride;
  82         }
  83
  84         table->array_size = table->ra_stride * table->dec_stride *
  85                 table->mag_stride * sizeof(struct astrodb_slist *);
  86         table->status_size = table->ra_stride * table->dec_stride *
  87                 table->mag_stride * sizeof(struct astrodb_tile_status);
  88         table->no_tiles = table->ra_stride * table->dec_stride *
  89                 table->mag_stride;
  90
  91         table->ra_stride_size =
  92                 fabs((table->db->ra_max - table->db->ra_min) / table->ra_stride);
  93         table->dec_stride_size =
  94                 fabs((table->db->dec_max - table->db->dec_min) / table->dec_stride);
  95         table->mag_stride_size =
  96                 fabs((table->db->mag_bright - table->db->mag_faint) / table->mag_stride);
  97
  98         slist = (struct astrodb_slist **) calloc(1, table->array_size);
  99         if (slist == 0) {
 100                 free(table);
 101                 return -ENOMEM;
 102         }
 103         table->objects = slist;
 104
 105         status = (struct astrodb_tile_status *) calloc(1, table->status_size);
 106         if (status == 0) {
 107                 free(slist);
 108                 free(table);
 109                 return -ENOMEM;
 110         }
 111         table->status = status;
 112
 113         for (i = 0; i < table->num_hmaps; i++) {
 114                 hash_object = (struct astrodb_slist **) calloc(1, table->array_size);
 115                 if (hash_object == NULL) {
 116                         free(slist);
 117                         free(status);
 118                         while(i--)
 119                                 free(table->hmaps[i-1].hobjects);
 120
 121                         free(table);
 122                         return -ENOMEM;
 123                 }
 124                 table->hmaps[i].hobjects = hash_object;
 125         }
 126
 127         return 0;
 128 }
 129
 130 /*! \fn static astrodb_table* create_table(astrodb_db*db, astrodb_dlist* desc, astrodb_dlist* files)
 131  * \brief Create a object dataset and populate fields
 132  */
 133 static struct astrodb_table *create_table(struct astrodb_db *db,
 134         struct astrodb_dlist *desc, struct astrodb_dlist *files)
 135 {
 136         struct astrodb_table *table;
 137
 138         table = calloc(1, sizeof(struct  astrodb_table));
 139         if (table == NULL)
 140                 return NULL;
 141
 142         bzero(table, sizeof(struct astrodb_table));
 143         table->byte_description =
 144                 ((struct astrodb_table_info *) files->data)->byte_description;
 145         table->file = ((struct astrodb_table_info *) files->data)->name;
 146         table->table_size = ((struct astrodb_table_info *) files->data)->records;
 147         table->record_size = ((struct astrodb_table_info *) files->data)->length;
 148         table->db = db;
 149         db->num_tables++;
 150         return table;
 151 }
 152
 153
 154
 155 /*! \fn astrodb_table *astrodb_table_create(astrodb_db *db, char *table_name, unsigned int flags)
 156  * \param db Catalog
 157  * \param table_name Dataset name (dataset file name in ReadMe)
 158  * \param flags Dataset creation flags.
 159  * \return A astrodb_table* for success or NULL on error
 160  *
 161  * Creates a dataset object.
 162  */
 163 struct astrodb_table *astrodb_table_create(struct astrodb_db *db,
 164                         char *table_name, unsigned int flags)
 165 {
 166         struct astrodb_dlist *files = db->info->files;
 167         struct astrodb_table *table = NULL;
 168         astrodb_info("Scanning for key fields in table: %s\n", table_name);
 169
 170         while (files) {
 171                 struct astrodb_table_info *table_info =
 172                         (struct  astrodb_table_info *) files->data;
 173                 struct astrodb_dlist *byte_desc = table_info->byte_description;
 174
 175                 /* create a tile array per file for files that have a byte desc */
 176                 if (byte_desc &&
 177                         !strncmp(table_name, table_info->name, strlen(table_name))) {
 178
 179                         table = create_table(db, byte_desc, files);
 180                         table->init_flags = flags;
 181                         astrodb_info("  scan: %s\n", table_info->name);
 182
 183                         db->tables = astrodb_dlist_append(db->tables,
 184                                                                 (char *) table);
 185                         return table;
 186                 }
 187                 files = files->tail;
 188         }
 189
 190         return NULL;
 191 }
 192
 193 /*! \fn int astrodb_table_open(astrodb_table * table, astrodb_progress progress, int ra, int dec, int mag)
 194  * \param table dataset
 195  * \param progress Progress callback
 196  * \param ra RA stride
 197  * \param dec Dec stride
 198  * \param mag Magnitude stride
 199  *
 200  * Initialise a dataset. This will download and import the raw data if necessary.
 201  */
 202 int insert_posn_mag(void *d, void *object);
 203
 204 int astrodb_table_open(struct astrodb_table *table, int ra, int dec, int mag)
 205 {
 206         struct stat stat_info;
 207         char file[FILE_LEN];
 208         char pstr[FILE_LEN];
 209         int ret;
 210
 211         sprintf(pstr, "Initialising table %s", table->file);
 212
 213         /* do we have to delete existing dataset */
 214         //if (flags & ADB_REFRESH)
 215         //      delete_table(astrodb_table* table);
 216
 217         //table_get_object_insert(table);
 218         table->object_insert = &insert_posn_mag;
 219         ret = init_table(table, ra, dec, mag);
 220         if (ret < 0) {
 221                 astrodb_error("failed to initialise table %d\n", ret);
 222                 return ret;
 223         }
 224
 225         /* check for local binary copy */
 226         sprintf(file, "%s%s%s", table->db->local_path, table->file, ".idx");
 227         ret = stat(file, &stat_info);
 228         if (ret == 0) {
 229                 ret = table_load_hdr(table);
 230                 if (ret < 0) {
 231                         astrodb_error("failed to load binary header %s %d\n",
 232                                 file, ret);
 233                         return ret;
 234                 }
 235                 ret = table_uncache_tiles(table);
 236                 if (ret < 0)
 237                         astrodb_error("failed to uncache table tiles %d\n",
 238                                 ret);
 239                 return ret;
 240         }
 241
 242         /* check for local ASCII copy */
 243         sprintf(file, "%s%s", table->db->local_path, table->file);
 244         ret = stat(file, &stat_info);
 245         if (ret == 0)
 246                 goto import;
 247
 248         /* check for local */
 249
 250         /* local binary or ASCII not available, so download ASCII */
 251         if (!(strncmp("http://", table->db->lib->remote, 7)) ||
 252                 !(strncmp("ftp://", table->db->lib->remote, 6))) {
 253                 ret = table_get_remote_dataset(table);
 254                 if (ret < 0) {
 255                         astrodb_error("failed to get remote table %d\n", ret);
 256                         return ret;
 257                 }
 258         }
 259 import:
 260         if (table->init_flags & ADB_MEM) {
 261                 ret = table_import(table, file);
 262                 if (ret < 0) {
 263                         astrodb_error("failed to import table %s %d\n", file,
 264                                 ret);
 265                         return ret;
 266                 }
 267         }
 268         if (table->init_flags & ADB_FILE) {
 269                 ret = table_save_hdr(table);
 270                 if (ret < 0) {
 271                         astrodb_error("failed to save table schema %d\n", ret);
 272                         return ret;
 273                 }
 274                 ret = table_cache_tiles(table);
 275         }
 276         return ret;
 277 }
 278
 279 /*! \fn void astrodb_table_close(astrodb_table* table)
 280  * \param table dataset
 281  *
 282  * Free dataset resources
 283  */
 284 void astrodb_table_close(struct astrodb_table *table)
 285 {
 286         int i, j;
 287         struct astrodb_db *db = table->db;
 288
 289         /* free hash maps */
 290         for (i = 0; i < table->num_hmaps; i++) {
 291                 for (j = 0; j < table->no_tiles; j++) {
 292
 293                         struct astrodb_slist *slist, *slist_;
 294                         slist_ = *(table->hmaps[i].hobjects + j);
 295
 296                         while (slist_) {
 297                                 slist = slist_;
 298                                 slist_ = slist_->tail;
 299                                 free(slist);
 300                         }
 301                 }
 302                 /* free array */
 303                 free(table->hmaps[i].hobjects);
 304         }
 305
 306         if (table->objects) {
 307                 /* free objects in array */
 308                 for (i = 0; i < table->no_tiles; i++)
 309                         astrodb_slist_foreach_free(*(table->objects + i), NULL, 0);
 310                 /* free array */
 311                 free(table->objects);
 312         }
 313
 314         if (table->status)
 315                 free(table->status);
 316
 317         db->tables = astrodb_dlist_free_object(db->tables, table);
 318 }
 319
 320 /*! \fn int astrodb_table_add_custom_field(astrodb_table* table, char* field)
 321  * \param table dataset
 322  * \param field Field name to add
 323  * \return 0 on success
 324  *
 325  * Add a custom field to a dataset for import.
 326  */
 327 int astrodb_table_add_custom_field(struct astrodb_table *table, char *field)
 328 {
 329         struct astrodb_table_column_info *byte = NULL;
 330         struct astrodb_schema_object *idx = NULL;
 331         struct astrodb_dlist *desc = table->byte_description;
 332         int full = 0;
 333
 334         /* do we add all the fields */
 335         if (!strncmp(field, "*", 1))
 336                 full = 1;
 337
 338         while (desc != NULL) {
 339
 340                 byte = (struct astrodb_table_column_info *) desc->data;
 341                 idx = &table->idx[table->object_fields];
 342
 343                 if ((!strncmp(field, byte->label, strlen(byte->label)) || full) &&
 344                         !table_is_field(table, byte->label)) {
 345
 346                         idx->s_offset = table->object_size;
 347                         idx->l_offset = byte->start;
 348                         idx->l_size = byte->end - byte->start + 1;
 349                         strncpy(idx->name, byte->explanation, 32);
 350                         strncpy(idx->units, byte->units, 16);
 351                         strncpy(idx->symbol, byte->label, 8);
 352                         idx->type = table_get_ctype(byte->type);
 353                         idx->s_size = table_get_csize(byte->type);
 354 #if 0
 355                         if (table->idx[table->object_fields].insert == NULL)
 356                                 table->idx[table->object_fields].insert =
 357                                         table_get_custom_insert(byte->type);
 358 #endif
 359                         table->object_fields++;
 360                         table->object_size += idx->s_size;
 361
 362                         if (!full)
 363                                 return 0;
 364                 }
 365                 desc = desc->tail;
 366         }
 367         if (full)
 368                 return 0;
 369
 370         astrodb_error("failed to add custom field %s\n", field);
 371         return -EINVAL;
 372 }
 373
 374
 375 /*! \fn void astrodb_table_put_objects(astrodb_slist *results)
 376  * \param results Results from astrodb_table_get_objects()
 377  *
 378  * Releases resources held by the results from astrodb_table_get_objects().
 379  */
 380 void astrodb_table_put_objects(struct astrodb_slist *results)
 381 {
 382         struct astrodb_slist *slist;
 383
 384         while (results) {
 385                 slist = results;
 386                 results = results->tail;
 387                 free(slist);
 388         }
 389 }
 390
 391 /*! \fn int astrodb_table_alt_column(astrodb_table* table, char* field, char* alt, int flags)
 392  * \param table dataset
 393  * \param field Primary field
 394  * \param alt Alternative field
 395  * \param flags flags
 396  *
 397  * Set an alternative import field if the primary field is blank.
 398  */
 399 int astrodb_table_alt_column(struct astrodb_table *table, char *field,
 400                                 char *alt, int flags)
 401 {
 402         int idx, err;
 403
 404         if (table->alt_fields >= TABLE_MAX_AIDX) {
 405                 astrodb_error("too many alt fields %s\n", field);
 406                 return -EINVAL;
 407         }
 408
 409         idx = table_find_field(table, field);
 410         if (idx < 0) {
 411                 astrodb_error("field not found %s\n", field);
 412                 return -EINVAL;
 413         }
 414
 415         err = table_add_alt_field(table, alt, idx);
 416         if (err < 0) {
 417                 astrodb_error("failed to add field %s at idx %d\n", field, idx);
 418                 return -EINVAL;
 419         }
 420
 421         return 0;
 422 }
 423
 424 /*! \fn void astrodb_table_clip_on_position (astrodb_table* table, double min_ra, double min_dec, double max_ra, double max_dec, double faint_mag,double bright_mag)
 425  * \param table dataset
 426  * \param min_ra Min RA
 427  * \param min_dec Min Dec
 428  * \param max_ra Max RA
 429  * \param max_dec Max Dec
 430  * \param faint_mag Faint Magnitude
 431  * \param bright_mag Bright Magnitude
 432  *
 433  * Clips the dataset to the within the given position range. Note: Clipping
 434  * boundaries are not exact and depend on the granularity of the
 435  * dataset position strides.
 436  */
 437 void astrodb_table_clip_on_position(struct astrodb_table *table,
 438                                         double min_ra, double min_dec,
 439                                         double max_ra, double max_dec,
 440                                         double faint_mag, double bright_mag)
 441 {
 442         /* check ra,deg & mag boundaries */
 443         if (min_ra < table->db->ra_min)
 444                 min_ra = table->db->ra_min;
 445         if (max_ra > table->db->ra_max)
 446                 max_ra = table->db->ra_max;
 447         if (min_dec < table->db->dec_min)
 448                 min_dec = table->db->dec_min;
 449         if (max_dec > table->db->dec_max)
 450                 max_dec = table->db->dec_max;
 451         if (faint_mag > table->db->mag_faint)
 452                 faint_mag = table->db->mag_faint;
 453         if (bright_mag < table->db->mag_bright)
 454                 bright_mag = table->db->mag_bright;
 455
 456         /* normalise boundaries */
 457         min_ra -= table->db->ra_min;
 458         max_ra -= table->db->ra_min;
 459         min_dec -= table->db->dec_min;
 460         max_dec -= table->db->dec_min;
 461         faint_mag -= table->db->mag_bright;
 462         bright_mag -= table->db->mag_bright;
 463
 464         table->fov = 0;
 465
 466         /* convert to stride boundaries */
 467         table->clip_min_ra = floor(min_ra / table->ra_stride_size);
 468         table->clip_max_ra = floor(max_ra / table->ra_stride_size);
 469         table->clip_min_dec = floor(min_dec / table->dec_stride_size);
 470         table->clip_max_dec = floor(max_dec / table->dec_stride_size);
 471
 472         table->clip_bright_mag = floor(bright_mag / table->mag_stride_size);
 473         table->clip_faint_mag = floor(faint_mag / table->mag_stride_size);
 474 }
 475
 476 /*! \fn void astrodb_table_clip_on_distance (astrodb_table* table, double min_AU, double max_AU)
 477  * \param table dataset
 478  * \param min_AU Min AU
 479  * \param max_AU Max AU
 480  *
 481  * Clips the dataset to the within the given distance range. Note: Clipping
 482  * boundaries are not exact and depend on the granularity of the
 483  * dataset distance stride.
 484  */
 485 void astrodb_table_clip_on_distance(struct astrodb_table *table,
 486                                         double min_AU, double max_AU)
 487 {
 488         /* check AU boundaries */
 489         if (min_AU < table->db->ra_min)
 490                 min_AU = table->db->ra_min;
 491         if (max_AU > table->db->ra_max)
 492                 max_AU = table->db->ra_max;
 493
 494         /* normalise boundaries */
 495         min_AU -= table->db->ra_min;
 496         max_AU -= table->db->ra_min;
 497
 498         table->fov = 0;
 499
 500         /* convert to stride boundaries */
 501         table->clip_min_ra = (int) min_AU / table->ra_stride_size;
 502         table->clip_max_ra = ((int) max_AU / table->ra_stride_size) + 1;
 503 }
 504
 505 /*! \fn void astrodb_table_clip_on_fov (astrodb_table* table, double ra, double dec, double radius,
 506  double faint_mag, double bright_mag);
 507  * \brief Set dataset clipping area based on field of view
 508  * \ingroup dataset
 509  */
 510 void astrodb_table_clip_on_fov (struct astrodb_table *table,
 511                                 double ra, double dec, double fov,
 512                                 double faint_mag, double bright_mag)
 513 {
 514         /* check boundaries */
 515         table->fov = fov;
 516         if (ra < table->db->ra_min)
 517                 ra = table->db->ra_min;
 518         if (ra > table->db->ra_max)
 519                 ra = table->db->ra_max;
 520         if (dec < table->db->dec_min)
 521                 dec = table->db->dec_min;
 522         if (dec > table->db->dec_max)
 523                 dec = table->db->dec_max;
 524         if (faint_mag > table->db->mag_faint)
 525                 faint_mag = table->db->mag_faint;
 526         if (bright_mag < table->db->mag_bright)
 527                 bright_mag = table->db->mag_bright;
 528
 529         table->clip_centre_ra = ra;
 530         table->clip_centre_dec = dec;
 531
 532         /* normalise boundaries */
 533         faint_mag -= table->db->mag_bright;
 534         bright_mag -= table->db->mag_bright;
 535
 536         /* convert to stride boundaries */
 537         table->clip_bright_mag = (int) bright_mag / table->mag_stride_size;
 538         table->clip_faint_mag = ((int) faint_mag / table->mag_stride_size) + 1;
 539 }
 540
 541 /*! \fn void astrodb_table_unclip (astrodb_table* table)
 542  * \param table dataset
 543  *
 544  * Unclip the dataset to it's full boundaries
 545  */
 546 void astrodb_table_unclip(struct astrodb_table *table)
 547 {
 548         int i,j,k, offset;
 549
 550         /* in stride */
 551         table->clip_min_ra = 0;
 552         table->clip_min_dec = 0;
 553         table->clip_max_ra = table->ra_stride - 1;
 554         table->clip_max_dec = table->dec_stride - 1;
 555         table->clip_faint_mag = table->mag_stride - 1;
 556         table->clip_bright_mag = 0;
 557         table->fov = 0;
 558
 559         for (i = 0; i <= table->clip_max_ra; i++) {
 560                 for (j = 0; j <= table->clip_max_dec; j++) {
 561                         for (k = 0; k <= table->clip_faint_mag; k++) {
 562                                 offset = table_calc_deep_offset(table, i, j, k);
 563                                 (table->status + offset)->flags &= ~ADB_SUSED;
 564                         }
 565                 }
 566         }
 567 }
 568
 569 /*! int astrodb_table_get_objects (astrodb_table* table, astrodb_progress progress, astrodb_slist **result, unsigned int src)
 570  * \param table dataset
 571  * \param progress Progress callback
 572  * \param results dataset objects
 573  * \param src Get objects flags
 574  * \return number of objects, or negative for failed
 575  *
 576  * Get objects from dataset based on clipping area. Results must be released
 577  * with astrodb_table_put_objects() after use.
 578  */
 579 int astrodb_table_get_objects(struct astrodb_table *table,
 580                                 struct astrodb_slist **result,
 581                                 unsigned int src)
 582 {
 583         if (table->fov) {
 584                 switch (src) {
 585                 case ADB_SMEM:
 586                         return table_get_objects_memc(table, result);
 587                 case ADB_SCACHE:
 588                         return table_get_objects_cachec(table, result);
 589                 case ADB_SLOCAL:
 590                         return table_get_objects_rawc(table, result);
 591                 case ADB_SONLINE:
 592                         return table_get_objects_onlinec(table, result);
 593                 default:
 594                         return ADB_ENOGET;
 595                 }
 596         } else {
 597                 switch (src) {
 598                 case ADB_SMEM:
 599                         return table_get_objects_mem(table, result);
 600                 case ADB_SCACHE:
 601                         return table_get_objects_cache(table, result);
 602                 case ADB_SLOCAL:
 603                         return table_get_objects_raw(table, result);
 604                 case ADB_SONLINE:
 605                         return table_get_objects_online(table, result);
 606                 default:
 607                         return ADB_ENOGET;
 608                 }
 609         }
 610 }
 611
 612 /*! \fn int astrodb_table_prune(astrodb_table* table)
 613  * \param table dataset
 614  * \return bytes free'd
 615  *
 616  * Free's oldest unused object tiles.
 617  */
 618 int astrodb_table_prune(struct astrodb_table *table)
 619 {
 620         return 0;
 621 }
 622
 623 /*! \fn int astrodb_table_get_size(astrodb_table* table)
 624  * \param table dataset
 625  * \return size in bytes
 626  *
 627  * Get dataset memory usage.
 628  */
 629 int astrodb_table_get_size(struct astrodb_table *table)
 630 {
 631         return table->object_size * table->db->db_size;
 632 }
 633
 634 /*! \fn void astrodb_table_set_status_posn (astrodb_table* table, double min_ra, double min_dec, double max_ra, double max_dec, double faint_mag,double bright_mag, unsigned int status)
 635  * \param table dataset
 636  * \param min_ra Min RA
 637  * \param min_dec Min Dec
 638  * \param max_ra Max RA
 639  * \param max_dec Max Dec
 640  * \param faint_mag Faint Magnitude
 641  * \param bright_mag Bright Magnitude
 642  * \param status Object status
 643  *
 644  * Set the status of objects within a given position range.
 645  */
 646 void astrodb_table_set_status_posn(struct astrodb_table *table,
 647                                         double min_ra, double min_dec,
 648                                         double max_ra, double max_dec,
 649                                         double faint_mag, double bright_mag,
 650                                         unsigned int status)
 651 {
 652         int i, j, k;
 653         int clip_min_ra, clip_max_ra, clip_min_dec, clip_max_dec,
 654         clip_bright_mag, clip_faint_mag;
 655
 656         clip_min_ra = table->clip_min_ra;
 657         clip_max_ra = table->clip_max_ra;
 658         clip_min_dec = table->clip_min_dec;
 659         clip_max_dec = table->clip_max_dec;
 660         clip_bright_mag = table->clip_bright_mag;
 661         clip_faint_mag = table->clip_faint_mag;
 662
 663         astrodb_table_clip_on_position(table, min_ra, min_dec,
 664                                         max_ra, max_dec, bright_mag, faint_mag);
 665
 666         for (i = table->clip_min_ra; i < table->clip_max_ra; i++) {
 667                 for (j = table->clip_min_dec; j < table->clip_max_dec; j++) {
 668                         for (k = table->clip_bright_mag;
 669                                 k < table->clip_faint_mag; k++) {
 670
 671                         }
 672                 }
 673         }
 674 }
 675
 676 /*! \fn void astrodb_table_set_status_dist (astrodb_table* table, double min_AU, double max_AU, unsigned int status)
 677  * \param table Dataset
 678  * \param min_AU Minimum distance in AU
 679  * \param max_AU Maximum distance in AU
 680  * \param status Status flags to set
 681  *
 682  * Set the status of objects within a given distance range.
 683  */
 684 void astrodb_table_set_status_dist(struct astrodb_table *table,
 685                                         double min_AU, double max_AU,
 686                                         unsigned int status)
 687 {
 688         int min_offset = (min_AU - table->db->ra_min) / table->ra_stride_size;
 689         int max_offset = (max_AU - table->db->ra_min) / table->ra_stride_size;
 690
 691         for (; min_offset < max_offset; min_offset++) {
 692                 ((struct astrodb_tile_status*)
 693                         (table->status + min_offset))->flags = status;
 694         }
 695 }
 696
 697 /*! \fn astrodb_tile_status astrodb_table_get_status_posn (astrodb_table* table, double ra, double dec, double mag)
 698  * \param table Dataset
 699  * \param ra RA in degrees
 700  * \param dec DEC in dergrees
 701  * \param mag Magnitude
 702  *
 703  * Get the status of objects within a tile at a give (ra,dec,mag) offset.
 704  */
 705 struct astrodb_tile_status
 706 astrodb_table_get_status_posn(struct astrodb_table *table,
 707                                 double ra, double dec, double mag)
 708 {
 709         int x, y, z, offset;
 710
 711         x = floor((ra - table->db->ra_min) / table->ra_stride_size);
 712         y = floor((dec - table->db->dec_min) / table->dec_stride_size);
 713         z = floor((mag - table->db->mag_bright) / table->mag_stride_size);
 714         offset = table_calc_deep_offset(table, x, y, z);
 715
 716         return *(table->status + offset);
 717 }
 718
 719 /*! \fn astrodb_tile_status astrodb_table_get_status_dist (astrodb_table* table, double AU)
 720  * \param table Dataset
 721  * \param AU Distance in AU
 722  * \returns Status
 723  *
 724  * Get the status of objects within a tile at a give distance offset.
 725  */
 726 struct astrodb_tile_status
 727 astrodb_table_get_status_dist(struct astrodb_table *table, double AU)
 728 {
 729         int offset = floor((AU - table->db->ra_min) / table->ra_stride_size);
 730         return *(table->status + offset);
 731 }
 732
 733 /*! \fn astrodb_ctype astrodb_table_get_column_type(astrodb_table* table, char* field)
 734  * \param table Dataset
 735  * \param field Dataset field name
 736  * \return C type
 737  *
 738  * Get the C type for a field within a dataset.
 739  */
 740 astrodb_ctype astrodb_table_get_column_type(struct astrodb_table *table,
 741                                                 char *field)
 742 {
 743         int i;
 744
 745         for (i = 0; i < table->object_fields; i++) {
 746                 if (!strcmp(table->idx[i].symbol, field))
 747                         return table->idx[i].type;
 748         }
 749
 750         return CT_NULL;
 751 }
 752
 753 /*! \fn int astrodb_table_get_column_offset(astrodb_table* table, char* field)
 754  * \param table Dataset
 755  * \param field Dataset field name
 756  * \return Field offset in bytes
 757  *
 758  * Gets the offset in bytes for a field within a dataset.
 759  */
 760 int astrodb_table_get_column_offset(struct astrodb_table *table, char *field)
 761 {
 762         int i;
 763
 764         /* check custom fields */
 765         for (i = 0; i < table->object_fields; i++) {
 766                 if (!strcmp(table->idx[i].symbol, field)) {
 767                         return table->idx[i].s_offset;
 768                 }
 769         }
 770
 771         astrodb_error("failed to find field %s\n", field);
 772         return -EINVAL;
 773 }
 774
 775 /*! \fn int astrodb_table_for_search_results_do(astrodb_slist* res, void(*func)(void* object, void* data), void* data)
 776  * \param res Results from get_objects
 777  * \param func Function to call per object in res
 778  * \param data Pointer to pass to func
 779  * \return 0 for success, negative for error
 780  *
 781  * Calls func for every object in res.
 782  */
 783
 784 int astrodb_table_for_search_results_do(struct astrodb_slist *results,
 785                                         void (*func) (void *object, void *data),
 786                                         void *data)
 787 {
 788         if (func == NULL)
 789                 return -EINVAL;
 790
 791         while (results) {
 792                 struct astrodb_slist *slist = results->data;
 793                 while (slist) {
 794                         func(slist->data, data);
 795                         slist = slist->tail;
 796                 }
 797                 results = results->tail;
 798         }
 799
 800         return 0;
 801 }
 802
 803
 804 /*! \fn int astrodb_table_get_row_size(astrodb_table* table);
 805  */
 806 int astrodb_table_get_row_size(struct astrodb_table *table)
 807 {
 808         return table->object_size;
 809 }
 810
 811
 812 /*! \fn void* astrodb_table_get_object (astrodb_table* table, char* id, char* field);
 813  * \param table dataset
 814  * \param id object id
 815  * \param field dataset field
 816  * \return object or NULL if not found
 817  *
 818  * Get an object based on it' ID
 819  */
 820 void* astrodb_table_get_object (struct astrodb_table *table, char *id,
 821                                 char *field)
 822 {
 823         struct astrodb_slist *slist = NULL;
 824         int offset, i;
 825
 826         for (i = 0; i < table->num_hmaps; i++) {
 827                 if (!strncmp(field, table->hmaps[i].field,
 828                         strlen(table->hmaps[i].field))) {
 829                         goto get_object;
 830                 }
 831         }
 832         astrodb_error("failed to find field %s\n", field);
 833         return NULL;
 834
 835 get_object:
 836         offset = table_calc_hash(id, strlen(id), table->no_tiles);
 837         slist = *(table->hmaps[i].hobjects + offset);
 838
 839         while (slist) {
 840                 char* object_id = slist->data + table->hmaps[i].offset;
 841                 if (!strstr(object_id, id))
 842                         return slist->data;
 843                 slist = slist->tail;
 844         }
 845
 846         astrodb_error("failed to find id %s\n", id);
 847         return NULL;
 848 }
 849
 850 /*! \fn int astrodb_table_hash_key(astrodb_table* table, char* field)
 851  * \param table dataset
 852  * \param field Field to be hashed.
 853  * \return 0 on success
 854  *
 855  * Add a field to be hashed for fast lookups.
 856  */
 857 int astrodb_table_hash_key(struct astrodb_table* table, char* key)
 858 {
 859         if (table->num_hmaps == HASH_MAPS) {
 860                 astrodb_error("too many hashed keys %s\n", key);
 861                 return -EINVAL;
 862         }
 863
 864         table->hmaps[table->num_hmaps].offset =
 865                 astrodb_table_get_column_offset(table, key);
 866         if (table->hmaps[table->num_hmaps].offset < 0) {
 867                 astrodb_error("invalid column offset %s\n", key);
 868                 return -EINVAL;
 869         }
 870
 871         strncpy(table->hmaps[table->num_hmaps].field, key, 8);
 872         table->num_hmaps++;
 873         return 0;
 874 }
 875
 876 /*! \fn int astrodb_table_register_schema(astrodb_table* table, astrodb_schema_object* idx, int idx_size);
 877  * \param table dataset
 878  * \param idx Object field index
 879  * \param idx_size Number of fields in index
 880  * \return 0 on success
 881  *
 882  * Register a new custom object type
 883  */
 884 int astrodb_table_register_schema(struct astrodb_table *table,
 885                                         struct astrodb_schema_object *idx,
 886                                         int idx_size, int object_size)
 887 {
 888         int n, i;
 889
 890         for (i = 0, n = 0; i < idx_size; i++) {
 891                 if (!table_add_custom_struct_field(table, &idx[i]))
 892                         n++;
 893         }
 894         table->object_size = object_size;
 895
 896         return n;
 897 }