src/math/sort.c

   1 /* PSPP - a program for statistical analysis.
   2    Copyright (C) 1997-9, 2000, 2006, 2009, 2011, 2012 Free Software Foundation, Inc.
   3
   4    This program is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation, either version 3 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
  16
  17 #include <config.h>
  18
  19 #include "math/sort.h"
  20
  21 #include <stdio.h>
  22
  23 #include "data/case.h"
  24 #include "data/casereader.h"
  25 #include "data/casewriter-provider.h"
  26 #include "data/casewriter.h"
  27 #include "data/settings.h"
  28 #include "data/subcase.h"
  29 #include "libpspp/array.h"
  30 #include "libpspp/assertion.h"
  31 #include "math/merge.h"
  32
  33 #include "gl/xalloc.h"
  34
  35 #include "gettext.h"
  36 #define _(msgid) gettext (msgid)
  37
  38 /* These should only be changed for testing purposes. */
  39 int min_buffers = 64;
  40 int max_buffers = INT_MAX;
  41
  42 struct sort_writer
  43   {
  44     struct caseproto *proto;
  45     struct subcase ordering;
  46     struct merge *merge;
  47     struct pqueue *pqueue;
  48
  49     struct casewriter *run;
  50     casenumber run_id;
  51     struct ccase *run_end;
  52   };
  53
  54 static struct casewriter_class sort_casewriter_class;
  55
  56 static struct pqueue *pqueue_create (const struct subcase *,
  57                                      const struct caseproto *);
  58 static void pqueue_destroy (struct pqueue *);
  59 static bool pqueue_is_full (const struct pqueue *);
  60 static bool pqueue_is_empty (const struct pqueue *);
  61 static void pqueue_push (struct pqueue *, struct ccase *, casenumber);
  62 static struct ccase *pqueue_pop (struct pqueue *, casenumber *);
  63
  64 static void output_record (struct sort_writer *);
  65
  66 struct casewriter *
  67 sort_create_writer (const struct subcase *ordering,
  68                     const struct caseproto *proto)
  69 {
  70   struct sort_writer *sort;
  71
  72   sort = xmalloc (sizeof *sort);
  73   sort->proto = caseproto_ref (proto);
  74   subcase_clone (&sort->ordering, ordering);
  75   sort->merge = merge_create (ordering, proto);
  76   sort->pqueue = pqueue_create (ordering, proto);
  77   sort->run = NULL;
  78   sort->run_id = 0;
  79   sort->run_end = NULL;
  80
  81   return casewriter_create (proto, &sort_casewriter_class, sort);
  82 }
  83
  84 static void
  85 sort_casewriter_write (struct casewriter *writer UNUSED, void *sort_,
  86                        struct ccase *c)
  87 {
  88   struct sort_writer *sort = sort_;
  89   bool next_run;
  90
  91   if (pqueue_is_full (sort->pqueue))
  92     output_record (sort);
  93
  94   next_run = (sort->run_end == NULL
  95               || subcase_compare_3way (&sort->ordering, c,
  96                                        &sort->ordering, sort->run_end) < 0);
  97   pqueue_push (sort->pqueue, c, sort->run_id + (next_run ? 1 : 0));
  98 }
  99
 100 static void
 101 sort_casewriter_destroy (struct casewriter *writer UNUSED, void *sort_)
 102 {
 103   struct sort_writer *sort = sort_;
 104
 105   subcase_destroy (&sort->ordering);
 106   merge_destroy (sort->merge);
 107   pqueue_destroy (sort->pqueue);
 108   casewriter_destroy (sort->run);
 109   case_unref (sort->run_end);
 110   caseproto_unref (sort->proto);
 111   free (sort);
 112 }
 113
 114 static struct casereader *
 115 sort_casewriter_convert_to_reader (struct casewriter *writer, void *sort_)
 116 {
 117   struct sort_writer *sort = sort_;
 118   struct casereader *output;
 119
 120   if (sort->run == NULL && sort->run_id == 0)
 121     {
 122       /* In-core sort. */
 123       sort->run = mem_writer_create (sort->proto);
 124       sort->run_id = 1;
 125     }
 126   while (!pqueue_is_empty (sort->pqueue))
 127     output_record (sort);
 128
 129   merge_append (sort->merge, casewriter_make_reader (sort->run));
 130   sort->run = NULL;
 131
 132   output = merge_make_reader (sort->merge);
 133   sort_casewriter_destroy (writer, sort);
 134   return output;
 135 }
 136
 137 static void
 138 output_record (struct sort_writer *sort)
 139 {
 140   struct ccase *min_case;
 141   casenumber min_run_id;
 142
 143   min_case = pqueue_pop (sort->pqueue, &min_run_id);
 144 #if 0
 145   printf ("\toutput: %f to run %d\n", case_num_idx (min_case, 0), min_run_id);
 146 #endif
 147
 148   if (sort->run_id != min_run_id && sort->run != NULL)
 149     {
 150       merge_append (sort->merge, casewriter_make_reader (sort->run));
 151       sort->run = NULL;
 152     }
 153   if (sort->run == NULL)
 154     {
 155       sort->run = tmpfile_writer_create (sort->proto);
 156       sort->run_id = min_run_id;
 157     }
 158
 159   case_unref (sort->run_end);
 160   sort->run_end = case_ref (min_case);
 161   casewriter_write (sort->run, min_case);
 162 }
 163
 164 static struct casewriter_class sort_casewriter_class =
 165   {
 166     sort_casewriter_write,
 167     sort_casewriter_destroy,
 168     sort_casewriter_convert_to_reader,
 169   };
 170 \f
 171 /* Reads all the cases from INPUT.  Sorts the cases according to
 172    ORDERING.  Returns the sorted cases in a new casereader.
 173    INPUT is destroyed by this function.
 174  */
 175 struct casereader *
 176 sort_execute (struct casereader *input, const struct subcase *ordering)
 177 {
 178   struct casewriter *output =
 179     sort_create_writer (ordering, casereader_get_proto (input));
 180   casereader_transfer (input, output);
 181   return casewriter_make_reader (output);
 182 }
 183
 184 /* Reads all the cases from INPUT.  Sorts the cases in ascending
 185    order according to VARIABLE.  Returns the sorted cases in a
 186    new casereader.  INPUT is destroyed by this function. */
 187 struct casereader *
 188 sort_execute_1var (struct casereader *input, const struct variable *var)
 189 {
 190   struct subcase sc;
 191   struct casereader *reader;
 192
 193   subcase_init_var (&sc, var, SC_ASCEND);
 194   reader = sort_execute (input, &sc);
 195   subcase_destroy (&sc);
 196   return reader;
 197 }
 198 \f
 199 struct pqueue
 200   {
 201     struct subcase ordering;
 202     struct pqueue_record *records;
 203     size_t record_cnt;          /* Current number of records. */
 204     size_t record_cap;          /* Space currently allocated for records. */
 205     size_t record_max;          /* Max space we are willing to allocate. */
 206     casenumber idx;
 207   };
 208
 209 struct pqueue_record
 210   {
 211     casenumber id;
 212     struct ccase *c;
 213     casenumber idx;
 214   };
 215
 216 static int compare_pqueue_records_minheap (const void *a, const void *b,
 217                                            const void *pq_);
 218
 219 static struct pqueue *
 220 pqueue_create (const struct subcase *ordering, const struct caseproto *proto)
 221 {
 222   struct pqueue *pq;
 223
 224   pq = xmalloc (sizeof *pq);
 225   subcase_clone (&pq->ordering, ordering);
 226   pq->record_max = settings_get_workspace_cases (proto);
 227   if (pq->record_max > max_buffers)
 228     pq->record_max = max_buffers;
 229   else if (pq->record_max < min_buffers)
 230     pq->record_max = min_buffers;
 231   pq->record_cnt = 0;
 232   pq->record_cap = 0;
 233   pq->records = NULL;
 234   pq->idx = 0;
 235
 236   return pq;
 237 }
 238
 239 static void
 240 pqueue_destroy (struct pqueue *pq)
 241 {
 242   if (pq != NULL)
 243     {
 244       while (!pqueue_is_empty (pq))
 245         {
 246           casenumber id;
 247           struct ccase *c = pqueue_pop (pq, &id);
 248           case_unref (c);
 249         }
 250       subcase_destroy (&pq->ordering);
 251       free (pq->records);
 252       free (pq);
 253     }
 254 }
 255
 256 static bool
 257 pqueue_is_full (const struct pqueue *pq)
 258 {
 259   return pq->record_cnt >= pq->record_max;
 260 }
 261
 262 static bool
 263 pqueue_is_empty (const struct pqueue *pq)
 264 {
 265   return pq->record_cnt == 0;
 266 }
 267
 268 static void
 269 pqueue_push (struct pqueue *pq, struct ccase *c, casenumber id)
 270 {
 271   struct pqueue_record *r;
 272
 273   assert (!pqueue_is_full (pq));
 274
 275   if (pq->record_cnt >= pq->record_cap)
 276     {
 277       pq->record_cap = pq->record_cap * 2;
 278       if (pq->record_cap < 16)
 279         pq->record_cap = 16;
 280       else if (pq->record_cap > pq->record_max)
 281         pq->record_cap = pq->record_max;
 282       pq->records = xrealloc (pq->records,
 283                               pq->record_cap * sizeof *pq->records);
 284     }
 285
 286   r = &pq->records[pq->record_cnt++];
 287   r->id = id;
 288   r->c = c;
 289   r->idx = pq->idx++;
 290
 291   push_heap (pq->records, pq->record_cnt, sizeof *pq->records,
 292              compare_pqueue_records_minheap, pq);
 293 }
 294
 295 static struct ccase *
 296 pqueue_pop (struct pqueue *pq, casenumber *id)
 297 {
 298   struct pqueue_record *r;
 299
 300   assert (!pqueue_is_empty (pq));
 301
 302   pop_heap (pq->records, pq->record_cnt--, sizeof *pq->records,
 303             compare_pqueue_records_minheap, pq);
 304
 305   r = &pq->records[pq->record_cnt];
 306   *id = r->id;
 307   return r->c;
 308 }
 309
 310 /* Compares record-run tuples A and B on id, then on case data,
 311    then on insertion order, in descending order. */
 312 static int
 313 compare_pqueue_records_minheap (const void *a_, const void *b_,
 314                                 const void *pq_)
 315 {
 316   const struct pqueue_record *a = a_;
 317   const struct pqueue_record *b = b_;
 318   const struct pqueue *pq = pq_;
 319   int result = a->id < b->id ? -1 : a->id > b->id;
 320   if (result == 0)
 321     result = subcase_compare_3way (&pq->ordering, a->c, &pq->ordering, b->c);
 322   if (result == 0)
 323     result = a->idx < b->idx ? -1 : a->idx > b->idx;
 324   return -result;
 325 }