scripts/compute_time_listings

   1 #!/bin/bash
   2 # The contents of this file are subject to the Common Public Attribution
   3 # License Version 1.0. (the "License"); you may not use this file except in
   4 # compliance with the License. You may obtain a copy of the License at
   5 # http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
   6 # License Version 1.1, but Sections 14 and 15 have been added to cover use of
   7 # software over a computer network and provide for limited attribution for the
   8 # Original Developer. In addition, Exhibit A has been modified to be consistent
   9 # with Exhibit B.
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
  13 # the specific language governing rights and limitations under the License.
  14 #
  15 # The Original Code is reddit.
  16 #
  17 # The Original Developer is the Initial Developer.  The Initial Developer of
  18 # the Original Code is reddit Inc.
  19 #
  20 # All portions of the code written by reddit are Copyright (c) 2006-2015 reddit
  21 # Inc. All Rights Reserved.
  22 ###############################################################################
  23
  24 set -e
  25
  26 # expects two environment variables
  27 #   REDDIT_ROOT = path to the root of the reddit public code; the directory with the Makefile
  28 #   REDDIT_INI = path to the ini file to use
  29 # which should be supplied via:
  30 source /etc/default/reddit
  31 # additionally, some configuration can be overridden in the environment
  32 export TMPDIR=${TMPDIR:-/tmp}
  33 export PGUSER=${PGUSER:-reddit}
  34 export PGHOST=${PGHOST:-localhost}
  35
  36 ## command line args
  37 # one of "link" or "comment"
  38 export THING_CLS="$1"
  39 # period of data to extract from postgres: e.g. "hour", "week", "year", "all"
  40 export INTERVAL="$2"
  41 # which period listings to update.
  42 # formatted as python tuple of strings: e.g. '("hour",)' or ("week", "all",) etc
  43 export TIMES="$3"
  44
  45 echo "Starting $THING_CLS processing"
  46
  47 THING_DUMP=$TMPDIR/$THING_CLS-$INTERVAL-thing.dump
  48 DATA_DUMP=$TMPDIR/$THING_CLS-$INTERVAL-data.dump
  49 function clean_up {
  50     rm -f $THING_DUMP $DATA_DUMP
  51 }
  52 trap clean_up EXIT
  53
  54 if [ -e $THING_DUMP ]; then
  55     echo cannot start because $THING_DUMP exists
  56     ls -l $THING_DUMP
  57     exit 1
  58 fi
  59 touch $THING_DUMP
  60
  61
  62 function run_query {
  63     psql -F"\t" -A -t -c "$1"
  64 }
  65
  66 function mrsort {
  67     sort -S200m
  68 }
  69
  70 function reddit {
  71     paster --plugin=r2 run $REDDIT_INI $REDDIT_ROOT/r2/lib/mr_top.py -c "$1"
  72 }
  73
  74 # Hack to let pg fetch all things with intervals
  75 if [ $INTERVAL = "all" ]; then
  76    export INTERVAL="century"
  77 fi
  78
  79 MINID=$(run_query "SELECT thing_id
  80                    FROM reddit_thing_$THING_CLS
  81                    WHERE
  82                       date > now() - interval '1 $INTERVAL' AND
  83                       date < now()
  84                    ORDER BY date
  85                    LIMIT 1")
  86 if [ -z $MINID ]; then
  87     echo \$MINID is empty. Replication is likely behind.
  88     exit 1
  89 fi
  90
  91 run_query "\\copy (SELECT thing_id, 'thing', '$THING_CLS', ups, downs, deleted, spam, extract(epoch from date)
  92                    FROM reddit_thing_$THING_CLS
  93                    WHERE
  94                        not deleted AND
  95                        thing_id >= $MINID
  96                   ) to $THING_DUMP"
  97
  98 run_query "\\copy (SELECT thing_id, 'data', '$THING_CLS', key, value
  99                    FROM reddit_data_$THING_CLS
 100                    WHERE
 101                        key IN ('url', 'sr_id', 'author_id') AND
 102                        thing_id >= $MINID
 103                   ) to $DATA_DUMP"
 104
 105 cat $THING_DUMP $DATA_DUMP |
 106     mrsort |
 107     reddit "join_things('$THING_CLS')" |
 108     reddit "time_listings($TIMES, '$THING_CLS')" |
 109     mrsort |
 110     reddit "write_permacache()"
 111
 112 echo 'Done.'