install-reddit.sh

   1 #!/bin/bash
   2 # The contents of this file are subject to the Common Public Attribution
   3 # License Version 1.0. (the "License"); you may not use this file except in
   4 # compliance with the License. You may obtain a copy of the License at
   5 # http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
   6 # License Version 1.1, but Sections 14 and 15 have been added to cover use of
   7 # software over a computer network and provide for limited attribution for the
   8 # Original Developer. In addition, Exhibit A has been modified to be consistent
   9 # with Exhibit B.
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
  13 # the specific language governing rights and limitations under the License.
  14 #
  15 # The Original Code is reddit.
  16 #
  17 # The Original Developer is the Initial Developer.  The Initial Developer of
  18 # the Original Code is reddit Inc.
  19 #
  20 # All portions of the code written by reddit are Copyright (c) 2006-2015 reddit
  21 # Inc. All Rights Reserved.
  22 ###############################################################################
  23
  24 ###############################################################################
  25 # reddit dev environment installer
  26 # --------------------------------
  27 # This script installs a reddit stack suitable for development. DO NOT run this
  28 # on a system that you use for other purposes as it might delete important
  29 # files, truncate your databases, and otherwise do mean things to you.
  30 #
  31 # By default, this script will install the reddit code in the current user's
  32 # home directory and all of its dependencies (including libraries and database
  33 # servers) at the system level. The installed reddit will expect to be visited
  34 # on the domain "reddit.local" unless specified otherwise.  Configuring name
  35 # resolution for the domain is expected to be done outside the installed
  36 # environment (e.g. in your host machine's /etc/hosts file) and is not
  37 # something this script handles.
  38 #
  39 # Several configuration options (listed in the "Configuration" section below)
  40 # are overridable with environment variables. e.g.
  41 #
  42 #    sudo REDDIT_DOMAIN=example.com ./install-reddit.sh
  43 #
  44 ###############################################################################
  45 set -e
  46
  47 ###############################################################################
  48 # Configuration
  49 ###############################################################################
  50 # which user to install the code for; defaults to the user invoking this script
  51 REDDIT_USER=${REDDIT_USER:-$SUDO_USER}
  52
  53 # the group to run reddit code as; must exist already
  54 REDDIT_GROUP=${REDDIT_GROUP:-nogroup}
  55
  56 # the root directory to base the install in. must exist already
  57 REDDIT_HOME=${REDDIT_HOME:-/home/$REDDIT_USER}
  58
  59 # the domain that you will connect to your reddit install with.
  60 # MUST contain a . in it somewhere as browsers won't do cookies for dotless
  61 # domains. an IP address will suffice if nothing else is available.
  62 REDDIT_DOMAIN=${REDDIT_DOMAIN:-reddit.local}
  63
  64 #The plugins to clone and register in the ini file
  65 REDDIT_PLUGINS=${REDDIT_PLUGINS:-meatspace about liveupdate}
  66
  67 ###############################################################################
  68 # Sanity Checks
  69 ###############################################################################
  70 if [[ $EUID -ne 0 ]]; then
  71     echo "ERROR: Must be run with root privileges."
  72     exit 1
  73 fi
  74
  75 # seriously! these checks are here for a reason. the packages from the
  76 # reddit ppa aren't built for anything but precise (12.04) right now, so
  77 # if you try and use this install script on another release you're gonna
  78 # have a bad time.
  79 source /etc/lsb-release
  80 if [ "$DISTRIB_ID" != "Ubuntu" -o "$DISTRIB_RELEASE" != "12.04" ]; then
  81     echo "ERROR: Only Ubuntu 12.04 is supported."
  82     exit 1
  83 fi
  84
  85 if [[ "2000000" -gt $(awk '/MemTotal/{print $2}' /proc/meminfo) ]]; then
  86     LOW_MEM_PROMPT="reddit requires at least 2GB of memory to work properly, continue anyway? [y/n] "
  87     read -er -n1 -p "$LOW_MEM_PROMPT" response
  88     if [[ "$response" != "y" ]]; then
  89       echo "Quitting."
  90       exit 1
  91     fi
  92 fi
  93
  94 ###############################################################################
  95 # Install prerequisites
  96 ###############################################################################
  97 set -x
  98
  99 # aptitude configuration
 100 APTITUDE_OPTIONS="-y"
 101 export DEBIAN_FRONTEND=noninteractive
 102
 103 # run an aptitude update to make sure python-software-properties
 104 # dependencies are found
 105 apt-get update
 106
 107 # add the reddit ppa for some custom packages
 108 apt-get install $APTITUDE_OPTIONS python-software-properties
 109 apt-add-repository -y ppa:reddit/ppa
 110
 111 # pin the ppa -- packages present in the ppa will take precedence over
 112 # ones in other repositories (unless further pinning is done)
 113 cat <<HERE > /etc/apt/preferences.d/reddit
 114 Package: *
 115 Pin: release o=LP-PPA-reddit
 116 Pin-Priority: 600
 117 HERE
 118
 119 # add the datastax cassandra repos
 120 echo deb http://debian.datastax.com/community stable main > /etc/apt/sources.list.d/cassandra.sources.list
 121 wget -qO- -L https://debian.datastax.com/debian/repo_key | sudo apt-key add -
 122
 123 # grab the new ppas' package listings
 124 apt-get update
 125
 126 # install prerequisites
 127 cat <<PACKAGES | xargs apt-get install $APTITUDE_OPTIONS
 128 netcat-openbsd
 129 git-core
 130
 131 python-dev
 132 python-setuptools
 133 python-routes=1.12.3-1ubuntu1
 134 python-pylons=1.0-2
 135 python-boto
 136 python-tz
 137 python-crypto
 138 python-babel
 139 cython
 140 python-sqlalchemy
 141 python-beautifulsoup
 142 python-chardet
 143 python-psycopg2
 144 python-pycassa
 145 python-imaging
 146 python-pycaptcha
 147 python-amqplib
 148 python-pylibmc
 149 python-bcrypt
 150 python-snudown
 151 python-l2cs
 152 python-lxml
 153 python-zope.interface
 154 python-kazoo
 155 python-stripe
 156 python-tinycss2
 157 python-unidecode
 158 python-mock
 159 python-yaml
 160
 161 python-flask
 162 geoip-bin
 163 geoip-database
 164 python-geoip
 165
 166 nodejs
 167 node-less
 168 node-uglify
 169 gettext
 170 make
 171 optipng
 172 jpegoptim
 173
 174 memcached
 175 postgresql
 176 postgresql-client
 177 rabbitmq-server
 178 cassandra=1.2.19
 179 haproxy
 180 nginx
 181 stunnel
 182 gunicorn
 183 sutro
 184 libpcre3-dev
 185 PACKAGES
 186
 187 # we don't want to upgrade to C* 2.0 yet, so we'll put it on hold
 188 apt-mark hold cassandra
 189
 190 ###############################################################################
 191 # Wait for all the services to be up
 192 ###############################################################################
 193 # cassandra doesn't auto-start after install
 194 service cassandra start
 195
 196 # check each port for connectivity
 197 echo "Waiting for services to be available, see source for port meanings..."
 198 # 11211 - memcache
 199 # 5432 - postgres
 200 # 5672 - rabbitmq
 201 # 9160 - cassandra
 202 for port in 11211 5432 5672 9160; do
 203     while ! nc -vz localhost $port; do
 204         sleep 1
 205     done
 206 done
 207
 208 ###############################################################################
 209 # Install the reddit source repositories
 210 ###############################################################################
 211 if [ ! -d $REDDIT_HOME/src ]; then
 212     mkdir -p $REDDIT_HOME/src
 213     chown $REDDIT_USER $REDDIT_HOME/src
 214 fi
 215
 216 function clone_reddit_repo {
 217     local destination=$REDDIT_HOME/src/${1}
 218     local repository_url=https://github.com/${2}.git
 219
 220     if [ ! -d $destination ]; then
 221         sudo -u $REDDIT_USER git clone $repository_url $destination
 222     fi
 223
 224     if [ -d $destination/upstart ]; then
 225         cp $destination/upstart/* /etc/init/
 226     fi
 227 }
 228
 229 function clone_reddit_plugin_repo {
 230     clone_reddit_repo $1 reddit/reddit-plugin-$1
 231 }
 232
 233 clone_reddit_repo reddit reddit/reddit
 234 clone_reddit_repo i18n reddit/reddit-i18n
 235 for plugin in $REDDIT_PLUGINS; do
 236     clone_reddit_plugin_repo $plugin
 237 done
 238
 239 ###############################################################################
 240 # Configure Cassandra
 241 ###############################################################################
 242 python <<END
 243 import pycassa
 244 sys = pycassa.SystemManager("localhost:9160")
 245
 246 if "reddit" not in sys.list_keyspaces():
 247     print "creating keyspace 'reddit'"
 248     sys.create_keyspace("reddit", "SimpleStrategy", {"replication_factor": "1"})
 249     print "done"
 250
 251 if "permacache" not in sys.get_keyspace_column_families("reddit"):
 252     print "creating column family 'permacache'"
 253     sys.create_column_family("reddit", "permacache")
 254     print "done"
 255 END
 256
 257 ###############################################################################
 258 # Configure PostgreSQL
 259 ###############################################################################
 260 SQL="SELECT COUNT(1) FROM pg_catalog.pg_database WHERE datname = 'reddit';"
 261 IS_DATABASE_CREATED=$(sudo -u postgres psql -t -c "$SQL")
 262
 263 if [ $IS_DATABASE_CREATED -ne 1 ]; then
 264     cat <<PGSCRIPT | sudo -u postgres psql
 265 CREATE DATABASE reddit WITH ENCODING = 'utf8' TEMPLATE template0 LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8';
 266 CREATE USER reddit WITH PASSWORD 'password';
 267 PGSCRIPT
 268 fi
 269
 270 sudo -u postgres psql reddit < $REDDIT_HOME/src/reddit/sql/functions.sql
 271
 272 ###############################################################################
 273 # Configure RabbitMQ
 274 ###############################################################################
 275 if ! rabbitmqctl list_vhosts | egrep "^/$"
 276 then
 277     rabbitmqctl add_vhost /
 278 fi
 279
 280 if ! rabbitmqctl list_users | egrep "^reddit"
 281 then
 282     rabbitmqctl add_user reddit reddit
 283 fi
 284
 285 rabbitmqctl set_permissions -p / reddit ".*" ".*" ".*"
 286
 287 ###############################################################################
 288 # Install and configure the reddit code
 289 ###############################################################################
 290 function install_reddit_repo {
 291     cd $REDDIT_HOME/src/$1
 292     sudo -u $REDDIT_USER python setup.py build
 293     python setup.py develop --no-deps
 294 }
 295
 296 install_reddit_repo reddit/r2
 297 install_reddit_repo i18n
 298 for plugin in $REDDIT_PLUGINS; do
 299     install_reddit_repo $plugin
 300 done
 301
 302 # generate binary translation files from source
 303 cd $REDDIT_HOME/src/i18n/
 304 sudo -u $REDDIT_USER make clean all
 305
 306 # this builds static files and should be run *after* languages are installed
 307 # so that the proper language-specific static files can be generated and after
 308 # plugins are installed so all the static files are available.
 309 cd $REDDIT_HOME/src/reddit/r2
 310 sudo -u $REDDIT_USER make clean all
 311
 312 plugin_str=$(echo -n "$REDDIT_PLUGINS" | tr " " ,)
 313 if [ ! -f development.update ]; then
 314     cat > development.update <<DEVELOPMENT
 315 # after editing this file, run "make ini" to
 316 # generate a new development.ini
 317
 318 [DEFAULT]
 319 # global debug flag -- displays pylons stacktrace rather than 500 page on error when true
 320 # WARNING: a pylons stacktrace allows remote code execution. Make sure this is false
 321 # if your server is publicly accessible.
 322 debug = true
 323
 324 disable_ads = true
 325 disable_captcha = true
 326 disable_ratelimit = true
 327 disable_require_admin_otp = true
 328
 329 page_cache_time = 0
 330
 331 domain = $REDDIT_DOMAIN
 332 oauth_domain = $REDDIT_DOMAIN
 333
 334 plugins = $plugin_str
 335
 336 media_provider = filesystem
 337 media_fs_root = /srv/www/media
 338 media_fs_base_url_http = http://%(domain)s/media/
 339
 340 [server:main]
 341 port = 8001
 342 DEVELOPMENT
 343     chown $REDDIT_USER development.update
 344 else
 345     sed -i "s/^plugins = .*$/plugins = $plugin_str/" $REDDIT_HOME/src/reddit/r2/development.update
 346     sed -i "s/^domain = .*$/domain = $REDDIT_DOMAIN/" $REDDIT_HOME/src/reddit/r2/development.update
 347     sed -i "s/^oauth_domain = .*$/oauth_domain = $REDDIT_DOMAIN/" $REDDIT_HOME/src/reddit/r2/development.update
 348 fi
 349
 350 sudo -u $REDDIT_USER make ini
 351
 352 if [ ! -L run.ini ]; then
 353     sudo -u $REDDIT_USER ln -nsf development.ini run.ini
 354 fi
 355
 356 ###############################################################################
 357 # some useful helper scripts
 358 ###############################################################################
 359 function helper-script() {
 360     cat > $1
 361     chmod 755 $1
 362 }
 363
 364 helper-script /usr/local/bin/reddit-run <<REDDITRUN
 365 #!/bin/bash
 366 exec paster --plugin=r2 run $REDDIT_HOME/src/reddit/r2/run.ini "\$@"
 367 REDDITRUN
 368
 369 helper-script /usr/local/bin/reddit-shell <<REDDITSHELL
 370 #!/bin/bash
 371 exec paster --plugin=r2 shell $REDDIT_HOME/src/reddit/r2/run.ini
 372 REDDITSHELL
 373
 374 helper-script /usr/local/bin/reddit-start <<REDDITSTART
 375 #!/bin/bash
 376 initctl emit reddit-start
 377 REDDITSTART
 378
 379 helper-script /usr/local/bin/reddit-stop <<REDDITSTOP
 380 #!/bin/bash
 381 initctl emit reddit-stop
 382 REDDITSTOP
 383
 384 helper-script /usr/local/bin/reddit-restart <<REDDITRESTART
 385 #!/bin/bash
 386 initctl emit reddit-restart TARGET=${1:-all}
 387 REDDITRESTART
 388
 389 helper-script /usr/local/bin/reddit-flush <<REDDITFLUSH
 390 #!/bin/bash
 391 echo flush_all | nc localhost 11211
 392 REDDITFLUSH
 393
 394 ###############################################################################
 395 # pixel and click server
 396 ###############################################################################
 397 mkdir -p /var/opt/reddit/
 398 chown $REDDIT_USER:$REDDIT_GROUP /var/opt/reddit/
 399
 400 mkdir -p /srv/www/pixel
 401 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/pixel
 402 cp $REDDIT_HOME/src/reddit/r2/r2/public/static/pixel.png /srv/www/pixel
 403
 404 if [ ! -f /etc/gunicorn.d/click.conf ]; then
 405     cat > /etc/gunicorn.d/click.conf <<CLICK
 406 CONFIG = {
 407     "mode": "wsgi",
 408     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 409     "user": "$REDDIT_USER",
 410     "group": "$REDDIT_USER",
 411     "args": (
 412         "--bind=unix:/var/opt/reddit/click.sock",
 413         "--workers=1",
 414         "tracker:application",
 415     ),
 416 }
 417 CLICK
 418 fi
 419
 420 service gunicorn start
 421
 422 ###############################################################################
 423 # nginx
 424 ###############################################################################
 425
 426 mkdir -p /srv/www/media
 427 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/media
 428
 429 cat > /etc/nginx/sites-available/reddit-media <<MEDIA
 430 server {
 431     listen 9000;
 432
 433     expires max;
 434
 435     location /media/ {
 436         alias /srv/www/media/;
 437     }
 438 }
 439 MEDIA
 440
 441 cat > /etc/nginx/sites-available/reddit-pixel <<PIXEL
 442 upstream click_server {
 443   server unix:/var/opt/reddit/click.sock fail_timeout=0;
 444 }
 445
 446 server {
 447   listen 8082;
 448
 449   log_format directlog '\$remote_addr - \$remote_user [\$time_local] '
 450                       '"\$request_method \$request_uri \$server_protocol" \$status \$body_bytes_sent '
 451                       '"\$http_referer" "\$http_user_agent"';
 452   access_log      /var/log/nginx/traffic/traffic.log directlog;
 453
 454   location / {
 455
 456     rewrite ^/pixel/of_ /pixel.png;
 457
 458     add_header Last-Modified "";
 459     add_header Pragma "no-cache";
 460
 461     expires -1;
 462     root /srv/www/pixel/;
 463   }
 464
 465   location /click {
 466     proxy_pass http://click_server;
 467   }
 468 }
 469 PIXEL
 470
 471 # remove the default nginx site that may conflict with haproxy
 472 rm -rf /etc/nginx/sites-enabled/default
 473 # put our config in place
 474 ln -nsf /etc/nginx/sites-available/reddit-media /etc/nginx/sites-enabled/
 475 ln -nsf /etc/nginx/sites-available/reddit-pixel /etc/nginx/sites-enabled/
 476
 477 # make the pixel log directory
 478 mkdir -p /var/log/nginx/traffic
 479
 480 # link the ini file for the Flask click tracker
 481 ln -nsf $REDDIT_HOME/src/reddit/r2/development.ini $REDDIT_HOME/src/reddit/scripts/production.ini
 482
 483 service nginx restart
 484
 485 ###############################################################################
 486 # haproxy
 487 ###############################################################################
 488 if [ -e /etc/haproxy/haproxy.cfg ]; then
 489     BACKUP_HAPROXY=$(mktemp /etc/haproxy/haproxy.cfg.XXX)
 490     echo "Backing up /etc/haproxy/haproxy.cfg to $BACKUP_HAPROXY"
 491     cat /etc/haproxy/haproxy.cfg > $BACKUP_HAPROXY
 492 fi
 493
 494 # make sure haproxy is enabled
 495 cat > /etc/default/haproxy <<DEFAULT
 496 ENABLED=1
 497 DEFAULT
 498
 499 # configure haproxy
 500 cat > /etc/haproxy/haproxy.cfg <<HAPROXY
 501 global
 502     maxconn 350
 503
 504 frontend frontend
 505     mode http
 506
 507     bind 0.0.0.0:80
 508     bind 127.0.0.1:8080
 509
 510     timeout client 24h
 511     option forwardfor except 127.0.0.1
 512     option httpclose
 513
 514     # make sure that requests have x-forwarded-proto: https iff tls
 515     reqidel ^X-Forwarded-Proto:.*
 516     acl is-ssl dst_port 8080
 517     reqadd X-Forwarded-Proto:\ https if is-ssl
 518
 519     # send websockets to sutro
 520     acl is-websocket hdr(Upgrade) -i WebSocket
 521     use_backend sutro if is-websocket
 522
 523     # send media stuff to the local nginx
 524     acl is-media path_beg /media/
 525     use_backend media if is-media
 526
 527     # send pixel stuff to local nginx
 528     acl is-pixel path_beg /pixel/
 529     acl is-click path_beg /click
 530     use_backend pixel if is-pixel || is-click
 531
 532     default_backend reddit
 533
 534 backend reddit
 535     mode http
 536     timeout connect 4000
 537     timeout server 30000
 538     timeout queue 60000
 539     balance roundrobin
 540
 541     server app01-8001 localhost:8001 maxconn 30
 542
 543 backend sutro
 544     mode http
 545     timeout connect 4s
 546     timeout server 24h
 547     balance roundrobin
 548
 549     server sutro localhost:8002 maxconn 250
 550
 551 backend media
 552     mode http
 553     timeout connect 4000
 554     timeout server 30000
 555     timeout queue 60000
 556     balance roundrobin
 557
 558     server nginx localhost:9000 maxconn 20
 559
 560 backend pixel
 561     mode http
 562     timeout connect 4000
 563     timeout server 30000
 564     timeout queue 60000
 565     balance roundrobin
 566
 567     server nginx localhost:8082 maxconn 20
 568 HAPROXY
 569
 570 # this will start it even if currently stopped
 571 service haproxy restart
 572
 573 ###############################################################################
 574 # stunnel
 575 ###############################################################################
 576 cat > /etc/stunnel/stunnel.conf <<STUNNELCONF
 577 foreground = no
 578
 579 ; replace these with real certificates
 580 cert = /etc/ssl/certs/ssl-cert-snakeoil.pem
 581 key = /etc/ssl/private/ssl-cert-snakeoil.key
 582
 583 ; protocol version and ciphers
 584 sslVersion = all
 585 ciphers = ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:ECDH-RSA-RC4-SHA:ECDH-ECDSA-RC4-SHA:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:SRP-DSS-AES-256-CBC-SHA:SRP-RSA-AES-256-CBC-SHA:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-CAMELLIA256-SHA:DHE-DSS-CAMELLIA256-SHA:ECDH-RSA-AES256-GCM-SHA384:ECDH-ECDSA-AES256-GCM-SHA384:ECDH-RSA-AES256-SHA384:ECDH-ECDSA-AES256-SHA384:ECDH-RSA-AES256-SHA:ECDH-ECDSA-AES256-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:PSK-AES256-CBC-SHA:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:SRP-DSS-AES-128-CBC-SHA:SRP-RSA-AES-128-CBC-SHA:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:DHE-RSA-SEED-SHA:DHE-DSS-SEED-SHA:DHE-RSA-CAMELLIA128-SHA:DHE-DSS-CAMELLIA128-SHA:ECDH-RSA-AES128-GCM-SHA256:ECDH-ECDSA-AES128-GCM-SHA256:ECDH-RSA-AES128-SHA256:ECDH-ECDSA-AES128-SHA256:ECDH-RSA-AES128-SHA:ECDH-ECDSA-AES128-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:SEED-SHA:CAMELLIA128-SHA:PSK-AES128-CBC-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5
 586 options = NO_SSLv2
 587 options = DONT_INSERT_EMPTY_FRAGMENTS
 588 options = CIPHER_SERVER_PREFERENCE
 589
 590 ; security
 591 chroot = /var/lib/stunnel4/
 592 setuid = stunnel4
 593 setgid = stunnel4
 594 pid = /stunnel4.pid
 595
 596 ; performance
 597 socket = l:TCP_NODELAY=1
 598 socket = r:TCP_NODELAY=1
 599
 600 ; logging
 601 output = /var/log/stunnel4/stunnel.log
 602 syslog = no
 603
 604 [https]
 605 accept = 443
 606 connect = 8080
 607 TIMEOUTclose = 0
 608 sslVersion = all
 609 ; this requires a patched version of stunnel which is in the reddit ppa
 610 xforwardedfor = yes
 611 STUNNELCONF
 612
 613 sed -i s/ENABLED=0/ENABLED=1/ /etc/default/stunnel4
 614
 615 service stunnel4 restart
 616
 617 ###############################################################################
 618 # sutro (websocket server)
 619 ###############################################################################
 620
 621 if [ ! -f /etc/sutro.ini ]; then
 622     cat > /etc/sutro.ini <<SUTRO
 623 [app:main]
 624 paste.app_factory = sutro.app:make_app
 625
 626 amqp.host = localhost
 627 amqp.port = 5672
 628 amqp.vhost = /
 629 amqp.username = reddit
 630 amqp.password = reddit
 631
 632 web.allowed_origins = $REDDIT_DOMAIN
 633 web.mac_secret = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5
 634 web.ping_interval = 300
 635
 636 stats.host =
 637 stats.port = 0
 638
 639 [server:main]
 640 use = egg:gunicorn#main
 641 worker_class = sutro.socketserver.SutroWorker
 642 workers = 1
 643 worker_connections = 250
 644 host = 127.0.0.1
 645 port = 8002
 646 graceful_timeout = 5
 647 forward_allow_ips = 127.0.0.1
 648
 649 [loggers]
 650 keys = root
 651
 652 [handlers]
 653 keys = syslog
 654
 655 [formatters]
 656 keys = generic
 657
 658 [logger_root]
 659 level = INFO
 660 handlers = syslog
 661
 662 [handler_syslog]
 663 class = handlers.SysLogHandler
 664 args = ("/dev/log", "local7")
 665 formatter = generic
 666 level = NOTSET
 667
 668 [formatter_generic]
 669 format = [%(name)s] %(message)s
 670 SUTRO
 671 fi
 672
 673 if [ ! -f /etc/init/sutro.conf ]; then
 674     cat > /etc/init/sutro.conf << UPSTART_SUTRO
 675 description "sutro websocket server"
 676
 677 stop on runlevel [!2345]
 678 start on runlevel [2345]
 679
 680 respawn
 681 respawn limit 10 5
 682 kill timeout 15
 683
 684 limit nofile 65535 65535
 685
 686 exec gunicorn_paster /etc/sutro.ini
 687 UPSTART_SUTRO
 688 fi
 689
 690 service sutro restart
 691
 692 ###############################################################################
 693 # geoip service
 694 ###############################################################################
 695 if [ ! -f /etc/gunicorn.d/geoip.conf ]; then
 696     cat > /etc/gunicorn.d/geoip.conf <<GEOIP
 697 CONFIG = {
 698     "mode": "wsgi",
 699     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 700     "user": "$REDDIT_USER",
 701     "group": "$REDDIT_USER",
 702     "args": (
 703         "--bind=127.0.0.1:5000",
 704         "--workers=1",
 705          "--limit-request-line=8190",
 706          "geoip_service:application",
 707     ),
 708 }
 709 GEOIP
 710 fi
 711
 712 service gunicorn start
 713
 714 ###############################################################################
 715 # Job Environment
 716 ###############################################################################
 717 CONSUMER_CONFIG_ROOT=$REDDIT_HOME/consumer-count.d
 718
 719 if [ ! -f /etc/default/reddit ]; then
 720     cat > /etc/default/reddit <<DEFAULT
 721 export REDDIT_ROOT=$REDDIT_HOME/src/reddit/r2
 722 export REDDIT_INI=$REDDIT_HOME/src/reddit/r2/run.ini
 723 export REDDIT_USER=$REDDIT_USER
 724 export REDDIT_GROUP=$REDDIT_GROUP
 725 export REDDIT_CONSUMER_CONFIG=$CONSUMER_CONFIG_ROOT
 726 alias wrap-job=$REDDIT_HOME/src/reddit/scripts/wrap-job
 727 alias manage-consumers=$REDDIT_HOME/src/reddit/scripts/manage-consumers
 728 DEFAULT
 729 fi
 730
 731 ###############################################################################
 732 # Queue Processors
 733 ###############################################################################
 734 mkdir -p $CONSUMER_CONFIG_ROOT
 735
 736 function set_consumer_count {
 737     if [ ! -f $CONSUMER_CONFIG_ROOT/$1 ]; then
 738         echo $2 > $CONSUMER_CONFIG_ROOT/$1
 739     fi
 740 }
 741
 742 set_consumer_count log_q 0
 743 set_consumer_count search_q 0
 744 set_consumer_count del_account_q 1
 745 set_consumer_count scraper_q 1
 746 set_consumer_count markread_q 1
 747 set_consumer_count commentstree_q 1
 748 set_consumer_count newcomments_q 1
 749 set_consumer_count vote_link_q 1
 750 set_consumer_count vote_comment_q 1
 751 set_consumer_count automoderator_q 0
 752
 753 chown -R $REDDIT_USER:$REDDIT_GROUP $CONSUMER_CONFIG_ROOT/
 754
 755 initctl emit reddit-stop
 756 initctl emit reddit-start
 757
 758 ###############################################################################
 759 # Cron Jobs
 760 ###############################################################################
 761 if [ ! -f /etc/cron.d/reddit ]; then
 762     cat > /etc/cron.d/reddit <<CRON
 763 0    3 * * * root /sbin/start --quiet reddit-job-update_sr_names
 764 30  16 * * * root /sbin/start --quiet reddit-job-update_reddits
 765 0    * * * * root /sbin/start --quiet reddit-job-update_promos
 766 */5  * * * * root /sbin/start --quiet reddit-job-clean_up_hardcache
 767 */2  * * * * root /sbin/start --quiet reddit-job-broken_things
 768 */2  * * * * root /sbin/start --quiet reddit-job-rising
 769 0    * * * * root /sbin/start --quiet reddit-job-trylater
 770
 771 # liveupdate
 772 *    * * * * root /sbin/start --quiet reddit-job-liveupdate_activity
 773
 774 # jobs that recalculate time-limited listings (e.g. top this year)
 775 PGPASSWORD=password
 776 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings link year '("hour", "day", "week", "month", "year")'
 777 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings comment year '("hour", "day", "week", "month", "year")'
 778
 779 # disabled by default, uncomment if you need these jobs
 780 #*    * * * * root /sbin/start --quiet reddit-job-email
 781 #0    0 * * * root /sbin/start --quiet reddit-job-update_gold_users
 782 CRON
 783 fi
 784
 785 ###############################################################################
 786 # All done!
 787 ###############################################################################
 788 cd $REDDIT_HOME
 789
 790 cat <<CONCLUSION
 791
 792 Congratulations! reddit is now installed.
 793
 794 The reddit application code is managed with upstart, to see what's currently
 795 running, run
 796
 797     sudo initctl list | grep reddit
 798
 799 Cron jobs start with "reddit-job-" and queue processors start with
 800 "reddit-consumer-". The crons are managed by /etc/cron.d/reddit. You can
 801 initiate a restart of all the consumers by running:
 802
 803     sudo reddit-restart
 804
 805 or target specific ones:
 806
 807     sudo reddit-restart scraper_q
 808
 809 See the GitHub wiki for more information on these jobs:
 810
 811 * https://github.com/reddit/reddit/wiki/Cron-jobs
 812 * https://github.com/reddit/reddit/wiki/Services
 813
 814 The reddit code can be shut down or started up with
 815
 816     sudo reddit-stop
 817     sudo reddit-start
 818
 819 And if you think caching might be hurting you, you can flush memcache with
 820
 821     reddit-flush
 822
 823 Now that the core of reddit is installed, you may want to do some additional
 824 steps:
 825
 826 * Ensure that $REDDIT_DOMAIN resolves to this machine.
 827
 828 * To populate the database with test data, run:
 829
 830     cd $REDDIT_HOME/src/reddit
 831     reddit-run scripts/inject_test_data.py -c 'inject_test_data()'
 832
 833 * Manually run reddit-job-update_reddits immediately after populating the db
 834   or adding your own subreddits.
 835 CONCLUSION