install-reddit.sh

   1 #!/bin/bash
   2 # The contents of this file are subject to the Common Public Attribution
   3 # License Version 1.0. (the "License"); you may not use this file except in
   4 # compliance with the License. You may obtain a copy of the License at
   5 # http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
   6 # License Version 1.1, but Sections 14 and 15 have been added to cover use of
   7 # software over a computer network and provide for limited attribution for the
   8 # Original Developer. In addition, Exhibit A has been modified to be consistent
   9 # with Exhibit B.
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
  13 # the specific language governing rights and limitations under the License.
  14 #
  15 # The Original Code is reddit.
  16 #
  17 # The Original Developer is the Initial Developer.  The Initial Developer of
  18 # the Original Code is reddit Inc.
  19 #
  20 # All portions of the code written by reddit are Copyright (c) 2006-2014 reddit
  21 # Inc. All Rights Reserved.
  22 ###############################################################################
  23
  24 ###############################################################################
  25 # reddit dev environment installer
  26 # --------------------------------
  27 # This script installs a reddit stack suitable for development. DO NOT run this
  28 # on a system that you use for other purposes as it might delete important
  29 # files, truncate your databases, and otherwise do mean things to you.
  30 #
  31 # By default, this script will install the reddit code in the current user's
  32 # home directory and all of its dependencies (including libraries and database
  33 # servers) at the system level. The installed reddit will expect to be visited
  34 # on the domain "reddit.local" unless specified otherwise.  Configuring name
  35 # resolution for the domain is expected to be done outside the installed
  36 # environment (e.g. in your host machine's /etc/hosts file) and is not
  37 # something this script handles.
  38 #
  39 # Several configuration options (listed in the "Configuration" section below)
  40 # are overridable with environment variables. e.g.
  41 #
  42 #    sudo REDDIT_DOMAIN=example.com ./install-reddit.sh
  43 #
  44 ###############################################################################
  45 set -e
  46
  47 ###############################################################################
  48 # Configuration
  49 ###############################################################################
  50 # which user to install the code for; defaults to the user invoking this script
  51 REDDIT_USER=${REDDIT_USER:-$SUDO_USER}
  52
  53 # the group to run reddit code as; must exist already
  54 REDDIT_GROUP=${REDDIT_GROUP:-nogroup}
  55
  56 # the root directory to base the install in. must exist already
  57 REDDIT_HOME=${REDDIT_HOME:-/home/$REDDIT_USER}
  58
  59 # the domain that you will connect to your reddit install with.
  60 # MUST contain a . in it somewhere as browsers won't do cookies for dotless
  61 # domains. an IP address will suffice if nothing else is available.
  62 REDDIT_DOMAIN=${REDDIT_DOMAIN:-reddit.local}
  63
  64 #The plugins to clone and register in the ini file
  65 REDDIT_PLUGINS=${REDDIT_PLUGINS:-meatspace about liveupdate}
  66
  67 ###############################################################################
  68 # Sanity Checks
  69 ###############################################################################
  70 if [[ $EUID -ne 0 ]]; then
  71     echo "ERROR: Must be run with root privileges."
  72     exit 1
  73 fi
  74
  75 # seriously! these checks are here for a reason. the packages from the
  76 # reddit ppa aren't built for anything but precise (12.04) right now, so
  77 # if you try and use this install script on another release you're gonna
  78 # have a bad time.
  79 source /etc/lsb-release
  80 if [ "$DISTRIB_ID" != "Ubuntu" -o "$DISTRIB_RELEASE" != "12.04" ]; then
  81     echo "ERROR: Only Ubuntu 12.04 is supported."
  82     exit 1
  83 fi
  84
  85 if [[ "2000000" -gt $(awk '/MemTotal/{print $2}' /proc/meminfo) ]]; then
  86     LOW_MEM_PROMPT="reddit requires at least 2GB of memory to work properly, continue anyway? [y/n] "
  87     read -er -n1 -p "$LOW_MEM_PROMPT" response
  88     if [[ "$response" != "y" ]]; then
  89       echo "Quitting."
  90       exit 1
  91     fi
  92 fi
  93
  94 ###############################################################################
  95 # Install prerequisites
  96 ###############################################################################
  97 set -x
  98
  99 # aptitude configuration
 100 APTITUDE_OPTIONS="-y"
 101 export DEBIAN_FRONTEND=noninteractive
 102
 103 # run an aptitude update to make sure python-software-properties
 104 # dependencies are found
 105 apt-get update
 106
 107 # add the reddit ppa for some custom packages
 108 apt-get install $APTITUDE_OPTIONS python-software-properties
 109 apt-add-repository -y ppa:reddit/ppa
 110
 111 # pin the ppa -- packages present in the ppa will take precedence over
 112 # ones in other repositories (unless further pinning is done)
 113 cat <<HERE > /etc/apt/preferences.d/reddit
 114 Package: *
 115 Pin: release o=LP-PPA-reddit
 116 Pin-Priority: 600
 117 HERE
 118
 119 # grab the new ppas' package listings
 120 apt-get update
 121
 122 # install prerequisites
 123 cat <<PACKAGES | xargs apt-get install $APTITUDE_OPTIONS
 124 netcat-openbsd
 125 git-core
 126
 127 python-dev
 128 python-setuptools
 129 python-routes
 130 python-pylons
 131 python-boto
 132 python-tz
 133 python-crypto
 134 python-babel
 135 cython
 136 python-sqlalchemy
 137 python-beautifulsoup
 138 python-chardet
 139 python-psycopg2
 140 python-pycassa
 141 python-imaging
 142 python-pycaptcha
 143 python-amqplib
 144 python-pylibmc
 145 python-bcrypt
 146 python-snudown
 147 python-l2cs
 148 python-lxml
 149 python-zope.interface
 150 python-kazoo
 151 python-stripe
 152 python-tinycss2
 153
 154 python-flask
 155 geoip-bin
 156 geoip-database
 157 python-geoip
 158
 159 nodejs
 160 node-less
 161 node-uglify
 162 gettext
 163 make
 164 optipng
 165 jpegoptim
 166
 167 memcached
 168 postgresql
 169 postgresql-client
 170 rabbitmq-server
 171 cassandra
 172 haproxy
 173 nginx
 174 stunnel
 175 gunicorn
 176 sutro
 177 libpcre3-dev
 178 PACKAGES
 179
 180 # paper over stack size issues with cassandra
 181 sed -i s/-Xss128k/-Xss228k/ /etc/cassandra/cassandra-env.sh
 182
 183 ###############################################################################
 184 # Wait for all the services to be up
 185 ###############################################################################
 186 # cassandra doesn't auto-start after install
 187 service cassandra start
 188
 189 # check each port for connectivity
 190 echo "Waiting for services to be available, see source for port meanings..."
 191 # 11211 - memcache
 192 # 5432 - postgres
 193 # 5672 - rabbitmq
 194 # 9160 - cassandra
 195 for port in 11211 5432 5672 9160; do
 196     while ! nc -vz localhost $port; do
 197         sleep 1
 198     done
 199 done
 200
 201 ###############################################################################
 202 # Install the reddit source repositories
 203 ###############################################################################
 204 if [ ! -d $REDDIT_HOME/src ]; then
 205     mkdir -p $REDDIT_HOME/src
 206     chown $REDDIT_USER $REDDIT_HOME/src
 207 fi
 208
 209 function clone_reddit_repo {
 210     local destination=$REDDIT_HOME/src/${1}
 211     local repository_url=https://github.com/${2}.git
 212
 213     if [ ! -d $destination ]; then
 214         sudo -u $REDDIT_USER git clone $repository_url $destination
 215     fi
 216
 217     if [ -d $destination/upstart ]; then
 218         cp $destination/upstart/* /etc/init/
 219     fi
 220 }
 221
 222 function clone_reddit_plugin_repo {
 223     clone_reddit_repo $1 reddit/reddit-plugin-$1
 224 }
 225
 226 clone_reddit_repo reddit reddit/reddit
 227 clone_reddit_repo i18n reddit/reddit-i18n
 228 for plugin in $REDDIT_PLUGINS; do
 229     clone_reddit_plugin_repo $plugin
 230 done
 231
 232 ###############################################################################
 233 # Configure Cassandra
 234 ###############################################################################
 235 if ! echo | cassandra-cli -h localhost -k reddit &> /dev/null; then
 236     echo "create keyspace reddit;" | cassandra-cli -h localhost -B
 237 fi
 238
 239 cat <<CASS | cassandra-cli -B -h localhost -k reddit || true
 240 create column family permacache with column_type = 'Standard' and
 241                                      comparator = 'BytesType';
 242 CASS
 243
 244 ###############################################################################
 245 # Configure PostgreSQL
 246 ###############################################################################
 247 SQL="SELECT COUNT(1) FROM pg_catalog.pg_database WHERE datname = 'reddit';"
 248 IS_DATABASE_CREATED=$(sudo -u postgres psql -t -c "$SQL")
 249
 250 if [ $IS_DATABASE_CREATED -ne 1 ]; then
 251     cat <<PGSCRIPT | sudo -u postgres psql
 252 CREATE DATABASE reddit WITH ENCODING = 'utf8' TEMPLATE template0 LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8';
 253 CREATE USER reddit WITH PASSWORD 'password';
 254 PGSCRIPT
 255 fi
 256
 257 sudo -u postgres psql reddit < $REDDIT_HOME/src/reddit/sql/functions.sql
 258
 259 ###############################################################################
 260 # Configure RabbitMQ
 261 ###############################################################################
 262 if ! rabbitmqctl list_vhosts | egrep "^/$"
 263 then
 264     rabbitmqctl add_vhost /
 265 fi
 266
 267 if ! rabbitmqctl list_users | egrep "^reddit"
 268 then
 269     rabbitmqctl add_user reddit reddit
 270 fi
 271
 272 rabbitmqctl set_permissions -p / reddit ".*" ".*" ".*"
 273
 274 ###############################################################################
 275 # Install and configure the reddit code
 276 ###############################################################################
 277 function install_reddit_repo {
 278     cd $REDDIT_HOME/src/$1
 279     sudo -u $REDDIT_USER python setup.py build
 280     python setup.py develop --no-deps
 281 }
 282
 283 install_reddit_repo reddit/r2
 284 install_reddit_repo i18n
 285 for plugin in $REDDIT_PLUGINS; do
 286     install_reddit_repo $plugin
 287 done
 288
 289 # generate binary translation files from source
 290 cd $REDDIT_HOME/src/i18n/
 291 sudo -u $REDDIT_USER make clean all
 292
 293 # this builds static files and should be run *after* languages are installed
 294 # so that the proper language-specific static files can be generated and after
 295 # plugins are installed so all the static files are available.
 296 cd $REDDIT_HOME/src/reddit/r2
 297 sudo -u $REDDIT_USER make clean all
 298
 299 plugin_str=$(echo -n "$REDDIT_PLUGINS" | tr " " ,)
 300 if [ ! -f development.update ]; then
 301     cat > development.update <<DEVELOPMENT
 302 # after editing this file, run "make ini" to
 303 # generate a new development.ini
 304
 305 [DEFAULT]
 306 debug = true
 307
 308 disable_ads = true
 309 disable_captcha = true
 310 disable_ratelimit = true
 311 disable_require_admin_otp = true
 312
 313 page_cache_time = 0
 314
 315 domain = $REDDIT_DOMAIN
 316
 317 plugins = $plugin_str
 318
 319 media_provider = filesystem
 320 media_fs_root = /srv/www/media
 321 media_fs_base_url_http = http://%(domain)s/media/
 322 media_fs_base_url_https = https://%(domain)s/media/
 323
 324 [server:main]
 325 port = 8001
 326 DEVELOPMENT
 327     chown $REDDIT_USER development.update
 328 else
 329     sed -i "s/^plugins = .*$/plugins = $plugin_str/" $REDDIT_HOME/src/reddit/r2/development.update
 330     sed -i "s/^domain = .*$/domain = $REDDIT_DOMAIN/" $REDDIT_HOME/src/reddit/r2/development.update
 331 fi
 332
 333 sudo -u $REDDIT_USER make ini
 334
 335 if [ ! -L run.ini ]; then
 336     sudo -u $REDDIT_USER ln -nsf development.ini run.ini
 337 fi
 338
 339 ###############################################################################
 340 # some useful helper scripts
 341 ###############################################################################
 342 function helper-script() {
 343     cat > $1
 344     chmod 755 $1
 345 }
 346
 347 helper-script /usr/local/bin/reddit-run <<REDDITRUN
 348 #!/bin/bash
 349 exec paster --plugin=r2 run $REDDIT_HOME/src/reddit/r2/run.ini "\$@"
 350 REDDITRUN
 351
 352 helper-script /usr/local/bin/reddit-shell <<REDDITSHELL
 353 #!/bin/bash
 354 exec paster --plugin=r2 shell $REDDIT_HOME/src/reddit/r2/run.ini
 355 REDDITSHELL
 356
 357 helper-script /usr/local/bin/reddit-start <<REDDITSTART
 358 #!/bin/bash
 359 initctl emit reddit-start
 360 REDDITSTART
 361
 362 helper-script /usr/local/bin/reddit-stop <<REDDITSTOP
 363 #!/bin/bash
 364 initctl emit reddit-stop
 365 REDDITSTOP
 366
 367 helper-script /usr/local/bin/reddit-restart <<REDDITRESTART
 368 #!/bin/bash
 369 initctl emit reddit-restart TARGET=${1:-all}
 370 REDDITRESTART
 371
 372 helper-script /usr/local/bin/reddit-flush <<REDDITFLUSH
 373 #!/bin/bash
 374 echo flush_all | nc localhost 11211
 375 REDDITFLUSH
 376
 377 ###############################################################################
 378 # pixel and click server
 379 ###############################################################################
 380 mkdir -p /var/opt/reddit/
 381 chown $REDDIT_USER:$REDDIT_GROUP /var/opt/reddit/
 382
 383 mkdir -p /srv/www/pixel
 384 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/pixel
 385 cp $REDDIT_HOME/src/reddit/r2/r2/public/static/pixel.png /srv/www/pixel
 386
 387 if [ ! -f /etc/gunicorn.d/click.conf ]; then
 388     cat > /etc/gunicorn.d/click.conf <<CLICK
 389 CONFIG = {
 390     "mode": "wsgi",
 391     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 392     "user": "$REDDIT_USER",
 393     "group": "$REDDIT_USER",
 394     "args": (
 395         "--bind=unix:/var/opt/reddit/click.sock",
 396         "--workers=1",
 397         "tracker:application",
 398     ),
 399 }
 400 CLICK
 401 fi
 402
 403 service gunicorn start
 404
 405 ###############################################################################
 406 # nginx
 407 ###############################################################################
 408
 409 mkdir -p /srv/www/media
 410 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/media
 411
 412 cat > /etc/nginx/sites-available/reddit-media <<MEDIA
 413 server {
 414     listen 9000;
 415
 416     expires max;
 417
 418     location /media/ {
 419         alias /srv/www/media/;
 420     }
 421 }
 422 MEDIA
 423
 424 cat > /etc/nginx/sites-available/reddit-pixel <<PIXEL
 425 upstream click_server {
 426   server unix:/var/opt/reddit/click.sock fail_timeout=0;
 427 }
 428
 429 server {
 430   listen 8082;
 431
 432   log_format directlog '\$remote_addr - \$remote_user [\$time_local] '
 433                       '"\$request_method \$request_uri \$server_protocol" \$status \$body_bytes_sent '
 434                       '"\$http_referer" "\$http_user_agent"';
 435   access_log      /var/log/nginx/traffic/traffic.log directlog;
 436
 437   location / {
 438
 439     rewrite ^/pixel/of_ /pixel.png;
 440
 441     add_header Last-Modified "";
 442     add_header Pragma "no-cache";
 443
 444     expires -1;
 445     root /srv/www/pixel/;
 446   }
 447
 448   location /click {
 449     proxy_pass http://click_server;
 450   }
 451 }
 452 PIXEL
 453
 454 # remove the default nginx site that may conflict with haproxy
 455 rm -rf /etc/nginx/sites-enabled/default
 456 # put our config in place
 457 ln -nsf /etc/nginx/sites-available/reddit-media /etc/nginx/sites-enabled/
 458 ln -nsf /etc/nginx/sites-available/reddit-pixel /etc/nginx/sites-enabled/
 459
 460 # make the pixel log directory
 461 mkdir -p /var/log/nginx/traffic
 462
 463 # link the ini file for the Flask click tracker
 464 ln -nsf $REDDIT_HOME/src/reddit/r2/development.ini $REDDIT_HOME/src/reddit/scripts/production.ini
 465
 466 service nginx restart
 467
 468 ###############################################################################
 469 # haproxy
 470 ###############################################################################
 471 if [ -e /etc/haproxy/haproxy.cfg ]; then
 472     BACKUP_HAPROXY=$(mktemp /etc/haproxy/haproxy.cfg.XXX)
 473     echo "Backing up /etc/haproxy/haproxy.cfg to $BACKUP_HAPROXY"
 474     cat /etc/haproxy/haproxy.cfg > $BACKUP_HAPROXY
 475 fi
 476
 477 # make sure haproxy is enabled
 478 cat > /etc/default/haproxy <<DEFAULT
 479 ENABLED=1
 480 DEFAULT
 481
 482 # configure haproxy
 483 cat > /etc/haproxy/haproxy.cfg <<HAPROXY
 484 global
 485     maxconn 350
 486
 487 frontend frontend
 488     mode http
 489
 490     bind 0.0.0.0:80
 491     bind 127.0.0.1:8080
 492
 493     timeout client 24h
 494     option forwardfor except 127.0.0.1
 495     option httpclose
 496
 497     # make sure that requests have x-forwarded-proto: https iff tls
 498     reqidel ^X-Forwarded-Proto:.*
 499     acl is-ssl dst_port 8080
 500     reqadd X-Forwarded-Proto:\ https if is-ssl
 501
 502     # send websockets to sutro
 503     acl is-websocket hdr(Upgrade) -i WebSocket
 504     use_backend sutro if is-websocket
 505
 506     # send media stuff to the local nginx
 507     acl is-media path_beg /media/
 508     use_backend media if is-media
 509
 510     # send pixel stuff to local nginx
 511     acl is-pixel path_beg /pixel/
 512     acl is-click path_beg /click
 513     use_backend pixel if is-pixel || is-click
 514
 515     default_backend reddit
 516
 517 backend reddit
 518     mode http
 519     timeout connect 4000
 520     timeout server 30000
 521     timeout queue 60000
 522     balance roundrobin
 523
 524     server app01-8001 localhost:8001 maxconn 30
 525
 526 backend sutro
 527     mode http
 528     timeout connect 4s
 529     timeout server 24h
 530     balance roundrobin
 531
 532     server sutro localhost:8002 maxconn 250
 533
 534 backend media
 535     mode http
 536     timeout connect 4000
 537     timeout server 30000
 538     timeout queue 60000
 539     balance roundrobin
 540
 541     server nginx localhost:9000 maxconn 20
 542
 543 backend pixel
 544     mode http
 545     timeout connect 4000
 546     timeout server 30000
 547     timeout queue 60000
 548     balance roundrobin
 549
 550     server nginx localhost:8082 maxconn 20
 551 HAPROXY
 552
 553 # this will start it even if currently stopped
 554 service haproxy restart
 555
 556 ###############################################################################
 557 # stunnel
 558 ###############################################################################
 559 cat > /etc/stunnel/stunnel.conf <<STUNNELCONF
 560 foreground = no
 561
 562 ; replace these with real certificates
 563 cert = /etc/ssl/certs/ssl-cert-snakeoil.pem
 564 key = /etc/ssl/private/ssl-cert-snakeoil.key
 565
 566 ; protocol version and ciphers
 567 sslVersion = all
 568 ciphers = ECDHE-RSA-RC4-SHA:ECDHE-ECDSA-RC4-SHA:ECDH-RSA-RC4-SHA:ECDH-ECDSA-RC4-SHA:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:SRP-DSS-AES-256-CBC-SHA:SRP-RSA-AES-256-CBC-SHA:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-CAMELLIA256-SHA:DHE-DSS-CAMELLIA256-SHA:ECDH-RSA-AES256-GCM-SHA384:ECDH-ECDSA-AES256-GCM-SHA384:ECDH-RSA-AES256-SHA384:ECDH-ECDSA-AES256-SHA384:ECDH-RSA-AES256-SHA:ECDH-ECDSA-AES256-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:PSK-AES256-CBC-SHA:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:SRP-DSS-AES-128-CBC-SHA:SRP-RSA-AES-128-CBC-SHA:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:DHE-RSA-SEED-SHA:DHE-DSS-SEED-SHA:DHE-RSA-CAMELLIA128-SHA:DHE-DSS-CAMELLIA128-SHA:ECDH-RSA-AES128-GCM-SHA256:ECDH-ECDSA-AES128-GCM-SHA256:ECDH-RSA-AES128-SHA256:ECDH-ECDSA-AES128-SHA256:ECDH-RSA-AES128-SHA:ECDH-ECDSA-AES128-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:SEED-SHA:CAMELLIA128-SHA:PSK-AES128-CBC-SHA:RC4-SHA:DES-CBC3-SHA:RC4-MD5
 569 options = NO_SSLv2
 570 options = DONT_INSERT_EMPTY_FRAGMENTS
 571 options = CIPHER_SERVER_PREFERENCE
 572
 573 ; security
 574 chroot = /var/lib/stunnel4/
 575 setuid = stunnel4
 576 setgid = stunnel4
 577 pid = /stunnel4.pid
 578
 579 ; performance
 580 socket = l:TCP_NODELAY=1
 581 socket = r:TCP_NODELAY=1
 582
 583 ; logging
 584 output = /var/log/stunnel4/stunnel.log
 585 syslog = no
 586
 587 [https]
 588 accept = 443
 589 connect = 8080
 590 TIMEOUTclose = 0
 591 sslVersion = all
 592 ; this requires a patched version of stunnel which is in the reddit ppa
 593 xforwardedfor = yes
 594 STUNNELCONF
 595
 596 sed -i s/ENABLED=0/ENABLED=1/ /etc/default/stunnel4
 597
 598 service stunnel4 restart
 599
 600 ###############################################################################
 601 # sutro (websocket server)
 602 ###############################################################################
 603
 604 if [ ! -f /etc/sutro.ini ]; then
 605     cat > /etc/sutro.ini <<SUTRO
 606 [app:main]
 607 paste.app_factory = sutro.app:make_app
 608
 609 amqp.host = localhost
 610 amqp.port = 5672
 611 amqp.vhost = /
 612 amqp.username = reddit
 613 amqp.password = reddit
 614
 615 web.allowed_origins = $REDDIT_DOMAIN
 616 web.mac_secret = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5
 617 web.ping_interval = 300
 618
 619 stats.host =
 620 stats.port = 0
 621
 622 [server:main]
 623 use = egg:gunicorn#main
 624 worker_class = sutro.socketserver.SutroWorker
 625 workers = 1
 626 worker_connections = 250
 627 host = 127.0.0.1
 628 port = 8002
 629 graceful_timeout = 5
 630 forward_allow_ips = 127.0.0.1
 631
 632 [loggers]
 633 keys = root
 634
 635 [handlers]
 636 keys = syslog
 637
 638 [formatters]
 639 keys = generic
 640
 641 [logger_root]
 642 level = INFO
 643 handlers = syslog
 644
 645 [handler_syslog]
 646 class = handlers.SysLogHandler
 647 args = ("/dev/log", "local7")
 648 formatter = generic
 649 level = NOTSET
 650
 651 [formatter_generic]
 652 format = [%(name)s] %(message)s
 653 SUTRO
 654 fi
 655
 656 if [ ! -f /etc/init/sutro.conf ]; then
 657     cat > /etc/init/sutro.conf << UPSTART_SUTRO
 658 description "sutro websocket server"
 659
 660 stop on runlevel [!2345]
 661 start on runlevel [2345]
 662
 663 respawn
 664 respawn limit 10 5
 665 kill timeout 15
 666
 667 limit nofile 65535 65535
 668
 669 exec gunicorn_paster /etc/sutro.ini
 670 UPSTART_SUTRO
 671 fi
 672
 673 service sutro restart
 674
 675 ###############################################################################
 676 # geoip service
 677 ###############################################################################
 678 if [ ! -f /etc/gunicorn.d/geoip.conf ]; then
 679     cat > /etc/gunicorn.d/geoip.conf <<GEOIP
 680 CONFIG = {
 681     "mode": "wsgi",
 682     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 683     "user": "$REDDIT_USER",
 684     "group": "$REDDIT_USER",
 685     "args": (
 686         "--bind=127.0.0.1:5000",
 687         "--workers=1",
 688          "--limit-request-line=8190",
 689          "geoip_service:application",
 690     ),
 691 }
 692 GEOIP
 693 fi
 694
 695 service gunicorn start
 696
 697 ###############################################################################
 698 # Job Environment
 699 ###############################################################################
 700 CONSUMER_CONFIG_ROOT=$REDDIT_HOME/consumer-count.d
 701
 702 if [ ! -f /etc/default/reddit ]; then
 703     cat > /etc/default/reddit <<DEFAULT
 704 export REDDIT_ROOT=$REDDIT_HOME/src/reddit/r2
 705 export REDDIT_INI=$REDDIT_HOME/src/reddit/r2/run.ini
 706 export REDDIT_USER=$REDDIT_USER
 707 export REDDIT_GROUP=$REDDIT_GROUP
 708 export REDDIT_CONSUMER_CONFIG=$CONSUMER_CONFIG_ROOT
 709 alias wrap-job=$REDDIT_HOME/src/reddit/scripts/wrap-job
 710 alias manage-consumers=$REDDIT_HOME/src/reddit/scripts/manage-consumers
 711 DEFAULT
 712 fi
 713
 714 ###############################################################################
 715 # Queue Processors
 716 ###############################################################################
 717 mkdir -p $CONSUMER_CONFIG_ROOT
 718
 719 function set_consumer_count {
 720     if [ ! -f $CONSUMER_CONFIG_ROOT/$1 ]; then
 721         echo $2 > $CONSUMER_CONFIG_ROOT/$1
 722     fi
 723 }
 724
 725 set_consumer_count log_q 0
 726 set_consumer_count cloudsearch_q 0
 727 set_consumer_count scraper_q 1
 728 set_consumer_count markread_q 1
 729 set_consumer_count commentstree_q 1
 730 set_consumer_count newcomments_q 1
 731 set_consumer_count vote_link_q 1
 732 set_consumer_count vote_comment_q 1
 733
 734 chown -R $REDDIT_USER:$REDDIT_GROUP $CONSUMER_CONFIG_ROOT/
 735
 736 initctl emit reddit-stop
 737 initctl emit reddit-start
 738
 739 ###############################################################################
 740 # Cron Jobs
 741 ###############################################################################
 742 if [ ! -f /etc/cron.d/reddit ]; then
 743     cat > /etc/cron.d/reddit <<CRON
 744 0    3 * * * root /sbin/start --quiet reddit-job-update_sr_names
 745 30  16 * * * root /sbin/start --quiet reddit-job-update_reddits
 746 0    * * * * root /sbin/start --quiet reddit-job-update_promos
 747 */5  * * * * root /sbin/start --quiet reddit-job-clean_up_hardcache
 748 */2  * * * * root /sbin/start --quiet reddit-job-broken_things
 749 */2  * * * * root /sbin/start --quiet reddit-job-rising
 750 0    * * * * root /sbin/start --quiet reddit-job-trylater
 751
 752 # liveupdate
 753 *    * * * * root /sbin/start --quiet reddit-job-liveupdate_activity
 754
 755 # jobs that recalculate time-limited listings (e.g. top this year)
 756 PGPASSWORD=password
 757 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings link year '("hour", "day", "week", "month", "year")'
 758 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings comment year '("hour", "day", "week", "month", "year")'
 759
 760 # disabled by default, uncomment if you need these jobs
 761 #*    * * * * root /sbin/start --quiet reddit-job-email
 762 #0    0 * * * root /sbin/start --quiet reddit-job-update_gold_users
 763 CRON
 764 fi
 765
 766 ###############################################################################
 767 # All done!
 768 ###############################################################################
 769 cd $REDDIT_HOME
 770
 771 cat <<CONCLUSION
 772
 773 Congratulations! reddit is now installed.
 774
 775 The reddit application code is managed with upstart, to see what's currently
 776 running, run
 777
 778     sudo initctl list | grep reddit
 779
 780 Cron jobs start with "reddit-job-" and queue processors start with
 781 "reddit-consumer-". The crons are managed by /etc/cron.d/reddit. You can
 782 initiate a restart of all the consumers by running:
 783
 784     sudo reddit-restart
 785
 786 or target specific ones:
 787
 788     sudo reddit-restart scraper_q
 789
 790 See the GitHub wiki for more information on these jobs:
 791
 792 * https://github.com/reddit/reddit/wiki/Cron-jobs
 793 * https://github.com/reddit/reddit/wiki/Services
 794
 795 The reddit code can be shut down or started up with
 796
 797     sudo reddit-stop
 798     sudo reddit-start
 799
 800 And if you think caching might be hurting you, you can flush memcache with
 801
 802     reddit-flush
 803
 804 Now that the core of reddit is installed, you may want to do some additional
 805 steps:
 806
 807 * Ensure that $REDDIT_DOMAIN resolves to this machine.
 808
 809 * To populate the database with test data, run:
 810
 811     cd $REDDIT_HOME/src/reddit/r2
 812     reddit-run r2/models/populatedb.py -c 'populate()'
 813
 814 * Manually run reddit-job-update_reddits immediately after populating the db
 815   or adding your own subreddits.
 816 CONCLUSION