install-reddit.sh

   1 #!/bin/bash
   2 # The contents of this file are subject to the Common Public Attribution
   3 # License Version 1.0. (the "License"); you may not use this file except in
   4 # compliance with the License. You may obtain a copy of the License at
   5 # http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
   6 # License Version 1.1, but Sections 14 and 15 have been added to cover use of
   7 # software over a computer network and provide for limited attribution for the
   8 # Original Developer. In addition, Exhibit A has been modified to be consistent
   9 # with Exhibit B.
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
  13 # the specific language governing rights and limitations under the License.
  14 #
  15 # The Original Code is reddit.
  16 #
  17 # The Original Developer is the Initial Developer.  The Initial Developer of
  18 # the Original Code is reddit Inc.
  19 #
  20 # All portions of the code written by reddit are Copyright (c) 2006-2015 reddit
  21 # Inc. All Rights Reserved.
  22 ###############################################################################
  23
  24 ###############################################################################
  25 # reddit dev environment installer
  26 # --------------------------------
  27 # This script installs a reddit stack suitable for development. DO NOT run this
  28 # on a system that you use for other purposes as it might delete important
  29 # files, truncate your databases, and otherwise do mean things to you.
  30 #
  31 # By default, this script will install the reddit code in the current user's
  32 # home directory and all of its dependencies (including libraries and database
  33 # servers) at the system level. The installed reddit will expect to be visited
  34 # on the domain "reddit.local" unless specified otherwise.  Configuring name
  35 # resolution for the domain is expected to be done outside the installed
  36 # environment (e.g. in your host machine's /etc/hosts file) and is not
  37 # something this script handles.
  38 #
  39 # Several configuration options (listed in the "Configuration" section below)
  40 # are overridable with environment variables. e.g.
  41 #
  42 #    sudo REDDIT_DOMAIN=example.com ./install-reddit.sh
  43 #
  44 ###############################################################################
  45 set -e
  46
  47 ###############################################################################
  48 # Configuration
  49 ###############################################################################
  50 # which user to install the code for; defaults to the user invoking this script
  51 REDDIT_USER=${REDDIT_USER:-$SUDO_USER}
  52
  53 # the group to run reddit code as; must exist already
  54 REDDIT_GROUP=${REDDIT_GROUP:-nogroup}
  55
  56 # the root directory to base the install in. must exist already
  57 REDDIT_HOME=${REDDIT_HOME:-/home/$REDDIT_USER}
  58
  59 # the domain that you will connect to your reddit install with.
  60 # MUST contain a . in it somewhere as browsers won't do cookies for dotless
  61 # domains. an IP address will suffice if nothing else is available.
  62 REDDIT_DOMAIN=${REDDIT_DOMAIN:-reddit.local}
  63
  64 #The plugins to clone and register in the ini file
  65 REDDIT_PLUGINS=${REDDIT_PLUGINS:-meatspace about liveupdate}
  66
  67 ###############################################################################
  68 # Sanity Checks
  69 ###############################################################################
  70 if [[ $EUID -ne 0 ]]; then
  71     echo "ERROR: Must be run with root privileges."
  72     exit 1
  73 fi
  74
  75 if [[ -z "$REDDIT_USER" ]]; then
  76     # in a production install, you'd want the code to be owned by root and run
  77     # by a less privileged user. this script is intended to build a development
  78     # install, so we expect the owner to run the app and not be root.
  79     cat <<END
  80 ERROR: You have not specified a user. This usually means you're running this
  81 script directly as root. It is not recommended to run reddit as the root user.
  82
  83 Please create a user to run reddit and set the REDDIT_USER variable
  84 appropriately.
  85 END
  86     exit 1
  87 fi
  88
  89 if [[ "amd64" != $(dpkg --print-architecture) ]]; then
  90     cat <<END
  91 ERROR: This host is running the $(dpkg --print-architecture) architecture!
  92
  93 Because of the pre-built dependencies in our PPA, and some extra picky things
  94 like ID generation in liveupdate, installing reddit is only supported on amd64
  95 architectures.
  96 END
  97     exit 1
  98 fi
  99
 100 # seriously! these checks are here for a reason. the packages from the
 101 # reddit ppa aren't built for anything but trusty (14.04) right now, so
 102 # if you try and use this install script on another release you're gonna
 103 # have a bad time.
 104 source /etc/lsb-release
 105 if [ "$DISTRIB_ID" != "Ubuntu" -o "$DISTRIB_RELEASE" != "14.04" ]; then
 106     echo "ERROR: Only Ubuntu 14.04 is supported."
 107     exit 1
 108 fi
 109
 110 if [[ "2000000" -gt $(awk '/MemTotal/{print $2}' /proc/meminfo) ]]; then
 111     LOW_MEM_PROMPT="reddit requires at least 2GB of memory to work properly, continue anyway? [y/n] "
 112     read -er -n1 -p "$LOW_MEM_PROMPT" response
 113     if [[ "$response" != "y" ]]; then
 114       echo "Quitting."
 115       exit 1
 116     fi
 117 fi
 118
 119 ###############################################################################
 120 # Install prerequisites
 121 ###############################################################################
 122 set -x
 123
 124 # aptitude configuration
 125 APTITUDE_OPTIONS="-y"
 126 export DEBIAN_FRONTEND=noninteractive
 127
 128 # run an aptitude update to make sure python-software-properties
 129 # dependencies are found
 130 apt-get update
 131
 132 # add the reddit ppa for some custom packages
 133 apt-get install $APTITUDE_OPTIONS python-software-properties
 134 apt-add-repository -y ppa:reddit/ppa
 135
 136 # pin the ppa -- packages present in the ppa will take precedence over
 137 # ones in other repositories (unless further pinning is done)
 138 cat <<HERE > /etc/apt/preferences.d/reddit
 139 Package: *
 140 Pin: release o=LP-PPA-reddit
 141 Pin-Priority: 600
 142 HERE
 143
 144 # add the datastax cassandra repos
 145 echo deb http://debian.datastax.com/community stable main > /etc/apt/sources.list.d/cassandra.sources.list
 146 wget -qO- -L https://debian.datastax.com/debian/repo_key | sudo apt-key add -
 147
 148 # grab the new ppas' package listings
 149 apt-get update
 150
 151 # install prerequisites
 152 cat <<PACKAGES | xargs apt-get install $APTITUDE_OPTIONS
 153 netcat-openbsd
 154 git-core
 155
 156 python-dev
 157 python-setuptools
 158 python-routes
 159 python-pylons
 160 python-boto
 161 python-tz
 162 python-crypto
 163 python-babel
 164 cython
 165 python-sqlalchemy
 166 python-beautifulsoup
 167 python-chardet
 168 python-psycopg2
 169 python-pycassa
 170 python-imaging
 171 python-pycaptcha
 172 python-amqplib
 173 python-pylibmc=1.2.2-1~trusty5
 174 python-bcrypt
 175 python-snudown
 176 python-l2cs
 177 python-lxml
 178 python-kazoo
 179 python-stripe
 180 python-tinycss2
 181 python-unidecode
 182 python-mock
 183 python-yaml
 184
 185 python-baseplate
 186
 187 python-flask
 188 geoip-bin
 189 geoip-database
 190 python-geoip
 191
 192 nodejs
 193 node-less
 194 node-uglify
 195 gettext
 196 make
 197 optipng
 198 jpegoptim
 199
 200 mcrouter
 201 memcached
 202 postgresql
 203 postgresql-client
 204 rabbitmq-server
 205 cassandra=1.2.19
 206 haproxy
 207 nginx
 208 gunicorn
 209 sutro
 210 libpcre3-dev
 211 PACKAGES
 212
 213 # we don't want to upgrade to C* 2.0 yet, so we'll put it on hold
 214 apt-mark hold cassandra
 215
 216 ###############################################################################
 217 # Wait for all the services to be up
 218 ###############################################################################
 219 # cassandra doesn't auto-start after install
 220 service cassandra start
 221
 222 # check each port for connectivity
 223 echo "Waiting for services to be available, see source for port meanings..."
 224 # 11211 - memcache
 225 # 5432 - postgres
 226 # 5672 - rabbitmq
 227 # 9160 - cassandra
 228 for port in 11211 5432 5672 9160; do
 229     while ! nc -vz localhost $port; do
 230         sleep 1
 231     done
 232 done
 233
 234 ###############################################################################
 235 # Install the reddit source repositories
 236 ###############################################################################
 237 if [ ! -d $REDDIT_HOME/src ]; then
 238     mkdir -p $REDDIT_HOME/src
 239     chown $REDDIT_USER $REDDIT_HOME/src
 240 fi
 241
 242 function clone_reddit_repo {
 243     local destination=$REDDIT_HOME/src/${1}
 244     local repository_url=https://github.com/${2}.git
 245
 246     if [ ! -d $destination ]; then
 247         sudo -u $REDDIT_USER -H git clone $repository_url $destination
 248     fi
 249
 250     if [ -d $destination/upstart ]; then
 251         cp $destination/upstart/* /etc/init/
 252     fi
 253 }
 254
 255 function clone_reddit_plugin_repo {
 256     clone_reddit_repo $1 reddit/reddit-plugin-$1
 257 }
 258
 259 clone_reddit_repo reddit reddit/reddit
 260 clone_reddit_repo i18n reddit/reddit-i18n
 261 for plugin in $REDDIT_PLUGINS; do
 262     clone_reddit_plugin_repo $plugin
 263 done
 264
 265 ###############################################################################
 266 # Configure Cassandra
 267 ###############################################################################
 268 python <<END
 269 import pycassa
 270 sys = pycassa.SystemManager("localhost:9160")
 271
 272 if "reddit" not in sys.list_keyspaces():
 273     print "creating keyspace 'reddit'"
 274     sys.create_keyspace("reddit", "SimpleStrategy", {"replication_factor": "1"})
 275     print "done"
 276
 277 if "permacache" not in sys.get_keyspace_column_families("reddit"):
 278     print "creating column family 'permacache'"
 279     sys.create_column_family("reddit", "permacache")
 280     print "done"
 281 END
 282
 283 ###############################################################################
 284 # Configure PostgreSQL
 285 ###############################################################################
 286 SQL="SELECT COUNT(1) FROM pg_catalog.pg_database WHERE datname = 'reddit';"
 287 IS_DATABASE_CREATED=$(sudo -u postgres psql -t -c "$SQL")
 288
 289 if [ $IS_DATABASE_CREATED -ne 1 ]; then
 290     cat <<PGSCRIPT | sudo -u postgres psql
 291 CREATE DATABASE reddit WITH ENCODING = 'utf8' TEMPLATE template0 LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8';
 292 CREATE USER reddit WITH PASSWORD 'password';
 293 PGSCRIPT
 294 fi
 295
 296 sudo -u postgres psql reddit < $REDDIT_HOME/src/reddit/sql/functions.sql
 297
 298 ###############################################################################
 299 # Configure mcrouter
 300 ###############################################################################
 301 if [ ! -d /etc/mcrouter ]; then
 302     mkdir -p /etc/mcrouter
 303 fi
 304
 305 if [ ! -f /etc/mcrouter/global.conf ]; then
 306     cat > /etc/mcrouter/global.conf <<MCROUTER
 307 {
 308   // route all valid prefixes to the local memcached
 309   "pools": {
 310     "local": {
 311       "servers": [
 312         "127.0.0.1:11211",
 313       ],
 314       "protocol": "ascii",
 315       "keep_routing_prefix": false,
 316     },
 317   },
 318   "route": {
 319     "type": "PrefixSelectorRoute",
 320     "policies": {
 321       "rend:": {
 322         "type": "PoolRoute",
 323         "pool": "local",
 324       },
 325       "page:": {
 326         "type": "PoolRoute",
 327         "pool": "local",
 328       },
 329       "pane:": {
 330         "type": "PoolRoute",
 331         "pool": "local",
 332       },
 333     },
 334     "wildcard": {
 335       "type": "NullRoute",
 336     },
 337   },
 338 }
 339 MCROUTER
 340 fi
 341
 342 ###############################################################################
 343 # Configure RabbitMQ
 344 ###############################################################################
 345 if ! rabbitmqctl list_vhosts | egrep "^/$"
 346 then
 347     rabbitmqctl add_vhost /
 348 fi
 349
 350 if ! rabbitmqctl list_users | egrep "^reddit"
 351 then
 352     rabbitmqctl add_user reddit reddit
 353 fi
 354
 355 rabbitmqctl set_permissions -p / reddit ".*" ".*" ".*"
 356
 357 ###############################################################################
 358 # Install and configure the reddit code
 359 ###############################################################################
 360 function install_reddit_repo {
 361     cd $REDDIT_HOME/src/$1
 362     sudo -u $REDDIT_USER python setup.py build
 363     python setup.py develop --no-deps
 364 }
 365
 366 install_reddit_repo reddit/r2
 367 install_reddit_repo i18n
 368 for plugin in $REDDIT_PLUGINS; do
 369     install_reddit_repo $plugin
 370 done
 371
 372 # generate binary translation files from source
 373 cd $REDDIT_HOME/src/i18n/
 374 sudo -u $REDDIT_USER make clean all
 375
 376 # this builds static files and should be run *after* languages are installed
 377 # so that the proper language-specific static files can be generated and after
 378 # plugins are installed so all the static files are available.
 379 cd $REDDIT_HOME/src/reddit/r2
 380 sudo -u $REDDIT_USER make clean all
 381
 382 plugin_str=$(echo -n "$REDDIT_PLUGINS" | tr " " ,)
 383 if [ ! -f development.update ]; then
 384     cat > development.update <<DEVELOPMENT
 385 # after editing this file, run "make ini" to
 386 # generate a new development.ini
 387
 388 [DEFAULT]
 389 # global debug flag -- displays pylons stacktrace rather than 500 page on error when true
 390 # WARNING: a pylons stacktrace allows remote code execution. Make sure this is false
 391 # if your server is publicly accessible.
 392 debug = true
 393
 394 disable_ads = true
 395 disable_captcha = true
 396 disable_ratelimit = true
 397 disable_require_admin_otp = true
 398
 399 page_cache_time = 0
 400
 401 domain = $REDDIT_DOMAIN
 402 oauth_domain = $REDDIT_DOMAIN
 403
 404 plugins = $plugin_str
 405
 406 media_provider = filesystem
 407 media_fs_root = /srv/www/media
 408 media_fs_base_url_http = http://%(domain)s/media/
 409
 410 [server:main]
 411 port = 8001
 412 DEVELOPMENT
 413     chown $REDDIT_USER development.update
 414 else
 415     sed -i "s/^plugins = .*$/plugins = $plugin_str/" $REDDIT_HOME/src/reddit/r2/development.update
 416     sed -i "s/^domain = .*$/domain = $REDDIT_DOMAIN/" $REDDIT_HOME/src/reddit/r2/development.update
 417     sed -i "s/^oauth_domain = .*$/oauth_domain = $REDDIT_DOMAIN/" $REDDIT_HOME/src/reddit/r2/development.update
 418 fi
 419
 420 sudo -u $REDDIT_USER make ini
 421
 422 if [ ! -L run.ini ]; then
 423     sudo -u $REDDIT_USER ln -nsf development.ini run.ini
 424 fi
 425
 426 ###############################################################################
 427 # some useful helper scripts
 428 ###############################################################################
 429 function helper-script() {
 430     cat > $1
 431     chmod 755 $1
 432 }
 433
 434 helper-script /usr/local/bin/reddit-run <<REDDITRUN
 435 #!/bin/bash
 436 exec paster --plugin=r2 run $REDDIT_HOME/src/reddit/r2/run.ini "\$@"
 437 REDDITRUN
 438
 439 helper-script /usr/local/bin/reddit-shell <<REDDITSHELL
 440 #!/bin/bash
 441 exec paster --plugin=r2 shell $REDDIT_HOME/src/reddit/r2/run.ini
 442 REDDITSHELL
 443
 444 helper-script /usr/local/bin/reddit-start <<REDDITSTART
 445 #!/bin/bash
 446 initctl emit reddit-start
 447 REDDITSTART
 448
 449 helper-script /usr/local/bin/reddit-stop <<REDDITSTOP
 450 #!/bin/bash
 451 initctl emit reddit-stop
 452 REDDITSTOP
 453
 454 helper-script /usr/local/bin/reddit-restart <<REDDITRESTART
 455 #!/bin/bash
 456 initctl emit reddit-restart TARGET=${1:-all}
 457 REDDITRESTART
 458
 459 helper-script /usr/local/bin/reddit-flush <<REDDITFLUSH
 460 #!/bin/bash
 461 echo flush_all | nc localhost 11211
 462 REDDITFLUSH
 463
 464 ###############################################################################
 465 # pixel and click server
 466 ###############################################################################
 467 mkdir -p /var/opt/reddit/
 468 chown $REDDIT_USER:$REDDIT_GROUP /var/opt/reddit/
 469
 470 mkdir -p /srv/www/pixel
 471 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/pixel
 472 cp $REDDIT_HOME/src/reddit/r2/r2/public/static/pixel.png /srv/www/pixel
 473
 474 if [ ! -f /etc/gunicorn.d/click.conf ]; then
 475     cat > /etc/gunicorn.d/click.conf <<CLICK
 476 CONFIG = {
 477     "mode": "wsgi",
 478     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 479     "user": "$REDDIT_USER",
 480     "group": "$REDDIT_USER",
 481     "args": (
 482         "--bind=unix:/var/opt/reddit/click.sock",
 483         "--workers=1",
 484         "tracker:application",
 485     ),
 486 }
 487 CLICK
 488 fi
 489
 490 service gunicorn start
 491
 492 ###############################################################################
 493 # nginx
 494 ###############################################################################
 495
 496 mkdir -p /srv/www/media
 497 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/media
 498
 499 cat > /etc/nginx/sites-available/reddit-media <<MEDIA
 500 server {
 501     listen 9000;
 502
 503     expires max;
 504
 505     location /media/ {
 506         alias /srv/www/media/;
 507     }
 508 }
 509 MEDIA
 510
 511 cat > /etc/nginx/sites-available/reddit-pixel <<PIXEL
 512 upstream click_server {
 513   server unix:/var/opt/reddit/click.sock fail_timeout=0;
 514 }
 515
 516 server {
 517   listen 8082;
 518
 519   log_format directlog '\$remote_addr - \$remote_user [\$time_local] '
 520                       '"\$request_method \$request_uri \$server_protocol" \$status \$body_bytes_sent '
 521                       '"\$http_referer" "\$http_user_agent"';
 522   access_log      /var/log/nginx/traffic/traffic.log directlog;
 523
 524   location / {
 525
 526     rewrite ^/pixel/of_ /pixel.png;
 527
 528     add_header Last-Modified "";
 529     add_header Pragma "no-cache";
 530
 531     expires -1;
 532     root /srv/www/pixel/;
 533   }
 534
 535   location /click {
 536     proxy_pass http://click_server;
 537   }
 538 }
 539 PIXEL
 540
 541 cat > /etc/nginx/sites-available/reddit-ssl <<SSL
 542 map \$http_upgrade \$connection_upgrade {
 543   default upgrade;
 544   ''      close;
 545 }
 546
 547 server {
 548     listen 443;
 549
 550     ssl on;
 551     ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
 552     ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
 553
 554     ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 555     ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
 556     ssl_prefer_server_ciphers on;
 557
 558     ssl_session_cache shared:SSL:1m;
 559
 560     location / {
 561         proxy_pass http://127.0.0.1:8080;
 562         proxy_set_header Host \$http_host;
 563         proxy_http_version 1.1;
 564         proxy_set_header X-Forwarded-For \$remote_addr;
 565         proxy_pass_header Server;
 566
 567         # allow websockets through if desired
 568         proxy_set_header Upgrade \$http_upgrade;
 569         proxy_set_header Connection \$connection_upgrade;
 570     }
 571 }
 572 SSL
 573
 574 # remove the default nginx site that may conflict with haproxy
 575 rm -rf /etc/nginx/sites-enabled/default
 576 # put our config in place
 577 ln -nsf /etc/nginx/sites-available/reddit-media /etc/nginx/sites-enabled/
 578 ln -nsf /etc/nginx/sites-available/reddit-pixel /etc/nginx/sites-enabled/
 579 ln -nsf /etc/nginx/sites-available/reddit-ssl /etc/nginx/sites-enabled/
 580
 581 # make the pixel log directory
 582 mkdir -p /var/log/nginx/traffic
 583
 584 # link the ini file for the Flask click tracker
 585 ln -nsf $REDDIT_HOME/src/reddit/r2/development.ini $REDDIT_HOME/src/reddit/scripts/production.ini
 586
 587 service nginx restart
 588
 589 ###############################################################################
 590 # haproxy
 591 ###############################################################################
 592 if [ -e /etc/haproxy/haproxy.cfg ]; then
 593     BACKUP_HAPROXY=$(mktemp /etc/haproxy/haproxy.cfg.XXX)
 594     echo "Backing up /etc/haproxy/haproxy.cfg to $BACKUP_HAPROXY"
 595     cat /etc/haproxy/haproxy.cfg > $BACKUP_HAPROXY
 596 fi
 597
 598 # make sure haproxy is enabled
 599 cat > /etc/default/haproxy <<DEFAULT
 600 ENABLED=1
 601 DEFAULT
 602
 603 # configure haproxy
 604 cat > /etc/haproxy/haproxy.cfg <<HAPROXY
 605 global
 606     maxconn 350
 607
 608 frontend frontend
 609     mode http
 610
 611     bind 0.0.0.0:80
 612     bind 127.0.0.1:8080
 613
 614     timeout client 24h
 615     option forwardfor except 127.0.0.1
 616     option httpclose
 617
 618     # make sure that requests have x-forwarded-proto: https iff tls
 619     reqidel ^X-Forwarded-Proto:.*
 620     acl is-ssl dst_port 8080
 621     reqadd X-Forwarded-Proto:\ https if is-ssl
 622
 623     # send websockets to sutro
 624     acl is-websocket hdr(Upgrade) -i WebSocket
 625     use_backend sutro if is-websocket
 626
 627     # send media stuff to the local nginx
 628     acl is-media path_beg /media/
 629     use_backend media if is-media
 630
 631     # send pixel stuff to local nginx
 632     acl is-pixel path_beg /pixel/
 633     acl is-click path_beg /click
 634     use_backend pixel if is-pixel || is-click
 635
 636     default_backend reddit
 637
 638 backend reddit
 639     mode http
 640     timeout connect 4000
 641     timeout server 30000
 642     timeout queue 60000
 643     balance roundrobin
 644
 645     server app01-8001 localhost:8001 maxconn 30
 646
 647 backend sutro
 648     mode http
 649     timeout connect 4s
 650     timeout server 24h
 651     balance roundrobin
 652
 653     server sutro localhost:8002 maxconn 250
 654
 655 backend media
 656     mode http
 657     timeout connect 4000
 658     timeout server 30000
 659     timeout queue 60000
 660     balance roundrobin
 661
 662     server nginx localhost:9000 maxconn 20
 663
 664 backend pixel
 665     mode http
 666     timeout connect 4000
 667     timeout server 30000
 668     timeout queue 60000
 669     balance roundrobin
 670
 671     server nginx localhost:8082 maxconn 20
 672 HAPROXY
 673
 674 # this will start it even if currently stopped
 675 service haproxy restart
 676
 677 ###############################################################################
 678 # sutro (websocket server)
 679 ###############################################################################
 680
 681 if [ ! -f /etc/sutro.ini ]; then
 682     cat > /etc/sutro.ini <<SUTRO
 683 [app:main]
 684 paste.app_factory = sutro.app:make_app
 685
 686 amqp.host = localhost
 687 amqp.port = 5672
 688 amqp.vhost = /
 689 amqp.username = reddit
 690 amqp.password = reddit
 691
 692 web.allowed_origins = $REDDIT_DOMAIN
 693 web.mac_secret = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5
 694 web.ping_interval = 300
 695
 696 stats.host =
 697 stats.port = 0
 698
 699 [server:main]
 700 use = egg:gunicorn#main
 701 worker_class = sutro.socketserver.SutroWorker
 702 workers = 1
 703 worker_connections = 250
 704 host = 127.0.0.1
 705 port = 8002
 706 graceful_timeout = 5
 707 forward_allow_ips = 127.0.0.1
 708
 709 [loggers]
 710 keys = root
 711
 712 [handlers]
 713 keys = syslog
 714
 715 [formatters]
 716 keys = generic
 717
 718 [logger_root]
 719 level = INFO
 720 handlers = syslog
 721
 722 [handler_syslog]
 723 class = handlers.SysLogHandler
 724 args = ("/dev/log", "local7")
 725 formatter = generic
 726 level = NOTSET
 727
 728 [formatter_generic]
 729 format = [%(name)s] %(message)s
 730 SUTRO
 731 fi
 732
 733 if [ ! -f /etc/init/sutro.conf ]; then
 734     cat > /etc/init/sutro.conf << UPSTART_SUTRO
 735 description "sutro websocket server"
 736
 737 stop on runlevel [!2345]
 738 start on runlevel [2345]
 739
 740 respawn
 741 respawn limit 10 5
 742 kill timeout 15
 743
 744 limit nofile 65535 65535
 745
 746 exec gunicorn_paster /etc/sutro.ini
 747 UPSTART_SUTRO
 748 fi
 749
 750 service sutro restart
 751
 752 ###############################################################################
 753 # geoip service
 754 ###############################################################################
 755 if [ ! -f /etc/gunicorn.d/geoip.conf ]; then
 756     cat > /etc/gunicorn.d/geoip.conf <<GEOIP
 757 CONFIG = {
 758     "mode": "wsgi",
 759     "working_dir": "$REDDIT_HOME/src/reddit/scripts",
 760     "user": "$REDDIT_USER",
 761     "group": "$REDDIT_USER",
 762     "args": (
 763         "--bind=127.0.0.1:5000",
 764         "--workers=1",
 765          "--limit-request-line=8190",
 766          "geoip_service:application",
 767     ),
 768 }
 769 GEOIP
 770 fi
 771
 772 service gunicorn start
 773
 774 ###############################################################################
 775 # Job Environment
 776 ###############################################################################
 777 CONSUMER_CONFIG_ROOT=$REDDIT_HOME/consumer-count.d
 778
 779 if [ ! -f /etc/default/reddit ]; then
 780     cat > /etc/default/reddit <<DEFAULT
 781 export REDDIT_ROOT=$REDDIT_HOME/src/reddit/r2
 782 export REDDIT_INI=$REDDIT_HOME/src/reddit/r2/run.ini
 783 export REDDIT_USER=$REDDIT_USER
 784 export REDDIT_GROUP=$REDDIT_GROUP
 785 export REDDIT_CONSUMER_CONFIG=$CONSUMER_CONFIG_ROOT
 786 alias wrap-job=$REDDIT_HOME/src/reddit/scripts/wrap-job
 787 alias manage-consumers=$REDDIT_HOME/src/reddit/scripts/manage-consumers
 788 DEFAULT
 789 fi
 790
 791 ###############################################################################
 792 # Queue Processors
 793 ###############################################################################
 794 mkdir -p $CONSUMER_CONFIG_ROOT
 795
 796 function set_consumer_count {
 797     if [ ! -f $CONSUMER_CONFIG_ROOT/$1 ]; then
 798         echo $2 > $CONSUMER_CONFIG_ROOT/$1
 799     fi
 800 }
 801
 802 set_consumer_count log_q 0
 803 set_consumer_count search_q 0
 804 set_consumer_count del_account_q 1
 805 set_consumer_count scraper_q 1
 806 set_consumer_count markread_q 1
 807 set_consumer_count commentstree_q 1
 808 set_consumer_count newcomments_q 1
 809 set_consumer_count vote_link_q 1
 810 set_consumer_count vote_comment_q 1
 811 set_consumer_count automoderator_q 0
 812
 813 chown -R $REDDIT_USER:$REDDIT_GROUP $CONSUMER_CONFIG_ROOT/
 814
 815
 816 ###############################################################################
 817 # Start everything up
 818 ###############################################################################
 819
 820 # the initial database setup should be done by one process rather than a bunch
 821 # vying with eachother to get there first
 822 reddit-run -c 'print "ok done"'
 823
 824 # ok, now start everything else up
 825 initctl emit reddit-stop
 826 initctl emit reddit-start
 827
 828 ###############################################################################
 829 # Cron Jobs
 830 ###############################################################################
 831 if [ ! -f /etc/cron.d/reddit ]; then
 832     cat > /etc/cron.d/reddit <<CRON
 833 0    3 * * * root /sbin/start --quiet reddit-job-update_sr_names
 834 30  16 * * * root /sbin/start --quiet reddit-job-update_reddits
 835 0    * * * * root /sbin/start --quiet reddit-job-update_promos
 836 */5  * * * * root /sbin/start --quiet reddit-job-clean_up_hardcache
 837 */2  * * * * root /sbin/start --quiet reddit-job-broken_things
 838 */2  * * * * root /sbin/start --quiet reddit-job-rising
 839 0    * * * * root /sbin/start --quiet reddit-job-trylater
 840
 841 # liveupdate
 842 *    * * * * root /sbin/start --quiet reddit-job-liveupdate_activity
 843
 844 # jobs that recalculate time-limited listings (e.g. top this year)
 845 PGPASSWORD=password
 846 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings link year '("hour", "day", "week", "month", "year")'
 847 */15 * * * * $REDDIT_USER $REDDIT_HOME/src/reddit/scripts/compute_time_listings comment year '("hour", "day", "week", "month", "year")'
 848
 849 # disabled by default, uncomment if you need these jobs
 850 #*    * * * * root /sbin/start --quiet reddit-job-email
 851 #0    0 * * * root /sbin/start --quiet reddit-job-update_gold_users
 852 CRON
 853 fi
 854
 855 ###############################################################################
 856 # All done!
 857 ###############################################################################
 858 cd $REDDIT_HOME
 859
 860 cat <<CONCLUSION
 861
 862 Congratulations! reddit is now installed.
 863
 864 The reddit application code is managed with upstart, to see what's currently
 865 running, run
 866
 867     sudo initctl list | grep reddit
 868
 869 Cron jobs start with "reddit-job-" and queue processors start with
 870 "reddit-consumer-". The crons are managed by /etc/cron.d/reddit. You can
 871 initiate a restart of all the consumers by running:
 872
 873     sudo reddit-restart
 874
 875 or target specific ones:
 876
 877     sudo reddit-restart scraper_q
 878
 879 See the GitHub wiki for more information on these jobs:
 880
 881 * https://github.com/reddit/reddit/wiki/Cron-jobs
 882 * https://github.com/reddit/reddit/wiki/Services
 883
 884 The reddit code can be shut down or started up with
 885
 886     sudo reddit-stop
 887     sudo reddit-start
 888
 889 And if you think caching might be hurting you, you can flush memcache with
 890
 891     reddit-flush
 892
 893 Now that the core of reddit is installed, you may want to do some additional
 894 steps:
 895
 896 * Ensure that $REDDIT_DOMAIN resolves to this machine.
 897
 898 * To populate the database with test data, run:
 899
 900     cd $REDDIT_HOME/src/reddit
 901     reddit-run scripts/inject_test_data.py -c 'inject_test_data()'
 902
 903 * Manually run reddit-job-update_reddits immediately after populating the db
 904   or adding your own subreddits.
 905 CONCLUSION