install/reddit.sh

   1 #!/bin/bash
   2 ###############################################################################
   3 # reddit dev environment installer
   4 # --------------------------------
   5 # This script installs a reddit stack suitable for development. DO NOT run this
   6 # on a system that you use for other purposes as it might delete important
   7 # files, truncate your databases, and otherwise do mean things to you.
   8 #
   9 # By default, this script will install the reddit code in the current user's
  10 # home directory and all of its dependencies (including libraries and database
  11 # servers) at the system level. The installed reddit will expect to be visited
  12 # on the domain "reddit.local" unless specified otherwise.  Configuring name
  13 # resolution for the domain is expected to be done outside the installed
  14 # environment (e.g. in your host machine's /etc/hosts file) and is not
  15 # something this script handles.
  16 #
  17 # Several configuration options (listed in the "Configuration" section below)
  18 # are overridable with environment variables. e.g.
  19 #
  20 #    sudo REDDIT_DOMAIN=example.com ./install/reddit.sh
  21 #
  22 ###############################################################################
  23
  24 # load configuration
  25 RUNDIR=$(dirname $0)
  26 source $RUNDIR/install.cfg
  27
  28
  29 ###############################################################################
  30 # Sanity Checks
  31 ###############################################################################
  32 if [[ $EUID -ne 0 ]]; then
  33     echo "ERROR: Must be run with root privileges."
  34     exit 1
  35 fi
  36
  37 if [[ -z "$REDDIT_USER" ]]; then
  38     # in a production install, you'd want the code to be owned by root and run
  39     # by a less privileged user. this script is intended to build a development
  40     # install, so we expect the owner to run the app and not be root.
  41     cat <<END
  42 ERROR: You have not specified a user. This usually means you're running this
  43 script directly as root. It is not recommended to run reddit as the root user.
  44
  45 Please create a user to run reddit and set the REDDIT_USER variable
  46 appropriately.
  47 END
  48     exit 1
  49 fi
  50
  51 if [[ "amd64" != $(dpkg --print-architecture) ]]; then
  52     cat <<END
  53 ERROR: This host is running the $(dpkg --print-architecture) architecture!
  54
  55 Because of the pre-built dependencies in our PPA, and some extra picky things
  56 like ID generation in liveupdate, installing reddit is only supported on amd64
  57 architectures.
  58 END
  59     exit 1
  60 fi
  61
  62 # seriously! these checks are here for a reason. the packages from the
  63 # reddit ppa aren't built for anything but trusty (14.04) right now, so
  64 # if you try and use this install script on another release you're gonna
  65 # have a bad time.
  66 source /etc/lsb-release
  67 if [ "$DISTRIB_ID" != "Ubuntu" -o "$DISTRIB_RELEASE" != "14.04" ]; then
  68     echo "ERROR: Only Ubuntu 14.04 is supported."
  69     exit 1
  70 fi
  71
  72 if [[ "2000000" -gt $(awk '/MemTotal/{print $2}' /proc/meminfo) ]]; then
  73     LOW_MEM_PROMPT="reddit requires at least 2GB of memory to work properly, continue anyway? [y/n] "
  74     read -er -n1 -p "$LOW_MEM_PROMPT" response
  75     if [[ "$response" != "y" ]]; then
  76       echo "Quitting."
  77       exit 1
  78     fi
  79 fi
  80
  81 REDDIT_AVAILABLE_PLUGINS=""
  82 for plugin in $REDDIT_PLUGINS; do
  83     if [ -d $REDDIT_SRC/$plugin ]; then
  84         if [[ -z "$REDDIT_PLUGINS" ]]; then
  85             REDDIT_AVAILABLE_PLUGINS+="$plugin"
  86         else
  87             REDDIT_AVAILABLE_PLUGINS+=" $plugin"
  88         fi
  89         echo "plugin $plugin found"
  90     else
  91         echo "plugin $plugin not found"
  92     fi
  93 done
  94
  95 ###############################################################################
  96 # Install prerequisites
  97 ###############################################################################
  98
  99 # install primary packages
 100 $RUNDIR/install_apt.sh
 101
 102 # install cassandra from datastax
 103 $RUNDIR/install_cassandra.sh
 104
 105 # install zookeeper
 106 $RUNDIR/install_zookeeper.sh
 107
 108 # install services (rabbitmq, postgres, memcached, etc.)
 109 $RUNDIR/install_services.sh
 110
 111 ###############################################################################
 112 # Install the reddit source repositories
 113 ###############################################################################
 114 if [ ! -d $REDDIT_SRC ]; then
 115     mkdir -p $REDDIT_SRC
 116     chown $REDDIT_USER $REDDIT_SRC
 117 fi
 118
 119 function copy_upstart {
 120     if [ -d ${1}/upstart ]; then
 121         cp ${1}/upstart/* /etc/init/
 122     fi
 123 }
 124
 125 function clone_reddit_repo {
 126     local destination=$REDDIT_SRC/${1}
 127     local repository_url=https://github.com/${2}.git
 128
 129     if [ ! -d $destination ]; then
 130         sudo -u $REDDIT_USER -H git clone $repository_url $destination
 131     fi
 132
 133     copy_upstart $destination
 134 }
 135
 136 function clone_reddit_service_repo {
 137     clone_reddit_repo $1 reddit/reddit-service-$1
 138 }
 139
 140 clone_reddit_repo reddit reddit/reddit
 141 clone_reddit_repo i18n reddit/reddit-i18n
 142 clone_reddit_service_repo websockets
 143 clone_reddit_service_repo activity
 144
 145 ###############################################################################
 146 # Configure Services
 147 ###############################################################################
 148
 149 # Configure Cassandra
 150 $RUNDIR/setup_cassandra.sh
 151
 152 # Configure PostgreSQL
 153 $RUNDIR/setup_postgres.sh
 154
 155 # Configure mcrouter
 156 $RUNDIR/setup_mcrouter.sh
 157
 158 # Configure RabbitMQ
 159 $RUNDIR/setup_rabbitmq.sh
 160
 161 ###############################################################################
 162 # Install and configure the reddit code
 163 ###############################################################################
 164 function install_reddit_repo {
 165     pushd $REDDIT_SRC/$1
 166     sudo -u $REDDIT_USER python setup.py build
 167     python setup.py develop --no-deps
 168     popd
 169 }
 170
 171 install_reddit_repo reddit/r2
 172 install_reddit_repo i18n
 173 for plugin in $REDDIT_AVAILABLE_PLUGINS; do
 174     copy_upstart $REDDIT_SRC/$plugin
 175     install_reddit_repo $plugin
 176 done
 177 install_reddit_repo websockets
 178 install_reddit_repo activity
 179
 180 # generate binary translation files from source
 181 sudo -u $REDDIT_USER make -C $REDDIT_SRC/i18n clean all
 182
 183 # this builds static files and should be run *after* languages are installed
 184 # so that the proper language-specific static files can be generated and after
 185 # plugins are installed so all the static files are available.
 186 pushd $REDDIT_SRC/reddit/r2
 187 sudo -u $REDDIT_USER make clean pyx
 188
 189 plugin_str=$(echo -n "$REDDIT_AVAILABLE_PLUGINS" | tr " " ,)
 190 if [ ! -f development.update ]; then
 191     cat > development.update <<DEVELOPMENT
 192 # after editing this file, run "make ini" to
 193 # generate a new development.ini
 194
 195 [DEFAULT]
 196 # global debug flag -- displays pylons stacktrace rather than 500 page on error when true
 197 # WARNING: a pylons stacktrace allows remote code execution. Make sure this is false
 198 # if your server is publicly accessible.
 199 debug = true
 200
 201 disable_ads = true
 202 disable_captcha = true
 203 disable_ratelimit = true
 204 disable_require_admin_otp = true
 205
 206 domain = $REDDIT_DOMAIN
 207 oauth_domain = $REDDIT_DOMAIN
 208
 209 plugins = $plugin_str
 210
 211 media_provider = filesystem
 212 media_fs_root = /srv/www/media
 213 media_fs_base_url_http = http://%(domain)s/media/
 214
 215 [server:main]
 216 port = 8001
 217 DEVELOPMENT
 218     chown $REDDIT_USER development.update
 219 else
 220     sed -i "s/^plugins = .*$/plugins = $plugin_str/" $REDDIT_SRC/reddit/r2/development.update
 221     sed -i "s/^domain = .*$/domain = $REDDIT_DOMAIN/" $REDDIT_SRC/reddit/r2/development.update
 222     sed -i "s/^oauth_domain = .*$/oauth_domain = $REDDIT_DOMAIN/" $REDDIT_SRC/reddit/r2/development.update
 223 fi
 224
 225 sudo -u $REDDIT_USER make ini
 226
 227 if [ ! -L run.ini ]; then
 228     sudo -u $REDDIT_USER ln -nsf development.ini run.ini
 229 fi
 230
 231 popd
 232
 233 ###############################################################################
 234 # some useful helper scripts
 235 ###############################################################################
 236 function helper-script() {
 237     cat > $1
 238     chmod 755 $1
 239 }
 240
 241 helper-script /usr/local/bin/reddit-run <<REDDITRUN
 242 #!/bin/bash
 243 exec paster --plugin=r2 run $REDDIT_SRC/reddit/r2/run.ini "\$@"
 244 REDDITRUN
 245
 246 helper-script /usr/local/bin/reddit-shell <<REDDITSHELL
 247 #!/bin/bash
 248 exec paster --plugin=r2 shell $REDDIT_SRC/reddit/r2/run.ini
 249 REDDITSHELL
 250
 251 helper-script /usr/local/bin/reddit-start <<REDDITSTART
 252 #!/bin/bash
 253 initctl emit reddit-start
 254 REDDITSTART
 255
 256 helper-script /usr/local/bin/reddit-stop <<REDDITSTOP
 257 #!/bin/bash
 258 initctl emit reddit-stop
 259 REDDITSTOP
 260
 261 helper-script /usr/local/bin/reddit-restart <<REDDITRESTART
 262 #!/bin/bash
 263 initctl emit reddit-restart TARGET=${1:-all}
 264 REDDITRESTART
 265
 266 helper-script /usr/local/bin/reddit-flush <<REDDITFLUSH
 267 #!/bin/bash
 268 echo flush_all | nc localhost 11211
 269 REDDITFLUSH
 270
 271 helper-script /usr/local/bin/reddit-serve <<REDDITSERVE
 272 #!/bin/bash
 273 exec paster serve --reload $REDDIT_SRC/reddit/r2/run.ini
 274 REDDITSERVE
 275
 276 ###############################################################################
 277 # pixel and click server
 278 ###############################################################################
 279 mkdir -p /var/opt/reddit/
 280 chown $REDDIT_USER:$REDDIT_GROUP /var/opt/reddit/
 281
 282 mkdir -p /srv/www/pixel
 283 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/pixel
 284 cp $REDDIT_SRC/reddit/r2/r2/public/static/pixel.png /srv/www/pixel
 285
 286 if [ ! -f /etc/gunicorn.d/click.conf ]; then
 287     cat > /etc/gunicorn.d/click.conf <<CLICK
 288 CONFIG = {
 289     "mode": "wsgi",
 290     "working_dir": "$REDDIT_SRC/reddit/scripts",
 291     "user": "$REDDIT_USER",
 292     "group": "$REDDIT_USER",
 293     "args": (
 294         "--bind=unix:/var/opt/reddit/click.sock",
 295         "--workers=1",
 296         "tracker:application",
 297     ),
 298 }
 299 CLICK
 300 fi
 301
 302 service gunicorn start
 303
 304 ###############################################################################
 305 # nginx
 306 ###############################################################################
 307
 308 mkdir -p /srv/www/media
 309 chown $REDDIT_USER:$REDDIT_GROUP /srv/www/media
 310
 311 cat > /etc/nginx/sites-available/reddit-media <<MEDIA
 312 server {
 313     listen 9000;
 314
 315     expires max;
 316
 317     location /media/ {
 318         alias /srv/www/media/;
 319     }
 320 }
 321 MEDIA
 322
 323 cat > /etc/nginx/sites-available/reddit-pixel <<PIXEL
 324 upstream click_server {
 325   server unix:/var/opt/reddit/click.sock fail_timeout=0;
 326 }
 327
 328 server {
 329   listen 8082;
 330
 331   log_format directlog '\$remote_addr - \$remote_user [\$time_local] '
 332                       '"\$request_method \$request_uri \$server_protocol" \$status \$body_bytes_sent '
 333                       '"\$http_referer" "\$http_user_agent"';
 334   access_log      /var/log/nginx/traffic/traffic.log directlog;
 335
 336   location / {
 337
 338     rewrite ^/pixel/of_ /pixel.png;
 339
 340     add_header Last-Modified "";
 341     add_header Pragma "no-cache";
 342
 343     expires -1;
 344     root /srv/www/pixel/;
 345   }
 346
 347   location /click {
 348     proxy_pass http://click_server;
 349   }
 350 }
 351 PIXEL
 352
 353 cat > /etc/nginx/sites-available/reddit-ssl <<SSL
 354 map \$http_upgrade \$connection_upgrade {
 355   default upgrade;
 356   ''      close;
 357 }
 358
 359 server {
 360     listen 443;
 361
 362     ssl on;
 363     ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
 364     ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
 365
 366     ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 367     ssl_ciphers EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
 368     ssl_prefer_server_ciphers on;
 369
 370     ssl_session_cache shared:SSL:1m;
 371
 372     location / {
 373         proxy_pass http://127.0.0.1:8080;
 374         proxy_set_header Host \$http_host;
 375         proxy_http_version 1.1;
 376         proxy_set_header X-Forwarded-For \$remote_addr;
 377         proxy_pass_header Server;
 378
 379         # allow websockets through if desired
 380         proxy_set_header Upgrade \$http_upgrade;
 381         proxy_set_header Connection \$connection_upgrade;
 382     }
 383 }
 384 SSL
 385
 386 # remove the default nginx site that may conflict with haproxy
 387 rm -rf /etc/nginx/sites-enabled/default
 388 # put our config in place
 389 ln -nsf /etc/nginx/sites-available/reddit-media /etc/nginx/sites-enabled/
 390 ln -nsf /etc/nginx/sites-available/reddit-pixel /etc/nginx/sites-enabled/
 391 ln -nsf /etc/nginx/sites-available/reddit-ssl /etc/nginx/sites-enabled/
 392
 393 # make the pixel log directory
 394 mkdir -p /var/log/nginx/traffic
 395
 396 # link the ini file for the Flask click tracker
 397 ln -nsf $REDDIT_SRC/reddit/r2/development.ini $REDDIT_SRC/reddit/scripts/production.ini
 398
 399 service nginx restart
 400
 401 ###############################################################################
 402 # haproxy
 403 ###############################################################################
 404 if [ -e /etc/haproxy/haproxy.cfg ]; then
 405     BACKUP_HAPROXY=$(mktemp /etc/haproxy/haproxy.cfg.XXX)
 406     echo "Backing up /etc/haproxy/haproxy.cfg to $BACKUP_HAPROXY"
 407     cat /etc/haproxy/haproxy.cfg > $BACKUP_HAPROXY
 408 fi
 409
 410 # make sure haproxy is enabled
 411 cat > /etc/default/haproxy <<DEFAULT
 412 ENABLED=1
 413 DEFAULT
 414
 415 # configure haproxy
 416 cat > /etc/haproxy/haproxy.cfg <<HAPROXY
 417 global
 418     maxconn 350
 419
 420 frontend frontend
 421     mode http
 422
 423     bind 0.0.0.0:80
 424     bind 127.0.0.1:8080
 425
 426     timeout client 24h
 427     option forwardfor except 127.0.0.1
 428     option httpclose
 429
 430     # make sure that requests have x-forwarded-proto: https iff tls
 431     reqidel ^X-Forwarded-Proto:.*
 432     acl is-ssl dst_port 8080
 433     reqadd X-Forwarded-Proto:\ https if is-ssl
 434
 435     # send websockets to the websocket service
 436     acl is-websocket hdr(Upgrade) -i WebSocket
 437     use_backend websockets if is-websocket
 438
 439     # send media stuff to the local nginx
 440     acl is-media path_beg /media/
 441     use_backend media if is-media
 442
 443     # send pixel stuff to local nginx
 444     acl is-pixel path_beg /pixel/
 445     acl is-click path_beg /click
 446     use_backend pixel if is-pixel || is-click
 447
 448     default_backend reddit
 449
 450 backend reddit
 451     mode http
 452     timeout connect 4000
 453     timeout server 30000
 454     timeout queue 60000
 455     balance roundrobin
 456
 457     server app01-8001 localhost:8001 maxconn 30
 458
 459 backend websockets
 460     mode http
 461     timeout connect 4s
 462     timeout server 24h
 463     balance roundrobin
 464
 465     server websockets localhost:9001 maxconn 250
 466
 467 backend media
 468     mode http
 469     timeout connect 4000
 470     timeout server 30000
 471     timeout queue 60000
 472     balance roundrobin
 473
 474     server nginx localhost:9000 maxconn 20
 475
 476 backend pixel
 477     mode http
 478     timeout connect 4000
 479     timeout server 30000
 480     timeout queue 60000
 481     balance roundrobin
 482
 483     server nginx localhost:8082 maxconn 20
 484 HAPROXY
 485
 486 # this will start it even if currently stopped
 487 service haproxy restart
 488
 489 ###############################################################################
 490 # websocket service
 491 ###############################################################################
 492
 493 if [ ! -f /etc/init/reddit-websockets.conf ]; then
 494     cat > /etc/init/reddit-websockets.conf << UPSTART_WEBSOCKETS
 495 description "websockets service"
 496
 497 stop on runlevel [!2345] or reddit-restart all or reddit-restart websockets
 498 start on runlevel [2345] or reddit-restart all or reddit-restart websockets
 499
 500 respawn
 501 respawn limit 10 5
 502 kill timeout 15
 503
 504 limit nofile 65535 65535
 505
 506 exec baseplate-serve2 --bind localhost:9001 $REDDIT_SRC/websockets/example.ini
 507 UPSTART_WEBSOCKETS
 508 fi
 509
 510 service reddit-websockets restart
 511
 512 ###############################################################################
 513 # activity service
 514 ###############################################################################
 515
 516 if [ ! -f /etc/init/reddit-activity.conf ]; then
 517     cat > /etc/init/reddit-activity.conf << UPSTART_ACTIVITY
 518 description "activity service"
 519
 520 stop on runlevel [!2345] or reddit-restart all or reddit-restart activity
 521 start on runlevel [2345] or reddit-restart all or reddit-restart activity
 522
 523 respawn
 524 respawn limit 10 5
 525 kill timeout 15
 526
 527 exec baseplate-serve2 --bind localhost:9002 $REDDIT_SRC/activity/example.ini
 528 UPSTART_ACTIVITY
 529 fi
 530
 531 service reddit-activity restart
 532
 533 ###############################################################################
 534 # geoip service
 535 ###############################################################################
 536 if [ ! -f /etc/gunicorn.d/geoip.conf ]; then
 537     cat > /etc/gunicorn.d/geoip.conf <<GEOIP
 538 CONFIG = {
 539     "mode": "wsgi",
 540     "working_dir": "$REDDIT_SRC/reddit/scripts",
 541     "user": "$REDDIT_USER",
 542     "group": "$REDDIT_USER",
 543     "args": (
 544         "--bind=127.0.0.1:5000",
 545         "--workers=1",
 546          "--limit-request-line=8190",
 547          "geoip_service:application",
 548     ),
 549 }
 550 GEOIP
 551 fi
 552
 553 service gunicorn start
 554
 555 ###############################################################################
 556 # Job Environment
 557 ###############################################################################
 558 CONSUMER_CONFIG_ROOT=$REDDIT_HOME/consumer-count.d
 559
 560 if [ ! -f /etc/default/reddit ]; then
 561     cat > /etc/default/reddit <<DEFAULT
 562 export REDDIT_ROOT=$REDDIT_SRC/reddit/r2
 563 export REDDIT_INI=$REDDIT_SRC/reddit/r2/run.ini
 564 export REDDIT_USER=$REDDIT_USER
 565 export REDDIT_GROUP=$REDDIT_GROUP
 566 export REDDIT_CONSUMER_CONFIG=$CONSUMER_CONFIG_ROOT
 567 alias wrap-job=$REDDIT_SRC/reddit/scripts/wrap-job
 568 alias manage-consumers=$REDDIT_SRC/reddit/scripts/manage-consumers
 569 DEFAULT
 570 fi
 571
 572 ###############################################################################
 573 # Queue Processors
 574 ###############################################################################
 575 mkdir -p $CONSUMER_CONFIG_ROOT
 576
 577 function set_consumer_count {
 578     if [ ! -f $CONSUMER_CONFIG_ROOT/$1 ]; then
 579         echo $2 > $CONSUMER_CONFIG_ROOT/$1
 580     fi
 581 }
 582
 583 set_consumer_count log_q 0
 584 set_consumer_count search_q 0
 585 set_consumer_count del_account_q 1
 586 set_consumer_count scraper_q 1
 587 set_consumer_count markread_q 1
 588 set_consumer_count commentstree_q 1
 589 set_consumer_count newcomments_q 1
 590 set_consumer_count vote_link_q 1
 591 set_consumer_count vote_comment_q 1
 592 set_consumer_count automoderator_q 0
 593 set_consumer_count butler_q 1
 594
 595 chown -R $REDDIT_USER:$REDDIT_GROUP $CONSUMER_CONFIG_ROOT/
 596
 597
 598 ###############################################################################
 599 # Start everything up
 600 ###############################################################################
 601
 602 # the initial database setup should be done by one process rather than a bunch
 603 # vying with eachother to get there first
 604 reddit-run -c 'print "ok done"'
 605
 606 # ok, now start everything else up
 607 initctl emit reddit-stop
 608 initctl emit reddit-start
 609
 610 ###############################################################################
 611 # Cron Jobs
 612 ###############################################################################
 613 if [ ! -f /etc/cron.d/reddit ]; then
 614     cat > /etc/cron.d/reddit <<CRON
 615 0    3 * * * root /sbin/start --quiet reddit-job-update_sr_names
 616 30  16 * * * root /sbin/start --quiet reddit-job-update_reddits
 617 0    * * * * root /sbin/start --quiet reddit-job-update_promos
 618 */5  * * * * root /sbin/start --quiet reddit-job-clean_up_hardcache
 619 */2  * * * * root /sbin/start --quiet reddit-job-broken_things
 620 */2  * * * * root /sbin/start --quiet reddit-job-rising
 621 0    * * * * root /sbin/start --quiet reddit-job-trylater
 622
 623 # liveupdate
 624 *    * * * * root /sbin/start --quiet reddit-job-liveupdate_activity
 625
 626 # jobs that recalculate time-limited listings (e.g. top this year)
 627 PGPASSWORD=password
 628 */15 * * * * $REDDIT_USER $REDDIT_SRC/reddit/scripts/compute_time_listings link year "['hour', 'day', 'week', 'month', 'year']"
 629 */15 * * * * $REDDIT_USER $REDDIT_SRC/reddit/scripts/compute_time_listings comment year "['hour', 'day', 'week', 'month', 'year']"
 630
 631 # disabled by default, uncomment if you need these jobs
 632 #*    * * * * root /sbin/start --quiet reddit-job-email
 633 #0    0 * * * root /sbin/start --quiet reddit-job-update_gold_users
 634 CRON
 635 fi
 636
 637 ###############################################################################
 638 # Complete plugin setup, if setup.sh exists
 639 ###############################################################################
 640 for plugin in $REDDIT_AVAILABLE_PLUGINS; do
 641     if [ -x $REDDIT_SRC/$plugin/setup.sh ]; then
 642         echo "Found setup.sh for $plugin; running setup script"
 643         $REDDIT_SRC/$plugin/setup.sh $REDDIT_SRC $REDDIT_USER
 644     fi
 645 done
 646
 647 ###############################################################################
 648 # Finished with install script
 649 ###############################################################################
 650 # print this out here. if vagrant's involved, it's gonna do more steps
 651 # afterwards and then re-run this script but that's ok.
 652 $RUNDIR/done.sh