build-all-cores.sh

   1 #!/bin/sh
   2 ./run-sbcl.sh --noprint --disable-debugger $* <<\EOF
   3 (defvar *configs-to-build* (cdr *posix-argv*))
   4
   5 (defparameter *jobs*
   6   (max 1
   7        #+unix
   8        (floor (sb-alien:alien-funcall
   9                (sb-alien:extern-alien "sysconf"
  10                                       (function sb-alien:long sb-alien:int))
  11                sb-unix::sc-nprocessors-onln)
  12               2)))
  13
  14 (let ((first (car *configs-to-build*)))
  15   (if (and (stringp first)
  16            (>= (length first) 2)
  17            (string= first "-j" :end1 2))
  18       ;; allow "-jN" as two args
  19       (cond ((= (length first) 2)
  20              (pop *configs-to-build*)
  21              (setq *jobs* (parse-integer (pop *configs-to-build*))))
  22             ((digit-char-p (char first 2)) ; or "-jN" as one arg
  23              (setq *jobs* (parse-integer first :start 2))
  24              (pop *configs-to-build*))
  25             (t
  26              (error "-j requires a number")))))
  27
  28 (when (> *jobs* 1) (format t "~&Using up to ~D job~:P~%" *jobs*))
  29
  30 ;;; I should probably have a way to add a fixed set of features in
  31 ;;; for an architecture, such as :LITTLE-ENDIAN for x86{-64}.
  32 ;;; This list is: arch-name and ((configuation-name feature ...) ...)
  33 (defparameter *all-configurations*
  34   '(("arm"    ("arm" :little-endian :largefile))
  35     ("arm64"  ("arm64" :little-endian :sb-thread)
  36               ("arm64-darwin" :darwin :bsd :unix :mach-o :little-endian :sb-thread :darwin-jit)
  37               ("arm64-reloc"  :little-endian :sb-thread :relocatable-static-space)
  38               ("arm64-immobile-space" :little-endian :sb-thread :immobile-space))
  39     ("mips"   ("mips" :largefile :little-endian))
  40     ("ppc"    ("ppc" :big-endian)
  41               ("ppc-thread" :big-endian :sb-thread))
  42     ("ppc64"  ("ppc64" :ppc64 :big-endian)) ; sb-thread is the default and required
  43     ("riscv"  ("riscv" :64-bit :little-endian :sb-thread))
  44     ("sparc"  ("sparc" :big-endian)
  45               ("sparc-sunos" :big-endian :unix :sunos :elf))
  46     ("x86"    ("x86" :little-endian :largefile (not :sb-eval) :sb-fasteval)
  47               ("x86-ascii" :little-endian :largefile (not :sb-unicode))
  48               ("x86-thread" :little-endian :largefile :sb-thread)
  49               ("x86-linux" :little-endian :largefile :sb-thread :linux :unix :elf :sb-thread))
  50     ("x86-64" ("x86-64" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256)
  51               ("x86-64-linux" :linux :unix :elf :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
  52                               (not :sb-eval) :sb-fasteval)
  53               ("x86-64-darwin" :darwin :bsd :unix :mach-o :little-endian :avx2 :gencgc
  54                                :sb-simd-pack :sb-simd-pack-256)
  55               ("x86-64-imm" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
  56                             :immobile-space)
  57               ("x86-64-permgen" :little-endian :avx2 :gencgc :sb-simd-pack :sb-simd-pack-256
  58                                 :permgen))))
  59
  60 (setq sb-ext:*evaluator-mode* :compile)
  61 (defun interpolate (string substitutions)
  62   (sb-int:collect ((fragments))
  63     (let ((start 0))
  64       (loop
  65        (let ((p (position #\{ string :start start)))
  66          (unless p
  67            (fragments (subseq string start))
  68            (return (apply #'concatenate 'string (fragments))))
  69          (fragments (subseq string start p))
  70          (let* ((end (the (not null) (position #\} string :start (1+ p))))
  71                 (var (subseq string (1+ p) end))
  72                 (pair (assoc var substitutions :test #'string=)))
  73            (fragments (the string (cdr pair)))
  74            (setq start (1+ end))))))))
  75
  76 (defun add-os-features (arch features)
  77   (if (intersection '(:unix :win32) features)
  78       features
  79       (let* ((arch-symbol (sb-int:keywordicate (string-upcase arch)))
  80              (os-features (case arch-symbol
  81                             ((:x86 :x86-64) '(:win32 :sb-thread :sb-safepoint))
  82                             (t '(:unix :linux :elf)))))
  83         (append os-features features))))
  84
  85 ;;; TODO: dependencies for each target based on build-order.lisp-expr
  86 (let
  87   ((new
  88     (with-output-to-string (makefile)
  89       (format makefile
  90           "SBCL=src/runtime/sbcl
  91 ARGS=--core output/sbcl.core --noinform --disable-debugger --noprint --no-userinit --no-sysinit
  92 SCRIPT1=crossbuild-runner/pass-1.lisp
  93 SCRIPT2=crossbuild-runner/pass-2.lisp
  94 DEPS1=crossbuild-runner/pass-1.lisp src/cold/build-order.lisp-expr~%")
  95   (dolist (arch+configs *all-configurations*)
  96     (dolist (config (cdr arch+configs))
  97       (destructuring-bind (config-name . features) config
  98         (setq features (add-os-features (car arch+configs) features))
  99         (format makefile
 100                 (interpolate
 101                  "~%obj/xbuild/{cfg}/xc.core: $(DEPS1)
 102         $(SBCL) $(ARGS) {cfg} {arch} \"{feat}\" < $(SCRIPT1)
 103 obj/xbuild/{cfg}.core: obj/xbuild/{cfg}/xc.core
 104         $(SBCL) $(ARGS) {cfg} < $(SCRIPT2)~%"
 105                  `(("cfg" . ,config-name)
 106                    ("arch" . ,(car arch+configs))
 107                    ;; features string can't contain a #\newline
 108                    ("feat" . ,(write-to-string features :pretty nil))))))))))
 109    (existing (with-open-file (stream "crossbuild-runner/Makefile")
 110                (let ((a (make-array (file-length stream) :element-type 'character)))
 111                  (read-sequence a stream)
 112                  a))))
 113  (unless (string= new existing)
 114     (with-open-file (stream "crossbuild-runner/Makefile" :direction :output :if-exists :supersede)
 115       (write-string new stream))
 116     (format t "~&Rewrote Makefile~%")))
 117
 118 (defun corefiles ()
 119   (mapcar (lambda (x) (format nil "obj/xbuild/~A.core" x))
 120           (or *configs-to-build*
 121               (mapcan (lambda (x) (mapcar 'car (cdr x)))
 122                       *all-configurations*))))
 123
 124 (ensure-directories-exist "obj/xbuild/")
 125 ;; if it's just one target this could exec() instead
 126 (defvar *process*
 127   (run-program "make"
 128                `(,(format nil "-j~D" *jobs*)
 129                  "-k"
 130                  "-fcrossbuild-runner/Makefile"
 131                  ,@(corefiles))
 132                :output t :error t
 133                :search t))
 134 (when (= (process-exit-code *process*) 0)
 135   (load "src/cold/shared" :verbose t)
 136   (load "validate-float.lisp")
 137   (dolist (pathname (directory "obj/xbuild/**/xfloat-math.lisp-expr"))
 138     (check-float-file pathname))
 139   (dolist (nbits '(30 61 63))
 140     (let* ((filename (format nil "xperfecthash~D.lisp-expr" nbits))
 141            (sources (directory (format nil "obj/xbuild/*/from-xc/~A" filename))))
 142       (when sources
 143         (funcall (intern "UPDATE-PERFECT-HASHFUNS" "SB-COLD")
 144                  sources filename))))
 145   (format t "~&Success~%"))
 146 (sb-ext:exit :code (sb-ext:process-exit-code *process*))
 147 EOF