bootstrap/config.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) Catalyst.Net Ltd 2019
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """
  19 Manage dependencies and bootstrap environments for Samba.
  20
  21 Config file for packages and templates.
  22
  23 Update the lists in this file to require new packages in the
  24 container images used in GitLab CI
  25
  26 Author: Joe Guo <joeg@catalyst.net.nz>
  27 """
  28 import os
  29 from os.path import abspath, dirname, join
  30 HERE = abspath(dirname(__file__))
  31 # output dir for rendered files
  32 OUT = join(HERE, 'generated-dists')
  33
  34
  35 # pkgs with same name in all packaging systems
  36 COMMON = [
  37     'acl',
  38     'attr',
  39     'autoconf',
  40     'binutils',
  41     'bison',
  42     'ccache',
  43     'curl',
  44     'chrpath',
  45     'codespell',
  46     'flex',
  47     'gcc',
  48     'gdb',
  49     'git',
  50     'gzip',
  51     'hostname',
  52     'htop',
  53     'jq',
  54     'lcov',
  55     'make',
  56     'patch',
  57     'perl',
  58     'psmisc',  # for pstree in test
  59     'rng-tools',
  60     'rsync',
  61     'sed',
  62     'shfmt',
  63     'sudo',  # docker images has no sudo by default
  64     'tar',
  65     'tree',
  66     'wget',
  67     'cargo',
  68 ]
  69
  70
  71 # define pkgs for all packaging systems in parallel
  72 # make it easier to find missing ones
  73 # use latest ubuntu and fedora as defaults
  74 # deb, rpm, ...
  75 PKGS = [
  76     # NAME1-dev, NAME2-devel
  77     ('lmdb-utils', 'lmdb'),
  78     ('mingw-w64', 'mingw64-gcc'),
  79     ('zlib1g-dev', 'zlib-devel'),
  80     ('libbsd-dev', 'libbsd-devel'),
  81     ('liburing-dev', 'liburing-devel'),
  82     ('libarchive-dev', 'libarchive-devel'),
  83     ('libblkid-dev', 'libblkid-devel'),
  84     ('libcap-dev', 'libcap-devel'),
  85     ('libacl1-dev', 'libacl-devel'),
  86     ('libattr1-dev', 'libattr-devel'),
  87     ('libutf8proc-dev', 'utf8proc-devel'),
  88     ('libssl-dev', 'openssl-devel'),
  89     ('libclang-dev', 'clang-devel'),
  90
  91     # libNAME1-dev, NAME2-devel
  92     ('libpopt-dev', 'popt-devel'),
  93     ('libreadline-dev', 'readline-devel'),
  94     ('libjansson-dev', 'jansson-devel'),
  95     ('liblmdb-dev', 'lmdb-devel'),
  96     ('libncurses5-dev', 'ncurses-devel'),
  97     # NOTE: Debian 7+ or Ubuntu 16.04+
  98     ('libsystemd-dev', 'systemd-devel'),
  99     ('libkrb5-dev', 'krb5-devel'),
 100     ('libldap2-dev', 'openldap-devel'),
 101     ('libcups2-dev', 'cups-devel'),
 102     ('libpam0g-dev', 'pam-devel'),
 103     ('libgpgme11-dev', 'gpgme-devel'),
 104     # NOTE: Debian 8+ and Ubuntu 14.04+
 105     ('libgnutls28-dev', 'gnutls-devel'),
 106     ('gnutls-bin', 'gnutls-utils'),
 107     ('libtasn1-bin', 'libtasn1-tools'),
 108     ('libtasn1-dev', 'libtasn1-devel'),
 109     ('', 'quota-devel'),
 110     ('uuid-dev', 'libuuid-devel'),
 111     ('libjs-jquery', ''),
 112     ('libavahi-common-dev', 'avahi-devel'),
 113     ('libdbus-1-dev', 'dbus-devel'),
 114     ('libpcap-dev', 'libpcap-devel'),
 115     ('libunwind-dev', 'libunwind-devel'),  # for back trace
 116     ('libglib2.0-dev', 'glib2-devel'),
 117     ('libicu-dev', 'libicu-devel'),
 118     ('heimdal-multidev', ''),
 119
 120     # NAME1, NAME2
 121     # for debian, locales provide locale support with language packs
 122     # ubuntu split language packs to language-pack-xx
 123     # for centos, glibc-common provide locale support with language packs
 124     # fedora split language packs  to glibc-langpack-xx
 125     ('locales', 'glibc-common'),  # required for locale
 126     ('language-pack-en', 'glibc-langpack-en'),  # we need en_US.UTF-8
 127     ('bind9utils', 'bind-utils'),
 128     ('dnsutils', ''),
 129     ('xsltproc', 'libxslt'),
 130     ('krb5-user', 'krb5-workstation'),
 131     ('krb5-config', ''),
 132     ('krb5-kdc', 'krb5-server'),
 133     ('apt-utils', 'yum-utils'),
 134     ('pkg-config', 'pkgconfig'),
 135     ('procps', 'procps-ng'),  # required for the free cmd in tests
 136     ('lsb-release', 'lsb-release'),  # we need lsb_release to show info
 137     ('', 'rpcgen'),  # required for test
 138     # refer: https://fedoraproject.org/wiki/Changes/SunRPCRemoval
 139     ('', 'libtirpc-devel'),  # for <rpc/rpc.h> header on fedora
 140     ('', 'rpcsvc-proto-devel'), # for <rpcsvc/rquota.h> header
 141     ('mawk', 'gawk'),
 142     ('', 'mold'),
 143     ('shellcheck', 'ShellCheck'),
 144     ('', 'crypto-policies-scripts'),
 145
 146     ('python3', 'python3'),
 147     ('python3-cryptography', 'python3-cryptography'), # for krb5 tests
 148     ('python3-dev', 'python3-devel'),
 149     ('python3-dbg', ''),
 150     ('python3-iso8601', 'python3-iso8601'),
 151     ('python3-gpg', 'python3-gpg'),  # defaults to ubuntu/fedora latest
 152     ('python3-markdown', 'python3-markdown'),
 153     ('python3-dnspython', 'python3-dns'),
 154     ('python3-pexpect', ''),  # for wintest only
 155     ('python3-pyasn1', 'python3-pyasn1'), # for krb5 tests
 156     ('python3-setproctitle', 'python3-setproctitle'),
 157     ('python3-requests', 'python3-requests'), # for cert auto enroll
 158
 159     ('', 'python3-libsemanage'),
 160     ('', 'python3-policycoreutils'),
 161
 162     # A copy of the `crypt` module that was removed in Python 3.13
 163     # See also https://bugzilla.samba.org/show_bug.cgi?id=15756
 164     ('', 'python3-crypt-r'),
 165
 166     # perl
 167     ('libparse-yapp-perl', 'perl-Parse-Yapp'),
 168     ('perl-modules', ''),
 169     ('', 'perl-FindBin'),
 170     ('', 'perl-Archive-Tar'),
 171     ('', 'perl-ExtUtils-MakeMaker'),
 172     ('', 'perl-Test-Base'),
 173     ('', 'perl-generators'),
 174     ('', 'perl-interpreter'),
 175
 176     # fs
 177     ('xfslibs-dev', 'xfsprogs-devel'), # for xfs quota support
 178     ('', 'glusterfs-api-devel'),
 179     ('glusterfs-common', 'glusterfs-devel'),
 180     ('libcephfs-dev', 'libcephfs-devel'),
 181
 182     # spotlight
 183     ('libtracker-sparql-2.0-dev', 'tracker-devel'),
 184
 185     # misc
 186     # @ means group for rpm, use fedora as rpm default
 187     ('build-essential', '@development-tools'),
 188     ('debhelper', ''),
 189     # rpm has no pkg for docbook-xml
 190     ('docbook-xml', 'docbook-dtds'),
 191     ('docbook-xsl', 'docbook-style-xsl'),
 192     ('libkeyutils-dev', 'keyutils-libs-devel'),
 193     ('', 'which'),
 194     ('xz-utils', 'xz')
 195 ]
 196
 197
 198 DEB_PKGS = COMMON + [pkg for pkg, _ in PKGS if pkg]
 199 RPM_PKGS = COMMON + [pkg for _, pkg in PKGS if pkg]
 200
 201 GENERATED_MARKER = r"""
 202 #
 203 # This file is generated by 'bootstrap/template.py --render'
 204 # See also bootstrap/config.py
 205 #
 206 """
 207
 208
 209 APT_BOOTSTRAP = r"""
 210 #!/bin/bash
 211 {GENERATED_MARKER}
 212 set -xueo pipefail
 213
 214 export DEBIAN_FRONTEND=noninteractive
 215 apt-get -y update
 216
 217 apt-get -y install \
 218     {pkgs}
 219
 220 apt-get -y autoremove
 221 apt-get -y autoclean
 222 apt-get -y clean
 223 """
 224
 225
 226 YUM_BOOTSTRAP = r"""
 227 #!/bin/bash
 228 {GENERATED_MARKER}
 229 set -xueo pipefail
 230
 231 yum update -y
 232 yum install -y epel-release
 233 yum install -y yum-plugin-copr
 234 yum copr enable -y sergiomb/SambaAD
 235 yum update -y
 236
 237 yum install -y \
 238     {pkgs}
 239
 240 yum clean all
 241
 242 if [ ! -f /usr/bin/python3 ]; then
 243     ln -sf /usr/bin/python3.6 /usr/bin/python3
 244 fi
 245 """
 246
 247 ROCKY8_DNF_BOOTSTRAP = r"""
 248 #!/bin/bash
 249 {GENERATED_MARKER}
 250 set -xueo pipefail
 251
 252 yum update -y
 253 yum install -y dnf-plugins-core
 254 yum install -y epel-release
 255 yum install -y centos-release-ceph-pacific
 256
 257 sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Ceph-*
 258 sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Ceph-*
 259 sed -i 's/$contentdir/centos/g' /etc/yum.repos.d/CentOS-Ceph-*
 260
 261 yum -v repolist all
 262 yum config-manager --set-enabled powertools -y
 263
 264 yum update -y
 265
 266 yum install -y \
 267     --setopt=install_weak_deps=False \
 268     --setopt=centos-ceph-pacific.module_hotfixes=true \
 269     {pkgs}
 270
 271 yum clean all
 272 """
 273
 274 CENTOS9S_DNF_BOOTSTRAP = r"""
 275 #!/bin/bash
 276 {GENERATED_MARKER}
 277 set -xueo pipefail
 278
 279 dnf update -y
 280 dnf install -y dnf-plugins-core
 281 dnf install -y epel-release
 282 dnf install -y centos-release-gluster9
 283
 284 dnf -v repolist all
 285 dnf config-manager --set-enabled crb -y
 286
 287 dnf update -y
 288
 289 dnf install -y \
 290     --setopt=install_weak_deps=False \
 291     {pkgs}
 292
 293 dnf clean all
 294 """
 295
 296 DNF_BOOTSTRAP = r"""
 297 #!/bin/bash
 298 {GENERATED_MARKER}
 299 set -xueo pipefail
 300
 301 dnf update -y
 302
 303 dnf install -y \
 304     --setopt=install_weak_deps=False \
 305     {pkgs}
 306
 307 dnf clean all
 308
 309 update-crypto-policies --set DEFAULT:AD-SUPPORT
 310 """
 311
 312 DNF_BOOTSTRAP_MIT = r"""
 313 #!/bin/bash
 314 {GENERATED_MARKER}
 315 set -xueo pipefail
 316
 317 dnf update -y
 318 dnf install -y dnf-plugins-core
 319 dnf copr -y enable abbra/krb5-test
 320 dnf update -y
 321
 322 dnf install -y \
 323     --setopt=install_weak_deps=False \
 324     {pkgs}
 325
 326 dnf clean all
 327 """
 328
 329 ZYPPER_BOOTSTRAP = r"""
 330 #!/bin/bash
 331 {GENERATED_MARKER}
 332 set -xueo pipefail
 333
 334 zypper --non-interactive refresh
 335 zypper --non-interactive update
 336 zypper --non-interactive install \
 337     --no-recommends \
 338     system-user-nobody \
 339     {pkgs}
 340
 341 zypper --non-interactive clean
 342
 343 if [ -f /usr/lib/mit/bin/krb5-config ]; then
 344     ln -sf /usr/lib/mit/bin/krb5-config /usr/bin/krb5-config
 345 fi
 346 """
 347
 348 # A generic shell script to setup locale
 349 LOCALE_SETUP = r"""
 350 #!/bin/bash
 351 {GENERATED_MARKER}
 352 set -xueo pipefail
 353
 354 # refer to /usr/share/i18n/locales
 355 INPUTFILE=en_US
 356 # refer to /usr/share/i18n/charmaps
 357 CHARMAP=UTF-8
 358 # locale to generate in /usr/lib/locale
 359 # glibc/localedef will normalize UTF-8 to utf8, follow the naming style
 360 LOCALE=$INPUTFILE.utf8
 361
 362 # if locale is already correct, exit
 363 ( locale | grep LC_ALL | grep -i $LOCALE ) && exit 0
 364
 365 # if locale not available, generate locale into /usr/lib/locale
 366 if ! ( locale --all-locales | grep -i $LOCALE )
 367 then
 368     # no-archive means create its own dir
 369     localedef --inputfile $INPUTFILE --charmap $CHARMAP --no-archive $LOCALE
 370 fi
 371
 372 # update locale conf and global env file
 373 # set both LC_ALL and LANG for safe
 374
 375 # update conf for Debian family
 376 FILE=/etc/default/locale
 377 if [ -f $FILE ]
 378 then
 379     echo LC_ALL="$LOCALE" > $FILE
 380     echo LANG="$LOCALE" >> $FILE
 381 fi
 382
 383 # update conf for RedHat family
 384 FILE=/etc/locale.conf
 385 if [ -f $FILE ]
 386 then
 387     # LC_ALL is not valid in this file, set LANG only
 388     echo LANG="$LOCALE" > $FILE
 389 fi
 390
 391 # update global env file
 392 FILE=/etc/environment
 393 if [ -f $FILE ]
 394 then
 395     # append LC_ALL if not exist
 396     grep LC_ALL $FILE || echo LC_ALL="$LOCALE" >> $FILE
 397     # append LANG if not exist
 398     grep LANG $FILE || echo LANG="$LOCALE" >> $FILE
 399 fi
 400 """
 401
 402
 403 DOCKERFILE = r"""
 404 {GENERATED_MARKER}
 405 FROM {docker_image}
 406
 407 # pass in with --build-arg while build
 408 ARG SHA1SUM
 409 RUN [ -n $SHA1SUM ] && echo $SHA1SUM > /sha1sum.txt
 410
 411 ADD *.sh /tmp/
 412 # need root permission, do it before USER samba
 413 RUN /tmp/bootstrap.sh && /tmp/locale.sh
 414
 415 # if ld.gold exists, force link it to ld
 416 RUN set -x; ! LD_GOLD=$(which ld.gold) || {{ LD=$(which ld) && ln -sf $LD_GOLD $LD && test -x $LD && echo "$LD is now $LD_GOLD"; }}
 417 # if ld.mold exists, force link it to ld (prefer mold over gold! ;-)
 418 RUN set -x; ! LD_MOLD=$(which ld.mold) || {{ LD=$(which ld) && ln -sf $LD_MOLD $LD && test -x $LD && echo "$LD is now $LD_MOLD"; }}
 419
 420 # make test can not work with root, so we have to create a new user
 421 RUN useradd -m -U -s /bin/bash samba && \
 422     mkdir -p /etc/sudoers.d && \
 423     echo "samba ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/samba
 424
 425 USER samba
 426 WORKDIR /home/samba
 427 # samba tests rely on this
 428 ENV USER=samba LC_ALL=en_US.utf8 LANG=en_US.utf8 LANGUAGE=en_US
 429 """
 430
 431 # Vagrantfile snippet for each dist
 432 VAGRANTFILE_SNIPPET = r"""
 433     config.vm.define "{name}" do |v|
 434         v.vm.box = "{vagrant_box}"
 435         v.vm.hostname = "{name}"
 436         v.vm.provision :shell, path: "{name}/bootstrap.sh"
 437         v.vm.provision :shell, path: "{name}/locale.sh"
 438     end
 439 """
 440
 441 # global Vagrantfile with snippets for all dists
 442 VAGRANTFILE_GLOBAL = r"""
 443 {GENERATED_MARKER}
 444
 445 Vagrant.configure("2") do |config|
 446     config.ssh.insert_key = false
 447
 448 {vagrantfile_snippets}
 449
 450 end
 451 """
 452
 453 DEB_DISTS = {
 454     'debian11': {
 455         'docker_image': 'debian:11',
 456         'vagrant_box': 'debian/bullseye64',
 457         'replace': {
 458             'language-pack-en': '',   # included in locales
 459             'shfmt': '',
 460             'cargo': '', # included cargo is broken
 461         }
 462     },
 463     'debian11-32bit': {
 464         'docker_image': 'debian:11',  # specify the platform in .gitlab-ci.yaml
 465         'vagrant_box': 'debian/bullseye32',
 466         'replace': {
 467             'language-pack-en': '',   # included in locales
 468             'shfmt': '',
 469             'cargo': '', # included cargo is broken
 470         }
 471     },
 472     'debian12': {
 473         'docker_image': 'debian:12',
 474         'vagrant_box': 'debian/bookworm64',
 475         'replace': {
 476             'language-pack-en': '',   # included in locales
 477             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 478             'cargo': '', # included cargo is broken
 479         }
 480     },
 481     'debian12-32bit': {
 482         'docker_image': 'registry-1.docker.io/i386/debian:12',
 483         'vagrant_box': 'debian/bookworm32',
 484         'replace': {
 485             'language-pack-en': '',   # included in locales
 486             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 487             'cargo': '', # included cargo is broken
 488         }
 489     },
 490     'ubuntu1804': {
 491         'docker_image': 'ubuntu:18.04',
 492         'vagrant_box': 'ubuntu/bionic64',
 493         'replace': {
 494             'liburing-dev': '',   # not available
 495             'shfmt': '',
 496         }
 497     },
 498     'ubuntu1804-32bit': {
 499         'docker_image': 'registry-1.docker.io/i386/ubuntu:18.04',
 500         'vagrant_box': 'ubuntu/bionic32',
 501         'replace': {
 502             'liburing-dev': '',   # not available
 503             'shfmt': '',
 504         }
 505     },
 506     'ubuntu2004': {
 507         'docker_image': 'ubuntu:20.04',
 508         'vagrant_box': 'ubuntu/focal64',
 509         'replace': {
 510             'liburing-dev': '',   # not available
 511             'shfmt': '',
 512         }
 513     },
 514     'ubuntu2204': {
 515         'docker_image': 'ubuntu:22.04',
 516         'vagrant_box': 'ubuntu/jammy64',
 517         'replace': {
 518             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 519         },
 520     }
 521 }
 522
 523
 524 RPM_DISTS = {
 525     'rocky8': {
 526         'docker_image': 'docker.io/library/rockylinux:8',
 527         'vagrant_box': 'rocky/8',
 528         'bootstrap': ROCKY8_DNF_BOOTSTRAP,
 529         'replace': {
 530             'lsb-release': 'redhat-lsb',
 531             '@development-tools': '"@Development Tools"',  # add quotes
 532             'lcov': '', # does not exist
 533             'perl-JSON-Parse': '', # does not exist?
 534             'perl-Test-Base': 'perl-Test-Simple',
 535             'perl-FindBin': '',
 536             'liburing-devel': '', # not available yet, Add me back, once available!
 537             'mold': '',
 538             'ShellCheck': '',
 539             'shfmt': '',
 540             'codespell': '',
 541             'python3-crypt-r': '',
 542         }
 543     },
 544     'centos9s': {
 545         'docker_image': 'quay.io/centos/centos:stream9',
 546         'vagrant_box': 'centos/stream9',
 547         'bootstrap': CENTOS9S_DNF_BOOTSTRAP,
 548         'replace': {
 549             'lsb-release': 'lsb_release',
 550             '@development-tools': '"@Development Tools"',  # add quotes
 551             'lcov': '', # does not exist
 552             'perl-JSON-Parse': '', # does not exist?
 553             'perl-Test-Base': 'perl-Test-Simple',
 554             'perl-FindBin': '',
 555             'mold': '',
 556             'ShellCheck': '',
 557             'shfmt': '',
 558             'codespell': '',
 559             'libcephfs-devel': '',  # not available anymore
 560             'curl': '',  # Use installed curl-minimal
 561             'python3-crypt-r': '',
 562         }
 563     },
 564     'fedora41': {
 565         'docker_image': 'quay.io/fedora/fedora:41',
 566         'vagrant_box': 'fedora/41-cloud-base',
 567         'bootstrap': DNF_BOOTSTRAP,
 568         'replace': {
 569             'lsb-release': 'redhat-lsb',
 570             'perl-FindBin': '',
 571             'python3-iso8601': 'python3-dateutil',
 572             'libtracker-sparql-2.0-dev': '',  # only tracker 3.x is available
 573         }
 574     },
 575     'opensuse155': {
 576         'docker_image': 'opensuse/leap:15.5',
 577         'vagrant_box': 'opensuse/openSUSE-15.5-x86_64',
 578         'bootstrap': ZYPPER_BOOTSTRAP,
 579         'replace': {
 580             '@development-tools': '',
 581             'dbus-devel': 'dbus-1-devel',
 582             'docbook-style-xsl': 'docbook-xsl-stylesheets',
 583             'glibc-common': 'glibc-locale',
 584             'glibc-locale-source': 'glibc-i18ndata',
 585             'glibc-langpack-en': '',
 586             'jansson-devel': 'libjansson-devel',
 587             'keyutils-libs-devel': 'keyutils-devel',
 588             'krb5-workstation': 'krb5-client',
 589             'python3-libsemanage': 'python3-semanage',
 590             'python3-crypt-r': '',
 591             'openldap-devel': 'openldap2-devel',
 592             'perl-Archive-Tar': 'perl-Archive-Tar-Wrapper',
 593             'perl-JSON-Parse': 'perl-JSON-XS',
 594             'perl-generators': '',
 595             'perl-interpreter': '',
 596             'perl-FindBin': '',
 597             'procps-ng': 'procps',
 598             'python3-iso8601': 'python3-python-dateutil',
 599             'python3-dns': 'python3-dnspython',
 600             'python3-markdown': 'python3-Markdown',
 601             'quota-devel': '',
 602             'glusterfs-api-devel': '',
 603             'gnutls-utils': 'gnutls',
 604             'libtasn1-tools': '', # asn1Parser is part of libtasn1
 605             'mold': '',
 606             'shfmt': '',
 607             'yum-utils': '',
 608         }
 609     }
 610 }
 611
 612
 613 DEB_FAMILY = {
 614     'name': 'deb',
 615     'pkgs': DEB_PKGS,
 616     'bootstrap': APT_BOOTSTRAP,  # family default
 617     'dists': DEB_DISTS,
 618 }
 619
 620
 621 RPM_FAMILY = {
 622     'name': 'rpm',
 623     'pkgs': RPM_PKGS,
 624     'bootstrap': YUM_BOOTSTRAP,  # family default
 625     'dists': RPM_DISTS,
 626 }
 627
 628
 629 YML_HEADER = r"""
 630 ---
 631 packages:
 632 """
 633
 634
 635 def expand_family_dists(family):
 636     dists = {}
 637     for name, config in family['dists'].items():
 638         config = config.copy()
 639         config['name'] = name
 640         config['home'] = join(OUT, name)
 641         config['family'] = family['name']
 642         config['GENERATED_MARKER'] = GENERATED_MARKER
 643
 644         # replace dist specific pkgs
 645         replace = config.get('replace', {})
 646         pkgs = []
 647         for pkg in family['pkgs']:
 648             pkg = replace.get(pkg, pkg)  # replace if exists or get self
 649             if pkg:
 650                 pkgs.append(pkg)
 651         pkgs.sort()
 652
 653         lines = ['  - {}'.format(pkg) for pkg in pkgs]
 654         config['packages.yml'] = YML_HEADER.lstrip() + os.linesep.join(lines)
 655
 656         sep = ' \\' + os.linesep + '    '
 657         config['pkgs'] = sep.join(pkgs)
 658
 659         # get dist bootstrap template or fall back to family default
 660         bootstrap_template = config.get('bootstrap', family['bootstrap'])
 661         config['bootstrap.sh'] = bootstrap_template.format(**config).strip()
 662         config['locale.sh'] = LOCALE_SETUP.format(**config).strip()
 663
 664         config['Dockerfile'] = DOCKERFILE.format(**config).strip()
 665         # keep the indent, no strip
 666         config['vagrantfile_snippet'] = VAGRANTFILE_SNIPPET.format(**config)
 667
 668         dists[name] = config
 669     return dists
 670
 671
 672 # expanded config for dists
 673 DEB_DISTS_EXP = expand_family_dists(DEB_FAMILY)
 674 RPM_DISTS_EXP = expand_family_dists(RPM_FAMILY)
 675
 676 # assemble all together
 677 DISTS = {}
 678 DISTS.update(DEB_DISTS_EXP)
 679 DISTS.update(RPM_DISTS_EXP)
 680
 681
 682 def render_vagrantfile(dists):
 683     """
 684     Render all snippets for each dist into global Vagrantfile.
 685
 686     Vagrant supports multiple vms in one Vagrantfile.
 687     This make it easier to manage the fleet, e.g:
 688
 689     start all: vagrant up
 690     start one: vagrant up ubuntu1804
 691
 692     All other commands apply to above syntax, e.g.: status, destroy, provision
 693     """
 694     # sort dists by name and put all vagrantfile snippets together
 695     snippets = [
 696         dists[dist]['vagrantfile_snippet']
 697         for dist in sorted(dists.keys())]
 698
 699     return VAGRANTFILE_GLOBAL.format(
 700             vagrantfile_snippets=''.join(snippets),
 701             GENERATED_MARKER=GENERATED_MARKER
 702             )
 703
 704
 705 VAGRANTFILE = render_vagrantfile(DISTS)
 706
 707
 708 # data we need to expose
 709 __all__ = ['DISTS', 'VAGRANTFILE', 'OUT']