timings/ElectricCloud/genscons.pl

   1 #!/usr/bin/perl
   2 #
   3 # genscons.pl
   4 #
   5 # This script generates a build tree with $ndirs + 1 directories, containing
   6 # $nfils source files each, and both SConstruct files and non-recursive
   7 # Makefiles to build the tree.
   8 #
   9 # Copyright (c) 2010 Electric Cloud, Inc.
  10 # All rights reserved.
  11 # Redistribution and use in source and binary forms, with or without
  12 # modification, are permitted provided that the following conditions are met:
  13 #
  14 #     * Redistributions of source code must retain the above copyright notice,
  15 #       this list of conditions and the following disclaimer.
  16 #     * Redistributions in binary form must reproduce the above copyright
  17 #       notice, this list of conditions and the following disclaimer in the
  18 #       documentation and/or other materials provided with the distribution.
  19 #     * Neither the name of Electric Cloud nor the names of its employees may
  20 #       be used to endorse or promote products derived from this software
  21 #       without specific prior written permission.
  22 #
  23 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  24 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26 # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  27 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  28 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  29 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  30 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  31 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  32 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33 # POSSIBILITY OF SUCH DAMAGE.
  34
  35 # These settings will generate 2,000 total C files, grouped in blocks of 20,
  36 # each of which does a couple of #includes.
  37
  38 $nlvls = 2; $ndirs = 3; $nfils = 500; $nlups = 2; $group = 20;
  39
  40 $rmdel = "rm -f";
  41 $OBJ = ".o";
  42
  43 # Check Variables settings
  44
  45 #if ( ($nfils % $group) != 0) {
  46 #  die "ERROR: The number of files ($nfils) must be a multiple of the group size ($group)";
  47 #}
  48
  49 sub init() {
  50
  51 #    use strict;
  52 #    use warnings FATAL => 'all';
  53     use Getopt::Std;
  54     my $opt_string = 'd:f:g:l:u:h';
  55     getopts ( "$opt_string", \%opt) or usage ();
  56
  57     &usage() if $opt{h};
  58
  59     $ndirs      = $opt{d} if $opt{d};
  60     $nfils      = $opt{f} if $opt{f};
  61     $group      = $opt{g} if $opt{g};
  62     $nlvls      = $opt{l} if $opt{l};
  63     $nlups      = $opt{u} if $opt{u};
  64
  65     return 0;
  66 }
  67
  68 sub usage () {
  69     print STDERR << "EOF";
  70
  71 usage: $0 [-l Levels] [-d Dirs] [-f Files] [-g Grouping] [-u Lookups]
  72           [-h]
  73
  74 -l Levels   : number of levels of directories             (default $nlvls)
  75 -d Dirs     : number of directories at each level         (default $ndirs)
  76 -f Files    : number of source files per directory        (default $nfils)
  77 -g Grouping : compile in groups of Grouping files         (default $group)
  78 -u Lookups  : number of lookups per source file           (default $nlups)
  79
  80 -h         : this help message
  81
  82 You can edit the default values in genscons.pl
  83
  84 EOF
  85     exit;
  86 }
  87
  88 # fmt
  89 #
  90 # Adds commas to long numbers to make them more readable.
  91 sub fmt {
  92     my ($value) = @_;
  93     my $running = 1;
  94     while ($running) {
  95         $value =~ s/([0-9])([0-9]{3})(?![0-9])/\1,\2/g;
  96         $running = ($1 =~ /[0-9]/);
  97     }
  98     return $value;
  99 }
 100
 101 # gen_incfile
 102 #
 103 # Generate a generic include file to keep the compiler busy.
 104 sub gen_incfile {
 105     my ($basedir, $filename, $idx) = @_;
 106
 107     open (INC, "> $filename") || die "Cannot open $filename for output.";
 108     print INC "#ifndef $filname[$idx]\n"
 109         . "#define $filname[$idx] \"$basedir\"\n\n"
 110         . "#include \"stdio.h\"\n\n";
 111     print INC "#endif\n";
 112     close (INC);
 113 }
 114
 115 # gen_cfile
 116 #
 117 # Generate a distinct C file to keep the compiler busy.
 118 sub gen_cfile {
 119     my ($basedir, $filename, $idx) = @_;
 120
 121     open (CFILE, "> $basedir/$filename")
 122         || die "Cannot open $basedir/$filename for output";
 123
 124     $buff = "#include <stdlib.h>\n";
 125     $buff .= "#include <$filname[$idx].h>\n";
 126
 127     $buff .= "#include <omega.h>\n";
 128     print CFILE $buff;
 129
 130     if ($group == 1) {
 131         print CFILE "int main (int argc, char * argv[]) {\n"
 132             . "\tint i, mb_out;\n"
 133             . "\tprintf (\"I am $basedir/%s\\n\", \"$filname[$idx]\""
 134             . ");\n"
 135             . "\treturn (0);\n}\n";
 136     }
 137     elsif ( ($group - ($fil[$idx] % $group)) == 1) {
 138         print CFILE "int printr_$filname[$idx] (char * fname) {\n"
 139             . "    printf (\"I am $basedir/%s\\n\", fname);\n"
 140             . "    return (0);\n}\n";
 141     }
 142     elsif ( ($fil[$idx] % $group) == 0) {
 143         $idx2 = $fil[$idx] + 1;
 144         print CFILE "extern int printr_$file[$idx2] (char * fname);\n"
 145             . "int main (int argc, char * argv[]) {\n"
 146             . "\tint i, mb_out;\n";
 147
 148         print CFILE "\tprintr_$file[$idx2] (\"$filname[$idx]\");\n"
 149             . "\n"
 150             . "\tmb_out = 0;\n"
 151             . "\tif (argc > 1) {\n"
 152             . "\t\tmb_out = atoi (argv[1]);\n"
 153             . "\t}\n"
 154             . "\tfor (i = 0; i < (mb_out * 16000); i++) {\n"
 155             . "\t\tprintf (\"%07d 9a123456789b123456789c12345"
 156             . "6789d123456789e123456789f12345678\\n\", i);\n"
 157             . "\t}\n"
 158             . "\texit (0);\n}\n";
 159     }
 160     else {
 161         $idx2 = $fil[$idx] + 1;
 162         print CFILE "extern int printr_$file[$idx2] (char * fname);\n"
 163             . "int printr_$filname[$idx] (char * fname) {\n"
 164             . "    printr_$file[$idx2] (fname);\n"
 165             . "    return (0);\n}\n";
 166     }
 167     close (CFILE);
 168 }
 169
 170 # mkdirs
 171 #
 172 # Recursive function for generating directories full of files to build, and
 173 # the makefiles that go with them.
 174 #
 175 sub mkdirs {
 176     my ($idx, $basedir, $thisLvl) = @_;
 177
 178     if ( ! -d $basedir ) {
 179         mkdir $basedir || die "Cannot create folder $basedir: $!";
 180     }
 181
 182     $relpath[$idx] = substr ($basedir, 8); # assumed top dir is "sconsbld"
 183     if ("$relpath[$idx]" eq "") {
 184         $relpath[$idx] = ".";
 185         $basestr = $basedir;
 186         $foo = "";
 187         $libdir = ".";
 188     }
 189     else {
 190         $basestr = $relpath[$idx];
 191         $basestr =~ s|/|_|g;
 192         $foo = substr($basedir,9) . "/";
 193         $libdir = substr($basedir,9);
 194     }
 195     $bstr[$idx] = $basestr;
 196
 197     $dirstr = $basedir;
 198     $dirstr =~ s|/|_|g;
 199
 200 #   $basedir is $relpath[$idx] with "sconsbld/" prepended so
 201 #   $dirstr is $basestr with "sconsbld_" prepended.
 202
 203     $cdstr = ".";
 204     for ($cdidx = 1; $cdidx < $thisLvl; $cdidx++) { $cdstr .= "/.."; }
 205
 206     $thissc[$idx] = "$basedir/SConstruct";
 207     $thismk[$idx] = "$basedir/Makefile";
 208     $fzero[$idx] = "$basedir/file0";
 209
 210     open (SC, "> $thissc[$idx]")
 211         || die "Cannot open $thissc[$idx] for output: $!";
 212     open (MK, "> $thismk[$idx]")
 213         || die "Cannot open $thismk[$idx] for output: $!";
 214
 215     print SC "import os\n"
 216         . "env = Environment(ENV = os.environ)\n\n";
 217
 218     my $cfgpath = "";
 219     for (my $junk = 1; $junk < $thisLvl; $junk++) { $cfgpath .= "../"; }
 220
 221     my $arkive = "archive";
 222
 223     if ($thisLvl == 1) { $mysubdir = "."; }
 224     else { $mysubdir = substr ($basedir, 8); }
 225
 226
 227     if (index ($basedir, "/") > 0) {
 228         @pieces = split (/\//, $basedir);
 229         $fileSuffix = "";
 230         for (my $ii =0; $ii <= $#pieces; $ii++) {
 231             $fileSuffix .= "_" . $pieces[$ii];
 232         }
 233     } else {
 234         $fileSuffix = "_" . $basedir;
 235     }
 236
 237     for (my $lupsIdx = 0; $lupsIdx < $nlups; $lupsIdx++) {
 238         printf SC ("env.Append (CPPPATH = ['./lup%03d$fileSuffix'])\n",
 239                    $lupsIdx);
 240         if ($lupsIdx == 0) {
 241             $eq = "=";
 242         } else {
 243             $eq = "+=";
 244         }
 245         printf MK ("${foo}%%.o: CPPFLAGS $eq -I${foo}lup%03d$fileSuffix\n",
 246                    $lupsIdx);
 247     }
 248     print SC "env.Append (LIBPATH = ['.'])\n\n";
 249     print MK "${foo}%: LDFLAGS = -L$libdir\n\n";
 250
 251     if ($thisLvl == 1) {
 252         print SC "\n"
 253             . "env.Help(\"\"\"\n"
 254             . "This build has parameters:\n"
 255             . "     number of levels  = $nlvls\n"
 256             . "     directories/level = $ndirs\n"
 257             . "     cfiles/directory  = $nfils\n"
 258             . "     lookups/source    = $nlups\n"
 259             . "     compiles grouped  = $group\n"
 260             . "\"\"\")\n";
 261
 262         print MK "${foo}%.a:\n";
 263         print MK "\tar rc \$@ \$^\n";
 264         print MK "\tranlib \$@\n\n";
 265         print MK "%.o: %.c\n";
 266         print MK "\t\$(CC) -MMD -o \$@ -c \$(CPPFLAGS) \$<\n\n";
 267         print MK "%: %.o\n";
 268         print MK "\t\$(CC) -o \$@ \$< \$(LDFLAGS) -l\$(notdir \$@)\n\n";
 269         print MK "CC=gcc\n\n";
 270         print MK "all:\n\t\@ps -eo vsz,rss,comm | fgrep make\n\n";
 271     }
 272     print SC "\n";
 273     print MK "\n";
 274
 275     # Create include directories for doing additional lookups.
 276     for (my $ii = 0; $ii < $nlups; $ii++) {
 277         $lupDir = sprintf ("lup%03d$fileSuffix", $ii);
 278         mkdir "$basedir/$lupDir" || die "Couldn't create $basedir/$lupDir: $!";
 279         $totald++;
 280     }
 281
 282     $scfcc = "";
 283     $scfar = "";
 284     $mkfcc = "";
 285     $mkfar = "";
 286
 287     ###
 288     ### generate the .c files and the .h files they include.
 289     ### Also generate the corresponding Makefile commands.
 290     ###
 291     for (my $filidx = 0; $filidx < $nfils; $filidx++) {
 292         $file[$filidx] = sprintf ("f%05d$fileSuffix", $filidx);
 293     }
 294     for ($fil[$idx] = 0; $fil[$idx] < $nfils; $fil[$idx]++) {
 295         $filname[$idx] = "$file[$fil[$idx]]";
 296
 297         $nextnum = substr ($filname[$idx], 1, 5);
 298
 299         if ($group == 1) {
 300 #
 301 #           Even when there are no groups, pre-compiled headers
 302 #           still apply.
 303 #
 304             print SC "env.Program ('$filname[$idx].c')\n\n";
 305         } #                end of $group == 1
 306 #
 307 #       Compile source files in groups.
 308 #       This removes unique lookups but adds some :: rules.
 309 #
 310         else {
 311             if ( ($fil[$idx] % $group) == 0) {
 312                 if ("$scfcc" ne "") {
 313                     print SC "$scfcc\n$scfar\n\n";
 314                     $scfcc = "";
 315                     $scfar = "";
 316                 }
 317                 if ("$mkfcc" ne "") {
 318                     print MK "$mkfcc\n$mkfar\n\n";
 319                     $mkfcc = "";
 320                     $mkfar = "";
 321                 }
 322
 323                 $groupFilename = "$filname[$idx]";
 324                 $nextnum = substr ($filname[$idx], 1, 5);
 325
 326                 $scfcc = "env.Program('$filname[$idx]',\n"
 327                     . "\tLIBS=['$filname[$idx]'])\n";
 328
 329                 $scfar = "env.Library ('$filname[$idx]',\n"
 330                     . "\t['$filname[$idx].c'";
 331
 332                 $mkfcc = "TARGETS += ${foo}$filname[$idx]\n${foo}$filname[$idx]: ${foo}$filname[$idx].o ${foo}lib$filname[$idx].a\n";
 333                 $mkfar = "${foo}lib$filname[$idx].a: ${foo}$filname[$idx].o";
 334
 335                 print MK "SRCS += ${foo}$filname[$idx].c\n";
 336                 $tmpfnam = $filname[$idx];
 337                 for ($filei = 1; $filei < $group; $filei++) {
 338                     $tmpfnum = sprintf ("%05d", $nextnum + $filei);
 339                     substr ($tmpfnam, 1, 5) = $tmpfnum;
 340
 341                     $scfar .= ",\n\t '$tmpfnam.c'";
 342                     $mkfar .= " ${foo}$tmpfnam.o";
 343                     print MK "SRCS += ${foo}$tmpfnam.c\n";
 344                 }
 345                 $scfar .= "])\n\n";
 346                 $mkfar .= "\n";
 347             }
 348
 349         } # end of handling of compiles for $group > 1
 350
 351         gen_incfile($basedir, "$basedir/$lupDir/$filname[$idx].h", $idx);
 352
 353         if ($fil[$idx] == 0) {
 354             open (INC, "> $basedir/$lupDir/omega.h")
 355             || die "Cannot open $basedir/$lupDir/omega.h for output.";
 356             print INC "// comment in dummy file.\n";
 357             close (INC);
 358         }
 359
 360         gen_cfile($basedir, "$filname[$idx].c", $idx);
 361     } # end of generation of source files and header files
 362
 363     if ($group > 1 && "$scfcc" ne "") {
 364 #
 365 #       create makefile commands for the leftover files
 366 #
 367         print SC "$scfcc\n$scfar\n\n";
 368         print MK "$mkfcc\n$mkfar\n\n";
 369     }
 370
 371     close (SC);
 372     close (MK);
 373
 374     # Recurse and create more subdirectories and their contents.
 375     if ($thisLvl < $nlvls) {
 376         $allsubs[$idx] = "";
 377         for ($dir[$idx] = 0; $dir[$idx] < $ndirs; $dir[$idx]++) {
 378             $dirname[$idx] = sprintf ("d${thisLvl}_%d", $dir[$idx]);
 379             $allsubs[$idx] .= "$dirname[$idx].subdir ";
 380 #
 381 #           divide the subdirectories into 2 lists
 382 #           The two lists are/can be treated differently in Makefile.util
 383 #
 384             if ($dir[$idx] < ($ndirs / 2)) {
 385                 if ("$dirs1[$idx]" eq "") { $dirs1[$idx] = "$dirname[$idx]"; }
 386                 elsif (index ($dirs1[$idx], $dirname[$idx]) < 0) {
 387                     $dirs1[$idx] .= " $dirname[$idx]";
 388                 }
 389             }
 390             else {
 391                 if ("$dirs2[$idx]" eq "") { $dirs2[$idx] = "$dirname[$idx]"; }
 392                 elsif (index ($dirs2[$idx], $dirname[$idx]) < 0) {
 393                     $dirs2[$idx] .= " $dirname[$idx]";
 394 #
 395 #                   The preceding elsif should really just be
 396 #                   "else" but when nlvls > 2, you start getting repetition
 397 #                   of directory names in $dirs1[$idx] and $dirs2[$idx].
 398 #                   Rather than figure out where the duplication is coming
 399 #                   from, just prevent it.
 400 #
 401                 }
 402             }
 403
 404             if ( ! -d "$basedir/$dirname[$idx]") {
 405                 mkdir "$basedir/$dirname[$idx]" ||
 406                     die "Couldn't create $basedir/$dirname[$idx]: $!";
 407                 $totald++;
 408             }
 409
 410             &mkdirs ($idx + 1, "$basedir/$dirname[$idx]", $thisLvl + 1);
 411             if ($thisLvl == 1) {
 412                 print "Finished folder $dirname[$idx] in $basedir at "
 413                     . `date`;
 414             }
 415         }
 416
 417     }
 418
 419     if ($thisLvl < $nlvls) {
 420         open (SC, ">> $thissc[$idx]")
 421             || die "Cannot open $thissc[$idx] for append: $!";
 422         open (MK, ">> $thismk[$idx]")
 423             || die "Cannot open $thismk[$idx] for append: $!";
 424
 425         print SC "SConscript([";
 426
 427         if (index ($dirs1[$idx], " ") > 0) {
 428             @subdirs = split (/ /, $dirs1[$idx]);
 429             for ($i = 0; $i <= $#subdirs; $i++) {
 430                 print SC "'$subdirs[$i]/SConstruct',\n\t";
 431                 print MK "include $subdirs[$i]/Makefile\n";
 432             }
 433         }
 434         else {
 435             print SC "'$dirs1[$idx]/SConstruct',\n\t";
 436             print MK "include $dirs1[$idx]/Makefile\n";
 437         }
 438         if (index ($dirs2[$idx], " ") > 0) {
 439             @subdirs = split (/ /, $dirs2[$idx]);
 440             for ($i = 0; $i < $#subdirs; $i++) {
 441                 print SC "'$subdirs[$i]/SConstruct',\n\t";
 442                 print MK "include $subdirs[$i]/Makefile\n";
 443             }
 444             print SC "'$subdirs[$#subdirs]/SConstruct'";
 445             print MK "include $subdirs[$#subdirs]/Makefile\n";
 446         }
 447         else {
 448             print SC "'$dirs2[$idx]/SConstruct'";
 449             print MK "include $dirs2[$idx]/Makefile\n";
 450         }
 451         print SC "])\n\n";
 452         print SC "\n";
 453         print MK "NUL=\n";
 454         print MK "SPACE=\$(NUL) \$(NUL)\n";
 455         print MK "define nl\n\$(SPACE)\n\$(SPACE)\nendef\n\n";
 456         print MK "all: \$(TARGETS)\n\n";
 457         print MK "clean:\n";
 458         print MK "\t\$(foreach tgt,\$(TARGETS),rm -f \$(tgt)\$(nl))\n";
 459         print MK "\tfor n in d1*; do rm -f \$\$n/*.o ; rm -f \$\$n/*.a;done\n";
 460         print MK "\trm -f *.o ; rm -f *.a\n\n";
 461         print MK "-include \$(SRCS:.c=.d)";
 462
 463         close (SC);
 464         close (MK);
 465     }
 466     return 0;
 467 }
 468
 469 $basedir = "sconsbld";
 470 if ( ! -d $basedir) { mkdir $basedir || die "Couldn't create $basedir: $!"; }
 471
 472 &init ();
 473
 474 $numdirs = 0; # dirs other than include dirs
 475 for ($i = 0; $i < $nlvls; $i++) { $numdirs += ($ndirs ** $i); }
 476
 477 $totldirs = $numdirs * ($nlups + 1); # dirs including include dirs
 478
 479 # total   = ( .c              ) + ( .h              ) + mkfiles + Makefile.util
 480 #                                                               + Makefile.cfg
 481 #                                                               + readme
 482 #                                                               + omega.h
 483 #                                                               + Makefile.clean
 484
 485 $totlfils = ($nfils * $numdirs) + ($nfils * $numdirs)
 486     + $numdirs + 3 + $numdirs;
 487
 488 $totlobjs = $nfils * $numdirs;
 489 $totlexes = $numdirs * ($nfils / $group);
 490
 491 $totllups = $nfils * $numdirs * $nlups / $group;
 492 $allfiles = $totlfils + $totlobjs + $totlexes;
 493
 494 # one rule for each group plus overhead of 10/makefile
 495 $nrules = ($numdirs * $nfils / $group) + ($numdirs * 10);
 496
 497 $txt1 = "Number of levels = $nlvls\n"
 498       . "Number of dirs / level = $ndirs\n"
 499       . "Number of source files / dir = $nfils\n"
 500       . "Number of lookups / source file = $nlups\n"
 501       . "Number of compiles grouped      = $group\n";
 502 print $txt1;
 503
 504 print $vartxt;
 505     $numMakefiles = 1;
 506
 507 $txt2 = "Expecting:\n"
 508       . "\tdirectories:   " . fmt($totldirs) . "\n"
 509       . "\tsource files:  " . fmt($numdirs * $nfils) . "\n"
 510       . "\tinclude files: " . fmt($numdirs * ($nfils + 1)) . "\n"
 511       . "\tmakefiles:     " . fmt($numdirs * $numMakefiles)
 512                             . " ($numMakefiles per directory)\n"
 513       . "\ttotal files:   " . fmt($totlfils) . "\n"
 514       . "\tlook-ups: >=   " . fmt($totllups) . "\n"
 515       . "\trules: >=      " . fmt($nrules) . "\n";
 516 print $txt2;
 517
 518 $txt3 = "When the build runs, " . fmt($totlobjs) . " object files & "
 519     . fmt($totlexes) . " executable(s)"
 520     . "\nwill be created, for a total of " . fmt($allfiles) . " files.\n";
 521 print $txt3;
 522
 523 # Using local archives the number of conflicts is about the number of compiles
 524 # which equals the number of archive writes.
 525 #
 526
 527 if (-d $basedir) {
 528     print "Cleaning up from a previous run of this perl script.\n\n";
 529    system ("rm -rf $basedir/*");
 530 }
 531
 532 ###
 533 ### Generate README.txt
 534 ###
 535 $readme = "$basedir/README.txt";
 536 open (README, "> $readme") || die "Cannot open $readme for output.";
 537
 538 $txt = "\nStarted at " . `date` . "\n";
 539 print $txt;
 540 print README $txt
 541            . $vartxt
 542            . $txt1
 543            . $txt2
 544            . $txt3 || die "Cannot write txt, vartxt, txt1 etc to README";
 545
 546 ###
 547 ### Do the heavy lifting
 548 ###
 549
 550 print "Start writing new files at " . `date` . "......\n";
 551
 552 $basedir0 = $basedir;
 553 &mkdirs (0, $basedir, 1);
 554
 555 ###
 556 ### Summarize the results to the README and the console
 557 ###
 558
 559 $txt = "\nFile creation ended at " . `date` . "\n";
 560 print $txt; print README $txt || die "Cannot print txt to README";
 561
 562 close (README);