added 'systemdir' key to rgdc.rc; we don't need to exclude 'std.' and company uncondi...
[rgdc.git] / rgdc.d
blobe1c9a90913dcac255cc63ed84a2998e62eb71d07
1 /*
2 * Copyright (C) 2008 by Andrei Alexandrescu
3 * Written by Andrei Alexandrescu, www.erdani.org
4 * Based on an idea by Georg Wrede
5 * Featuring improvements suggested by Christopher Wright
6 * Windows port using bug fixes and suggestions by Adam Ruppe
8 * Code uglification: Ketmar // Invisible Vector
10 * Distributed under the Boost Software License, Version 1.0.
11 * (See accompanying file LICENSE_1_0.txt or copy at
12 * http://www.boost.org/LICENSE_1_0.txt)
14 // -fversion=no_pragma_lib to disable non-standard -fwrite-pragma-lib arg
16 import
17 std.c.stdlib,
18 std.algorithm,
19 std.array,
20 std.datetime,
21 std.digest.md,
22 std.exception,
23 std.file,
24 std.getopt,
25 std.parallelism,
26 std.path,
27 std.process,
28 std.range,
29 std.regex,
30 std.stdio,
31 std.string,
32 std.typetuple;
35 private enum objExt = ".o";
36 private enum binExt = "";
37 private enum libExt = ".a";
38 private enum altDirSeparator = "";
40 private bool chatty, buildOnly, dryRun, force;
41 private string exe;
44 private string compilerBinary = "gdc";
45 private string[] compilerExtraFlags = ["-pipe"];
47 private bool optDynamic = false;
48 private bool optShowCommands = false;
51 ////////////////////////////////////////////////////////////////////////////////
52 private void yap(size_t line=__LINE__, T...) (auto ref T stuff) {
53 if (!chatty) return;
54 debug stderr.writeln(line, ": ", stuff);
55 else stderr.writeln(stuff);
59 ////////////////////////////////////////////////////////////////////////////////
60 private string[] includeDirs; // -I
61 private string[] importDirs; // -J
62 private bool[string] systemDirs; // to ignore
65 private string expandString (string s) {
66 auto pos = s.indexOf("$(");
67 if (pos >= 0) {
68 string res;
69 while (pos >= 0) {
70 res ~= s[0..pos];
71 s = s[pos+2..$];
72 auto epos = s.indexOf(')');
73 if (epos < 1) throw new Exception("invalid envvar name");
74 auto v = environment.get(s[0..epos], "");
75 res ~= v;
76 s = s[epos+1..$];
77 pos = s.indexOf("$(");
79 res ~= s;
80 return res;
82 return s;
86 private void readRC (string fname) {
87 auto reader = File(fname);
88 scope(exit) collectException(reader.close()); // don't care for errors
89 foreach (string line; lines(reader)) {
90 if (line.startsWith("include=")) {
91 auto val = expandString(line[8..$].strip);
92 val = buildNormalizedPath(val);
93 if (val.length > 0) {
94 val = "-I"~val;
95 if (!includeDirs.canFind(val)) includeDirs ~= val;
97 } else if (line.startsWith("import=")) {
98 auto val = expandString(line[7..$].strip);
99 val = buildNormalizedPath(val);
100 if (val.length > 0) {
101 val = "-J"~val;
102 if (!importDirs.canFind(val)) importDirs ~= val;
104 } else if (line.startsWith("systemdir=")) {
105 auto val = expandString(line[10..$].strip);
106 val = buildNormalizedPath(val);
107 if (val.length > 0) systemDirs[val] = true;
113 ////////////////////////////////////////////////////////////////////////////////
114 private File lockFile;
116 private void lockWorkPath (string workPath) {
117 string lockFileName = buildPath(workPath, "rgdc.lock");
118 if (!dryRun) lockFile.open(lockFileName, "w");
119 yap("lock ", lockFile.name);
120 if (!dryRun) lockFile.lock();
124 private void unlockWorkPath () {
125 yap("unlock ", lockFile.name);
126 if (!dryRun) {
127 lockFile.unlock();
128 lockFile.close();
133 ////////////////////////////////////////////////////////////////////////////////
134 // Run a program optionally writing the command line first
135 // If "replace" is true and the OS supports it, replace the current process.
136 private int run (string[] args, string output=null, bool replace=false) {
137 import std.conv;
138 yap(replace ? "exec " : "spawn ", args.text);
139 if (optShowCommands) stderr.writeln("run: ", args.join(" ")); //TODO: proper quoting
140 if (dryRun) return 0;
141 if (replace && !output) {
142 import std.c.process;
143 auto argv = args.map!toStringz.chain(null.only).array;
144 return execv(argv[0], argv.ptr);
146 File outputFile;
147 if (output) {
148 outputFile = File(output, "wb");
149 } else {
150 outputFile = stdout;
152 auto process = spawnProcess(args, stdin, outputFile);
153 return process.wait();
157 private int runSilent (string[] args) {
158 import std.conv;
159 yap("spawn ", args.text);
160 if (optShowCommands) stderr.writeln("run: ", args.join(" ")); //TODO: proper quoting
161 if (dryRun) return 0;
162 auto process = spawnProcess(args);
163 return process.wait();
167 private int exec (string[] args) {
168 return run(args, null, true);
172 ////////////////////////////////////////////////////////////////////////////////
173 private @property string myOwnTmpDir () {
174 import core.sys.posix.unistd;
175 auto tmpRoot = format("/tmp/.rgdc-%d", getuid());
176 yap("mkdirRecurse ", tmpRoot);
177 if (!dryRun) mkdirRecurse(tmpRoot);
178 return tmpRoot;
182 private string getWorkPath (in string root, in string[] compilerFlags) {
183 static string workPath;
184 if (workPath) return workPath;
185 enum string[] irrelevantSwitches = ["--help", "-ignore", "-quiet", "-v"];
186 MD5 context;
187 context.start();
188 context.put(getcwd().representation);
189 context.put(root.representation);
190 foreach (flag; compilerFlags) {
191 if (irrelevantSwitches.canFind(flag)) continue;
192 context.put(flag.representation);
194 auto digest = context.finish();
195 string hash = toHexString(digest);
196 const tmpRoot = myOwnTmpDir;
197 workPath = buildPath(tmpRoot, "rgdc-"~baseName(root)~'-'~hash);
198 yap("mkdirRecurse ", workPath);
199 if (!dryRun) mkdirRecurse(workPath);
200 return workPath;
204 ////////////////////////////////////////////////////////////////////////////////
205 // Rebuild the executable fullExe starting from modules in myDeps
206 // passing the compiler flags compilerFlags. Generates one large
207 // object file.
208 private int rebuild (string root, string fullExe,
209 string workDir, string objDir, in string[string] myDeps,
210 string[] compilerFlags, bool addStubMain)
212 // Delete the old executable before we start building.
213 yap("stat ", fullExe);
214 if (!dryRun && exists(fullExe)) {
215 yap("rm ", fullExe);
216 try {
217 remove(fullExe);
218 } catch (FileException e) {
219 // This can occur on Windows if the executable is locked.
220 // Although we can't delete the file, we can still rename it.
221 auto oldExe = "%s.%s-%s.old".format(fullExe, Clock.currTime.stdTime, thisProcessID);
222 yap("mv ", fullExe, " ", oldExe);
223 rename(fullExe, oldExe);
226 auto fullExeTemp = fullExe~".tmp";
228 string[] buildTodo () {
229 string outExe = (fullExeTemp[0] != '/' ? objDir~"/"~fullExeTemp : fullExeTemp);
230 auto todo =
231 compilerFlags~
232 ["-o", outExe]~
233 ["-I"~dirName(root)]~
234 ["-J"~dirName(root)]~
235 includeDirs~
236 importDirs/*~
237 [root]*/;
238 foreach (k, objectFile; myDeps) if (objectFile !is null) todo ~= [k];
239 // Need to add void main(){}?
240 if (addStubMain) {
241 auto stubMain = buildPath(myOwnTmpDir, "stubmain.d");
242 std.file.write(stubMain, "void main(){}");
243 todo ~= [stubMain];
245 return todo;
248 auto todo = buildTodo();
249 auto commandLength = escapeShellCommand(todo).length;
250 string[] eflags2;
251 if (optDynamic) {
252 eflags2 ~= "-fPIC";
253 } else {
254 eflags2 ~= "-static-libphobos";
256 // get list of pragma(lib)s
257 string[] prlibs;
258 version(no_pragma_lib) {} else {
259 string prfname = objDir~"/"~fullExeTemp.baseName~".link";
260 //writeln(prfname);
261 collectException(remove(prfname));
262 immutable r1 = run(
263 [compilerBinary]~
264 compilerExtraFlags~
265 ["-fwrite-pragma-libs="~prfname]~
266 todo);
267 try {
268 auto reader = File(prfname, "r");
269 foreach (string line; lines(reader)) {
270 auto s = line.strip;
271 if (!prlibs.canFind(s)) prlibs ~= s.idup;
273 } catch (Exception) {}
274 if (prlibs.length) {
275 try {
276 auto fo = File(prfname, "w");
277 foreach (s; prlibs) fo.writeln(s);
278 } catch (Exception) {
279 prfname = null;
281 } else {
282 prfname = null;
284 if (prfname.length > 0) {
285 prlibs = ["-Wl,@"~prfname];
286 } else {
287 prlibs.length = 0;
290 if (commandLength+compilerBinary.length+compilerExtraFlags.join(" ").length+eflags2.join(" ").length+
291 prlibs.join(" ").length > 32700) throw new Exception("SHIT!");
292 immutable result = run([compilerBinary]~compilerExtraFlags~eflags2~todo~prlibs);
293 if (result) {
294 // build failed
295 if (exists(fullExeTemp)) remove(fullExeTemp);
296 return result;
298 // clean up the dir containing the object file, just not in dry
299 // run mode because we haven't created any!
300 if (!dryRun) {
301 yap("stat ", objDir);
302 if (objDir.exists && objDir.startsWith(workDir)) {
303 yap("rmdirRecurse ", objDir);
304 // We swallow the exception because of a potential race: two
305 // concurrently-running scripts may attempt to remove this
306 // directory. One will fail.
307 collectException(rmdirRecurse(objDir));
309 yap("mv ", fullExeTemp, " ", fullExe);
310 rename(fullExeTemp, fullExe);
312 return 0;
316 ////////////////////////////////////////////////////////////////////////////////
317 // we don't need std exclusions anymore, we have 'systemdir' key in .rc file for that
318 //private string[] exclusions = ["std", "etc", "core", "tango"]; // packages that are to be excluded
319 private string[] exclusions = []; // packages that are to be excluded
322 // Given module rootModule, returns a mapping of all dependees .d
323 // source filenames to their corresponding .o files sitting in
324 // directory workDir. The mapping is obtained by running dmd -v against
325 // rootModule.
326 private string[string] getDependencies (string rootModule, string workDir, string objDir, string[] compilerFlags) {
327 immutable depsFilename = buildPath(workDir, "rgdc.deps");
329 string[string] readDepsFile (/*string depsFilename, string objDir="."*/) {
331 bool[string] sysDirCache; // cache all modules that are in systemdirs
333 bool inALibrary (string source, string object) {
334 if (object.endsWith(".di") || source == "object" || source == "gcstats") return true;
335 foreach (string exclusion; exclusions) if (source.startsWith(exclusion~'.')) return true;
336 return false;
339 bool isInSystemDir (string path) {
340 path = buildNormalizedPath(path);
341 if (path in sysDirCache) return true;
342 //writeln("isInSystemDir: path=", path, "; dirs=", systemDirs);
343 foreach (auto k; systemDirs.byKey) {
344 if (path.length > k.length && path.startsWith(k) && path[k.length] == '/') {
345 sysDirCache[path.idup] = true;
346 //writeln("in system dir: ", path);
347 return true;
350 return false;
353 string d2obj (string dfile) { return buildPath(objDir, dfile.baseName.chomp(".d")~objExt); }
355 string findLib (string libName) {
356 // This can't be 100% precise without knowing exactly where the linker
357 // will look for libraries (which requires, but is not limited to,
358 // parsing the linker's command line (as specified in dmd.conf/sc.ini).
359 // Go for best-effort instead.
360 string[] dirs = ["."];
361 foreach (envVar; ["LIB", "LIBRARY_PATH", "LD_LIBRARY_PATH"]) dirs ~= environment.get(envVar, "").split(pathSeparator);
362 string[] names = ["lib"~libName~".so", "lib"~libName~".a"];
363 dirs ~= ["/lib", "/usr/lib", "/usr/local/lib"];
364 foreach (dir; dirs) {
365 foreach (name; names) {
366 auto path = buildPath(dir, name);
367 if (path.exists) {
368 if (path.extension == ".so") return "-l"~path.baseName.stripExtension[3..$];
369 return absolutePath(path);
373 return null;
376 yap("read ", depsFilename);
377 auto depsReader = File(depsFilename);
378 scope(exit) collectException(depsReader.close()); // don't care for errors
379 // Fetch all dependencies and append them to myDeps
380 static auto modInfoRE = ctRegex!(r"^(.+?)\s+\(([^)]+)\)");
381 //std.stdio (/opt/gdc/include/d/4.9.1/std/stdio.d) : private : object (/opt/gdc/include/d/4.9.1/object.di)
382 static auto modImportRE = ctRegex!(r"^(.+?)\s+\(([^)]+)\)\s+:\s+string\s+:\s+[^(]+\(([^)]+)\)");
383 //hello (hello.d) : string : zm (/mnt/tigerclaw/D/prj/rgdc/rgdc_native/zm)
384 string[string] result;
385 foreach (string line; lines(depsReader)) {
386 string modName, modPath, filePath;
387 auto modImportMatch = match(line, modImportRE);
388 if (modImportMatch.empty) {
389 auto modInfoMatch = match(line, modInfoRE);
390 if (modInfoMatch.empty) continue;
391 auto modInfoCaps = modInfoMatch.captures;
392 // [1]: module name
393 // [2]: module path
394 modName = modInfoCaps[1];
395 modPath = modInfoCaps[2];
396 } else {
397 auto modImportCaps = modImportMatch.captures;
398 // [1]: module name
399 // [2]: module path
400 // [3]: file path
401 modName = modImportCaps[1];
402 modPath = modImportCaps[2];
403 filePath = modImportCaps[3];
405 if (filePath.length && !isInSystemDir(filePath)) result[filePath] = null;
406 //if (inALibrary(modName, modPath) || isInSystemDir(modPath)) continue;
407 if (isInSystemDir(modPath)) continue;
408 result[modPath] = d2obj(modPath);
410 return result;
413 // Check if the old dependency file is fine
414 if (!force) {
415 yap("stat ", depsFilename);
416 if (exists(depsFilename)) {
417 // See if the deps file is still in good shape
418 auto deps = readDepsFile();
419 auto allDeps = chain(rootModule.only, deps.byKey).array;
420 bool mustRebuildDeps = allDeps.anyNewerThan(timeLastModified(depsFilename));
421 if (!mustRebuildDeps) return deps; // Cool, we're in good shape
422 // Fall through to rebuilding the deps file
426 immutable rootDir = dirName(rootModule);
428 // Collect dependencies
429 auto depsGetter =
430 // "cd "~shellQuote(rootDir)~" && "
431 [compilerBinary]~compilerExtraFlags~compilerFlags~
433 "-c",
434 "-o", "/dev/null",
435 "-fdeps="~depsFilename,
436 rootModule,
437 "-I"~rootDir,
438 "-J"~rootDir
439 ]~includeDirs~importDirs;
441 scope(failure) {
442 // Delete the deps file on failure, we don't want to be fooled
443 // by it next time we try
444 collectException(std.file.remove(depsFilename));
447 immutable depsExitCode = runSilent(depsGetter);
448 if (depsExitCode) {
449 stderr.writefln("Failed: %s", depsGetter);
450 collectException(std.file.remove(depsFilename));
451 exit(depsExitCode);
454 return (dryRun ? null : readDepsFile());
458 ////////////////////////////////////////////////////////////////////////////////
459 // Is any file newer than the given file?
460 private bool anyNewerThan (in string[] files, in string file) {
461 yap("stat ", file);
462 return files.anyNewerThan(file.timeLastModified);
466 // Is any file newer than the given file?
467 private bool anyNewerThan (in string[] files, SysTime t) {
468 // Experimental: running newerThan in separate threads, one per file
469 if (false) {
470 foreach (source; files) if (source.newerThan(t)) return true;
471 return false;
472 } else {
473 bool result;
474 foreach (source; taskPool.parallel(files)) if (!result && source.newerThan(t)) result = true;
475 return result;
481 * If force is true, returns true. Otherwise, if source and target both
482 * exist, returns true iff source's timeLastModified is strictly greater
483 * than target's. Otherwise, returns true.
485 private bool newerThan (string source, string target) {
486 if (force) return true;
487 yap("stat ", target);
488 return source.newerThan(timeLastModified(target, SysTime(0)));
492 private bool newerThan (string source, SysTime target) {
493 if (force) return true;
494 try {
495 yap("stat ", source);
496 return DirEntry(source).timeLastModified > target;
497 } catch (Exception) {
498 // File not there, consider it newer
499 return true;
504 ////////////////////////////////////////////////////////////////////////////////
505 private @property string helpString () {
506 return
507 "rgdc build "~thisVersion~"
508 Usage: rgdc [RDMD AND DMD OPTIONS]... program [PROGRAM OPTIONS]...
509 Builds (with dependents) and runs a D program.
510 Example: rgdc -release myprog --myprogparm 5
512 Any option to be passed to the compiler must occur before the program name. In
513 addition to compiler options, rgdc recognizes the following options:
514 --build-only just build the executable, don't run it
515 --chatty write compiler commands to stdout before executing them
516 --compiler=comp use the specified compiler
517 --dry-run do not compile, just show what commands would be run (implies --chatty)
518 --exclude=package exclude a package from the build (multiple --exclude allowed)
519 --force force a rebuild even if apparently not necessary
520 --help this message
521 --main add a stub main program to the mix (e.g. for unittesting)
522 --shebang rgdc is in a shebang line (put as first argument)
527 ////////////////////////////////////////////////////////////////////////////////
528 private @property string thisVersion () {
529 enum d = __DATE__;
530 enum month = d[0..3],
531 day = d[4] == ' ' ? "0"~d[5] : d[4..6],
532 year = d[7..$];
533 enum monthNum
534 = month == "Jan" ? "01"
535 : month == "Feb" ? "02"
536 : month == "Mar" ? "03"
537 : month == "Apr" ? "04"
538 : month == "May" ? "05"
539 : month == "Jun" ? "06"
540 : month == "Jul" ? "07"
541 : month == "Aug" ? "08"
542 : month == "Sep" ? "09"
543 : month == "Oct" ? "10"
544 : month == "Nov" ? "11"
545 : month == "Dec" ? "12"
546 : "";
547 static assert(month != "", "Unknown month "~month);
548 return year[0]~year[1..$]~monthNum~day;
552 private string which (string path) {
553 yap("which ", path);
554 if (path.canFind(dirSeparator) || altDirSeparator != "" && path.canFind(altDirSeparator)) return path;
555 string[] extensions = [""];
556 foreach (extension; extensions) {
557 foreach (envPath; environment["PATH"].splitter(pathSeparator)) {
558 string absPath = buildPath(envPath, path~extension);
559 yap("stat ", absPath);
560 if (exists(absPath) && isFile(absPath)) return absPath;
563 throw new FileException(path, "File not found in PATH");
567 private size_t indexOfProgram (string[] args) {
568 foreach(i, arg; args[1..$]) {
569 if (!arg.startsWith('-', '@') && !arg.endsWith(".o", ".a")) return i+1;
571 return args.length;
575 int main(string[] args) {
576 // special args
577 for (size_t f = 1; f < args.length; ++f) {
578 if (args[f] == "-dynamic" || args[f] == "--dynamic") {
579 optDynamic = true;
580 args = args[0..f]~args[f+1..$];
581 continue;
583 if (args[f] == "-static" || args[f] == "--static") {
584 optDynamic = false;
585 args = args[0..f]~args[f+1..$];
586 continue;
590 //writeln("Invoked with: ", args);
591 if (args.length > 1 && args[1].startsWith("--shebang ", "--shebang=")) {
592 // multiple options wrapped in one
593 auto a = args[1]["--shebang ".length..$];
594 args = args[0..1]~std.string.split(a)~args[2..$];
597 // Continue parsing the command line; now get rgdc's own arguments
598 auto programPos = indexOfProgram(args);
599 assert(programPos > 0);
600 auto argsBeforeProgram = args[0..programPos];
602 bool bailout; // bailout set by functions called in getopt if program should exit
603 bool addStubMain; // set by --main
604 getopt(argsBeforeProgram,
605 std.getopt.config.caseSensitive,
606 std.getopt.config.passThrough,
607 "build-only", &buildOnly,
608 "chatty", &chatty,
609 "compiler", &compilerBinary,
610 "dry-run", &dryRun,
611 "exclude", &exclusions,
612 "force", &force,
613 "help", { writeln(helpString); bailout = true; },
614 "main", &addStubMain,
615 "show-commands", &optShowCommands
617 if (bailout) return 0;
618 if (dryRun) chatty = true; // dry-run implies chatty
620 // no code on command line => require a source file
621 if (programPos == args.length) {
622 write(helpString);
623 return 1;
626 auto
627 root = args[programPos].chomp(".d")~".d",
628 exeBasename = root.baseName(".d"),
629 exeDirname = root.dirName,
630 programArgs = args[programPos+1..$];
632 assert(argsBeforeProgram.length >= 1);
633 auto compilerFlags = argsBeforeProgram[1..$];
635 bool lib = compilerFlags.canFind("-lib");
636 string outExt = (lib ? libExt : binExt);
638 //collectException(readRC(thisExePath.setExtension(".rc")));
639 collectException(readRC("rgdc.rc"));
641 // --build-only implies the user would like a binary in the program's directory
642 if (buildOnly && !exe) exe = exeDirname~dirSeparator;
644 if (exe && exe.endsWith(dirSeparator)) {
645 // user specified a directory, complete it to a file
646 exe = buildPath(exe, exeBasename)~outExt;
649 // Compute the object directory and ensure it exists
650 immutable workDir = getWorkPath(root, compilerFlags);
651 lockWorkPath(workDir); // will be released by the OS on process exit
652 string objDir = buildPath(workDir, "objs");
653 yap("mkdirRecurse ", objDir);
654 if (!dryRun) mkdirRecurse(objDir);
656 if (lib) {
657 // When building libraries, DMD does not generate object files.
658 // Instead, it uses the -od parameter as the location for the library file.
659 // Thus, override objDir (which is normally a temporary directory)
660 // to be the target output directory.
661 objDir = exe.dirName;
664 // Fetch dependencies
665 const myDeps = getDependencies(root, workDir, objDir, compilerFlags);
667 // Compute executable name, check for freshness, rebuild
669 We need to be careful about using -o. Normally the generated
670 executable is hidden in the unique directory workDir. But if the
671 user forces generation in a specific place by using -od or -of,
672 the time of the binary can't be used to check for freshness
673 because the user may change e.g. the compile option from one run
674 to the next, yet the generated binary's datetime stays the
675 same. In those cases, we'll use a dedicated file called ".built"
676 and placed in workDir. Upon a successful build, ".built" will be
677 touched. See also
678 http://d.puremagic.com/issues/show_bug.cgi?id=4814
680 string buildWitness;
681 SysTime lastBuildTime = SysTime.min;
682 if (exe) {
683 // user-specified exe name
684 buildWitness = buildPath(workDir, ".built");
685 if (!exe.newerThan(buildWitness)) {
686 // Both exe and buildWitness exist, and exe is older than
687 // buildWitness. This is the only situation in which we
688 // may NOT need to recompile.
689 lastBuildTime = buildWitness.timeLastModified(SysTime.min);
691 } else {
692 exe = buildPath(workDir, exeBasename)~outExt;
693 buildWitness = exe;
694 lastBuildTime = buildWitness.timeLastModified(SysTime.min);
697 // Have at it
698 if (chain(root.only, myDeps.byKey).array.anyNewerThan(lastBuildTime)) {
699 immutable result = rebuild(root, exe, workDir, objDir, myDeps, compilerFlags, addStubMain);
700 if (result) return result;
701 // Touch the build witness to track the build time
702 if (buildWitness != exe) {
703 yap("touch ", buildWitness);
704 std.file.write(buildWitness, "");
708 if (buildOnly) {
709 // Pretty much done!
710 return 0;
713 // release lock on workDir before launching the user's program
714 unlockWorkPath();
716 // run
717 return exec(exe~programArgs);