nel/tools/build_gamedata/processes/rbank/2_build.py

   1 #!/usr/bin/python
   2 #
   3 # \file 2_build.py
   4 # \brief Build rbank
   5 # \date 2009-03-10-22-43-GMT
   6 # \author Jan Boon (Kaetemi)
   7 # Python port of game data build pipeline.
   8 # Build rbank
   9 #
  10 # NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
  11 # Copyright (C) 2009-2014  by authors
  12 #
  13 # This program is free software: you can redistribute it and/or modify
  14 # it under the terms of the GNU Affero General Public License as
  15 # published by the Free Software Foundation, either version 3 of the
  16 # License, or (at your option) any later version.
  17 #
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU Affero General Public License for more details.
  22 #
  23 # You should have received a copy of the GNU Affero General Public License
  24 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  25 #
  26
  27 import time, sys, os, shutil, subprocess, distutils.dir_util
  28 sys.path.append("../../configuration")
  29
  30 if os.path.isfile("log.log"):
  31         os.remove("log.log")
  32 log = open("log.log", "w")
  33 from scripts import *
  34 from buildsite import *
  35 from process import *
  36 from tools import *
  37 from directories import *
  38
  39 printLog(log, "")
  40 printLog(log, "-------")
  41 printLog(log, "--- Build rbank")
  42 printLog(log, "-------")
  43 printLog(log, time.strftime("%Y-%m-%d %H:%MGMT", time.gmtime(time.time())))
  44 printLog(log, "")
  45
  46 # Find tools
  47 BuildIgBoxes = findTool(log, ToolDirectories, BuildIgBoxesTool, ToolSuffix)
  48 ExecTimeout = findTool(log, ToolDirectories, ExecTimeoutTool, ToolSuffix)
  49 BuildRbank = findTool(log, ToolDirectories, BuildRbankTool, ToolSuffix)
  50 GetNeighbors = findTool(log, ToolDirectories, GetNeighborsTool, ToolSuffix)
  51 BuildIndoorRbank = findTool(log, ToolDirectories, BuildIndoorRbankTool, ToolSuffix)
  52 # AiBuildWmap = findTool(log, ToolDirectories, AiBuildWmapTool, ToolSuffix)
  53 printLog(log, "")
  54
  55 # Build rbank bbox
  56 printLog(log, ">>> Build rbank bbox <<<")
  57 tempBbox = ExportBuildDirectory + "/" + RbankBboxBuildDirectory + "/temp.bbox"
  58 rebuiltBbox = False
  59 if BuildIgBoxes == "":
  60         toolLogFail(log, BuildIgBoxesTool, ToolSuffix)
  61 else:
  62         mkPath(log, ExportBuildDirectory + "/" + RbankBboxBuildDirectory)
  63         needUpdateIg = needUpdateMultiDirNoSubdirFile(log, ExportBuildDirectory, IgLookupDirectories, tempBbox)
  64         if needUpdateIg:
  65                 printLog(log, "DETECT UPDATE IG->Bbox")
  66         else:
  67                 printLog(log, "DETECT SKIP IG->Bbox")
  68         needUpdateShape = needUpdateMultiDirNoSubdirFile(log, ExportBuildDirectory, ShapeLookupDirectories, tempBbox)
  69         if needUpdateShape:
  70                 printLog(log, "DETECT UPDATE Shape->Bbox")
  71         else:
  72                 printLog(log, "DETECT SKIP Shape->Bbox")
  73         if needUpdateIg or needUpdateShape:
  74                 rebuiltBbox = True
  75                 printLog(log, "DETECT DECIDE UPDATE")
  76                 cf = open("build_ig_boxes.cfg", "w")
  77                 cf.write("\n")
  78                 cf.write("Pathes = {\n")
  79                 for dir in IgLookupDirectories:
  80                         mkPath(log, ExportBuildDirectory + "/" + dir)
  81                         cf.write("\t\"" + ExportBuildDirectory + "/" + dir + "\", \n")
  82                 for dir in ShapeLookupDirectories:
  83                         mkPath(log, ExportBuildDirectory + "/" + dir)
  84                         cf.write("\t\"" + ExportBuildDirectory + "/" + dir + "\", \n")
  85                 cf.write("};\n")
  86                 cf.write("\n")
  87                 cf.write("IGs = {\n")
  88                 for dir in IgLookupDirectories:
  89                         files = findFiles(log, ExportBuildDirectory + "/" + dir, "", ".ig")
  90                         for file in files:
  91                                 cf.write("\t\"" + os.path.basename(file)[0:-len(".ig")] + "\", \n")
  92                 cf.write("};\n")
  93                 cf.write("\n")
  94                 cf.write("Output = \"" + tempBbox + "\";\n")
  95                 cf.write("\n")
  96                 cf.close()
  97                 subprocess.call([ BuildIgBoxes ])
  98                 os.remove("build_ig_boxes.cfg")
  99         else:
 100                 printLog(log, "DETECT DECIDE SKIP")
 101                 printLog(log, "SKIP *")
 102 printLog(log, "")
 103
 104 printLog(log, ">>> Build rbank build config <<<")
 105 cf = open("build_rbank.cfg", "w")
 106 cf.write("\n")
 107 cf.write("// Rbank settings\n")
 108 cf.write("\n")
 109 cf.write("Verbose = " + str(RBankVerbose) + ";\n")
 110 cf.write("CheckConsistency = " + str(RBankConsistencyCheck) + ";\n")
 111 mkPath(log, ExportBuildDirectory + "/" + ZoneWeldBuildDirectory)
 112 cf.write("ZonePath = \"" + ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/\";\n")
 113 mkPath(log, ExportBuildDirectory + "/" + SmallbankExportDirectory)
 114 cf.write("BanksPath = \"" + ExportBuildDirectory + "/" + SmallbankExportDirectory + "/\";\n")
 115 cf.write("Bank = \"" + ExportBuildDirectory + "/" + SmallbankExportDirectory + "/" + BankTileBankName + ".smallbank\";\n")
 116 cf.write("ZoneExt = \".zonew\";\n")
 117 cf.write("ZoneNHExt = \".zonenhw\";\n")
 118 cf.write("IGBoxes = \"" + tempBbox + "\";\n")
 119 mkPath(log, LeveldesignWorldDirectory)
 120 cf.write("LevelDesignWorldPath = \"" + LeveldesignWorldDirectory + "\";\n")
 121 mkPath(log, ExportBuildDirectory + "/" + IgLandBuildDirectory)
 122 cf.write("IgLandPath = \"" + ExportBuildDirectory + "/" + IgLandBuildDirectory + "\";\n")
 123 mkPath(log, ExportBuildDirectory + "/" + IgOtherBuildDirectory)
 124 cf.write("IgVillagePath = \"" + ExportBuildDirectory + "/" + IgOtherBuildDirectory + "\";\n")
 125 cf.write("\n")
 126 mkPath(log, ExportBuildDirectory + "/" + RbankTessellationBuildDirectory)
 127 cf.write("TessellationPath = \"" + ExportBuildDirectory + "/" + RbankTessellationBuildDirectory + "/\";\n")
 128 cf.write("TessellateLevel = " + str(BuildQuality) + ";\n") # BuildQuality
 129 cf.write("\n")
 130 cf.write("WaterThreshold = 1.0;\n")
 131 cf.write("\n")
 132 cf.write("OutputRootPath = \"" + ExportBuildDirectory + "/\";\n")
 133 mkPath(log, ExportBuildDirectory + "/" + RbankSmoothBuildDirectory)
 134 cf.write("SmoothDirectory = \"" + RbankSmoothBuildDirectory + "/\";\n")
 135 mkPath(log, ExportBuildDirectory + "/" + RbankRawBuildDirectory)
 136 cf.write("RawDirectory = \"" + RbankRawBuildDirectory + "/\";\n")
 137 cf.write("\n")
 138 cf.write("ReduceSurfaces = " + str(RbankReduceSurfaces) + ";\n")
 139 cf.write("SmoothBorders = " + str(RbankSmoothBorders) + ";\n")
 140 cf.write("\n")
 141 cf.write("ComputeElevation = " + str(RbankComputeElevation) + ";\n")
 142 cf.write("ComputeLevels = " + str(RbankComputeLevels) + ";\n")
 143 cf.write("\n")
 144 cf.write("LinkElements = " + str(RbankLinkElements) + ";\n")
 145 cf.write("\n")
 146 cf.write("CutEdges = " + str(RbankCutEdges) + ";\n")
 147 cf.write("\n")
 148 cf.write("UseZoneSquare = " + str(RbankUseZoneSquare) + ";\n")
 149 cf.write("\n")
 150 cf.write("// The whole landscape\n")
 151 cf.write("ZoneUL = \"" + RbankZoneUl + "\";\n")
 152 cf.write("ZoneDR = \"" + RbankZoneDr + "\";\n")
 153 cf.write("\n")
 154 mkPath(log, ExportBuildDirectory + "/" + RbankPreprocBuildDirectory)
 155 cf.write("PreprocessDirectory = \"" + ExportBuildDirectory + "/" + RbankPreprocBuildDirectory + "/\";\n")
 156 cf.write("\n")
 157 cf.write("// The global retriever processing settings\n")
 158 cf.write("GlobalRetriever = \"temp.gr\";\n")
 159 cf.write("RetrieverBank = \"temp.rbank\";\n")
 160 cf.write("\n")
 161 cf.write("GlobalUL = \"" + RbankZoneUl + "\";\n")
 162 cf.write("GlobalDR = \"" + RbankZoneDr + "\";\n")
 163 cf.write("\n")
 164 cf.write("// Which kind of stuff to do\n")
 165 cf.write("TessellateZones = 0;\n")
 166 cf.write("MoulineZones = 0;\n")
 167 cf.write("ProcessRetrievers = 0;\n")
 168 cf.write("ProcessGlobal = 0;\n")
 169 cf.write("\n")
 170 cf.write("Zones = {\n")
 171 mkPath(log, ExportBuildDirectory + "/" + ZoneWeldBuildDirectory)
 172 files = findFiles(log, ExportBuildDirectory + "/" + ZoneWeldBuildDirectory, "", ".zonew")
 173 for file in files:
 174         cf.write("\t\"" + os.path.basename(file) + "\", \n")
 175 cf.write("};\n")
 176 cf.write("\n")
 177 cf.write("Pathes = {\n")
 178 mkPath(log, WorldEditorFilesDirectory);
 179 cf.write("\t\"" + WorldEditorFilesDirectory + "\", \n");
 180 for dir in IgLookupDirectories:
 181         mkPath(log, ExportBuildDirectory + "/" + dir)
 182         cf.write("\t\"" + ExportBuildDirectory + "/" + dir + "\", \n")
 183 for dir in ShapeLookupDirectories:
 184         mkPath(log, ExportBuildDirectory + "/" + dir)
 185         cf.write("\t\"" + ExportBuildDirectory + "/" + dir + "\", \n")
 186 cf.write("};\n")
 187 cf.write("\n")
 188 cf.close()
 189 printLog(log, "")
 190
 191 printLog(log, ">>> Build rbank check prims <<<")
 192 if BuildRbank == "":
 193         toolLogFail(log, BuildRbankTool, ToolSuffix)
 194 elif ExecTimeout == "":
 195         toolLogFail(log, ExecTimeoutTool, ToolSuffix)
 196 else:
 197         subprocess.call([ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-C", "-p", "-g" ])
 198 printLog(log, "")
 199
 200 printLog(log, ">>> Build rbank process all passes <<<")
 201 if BuildRbank == "":
 202         toolLogFail(log, BuildRbankTool, ToolSuffix)
 203 if GetNeighbors == "":
 204         toolLogFail(log, GetNeighborsTool, ToolSuffix)
 205 elif ExecTimeout == "":
 206         toolLogFail(log, ExecTimeoutTool, ToolSuffix)
 207 else:
 208         zonefiles = findFiles(log, ExportBuildDirectory + "/" + ZoneWeldBuildDirectory, "", ".zonew")
 209         zonesToBuild = []
 210         for zonefile in zonefiles:
 211                 zone = os.path.basename(zonefile)[0:-len(".zonew")]
 212                 lr1 = ExportBuildDirectory + "/" + RbankSmoothBuildDirectory + "/" + zone + ".lr"
 213                 nearzones = subprocess.Popen([ GetNeighbors, zone ], stdout = subprocess.PIPE).communicate()[0].strip().split(" ")
 214                 printLog(log, "ZONE " + zone + ": " + str(nearzones))
 215                 zoneToBuild = 0
 216                 for nearzone in nearzones:
 217                         sourcePath = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + nearzone + ".zonew"
 218                         if (os.path.isfile(sourcePath)):
 219                                 if (rebuiltBbox or needUpdate(log, sourcePath, lr1)):
 220                                         zoneToBuild = 1
 221                                         zonesToBuild.append(os.path.basename(zonefile))
 222                 sourcePath = ExportBuildDirectory + "/" + ZoneWeldBuildDirectory + "/" + zone + ".zonew"
 223                 if zoneToBuild:
 224                         printLog(log, sourcePath + " -> " + lr1)
 225                         # subprocess.call([ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-c", "-P", "-g", os.path.basename(zonefile) ])
 226                 else:
 227                         printLog(log, "SKIP " + lr1)
 228         while len(zonesToBuild) > 0:
 229                 processCommand = [ ExecTimeout, str(RbankBuildTesselTimeout), BuildRbank, "-c", "-P", "-g" ]
 230                 processCommand.extend(zonesToBuild[:min(len(zonesToBuild), 64)])
 231                 if len(zonesToBuild) > 64:
 232                         zonesToBuild = zonesToBuild[64:]
 233                 else:
 234                         zonesToBuild = []
 235                 print processCommand
 236                 callParallelProcess(processCommand)
 237         flushParallelProcesses()
 238 printLog(log, "")
 239
 240 printLog(log, ">>> Detect modifications to rebuild lr <<<")
 241 needUpdateCmbLr = needUpdateDirByTagLog(log, ExportBuildDirectory + "/" + RBankCmbExportDirectory, ".cmb", ExportBuildDirectory + "/" + RbankRetrieversBuildDirectory, ".lr")
 242 if needUpdateCmbLr:
 243         printLog(log, "DETECT UPDATE Cmb->Lr")
 244 else:
 245         printLog(log, "DETECT SKIP Cmb->Lr")
 246 needUpdateCmbRbank = needUpdateDirNoSubdirFile(log, ExportBuildDirectory + "/" + RBankCmbExportDirectory, ExportBuildDirectory + "/" + RbankOutputBuildDirectory + "/" + RbankRbankName + ".rbank")
 247 if needUpdateCmbRbank:
 248         printLog(log, "DETECT UPDATE Cmb->Rbank")
 249 else:
 250         printLog(log, "DETECT SKIP Cmb->Rbank")
 251 needUpdateLrRbank = needUpdateDirNoSubdirFile(log, ExportBuildDirectory + "/" + RbankSmoothBuildDirectory, ExportBuildDirectory + "/" + RbankOutputBuildDirectory + "/" + RbankRbankName + ".rbank")
 252 if needUpdateLrRbank:
 253         printLog(log, "DETECT UPDATE Lr->Rbank")
 254 else:
 255         printLog(log, "DETECT SKIP Lr->Rbank")
 256 needUpdateBboxRbank = needUpdate(log, tempBbox, ExportBuildDirectory + "/" + RbankOutputBuildDirectory + "/" + RbankRbankName + ".rbank")
 257 if needUpdateBboxRbank:
 258         printLog(log, "DETECT UPDATE Lr->Rbank")
 259 else:
 260         printLog(log, "DETECT SKIP Lr->Rbank")
 261
 262 if rebuiltBbox or needUpdateCmbLr or needUpdateCmbRbank or needUpdateLrRbank or needUpdateBboxRbank:
 263         printLog(log, "DETECT DECIDE UPDATE")
 264         printLog(log, ">>> Build rbank process global <<<") # This generates temp lr files. TODO: Check if the LR changed?
 265         if BuildRbank == "":
 266                 toolLogFail(log, BuildRbankTool, ToolSuffix)
 267         elif ExecTimeout == "":
 268                 toolLogFail(log, ExecTimeoutTool, ToolSuffix)
 269         else:
 270                 subprocess.call([ ExecTimeout, str(RbankBuildProcglobalTimeout), BuildRbank, "-c", "-P", "-G" ])
 271         printLog(log, "")
 272         os.remove("build_rbank.cfg")
 273
 274         printLog(log, ">>> Build rbank indoor <<<") # This generates the retrievers for the ig that have the cmb export
 275         if BuildIndoorRbank == "":
 276                 toolLogFail(log, BuildIndoorRbankTool, ToolSuffix)
 277         elif ExecTimeout == "":
 278                 toolLogFail(log, ExecTimeoutTool, ToolSuffix)
 279         else:
 280                 retrieversDir = ExportBuildDirectory + "/" + RbankRetrieversBuildDirectory
 281                 mkPath(log, retrieversDir)
 282                 removeFilesRecursiveExt(log, retrieversDir, ".rbank")
 283                 removeFilesRecursiveExt(log, retrieversDir, ".gr")
 284                 removeFilesRecursiveExt(log, retrieversDir, ".lr")
 285                 cf = open("build_indoor_rbank.cfg", "w")
 286                 cf.write("\n")
 287                 mkPath(log, ExportBuildDirectory + "/" + RBankCmbExportDirectory)
 288                 cf.write("MeshPath = \"" + ExportBuildDirectory + "/" + RBankCmbExportDirectory + "/\";\n")
 289                 # cf.write("Meshes = { };\n")
 290                 cf.write("Meshes = \n")
 291                 cf.write("{\n")
 292                 meshFiles = findFilesNoSubdir(log, ExportBuildDirectory + "/" + RBankCmbExportDirectory, ".cmb")
 293                 lenCmbExt = len(".cmb")
 294                 for file in meshFiles:
 295                         cf.write("\t\"" + file[0:-lenCmbExt] + "\", \n")
 296                 cf.write("};\n")
 297                 cf.write("OutputPath = \"" + retrieversDir + "/\";\n")
 298                 # mkPath(log, ExportBuildDirectory + "/" + RbankOutputBuildDirectory)
 299                 # cf.write("OutputPath = \"" + ExportBuildDirectory + "/" + RbankOutputBuildDirectory + "/\";\n")
 300                 cf.write("OutputPrefix = \"unused\";\n")
 301                 cf.write("Merge = 1;\n")
 302                 mkPath(log, ExportBuildDirectory + "/" + RbankSmoothBuildDirectory)
 303                 cf.write("MergePath = \"" + ExportBuildDirectory + "/" + RbankSmoothBuildDirectory + "/\";\n")
 304                 cf.write("MergeInputPrefix = \"temp\";\n")
 305                 cf.write("MergeOutputPrefix = \"tempMerged\";\n")
 306                 # cf.write("MergeOutputPrefix = \"" + RbankRbankName + "\";\n")
 307                 cf.write("AddToRetriever = 1;\n")
 308                 cf.write("\n")
 309                 cf.close()
 310                 subprocess.call([ ExecTimeout, str(RbankBuildIndoorTimeout), BuildIndoorRbank ])
 311                 os.remove("build_indoor_rbank.cfg")
 312         printLog(log, "")
 313
 314         retrieversDir = ExportBuildDirectory + "/" + RbankRetrieversBuildDirectory
 315         mkPath(log, retrieversDir)
 316         outputDir = ExportBuildDirectory + "/" + RbankOutputBuildDirectory
 317         mkPath(log, outputDir)
 318         printLog(log, ">>> Move gr, rbank and lr <<<") # This simply renames everything
 319         if needUpdateDirNoSubdir(log, retrieversDir, outputDir):
 320                 removeFilesRecursiveExt(log, outputDir, ".rbank")
 321                 removeFilesRecursiveExt(log, outputDir, ".gr")
 322                 removeFilesRecursiveExt(log, outputDir, ".lr")
 323                 copyFilesRenamePrefixExt(log, retrieversDir, outputDir, "tempMerged", RbankRbankName, ".rbank")
 324                 copyFilesRenamePrefixExt(log, retrieversDir, outputDir, "tempMerged", RbankRbankName, ".gr")
 325                 copyFilesRenamePrefixExt(log, retrieversDir, outputDir, "tempMerged_", RbankRbankName + "_", ".lr")
 326         else:
 327                 printLog(log, "SKIP *")
 328 else:
 329         printLog(log, "DETECT DECIDE SKIP")
 330         printLog(log, "SKIP *")
 331
 332 # Remove pacs.packed_prims when done
 333 if os.path.isfile("pacs.packed_prims"):
 334         os.remove("pacs.packed_prims")
 335
 336 log.close()
 337
 338
 339 # end of file