nel/tools/build_gamedata/9_upload.py

   1 #!/usr/bin/python
   2 #
   3 # \file 9_upload.py
   4 # \brief Upload data to servers
   5 # \date 2009-02-18 16:19GMT
   6 # \author Jan Boon (Kaetemi)
   7 # Game data build pipeline.
   8 # Upload data to servers
   9 #
  10 # NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
  11 # Copyright (C) 2011  Kaetemi
  12 #
  13 # This program is free software: you can redistribute it and/or modify
  14 # it under the terms of the GNU Affero General Public License as
  15 # published by the Free Software Foundation, either version 3 of the
  16 # License, or (at your option) any later version.
  17 #
  18 # This program is distributed in the hope that it will be useful,
  19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21 # GNU Affero General Public License for more details.
  22 #
  23 # You should have received a copy of the GNU Affero General Public License
  24 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  25 #
  26
  27 import time, sys, os, shutil, subprocess, distutils.dir_util
  28 sys.path.append("configuration")
  29
  30 if os.path.isfile("log.log"):
  31         os.remove("log.log")
  32 log = open("log.log", "w")
  33 from scripts import *
  34 from buildsite_local import *
  35 from tools import *
  36
  37 try:
  38         from upload import *
  39 except ImportError:
  40         # Not documenting this. Because we can.
  41         printLog(log, "ERROR Upload not configured, bye.")
  42         exit()
  43
  44 sys.path.append(WorkspaceDirectory)
  45 from projects import *
  46
  47 # Log error
  48 printLog(log, "")
  49 printLog(log, "-------")
  50 printLog(log, "--- Upload data to servers")
  51 printLog(log, "-------")
  52 printLog(log, time.strftime("%Y-%m-%d %H:%MGMT", time.gmtime(time.time())))
  53 printLog(log, "")
  54
  55 # Find tools
  56 # Not documenting this. Because we can.
  57 Psftp = findFileMultiDir(log, ToolDirectories + WindowsExeDllCfgDirectories, UploadPsftpTool)
  58 printLog(log, "PSFTP " + Psftp)
  59
  60 def downloadVersionTag(server, user, sshkey, dir):
  61         if os.path.isfile("upload.tag"):
  62                 os.remove("upload.tag")
  63         if os.path.isfile("upload.batch"):
  64                 os.remove("upload.batch")
  65         ub = open("upload.batch", "w")
  66         ub.write("cd " + dir + "\n")
  67         ub.write("get upload.tag upload.tag\n")
  68         ub.write("quit\n")
  69         ub.close()
  70         subprocess.call([ Psftp, "-b", "upload.batch", "-i", sshkey, user + "@" + server ])
  71         os.remove("upload.batch")
  72         if os.path.isfile("upload.tag"):
  73                 ft = open("upload.tag")
  74                 result = float(ft.read()) # float, really
  75                 ft.close()
  76                 os.remove("upload.tag")
  77                 printLog(log, "INFO Upload tag is " + str(result))
  78                 return result
  79         else:
  80                 printLog(log, "WARNING Upload tag not found, uploading everything")
  81                 return 0
  82
  83 def isDirectoryNeeded(ft, dir):
  84         files = os.listdir(dir)
  85         for fileName in files:
  86                 if isLegalFileName(fileName):
  87                         fileFull = dir + "/" + fileName
  88                         if os.path.isfile(fileFull):
  89                                 nftf = os.stat(fileFull).st_mtime
  90                                 if nftf > ft:
  91                                         return True
  92                         elif os.path.isdir(fileFull):
  93                                 if isDirectoryNeeded(ft, fileFull):
  94                                         return True
  95                         elif not os.path.isdir(fileFull):
  96                                 printLog(log, "isDirectoryNeeded: file not dir or file?!" + fileFull)
  97         return False
  98
  99 def listDirectoryUpload(ft, ub, udb, dir):
 100         nft = 0
 101         files = os.listdir(dir)
 102         for fileName in files:
 103                 if isLegalFileName(fileName):
 104                         fileFull = dir + "/" + fileName
 105                         if os.path.isfile(fileFull):
 106                                 nftf = os.stat(fileFull).st_mtime
 107                                 if nftf > ft:
 108                                         ub.write("put " + fileFull + " " + fileName + "\n")
 109                                 if nftf > nft:
 110                                         nft = nftf
 111                         elif os.path.isdir(fileFull):
 112                                 if isDirectoryNeeded(ft, fileFull):
 113                                         udb.write("mkdir " + fileName + "\n")
 114                                         ub.write("cd " + fileName + "\n")
 115                                         udb.write("cd " + fileName + "\n")
 116                                         nft2 = listDirectoryUpload(ft, ub, udb, fileFull)
 117                                         if (nft2 > nft):
 118                                                 nft = nft2
 119                                         ub.write("cd ..\n")
 120                                         udb.write("cd ..\n")
 121                         elif not os.path.isdir(fileFull):
 122                                 printLog(log, "listDirectoryUpload: file not dir or file?!" + fileFull)
 123         return nft
 124
 125 def uploadSftp(server, user, sshkey, dir_to, dir_from, addcmd):
 126         ft = downloadVersionTag(server, user, sshkey, dir_to)
 127         if isDirectoryNeeded(ft, dir_from):
 128                 if os.path.isfile("upload_dir.batch"):
 129                         os.remove("upload_dir.batch")
 130                 if os.path.isfile("upload.batch"):
 131                         os.remove("upload.batch")
 132                 udb = open("upload_dir.batch", "w")
 133                 udb.write("cd " + dir_to + "\n")
 134                 ub = open("upload.batch", "w")
 135                 ub.write("cd " + dir_to + "\n")
 136                 for ac in addcmd:
 137                         ub.write(ac + "\n")
 138                 ftn = listDirectoryUpload(ft, ub, udb, dir_from)
 139                 if (ft > ftn):
 140                         ftn = ft
 141                 nft = open("upload.tag", "w")
 142                 nft.write(str(ftn))
 143                 nft.close()
 144                 ub.write("put upload.tag upload.tag\n")
 145                 ub.write("quit\n")
 146                 ub.close()
 147                 udb.write("quit\n")
 148                 udb.close()
 149                 subprocess.call([ Psftp, "-be", "-b", "upload_dir.batch", "-i", sshkey, user + "@" + server ])
 150                 subprocess.call([ Psftp, "-b", "upload.batch", "-i", sshkey, user + "@" + server ])
 151                 os.remove("upload_dir.batch")
 152                 os.remove("upload.batch")
 153                 os.remove("upload.tag")
 154         else:
 155                 printLog(log, "SKIP " + dir_to)
 156
 157 printLog(log, ">>> Upload patch <<<")
 158 for target in UploadPatch:
 159         uploadSftp(target[0], target[1], target[2], target[3], ClientPatchDirectory + "/patch", [ ])
 160
 161 printLog(log, ">>> Upload data_shard <<<")
 162 for target in UploadShard:
 163         uploadSftp(target[0], target[1], target[2], target[3], DataShardDirectory, [ "rm *.packed_sheets", "rm primitive_cache/*.binprim" ])
 164
 165 printLog(log, ">>> Upload data_common <<<")
 166 for target in UploadCommon:
 167         uploadSftp(target[0], target[1], target[2], target[3], DataCommonDirectory, [ ])
 168
 169 printLog(log, ">>> Upload data_leveldesign/leveldesign <<<")
 170 for target in UploadLeveldesign:
 171         uploadSftp(target[0], target[1], target[2], target[3], LeveldesignDirectory, [ ])
 172
 173 printLog(log, ">>> Upload data_leveldesign/primitives <<<")
 174 for target in UploadPrimitives:
 175         uploadSftp(target[0], target[1], target[2], target[3], PrimitivesDirectory, [ ])
 176
 177 log.close()
 178 if os.path.isfile("9_upload.log"):
 179         os.remove("9_upload.log")
 180 shutil.copy("log.log", time.strftime("%Y-%m-%d-%H-%M-GMT", time.gmtime(time.time())) + "_upload.log")
 181 shutil.move("log.log", "9_upload.log")