kBuild/footer-pass2-fetches.kmk

   1 # $Id$
   2 ## @file
   3 # kBuild - Footer - Target lists - Pass 2 - Fetches.
   4 #
   5
   6 #
   7 # Copyright (c) 2004-2017 knut st. osmundsen <bird-kBuild-spam-xviiv@anduin.net>
   8 #
   9 # This file is part of kBuild.
  10 #
  11 # kBuild is free software; you can redistribute it and/or modify
  12 # it under the terms of the GNU General Public License as published by
  13 # the Free Software Foundation; either version source of the License, or
  14 # (at your option) any later version.
  15 #
  16 # kBuild is distributed in the hope that it will be useful,
  17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 # GNU General Public License for more details.
  20 #
  21 # You should have received a copy of the GNU General Public License
  22 # along with kBuild; if not, write to the Free Software
  23 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24 #
  25 #
  26 # As a special exception you are granted permission to include this file, via
  27 # the kmk include directive, as you wish without this in itself causing the
  28 # resulting makefile, program or whatever to be covered by the GPL license.
  29 # This exception does not however invalidate any other reasons why the makefile,
  30 # program, whatever should not be covered the GPL.
  31 #
  32 #
  33
  34
  35 ## @page pg_fetches     Fetching Tools, Sources and Similar.
  36 #
  37 # The targets listed in the the FETCHES target list have the following attributes:
  38 #       NAME
  39 #       SOURCES
  40 #       INST
  41 #       FETCHTOOL
  42 #       FETCHFLAGS
  43 #       FETCHDIR
  44 #       UNPACKTOOL
  45 #       UNPACKFLAGS
  46 #
  47 # As usual the target name is an alias for 'creating' the target. Other
  48 # aliases are:
  49 #       pass_fetches
  50 #       fetch
  51 #       unfetch
  52 #       download
  53 #       unpack
  54 #
  55 # @remark
  56 #
  57 #   This is a little bit complex because we must guarantee that if a source file
  58 #   changes only sligtly we must refetch it and to a proper unpacking of it. It
  59 #   is also a desire that fetched archives and unpacked files can be deleted to
  60 #   save space.
  61 #
  62 #   Thus, we must be able to cleanup what we've unpacked should any of the
  63 #   sources be removed. We do this by maintaining a file listing the files
  64 #   and directories that was unpacked. This operation is named 'unfetch'.
  65 #
  66 #   We make use of the SIZE and MD5 attributes for each of the sources to
  67 #   create a digest that is stored in the primary target file. Subsequent
  68 #   runswill compare their digest with it to decide if a refetch is required.
  69 #   When a refetch is found necessary, an 'unfetch' is performed first to
  70 #       clean out old files and directores. Note even changes in source order
  71 #   will cause a refetch due to the way the digest is constructed and
  72 #   evaluated.
  73 #
  74 #   By not depending directly on the archives (nor on any unpacked files)
  75 #       but on a goal made up from the archive name, size and md5, we allow
  76 #   the user to delete the archives. Naturally, this means we'll have to
  77 #   check and fetch missing archives before attempting to unpack them.
  78 #
  79 # @remark
  80 #
  81 #       This feature will *NOT* work correctly with vanilla GNU make becuase
  82 #   it makes use of includedep to avoid too many unnecessary files.
  83 #
  84 # @todo
  85 #   0. Move the fetches out into a unit.
  86 #   1. Download corruption / continuation.
  87 #   2. It's quite possible that there is one too many indirect dependency now...
  88 #
  89
  90 ## generates the fetch rule
  91 define def_fetch_src_fetch_rule
  92 # Indirect goal for downloading something.
  93 .PRECIOUS: $(out)
  94 $(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE)  : \
  95                 | $($(target)_$(srcname)_FETCH_2_DEPORD)
  96         %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
  97         @## @todo do fancy stuff like download continuation.
  98         $$(QUIET)$$(RM) -f -- $(out)
  99         $(cmds)
 100         $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))
 101
 102
 103
 104 # Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
 105 $(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
 106         %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
 107         $$(QUIET)$$(RM) -f -- $$@
 108         @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
 109         @## @todo do fancy stuff like download continuation.
 110         $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
 111                   || (   $$(RM_EXT) -f $(out) \
 112                       && $$(MAKE)      $(out) -f $(MAKEFILE) --no-print-directory )
 113         $$(QUIET2)$$(APPEND) $$@
 114
 115 _TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)
 116
 117 # Just a little precaution.
 118 .NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)
 119
 120 endef # def_fetch_src_fetch_rule
 121 $(eval-opt-var def_fetch_src_fetch_rule)
 122
 123 ## generates the unpack rule
 124 define def_fetch_src_unpack_rule
 125 # This is the unpack rule. it has an order-only dependency on the download check.
 126 $(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
 127                 $($(target)_$(srcname)_UNPACK_2_DEPEND) \
 128                 | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
 129                 $(archive).checked_$(md5)_$(size) \
 130                 $(dir $(out))
 131         %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
 132         $$(QUIET)$$(RM) -f -- $(out)
 133         $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
 134         @# if the source archive doesn't exist fetch it (may have been deleted to save space).
 135         $$(QUIET)test -f $(archive) \
 136                  || (   $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
 137                      && $$(MAKE)      $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
 138         $(cmds)
 139         $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
 140         $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))
 141
 142 $(eval _TARGET_$(target)_UNPACKED += $(out))
 143 _TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)
 144
 145 .NOTPARALLEL: $(out)
 146
 147 endef # def_fetch_src_unpack_rule
 148 $(eval-opt-var def_fetch_src_unpack_rule)
 149
 150 ## Processes a fetch source
 151 #
 152 define def_fetch_src
 153 #$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')
 154
 155 # common
 156 local srcname := $(notdir $(source))
 157 local inst := $(firstword \
 158         $($(target)_$(source)_INST)\
 159         $($(target)_$(srcname)_INST)\
 160         $($(source)_INST)\
 161         $($(srcname)_INST)\
 162         $($(target)_INST)\
 163 )
 164 ifneq ($(patsubst %/,ok,$(inst)),ok)
 165 $(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
 166 endif
 167 ## @todo Install-revamp: FIXME
 168 INSTARGET_$(target)_$(srcname) := $(inst)
 169 local fetchdir := $(firstword \
 170         $($(target)_$(source)_FETCHDIR)\
 171         $($(target)_$(srcname)_FETCHDIR)\
 172         $($(source)_FETCHDIR)\
 173         $($(srcname)_FETCHDIR)\
 174         $($(target)_FETCHDIR)\
 175         $(FETCHDIR)\
 176         $(PATH_TARGET)\
 177 )
 178 local deps := \
 179         $($(target)_$(source)_DEPS)\
 180         $($(target)_$(srcname)_DEPS)\
 181         $($(source)_DEPS)\
 182         $($(srcname)_DEPS)\
 183         $($(target)_DEPS)
 184 local orderdeps := \
 185         $($(target)_$(source)_ORDERDEPS)\
 186         $($(target)_$(srcname)_ORDERDEPS)\
 187         $($(source)_ORDERDEPS)\
 188         $($(srcname)_ORDERDEPS)\
 189         $($(target)_ORDERDEPS)
 190 local md5 := $(firstword \
 191         $($(target)_$(source)_MD5)\
 192         $($(target)_$(srcname)_MD5)\
 193         $($(source)_MD5)\
 194         $($(srcname)_MD5)\
 195         $($(target)_MD5)\
 196 )
 197 local size := $(firstword \
 198         $($(target)_$(source)_SIZE)\
 199         $($(target)_$(srcname)_SIZE)\
 200         $($(source)_SIZE)\
 201         $($(srcname)_SIZE)\
 202         $($(target)_SIZE)\
 203 )
 204 clean_files += \
 205         $($(target)_$(source)_CLEAN)\
 206         $($(target)_$(srcname)_CLEAN)\
 207         $($(source)_CLEAN)\
 208         $($(srcname)_CLEAN)
 209 local dep := # not legal for fetch and unpack tools
 210
 211
 212 #
 213 # The fetching.
 214 #
 215 local out := $(fetchdir)/$(srcname)
 216 local archive := $(out)
 217 $(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
 218 $(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))
 219
 220 local dirdep := $(call DIRDEP,$(fetchdir))
 221 local tool := $(firstword \
 222         $($(target)_$(source)_FETCHTOOL)\
 223         $($(target)_$(srcname)_FETCHTOOL)\
 224         $($(target)_$(source)_TOOL)\
 225         $($(target)_$(srcname)_TOOL)\
 226         $($(source)_FETCHTOOL)\
 227         $($(srcname)_FETCHTOOL)\
 228         $($(source)_TOOL)\
 229         $($(srcname)_TOOL)\
 230         $($(target)_FETCHTOOL)\
 231         $($(target)_TOOL)\
 232         )
 233 local flags :=\
 234         $(TOOL_$(tool)_FETCHFLAGS)\
 235         $(FETCHFLAGS)\
 236         $($(target)_FETCHFLAGS)\
 237         $($(srcname)_FETCHFLAGS)\
 238         $($(source)_FETCHFLAGS)\
 239         $($(target)_$(srcname)_FETCHFLAGS)\
 240         $($(target)_$(source)_FETCHFLAGS)
 241
 242 #$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))
 243
 244 ifndef TOOL_$(tool)_FETCH_CMDS
 245 $(warning kBuild: tools: \
 246         1 $($(target)_$(source)_FETCHTOOL)\
 247         2 $($(target)_$(srcname)_FETCHTOOL)\
 248         3 $($(target)_$(source)_TOOL)\
 249         4 $($(target)_$(srcname)_TOOL)\
 250         5 $($(source)_FETCHTOOL)\
 251         6 $($(srcname)_FETCHTOOL)\
 252         7 $($(source)_TOOL)\
 253         8 $($(srcname)_TOOL)\
 254         9 $($(target)_FETCHTOOL)\
 255         10 $($(target)_TOOL) )
 256 $(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
 257 endif
 258
 259 # call the tool
 260 local cmds := $(TOOL_$(tool)_FETCH_CMDS)
 261 $(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
 262 $(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
 263 $(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
 264 $(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)
 265
 266 # generate the fetch rule.
 267 $(eval $(def_fetch_src_fetch_rule))
 268
 269
 270 #
 271 # The unpacking / installing.
 272 #
 273 local out := $(inst)_kBuild_$(name)_$(srcname)_unpacked.lst
 274 local dirdep := $(call DIRDEP,$(inst))
 275 local tool := $(firstword \
 276         $($(target)_$(source)_UNPACKTOOL)\
 277         $($(target)_$(srcname)_UNPACKTOOL)\
 278         $($(target)_$(source)_TOOL)\
 279         $($(target)_$(srcname)_TOOL)\
 280         $($(source)_UNPACKTOOL)\
 281         $($(srcname)_UNPACKTOOL)\
 282         $($(source)_TOOL)\
 283         $($(srcname)_TOOL)\
 284         $($(target)_UNPACKTOOL)\
 285         $($(target)_TOOL) \
 286         )
 287 ifeq ($(tool),)
 288 local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
 289 $(evalval def_tools_include)
 290 endif
 291 local flags :=\
 292         $(TOOL_$(tool)_UNPACKFLAGS)\
 293         $(UNPACKFLAGS)\
 294         $($(target)_UNPACKFLAGS)\
 295         $($(srcname)_UNPACKFLAGS)\
 296         $($(source)_UNPACKFLAGS)\
 297         $($(target)_$(srcname)_UNPACKFLAGS)\
 298         $($(target)_$(source)_UNPACKFLAGS)
 299
 300 #$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
 301 ifndef TOOL_$(tool)_UNPACK_CMDS
 302 $(warning kBuild: tools: \
 303         1 $($(target)_$(source)_UNPACKTOOL)\
 304         2 $($(target)_$(srcname)_UNPACKTOOL)\
 305         3 $($(target)_$(source)_TOOL)\
 306         4 $($(target)_$(srcname)_TOOL)\
 307         5 $($(source)_UNPACKTOOL)\
 308         6 $($(srcname)_UNPACKTOOL)\
 309         7 $($(source)_TOOL)\
 310         8 $($(srcname)_TOOL)\
 311         9 $($(target)_UNPACKTOOL)\
 312         10 $($(target)_TOOL) \
 313         11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
 314         )
 315 $(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
 316 endif
 317
 318 # call the tool
 319 local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
 320 $(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
 321 $(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
 322 $(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
 323 $(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)
 324
 325 # generate the fetch rule.
 326 $(eval $(def_fetch_src_unpack_rule))
 327
 328 _DIRS += $(inst) $(fetchdir)
 329
 330 endef # def_fetch_src
 331 $(eval-opt-var def_fetch_src)
 332
 333
 334 ##
 335 # Define the target level rules for a fetch.
 336 # @param        target
 337 # @param        out
 338 # @param        inst
 339 # @param        _TARGET_$(target)_UNPACKED
 340 # @param        _TARGET_$(target)_DIGEST
 341 # @param        bld_trg
 342 # @param        bld_trg_arch
 343 define def_fetch_rules
 344
 345 $(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
 346         %$$(call MSG_FETCH_OK,$(target))
 347         $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
 348         $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
 349         $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
 350         $$(QUIET)$$(MV) -f -- $$@.tmp $$@
 351
 352 $(out)_unfetched:
 353         %$$(call MSG_UNFETCH,$(target))
 354         $$(QUIET)$$(RM) -f -- $$(qaddprefix sh,$(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))
 355         $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
 356                 $$(qaddprefix sh,$(inst),$$(qsortfiles ,$$(qdir ,$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d' -e 's/ /\\ /g'))))
 357         $$(QUIET)$$(RM) -f -- $(out).lst $(out)
 358         $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)
 359
 360 $(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
 361         $$(QUIET)$$(RM) -f -- $$@
 362         %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
 363                 ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
 364         $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
 365 if $(KBUILD_KMK_REVISION) > 2911
 366         $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))"
 367 endif
 368         $$(QUIET)$$(if  $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
 369         $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")
 370
 371 .NOTPARALLEL: $(out).lst $(out)_unfetched $(out)
 372
 373 endef
 374
 375
 376 ##
 377 # Deal with one fetch target.
 378 # @param        target
 379 # @param        bld_trg
 380 # @param        bld_trg_arch
 381 define def_fetch
 382 # common
 383 local name := $(firstword $($(target)_NAME) $(target))
 384 ## @todo Install-revamp: FIXME
 385 INSTARGET_$(target) := $($(target)_INST)
 386 ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
 387 $(error kBuild: Bad or missing INST property for target '$(target)'. \
 388         $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
 389 endif
 390 _TARGET_$(target)_FETCHED :=
 391 _TARGET_$(target)_UNPACKED :=
 392 _TARGET_$(target)_DIGEST :=
 393 local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))
 394
 395 # The 'sources'.
 396 #$ (warning dbg fetch: target=$(target) name=$(name) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
 397 $(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
 398         $(evalval def_fetch_src))
 399
 400 # The target.
 401 local inst := $(INSTARGET_$(target))
 402 local out := $(inst)_kBuild_fetch_$(name)
 403
 404 $(eval includedep $(out))
 405
 406 $(eval $(def_fetch_rules))
 407
 408 # Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
 409 $(target): $(out)
 410 $(target)_unfetch: $(out)_unfetched
 411
 412 _FETCHES += $(out)
 413 _DOWNLOADS += $(_TARGET_$(target)_FETCHED)
 414 _UNPACKS += $(_TARGET_$(target)_UNPACKED)
 415 _UNFETCHES += $(out)_unfetched
 416 _DIRS += $(inst)
 417 _CLEAN_FILES += $(clean_files)
 418
 419 endef
 420 $(eval-opt-var def_fetch)
 421
 422 # Walk the FETCH target lists.
 423 bld_trg := $(KBUILD_TARGET)
 424 bld_trg_arch := $(KBUILD_TARGET_ARCH)
 425 $(foreach target, $(_ALL_FETCHES), \
 426         $(evalvalctx def_fetch))
 427
 428 # some aliases.
 429 download: $(_DOWNLOADS)
 430 unpack: $(_UNPACKS)
 431 fetch: $(_FETCHES)
 432 unfetch: $(_UNFETCHES)
 433
 434 ifdef KBUILD_PROFILE_SELF
 435  $(evalcall def_profile_self, done fetching targets)
 436 endif