hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #     http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17
  18 set -e
  19 function usage {
  20   echo "Usage: ${0} [options] [/path/to/some/example.jar:/path/to/another/example/created.jar]"
  21   echo ""
  22   echo "  accepts a single command line argument with a colon separated list of"
  23   echo "  paths to jars to check. Iterates through each such passed jar and checks"
  24   echo "  all the contained paths to make sure they follow the below constructed"
  25   echo "  safe list."
  26   echo ""
  27   echo "    --allow-hadoop     Include stuff from the Apache Hadoop project in the list"
  28   echo "                       of allowed jar contents. default: false"
  29   echo "    --debug            print more info to stderr"
  30   exit 1
  31 }
  32 # if no args specified, show usage
  33 if [ $# -lt 1 ]; then
  34   usage
  35 fi
  36
  37 # Get arguments
  38 declare allow_hadoop
  39 declare debug
  40 while [ $# -gt 0 ]
  41 do
  42   case "$1" in
  43     --allow-hadoop) shift; allow_hadoop="true";;
  44     --debug) shift; debug="true";;
  45     --) shift; break;;
  46     -*) usage ;;
  47     *)  break;;  # terminate while loop
  48   esac
  49 done
  50
  51 if [ -z "${JAVA_HOME}" ]; then
  52   echo "[ERROR] Must have JAVA_HOME defined." 1>&2
  53   exit 1
  54 fi
  55 JAR="${JAVA_HOME}/bin/jar"
  56
  57 # should still have jars to check.
  58 if [ $# -lt 1 ]; then
  59   usage
  60 fi
  61 if [ -n "${debug}" ]; then
  62   echo "[DEBUG] Checking on jars: $*" >&2
  63   echo "jar command is: $(which jar)" >&2
  64   echo "grep command is: $(which grep)" >&2
  65   grep -V >&2 || true
  66 fi
  67
  68 IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
  69
  70 # we have to allow the directories that lead to the hbase dirs
  71 allowed_expr="(^org/$|^org/apache/$|^org/apache/hadoop/$"
  72 # We allow the following things to exist in our client artifacts:
  73 #   * classes in packages that start with org.apache.hadoop.hbase, which by
  74 #     convention should be in a path that looks like org/apache/hadoop/hbase
  75 allowed_expr+="|^org/apache/hadoop/hbase"
  76 #   * classes in packages that start with org.apache.hbase
  77 allowed_expr+="|^org/apache/hbase/"
  78 #   * whatever in the "META-INF" directory
  79 allowed_expr+="|^META-INF/"
  80 #   * the folding tables from jcodings
  81 allowed_expr+="|^tables/"
  82 #   * contents of hbase-webapps
  83 allowed_expr+="|^hbase-webapps/"
  84 #   * HBase's default configuration files, which have the form
  85 #     "_module_-default.xml"
  86 allowed_expr+="|^hbase-default.xml$"
  87 # public suffix list used by httpcomponents
  88 allowed_expr+="|^mozilla/$"
  89 allowed_expr+="|^mozilla/public-suffix-list.txt$"
  90 # Comes from commons-configuration, not sure if relocatable.
  91 allowed_expr+="|^digesterRules.xml$"
  92 allowed_expr+="|^properties.dtd$"
  93 allowed_expr+="|^PropertyList-1.0.dtd$"
  94 # Shaded jetty resources
  95 allowed_expr+="|^about.html$"
  96 allowed_expr+="|^jetty-dir.css$"
  97
  98
  99 if [ -n "${allow_hadoop}" ]; then
 100   #   * classes in packages that start with org.apache.hadoop, which by
 101   #     convention should be in a path that looks like org/apache/hadoop
 102   allowed_expr+="|^org/apache/hadoop/"
 103   #   * Hadoop's default configuration files, which have the form
 104   #     "_module_-default.xml"
 105   allowed_expr+="|^[^-]*-default.xml$"
 106   #   * Hadoop's versioning properties files, which have the form
 107   #     "_module_-version-info.properties"
 108   allowed_expr+="|^[^-]*-version-info.properties$"
 109   #   * Hadoop's application classloader properties file.
 110   allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
 111 else
 112   # We have some classes for integrating with the Hadoop Metrics2 system
 113   # that have to be in a particular package space due to access rules.
 114   allowed_expr+="|^org/apache/hadoop/metrics2"
 115 fi
 116
 117
 118 allowed_expr+=")"
 119 declare -i bad_artifacts=0
 120 declare -a bad_contents
 121 for artifact in "${artifact_list[@]}"; do
 122   bad_contents=($("${JAR}" tf "${artifact}" | grep -v -E "${allowed_expr}" || true))
 123   class_count=$("${JAR}" tf "${artifact}" | grep -c -E '\.class$' || true)
 124   if [ ${#bad_contents[@]} -eq 0 ] && [ "${class_count}" -lt 1 ]; then
 125     bad_contents=("The artifact contains no java class files.")
 126   fi
 127   if [ ${#bad_contents[@]} -gt 0 ]; then
 128     echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
 129     echo "    Please check the following and either correct the build or update"
 130     echo "    the allowed list with reasoning."
 131     echo ""
 132     for bad_line in "${bad_contents[@]}"; do
 133       echo "    ${bad_line}"
 134     done
 135     bad_artifacts=${bad_artifacts}+1
 136   else
 137     echo "[INFO] Artifact looks correct: '$(basename "${artifact}")'"
 138   fi
 139 done
 140
 141 # if there was atleast one bad artifact, exit with failure
 142 if [ "${bad_artifacts}" -gt 0 ]; then
 143   exit 1
 144 fi