bin/custom-apt-cruft-check

   1 #!/usr/bin/env ruby
   2
   3 # This script reports which binary/source packages that can be safely
   4 # deleted from one of the main APTs suite in our custom repo. It requires a
   5 # .build-manifest as the source for which packages that are used
   6 # during build and thus cannot be deleted.
   7
   8 begin
   9   require 'debian'
  10 rescue LoadError
  11   raise 'please install the ruby-debian package'
  12 end
  13 require 'open-uri'
  14 require 'optparse'
  15 require 'yaml'
  16
  17 class NoSource < StandardError
  18 end
  19
  20 class AptSources
  21   def initialize(suite)
  22     @apt_sources = []
  23     apt_repo_hostnames = [
  24       'deb.tails.boum.org',
  25       'umjqavufhoix3smyq6az2sx4istmuvsgmz4bq5u5x56rnayejoo6l2qd.onion',
  26     ]
  27     ['main', 'contrib', 'non-free'].each do |repo|
  28       apt_repo_filenames = apt_repo_hostnames.map do |hostname|
  29         "/var/lib/apt/lists/#{hostname}_dists_#{suite}_#{repo}_source_Sources"
  30       end
  31       apt_repo_filename = apt_repo_filenames.find do |filename|
  32         File.exist?(filename)
  33       end
  34       next if apt_repo_filename.nil?
  35
  36       @apt_sources << Debian::Sources.new(apt_repo_filename)
  37     end
  38     return unless @apt_sources.empty?
  39
  40     raise "could not find Tails custom APT repo's sources, " \
  41           "please add this to your APT sources:\n" \
  42           'deb-src [arch=amd64] http://deb.tails.boum.org/ ' \
  43           "#{suite} main contrib non-free"
  44   end
  45
  46   def source_package(package)
  47     matches = []
  48     @apt_sources.each do |repo|
  49       repo.each_package do |dsc|
  50         # The -dbg(sym) packages are not listed, so we look for the
  51         # original package's source instead, which will be the same.
  52         matches << dsc.package if dsc.binary.include?(package.sub(/-dbg(sym)?$/, ''))
  53       end
  54     end
  55     raise NoSource, "found no source package for #{package}" if matches.size.zero?
  56
  57     raise "found multiple source packages for #{package}: #{matches.join(', ')}" \
  58       if matches.uniq.size > 1
  59
  60     matches.first
  61   end
  62 end
  63
  64 Options = Struct.new(:suite, :build_manifest, keyword_init: true)
  65
  66 class Parser
  67   def self.parse(options)
  68     args = Options.new(suite: nil, build_manifest: nil)
  69
  70     opt_parser = OptionParser.new do |opts|
  71       opts.on(
  72         '--suite SUITE',
  73         'Look for cruft in APT suite SUITE'
  74       ) do |suite|
  75         args.suite = suite
  76       end
  77       opts.on(
  78         '--build-manifest MANIFEST',
  79         'Use specified build manifest instead of downloading the latest one'
  80       ) do |build_manifest|
  81         args.build_manifest = build_manifest
  82       end
  83       opts.on('-h', '--help', 'Prints this help') do
  84         puts opts
  85         exit
  86       end
  87     end
  88     opt_parser.parse!(options)
  89
  90     !args.suite.nil? or raise 'Please use --suite SUITE'
  91
  92     args
  93   end
  94 end
  95 options = Parser.parse(ARGV)
  96
  97 allowed_suites = ['stable', 'devel', 'testing']
  98 unless allowed_suites.include?(options.suite)
  99   raise "we only support checking the following' " \
 100         "custom APT suites: #{allowed_suites.join(', ')}"
 101 end
 102
 103 if options.build_manifest.nil?
 104   url = "https://nightly.tails.boum.org/build_Tails_ISO_#{options.suite}/lastSuccessful/archive/latest.build-manifest"
 105   begin
 106     manifest = YAML.safe_load(
 107       URI.parse(url).open.read
 108     )
 109   rescue OpenURI::HTTPError
 110     raise "got HTTP 404 when attempting to fetch: #{url}\n" \
 111           'Please try again in a while -- Jenkins sometimes needs some time ' \
 112           'to create the latest.build-manifest symlink after a build completes'
 113   end
 114 else
 115   manifest = YAML.load_file(options.build_manifest)
 116 end
 117
 118 APT = AptSources.new(options.suite).freeze
 119 all_source_packages = []
 120 used_source_packages = []
 121 binary_cruft_candidates = []
 122 no_source_packages = []
 123
 124 custom_packages = `ssh reprepro@incoming.deb.tails.boum.org \
 125                        reprepro list #{options.suite}`
 126 custom_packages.each_line(chomp: true) do |line|
 127   type, name, version = line.split
 128   if type['source']
 129     all_source_packages << name
 130   else
 131     installed = manifest['packages']['binary'].find { |x| x['package'] == name }
 132     if installed.nil? || version != installed['version']
 133       binary_cruft_candidates << name
 134     else
 135       begin
 136         used_source_packages << APT.source_package(name)
 137       rescue NoSource
 138         no_source_packages << name
 139       end
 140     end
 141   end
 142 end
 143
 144 source_cruft = all_source_packages.uniq - used_source_packages
 145 binary_cruft = []
 146 binary_cruft_candidates.each do |p|
 147   begin
 148     next if used_source_packages.include?(APT.source_package(p))
 149   rescue NoSource
 150     # If we don't have a source for a package, it should be a package
 151     # we forgot to clean up when we removed its sources.
 152   end
 153   binary_cruft << p
 154 end
 155
 156 def puts_list(list)
 157   list.each_with_index { |item, i| puts " #{i + 1}. `#{item}`" }
 158 end
 159
 160 unless binary_cruft.empty?
 161   puts "## Binary packages that are not used\n\n"
 162   puts_list(binary_cruft)
 163   puts
 164   puts "### Clean up command\n\n" \
 165        '    ssh reprepro@incoming.deb.tails.boum.org ' \
 166        "reprepro remove #{options.suite} #{binary_cruft.join(' ')}"
 167   puts
 168 end
 169
 170 unless no_source_packages.empty?
 171   puts "## Binary packages that are used but lack source\n\n"
 172   puts_list(no_source_packages)
 173   puts
 174   puts '**Please investigate!** Most likely you want to just upload the ' \
 175        'missing source packages, or these binary packages are installed ' \
 176        "by mistake and should be considered as cruft and removed with:\n\n" \
 177        '    ssh reprepro@incoming.deb.tails.boum.org ' \
 178        "reprepro remove #{options.suite} #{no_source_packages.join(' ')}"
 179   puts
 180 end
 181
 182 unless source_cruft.empty?
 183   puts "## Source packages that are not used\n\n"
 184   puts_list(source_cruft)
 185   puts
 186   puts "### Clean up command\n\n" \
 187        '    ssh reprepro@incoming.deb.tails.boum.org ' \
 188        "reprepro removesrcs #{options.suite} #{source_cruft.join(' ')}"
 189   puts
 190 end