bin/custom-apt-cruft-check

   1 #!/usr/bin/env ruby
   2
   3 # This script reports which binary/source packages that can be safely
   4 # deleted from one of the main APTs suite in our custom repo. It requires a
   5 # .build-manifest as the source for which packages that are used
   6 # during build and thus cannot be deleted.
   7
   8 begin
   9   require 'debian'
  10 rescue LoadError
  11   raise 'please install the ruby-debian package'
  12 end
  13 require 'open-uri'
  14 require 'optparse'
  15 require 'yaml'
  16
  17 class NoSource < StandardError
  18 end
  19
  20 def source_package(package)
  21   matches = []
  22   APT_SOURCES.each_package do |dsc|
  23     # The -dbg(sym) packages are not listed, so we look for the
  24     # original package's source instead, which will be the same.
  25     matches << dsc if dsc.binary.include?(package.sub(/-dbg(sym)?$/, ''))
  26   end
  27   raise NoSource, "found no source package for #{package}" if matches.size.zero?
  28
  29   raise "found multiple source packages for #{package}" if matches.size > 1
  30
  31   matches.first.package
  32 end
  33
  34 def binary_packages(package)
  35   APT_SOURCES[package].binary
  36 end
  37
  38 Options = Struct.new(:suite, :build_manifest, keyword_init: true)
  39
  40 class Parser
  41   def self.parse(options)
  42     args = Options.new(suite: nil, build_manifest: nil)
  43
  44     opt_parser = OptionParser.new do |opts|
  45       opts.on(
  46         '--suite SUITE',
  47         'Look for cruft in APT suite SUITE'
  48       ) do |suite|
  49         args.suite = suite
  50       end
  51       opts.on(
  52         '--build-manifest MANIFEST',
  53         'Use specified build manifest instead of downloading the latest one'
  54       ) do |build_manifest|
  55         args.build_manifest = build_manifest
  56       end
  57       opts.on('-h', '--help', 'Prints this help') do
  58         puts opts
  59         exit
  60       end
  61     end
  62     opt_parser.parse!(options)
  63
  64     !args.suite.nil? or raise 'Please use --suite SUITE'
  65
  66     args
  67   end
  68 end
  69 options = Parser.parse(ARGV)
  70
  71 allowed_suites = ['stable', 'devel']
  72 unless allowed_suites.include?(options.suite)
  73   raise "we only support checking the following' " \
  74         "custom APT suites: #{allowed_suites.join(', ')}"
  75 end
  76 begin
  77   apt_repo_hostnames = [
  78     'deb.tails.boum.org',
  79     'umjqavufhoix3smyq6az2sx4istmuvsgmz4bq5u5x56rnayejoo6l2qd.onion',
  80   ]
  81   apt_repo_filenames = apt_repo_hostnames.map do |hostname|
  82     "/var/lib/apt/lists/#{hostname}_dists_#{options.suite}_main_source_Sources"
  83   end
  84   apt_repo_filename = apt_repo_filenames.find do |filename|
  85     File.exist?(filename)
  86   end
  87   APT_SOURCES = Debian::Sources.new(apt_repo_filename).freeze
  88 rescue
  89   raise "could not find Tails custom APT repo's sources, " \
  90         "please add this to your APT sources:\n" \
  91         "deb-src [arch=amd64] http://deb.tails.boum.org/ #{options.suite} main"
  92 end
  93
  94 if options.build_manifest.nil?
  95   url = "https://nightly.tails.boum.org/build_Tails_ISO_#{options.suite}/lastSuccessful/archive/latest.build-manifest"
  96   begin
  97     manifest = YAML.safe_load(
  98       URI.open(url).read
  99     )
 100   rescue OpenURI::HTTPError
 101     raise "got HTTP 404 when attempting to fetch: #{url}\n" \
 102           'Please try again in a while -- Jenkins sometimes needs some time ' \
 103           'to create the latest.build-manifest symlink after a build completes'
 104   end
 105 else
 106   manifest = YAML.load_file(options.build_manifest)
 107 end
 108
 109 all_source_packages = []
 110 used_source_packages = []
 111 binary_cruft_candidates = []
 112
 113 custom_packages = `ssh reprepro@incoming.deb.tails.boum.org reprepro list #{options.suite}`
 114 custom_packages.each_line(chomp: true) do |line|
 115   type, name, version = line.split
 116   if type['source']
 117     all_source_packages << name
 118   else
 119     installed = manifest['packages']['binary'].find { |x| x['package'] == name }
 120     if installed.nil? || version != installed['version']
 121       binary_cruft_candidates << name
 122     else
 123       used_source_packages << source_package(name)
 124     end
 125   end
 126 end
 127
 128 source_cruft = all_source_packages.uniq - used_source_packages
 129 binary_cruft = []
 130 binary_cruft_candidates.each do |p|
 131   begin
 132     next if used_source_packages.include?(source_package(p))
 133   rescue NoSource
 134     # If we don't have a source for a package, it should be a package
 135     # we forgot to clean up when we removed its sources.
 136   end
 137   binary_cruft << p
 138 end
 139
 140 unless binary_cruft.empty?
 141   puts 'Binary packages that are not used:'
 142   binary_cruft.each { |p| puts " - #{p}" }
 143   puts
 144   puts "  Clean up command:\n" \
 145        '  ssh reprepro@incoming.deb.tails.boum.org ' \
 146        "reprepro remove #{options.suite} #{binary_cruft.join(' ')}"
 147   puts
 148 end
 149
 150 unless source_cruft.empty?
 151   puts 'Source packages that are not used:'
 152   source_cruft.each { |p| puts " - #{p}" }
 153   puts
 154   puts "  Clean up command:\n" \
 155        '  ssh reprepro@incoming.deb.tails.boum.org ' \
 156        "reprepro removesrcs #{options.suite} #{source_cruft.join(' ')}"
 157 end