ntpclients/ntpviz.py

   1 #! @PYSHEBANG@
   2 # -*- coding: utf-8 -*-
   3 """\
   4 ntpviz - visualizer for NTP log files
   5
   6 ntpviz [-d LOGDIR]
   7        [-D DLVL | --debug DLVL]
   8        [-c | --clip]
   9        [-e endtime]
  10        [-g]
  11        [-n name]
  12        [-N | --nice]
  13        [-o OUTDIR]
  14        [-p DAYS]
  15        [-s starttime]
  16        [-w SIZE | --width SIZE]
  17        [-T terminal]
  18        [--all-peer-jitters |
  19         --all-peer-offsets |
  20         --local-error |
  21         --local-freq-temps |
  22         --local-gps |
  23         --local-jitter |
  24         --local-offset |
  25         --local-offset-histogram |
  26         --local-offset-multiplot |
  27         --local-stability |
  28         --local-temps |
  29         --peer-jitters=hosts |
  30         --peer-offsets=hosts |
  31        ]
  32        [-V | --version]
  33        [@OPTIONFILE]
  34
  35 See the manual page for details.
  36
  37 Python by ESR, concept and gnuplot code by Dan Drown.
  38 """
  39
  40 # Copyright the NTPsec project contributors
  41 #
  42 # SPDX-License-Identifier: BSD-2-Clause
  43
  44 from __future__ import print_function, division
  45
  46 import atexit
  47 import binascii
  48 import collections
  49 import csv
  50 import datetime
  51 import math
  52 import re
  53 import os
  54 import socket
  55 import sys
  56 import subprocess
  57 import tempfile
  58 try:
  59     import argparse
  60 except ImportError:
  61     sys.stderr.write("""
  62 ntpviz: ERROR:  can't find the Python argparse module
  63         If your Python version is < 2.7, then manual installation is needed:
  64         # pip install argparse
  65 """)
  66     sys.exit(1)
  67
  68 if sys.version_info[0] == 2:
  69     import codecs
  70
  71     # force UTF-8 strings, otherwise some systems crash on micro.
  72     reload(sys)           # why?
  73     sys.setdefaultencoding('utf8')
  74
  75     def open(file, mode='r', buffering=-1, encoding=None, errors=None):
  76         "Redefine open()"
  77         return(codecs.open(filename=file, mode=mode, encoding=encoding,
  78                errors=errors, buffering=buffering))
  79
  80 # believe it or not, Python has no way to make a simple constant!
  81 MS_PER_S = 1e3          # milliseconds per second
  82 NS_PER_S = 1e9          # nanoseconds per second
  83 US_PER_S = 1e6          # microseconds per second
  84 S_PER_MS = 1.0e-3       # seconds per millisecond
  85 S_PER_NS = 1.0e-9       # seconds per nanosecond
  86 S_PER_US = 1.0e-6       # seconds per microsecond
  87
  88 # table to translate refclock names
  89 refclock_name = {'127.127.20.0': 'NMEA(0)',
  90                  '127.127.20.1': 'NMEA(1)',
  91                  '127.127.20.2': 'NMEA(2)',
  92                  '127.127.20.3': 'NMEA(3)',
  93                  '127.127.22.0': 'PPS(0)',
  94                  '127.127.22.1': 'PPS(1)',
  95                  '127.127.22.2': 'PPS(2)',
  96                  '127.127.22.3': 'PPS(3)',
  97                  '127.127.28.0': 'SHM(0)',
  98                  '127.127.28.1': 'SHM(1)',
  99                  '127.127.28.2': 'SHM(2)',
 100                  '127.127.28.3': 'SHM(3)',
 101                  '127.127.46.0': 'GPS(0)',
 102                  '127.127.46.1': 'GPS(1)',
 103                  '127.127.46.2': 'GPS(2)',
 104                  '127.127.46.3': 'GPS(3)'}
 105
 106
 107 # Gack, python before 3.2 has no defined tzinfo for utc...
 108 # define our own
 109 class UTC(datetime.tzinfo):
 110     """UTC"""
 111
 112     def utcoffset(self, dt):
 113         return datetime.timedelta(0)
 114
 115     def tzname(self, dt):
 116         return "UTC"
 117
 118     def dst(self, dt):
 119         return datetime.timedelta(0)
 120
 121
 122 try:
 123     import ntp.statfiles
 124     import ntp.util
 125 except ImportError as e:
 126     sys.stderr.write("ntpviz: ERROR: can't find Python NTP library.\n%s\n"
 127                      "Check your PYTHONPATH\n" % e)
 128     sys.exit(1)
 129
 130 # check Python version
 131 Python26 = False
 132 if (3 > sys.version_info[0]) and (7 > sys.version_info[1]):
 133     # running under Python version before 2.7
 134     Python26 = True
 135
 136
 137 # overload ArgumentParser
 138 class MyArgumentParser(argparse.ArgumentParser):
 139     "class to parse arguments"
 140
 141     def convert_arg_line_to_args(self, arg_line):
 142         '''Make options file more tolerant'''
 143         # strip out trailing comments
 144         arg_line = re.sub(r'\s+#.*$', '', arg_line)
 145
 146         # ignore blank lines
 147         if not arg_line:
 148             return []
 149         # ignore comment lines
 150         if '#' == arg_line[0]:
 151             return []
 152
 153         return arg_line.split()
 154
 155
 156 def print_profile():
 157     """called by atexit() on normal exit to print profile data"""
 158     pr.disable()
 159     pr.print_stats('tottime')
 160     pr.print_stats('cumtime')
 161
 162
 163 # standard deviation class
 164 # use this until we can guarantee Python 3.4 and the statistics module
 165 # http://stackoverflow.com/questions/15389768/standard-deviation-of-a-list#21505523
 166
 167 # class to calc:
 168 #   Mean, Variance, Standard Deviation, Skewness and Kurtosis
 169
 170 class RunningStats(object):
 171     "Calculate mean, variance, sigma, skewness and kurtosis"
 172
 173     def __init__(self, values):
 174         self.num = len(values)     # number of samples
 175         self.mu = 0.0              # simple arithmetic mean
 176         self.variance = 0.0        # variance
 177         self.sigma = 0.0           # aka standard deviation
 178         self.skewness = 0.0
 179         self.kurtosis = 3.0
 180
 181         if 0 >= self.num:
 182             # no data??
 183             return
 184
 185         self.mu = sum(values) / self.num
 186         self.variance = sum(pow((v-self.mu), 2) for v in values) / self.num
 187         self.sigma = math.sqrt(self.variance)
 188
 189         if math.isnan(self.sigma) or 1e-12 >= abs(self.sigma):
 190             # punt
 191             self.skewness = float('nan')
 192             self.kurtosis = float('nan')
 193             return
 194
 195         m3 = 0
 196         m4 = 0
 197         for val in values:
 198             m3 += pow(val - self.sigma, 3)
 199             m4 += pow(val - self.sigma, 4)
 200
 201         self.skewness = m3 / (self.num * pow(self.sigma, 3))
 202         self.kurtosis = m4 / (self.num * pow(self.sigma, 4))
 203
 204 # end standard deviation class
 205
 206
 207 # class for calced values
 208 class VizStats(ntp.statfiles.NTPStats):
 209     "Class for calculated values"
 210
 211     percs = {}          # dictionary of percentages
 212     title = ''          # title
 213     unit = 's'          # display units: s, ppm, etc.
 214     skip_summary = False
 215     clipped = False
 216     multiplier = 1
 217
 218     # observe RFC 4180, end lines with CRLF
 219     csv_head = ["Name", "Min", "1%", "5%", "50%", "95%", "99%", "Max", "",
 220                 "90% Range", "98% Range", "StdDev", "", "Mean", "Units",
 221                 "Skewness", "Kurtosis"]
 222
 223     table_head = """\
 224 <br>
 225 <table>
 226 <thead>
 227   <tr style="font-weight:bold;text-align:left;">
 228     <td style="width:300px;"></td>
 229     <td colspan=8> Percentiles......</td>
 230     <td colspan=3> Ranges......</td>
 231     <td colspan=3></td>
 232     <td style="text-align:right;">Skew-</td>
 233     <td style="text-align:right;">Kurt-</td>
 234   </tr>
 235   <tr style="font-weight:bold;text-align:right;">
 236     <td style="text-align:left;">Name</td>
 237     <td>Min</td><td>1%</td><td>5%</td><td>50%</td><td>95%</td>
 238     <td>99%</td><td>Max</td> <td style="width:10px;">&nbsp;</td>
 239     <td>90%</td><td>98%</td><td>StdDev</td>
 240     <td style="width:10px;">&nbsp;</td><td>Mean</td><td>Units</td>
 241     <td>ness</td><td>osis</td>
 242   </tr>
 243 </thead>
 244 """
 245     table_tail = """\
 246 </table>
 247 """
 248
 249     def __init__(self, values, title, freq=0, units=''):
 250
 251         values.sort()
 252         self.percs = self.percentiles((100, 99, 95, 50, 5, 1, 0), values)
 253
 254         # find the target for autoranging
 255         if args.clip:
 256             # keep 99% and 1% under 999 in selected units
 257             # clip to 1% and 99%
 258             target = max(self.percs["p99"], -self.percs["p1"])
 259         else:
 260             # keep 99% and 1% under 999 in selected units
 261             # but do not let 100% and 1% go over 5000 in selected units
 262             target = max(self.percs["p99"], -self.percs["p1"],
 263                          self.percs["p100"]/5, -self.percs["p0"]/5)
 264
 265         if units:
 266             # fixed scale
 267             self.multiplier = 1
 268             self.unit = units
 269
 270         elif 1 <= target:
 271             self.multiplier = 1
 272             if freq:
 273                 # go to ppm
 274                 self.unit = "ppm"
 275             else:
 276                 # go to seconds
 277                 self.unit = "s"
 278
 279         elif S_PER_MS <= target:
 280             self.multiplier = MS_PER_S
 281             if freq:
 282                 # go to ppb
 283                 self.unit = "ppb"
 284             else:
 285                 # go to millisec
 286                 self.unit = "ms"
 287
 288         elif S_PER_US <= target:
 289             self.multiplier = US_PER_S
 290             if freq:
 291                 self.unit = "10e-12"
 292             else:
 293                 # go to microsec
 294                 self.unit = "µs"
 295
 296         else:
 297             self.multiplier = NS_PER_S
 298             if freq:
 299                 self.unit = "10e-15"
 300             else:
 301                 # go to nanosec
 302                 self.unit = "ns"
 303
 304         sts = RunningStats(values)
 305         self.percs["mu"] = sts.mu
 306         self.percs["pstd"] = sts.sigma
 307
 308         self.title = title
 309
 310         # calculate ranges
 311         self.percs["r90"] = self.percs["p95"] - self.percs["p5"]
 312         self.percs["r98"] = self.percs["p99"] - self.percs["p1"]
 313
 314         # calculate mean +/- std dev
 315         self.percs["m1sigma"] = self.percs["mu"] - self.percs["pstd"]
 316         self.percs["p1sigma"] = self.percs["mu"] + self.percs["pstd"]
 317
 318         # pretty print the values
 319         self.percs_f = {}
 320         for k, v in self.percs.items():
 321             # range the data
 322             v *= self.multiplier
 323             self.percs[k] = round(v, 4)
 324             if 'ppm' == self.unit and 0.020 > abs(self.percs[k]):
 325                 fmt = ".4f"
 326             else:
 327                 fmt = ".3f"
 328             if not Python26:
 329                 # Python 2.6 does not undertand the comma format option
 330                 fmt = "," + fmt
 331             self.percs_f[k] = format(v, fmt)
 332
 333         # don't scale skewness and kurtosis
 334         self.percs["skew"] = sts.skewness
 335         self.percs["kurt"] = sts.kurtosis
 336         if '°C' == units:
 337             # skip for temperatures.
 338             self.percs_f["skew"] = ''
 339             self.percs_f["kurt"] = ''
 340         else:
 341             self.percs_f["skew"] = format(self.percs["skew"], "6.4g")
 342             self.percs_f["kurt"] = format(self.percs["kurt"], "6.4g")
 343
 344         if args.clip:
 345             self.percs["min_y"] = self.percs["p1"]
 346             self.percs["max_y"] = self.percs["p99"]
 347             self.percs["clipped"] = " (clipped)"
 348         else:
 349             self.percs["min_y"] = self.percs["p0"]
 350             self.percs["max_y"] = self.percs["p100"]
 351             self.percs["clipped"] = ""
 352
 353         self.fmt = gnuplot_fmt(self.percs["min_y"], self.percs["max_y"])
 354
 355         # Python is stupid about nested objects, so add in some other stuff
 356         self.percs_f["fmt"] = self.percs["fmt"] = self.fmt
 357         self.percs_f["multiplier"] = self.percs["multiplier"] = self.multiplier
 358         self.percs_f["title"] = self.percs["title"] = self.title
 359         self.percs_f["unit"] = self.percs["unit"] = self.unit
 360
 361         s = ["%(title)s", "%(p0)s", "%(p1)s", "%(p5)s", "%(p50)s", "%(p95)s",
 362              " %(p99)s", "%(p100)s", "", "%(r90)s", "%(r98)s", "%(pstd)s",
 363              "", "%(mu)s", "%(unit)s", "%(skew)s", "%(kurt)s", ]
 364
 365         # csv is raw, html table is autoranged
 366         self.csv = [x % self.percs for x in s]
 367         self.table = [x % self.percs_f for x in s]
 368         self.table = "</td>\n <td>".join(self.table)
 369
 370         self.table = '''\
 371 <tr>
 372  <td style="text-align:left;">%s</td>
 373 </tr>
 374 ''' % self.table
 375
 376         return
 377
 378
 379 def gnuplot_fmt(min_val, max_val):
 380     "return optimal gnuplot format"
 381     span = max_val - min_val
 382     if 6 <= span:
 383         fmt = '%.0f'
 384     elif 0.6 <= span:
 385         fmt = '%.1f'
 386     elif 0.1 <= span:
 387         fmt = '%.2f'
 388     else:
 389         fmt = '%.3f'
 390     return fmt
 391
 392
 393 # end calc things now
 394
 395 # RMS frequency jitter - Deviation from root-mean-square linear approximation?
 396 # Investigate.
 397
 398 def gnuplot(template, outfile=None):
 399     "Run a specified gnuplot program."
 400
 401     if not template:
 402         # silently ignore empty plots
 403         return ''
 404
 405     if outfile is None:
 406         out = None
 407     else:
 408         if 2 <= args.debug_level:
 409             sys.stderr.write("ntpviz: INFO: sending plot output "
 410                              "to %s\n" % outfile)
 411         out = open(outfile, "w", encoding='utf-8')
 412     ##
 413
 414     # can be 30% faster to write to a tmp file than to pipe to gnuplot
 415     # bonus, we can keep the plot file for debug.
 416     if sys.version_info[0] == 2:
 417         tmp_file = tempfile.NamedTemporaryFile(mode='w',
 418                                                suffix='.plt', delete=False)
 419     else:
 420         tmp_file = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8',
 421                                                suffix='.plt', delete=False)
 422     # note that tmp_file is a file handle, it is not a file object
 423     tmp_file.write(template)
 424     tmp_file.close()
 425
 426     # shell=True is a security hazard, do not use
 427     try:
 428         rcode = subprocess.call(['gnuplot', tmp_file.name], stdout=out)
 429     except OSError as e:
 430         if e.errno == os.errno.ENOENT:
 431             # gnuplot not found
 432             sys.stderr.write("ntpviz: ERROR: gnuplot not found in path\n")
 433         else:
 434             # Something else went wrong while trying to run gnuplot
 435             sys.stderr.write("ntpviz: ERROR: gnuplot failed\n")
 436         raise SystemExit(1)
 437
 438     if 0 != rcode:
 439         sys.stderr.write("ntpviz: WARNING: plot returned %s\n" % rcode)
 440         sys.stderr.write("ntpviz: WARNING: plot file %s\n" % tmp_file.name)
 441     elif 2 <= args.debug_level:
 442         sys.stderr.write("ntpviz: INFO: plot file %s\n" % tmp_file.name)
 443     else:
 444         # remove tmp file
 445         os.remove(tmp_file.name)
 446
 447     return rcode
 448
 449
 450 class NTPViz(ntp.statfiles.NTPStats):
 451     "Class for visualizing statistics from a single server."
 452
 453     # Python takes single quotes here. Since no % substitution
 454     Common = """\
 455 set grid
 456 set autoscale xfixmin
 457 set autoscale xfixmax
 458 set xdata time
 459 set xlabel "Time UTC"
 460 set xtics format "%d %b\\n%H:%MZ"
 461 set timefmt "%s"
 462 set lmargin 10
 463 set rmargin 10
 464 """
 465
 466     def __init__(self, statsdir,
 467                  sitename=None, period=None, starttime=None, endtime=None):
 468         ntp.statfiles.NTPStats.__init__(self, statsdir=statsdir,
 469                                         sitename=sitename,
 470                                         period=period,
 471                                         starttime=starttime,
 472                                         endtime=endtime)
 473
 474     def plot_slice(self, rows, item1, item2=None):
 475         "slice 0,item1, maybe item2, from rows, ready for gnuplot"
 476         # speed up by only sending gnuplot the data it will actually use
 477         # WARNING: this is hot code, only modify if you profile
 478         # since we are looping the data, get the values too
 479         plot_data = ''
 480         last_time = 0
 481         values1 = []
 482         values2 = []
 483         if item2:
 484             for row in rows:
 485                 try:
 486                     val1 = float(row[item1])
 487                     val2 = float(row[item2])
 488                     if 2200000 < row[0] - last_time:
 489                         # more than 2,200 seconds between points
 490                         # data loss, add a break in the plot line
 491                         plot_data += '\n'
 492                     # fields: time, fld1, and fld2
 493                     plot_data += row[1] + ' ' + row[item1] + ' ' \
 494                         + row[item2] + '\n'
 495                     last_time = row[0]
 496                 except IndexError:
 497                     continue
 498                 except ValueError:
 499                     continue
 500                 # both values are good, append them.
 501                 values1.append(val1)
 502                 values2.append(val2)
 503         else:
 504             for row in rows:
 505                 try:
 506                     values1.append(float(row[item1]))
 507                     if 2200000 < row[0] - last_time:
 508                         # more than 2,200 seconds between points
 509                         # data loss, add a break in the plot line
 510                         plot_data += '\n'
 511                     # fields: time, fld
 512                     plot_data += row[1] + ' ' + row[item1] + '\n'
 513                     last_time = row[0]
 514                 except IndexError:
 515                     pass
 516                 except ValueError:
 517                     pass
 518
 519         # I know you want to replace the plot_data string concat with
 520         # or more join()s, do not do it, it is slower
 521         # next you'll want to try %-substitution.  it too is slower
 522         plot_data += "e\n"
 523         if item2:
 524             return (plot_data, values1, values2)
 525
 526         return (plot_data, values1)
 527
 528     def local_offset_gnuplot(self):
 529         "Generate gnuplot code graphing local clock loop statistics"
 530         if not self.loopstats:
 531             sys.stderr.write("ntpviz: WARNING: no loopstats to graph\n")
 532             return ''
 533
 534         # speed up by only sending gnuplot the data it will actually use
 535         # fields: time, time offset, freq offset
 536         (plot_data, values, values_f) = self.plot_slice(self.loopstats, 2, 3)
 537
 538         # compute clock offset
 539         stats = VizStats(values, "Local Clock Time Offset")
 540
 541         # compute frequency offset
 542         stats_f = VizStats(values_f, "Local Clock Frequency Offset", freq=1)
 543
 544         out = stats.percs
 545         out["fmt_f"] = stats_f.percs["fmt"]
 546         out["fmt"] = stats.percs["fmt"]
 547         out["max_y2"] = stats_f.percs["max_y"]
 548         out["min_y2"] = stats_f.percs["min_y"]
 549         out["multiplier_f"] = stats_f.percs["multiplier"]
 550         out["sitename"] = self.sitename
 551         out['size'] = args.img_size
 552         out['terminal'] = args.terminal
 553         out["unit_f"] = stats_f.percs["unit"]
 554
 555         plot_template = NTPViz.Common + """\
 556 set terminal %(terminal)s size %(size)s
 557 set title "%(sitename)s: Local Clock Time/Frequency Offsets%(clipped)s"
 558 set ytics format "%(fmt)s %(unit)s" nomirror textcolor rgb '#0060ad'
 559 set yrange [%(min_y)s:%(max_y)s]
 560 set y2tics format "%(fmt_f)s %(unit_f)s" nomirror textcolor rgb '#dd181f'
 561 set y2range [%(min_y2)s:%(max_y2)s]
 562 set key top right
 563 set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
 564 set style line 2 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
 565 plot \
 566  "-" using 1:($2*%(multiplier)s) title "clock offset %(unit)s" \
 567      with linespoints ls 1, \
 568  "-" using 1:($3*%(multiplier_f)s) title "frequency offset %(unit_f)s" \
 569      with linespoints ls 2 axis x1y2
 570 """ % out
 571
 572         exp = """\
 573 <p>The time and frequency offsets between the ntpd calculated time
 574 and the local system clock.  Showing frequency offset (red, in parts
 575 per million, scale on right) and the time offset (blue, in μs, scale
 576 on left).  Quick changes in time offset will lead to larger frequency
 577 offsets.</p>
 578
 579 <p>These are fields 3 (time) and 4 (frequency) from the loopstats log
 580 file.</p>
 581
 582 """
 583
 584         ret = {'html': VizStats.table_head + stats.table +
 585                stats_f.table + VizStats.table_tail + exp,
 586                'plot': plot_template + plot_data + plot_data,
 587                'stats': [stats, stats_f],
 588                'title': "Local Clock Time/Frequency Offsets"}
 589         return ret
 590
 591     def local_freq_temps_plot(self):
 592         "Generate gnuplot code graphing local frequency and temps"
 593         if not self.loopstats:
 594             sys.stderr.write("ntpviz: WARNING: no loopstats to graph\n")
 595             return ''
 596
 597         tempsmap = self.tempssplit()
 598         tempslist = list(tempsmap.keys())
 599         tempslist.sort()
 600         if not tempsmap or not tempslist:
 601             sys.stderr.write("ntpviz: WARNING: no temps to graph\n")
 602             return ''
 603
 604         # speed up by only sending gnuplot the data it will actually use
 605         # fields: time, freq offset
 606         (plot_data, values_f) = self.plot_slice(self.loopstats, 3)
 607
 608         # compute frequency offset
 609         stats_f = VizStats(values_f, "Local Clock Frequency Offset", freq=1)
 610
 611         stats = [stats_f]
 612         table = ''
 613         plot_data_t = ''
 614         max_temp = -300
 615         min_temp = 1000
 616         for key in tempslist:
 617             # speed up by only sending gnuplot the data it will actually use
 618             # fields: time, temp
 619             (p, v) = self.plot_slice(tempsmap[key], 3)
 620             plot_data_t += p
 621             s = VizStats(v, 'Temp %s' % key, units='°C')
 622             max_temp = max(s.percs["max_y"], max_temp)
 623             min_temp = min(s.percs["min_y"], min_temp)
 624             table += s.table
 625             stats.append(s)
 626
 627         # out = stats.percs
 628         out = {}
 629         if args.clip:
 630             out["clipped"] = " (clipped)"
 631         else:
 632             out["clipped"] = ""
 633         out["fmt_f"] = stats_f.percs["fmt"]
 634         out['fmt'] = gnuplot_fmt(min_temp, max_temp)
 635         out["max_y2"] = stats_f.percs["max_y"]
 636         out["min_y2"] = stats_f.percs["min_y"]
 637         out["multiplier_f"] = stats_f.percs["multiplier"]
 638         out["sitename"] = self.sitename
 639         out['size'] = args.img_size
 640         out['terminal'] = args.terminal
 641         out["unit"] = '°C'
 642         out["unit_f"] = stats_f.percs["unit"]
 643
 644         # let temp autoscale
 645         # set yrange [%(min_y)s:%(max_y)s]
 646         plot_template = NTPViz.Common + """\
 647 set terminal %(terminal)s size %(size)s
 648 set title "%(sitename)s: Local Frequency Offset/Temps%(clipped)s"
 649 set ytics format "%(fmt)s %(unit)s" nomirror textcolor rgb '#0060ad'
 650 set y2tics format "%(fmt_f)s %(unit_f)s" nomirror textcolor rgb '#dd181f'
 651 set y2range [%(min_y2)s:%(max_y2)s]
 652 set key top right
 653 set style line 1 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
 654 plot \
 655  "-" using 1:($2*%(multiplier_f)s) title "frequency offset %(unit_f)s" \
 656      with linespoints ls 1 axis x1y2, \
 657 """ % out
 658
 659         for key in tempslist:
 660             out['key'] = key
 661             plot_template += "'-' using 1:2 title '%(key)s' with line, \\\n" \
 662                              % out
 663
 664         # strip trailing ", \n"
 665         plot_template = plot_template[:-4] + "\n"
 666
 667         exp = """\
 668 <p>The frequency offsets and temperatures.
 669 Showing frequency offset (red, in parts
 670 per million, scale on right) and the temperatures.</p>
 671
 672 <p>These are field 4 (frequency) from the loopstats log
 673 file, and field 3 from the tempstats log file.</p>
 674
 675 """
 676
 677         ret = {'html': VizStats.table_head + stats_f.table +
 678                table + VizStats.table_tail + exp,
 679                'plot': plot_template + plot_data + plot_data_t,
 680                'stats': stats,
 681                'title': "Local Frequency/Temp"}
 682         return ret
 683
 684     def local_temps_gnuplot(self):
 685         "Generate gnuplot code graphing local temperature statistics"
 686         sitename = self.sitename
 687         tempsmap = self.tempssplit()
 688         tempslist = list(tempsmap.keys())
 689         tempslist.sort()
 690
 691         if not tempsmap or not tempslist:
 692             sys.stderr.write("ntpviz: WARNING: no temps to graph\n")
 693             return ''
 694
 695         stats = []
 696         plot_data = ''
 697         max_temp = -300
 698         min_temp = 1000
 699         for key in tempslist:
 700             # speed up by only sending gnuplot the data it will actually use
 701             # fields: time, temp
 702             (p, v) = self.plot_slice(tempsmap[key], 3)
 703             s = VizStats(v, 'Temp %s' % key, units='°C')
 704             max_temp = max(s.percs["max_y"], max_temp)
 705             min_temp = min(s.percs["min_y"], min_temp)
 706             plot_data += p
 707
 708         out = {}
 709         out['fmt'] = gnuplot_fmt(min_temp, max_temp)
 710         out['sitename'] = sitename
 711         out['size'] = args.img_size
 712         out['terminal'] = args.terminal
 713
 714         plot_template = NTPViz.Common + """\
 715 set terminal %(terminal)s size %(size)s
 716 set title "%(sitename)s: Local Temperatures"
 717 set ytics format "%(fmt)s °C" nomirror textcolor rgb '#0060ad'
 718 set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
 719 set key top right
 720 plot \\
 721 """ % out
 722
 723         for key in tempslist:
 724             out['key'] = key
 725             plot_template += "'-' using 1:2 title '%(key)s' with line, \\\n" \
 726                              % out
 727
 728         # strip the trailing ", \n"
 729         plot_template = plot_template[:-4] + "\n"
 730         exp = """\
 731 <p>Local temperatures.  These will be site-specific depending upon what
 732 temperature sensors you collect data from.
 733 Temperature changes affect the local clock crystal frequency and
 734 stability.  The math of how temperature changes frequency is
 735 complex, and also depends on crystal aging.  So there is no easy
 736 way to correct for it in software.  This is the single most important
 737 component of frequency drift.</p>
 738 <p>The Local Temperatures are from field 3 from the tempstats log file.</p>
 739 """
 740
 741         ret = {'html': exp, 'stats': stats}
 742         ret['title'] = "Local Temperatures"
 743         ret['plot'] = plot_template + plot_data
 744         return ret
 745
 746     def local_gps_gnuplot(self):
 747         "Generate gnuplot code graphing local GPS statistics"
 748         sitename = self.sitename
 749         gpsmap = self.gpssplit()
 750         gpslist = list(gpsmap.keys())
 751         gpslist.sort()
 752
 753         if not gpsmap or not gpslist:
 754             if 1 <= args.debug_level:
 755                 sys.stderr.write("ntpviz: INFO: no GPS data to graph\n")
 756             return ''
 757
 758         # build the output dictionary, because Python can not format
 759         # complex objects.
 760         values_nsat = []
 761         values_tdop = []
 762         plot_data = ""
 763         for key in gpslist:
 764             # fields: time, TDOP, nSats
 765             (ps, values_tdop, values_nsat) = self.plot_slice(gpsmap[key], 3, 4)
 766             plot_data += ps
 767
 768         stats = VizStats(values_nsat, "nSats", units='nSat')
 769         stats_tdop = VizStats(values_tdop, "TDOP", units=' ')
 770
 771         out = stats_tdop.percs
 772         out['sitename'] = sitename
 773         out['size'] = args.img_size
 774         if out['min_y'] == out['max_y']:
 775             # some GPS always output the same TDOP
 776             if 0 == out['min_y']:
 777                 # scale 0:1
 778                 out['max_y'] = 1
 779             else:
 780                 # scale +/- 20%
 781                 out['min_y'] = out['max_y'] * 0.8
 782                 out['max_y'] = out['max_y'] * 1.2
 783         elif 2 > out['min_y']:
 784             # scale 0:max_x
 785             out['min_y'] = 0
 786
 787         # recalc fmt
 788         out['fmt'] = gnuplot_fmt(out["min_y"], out["max_y"])
 789         out['terminal'] = args.terminal
 790
 791         plot_template = NTPViz.Common + """\
 792 set terminal %(terminal)s size %(size)s
 793 set title "%(sitename)s: Local GPS%(clipped)s
 794 set ytics format "%(fmt)s TDOP" nomirror textcolor rgb '#0060ad'
 795 set yrange [%(min_y)s:%(max_y)s]
 796 set y2tics format "%%4.1f nSat"  nomirror textcolor rgb '#dd181f'
 797 set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
 798 set style line 2 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
 799 set key top right
 800 plot \\
 801 """ % out
 802
 803         for key in gpslist:
 804             plot_template += """\
 805 '-' using 1:2 title '%s TDOP' with line ls 1, \\
 806 '-' using 1:3 title '%s nSat' with line ls 2 axis x1y2, \\
 807 """ % (key, key)
 808
 809         # strip the trailing ", \\n"
 810         plot_template = plot_template[:-4] + "\n"
 811         exp = """\
 812 <p>Local GPS.  The Time Dilution of Precision (TDOP) is plotted in blue.
 813 The number of visible satellites (nSat) is plotted in red.</p>
 814 <p>TDOP is field 3, and nSats is field 4, from the gpsd log file.  The
 815 gpsd log file is created by the ntploggps program.</p>
 816 <p>TDOP is a dimensionless error factor.  Smaller numbers are better.
 817 TDOP ranges from 1 (ideal), 2 to 5 (good), to greater than 20 (poor).
 818 Some GNSS receivers report TDOP less than one which is theoretically
 819 impossible.</p>
 820 """
 821
 822         ret = {'html': VizStats.table_head + stats.table +
 823                stats_tdop.table + VizStats.table_tail + exp,
 824                'stats': [stats, stats_tdop],
 825                'title': "Local GPS",
 826                'plot': plot_template + plot_data + plot_data}
 827         return ret
 828
 829     def local_error_gnuplot(self):
 830         "Plot the local clock frequency error."
 831         if not self.loopstats:
 832             sys.stderr.write("ntpviz: WARNING: no loopstats to graph\n")
 833             return ''
 834
 835         # grab and sort the values, no need for the timestamp, etc.
 836
 837         # speed up by only sending gnuplot the data it will actually use
 838         # fields: time, freq error
 839         (plot_data, values) = self.plot_slice(self.loopstats, 3)
 840
 841         # compute frequency offset
 842         stats = VizStats(values, "Local Clock Frequency Offset", freq=1,)
 843
 844         # build the output dictionary, because Python can not format
 845         # complex objects.
 846         out = stats.percs
 847         out["fmt"] = stats.percs["fmt"]
 848         out["sitename"] = self.sitename
 849         out['size'] = args.img_size
 850         out['terminal'] = args.terminal
 851
 852         plot_template = NTPViz.Common + """\
 853 set terminal %(terminal)s size %(size)s
 854 set title "%(sitename)s: Local Clock Frequency Offset%(clipped)s"
 855 set ytics format "%(fmt)s %(unit)s" nomirror
 856 set yrange [%(min_y)s:%(max_y)s]
 857 set key bottom right
 858 set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
 859 set style line 2 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
 860 plot \
 861  "-" using 1:($2 * %(multiplier)s) title "local clock error" \
 862      with linespoints ls 2, \
 863  %(p99)s title "99th percentile", \
 864  %(p95)s title "95th percentile", \
 865  %(p5)s title "5th percentile", \
 866  %(p1)s title "1st percentile"
 867 """ % out
 868
 869         exp = """\
 870 <p>The frequency offset of the local clock (aka drift).  The
 871 graph includes percentile data to show how much the frequency changes
 872 over a longer period of time.  The majority of this change should come
 873 from temperature changes (ex: HVAC, the weather, CPU usage causing local
 874 heating).</p>
 875
 876 <p>Smaller changes are better.  An ideal result would be a horizontal
 877 line at 0ppm.  Expected values of 99%-1% percentiles: 0.4ppm</p>
 878
 879 <p>The Frequency Offset comes from field 4 of the loopstats log file.</p>
 880 """
 881         ret = {'html': VizStats.table_head + stats.table +
 882                VizStats.table_tail + exp,
 883                'plot': plot_template + plot_data,
 884                'stats': [stats],
 885                'title': "Local Clock Frequency Offset"}
 886         return ret
 887
 888     def loopstats_gnuplot(self, fld, title, legend, freq):
 889         "Generate gnuplot code of a given loopstats field"
 890         if not self.loopstats:
 891             sys.stderr.write("ntpviz: WARNING: no loopstats to graph\n")
 892             return ''
 893
 894         # speed up by only sending gnuplot the data it will actually use
 895         # fields: time, fld
 896         (plot_data, values) = self.plot_slice(self.loopstats, fld)
 897
 898         # process the values
 899         stats = VizStats(values, title, freq=freq)
 900
 901         # build the output dictionary, because Python can not format
 902         # complex objects.
 903         out = stats.percs
 904         out["fld"] = fld
 905         out["fmt"] = stats.percs["fmt"]
 906         out["legend"] = legend
 907         out["min_y"] = '0'
 908         out["sitename"] = self.sitename
 909         out['size'] = args.img_size
 910         out['terminal'] = args.terminal
 911
 912         if freq:
 913             exp = """\
 914 <p>The RMS Frequency Jitter (aka wander) of the local
 915 clock's frequency.  In other words, how fast the local clock changes
 916 frequency.</p>
 917
 918 <p>Lower is better.  An ideal clock would be a horizontal line at
 919 0ppm.</p>
 920
 921 <p> RMS Frequency Jitter is field 6 in the loopstats log file.</p>
 922 """
 923         else:
 924             exp = """\
 925 <p>The RMS Jitter of the local clock offset.  In other words,
 926 how fast the local clock offset is changing.</p>
 927
 928 <p>Lower is better.  An ideal system would be a horizontal line at 0μs.</p>
 929
 930 <p>RMS jitter is field 5 in the loopstats log file.</p>
 931 """
 932
 933         plot_template = NTPViz.Common + """\
 934 set terminal %(terminal)s size %(size)s
 935 set title "%(sitename)s: %(title)s%(clipped)s"
 936 set ytics format "%(fmt)s %(unit)s" nomirror
 937 set yrange [%(min_y)s:%(max_y)s]
 938 set key top right
 939 set style line 1 lc rgb '#0060ad' lt 1 lw 1 pt 7 ps 0   # --- blue
 940 set style line 2 lc rgb '#dd181f' lt 1 lw 1 pt 5 ps 0   # --- red
 941 plot \
 942  "-" using 1:($2*%(multiplier)s) title "%(legend)s" with linespoints ls 1, \
 943  %(p99)s title "99th percentile", \
 944  %(p95)s title "95th percentile", \
 945  %(p5)s title "5th percentile", \
 946  %(p1)s title "1st percentile"
 947 """ % out
 948
 949         ret = {'html': VizStats.table_head + stats.table +
 950                VizStats.table_tail + exp,
 951                'plot': plot_template + plot_data,
 952                'stats': [stats],
 953                'title': title}
 954         return ret
 955
 956     def local_offset_jitter_gnuplot(self):
 957         "Generate gnuplot code of local clock loop standard deviation"
 958         return self.loopstats_gnuplot(4, "Local RMS Time Jitter", "Jitter", 0)
 959
 960     def local_offset_stability_gnuplot(self):
 961         "Generate gnuplot code graphing local clock stability"
 962         return self.loopstats_gnuplot(5, "Local RMS Frequency Jitter",
 963                                       "Stability", 1)
 964
 965     def peerstats_gnuplot(self, peerlist, fld, title, ptype):
 966         "Plot a specified field from peerstats."
 967
 968         peerdict = self.peersplit()
 969         if not peerlist:
 970             peerlist = list(peerdict.keys())
 971             if not peerlist:
 972                 sys.stderr.write("ntpviz: WARNING: no server data to graph\n")
 973                 return ''
 974         peerlist.sort()  # For stability of output
 975         namelist = []    # peer names
 976
 977         ip_todo = []
 978         for key in peerlist:
 979             # Trickiness - we allow peerlist elements to be DNS names.
 980             # The socket.gethostbyname() call maps DNS names to IP addresses,
 981             # passing through literal IPv4 addresses unaltered.  However,
 982             # it barfs on either literal IPv6 addresses or refclock names.
 983             try:
 984                 ip = socket.gethostbyname(key)
 985                 namelist.append(key)
 986             except socket.error:
 987                 # ignore it
 988                 ip = key
 989                 # socket.getfqdn() is also flakey...
 990                 namelist.append(socket.getfqdn(key))
 991
 992             if ip in peerdict:
 993                 ip_todo.append(ip)
 994             else:
 995                 # can this ever happen?
 996                 sys.stderr.write("ntpviz: ERROR: No such server as %s" % key)
 997                 raise SystemExit(1)
 998
 999         rtt = 0
1000         percentages = ""
1001         stats = []
1002         if len(peerlist) == 1:
1003             # only one peer
1004             if "offset" == ptype:
1005                 # doing offset, not jitter
1006                 rtt = 1
1007                 if peerlist[0].startswith("127.127."):
1008                     # don't do rtt for reclocks
1009                     rtt = 0
1010                     title = "Refclock Offset " + str(peerlist[0])
1011                     exp = """\
1012 <p>The offset of a local refclock in seconds.  This is
1013 useful to see how the measured offset is behaving.</p>
1014
1015 <p>Closer to 0s is better.  An ideal system would be a horizontal line
1016 at 0s. Typical 90% ranges may be: local serial GPS 200 ms; local PPS
1017 20µs.</p>
1018
1019 <p>Clock Offset is field 5 in the peerstats log file.</p>
1020 """
1021                 else:
1022                     title = "Server Offset " + str(peerlist[0])
1023                     exp = """\
1024 <p>The offset of a server in seconds.  This is
1025 useful to see how the measured offset is behaving.</p>
1026
1027 <p>The chart also plots offset±rtt,  where rtt is the round trip time
1028 to the server.  NTP can not really know the offset of a remote chimer,
1029 NTP computes it by subtracting rtt/2 from the offset.  Plotting the
1030 offset±rtt reverses this calculation to more easily see the effects of
1031 rtt changes.</p>
1032
1033 <p>Closer to 0s is better.  An ideal system would be a horizontal line
1034 at 0s. Typical 90% ranges may be: local LAN server 80µs; 90% ranges for
1035 WAN server may be 4ms and much larger. </p>
1036
1037 <p>Clock Offset is field 5 in the peerstats log file.  The Round Trip
1038 Time (rtt) is field 6 in the peerstats log file.</p>
1039 """
1040
1041             else:
1042                 # doing jitter, not offset
1043                 if peerlist[0].startswith("127.127."):
1044                     title = "Refclock RMS Jitter " + str(peerlist[0])
1045                     exp = """\
1046 <p>The RMS Jitter of a local refclock.  Jitter is the
1047 current estimated dispersion, in other words the variation in offset
1048 between samples.</p>
1049
1050 <p>Closer to 0s is better.  An ideal system would be a horizontal
1051 line at 0s.</p>
1052
1053 <p>RMS Jitter is field 8 in the peerstats log file.</p>
1054 """
1055                 else:
1056                     title = "Server Jitter " + str(peerlist[0])
1057                     exp = """\
1058 <p>The RMS Jitter of a server.  Jitter is the
1059 current estimated dispersion, in other words the variation in offset
1060 between samples.</p>
1061
1062 <p>Closer to 0s is better.  An ideal system would be a horizontal line
1063 at 0s.</p>
1064
1065 <p>RMS Jitter is field 8 in the peerstats log file.</p>
1066 """
1067
1068             if namelist[0] and peerlist[0] != namelist[0]:
1069                 # append hostname, if we have it
1070                 # after stats to keep summary short
1071                 title += " (%s)" % namelist[0]
1072
1073         else:
1074             # many peers
1075             title += "s"
1076
1077             if "offset" == ptype:
1078                 title = "Server Offsets"
1079                 exp = """\
1080 <p>The offset of all refclocks and servers.
1081 This can be useful to see if offset changes are happening in
1082 a single clock or all clocks together.</p>
1083
1084 <p>Clock Offset is field 5 in the peerstats log file.</p>
1085 """
1086             else:
1087                 title = "Server Jitters"
1088                 exp = """\
1089 <p>The RMS Jitter of all refclocks and servers. Jitter is the
1090 current estimated dispersion, in other words the variation in offset
1091 between samples.</p>
1092
1093 <p>Closer to 0s is better.  An ideal system would be a horizontal line
1094 at 0s.</p>
1095
1096 <p>RMS Jitter is field 8 in the peerstats log file.</p>
1097 """
1098
1099         if len(peerlist) == 1:
1100             if peerlist[0] in refclock_name:
1101                 title += ' ' + refclock_name[peerlist[0]]
1102
1103         plot_data = ""
1104         for ip in ip_todo:
1105             # 20% speed up by only sending gnuplot the data it will
1106             # actually use
1107             if rtt:
1108                 # fields: time, fld, and rtt
1109                 pt = self.plot_slice(peerdict[ip], fld, 5)
1110                 plot_data += pt[0]
1111             else:
1112                 # fields: time, fld
1113                 pt = self.plot_slice(peerdict[ip], fld)
1114                 plot_data += pt[0]
1115
1116         stats = VizStats(pt[1], title)
1117         if len(peerlist) == 1:
1118             percentages = " %(p50)s title '50th percentile', " % stats.percs
1119         else:
1120             # skip stats on peers/offsets plots
1121             stats.skip_summary = True
1122             stats.table = ''
1123
1124         out = stats.percs
1125         out["fmt"] = stats.percs["fmt"]
1126         out['sitename'] = self.sitename
1127         out['size'] = args.img_size
1128         out['terminal'] = args.terminal
1129         out['title'] = title
1130
1131         if 6 >= len(peerlist):
1132             out['set_key'] = "set key top right"
1133         elif 12 >= len(peerlist):
1134             # getting crowded
1135             out['set_key'] = "set key bmargin"
1136         else:
1137             # too many keys to show
1138             out['set_key'] = "set key off"
1139
1140         plot_template = NTPViz.Common + """\
1141 set terminal %(terminal)s size %(size)s
1142 set title "%(sitename)s: %(title)s%(clipped)s"
1143 set ylabel ""
1144 set ytics format "%(fmt)s %(unit)s" nomirror
1145 set yrange [%(min_y)s:%(max_y)s]
1146 %(set_key)s
1147 plot \
1148 """ % out
1149
1150         plot_template += percentages
1151         for key in peerlist:
1152             out['label'] = self.ip_label(key)
1153             plot_template += "'-' using 1:($2*%(multiplier)s) " \
1154                              " title '%(label)s' with line, \\\n" % out
1155
1156         if 1 == rtt:
1157             plot_template += """\
1158 '-' using 1:(($2+$3/2)*%(multiplier)s) title 'offset+rtt/2' with line, \\
1159 '-' using 1:(($2-$3/2)*%(multiplier)s) title 'offset-rtt/2' with line
1160 """ % stats.percs
1161             # sadly, gnuplot needs 3 identical copies of the data.
1162             plot_template += plot_data + plot_data
1163         else:
1164             # strip the trailing ", \n"
1165             plot_template = plot_template[:-4] + "\n"
1166
1167         if len(peerlist) == 1:
1168             # skip stats for multiplots
1169             html = VizStats.table_head + stats.table \
1170                 + VizStats.table_tail + exp,
1171         else:
1172             html = exp
1173
1174         ret = {'html': html,
1175                'plot': plot_template + plot_data,
1176                'stats': [stats],
1177                'title': title}
1178         return ret
1179
1180     def peer_offsets_gnuplot(self, peerlist=None):
1181         "gnuplot Peer Offsets"
1182         return self.peerstats_gnuplot(peerlist, 4, "Server Offset",
1183                                       "offset")
1184
1185     def peer_jitters_gnuplot(self, peerlist=None):
1186         "gnuplot Peer Jitters"
1187         return self.peerstats_gnuplot(peerlist, 7, "Server Jitter",
1188                                       "jitter")
1189
1190     def local_offset_histogram_gnuplot(self):
1191         "Plot a histogram of clock offset values from loopstats."
1192         if not self.loopstats:
1193             sys.stderr.write("ntpviz: WARNING: no loopstats to graph\n")
1194             return ''
1195
1196         # TODO normalize to 0 to 100?
1197
1198         # grab and sort the values, no need for the timestamp, etc.
1199         values = [float(row[2]) for row in self.loopstats]
1200         stats = VizStats(values, 'Local Clock Offset')
1201         out = stats.percs
1202         out["fmt_x"] = stats.percs["fmt"]
1203         out['sitename'] = self.sitename
1204         # flip the axis
1205         out['min_x'] = out['min_y']
1206         out['max_x'] = out['max_y']
1207
1208         rnd1 = 7        # round to 100 ns boxes
1209         out['boxwidth'] = 1e-7
1210
1211         # between -10us and 10us
1212         if 1e-5 > stats.percs["p99"] and -1e-5 < stats.percs["p1"]:
1213             # go to nanosec
1214             rnd1 = 9        # round to 1 ns boxes
1215             out['boxwidth'] = S_PER_NS
1216
1217         # Python 2.6  has no collections.Counter(), so fake it.
1218         cnt = collections.defaultdict(int)
1219         for value in values:
1220             # put into buckets
1221             # for a +/- 50 microSec range that is 1,000 buckets to plot
1222             cnt[round(value, rnd1)] += 1
1223
1224         sigma = True
1225         if args.clip:
1226             if stats.percs['p1sigma'] > stats.percs['p99'] or \
1227                stats.percs['m1sigma'] < stats.percs['p1']:
1228                 # sigma out of range, do not plot
1229                 sigma = ''
1230
1231         out['sigma'] = ''
1232         if sigma:
1233             # plus/minus of one sigma range
1234             out['sigma'] = """\
1235 set style arrow 1 nohead
1236 set linestyle 1 linecolor rgb "#009900"
1237 set arrow from %(m1sigma)s,graph 0 to %(m1sigma)s,graph 0.90 as 1 ls 1
1238 set arrow from %(p1sigma)s,graph 0 to %(p1sigma)s,graph 0.90 as 1 ls 1
1239 set label 1 "-1σ" at %(m1sigma)s, graph 0.96 left front offset -1,-1 \
1240     textcolor rgb "#009900"
1241 set label 2 "+1σ" at %(p1sigma)s, graph 0.96 left front offset -1,-1 \
1242     textcolor rgb "#009900"
1243 """ % out
1244
1245         out['size'] = args.img_size
1246         out['terminal'] = args.terminal
1247
1248         # in 2016, 25% of screens are 1024x768, 42% are 1388x768
1249         # but leave some room for the browser frame
1250         plot_template = '''\
1251 set terminal %(terminal)s size %(size)s
1252 set title "%(sitename)s: Local Clock Time Offset Histogram%(clipped)s"
1253 set grid
1254 set boxwidth %(boxwidth)s
1255 set xtics format "%(fmt_x)s %(unit)s" nomirror
1256 set xrange [%(min_x)s:%(max_x)s]
1257 set yrange [0:*]
1258 set style arrow 3 nohead
1259 set arrow from %(p99)s,graph 0 to %(p99)s,graph 0.30 as 3
1260 set style arrow 4 nohead
1261 set arrow from %(p95)s,graph 0 to %(p95)s,graph 0.45 as 4
1262 set style arrow 5 nohead
1263 set arrow from %(p5)s,graph 0 to %(p5)s,graph 0.45 as 5
1264 set style arrow 6 nohead
1265 set arrow from %(p1)s,graph 0 to %(p1)s,graph 0.30 as 6
1266 set key off
1267 set lmargin 10
1268 set rmargin 10
1269 set style fill solid 0.5
1270 set label 3 "99%%" at %(p99)s, graph 0.35  left front offset -1,-1
1271 set label 4 "95%%" at %(p95)s, graph 0.50  left front offset -1,-1
1272 set label 5 "1%%" at %(p1)s, graph 0.35  left front offset -1,-1
1273 set label 6 "5%%" at %(p5)s, graph 0.50  left front offset -1,-1
1274 %(sigma)s
1275 plot \
1276  "-" using ($1 * %(multiplier)s):2 title "histogram" with boxes
1277 ''' % out
1278
1279         histogram_data = ["%s %s\n" % (k, v) for k, v in cnt.items()]
1280
1281         exp = """\
1282 <p>The clock offsets of the local clock as a histogram.</p>
1283
1284 <p>The Local Clock Offset is field 3 from the loopstats log file.</p>
1285 """
1286         # don't return stats, it's just a dupe
1287         ret = {'html': VizStats.table_head + stats.table +
1288                VizStats.table_tail + exp,
1289                'plot': plot_template + "".join(histogram_data) + "e\n",
1290                'stats': [],
1291                'title': "Local Clock Time Offset Histogram"}
1292         return ret
1293
1294
1295 # Multiplotting can't live inside NTPViz because it consumes a list
1296 # of such objects, not a single one.
1297
1298
1299 def local_offset_multiplot(statlist):
1300     "Plot comparative local offsets for a list of NTPViz objects."
1301
1302     out = {}
1303     out['size'] = args.img_size
1304     out['terminal'] = args.terminal
1305
1306     plot = NTPViz.Common + '''\
1307 set terminal %(terminal)s size %(size)s
1308 set title "Multiplot Local Clock Offsets"
1309 set ytics format "%%1.2f μs" nomirror textcolor rgb "#0060ad"
1310 set key bottom right box
1311 plot \\
1312 ''' % out
1313     # FIXME: probably need to be more flexible about computing the plot label
1314     sitenames = [os.path.basename(os.path.dirname(dr))
1315                  for dr in args.statsdirs]
1316     for (i, stats) in enumerate(statlist):
1317         plot += '"-" using 1:($2*1000000) title "%s clock offset μs"  ' \
1318                 'with linespoints, \\\n' % (sitenames[i])
1319     plot = plot[:-4] + "\n"
1320
1321     plot_data = ''
1322     for stats in statlist:
1323         # speed up by only sending gnuplot the data it will actually use
1324         # fields: time, offset
1325         pt = stats.plot_slice(stats.loopstats, 2)
1326         plot_data += pt[0]
1327
1328     ret = {'html': '', 'stats': []}
1329     ret['title'] = "Multiplot"
1330     ret['plot'] = plot + plot_data
1331     return ret
1332
1333
1334 # here is how to create the base64 from an image file:
1335 # with open("path/to/file.png", "rb") as f:
1336 #    data = f.read()
1337 #    print data.encode("base64")
1338 #
1339 ntpsec_logo = """
1340 iVBORw0KGgoAAAANSUhEUgAAAEAAAABKCAQAAACh+5ozAAAABGdBTUEAALGPC/xhBQAAAAFzUkdC
1341 AK7OHOkAAAAgY0hSTQAAeiYAAICEAAD6AAAAgOgAAHUwAADqYAAAOpgAABdwnLpRPAAAAAJiS0dE
1342 AP7wiPwpAAAACXBIWXMAAAsTAAALEwEAmpwYAAAFKElEQVRo3s2ZT0wcVRzHPzMLKCwsNgqLkYPS
1343 xBjbRF3TcKlC4VAhFU0AdRN7a+zBEsUEL0qImqoxMTWhBzEkTdqmREhMCgpeiiV6KVE46MVE1KQg
1344 uxv/df81tLvzPOzsMjs7sztvd7b4ndPsfPf3vu/33vv93vs9yGCIJMLyWaKJXTSxZMMTCITilJ1k
1345 KENRdeoB6rHGYboNb80cpAjEQZoNr90ctiHWcyBfgD0aCZTk2CFAYylKTd7bVZYNknycwGf5ryjT
1346 RE2/OWVr9Bh9ahbwnuGtnRdsTZ5h0/Rbhr1PDYhNUZyt2guwRjdazi8+G0lZeMWoeExna3mzxwbO
1347 BDgwlIWQYhefhCkSNl8SpCpkO/JAiHFO00D+kCokGa8JpRyylSTjIlSeAPiC7/AU/JomknLM9qRb
1348 Ijv8XaaANNs4hyU7VcJE6UBUZeR7wLjgqgXT4jQL6JYw5Qqy/U3e6YazLWY9cJ5DDOc+/kvU9aHQ
1349 8HFP7m2O8/kCwoyQYgAvAD8xwja1rjUugA7e15NzgnlGCRfSvATZII1A4yv1KIqL/R/iF9IIBCGC
1350 itfOtEoHs/qeJURQ90elaGOCbQSCtLKhDOd/LJTiZ1KfDXGW+aFiP2h00o8CJJhX3m75PabdLMZX
1351 jIrdfIq6vhDDhFxtfkV9xtqXlrmgjltzHGIMSBMhXcEAeGjFAyxrX1sTLAXcAvTsHuE5tixjgga6
1352 NA92OUXjAS5zfzGFpXZEabb5w7Jn99LMAI3EmecGf9n4SS3lPydbskKjD3GcIM3ch4c0Y9xghgv8
1353 hiZvrBwBg3zIgwj+1FN9LfsZ52Uu8ikhWWPyAoY5Swu/coEZYmio+DhGD31M8CgjViG2PEwgEFyn
1354 3dR8GMEsHahAF+/SBezGjkums1A71xEIJtwR0K837zdwdk0HiRNnQE6ATNL1cpJWFjll4+YF5vFy
1355 Qi6DyAhop5MkU0Rsvsd5hzC99FZLwAB+NlktwtjkGg08US0BDcDlogstwRoQkBkE2WVYePw6ondD
1356 ZZUFAALssz2mVSwgHzFCPMwjAHhoY1HehKyAAF5D76aZNXyL6nF/jX+qI2CdJJ2087Ohyfw6iZcA
1357 sOZ8AOQm4Sqb+HmpCKOXXhKsS9iUEhDiEnCc/TbfWzmJlytcqZYAuMgG+/kgF4qN8HOWfiJMyQxA
1358 MRRLRoscy0s62e18GNOmu3QukF0Fc8AkfTzFN6zwJXEET9LF83QQ4RRz7vTe3gOg0McCMQQpQmyx
1359 RRRBnAX6LPa9rnsABEt8yxG6eFavC8dZYYqrxMvpZ3mRMM4Ci3ycqwhFC+qmVRYAsvWjsgX4GC2/
1360 d5SurNoK8Oo1ch9vuNFP+XN2kJjLR9Nh64asPNDEa7xKIxVNLgN8+PAzCVZRwurEGuQzGoEwr7Ni
1361 USmVQ5ouPsFPpgzkIFBlD+a2TpOF6txmPtXVMpkTCZ5d2jaDblaoABjUqy4mCcZ2+jlHK3CTt/gc
1362 xdUqmUDwIqepBzY4ykahgFbO0Q9AirCp6u8OFPz6qpvhlcLMMeZ6Wcr+iSu5E+TuTGvIyqzuA4BX
1363 5E5P5kAUrZuucSP42CDl2zHdLhYI2DmzsylhURYFd5F7fmOy5wJqaFbb7h5Q65PdGoDvrtEqz4HM
1364 APTUfn97HZW4whKPKy14sgvf9QhoQi7ARImi8KNSlZAjgewqcCfzy0DfrGUFTPORi1c0pXGbNzOb
1365 vV0PuFZgdAjd4/+DZZjBnbgzNSJ3f7rnq0AltrcCPMR4mro9a3/9Pwl2Z1Rsm9zNAAAAJXRFWHRk
1366 YXRlOmNyZWF0ZQAyMDE1LTA2LTI5VDE4OjMwOjA3LTA0OjAwZxkj2wAAACV0RVh0ZGF0ZTptb2Rp
1367 ZnkAMjAxNS0wNi0yOVQxODozMDowNy0wNDowMBZEm2cAAAAASUVORK5CYII=
1368 """
1369
1370 ntpsec_ico = """\
1371 AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAAAAAAAAAAAAAAAAA
1372 AAAAAAAAAAAAAAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/
1373 AAAA/wAAAP8AAAD/AAAAAAAAAP+fn59gn5+fYJ+fn2Cfn59gn5+fYJ+fn2Cfn59gn5+fYJ+fn2Cf
1374 n59gn5+fYJ+fn2B/f39/AAAA/wAAAAAAAAAAAAAA/5+fn2Cfn59gn5+fYJ+fn2Cfn59gn5+fYJ+f
1375 n2Cfn59gn5+fYJ+fn2Cfn59gAAAA/wAAAAAAAAAAAAAAAAAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA
1376 /wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAAAAAAAAAAAAAAAAAAAAAAA/5+fn2Cfn59g
1377 n5+fYJ+fn2Cfn59gn5+fYJ+fn2Cfn59gn5+fYAAAAP8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP+f
1378 n59gn5+fYAAAAP8AAAD/AAAA/wAAAP8AAAD/n5+fYJ+fn2AAAAD/AAAAAAAAAAAAAAAAAAAAAAAA
1379 AAAAAAD/n5+fYAAAAP8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP+fn59gAAAA/wAAAAAAAAAAAAAA
1380 AAAAAAAAAAAAAAAA/5+fn2AAAAD/AAAAAAAAAAAAAAD/AAAA/01NTWAAAAD/n5+fYAAAAP8AAAAA
1381 AAAAAAAAAAAAAAAAAAAAAAAAAP+fn59gAAAA/wAAAAAAAAAAAAAA/wAAAAAAAAAAAAAA/5+fn2AA
1382 AAD/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/n5+fYAAAAP8AAAAAAAAAAE1NTWAAAAAAAAAAAAAA
1383 AP+fn59gAAAA/wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/5+fn2Cfn59gAAAA/wAAAP8AAAD/AAAA
1384 /wAAAP+fn59gn5+fYAAAAP8AAAAAAAAAAAAAAAAAAAAAAAAAAAEBAf+fn59gn5+fYJ+fn2Cfn59g
1385 n5+fYJ+fn2Cfn59gn5+fYJ+fn2AAAAD/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/AAAA/wAAAP8A
1386 AAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAAAAAAAAAAAAAAAAAAAAAAD/AAAAAAAA
1387 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/AAAAAAAAAAAAAAAAAAAA
1388 /wAAAAAAAAD/AAAA/wAAAP8AAAAAAAAAAAAAAAAAAAD/AAAA/wAAAP8AAAAAAAAA/wAAAAAAAAAA
1389 AAAAAAAAAP8AAAD/AAAA/wAAAAAAAAD/AAAA/wAAAP8AAAD/AAAA/wAAAAAAAAD/AAAA/wAAAP8A
1390 AAAAgAAAAL/+AADf/QAAwAEAAO/7AADsGwAA6+sAAOsrAADrawAA6+sAAOwbAADv+wAA4AMAAN/9
1391 AADRxQAAxBEAAA==
1392 """
1393
1394 if __name__ == '__main__':
1395     bin_ver = "ntpsec-@NTPSEC_VERSION_EXTENDED@"
1396     ntp.util.stdversioncheck(bin_ver)
1397
1398     parser = MyArgumentParser(description="ntpd stats visualizer",
1399                               fromfile_prefix_chars='@',
1400                               epilog="""
1401 You can place command line options in a file, one per line.
1402
1403 See the manual page for details.
1404
1405 Python by ESR, concept and gnuplot code by Dan Drown.
1406 """)
1407     group = parser.add_mutually_exclusive_group()
1408
1409     parser.add_argument('-c', '--clip',
1410                         action="store_true",
1411                         dest='clip',
1412                         help="Clip plots at 1%% and 99%%")
1413     parser.add_argument('-d', '--datadir',
1414                         default="/var/log/ntpstats",
1415                         dest='statsdirs',
1416                         help="one or more log file directories to read",
1417                         type=str)
1418     parser.add_argument('-D', '--debug',
1419                         default=0,
1420                         dest='debug_level',
1421                         help="debug level, 0 (none) to 9 (most)",
1422                         type=int)
1423     parser.add_argument('-e', '--endtime',
1424                         dest='endtime',
1425                         help="End time in POSIX (seconds) or ISO 8601",
1426                         type=str)
1427     parser.add_argument('-g', '--generate',
1428                         action="store_true",
1429                         dest='generate',
1430                         help="Run through gnuplot to make plot images")
1431     parser.add_argument('-n', '--name',
1432                         default=socket.getfqdn(),
1433                         dest='sitename',
1434                         help="sitename (title)",
1435                         type=str)
1436     # some OS do not support os.nice()
1437     try:
1438         os.nice(0)
1439         parser.add_argument('-N', '--nice',
1440                             action="store_true",
1441                             dest='nice',
1442                             help="Run as lowest priority")
1443     except OSError:
1444         pass
1445
1446     parser.add_argument('-o', '--outdir',
1447                         default="ntpgraphs",
1448                         dest='outdir',
1449                         help="output directory",
1450                         type=str)
1451     parser.add_argument('-p', '--period',
1452                         default=7,    # default to 7 days
1453                         dest='period',
1454                         help="period in days to graph (float)",
1455                         type=float)
1456     parser.add_argument('-s', '--starttime',
1457                         dest='starttime',
1458                         help="Start time in POSIX (seconds) or ISO 8601",
1459                         type=str)
1460     parser.add_argument('-T', '--terminal',
1461                         default='png',
1462                         dest='terminal',
1463                         help="gnuplot terminal type for graphs",
1464                         type=str)
1465     parser.add_argument('-w', '--width',
1466                         choices=['s', 'm', 'l'],
1467                         default='m',
1468                         dest='width',
1469                         help="Image width: s, m, or l",
1470                         type=str)
1471     group.add_argument('--all-peer-jitters',
1472                        action="store_true",
1473                        dest='show_peer_jitters',
1474                        help="Plot all peer jitters")
1475     group.add_argument('--all-peer-offsets',
1476                        action="store_true",
1477                        dest='show_peer_offsets',
1478                        help="Plot all peer offsets")
1479     group.add_argument('--local-error',
1480                        action="store_true",
1481                        dest='show_local_error',
1482                        help="Plot local clock frequency offsets")
1483     group.add_argument('--local-freq-temps',
1484                        action="store_true",
1485                        dest='show_freq_temps',
1486                        help="Plot local frequency vs temperature data")
1487     group.add_argument('--local-gps',
1488                        action="store_true",
1489                        dest='show_gps',
1490                        help="Plot gpsd TDOP and nSats")
1491     group.add_argument('--local-jitter',
1492                        action="store_true",
1493                        dest='show_local_jitter',
1494                        help="Plot clock time jitter")
1495     group.add_argument('--local-offset',
1496                        action="store_true",
1497                        dest='show_local_offset',
1498                        help="Plot Clock frequency offset")
1499     group.add_argument('--local-offset-histogram',
1500                        action="store_true",
1501                        dest='show_local_offset_histogram',
1502                        help="Plot histogram of loopstats time offsets")
1503     group.add_argument('--local-offset-multiplot',
1504                        action="store_true",
1505                        dest='show_local_offset_multiplot',
1506                        help="Plot comparative local offsets for "
1507                             "multiple directories")
1508     group.add_argument('--local-stability',
1509                        action="store_true",
1510                        dest='show_local_stability',
1511                        help="Plot RMS frequency-jitter")
1512     group.add_argument('--local-temps',
1513                        action="store_true",
1514                        dest='show_temps',
1515                        help="Plot local temperature data")
1516     group.add_argument('--peer-jitters',
1517                        default='',
1518                        dest='peer_jitters',
1519                        help="Plot peer jitters.  Comma separated host list.",
1520                        type=str)
1521     group.add_argument('--peer-offsets',
1522                        default='',
1523                        dest='peer_offsets',
1524                        help="Plot peer offsets.  Comma separated host list.",
1525                        type=str)
1526     parser.add_argument('-V', '--version',
1527                         action="version",
1528                         version="ntpviz %s" % ntp.util.stdversion())
1529
1530     args = parser.parse_args()
1531
1532     if args.nice:
1533         # run at lowest possible priority
1534         nice = os.nice(19)
1535         if args.debug_level:
1536             sys.stderr.write("ntpviz: INFO: Now running at nice %s\n" % nice)
1537
1538     if 's' == args.width:
1539         # fit in 1024x768 browser
1540         # in 2016 this is 22% of all browsers
1541         args.img_size = '1000,720'
1542     elif 'l' == args.width:
1543         # fit in 1920x1080 browser
1544         args.img_size = '1850,1000'
1545     else:
1546         # fit in 1388x768 browser
1547         # in 2016 this is 42% of all browsers
1548         args.img_size = '1340,720'
1549
1550     # figure out plot image file extension
1551     term_map = {'gif': '.gif',
1552                 'jpeg': '.jpg',
1553                 'pngcairo': '.png',
1554                 'png': '.png',
1555                 'svg': '.svg',
1556                }
1557     if args.terminal in term_map:
1558         args.img_ext = term_map[args.terminal]
1559     else:
1560         sys.stderr.write("ntpviz: ERROR: Unknown terminal type: %s\n" %
1561                          args.terminal)
1562         raise SystemExit(1)
1563
1564     args.period = int(float(args.period) * ntp.statfiles.NTPStats.SecondsInDay)
1565     if args.endtime is not None:
1566         args.endtime = ntp.statfiles.iso_to_posix(args.endtime)
1567     if args.starttime is not None:
1568         args.starttime = ntp.statfiles.iso_to_posix(args.starttime)
1569
1570     args.statsdirs = [os.path.expanduser(path)
1571                       for path in args.statsdirs.split(",")]
1572
1573     if args.show_peer_offsets:
1574         args.show_peer_offsets = []
1575     elif args.peer_offsets:
1576         args.show_peer_offsets = args.peer_offsets.split(",")
1577     else:
1578         args.show_peer_offsets = None
1579
1580     if args.show_peer_jitters:
1581         args.show_peer_jitters = []
1582     elif args.peer_jitters:
1583         args.show_peer_jitters = args.peer_jitters.split(",")
1584     else:
1585         args.show_peer_jitters = None
1586
1587     if 0 < args.debug_level:
1588         sys.stderr.write("ntpviz: INFO: now running at debug: %s\n" %
1589                          args.debug_level)
1590         sys.stderr.write("ntpviz: INFO: Version: %s\n" % ntp.util.stdversion())
1591         sys.stderr.write("ntpviz: INFO: Parsed Options %s\n" % args)
1592
1593         if 9 == args.debug_level:
1594             # crazy debug, also profile
1595             import cProfile
1596             pr = cProfile.Profile()
1597             pr.enable()
1598
1599             # register to dump debug on all normal exits
1600             atexit.register(print_profile)
1601
1602     nice = 19       # always run nicely
1603     if 0 != nice:
1604         try:
1605             import psutil
1606             # set ionice() to idle
1607             p = psutil.Process(os.getpid())
1608             try:
1609                 p.ionice(psutil.IOPRIO_CLASS_IDLE)
1610             except AttributeError:
1611                 sys.stderr.write("ntpviz: INFO: ionice is not available\n")
1612
1613         except ImportError:
1614             if 0 < args.debug_level:
1615                 sys.stderr.write("ntpviz: INFO: psutil not found\n")
1616
1617         # set nice()
1618         nice = os.nice(nice)
1619         if 2 < args.debug_level:
1620             sys.stderr.write("ntpviz: INFO: now running at nice: %s\n" % nice)
1621
1622     for fontpath in ("@PREFIX@/share/fonts/liberation",
1623                      "@PREFIX@/share/fonts/liberation-fonts",
1624                      "@PREFIX@/share/fonts/truetype/liberation",
1625                      "/usr/share/fonts/liberation",
1626                      "/usr/share/fonts/liberation-fonts",
1627                      "/usr/share/fonts/truetype/liberation"):
1628
1629         if os.path.exists(fontpath):
1630             os.environ["GDFONTPATH"] = fontpath
1631             break
1632     else:
1633         sys.stderr.write(
1634             "ntpviz: WARNING: liberation truetype fonts not found\n")
1635     os.environ["GNUPLOT_DEFAULT_GDFONT"] = "LiberationSans-Regular"
1636
1637     plot = None
1638
1639     if 1 == len(args.statsdirs):
1640         statlist = [NTPViz(statsdir=args.statsdirs[0], sitename=args.sitename,
1641                            period=args.period, starttime=args.starttime,
1642                            endtime=args.endtime)]
1643     else:
1644         statlist = [NTPViz(statsdir=d, sitename=d,
1645                            period=args.period, starttime=args.starttime,
1646                            endtime=args.endtime)
1647                     for d in args.statsdirs]
1648
1649     if len(statlist) == 1:
1650         stats = statlist[0]
1651
1652         if args.show_local_offset or \
1653            args.show_local_error or \
1654            args.show_local_jitter or \
1655            args.show_local_stability or \
1656            args.show_local_offset_histogram:
1657             if not stats.loopstats:
1658                 sys.stderr.write("ntpviz: ERROR: missing loopstats data\n")
1659                 raise SystemExit(1)
1660
1661             if args.show_local_offset:
1662                 plot = stats.local_offset_gnuplot()
1663             elif args.show_local_error:
1664                 plot = stats.local_error_gnuplot()
1665             elif args.show_local_jitter:
1666                 plot = stats.local_offset_jitter_gnuplot()
1667             elif args.show_local_stability:
1668                 plot = stats.local_offset_stability_gnuplot()
1669             elif args.show_local_offset_histogram:
1670                 plot = stats.local_offset_histogram_gnuplot()
1671
1672         if args.show_peer_offsets is not None or \
1673            args.show_peer_jitters is not None:
1674             if not stats.peerstats:
1675                 sys.stderr.write("ntpviz: ERROR:  missing peerstats data\n")
1676                 raise SystemExit(1)
1677             if args.show_peer_offsets is not None:
1678                 plot = stats.peer_offsets_gnuplot(args.show_peer_offsets)
1679             if args.show_peer_jitters is not None:
1680                 plot = stats.peer_jitters_gnuplot(args.show_peer_jitters)
1681
1682         if args.show_freq_temps:
1683             if not stats.temps:
1684                 sys.stderr.write("ntpviz: ERROR: missing temps data\n")
1685                 raise SystemExit(1)
1686             plot = stats.local_freq_temps_plot()
1687
1688         if args.show_temps:
1689             if not stats.temps:
1690                 sys.stderr.write("ntpviz: ERROR: missing temps data\n")
1691                 raise SystemExit(1)
1692             plot = stats.local_temps_gnuplot()
1693
1694         if args.show_gps:
1695             if not stats.gpsd:
1696                 sys.stderr.write("ntpviz: ERROR: missing gps data\n")
1697                 raise SystemExit(1)
1698             plot = stats.local_gps_gnuplot()
1699
1700     if args.show_local_offset_multiplot:
1701         plot = local_offset_multiplot(statlist)
1702
1703     if plot is not None:
1704         # finish up the plot, and exit
1705         if args.generate:
1706             gnuplot(plot['plot'])
1707         else:
1708             sys.stdout.write(plot['plot'])
1709         raise SystemExit(0)
1710
1711     # Fall through to HTML code generation
1712     if not os.path.isdir(args.outdir):
1713         try:
1714             os.mkdir(args.outdir)
1715         except SystemError:
1716             sys.stderr.write("ntpviz: ERROR: %s can't be created.\n"
1717                              % args.outdir)
1718             raise SystemExit(1)
1719
1720     # if no ntpsec favicon.ico, write one.
1721     ico_filename = os.path.join(args.outdir, "favicon.ico")
1722     if not os.path.lexists(ico_filename):
1723         with open(ico_filename, "wb") as wp:
1724             wp.write(binascii.a2b_base64(ntpsec_ico))
1725
1726     # if no ntpsec logo, write one.
1727     logo_filename = os.path.join(args.outdir, "ntpsec-logo.png")
1728     if not os.path.lexists(logo_filename):
1729         with open(logo_filename, "wb") as wp:
1730             wp.write(binascii.a2b_base64(ntpsec_logo))
1731
1732     # report_time = datetime.datetime.utcnow() # the time now is...
1733     report_time = datetime.datetime.now(UTC())   # the time now is...
1734     report_time = report_time.strftime("%c %Z")  # format it nicely
1735
1736     title = args.sitename
1737
1738     index_header = '''\
1739 <!DOCTYPE html>
1740 <html lang="en">
1741 <head>
1742 <link rel="shortcut icon" href="favicon.ico">
1743 <meta charset="UTF-8">
1744 <meta http-equiv="refresh" content="1800">
1745 <meta name="expires" content="0">
1746 <title>%(title)s</title>
1747 <style>
1748 dt {
1749     font-weight: bold;
1750     margin-left: 20px;
1751 }
1752 dd {
1753     margin-top: 4px;
1754     margin-bottom: 10px;
1755 }
1756 table {
1757     text-align: right;
1758     width: 1300px;
1759     border-collapse: collapse;
1760 }
1761 thead {
1762     font-weight: bold;
1763 }
1764 tbody tr {
1765     vertical-align: top;
1766 }
1767 tbody tr:nth-child(6n+4),
1768 tbody tr:nth-child(6n+5),
1769 tbody tr:nth-child(6n+6) {
1770     background-color: rgba(0,255,0,0.2);
1771 }
1772 .section {
1773     color: #000000;
1774     text-decoration: none;
1775 }
1776 .section .site-title:visited {
1777     color: #000000;
1778 }
1779 </style>
1780 </head>
1781 <body>
1782 <div style="width:910px">
1783 <a href='https://www.ntpsec.org/'>
1784 <img src="ntpsec-logo.png" alt="NTPsec" style="float:left;margin:20px 70px;">
1785 </a>
1786 <div>
1787 <h1 style="margin-bottom:10px;">%(title)s</h1>
1788 <b>Report generated:</b> %(report_time)s <br>
1789 ''' % locals()
1790
1791     # Ugh.  Not clear what to do in the multiplot case
1792     if len(statlist) == 1:
1793         start_time = datetime.datetime.utcfromtimestamp(
1794             stats.starttime).strftime('%c')
1795         end_time = datetime.datetime.utcfromtimestamp(
1796             stats.endtime).strftime('%c')
1797
1798         index_header += '<b>Start Time:</b> %s UTC<br>\n' \
1799                         '<b>End Time:</b> %s UTC<br>\n' \
1800             % (start_time, end_time)
1801         if 1 > stats.period:
1802             # less than a day, report hours
1803             index_header += ('<b>Report Period:</b> %1.1f hours <br>\n' %
1804                              (float(stats.period) / (24 * 60)))
1805         else:
1806             # more than a day, report days
1807             index_header += ('<b>Report Period:</b> %1.1f days <br>\n' %
1808                              (float(stats.period) /
1809                               ntp.statfiles.NTPStats.SecondsInDay))
1810
1811     if args.clip:
1812         index_header += """\
1813 <span style="color:red;font-weight:bold;">Warning: plots clipped</span><br>
1814 """
1815
1816     index_header += '</div>\n<div style="clear:both;"></div>'
1817
1818     index_trailer = '''\
1819 <h2>Glossary:</h2>
1820
1821 <dl>
1822 <dt>frequency offset:</dt>
1823 <dd>The difference between the ntpd calculated frequency and the local
1824 system clock frequency (usually in parts per million, ppm)</dd>
1825
1826 <dt>jitter, dispersion:</dt>
1827 <dd>The short term change in a value.  NTP measures Local Time Jitter,
1828 Refclock Jitter, and Server Jitter in seconds.  Local Frequency Jitter is
1829 in ppm or ppb.
1830 </dd>
1831
1832 <dt>kurtosis, Kurt:</dt>
1833 <dd>The kurtosis of a random variable X is the fourth standardized
1834 moment and is a dimension-less ratio. ntpviz uses the Pearson's moment
1835 coefficient of kurtosis.  A normal distribution has a kurtosis of three.
1836 NIST describes a kurtosis over three as "heavy tailed" and one under
1837 three as "light tailed".</dd>
1838
1839 <dt>ms, millisecond:</dt>
1840 <dd>One thousandth of a second = 0.001 seconds, 1e-3 seconds</dd>
1841
1842 <dt>mu, mean:</dt>
1843 <dd>The arithmetic mean: the sum of all the values divided by the
1844 number of values.
1845 The formula for mu is: "mu = (∑x<sub>i</sub>) / N".
1846 Where x<sub>i</sub> denotes the data points and N is the number of data
1847 points.</dd>
1848
1849 <dt>ns, nanosecond:</dt>
1850 <dd>One billionth of a second, also one thousandth of a microsecond,
1851 0.000000001 seconds and 1e-9 seconds.</dd>
1852
1853 <dt>percentile:</dt>
1854 <dd>The value below which a given percentage of values fall.</dd>
1855
1856 <dt>ppb, parts per billion:</dt>
1857 <dd>Ratio between two values. These following are all the same:
1858     1 ppb, one in one billion, 1/1,000,000,000, 0.000,000,001, 1e-9 and
1859     0.000,000,1%</dd>
1860
1861 <dt>ppm, parts per million:</dt>
1862 <dd>Ratio between two values. These following are all the same:
1863     1 ppm, one in one million, 1/1,000,000, 0.000,001, and 0.000,1%</dd>
1864
1865 <dt>‰, parts per thousand:</dt>
1866 <dd>Ratio between two values. These following are all the same:
1867     1 ‰. one in one thousand, 1/1,000, 0.001, and 0.1%</dd>
1868
1869 <dt>refclock:</dt>
1870 <dd>Reference clock, a local GPS module or other local source of time.</dd>
1871
1872 <dt>remote clock:</dt>
1873 <dd>Any clock reached over the network, LAN or WAN.  Also called a peer
1874 or server.</dd>
1875
1876 <dt>time offset:</dt>
1877 <dd>The difference between the ntpd calculated time and the local system
1878  clock's time.  Also called phase offset.</dd>
1879
1880 <dt>σ, sigma:</dt>
1881 <dd>Sigma denotes the standard deviation (SD) and is centered on the
1882 arithmetic mean of the data set. The SD is simply the square root of
1883 the variance of the data set.  Two sigma is simply twice the standard
1884 deviation.  Three sigma is three times sigma.  Smaller is better.<br>
1885 The formula for sigma is: "σ = √[ ∑(x<sub>i</sub>-mu)^2 / N ]".
1886 Where x<sub>i</sub> denotes the data points and N is the number of data
1887 points.</dd>
1888
1889 <dt>skewness, Skew:</dt>
1890 <dd>The skewness of a random variable X is the third standardized
1891 moment and is a dimension-less ratio. ntpviz uses the Pearson's moment
1892 coefficient of skewness.  Wikipedia describes it best: "The qualitative
1893 interpretation of the skew is complicated and unintuitive."<br> A normal
1894 distribution has a skewness of zero. </dd>
1895
1896 <dt>upstream clock:</dt>
1897 <dd>Any server or reference clock used as a source of time.</dd>
1898
1899 <dt>µs, us, microsecond:</dt>
1900 <dd>One millionth of a second, also one thousandth of a millisecond,
1901 0.000,001 seconds, and 1e-6 seconds.</dd>
1902 </dl>
1903
1904 <br>
1905 <br>
1906 <br>
1907 <div style="float:left">
1908 This page autogenerated by
1909 <a href="https://docs.ntpsec.org/latest/ntpviz.html">
1910 ntpviz</a>, part of the <a href="https://www.ntpsec.org/">NTPsec project</a>
1911 </div>
1912 <div style="float:left;margin-left:350px;">
1913     <a href="https://validator.w3.org/nu/">
1914     <img src="https://www.w3.org/html/logo/downloads/HTML5_Logo_32.png"
1915         alt="html 5">
1916     </a>
1917 &nbsp;&nbsp;
1918     <a href="https://jigsaw.w3.org/css-validator/check/referer">
1919         <img style="border:0;width:88px;height:31px"
1920             src="https://jigsaw.w3.org/css-validator/images/vcss"
1921             alt="Valid CSS!" />
1922     </a>
1923 </div>
1924 <div style="clear:both;"></div>
1925 </div>
1926 </body>
1927 </html>
1928 '''
1929     imagewrapper = "<img src='%%s%s' alt='%%s plot'>\n" % args.img_ext
1930
1931     # buffer the index.html output so the index.html is not empty
1932     # during the run
1933     index_buffer = index_header
1934     # if header file, add it to index.html
1935     header = os.path.join(args.outdir, "header")
1936     if os.path.isfile(header):
1937         try:
1938             header_file = open(header, 'r', encoding='utf-8')
1939             header_txt = header_file.read()
1940             index_buffer += '<br>\n' + header_txt + '\n'
1941         except IOError:
1942             pass
1943
1944     if len(statlist) > 1:
1945         index_buffer += local_offset_multiplot(statlist)
1946     else:
1947         # imagepairs in the order of the html entries
1948         imagepairs = [
1949             ("local-offset", stats.local_offset_gnuplot()),
1950             # skipa next one, redundant to one above
1951             # ("local-error", stats.local_error_gnuplot()),
1952             ("local-jitter", stats.local_offset_jitter_gnuplot()),
1953             ("local-stability", stats.local_offset_stability_gnuplot()),
1954             ("local-offset-histogram", stats.local_offset_histogram_gnuplot()),
1955             ("local-temps", stats.local_temps_gnuplot()),
1956             ("local-freq-temps", stats.local_freq_temps_plot()),
1957             ("local-gps", stats.local_gps_gnuplot()),
1958             ("peer-offsets", stats.peer_offsets_gnuplot()),
1959         ]
1960
1961         peerlist = list(stats.peersplit().keys())
1962         # sort for output order stability
1963         peerlist.sort()
1964         for key in peerlist:
1965             imagepairs.append(("peer-offset-" + key,
1966                                stats.peer_offsets_gnuplot([key])))
1967
1968         imagepairs.append(("peer-jitters",
1969                            stats.peer_jitters_gnuplot()))
1970         for key in peerlist:
1971             imagepairs.append(("peer-jitter-" + key,
1972                                stats.peer_jitters_gnuplot([key])))
1973
1974         stats = []
1975         for (imagename, image) in imagepairs:
1976             if not image:
1977                 continue
1978             if 1 <= args.debug_level:
1979                 sys.stderr.write("ntpviz: plotting %s\n" % image['title'])
1980             stats.append(image['stats'])
1981             # give each H2 an unique ID.
1982             div_id = image['title'].lower().replace(' ', '_').replace(':', '_')
1983
1984             index_buffer += """\
1985 <div id="%s">\n<h2><a class="section" href="#%s">%s</a></h2>
1986 """ % (div_id, div_id, image['title'])
1987
1988             div_name = imagename.replace('-', ' ')
1989             # Windows hates colons in filename
1990             imagename = imagename.replace(':', '-')
1991             index_buffer += imagewrapper % (imagename, div_name)
1992
1993             if image['html']:
1994                 index_buffer += "<div>\n%s</div>\n" % image['html']
1995             index_buffer += "<br><br>\n"
1996             gnuplot(image['plot'], os.path.join(args.outdir,
1997                     imagename + args.img_ext))
1998             index_buffer += "</div>\n"
1999
2000     # dump stats
2001     csvs = []
2002
2003     stats_to_output = {}
2004     for stat in stats:
2005         if [] == stat:
2006             continue
2007         for sta in stat:
2008             if sta.skip_summary:
2009                 continue
2010             # This removes duplicates
2011             stats_to_output[sta.title] = sta
2012
2013     index_buffer += '<div id="Summary">\n' \
2014         '<h2><a class="section" href="#Summary">Summary</a></h2>\n'
2015     index_buffer += VizStats.table_head
2016
2017     for key in sorted(stats_to_output.keys()):
2018         index_buffer += str(stats_to_output[key].table)
2019         csvs.append(stats_to_output[key].csv)
2020
2021     # RFC 4180 specifies the mime-type of a csv: text/csv
2022     # your webserver should be programmed the same
2023     index_buffer += VizStats.table_tail
2024     index_buffer += """\
2025 <a href="summary.csv" target="_blank"
2026   type="text/csv;charset=UTF-8;header=present">Summary as CSV file</a><br>
2027 </div>
2028 """
2029
2030     # if footer file, add it to index.html
2031     footer = os.path.join(args.outdir, "footer")
2032     if os.path.isfile(footer):
2033         try:
2034             footer_file = open(footer, 'r', encoding='utf-8')
2035             footer_txt = footer_file.read()
2036             index_buffer += '<br>\n' + footer_txt + '\n'
2037         except IOError:
2038             pass
2039     index_buffer += index_trailer
2040
2041     # and send the file buffer
2042     index_filename = os.path.join(args.outdir, "index.html")
2043     with open(index_filename + ".tmp", "w", encoding='utf-8') as ifile:
2044         ifile.write(index_buffer)
2045
2046     # create csv file, as a tmp file
2047     csv_filename = os.path.join(args.outdir, "summary.csv")
2048     with open(csv_filename + ".tmp", "w", encoding='utf-8') as csv_file:
2049         csv_ob = csv.writer(csv_file)
2050         csv_ob.writerow(VizStats.csv_head)
2051         for row in csvs:
2052             csv_ob.writerow(row)
2053
2054     # move new index and summary into place
2055     # windows python 2.7, 3.6 has no working rename, so delete and move
2056     try:
2057         os.remove(csv_filename)
2058         os.remove(index_filename)
2059     except OSError:
2060         pass
2061
2062     os.rename(csv_filename + ".tmp", csv_filename)
2063     os.rename(index_filename + ".tmp", index_filename)
2064
2065
2066 # end