sys/kern/kern_cctr.c

   1 /*      $NetBSD: kern_cctr.c,v 1.8 2008/05/19 17:06:02 ad Exp $ */
   2
   3 /*-
   4  * Copyright (c) 2006, 2008 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * re-implementation of TSC for MP systems merging cc_microtime and
   8  * TSC for timecounters by Frank Kardel
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 /* basic calibration ideas are (kern_microtime.c): */
  33 /******************************************************************************
  34  *                                                                            *
  35  * Copyright (c) David L. Mills 1993, 1994                                    *
  36  *                                                                            *
  37  * Permission to use, copy, modify, and distribute this software and its      *
  38  * documentation for any purpose and without fee is hereby granted, provided  *
  39  * that the above copyright notice appears in all copies and that both the    *
  40  * copyright notice and this permission notice appear in supporting           *
  41  * documentation, and that the name University of Delaware not be used in     *
  42  * advertising or publicity pertaining to distribution of the software        *
  43  * without specific, written prior permission.  The University of Delaware    *
  44  * makes no representations about the suitability this software for any       *
  45  * purpose.  It is provided "as is" without express or implied warranty.      *
  46  *                                                                            *
  47  ******************************************************************************/
  48
  49 /* reminiscents from older version of this file are: */
  50 /*-
  51  * Copyright (c) 1998-2003 Poul-Henning Kamp
  52  * All rights reserved.
  53  *
  54  * Redistribution and use in source and binary forms, with or without
  55  * modification, are permitted provided that the following conditions
  56  * are met:
  57  * 1. Redistributions of source code must retain the above copyright
  58  *    notice, this list of conditions and the following disclaimer.
  59  * 2. Redistributions in binary form must reproduce the above copyright
  60  *    notice, this list of conditions and the following disclaimer in the
  61  *    documentation and/or other materials provided with the distribution.
  62  *
  63  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  64  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  65  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  66  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  67  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  68  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  69  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  70  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  71  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  72  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  73  * SUCH DAMAGE.
  74  */
  75
  76 #include <sys/cdefs.h>
  77 /* __FBSDID("$FreeBSD: src/sys/i386/i386/tsc.c,v 1.204 2003/10/21 18:28:34 silby Exp $"); */
  78 __KERNEL_RCSID(0, "$NetBSD: kern_cctr.c,v 1.8 2008/05/19 17:06:02 ad Exp $");
  79
  80 #include <sys/param.h>
  81 #include <sys/systm.h>
  82 #include <sys/sysctl.h>
  83 #include <sys/time.h>
  84 #include <sys/timetc.h>
  85 #include <sys/kernel.h>
  86 #include <sys/power.h>
  87 #include <sys/cpu.h>
  88 #include <machine/cpu_counter.h>
  89
  90 /* XXX make cc_timecounter.tc_frequency settable by sysctl() */
  91
  92 static timecounter_pps_t cc_calibrate;
  93
  94 void cc_calibrate_cpu(struct cpu_info *);
  95
  96 static int64_t cc_cal_val;  /* last calibrate time stamp */
  97
  98 static struct timecounter cc_timecounter = {
  99         .tc_get_timecount       = cc_get_timecount,
 100         .tc_poll_pps            = cc_calibrate,
 101         .tc_counter_mask        = ~0u,
 102         .tc_frequency           = 0,
 103         .tc_name                = "unkown cycle counter",
 104         /*
 105          * don't pick cycle counter automatically
 106          * if frequency changes might affect cycle counter
 107          */
 108         .tc_quality             = -100000,
 109
 110         .tc_priv                = NULL,
 111         .tc_next                = NULL
 112 };
 113
 114 /*
 115  * initialize cycle counter based timecounter
 116  */
 117 struct timecounter *
 118 cc_init(timecounter_get_t getcc, uint64_t freq, const char *name, int quality)
 119 {
 120
 121         if (getcc != NULL)
 122                 cc_timecounter.tc_get_timecount = getcc;
 123
 124         cc_timecounter.tc_frequency = freq;
 125         cc_timecounter.tc_name = name;
 126         cc_timecounter.tc_quality = quality;
 127         tc_init(&cc_timecounter);
 128
 129         return &cc_timecounter;
 130 }
 131
 132 /*
 133  * pick up tick count scaled to reference tick count
 134  */
 135 u_int
 136 cc_get_timecount(struct timecounter *tc)
 137 {
 138         struct cpu_info *ci;
 139         int64_t rcc, cc, ncsw;
 140         u_int gen;
 141
 142  retry:
 143         ncsw = curlwp->l_ncsw;
 144         __insn_barrier();
 145         ci = curcpu();
 146         if (ci->ci_cc.cc_denom == 0) {
 147                 /*
 148                  * This is our first time here on this CPU.  Just
 149                  * start with reasonable initial values.
 150                  */
 151                 ci->ci_cc.cc_cc    = cpu_counter32();
 152                 ci->ci_cc.cc_val   = 0;
 153                 if (ci->ci_cc.cc_gen == 0)
 154                         ci->ci_cc.cc_gen++;
 155
 156                 ci->ci_cc.cc_denom = cpu_frequency(ci);
 157                 if (ci->ci_cc.cc_denom == 0)
 158                         ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
 159                 ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
 160         }
 161
 162         /*
 163          * read counter and re-read when the re-calibration
 164          * strikes inbetween
 165          */
 166         do {
 167                 /* pick up current generation number */
 168                 gen = ci->ci_cc.cc_gen;
 169
 170                 /* determine local delta ticks */
 171                 cc = cpu_counter32() - ci->ci_cc.cc_cc;
 172                 if (cc < 0)
 173                         cc += 0x100000000LL;
 174
 175                 /* scale to primary */
 176                 rcc = (cc * ci->ci_cc.cc_delta) / ci->ci_cc.cc_denom
 177                     + ci->ci_cc.cc_val;
 178         } while (gen == 0 || gen != ci->ci_cc.cc_gen);
 179         __insn_barrier();
 180         if (ncsw != curlwp->l_ncsw) {
 181                 /* Was preempted */
 182                 goto retry;
 183         }
 184
 185         return rcc;
 186 }
 187
 188 /*
 189  * called once per clock tick via the pps callback
 190  * for the calibration of the TSC counters.
 191  * it is called only for the PRIMARY cpu. all
 192  * other cpus are called via a broadcast IPI
 193  * calibration interval is 1 second - we call
 194  * the calibration code only every hz calls
 195  */
 196 static void
 197 cc_calibrate(struct timecounter *tc)
 198 {
 199         static int calls;
 200         struct cpu_info *ci;
 201
 202         KASSERT(kpreempt_disabled());
 203
 204          /*
 205           * XXX: for high interrupt frequency
 206           * support: ++calls < hz / tc_tick
 207           */
 208         if (++calls < hz)
 209                 return;
 210
 211         calls = 0;
 212         ci = curcpu();
 213         /* pick up reference ticks */
 214         cc_cal_val = cpu_counter32();
 215
 216 #if defined(MULTIPROCESSOR)
 217         cc_calibrate_mp(ci);
 218 #endif
 219         cc_calibrate_cpu(ci);
 220 }
 221
 222 /*
 223  * This routine is called about once per second directly by the master
 224  * processor and via an interprocessor interrupt for other processors.
 225  * It determines the CC frequency of each processor relative to the
 226  * master clock and the time this determination is made.  These values
 227  * are used by cc_get_timecount() to interpolate the ticks between
 228  * timer interrupts.  Note that we assume the kernel variables have
 229  * been zeroed early in life.
 230  */
 231 void
 232 cc_calibrate_cpu(struct cpu_info *ci)
 233 {
 234         u_int   gen;
 235         int64_t val;
 236         int64_t delta, denom;
 237         int s;
 238 #ifdef TIMECOUNTER_DEBUG
 239         int64_t factor, old_factor;
 240 #endif
 241         val = cc_cal_val;
 242
 243         s = splhigh();
 244         /* create next generation number */
 245         gen = ci->ci_cc.cc_gen;
 246         gen++;
 247         if (gen == 0)
 248                 gen++;
 249
 250         /* update in progress */
 251         ci->ci_cc.cc_gen = 0;
 252
 253         denom = ci->ci_cc.cc_cc;
 254         ci->ci_cc.cc_cc = cpu_counter32();
 255
 256         if (ci->ci_cc.cc_denom == 0) {
 257                 /*
 258                  * This is our first time here on this CPU.  Just
 259                  * start with reasonable initial values.
 260                  */
 261                 ci->ci_cc.cc_val = val;
 262                 ci->ci_cc.cc_denom = cpu_frequency(ci);
 263                 if (ci->ci_cc.cc_denom == 0)
 264                         ci->ci_cc.cc_denom = cc_timecounter.tc_frequency;
 265                 ci->ci_cc.cc_delta = ci->ci_cc.cc_denom;
 266                 ci->ci_cc.cc_gen = gen;
 267                 splx(s);
 268                 return;
 269         }
 270
 271 #ifdef TIMECOUNTER_DEBUG
 272         old_factor = (ci->ci_cc.cc_delta * 1000 ) / ci->ci_cc.cc_denom;
 273 #endif
 274
 275         /* local ticks per period */
 276         denom = ci->ci_cc.cc_cc - denom;
 277         if (denom < 0)
 278                 denom += 0x100000000LL;
 279
 280         ci->ci_cc.cc_denom = denom;
 281
 282         /* reference ticks per period */
 283         delta = val - ci->ci_cc.cc_val;
 284         if (delta < 0)
 285                 delta += 0x100000000LL;
 286
 287         ci->ci_cc.cc_val = val;
 288         ci->ci_cc.cc_delta = delta;
 289
 290         /* publish new generation number */
 291         ci->ci_cc.cc_gen = gen;
 292         splx(s);
 293
 294 #ifdef TIMECOUNTER_DEBUG
 295         factor = (delta * 1000) / denom - old_factor;
 296         if (factor < 0)
 297                 factor = -factor;
 298
 299         if (factor > old_factor / 10)
 300                 printf("cc_calibrate_cpu[%u]: 10%% exceeded - delta %"
 301                     PRId64 ", denom %" PRId64 ", factor %" PRId64
 302                     ", old factor %" PRId64"\n", ci->ci_index,
 303                     delta, denom, (delta * 1000) / denom, old_factor);
 304 #endif /* TIMECOUNTER_DEBUG */
 305 }