Documentation/ia64/err_inject.txt

   1
   2 IPF Machine Check (MC) error inject tool
   3 ========================================
   4
   5 IPF Machine Check (MC) error inject tool is used to inject MC
   6 errors from Linux. The tool is a test bed for IPF MC work flow including
   7 hardware correctable error handling, OS recoverable error handling, MC
   8 event logging, etc.
   9
  10 The tool includes two parts: a kernel driver and a user application
  11 sample. The driver provides interface to PAL to inject error
  12 and query error injection capabilities. The driver code is in
  13 arch/ia64/kernel/err_inject.c. The application sample (shown below)
  14 provides a combination of various errors and calls the driver's interface
  15 (sysfs interface) to inject errors or query error injection capabilities.
  16
  17 The tool can be used to test Intel IPF machine MC handling capabilities.
  18 It's especially useful for people who can not access hardware MC injection
  19 tool to inject error. It's also very useful to integrate with other
  20 software test suits to do stressful testing on IPF.
  21
  22 Below is a sample application as part of the whole tool. The sample
  23 can be used as a working test tool. Or it can be expanded to include
  24 more features. It also can be a integrated into a library or other user
  25 application to have more thorough test.
  26
  27 The sample application takes err.conf as error configuration input. GCC
  28 compiles the code. After you install err_inject driver, you can run
  29 this sample application to inject errors.
  30
  31 Errata: Itanium 2 Processors Specification Update lists some errata against
  32 the pal_mc_error_inject PAL procedure. The following err.conf has been tested
  33 on latest Montecito PAL.
  34
  35 err.conf:
  36
  37 #This is configuration file for err_inject_tool.
  38 #The format of the each line is:
  39 #cpu, loop, interval, err_type_info, err_struct_info, err_data_buffer
  40 #where
  41 #       cpu: logical cpu number the error will be inject in.
  42 #       loop: times the error will be injected.
  43 #       interval: In second. every so often one error is injected.
  44 #       err_type_info, err_struct_info: PAL parameters.
  45 #
  46 #Note: All values are hex w/o or w/ 0x prefix.
  47
  48
  49 #On cpu2, inject only total 0x10 errors, interval 5 seconds
  50 #corrected, data cache, hier-2, physical addr(assigned by tool code).
  51 #working on Montecito latest PAL.
  52 2, 10, 5, 4101, 95
  53
  54 #On cpu4, inject and consume total 0x10 errors, interval 5 seconds
  55 #corrected, data cache, hier-2, physical addr(assigned by tool code).
  56 #working on Montecito latest PAL.
  57 4, 10, 5, 4109, 95
  58
  59 #On cpu15, inject and consume total 0x10 errors, interval 5 seconds
  60 #recoverable, DTR0, hier-2.
  61 #working on Montecito latest PAL.
  62 0xf, 0x10, 5, 4249, 15
  63
  64 The sample application source code:
  65
  66 err_injection_tool.c:
  67
  68 /*
  69  * This program is free software; you can redistribute it and/or modify
  70  * it under the terms of the GNU General Public License as published by
  71  * the Free Software Foundation; either version 2 of the License, or
  72  * (at your option) any later version.
  73  *
  74  * This program is distributed in the hope that it will be useful, but
  75  * WITHOUT ANY WARRANTY; without even the implied warranty of
  76  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  77  * NON INFRINGEMENT.  See the GNU General Public License for more
  78  * details.
  79  *
  80  * You should have received a copy of the GNU General Public License
  81  * along with this program; if not, write to the Free Software
  82  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  83  *
  84  * Copyright (C) 2006 Intel Co
  85  *      Fenghua Yu <fenghua.yu@intel.com>
  86  *
  87  */
  88 #include <sys/types.h>
  89 #include <sys/stat.h>
  90 #include <fcntl.h>
  91 #include <stdio.h>
  92 #include <sched.h>
  93 #include <unistd.h>
  94 #include <stdlib.h>
  95 #include <stdarg.h>
  96 #include <string.h>
  97 #include <errno.h>
  98 #include <time.h>
  99 #include <sys/ipc.h>
 100 #include <sys/sem.h>
 101 #include <sys/wait.h>
 102 #include <sys/mman.h>
 103 #include <sys/shm.h>
 104
 105 #define MAX_FN_SIZE             256
 106 #define MAX_BUF_SIZE            256
 107 #define DATA_BUF_SIZE           256
 108 #define NR_CPUS                 512
 109 #define MAX_TASK_NUM            2048
 110 #define MIN_INTERVAL            5       // seconds
 111 #define ERR_DATA_BUFFER_SIZE    3       // Three 8-byte.
 112 #define PARA_FIELD_NUM          5
 113 #define MASK_SIZE               (NR_CPUS/64)
 114 #define PATH_FORMAT "/sys/devices/system/cpu/cpu%d/err_inject/"
 115
 116 int sched_setaffinity(pid_t pid, unsigned int len, unsigned long *mask);
 117
 118 int verbose;
 119 #define vbprintf if (verbose) printf
 120
 121 int log_info(int cpu, const char *fmt, ...)
 122 {
 123         FILE *log;
 124         char fn[MAX_FN_SIZE];
 125         char buf[MAX_BUF_SIZE];
 126         va_list args;
 127
 128         sprintf(fn, "%d.log", cpu);
 129         log=fopen(fn, "a+");
 130         if (log==NULL) {
 131                 perror("Error open:");
 132                 return -1;
 133         }
 134
 135         va_start(args, fmt);
 136         vprintf(fmt, args);
 137         memset(buf, 0, MAX_BUF_SIZE);
 138         vsprintf(buf, fmt, args);
 139         va_end(args);
 140
 141         fwrite(buf, sizeof(buf), 1, log);
 142         fclose(log);
 143
 144         return 0;
 145 }
 146
 147 typedef unsigned long u64;
 148 typedef unsigned int  u32;
 149
 150 typedef union err_type_info_u {
 151         struct {
 152                 u64     mode            : 3,    /* 0-2 */
 153                         err_inj         : 3,    /* 3-5 */
 154                         err_sev         : 2,    /* 6-7 */
 155                         err_struct      : 5,    /* 8-12 */
 156                         struct_hier     : 3,    /* 13-15 */
 157                         reserved        : 48;   /* 16-63 */
 158         } err_type_info_u;
 159         u64     err_type_info;
 160 } err_type_info_t;
 161
 162 typedef union err_struct_info_u {
 163         struct {
 164                 u64     siv             : 1,    /* 0     */
 165                         c_t             : 2,    /* 1-2   */
 166                         cl_p            : 3,    /* 3-5   */
 167                         cl_id           : 3,    /* 6-8   */
 168                         cl_dp           : 1,    /* 9     */
 169                         reserved1       : 22,   /* 10-31 */
 170                         tiv             : 1,    /* 32    */
 171                         trigger         : 4,    /* 33-36 */
 172                         trigger_pl      : 3,    /* 37-39 */
 173                         reserved2       : 24;   /* 40-63 */
 174         } err_struct_info_cache;
 175         struct {
 176                 u64     siv             : 1,    /* 0     */
 177                         tt              : 2,    /* 1-2   */
 178                         tc_tr           : 2,    /* 3-4   */
 179                         tr_slot         : 8,    /* 5-12  */
 180                         reserved1       : 19,   /* 13-31 */
 181                         tiv             : 1,    /* 32    */
 182                         trigger         : 4,    /* 33-36 */
 183                         trigger_pl      : 3,    /* 37-39 */
 184                         reserved2       : 24;   /* 40-63 */
 185         } err_struct_info_tlb;
 186         struct {
 187                 u64     siv             : 1,    /* 0     */
 188                         regfile_id      : 4,    /* 1-4   */
 189                         reg_num         : 7,    /* 5-11  */
 190                         reserved1       : 20,   /* 12-31 */
 191                         tiv             : 1,    /* 32    */
 192                         trigger         : 4,    /* 33-36 */
 193                         trigger_pl      : 3,    /* 37-39 */
 194                         reserved2       : 24;   /* 40-63 */
 195         } err_struct_info_register;
 196         struct {
 197                 u64     reserved;
 198         } err_struct_info_bus_processor_interconnect;
 199         u64     err_struct_info;
 200 } err_struct_info_t;
 201
 202 typedef union err_data_buffer_u {
 203         struct {
 204                 u64     trigger_addr;           /* 0-63         */
 205                 u64     inj_addr;               /* 64-127       */
 206                 u64     way             : 5,    /* 128-132      */
 207                         index           : 20,   /* 133-152      */
 208                                         : 39;   /* 153-191      */
 209         } err_data_buffer_cache;
 210         struct {
 211                 u64     trigger_addr;           /* 0-63         */
 212                 u64     inj_addr;               /* 64-127       */
 213                 u64     way             : 5,    /* 128-132      */
 214                         index           : 20,   /* 133-152      */
 215                         reserved        : 39;   /* 153-191      */
 216         } err_data_buffer_tlb;
 217         struct {
 218                 u64     trigger_addr;           /* 0-63         */
 219         } err_data_buffer_register;
 220         struct {
 221                 u64     reserved;               /* 0-63         */
 222         } err_data_buffer_bus_processor_interconnect;
 223         u64 err_data_buffer[ERR_DATA_BUFFER_SIZE];
 224 } err_data_buffer_t;
 225
 226 typedef union capabilities_u {
 227         struct {
 228                 u64     i               : 1,
 229                         d               : 1,
 230                         rv              : 1,
 231                         tag             : 1,
 232                         data            : 1,
 233                         mesi            : 1,
 234                         dp              : 1,
 235                         reserved1       : 3,
 236                         pa              : 1,
 237                         va              : 1,
 238                         wi              : 1,
 239                         reserved2       : 20,
 240                         trigger         : 1,
 241                         trigger_pl      : 1,
 242                         reserved3       : 30;
 243         } capabilities_cache;
 244         struct {
 245                 u64     d               : 1,
 246                         i               : 1,
 247                         rv              : 1,
 248                         tc              : 1,
 249                         tr              : 1,
 250                         reserved1       : 27,
 251                         trigger         : 1,
 252                         trigger_pl      : 1,
 253                         reserved2       : 30;
 254         } capabilities_tlb;
 255         struct {
 256                 u64     gr_b0           : 1,
 257                         gr_b1           : 1,
 258                         fr              : 1,
 259                         br              : 1,
 260                         pr              : 1,
 261                         ar              : 1,
 262                         cr              : 1,
 263                         rr              : 1,
 264                         pkr             : 1,
 265                         dbr             : 1,
 266                         ibr             : 1,
 267                         pmc             : 1,
 268                         pmd             : 1,
 269                         reserved1       : 3,
 270                         regnum          : 1,
 271                         reserved2       : 15,
 272                         trigger         : 1,
 273                         trigger_pl      : 1,
 274                         reserved3       : 30;
 275         } capabilities_register;
 276         struct {
 277                 u64     reserved;
 278         } capabilities_bus_processor_interconnect;
 279 } capabilities_t;
 280
 281 typedef struct resources_s {
 282         u64     ibr0            : 1,
 283                 ibr2            : 1,
 284                 ibr4            : 1,
 285                 ibr6            : 1,
 286                 dbr0            : 1,
 287                 dbr2            : 1,
 288                 dbr4            : 1,
 289                 dbr6            : 1,
 290                 reserved        : 48;
 291 } resources_t;
 292
 293
 294 long get_page_size(void)
 295 {
 296         long page_size=sysconf(_SC_PAGESIZE);
 297         return page_size;
 298 }
 299
 300 #define PAGE_SIZE (get_page_size()==-1?0x4000:get_page_size())
 301 #define SHM_SIZE (2*PAGE_SIZE*NR_CPUS)
 302 #define SHM_VA 0x2000000100000000
 303
 304 int shmid;
 305 void *shmaddr;
 306
 307 int create_shm(void)
 308 {
 309         key_t key;
 310         char fn[MAX_FN_SIZE];
 311
 312         /* cpu0 is always existing */
 313         sprintf(fn, PATH_FORMAT, 0);
 314         if ((key = ftok(fn, 's')) == -1) {
 315                 perror("ftok");
 316                 return -1;
 317         }
 318
 319         shmid = shmget(key, SHM_SIZE, 0644 | IPC_CREAT);
 320         if (shmid == -1) {
 321                 if (errno==EEXIST) {
 322                         shmid = shmget(key, SHM_SIZE, 0);
 323                         if (shmid == -1) {
 324                                 perror("shmget");
 325                                 return -1;
 326                         }
 327                 }
 328                 else {
 329                         perror("shmget");
 330                         return -1;
 331                 }
 332         }
 333         vbprintf("shmid=%d", shmid);
 334
 335         /* connect to the segment: */
 336         shmaddr = shmat(shmid, (void *)SHM_VA, 0);
 337         if (shmaddr == (void*)-1) {
 338                 perror("shmat");
 339                 return -1;
 340         }
 341
 342         memset(shmaddr, 0, SHM_SIZE);
 343         mlock(shmaddr, SHM_SIZE);
 344
 345         return 0;
 346 }
 347
 348 int free_shm()
 349 {
 350         munlock(shmaddr, SHM_SIZE);
 351         shmdt(shmaddr);
 352         semctl(shmid, 0, IPC_RMID);
 353
 354         return 0;
 355 }
 356
 357 #ifdef _SEM_SEMUN_UNDEFINED
 358 union semun
 359 {
 360         int val;
 361         struct semid_ds *buf;
 362         unsigned short int *array;
 363         struct seminfo *__buf;
 364 };
 365 #endif
 366
 367 u32 mode=1; /* 1: physical mode; 2: virtual mode. */
 368 int one_lock=1;
 369 key_t key[NR_CPUS];
 370 int semid[NR_CPUS];
 371
 372 int create_sem(int cpu)
 373 {
 374         union semun arg;
 375         char fn[MAX_FN_SIZE];
 376         int sid;
 377
 378         sprintf(fn, PATH_FORMAT, cpu);
 379         sprintf(fn, "%s/%s", fn, "err_type_info");
 380         if ((key[cpu] = ftok(fn, 'e')) == -1) {
 381                 perror("ftok");
 382                 return -1;
 383         }
 384
 385         if (semid[cpu]!=0)
 386                 return 0;
 387
 388         /* clear old semaphore */
 389         if ((sid = semget(key[cpu], 1, 0)) != -1)
 390                 semctl(sid, 0, IPC_RMID);
 391
 392         /* get one semaphore */
 393         if ((semid[cpu] = semget(key[cpu], 1, IPC_CREAT | IPC_EXCL)) == -1) {
 394                 perror("semget");
 395                 printf("Please remove semaphore with key=0x%lx, then run the tool.\n",
 396                         (u64)key[cpu]);
 397                 return -1;
 398         }
 399
 400         vbprintf("semid[%d]=0x%lx, key[%d]=%lx\n",cpu,(u64)semid[cpu],cpu,
 401                 (u64)key[cpu]);
 402         /* initialize the semaphore to 1: */
 403         arg.val = 1;
 404         if (semctl(semid[cpu], 0, SETVAL, arg) == -1) {
 405                 perror("semctl");
 406                 return -1;
 407         }
 408
 409         return 0;
 410 }
 411
 412 static int lock(int cpu)
 413 {
 414         struct sembuf lock;
 415
 416         lock.sem_num = cpu;
 417         lock.sem_op = 1;
 418         semop(semid[cpu], &lock, 1);
 419
 420         return 0;
 421 }
 422
 423 static int unlock(int cpu)
 424 {
 425         struct sembuf unlock;
 426
 427         unlock.sem_num = cpu;
 428         unlock.sem_op = -1;
 429         semop(semid[cpu], &unlock, 1);
 430
 431         return 0;
 432 }
 433
 434 void free_sem(int cpu)
 435 {
 436         semctl(semid[cpu], 0, IPC_RMID);
 437 }
 438
 439 int wr_multi(char *fn, unsigned long *data, int size)
 440 {
 441         int fd;
 442         char buf[MAX_BUF_SIZE];
 443         int ret;
 444
 445         if (size==1)
 446                 sprintf(buf, "%lx", *data);
 447         else if (size==3)
 448                 sprintf(buf, "%lx,%lx,%lx", data[0], data[1], data[2]);
 449         else {
 450                 fprintf(stderr,"write to file with wrong size!\n");
 451                 return -1;
 452         }
 453
 454         fd=open(fn, O_RDWR);
 455         if (!fd) {
 456                 perror("Error:");
 457                 return -1;
 458         }
 459         ret=write(fd, buf, sizeof(buf));
 460         close(fd);
 461         return ret;
 462 }
 463
 464 int wr(char *fn, unsigned long data)
 465 {
 466         return wr_multi(fn, &data, 1);
 467 }
 468
 469 int rd(char *fn, unsigned long *data)
 470 {
 471         int fd;
 472         char buf[MAX_BUF_SIZE];
 473
 474         fd=open(fn, O_RDONLY);
 475         if (fd<0) {
 476                 perror("Error:");
 477                 return -1;
 478         }
 479         read(fd, buf, MAX_BUF_SIZE);
 480         *data=strtoul(buf, NULL, 16);
 481         close(fd);
 482         return 0;
 483 }
 484
 485 int rd_status(char *path, int *status)
 486 {
 487         char fn[MAX_FN_SIZE];
 488         sprintf(fn, "%s/status", path);
 489         if (rd(fn, (u64*)status)<0) {
 490                 perror("status reading error.\n");
 491                 return -1;
 492         }
 493
 494         return 0;
 495 }
 496
 497 int rd_capabilities(char *path, u64 *capabilities)
 498 {
 499         char fn[MAX_FN_SIZE];
 500         sprintf(fn, "%s/capabilities", path);
 501         if (rd(fn, capabilities)<0) {
 502                 perror("capabilities reading error.\n");
 503                 return -1;
 504         }
 505
 506         return 0;
 507 }
 508
 509 int rd_all(char *path)
 510 {
 511         unsigned long err_type_info, err_struct_info, err_data_buffer;
 512         int status;
 513         unsigned long capabilities, resources;
 514         char fn[MAX_FN_SIZE];
 515
 516         sprintf(fn, "%s/err_type_info", path);
 517         if (rd(fn, &err_type_info)<0) {
 518                 perror("err_type_info reading error.\n");
 519                 return -1;
 520         }
 521         printf("err_type_info=%lx\n", err_type_info);
 522
 523         sprintf(fn, "%s/err_struct_info", path);
 524         if (rd(fn, &err_struct_info)<0) {
 525                 perror("err_struct_info reading error.\n");
 526                 return -1;
 527         }
 528         printf("err_struct_info=%lx\n", err_struct_info);
 529
 530         sprintf(fn, "%s/err_data_buffer", path);
 531         if (rd(fn, &err_data_buffer)<0) {
 532                 perror("err_data_buffer reading error.\n");
 533                 return -1;
 534         }
 535         printf("err_data_buffer=%lx\n", err_data_buffer);
 536
 537         sprintf(fn, "%s/status", path);
 538         if (rd("status", (u64*)&status)<0) {
 539                 perror("status reading error.\n");
 540                 return -1;
 541         }
 542         printf("status=%d\n", status);
 543
 544         sprintf(fn, "%s/capabilities", path);
 545         if (rd(fn,&capabilities)<0) {
 546                 perror("capabilities reading error.\n");
 547                 return -1;
 548         }
 549         printf("capabilities=%lx\n", capabilities);
 550
 551         sprintf(fn, "%s/resources", path);
 552         if (rd(fn, &resources)<0) {
 553                 perror("resources reading error.\n");
 554                 return -1;
 555         }
 556         printf("resources=%lx\n", resources);
 557
 558         return 0;
 559 }
 560
 561 int query_capabilities(char *path, err_type_info_t err_type_info,
 562                         u64 *capabilities)
 563 {
 564         char fn[MAX_FN_SIZE];
 565         err_struct_info_t err_struct_info;
 566         err_data_buffer_t err_data_buffer;
 567
 568         err_struct_info.err_struct_info=0;
 569         memset(err_data_buffer.err_data_buffer, -1, ERR_DATA_BUFFER_SIZE*8);
 570
 571         sprintf(fn, "%s/err_type_info", path);
 572         wr(fn, err_type_info.err_type_info);
 573         sprintf(fn, "%s/err_struct_info", path);
 574         wr(fn, 0x0);
 575         sprintf(fn, "%s/err_data_buffer", path);
 576         wr_multi(fn, err_data_buffer.err_data_buffer, ERR_DATA_BUFFER_SIZE);
 577
 578         // Fire pal_mc_error_inject procedure.
 579         sprintf(fn, "%s/call_start", path);
 580         wr(fn, mode);
 581
 582         if (rd_capabilities(path, capabilities)<0)
 583                 return -1;
 584
 585         return 0;
 586 }
 587
 588 int query_all_capabilities()
 589 {
 590         int status;
 591         err_type_info_t err_type_info;
 592         int err_sev, err_struct, struct_hier;
 593         int cap=0;
 594         u64 capabilities;
 595         char path[MAX_FN_SIZE];
 596
 597         err_type_info.err_type_info=0;                  // Initial
 598         err_type_info.err_type_info_u.mode=0;           // Query mode;
 599         err_type_info.err_type_info_u.err_inj=0;
 600
 601         printf("All capabilities implemented in pal_mc_error_inject:\n");
 602         sprintf(path, PATH_FORMAT ,0);
 603         for (err_sev=0;err_sev<3;err_sev++)
 604                 for (err_struct=0;err_struct<5;err_struct++)
 605                         for (struct_hier=0;struct_hier<5;struct_hier++)
 606         {
 607                 status=-1;
 608                 capabilities=0;
 609                 err_type_info.err_type_info_u.err_sev=err_sev;
 610                 err_type_info.err_type_info_u.err_struct=err_struct;
 611                 err_type_info.err_type_info_u.struct_hier=struct_hier;
 612
 613                 if (query_capabilities(path, err_type_info, &capabilities)<0)
 614                         continue;
 615
 616                 if (rd_status(path, &status)<0)
 617                         continue;
 618
 619                 if (status==0) {
 620                         cap=1;
 621                         printf("For err_sev=%d, err_struct=%d, struct_hier=%d: ",
 622                                 err_sev, err_struct, struct_hier);
 623                         printf("capabilities 0x%lx\n", capabilities);
 624                 }
 625         }
 626         if (!cap) {
 627                 printf("No capabilities supported.\n");
 628                 return 0;
 629         }
 630
 631         return 0;
 632 }
 633
 634 int err_inject(int cpu, char *path, err_type_info_t err_type_info,
 635                 err_struct_info_t err_struct_info,
 636                 err_data_buffer_t err_data_buffer)
 637 {
 638         int status;
 639         char fn[MAX_FN_SIZE];
 640
 641         log_info(cpu, "err_type_info=%lx, err_struct_info=%lx, ",
 642                 err_type_info.err_type_info,
 643                 err_struct_info.err_struct_info);
 644         log_info(cpu,"err_data_buffer=[%lx,%lx,%lx]\n",
 645                 err_data_buffer.err_data_buffer[0],
 646                 err_data_buffer.err_data_buffer[1],
 647                 err_data_buffer.err_data_buffer[2]);
 648         sprintf(fn, "%s/err_type_info", path);
 649         wr(fn, err_type_info.err_type_info);
 650         sprintf(fn, "%s/err_struct_info", path);
 651         wr(fn, err_struct_info.err_struct_info);
 652         sprintf(fn, "%s/err_data_buffer", path);
 653         wr_multi(fn, err_data_buffer.err_data_buffer, ERR_DATA_BUFFER_SIZE);
 654
 655         // Fire pal_mc_error_inject procedure.
 656         sprintf(fn, "%s/call_start", path);
 657         wr(fn,mode);
 658
 659         if (rd_status(path, &status)<0) {
 660                 vbprintf("fail: read status\n");
 661                 return -100;
 662         }
 663
 664         if (status!=0) {
 665                 log_info(cpu, "fail: status=%d\n", status);
 666                 return status;
 667         }
 668
 669         return status;
 670 }
 671
 672 static int construct_data_buf(char *path, err_type_info_t err_type_info,
 673                 err_struct_info_t err_struct_info,
 674                 err_data_buffer_t *err_data_buffer,
 675                 void *va1)
 676 {
 677         char fn[MAX_FN_SIZE];
 678         u64 virt_addr=0, phys_addr=0;
 679
 680         vbprintf("va1=%lx\n", (u64)va1);
 681         memset(&err_data_buffer->err_data_buffer_cache, 0, ERR_DATA_BUFFER_SIZE*8);
 682
 683         switch (err_type_info.err_type_info_u.err_struct) {
 684                 case 1: // Cache
 685                         switch (err_struct_info.err_struct_info_cache.cl_id) {
 686                                 case 1: //Virtual addr
 687                                         err_data_buffer->err_data_buffer_cache.inj_addr=(u64)va1;
 688                                         break;
 689                                 case 2: //Phys addr
 690                                         sprintf(fn, "%s/virtual_to_phys", path);
 691                                         virt_addr=(u64)va1;
 692                                         if (wr(fn,virt_addr)<0)
 693                                                 return -1;
 694                                         rd(fn, &phys_addr);
 695                                         err_data_buffer->err_data_buffer_cache.inj_addr=phys_addr;
 696                                         break;
 697                                 default:
 698                                         printf("Not supported cl_id\n");
 699                                         break;
 700                         }
 701                         break;
 702                 case 2: //  TLB
 703                         break;
 704                 case 3: //  Register file
 705                         break;
 706                 case 4: //  Bus/system interconnect
 707                 default:
 708                         printf("Not supported err_struct\n");
 709                         break;
 710         }
 711
 712         return 0;
 713 }
 714
 715 typedef struct {
 716         u64 cpu;
 717         u64 loop;
 718         u64 interval;
 719         u64 err_type_info;
 720         u64 err_struct_info;
 721         u64 err_data_buffer[ERR_DATA_BUFFER_SIZE];
 722 } parameters_t;
 723
 724 parameters_t line_para;
 725 int para;
 726
 727 static int empty_data_buffer(u64 *err_data_buffer)
 728 {
 729         int empty=1;
 730         int i;
 731
 732         for (i=0;i<ERR_DATA_BUFFER_SIZE; i++)
 733            if (err_data_buffer[i]!=-1)
 734                 empty=0;
 735
 736         return empty;
 737 }
 738
 739 int err_inj()
 740 {
 741         err_type_info_t err_type_info;
 742         err_struct_info_t err_struct_info;
 743         err_data_buffer_t err_data_buffer;
 744         int count;
 745         FILE *fp;
 746         unsigned long cpu, loop, interval, err_type_info_conf, err_struct_info_conf;
 747         u64 err_data_buffer_conf[ERR_DATA_BUFFER_SIZE];
 748         int num;
 749         int i;
 750         char path[MAX_FN_SIZE];
 751         parameters_t parameters[MAX_TASK_NUM]={};
 752         pid_t child_pid[MAX_TASK_NUM];
 753         time_t current_time;
 754         int status;
 755
 756         if (!para) {
 757             fp=fopen("err.conf", "r");
 758             if (fp==NULL) {
 759                 perror("Error open err.conf");
 760                 return -1;
 761             }
 762
 763             num=0;
 764             while (!feof(fp)) {
 765                 char buf[256];
 766                 memset(buf,0,256);
 767                 fgets(buf, 256, fp);
 768                 count=sscanf(buf, "%lx, %lx, %lx, %lx, %lx, %lx, %lx, %lx\n",
 769                                 &cpu, &loop, &interval,&err_type_info_conf,
 770                                 &err_struct_info_conf,
 771                                 &err_data_buffer_conf[0],
 772                                 &err_data_buffer_conf[1],
 773                                 &err_data_buffer_conf[2]);
 774                 if (count!=PARA_FIELD_NUM+3) {
 775                         err_data_buffer_conf[0]=-1;
 776                         err_data_buffer_conf[1]=-1;
 777                         err_data_buffer_conf[2]=-1;
 778                         count=sscanf(buf, "%lx, %lx, %lx, %lx, %lx\n",
 779                                 &cpu, &loop, &interval,&err_type_info_conf,
 780                                 &err_struct_info_conf);
 781                         if (count!=PARA_FIELD_NUM)
 782                                 continue;
 783                 }
 784
 785                 parameters[num].cpu=cpu;
 786                 parameters[num].loop=loop;
 787                 parameters[num].interval= interval>MIN_INTERVAL
 788                                           ?interval:MIN_INTERVAL;
 789                 parameters[num].err_type_info=err_type_info_conf;
 790                 parameters[num].err_struct_info=err_struct_info_conf;
 791                 memcpy(parameters[num++].err_data_buffer,
 792                         err_data_buffer_conf,ERR_DATA_BUFFER_SIZE*8) ;
 793
 794                 if (num>=MAX_TASK_NUM)
 795                         break;
 796             }
 797         }
 798         else {
 799                 parameters[0].cpu=line_para.cpu;
 800                 parameters[0].loop=line_para.loop;
 801                 parameters[0].interval= line_para.interval>MIN_INTERVAL
 802                                           ?line_para.interval:MIN_INTERVAL;
 803                 parameters[0].err_type_info=line_para.err_type_info;
 804                 parameters[0].err_struct_info=line_para.err_struct_info;
 805                 memcpy(parameters[0].err_data_buffer,
 806                         line_para.err_data_buffer,ERR_DATA_BUFFER_SIZE*8) ;
 807
 808                 num=1;
 809         }
 810
 811         /* Create semaphore: If one_lock, one semaphore for all processors.
 812            Otherwise, one semaphore for each processor. */
 813         if (one_lock) {
 814                 if (create_sem(0)) {
 815                         printf("Can not create semaphore...exit\n");
 816                         free_sem(0);
 817                         return -1;
 818                 }
 819         }
 820         else {
 821                 for (i=0;i<num;i++) {
 822                    if (create_sem(parameters[i].cpu)) {
 823                         printf("Can not create semaphore for cpu%d...exit\n",i);
 824                         free_sem(parameters[num].cpu);
 825                         return -1;
 826                    }
 827                 }
 828         }
 829
 830         /* Create a shm segment which will be used to inject/consume errors on.*/
 831         if (create_shm()==-1) {
 832                 printf("Error to create shm...exit\n");
 833                 return -1;
 834         }
 835
 836         for (i=0;i<num;i++) {
 837                 pid_t pid;
 838
 839                 current_time=time(NULL);
 840                 log_info(parameters[i].cpu, "\nBegine at %s", ctime(&current_time));
 841                 log_info(parameters[i].cpu, "Configurations:\n");
 842                 log_info(parameters[i].cpu,"On cpu%ld: loop=%lx, interval=%lx(s)",
 843                         parameters[i].cpu,
 844                         parameters[i].loop,
 845                         parameters[i].interval);
 846                 log_info(parameters[i].cpu," err_type_info=%lx,err_struct_info=%lx\n",
 847                         parameters[i].err_type_info,
 848                         parameters[i].err_struct_info);
 849
 850                 sprintf(path, PATH_FORMAT, (int)parameters[i].cpu);
 851                 err_type_info.err_type_info=parameters[i].err_type_info;
 852                 err_struct_info.err_struct_info=parameters[i].err_struct_info;
 853                 memcpy(err_data_buffer.err_data_buffer,
 854                         parameters[i].err_data_buffer,
 855                         ERR_DATA_BUFFER_SIZE*8);
 856
 857                 pid=fork();
 858                 if (pid==0) {
 859                         unsigned long mask[MASK_SIZE];
 860                         int j, k;
 861
 862                         void *va1, *va2;
 863
 864                         /* Allocate two memory areas va1 and va2 in shm */
 865                         va1=shmaddr+parameters[i].cpu*PAGE_SIZE;
 866                         va2=shmaddr+parameters[i].cpu*PAGE_SIZE+PAGE_SIZE;
 867
 868                         vbprintf("va1=%lx, va2=%lx\n", (u64)va1, (u64)va2);
 869                         memset(va1, 0x1, PAGE_SIZE);
 870                         memset(va2, 0x2, PAGE_SIZE);
 871
 872                         if (empty_data_buffer(err_data_buffer.err_data_buffer))
 873                                 /* If not specified yet, construct data buffer
 874                                  * with va1
 875                                  */
 876                                 construct_data_buf(path, err_type_info,
 877                                         err_struct_info, &err_data_buffer,va1);
 878
 879                         for (j=0;j<MASK_SIZE;j++)
 880                                 mask[j]=0;
 881
 882                         cpu=parameters[i].cpu;
 883                         k = cpu%64;
 884                         j = cpu/64;
 885                         mask[j]=1<<k;
 886
 887                         if (sched_setaffinity(0, MASK_SIZE*8, mask)==-1) {
 888                                 perror("Error sched_setaffinity:");
 889                                 return -1;
 890                         }
 891
 892                         for (j=0; j<parameters[i].loop; j++) {
 893                                 log_info(parameters[i].cpu,"Injection ");
 894                                 log_info(parameters[i].cpu,"on cpu%ld: #%d/%ld ",
 895
 896                                         parameters[i].cpu,j+1, parameters[i].loop);
 897
 898                                 /* Hold the lock */
 899                                 if (one_lock)
 900                                         lock(0);
 901                                 else
 902                                 /* Hold lock on this cpu */
 903                                         lock(parameters[i].cpu);
 904
 905                                 if ((status=err_inject(parameters[i].cpu,
 906                                            path, err_type_info,
 907                                            err_struct_info, err_data_buffer))
 908                                            ==0) {
 909                                         /* consume the error for "inject only"*/
 910                                         memcpy(va2, va1, PAGE_SIZE);
 911                                         memcpy(va1, va2, PAGE_SIZE);
 912                                         log_info(parameters[i].cpu,
 913                                                 "successful\n");
 914                                 }
 915                                 else {
 916                                         log_info(parameters[i].cpu,"fail:");
 917                                         log_info(parameters[i].cpu,
 918                                                 "status=%d\n", status);
 919                                         unlock(parameters[i].cpu);
 920                                         break;
 921                                 }
 922                                 if (one_lock)
 923                                 /* Release the lock */
 924                                         unlock(0);
 925                                 /* Release lock on this cpu */
 926                                 else
 927                                         unlock(parameters[i].cpu);
 928
 929                                 if (j < parameters[i].loop-1)
 930                                         sleep(parameters[i].interval);
 931                         }
 932                         current_time=time(NULL);
 933                         log_info(parameters[i].cpu, "Done at %s", ctime(&current_time));
 934                         return 0;
 935                 }
 936                 else if (pid<0) {
 937                         perror("Error fork:");
 938                         continue;
 939                 }
 940                 child_pid[i]=pid;
 941         }
 942         for (i=0;i<num;i++)
 943                 waitpid(child_pid[i], NULL, 0);
 944
 945         if (one_lock)
 946                 free_sem(0);
 947         else
 948                 for (i=0;i<num;i++)
 949                         free_sem(parameters[i].cpu);
 950
 951         printf("All done.\n");
 952
 953         return 0;
 954 }
 955
 956 void help()
 957 {
 958         printf("err_inject_tool:\n");
 959         printf("\t-q: query all capabilities. default: off\n");
 960         printf("\t-m: procedure mode. 1: physical 2: virtual. default: 1\n");
 961         printf("\t-i: inject errors. default: off\n");
 962         printf("\t-l: one lock per cpu. default: one lock for all\n");
 963         printf("\t-e: error parameters:\n");
 964         printf("\t\tcpu,loop,interval,err_type_info,err_struct_info[,err_data_buffer[0],err_data_buffer[1],err_data_buffer[2]]\n");
 965         printf("\t\t   cpu: logical cpu number the error will be inject in.\n");
 966         printf("\t\t   loop: times the error will be injected.\n");
 967         printf("\t\t   interval: In second. every so often one error is injected.\n");
 968         printf("\t\t   err_type_info, err_struct_info: PAL parameters.\n");
 969         printf("\t\t   err_data_buffer: PAL parameter. Optional. If not present,\n");
 970         printf("\t\t                    it's constructed by tool automatically. Be\n");
 971         printf("\t\t                    careful to provide err_data_buffer and make\n");
 972         printf("\t\t                    sure it's working with the environment.\n");
 973         printf("\t    Note:no space between error parameters.\n");
 974         printf("\t    default: Take error parameters from err.conf instead of command line.\n");
 975         printf("\t-v: verbose. default: off\n");
 976         printf("\t-h: help\n\n");
 977         printf("The tool will take err.conf file as ");
 978         printf("input to inject single or multiple errors ");
 979         printf("on one or multiple cpus in parallel.\n");
 980 }
 981
 982 int main(int argc, char **argv)
 983 {
 984         char c;
 985         int do_err_inj=0;
 986         int do_query_all=0;
 987         int count;
 988         u32 m;
 989
 990         /* Default one lock for all cpu's */
 991         one_lock=1;
 992         while ((c = getopt(argc, argv, "m:iqvhle:")) != EOF)
 993                 switch (c) {
 994                         case 'm':       /* Procedure mode. 1: phys 2: virt */
 995                                 count=sscanf(optarg, "%x", &m);
 996                                 if (count!=1 || (m!=1 && m!=2)) {
 997                                         printf("Wrong mode number.\n");
 998                                         help();
 999                                         return -1;
1000                                 }
1001                                 mode=m;
1002                                 break;
1003                         case 'i':       /* Inject errors */
1004                                 do_err_inj=1;
1005                                 break;
1006                         case 'q':       /* Query */
1007                                 do_query_all=1;
1008                                 break;
1009                         case 'v':       /* Verbose */
1010                                 verbose=1;
1011                                 break;
1012                         case 'l':       /* One lock per cpu */
1013                                 one_lock=0;
1014                                 break;
1015                         case 'e':       /* error arguments */
1016                                 /* Take parameters:
1017                                  * #cpu, loop, interval, err_type_info, err_struct_info[, err_data_buffer]
1018                                  * err_data_buffer is optional. Recommend not to specify
1019                                  * err_data_buffer. Better to use tool to generate it.
1020                                  */
1021                                 count=sscanf(optarg,
1022                                         "%lx, %lx, %lx, %lx, %lx, %lx, %lx, %lx\n",
1023                                         &line_para.cpu,
1024                                         &line_para.loop,
1025                                         &line_para.interval,
1026                                         &line_para.err_type_info,
1027                                         &line_para.err_struct_info,
1028                                         &line_para.err_data_buffer[0],
1029                                         &line_para.err_data_buffer[1],
1030                                         &line_para.err_data_buffer[2]);
1031                                 if (count!=PARA_FIELD_NUM+3) {
1032                                     line_para.err_data_buffer[0]=-1,
1033                                     line_para.err_data_buffer[1]=-1,
1034                                     line_para.err_data_buffer[2]=-1;
1035                                     count=sscanf(optarg, "%lx, %lx, %lx, %lx, %lx\n",
1036                                                 &line_para.cpu,
1037                                                 &line_para.loop,
1038                                                 &line_para.interval,
1039                                                 &line_para.err_type_info,
1040                                                 &line_para.err_struct_info);
1041                                     if (count!=PARA_FIELD_NUM) {
1042                                         printf("Wrong error arguments.\n");
1043                                         help();
1044                                         return -1;
1045                                     }
1046                                 }
1047                                 para=1;
1048                                 break;
1049                         continue;
1050                                 break;
1051                         case 'h':
1052                                 help();
1053                                 return 0;
1054                         default:
1055                                 break;
1056                 }
1057
1058         if (do_query_all)
1059                 query_all_capabilities();
1060         if (do_err_inj)
1061                 err_inj();
1062
1063         if (!do_query_all &&  !do_err_inj)
1064                 help();
1065
1066         return 0;
1067 }
1068