gcc/d/dmd/tokens.d

   1 /**
   2  * Defines lexical tokens.
   3  *
   4  * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
   5  *
   6  * Copyright:   Copyright (C) 1999-2021 by The D Language Foundation, All Rights Reserved
   7  * Authors:     $(LINK2 http://www.digitalmars.com, Walter Bright)
   8  * License:     $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
   9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
  10  * Documentation:  https://dlang.org/phobos/dmd_tokens.html
  11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
  12  */
  13
  14 module dmd.tokens;
  15
  16 import core.stdc.ctype;
  17 import core.stdc.stdio;
  18 import core.stdc.string;
  19 import dmd.globals;
  20 import dmd.identifier;
  21 import dmd.root.ctfloat;
  22 import dmd.common.outbuffer;
  23 import dmd.root.rmem;
  24 import dmd.utf;
  25
  26 enum TOK : ushort
  27 {
  28     reserved,
  29
  30     // Other
  31     leftParenthesis,
  32     rightParenthesis,
  33     leftBracket,
  34     rightBracket,
  35     leftCurly,
  36     rightCurly,
  37     colon,
  38     negate,
  39     semicolon,
  40     dotDotDot,
  41     endOfFile,
  42     cast_,
  43     null_,
  44     assert_,
  45     true_,
  46     false_,
  47     array,
  48     call,
  49     address,
  50     type,
  51     throw_,
  52     new_,
  53     delete_,
  54     star,
  55     symbolOffset,
  56     variable,
  57     dotVariable,
  58     dotIdentifier,
  59     dotTemplateInstance,
  60     dotType,
  61     slice,
  62     arrayLength,
  63     version_,
  64     module_,
  65     dollar,
  66     template_,
  67     dotTemplateDeclaration,
  68     declaration,
  69     typeof_,
  70     pragma_,
  71     dSymbol,
  72     typeid_,
  73     uadd,
  74     remove,
  75     newAnonymousClass,
  76     comment,
  77     arrayLiteral,
  78     assocArrayLiteral,
  79     structLiteral,
  80     classReference,
  81     thrownException,
  82     delegatePointer,
  83     delegateFunctionPointer,
  84
  85     // Operators
  86     lessThan = 54,
  87     greaterThan,
  88     lessOrEqual,
  89     greaterOrEqual,
  90     equal,
  91     notEqual,
  92     identity,
  93     notIdentity,
  94     index,
  95     is_,
  96
  97     leftShift = 64,
  98     rightShift,
  99     leftShiftAssign,
 100     rightShiftAssign,
 101     unsignedRightShift,
 102     unsignedRightShiftAssign,
 103     concatenate,
 104     concatenateAssign, // ~=
 105     concatenateElemAssign,
 106     concatenateDcharAssign,
 107     add,
 108     min,
 109     addAssign,
 110     minAssign,
 111     mul,
 112     div,
 113     mod,
 114     mulAssign,
 115     divAssign,
 116     modAssign,
 117     and,
 118     or,
 119     xor,
 120     andAssign,
 121     orAssign,
 122     xorAssign,
 123     assign,
 124     not,
 125     tilde,
 126     plusPlus,
 127     minusMinus,
 128     construct,
 129     blit,
 130     dot,
 131     comma,
 132     question,
 133     andAnd,
 134     orOr,
 135     prePlusPlus,
 136     preMinusMinus,
 137
 138     // Numeric literals
 139     int32Literal = 104,
 140     uns32Literal,
 141     int64Literal,
 142     uns64Literal,
 143     int128Literal,
 144     uns128Literal,
 145     float32Literal,
 146     float64Literal,
 147     float80Literal,
 148     imaginary32Literal,
 149     imaginary64Literal,
 150     imaginary80Literal,
 151
 152     // Char constants
 153     charLiteral = 116,
 154     wcharLiteral,
 155     dcharLiteral,
 156
 157     // Leaf operators
 158     identifier = 119,
 159     string_,
 160     hexadecimalString,
 161     this_,
 162     super_,
 163     halt,
 164     tuple,
 165     error,
 166
 167     // Basic types
 168     void_ = 127,
 169     int8,
 170     uns8,
 171     int16,
 172     uns16,
 173     int32,
 174     uns32,
 175     int64,
 176     uns64,
 177     int128,
 178     uns128,
 179     float32,
 180     float64,
 181     float80,
 182     imaginary32,
 183     imaginary64,
 184     imaginary80,
 185     complex32,
 186     complex64,
 187     complex80,
 188     char_,
 189     wchar_,
 190     dchar_,
 191     bool_,
 192
 193     // Aggregates
 194     struct_ = 151,
 195     class_,
 196     interface_,
 197     union_,
 198     enum_,
 199     import_,
 200     alias_,
 201     override_,
 202     delegate_,
 203     function_,
 204     mixin_,
 205     align_,
 206     extern_,
 207     private_,
 208     protected_,
 209     public_,
 210     export_,
 211     static_,
 212     final_,
 213     const_,
 214     abstract_,
 215     debug_,
 216     deprecated_,
 217     in_,
 218     out_,
 219     inout_,
 220     lazy_,
 221     auto_,
 222     package_,
 223     immutable_,
 224
 225     // Statements
 226     if_ = 181,
 227     else_,
 228     while_,
 229     for_,
 230     do_,
 231     switch_,
 232     case_,
 233     default_,
 234     break_,
 235     continue_,
 236     with_,
 237     synchronized_,
 238     return_,
 239     goto_,
 240     try_,
 241     catch_,
 242     finally_,
 243     asm_,
 244     foreach_,
 245     foreach_reverse_,
 246     scope_,
 247     onScopeExit,
 248     onScopeFailure,
 249     onScopeSuccess,
 250
 251     // Contracts
 252     invariant_ = 205,
 253
 254     // Testing
 255     unittest_,
 256
 257     // Added after 1.0
 258     argumentTypes,
 259     ref_,
 260     macro_,
 261
 262     parameters = 210,
 263     traits,
 264     overloadSet,
 265     pure_,
 266     nothrow_,
 267     gshared,
 268     line,
 269     file,
 270     fileFullPath,
 271     moduleString,   // __MODULE__
 272     functionString, // __FUNCTION__
 273     prettyFunction, // __PRETTY_FUNCTION__
 274     shared_,
 275     at,
 276     pow,
 277     powAssign,
 278     goesTo,
 279     vector,
 280     pound,
 281
 282     interval = 229,
 283     voidExpression,
 284     cantExpression,
 285     showCtfeContext,
 286
 287     objcClassReference,
 288     vectorArray,
 289
 290     arrow,      // ->
 291     colonColon, // ::
 292     wchar_tLiteral,
 293     compoundLiteral, // ( type-name ) { initializer-list }
 294
 295     // C only keywords
 296     inline,
 297     register,
 298     restrict,
 299     signed,
 300     sizeof_,
 301     typedef_,
 302     unsigned,
 303     volatile,
 304     _Alignas,
 305     _Alignof,
 306     _Atomic,
 307     _Bool,
 308     _Complex,
 309     _Generic,
 310     _Imaginary,
 311     _Noreturn,
 312     _Static_assert,
 313     _Thread_local,
 314
 315     // C only extended keywords
 316     __cdecl,
 317     __declspec,
 318     __attribute__,
 319 }
 320
 321 enum FirstCKeyword = TOK.inline;
 322
 323 // Assert that all token enum members have consecutive values and
 324 // that none of them overlap
 325 static assert(() {
 326     foreach (idx, enumName; __traits(allMembers, TOK)) {
 327        static if (idx != __traits(getMember, TOK, enumName)) {
 328            pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
 329            static assert(0);
 330        }
 331     }
 332     return true;
 333 }());
 334
 335 /****************************************
 336  */
 337
 338 private immutable TOK[] keywords =
 339 [
 340     TOK.this_,
 341     TOK.super_,
 342     TOK.assert_,
 343     TOK.null_,
 344     TOK.true_,
 345     TOK.false_,
 346     TOK.cast_,
 347     TOK.new_,
 348     TOK.delete_,
 349     TOK.throw_,
 350     TOK.module_,
 351     TOK.pragma_,
 352     TOK.typeof_,
 353     TOK.typeid_,
 354     TOK.template_,
 355     TOK.void_,
 356     TOK.int8,
 357     TOK.uns8,
 358     TOK.int16,
 359     TOK.uns16,
 360     TOK.int32,
 361     TOK.uns32,
 362     TOK.int64,
 363     TOK.uns64,
 364     TOK.int128,
 365     TOK.uns128,
 366     TOK.float32,
 367     TOK.float64,
 368     TOK.float80,
 369     TOK.bool_,
 370     TOK.char_,
 371     TOK.wchar_,
 372     TOK.dchar_,
 373     TOK.imaginary32,
 374     TOK.imaginary64,
 375     TOK.imaginary80,
 376     TOK.complex32,
 377     TOK.complex64,
 378     TOK.complex80,
 379     TOK.delegate_,
 380     TOK.function_,
 381     TOK.is_,
 382     TOK.if_,
 383     TOK.else_,
 384     TOK.while_,
 385     TOK.for_,
 386     TOK.do_,
 387     TOK.switch_,
 388     TOK.case_,
 389     TOK.default_,
 390     TOK.break_,
 391     TOK.continue_,
 392     TOK.synchronized_,
 393     TOK.return_,
 394     TOK.goto_,
 395     TOK.try_,
 396     TOK.catch_,
 397     TOK.finally_,
 398     TOK.with_,
 399     TOK.asm_,
 400     TOK.foreach_,
 401     TOK.foreach_reverse_,
 402     TOK.scope_,
 403     TOK.struct_,
 404     TOK.class_,
 405     TOK.interface_,
 406     TOK.union_,
 407     TOK.enum_,
 408     TOK.import_,
 409     TOK.mixin_,
 410     TOK.static_,
 411     TOK.final_,
 412     TOK.const_,
 413     TOK.alias_,
 414     TOK.override_,
 415     TOK.abstract_,
 416     TOK.debug_,
 417     TOK.deprecated_,
 418     TOK.in_,
 419     TOK.out_,
 420     TOK.inout_,
 421     TOK.lazy_,
 422     TOK.auto_,
 423     TOK.align_,
 424     TOK.extern_,
 425     TOK.private_,
 426     TOK.package_,
 427     TOK.protected_,
 428     TOK.public_,
 429     TOK.export_,
 430     TOK.invariant_,
 431     TOK.unittest_,
 432     TOK.version_,
 433     TOK.argumentTypes,
 434     TOK.parameters,
 435     TOK.ref_,
 436     TOK.macro_,
 437     TOK.pure_,
 438     TOK.nothrow_,
 439     TOK.gshared,
 440     TOK.traits,
 441     TOK.vector,
 442     TOK.overloadSet,
 443     TOK.file,
 444     TOK.fileFullPath,
 445     TOK.line,
 446     TOK.moduleString,
 447     TOK.functionString,
 448     TOK.prettyFunction,
 449     TOK.shared_,
 450     TOK.immutable_,
 451
 452     // C only keywords
 453     TOK.inline,
 454     TOK.register,
 455     TOK.restrict,
 456     TOK.signed,
 457     TOK.sizeof_,
 458     TOK.typedef_,
 459     TOK.unsigned,
 460     TOK.volatile,
 461     TOK._Alignas,
 462     TOK._Alignof,
 463     TOK._Atomic,
 464     TOK._Bool,
 465     TOK._Complex,
 466     TOK._Generic,
 467     TOK._Imaginary,
 468     TOK._Noreturn,
 469     TOK._Static_assert,
 470     TOK._Thread_local,
 471
 472     // C only extended keywords
 473     TOK.__cdecl,
 474     TOK.__declspec,
 475     TOK.__attribute__,
 476 ];
 477
 478 // Initialize the identifier pool
 479 shared static this() nothrow
 480 {
 481     Identifier.initTable();
 482     foreach (kw; keywords)
 483     {
 484         //printf("keyword[%d] = '%s'\n",kw, tochars[kw].ptr);
 485         Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw);
 486     }
 487 }
 488
 489 /************************************
 490  * This is used to pick the C keywords out of the tokens.
 491  * If it's not a C keyword, then it's an identifier.
 492  */
 493 static immutable TOK[TOK.max + 1] Ckeywords =
 494 () {
 495     with (TOK)
 496     {
 497         TOK[TOK.max + 1] tab = identifier;  // default to identifier
 498         enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
 499                        enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
 500                        restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
 501                        union_, unsigned, void_, volatile, while_, asm_,
 502                        _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
 503                        _Static_assert, _Thread_local, __cdecl, __declspec, __attribute__ ];
 504
 505         foreach (kw; Ckwds)
 506             tab[kw] = cast(TOK) kw;
 507
 508         return tab;
 509     }
 510 } ();
 511
 512
 513 /***********************************************************
 514  */
 515 extern (C++) struct Token
 516 {
 517     Token* next;
 518     Loc loc;
 519     const(char)* ptr; // pointer to first character of this token within buffer
 520     TOK value;
 521     const(char)[] blockComment; // doc comment string prior to this token
 522     const(char)[] lineComment; // doc comment for previous token
 523
 524     union
 525     {
 526         // Integers
 527         sinteger_t intvalue;
 528         uinteger_t unsvalue;
 529         // Floats
 530         real_t floatvalue;
 531
 532         struct
 533         {
 534             const(char)* ustring; // UTF8 string
 535             uint len;
 536             ubyte postfix; // 'c', 'w', 'd'
 537         }
 538
 539         Identifier ident;
 540     }
 541
 542     extern (D) private static immutable string[TOK.max + 1] tochars =
 543     [
 544         // Keywords
 545         TOK.this_: "this",
 546         TOK.super_: "super",
 547         TOK.assert_: "assert",
 548         TOK.null_: "null",
 549         TOK.true_: "true",
 550         TOK.false_: "false",
 551         TOK.cast_: "cast",
 552         TOK.new_: "new",
 553         TOK.delete_: "delete",
 554         TOK.throw_: "throw",
 555         TOK.module_: "module",
 556         TOK.pragma_: "pragma",
 557         TOK.typeof_: "typeof",
 558         TOK.typeid_: "typeid",
 559         TOK.template_: "template",
 560         TOK.void_: "void",
 561         TOK.int8: "byte",
 562         TOK.uns8: "ubyte",
 563         TOK.int16: "short",
 564         TOK.uns16: "ushort",
 565         TOK.int32: "int",
 566         TOK.uns32: "uint",
 567         TOK.int64: "long",
 568         TOK.uns64: "ulong",
 569         TOK.int128: "cent",
 570         TOK.uns128: "ucent",
 571         TOK.float32: "float",
 572         TOK.float64: "double",
 573         TOK.float80: "real",
 574         TOK.bool_: "bool",
 575         TOK.char_: "char",
 576         TOK.wchar_: "wchar",
 577         TOK.dchar_: "dchar",
 578         TOK.imaginary32: "ifloat",
 579         TOK.imaginary64: "idouble",
 580         TOK.imaginary80: "ireal",
 581         TOK.complex32: "cfloat",
 582         TOK.complex64: "cdouble",
 583         TOK.complex80: "creal",
 584         TOK.delegate_: "delegate",
 585         TOK.function_: "function",
 586         TOK.is_: "is",
 587         TOK.if_: "if",
 588         TOK.else_: "else",
 589         TOK.while_: "while",
 590         TOK.for_: "for",
 591         TOK.do_: "do",
 592         TOK.switch_: "switch",
 593         TOK.case_: "case",
 594         TOK.default_: "default",
 595         TOK.break_: "break",
 596         TOK.continue_: "continue",
 597         TOK.synchronized_: "synchronized",
 598         TOK.return_: "return",
 599         TOK.goto_: "goto",
 600         TOK.try_: "try",
 601         TOK.catch_: "catch",
 602         TOK.finally_: "finally",
 603         TOK.with_: "with",
 604         TOK.asm_: "asm",
 605         TOK.foreach_: "foreach",
 606         TOK.foreach_reverse_: "foreach_reverse",
 607         TOK.scope_: "scope",
 608         TOK.struct_: "struct",
 609         TOK.class_: "class",
 610         TOK.interface_: "interface",
 611         TOK.union_: "union",
 612         TOK.enum_: "enum",
 613         TOK.import_: "import",
 614         TOK.mixin_: "mixin",
 615         TOK.static_: "static",
 616         TOK.final_: "final",
 617         TOK.const_: "const",
 618         TOK.alias_: "alias",
 619         TOK.override_: "override",
 620         TOK.abstract_: "abstract",
 621         TOK.debug_: "debug",
 622         TOK.deprecated_: "deprecated",
 623         TOK.in_: "in",
 624         TOK.out_: "out",
 625         TOK.inout_: "inout",
 626         TOK.lazy_: "lazy",
 627         TOK.auto_: "auto",
 628         TOK.align_: "align",
 629         TOK.extern_: "extern",
 630         TOK.private_: "private",
 631         TOK.package_: "package",
 632         TOK.protected_: "protected",
 633         TOK.public_: "public",
 634         TOK.export_: "export",
 635         TOK.invariant_: "invariant",
 636         TOK.unittest_: "unittest",
 637         TOK.version_: "version",
 638         TOK.argumentTypes: "__argTypes",
 639         TOK.parameters: "__parameters",
 640         TOK.ref_: "ref",
 641         TOK.macro_: "macro",
 642         TOK.pure_: "pure",
 643         TOK.nothrow_: "nothrow",
 644         TOK.gshared: "__gshared",
 645         TOK.traits: "__traits",
 646         TOK.vector: "__vector",
 647         TOK.overloadSet: "__overloadset",
 648         TOK.file: "__FILE__",
 649         TOK.fileFullPath: "__FILE_FULL_PATH__",
 650         TOK.line: "__LINE__",
 651         TOK.moduleString: "__MODULE__",
 652         TOK.functionString: "__FUNCTION__",
 653         TOK.prettyFunction: "__PRETTY_FUNCTION__",
 654         TOK.shared_: "shared",
 655         TOK.immutable_: "immutable",
 656
 657         TOK.endOfFile: "End of File",
 658         TOK.leftCurly: "{",
 659         TOK.rightCurly: "}",
 660         TOK.leftParenthesis: "(",
 661         TOK.rightParenthesis: ")",
 662         TOK.leftBracket: "[",
 663         TOK.rightBracket: "]",
 664         TOK.semicolon: ";",
 665         TOK.colon: ":",
 666         TOK.comma: ",",
 667         TOK.dot: ".",
 668         TOK.xor: "^",
 669         TOK.xorAssign: "^=",
 670         TOK.assign: "=",
 671         TOK.construct: "=",
 672         TOK.blit: "=",
 673         TOK.lessThan: "<",
 674         TOK.greaterThan: ">",
 675         TOK.lessOrEqual: "<=",
 676         TOK.greaterOrEqual: ">=",
 677         TOK.equal: "==",
 678         TOK.notEqual: "!=",
 679         TOK.not: "!",
 680         TOK.leftShift: "<<",
 681         TOK.rightShift: ">>",
 682         TOK.unsignedRightShift: ">>>",
 683         TOK.add: "+",
 684         TOK.min: "-",
 685         TOK.mul: "*",
 686         TOK.div: "/",
 687         TOK.mod: "%",
 688         TOK.slice: "..",
 689         TOK.dotDotDot: "...",
 690         TOK.and: "&",
 691         TOK.andAnd: "&&",
 692         TOK.or: "|",
 693         TOK.orOr: "||",
 694         TOK.array: "[]",
 695         TOK.index: "[i]",
 696         TOK.address: "&",
 697         TOK.star: "*",
 698         TOK.tilde: "~",
 699         TOK.dollar: "$",
 700         TOK.plusPlus: "++",
 701         TOK.minusMinus: "--",
 702         TOK.prePlusPlus: "++",
 703         TOK.preMinusMinus: "--",
 704         TOK.type: "type",
 705         TOK.question: "?",
 706         TOK.negate: "-",
 707         TOK.uadd: "+",
 708         TOK.variable: "var",
 709         TOK.addAssign: "+=",
 710         TOK.minAssign: "-=",
 711         TOK.mulAssign: "*=",
 712         TOK.divAssign: "/=",
 713         TOK.modAssign: "%=",
 714         TOK.leftShiftAssign: "<<=",
 715         TOK.rightShiftAssign: ">>=",
 716         TOK.unsignedRightShiftAssign: ">>>=",
 717         TOK.andAssign: "&=",
 718         TOK.orAssign: "|=",
 719         TOK.concatenateAssign: "~=",
 720         TOK.concatenateElemAssign: "~=",
 721         TOK.concatenateDcharAssign: "~=",
 722         TOK.concatenate: "~",
 723         TOK.call: "call",
 724         TOK.identity: "is",
 725         TOK.notIdentity: "!is",
 726         TOK.identifier: "identifier",
 727         TOK.at: "@",
 728         TOK.pow: "^^",
 729         TOK.powAssign: "^^=",
 730         TOK.goesTo: "=>",
 731         TOK.pound: "#",
 732         TOK.arrow: "->",
 733         TOK.colonColon: "::",
 734
 735         // For debugging
 736         TOK.error: "error",
 737         TOK.dotIdentifier: "dotid",
 738         TOK.dotTemplateDeclaration: "dottd",
 739         TOK.dotTemplateInstance: "dotti",
 740         TOK.dotVariable: "dotvar",
 741         TOK.dotType: "dottype",
 742         TOK.symbolOffset: "symoff",
 743         TOK.arrayLength: "arraylength",
 744         TOK.arrayLiteral: "arrayliteral",
 745         TOK.assocArrayLiteral: "assocarrayliteral",
 746         TOK.structLiteral: "structliteral",
 747         TOK.string_: "string",
 748         TOK.dSymbol: "symbol",
 749         TOK.tuple: "tuple",
 750         TOK.declaration: "declaration",
 751         TOK.onScopeExit: "scope(exit)",
 752         TOK.onScopeSuccess: "scope(success)",
 753         TOK.onScopeFailure: "scope(failure)",
 754         TOK.delegatePointer: "delegateptr",
 755
 756         // Finish up
 757         TOK.reserved: "reserved",
 758         TOK.remove: "remove",
 759         TOK.newAnonymousClass: "newanonclass",
 760         TOK.comment: "comment",
 761         TOK.classReference: "classreference",
 762         TOK.thrownException: "thrownexception",
 763         TOK.delegateFunctionPointer: "delegatefuncptr",
 764         TOK.int32Literal: "int32v",
 765         TOK.uns32Literal: "uns32v",
 766         TOK.int64Literal: "int64v",
 767         TOK.uns64Literal: "uns64v",
 768         TOK.int128Literal: "int128v",
 769         TOK.uns128Literal: "uns128v",
 770         TOK.float32Literal: "float32v",
 771         TOK.float64Literal: "float64v",
 772         TOK.float80Literal: "float80v",
 773         TOK.imaginary32Literal: "imaginary32v",
 774         TOK.imaginary64Literal: "imaginary64v",
 775         TOK.imaginary80Literal: "imaginary80v",
 776         TOK.charLiteral: "charv",
 777         TOK.wcharLiteral: "wcharv",
 778         TOK.dcharLiteral: "dcharv",
 779         TOK.wchar_tLiteral: "wchar_tv",
 780         TOK.compoundLiteral: "compoundliteral",
 781
 782         TOK.halt: "halt",
 783         TOK.hexadecimalString: "xstring",
 784
 785         TOK.interval: "interval",
 786         TOK.voidExpression: "voidexp",
 787         TOK.cantExpression: "cantexp",
 788         TOK.showCtfeContext : "showCtfeContext",
 789
 790         TOK.objcClassReference: "class",
 791         TOK.vectorArray: "vectorarray",
 792
 793         // C only keywords
 794         TOK.inline    : "inline",
 795         TOK.register  : "register",
 796         TOK.restrict  : "restrict",
 797         TOK.signed    : "signed",
 798         TOK.sizeof_   : "sizeof",
 799         TOK.typedef_  : "typedef",
 800         TOK.unsigned  : "unsigned",
 801         TOK.volatile  : "volatile",
 802         TOK._Alignas  : "_Alignas",
 803         TOK._Alignof  : "_Alignof",
 804         TOK._Atomic   : "_Atomic",
 805         TOK._Bool     : "_Bool",
 806         TOK._Complex  : "_Complex",
 807         TOK._Generic  : "_Generic",
 808         TOK._Imaginary: "_Imaginary",
 809         TOK._Noreturn : "_Noreturn",
 810         TOK._Static_assert : "_Static_assert",
 811         TOK._Thread_local  : "_Thread_local",
 812
 813         // C only extended keywords
 814         TOK.__cdecl        : "__cdecl",
 815         TOK.__declspec     : "__declspec",
 816         TOK.__attribute__  : "__attribute__",
 817     ];
 818
 819     static assert(() {
 820         foreach (s; tochars)
 821             assert(s.length);
 822         return true;
 823     }());
 824
 825 nothrow:
 826
 827     int isKeyword() const
 828     {
 829         foreach (kw; keywords)
 830         {
 831             if (kw == value)
 832                 return 1;
 833         }
 834         return 0;
 835     }
 836
 837     /****
 838      * Set to contents of ptr[0..length]
 839      * Params:
 840      *  ptr = pointer to string
 841      *  length = length of string
 842      */
 843     void setString(const(char)* ptr, size_t length)
 844     {
 845         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
 846         memcpy(s, ptr, length);
 847         s[length] = 0;
 848         ustring = s;
 849         len = cast(uint)length;
 850         postfix = 0;
 851     }
 852
 853     /****
 854      * Set to contents of buf
 855      * Params:
 856      *  buf = string (not zero terminated)
 857      */
 858     void setString(const ref OutBuffer buf)
 859     {
 860         setString(cast(const(char)*)buf[].ptr, buf.length);
 861     }
 862
 863     /****
 864      * Set to empty string
 865      */
 866     void setString()
 867     {
 868         ustring = "";
 869         len = 0;
 870         postfix = 0;
 871     }
 872
 873     extern (C++) const(char)* toChars() const
 874     {
 875         __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer;
 876         const(char)* p = &buffer[0];
 877         switch (value)
 878         {
 879         case TOK.int32Literal:
 880             sprintf(&buffer[0], "%d", cast(d_int32)intvalue);
 881             break;
 882         case TOK.uns32Literal:
 883         case TOK.charLiteral:
 884         case TOK.wcharLiteral:
 885         case TOK.dcharLiteral:
 886         case TOK.wchar_tLiteral:
 887             sprintf(&buffer[0], "%uU", cast(d_uns32)unsvalue);
 888             break;
 889         case TOK.int64Literal:
 890             sprintf(&buffer[0], "%lldL", cast(long)intvalue);
 891             break;
 892         case TOK.uns64Literal:
 893             sprintf(&buffer[0], "%lluUL", cast(ulong)unsvalue);
 894             break;
 895         case TOK.float32Literal:
 896             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 897             strcat(&buffer[0], "f");
 898             break;
 899         case TOK.float64Literal:
 900             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 901             break;
 902         case TOK.float80Literal:
 903             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 904             strcat(&buffer[0], "L");
 905             break;
 906         case TOK.imaginary32Literal:
 907             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 908             strcat(&buffer[0], "fi");
 909             break;
 910         case TOK.imaginary64Literal:
 911             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 912             strcat(&buffer[0], "i");
 913             break;
 914         case TOK.imaginary80Literal:
 915             CTFloat.sprint(&buffer[0], 'g', floatvalue);
 916             strcat(&buffer[0], "Li");
 917             break;
 918         case TOK.string_:
 919             {
 920                 OutBuffer buf;
 921                 buf.writeByte('"');
 922                 for (size_t i = 0; i < len;)
 923                 {
 924                     dchar c;
 925                     utf_decodeChar(ustring[0 .. len], i, c);
 926                     switch (c)
 927                     {
 928                     case 0:
 929                         break;
 930                     case '"':
 931                     case '\\':
 932                         buf.writeByte('\\');
 933                         goto default;
 934                     default:
 935                         if (c <= 0x7F)
 936                         {
 937                             if (isprint(c))
 938                                 buf.writeByte(c);
 939                             else
 940                                 buf.printf("\\x%02x", c);
 941                         }
 942                         else if (c <= 0xFFFF)
 943                             buf.printf("\\u%04x", c);
 944                         else
 945                             buf.printf("\\U%08x", c);
 946                         continue;
 947                     }
 948                     break;
 949                 }
 950                 buf.writeByte('"');
 951                 if (postfix)
 952                     buf.writeByte(postfix);
 953                 buf.writeByte(0);
 954                 p = buf.extractSlice().ptr;
 955             }
 956             break;
 957         case TOK.hexadecimalString:
 958             {
 959                 OutBuffer buf;
 960                 buf.writeByte('x');
 961                 buf.writeByte('"');
 962                 foreach (size_t i; 0 .. len)
 963                 {
 964                     if (i)
 965                         buf.writeByte(' ');
 966                     buf.printf("%02x", ustring[i]);
 967                 }
 968                 buf.writeByte('"');
 969                 if (postfix)
 970                     buf.writeByte(postfix);
 971                 buf.writeByte(0);
 972                 p = buf.extractSlice().ptr;
 973                 break;
 974             }
 975         case TOK.identifier:
 976         case TOK.enum_:
 977         case TOK.struct_:
 978         case TOK.import_:
 979         case TOK.wchar_:
 980         case TOK.dchar_:
 981         case TOK.bool_:
 982         case TOK.char_:
 983         case TOK.int8:
 984         case TOK.uns8:
 985         case TOK.int16:
 986         case TOK.uns16:
 987         case TOK.int32:
 988         case TOK.uns32:
 989         case TOK.int64:
 990         case TOK.uns64:
 991         case TOK.int128:
 992         case TOK.uns128:
 993         case TOK.float32:
 994         case TOK.float64:
 995         case TOK.float80:
 996         case TOK.imaginary32:
 997         case TOK.imaginary64:
 998         case TOK.imaginary80:
 999         case TOK.complex32:
1000         case TOK.complex64:
1001         case TOK.complex80:
1002         case TOK.void_:
1003             p = ident.toChars();
1004             break;
1005         default:
1006             p = toChars(value);
1007             break;
1008         }
1009         return p;
1010     }
1011
1012     static const(char)* toChars(uint value)
1013     {
1014         return toString(value).ptr;
1015     }
1016
1017     extern (D) static string toString(uint value) pure nothrow @nogc @safe
1018     {
1019         return tochars[value];
1020     }
1021 }
1022