2 * Defines lexical tokens.
4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
6 * Copyright: Copyright (C) 1999-2021 by The D Language Foundation, All Rights Reserved
7 * Authors: $(LINK2 http://www.digitalmars.com, Walter Bright)
8 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
9 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
10 * Documentation: https://dlang.org/phobos/dmd_tokens.html
11 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
16 import core
.stdc
.ctype
;
17 import core
.stdc
.stdio
;
18 import core
.stdc
.string
;
20 import dmd
.identifier
;
21 import dmd
.root
.ctfloat
;
22 import dmd
.common
.outbuffer
;
67 dotTemplateDeclaration
,
83 delegateFunctionPointer
,
102 unsignedRightShiftAssign
,
104 concatenateAssign
, // ~=
105 concatenateElemAssign
,
106 concatenateDcharAssign
,
271 moduleString
, // __MODULE__
272 functionString
, // __FUNCTION__
273 prettyFunction
, // __PRETTY_FUNCTION__
293 compoundLiteral
, // ( type-name ) { initializer-list }
315 // C only extended keywords
352 dotTemplateDeclaration
,
367 delegateFunctionPointer
,
386 unsignedRightShiftAssign
,
388 concatenateAssign
, // ~=
389 concatenateElemAssign
,
390 concatenateDcharAssign
,
453 moduleString
, // __MODULE__
454 functionString
, // __FUNCTION__
455 prettyFunction
, // __PRETTY_FUNCTION__
467 compoundLiteral
, // ( type-name ) { initializer-list }
472 enum FirstCKeyword
= TOK
.inline
;
474 // Assert that all token enum members have consecutive values and
475 // that none of them overlap
477 foreach (idx
, enumName
; __traits(allMembers
, TOK
)) {
478 static if (idx
!= __traits(getMember
, TOK
, enumName
)) {
479 pragma(msg
, "Error: Expected TOK.", enumName
, " to be ", idx
, " but is ", __traits(getMember
, TOK
, enumName
));
486 /****************************************
489 private immutable TOK
[] keywords
=
552 TOK
.foreach_reverse_
,
623 // C only extended keywords
629 // Initialize the identifier pool
630 shared static this() nothrow
632 Identifier
.initTable();
633 foreach (kw
; keywords
)
635 //printf("keyword[%d] = '%s'\n",kw, tochars[kw].ptr);
636 Identifier
.idPool(Token
.tochars
[kw
].ptr
, Token
.tochars
[kw
].length
, cast(uint)kw
);
640 /************************************
641 * This is used to pick the C keywords out of the tokens.
642 * If it's not a C keyword, then it's an identifier.
644 static immutable TOK
[TOK
.max
+ 1] Ckeywords
=
648 TOK
[TOK
.max
+ 1] tab
= identifier
; // default to identifier
649 enum Ckwds
= [ auto_
, break_
, case_
, char_
, const_
, continue_
, default_
, do_
, float64
, else_
,
650 enum_
, extern_
, float32
, for_
, goto_
, if_
, inline
, int32
, int64
, register
,
651 restrict
, return_
, int16
, signed
, sizeof_
, static_
, struct_
, switch_
, typedef_
,
652 union_
, unsigned
, void_
, volatile, while_
, asm_
,
653 _Alignas
, _Alignof
, _Atomic
, _Bool
, _Complex
, _Generic
, _Imaginary
, _Noreturn
,
654 _Static_assert
, _Thread_local
, __cdecl
, __declspec
, __attribute__
];
657 tab
[kw
] = cast(TOK
) kw
;
664 /***********************************************************
666 extern (C
++) struct Token
670 const(char)* ptr
; // pointer to first character of this token within buffer
672 const(char)[] blockComment
; // doc comment string prior to this token
673 const(char)[] lineComment
; // doc comment for previous token
685 const(char)* ustring
; // UTF8 string
687 ubyte postfix
; // 'c', 'w', 'd'
693 extern (D
) private static immutable string
[TOK
.max
+ 1] tochars
=
698 TOK
.assert_
: "assert",
704 TOK
.delete_
: "delete",
706 TOK
.module_
: "module",
707 TOK
.pragma_
: "pragma",
708 TOK
.typeof_
: "typeof",
709 TOK
.typeid_
: "typeid",
710 TOK
.template_
: "template",
722 TOK
.float32
: "float",
723 TOK
.float64
: "double",
729 TOK
.imaginary32
: "ifloat",
730 TOK
.imaginary64
: "idouble",
731 TOK
.imaginary80
: "ireal",
732 TOK
.complex32
: "cfloat",
733 TOK
.complex64
: "cdouble",
734 TOK
.complex80
: "creal",
735 TOK
.delegate_
: "delegate",
736 TOK
.function_
: "function",
743 TOK
.switch_
: "switch",
745 TOK
.default_
: "default",
747 TOK
.continue_
: "continue",
748 TOK
.synchronized_
: "synchronized",
749 TOK
.return_
: "return",
753 TOK
.finally_
: "finally",
756 TOK
.foreach_
: "foreach",
757 TOK
.foreach_reverse_
: "foreach_reverse",
759 TOK
.struct_
: "struct",
761 TOK
.interface_
: "interface",
764 TOK
.import_
: "import",
766 TOK
.static_
: "static",
770 TOK
.override_
: "override",
771 TOK
.abstract_
: "abstract",
773 TOK
.deprecated_
: "deprecated",
780 TOK
.extern_
: "extern",
781 TOK
.private_
: "private",
782 TOK
.package_
: "package",
783 TOK
.protected_
: "protected",
784 TOK
.public_
: "public",
785 TOK
.export_
: "export",
786 TOK
.invariant_
: "invariant",
787 TOK
.unittest_
: "unittest",
788 TOK
.version_
: "version",
789 TOK
.argumentTypes
: "__argTypes",
790 TOK
.parameters
: "__parameters",
794 TOK
.nothrow_
: "nothrow",
795 TOK
.gshared
: "__gshared",
796 TOK
.traits
: "__traits",
797 TOK
.vector
: "__vector",
798 TOK
.overloadSet
: "__overloadset",
799 TOK
.file
: "__FILE__",
800 TOK
.fileFullPath
: "__FILE_FULL_PATH__",
801 TOK
.line
: "__LINE__",
802 TOK
.moduleString
: "__MODULE__",
803 TOK
.functionString
: "__FUNCTION__",
804 TOK
.prettyFunction
: "__PRETTY_FUNCTION__",
805 TOK
.shared_
: "shared",
806 TOK
.immutable_
: "immutable",
808 TOK
.endOfFile
: "End of File",
811 TOK
.leftParenthesis
: "(",
812 TOK
.rightParenthesis
: ")",
813 TOK
.leftBracket
: "[",
814 TOK
.rightBracket
: "]",
825 TOK
.greaterThan
: ">",
826 TOK
.lessOrEqual
: "<=",
827 TOK
.greaterOrEqual
: ">=",
832 TOK
.rightShift
: ">>",
833 TOK
.unsignedRightShift
: ">>>",
840 TOK
.dotDotDot
: "...",
852 TOK
.minusMinus
: "--",
853 TOK
.prePlusPlus
: "++",
854 TOK
.preMinusMinus
: "--",
865 TOK
.leftShiftAssign
: "<<=",
866 TOK
.rightShiftAssign
: ">>=",
867 TOK
.unsignedRightShiftAssign
: ">>>=",
870 TOK
.concatenateAssign
: "~=",
871 TOK
.concatenateElemAssign
: "~=",
872 TOK
.concatenateDcharAssign
: "~=",
873 TOK
.concatenate
: "~",
876 TOK
.notIdentity
: "!is",
877 TOK
.identifier
: "identifier",
880 TOK
.powAssign
: "^^=",
884 TOK
.colonColon
: "::",
888 TOK
.dotIdentifier
: "dotid",
889 TOK
.dotTemplateDeclaration
: "dottd",
890 TOK
.dotTemplateInstance
: "dotti",
891 TOK
.dotVariable
: "dotvar",
892 TOK
.dotType
: "dottype",
893 TOK
.symbolOffset
: "symoff",
894 TOK
.arrayLength
: "arraylength",
895 TOK
.arrayLiteral
: "arrayliteral",
896 TOK
.assocArrayLiteral
: "assocarrayliteral",
897 TOK
.structLiteral
: "structliteral",
898 TOK
.string_
: "string",
899 TOK
.dSymbol
: "symbol",
901 TOK
.declaration
: "declaration",
902 TOK
.onScopeExit
: "scope(exit)",
903 TOK
.onScopeSuccess
: "scope(success)",
904 TOK
.onScopeFailure
: "scope(failure)",
905 TOK
.delegatePointer
: "delegateptr",
908 TOK
.reserved
: "reserved",
909 TOK
.remove
: "remove",
910 TOK
.newAnonymousClass
: "newanonclass",
911 TOK
.comment
: "comment",
912 TOK
.classReference
: "classreference",
913 TOK
.thrownException
: "thrownexception",
914 TOK
.delegateFunctionPointer
: "delegatefuncptr",
915 TOK
.int32Literal
: "int32v",
916 TOK
.uns32Literal
: "uns32v",
917 TOK
.int64Literal
: "int64v",
918 TOK
.uns64Literal
: "uns64v",
919 TOK
.int128Literal
: "int128v",
920 TOK
.uns128Literal
: "uns128v",
921 TOK
.float32Literal
: "float32v",
922 TOK
.float64Literal
: "float64v",
923 TOK
.float80Literal
: "float80v",
924 TOK
.imaginary32Literal
: "imaginary32v",
925 TOK
.imaginary64Literal
: "imaginary64v",
926 TOK
.imaginary80Literal
: "imaginary80v",
927 TOK
.charLiteral
: "charv",
928 TOK
.wcharLiteral
: "wcharv",
929 TOK
.dcharLiteral
: "dcharv",
930 TOK
.wchar_tLiteral
: "wchar_tv",
931 TOK
.compoundLiteral
: "compoundliteral",
934 TOK
.hexadecimalString
: "xstring",
936 TOK
.interval
: "interval",
937 TOK
.voidExpression
: "voidexp",
938 TOK
.cantExpression
: "cantexp",
939 TOK
.showCtfeContext
: "showCtfeContext",
941 TOK
.objcClassReference
: "class",
942 TOK
.vectorArray
: "vectorarray",
945 TOK
.inline
: "inline",
946 TOK
.register
: "register",
947 TOK
.restrict
: "restrict",
948 TOK
.signed
: "signed",
949 TOK
.sizeof_
: "sizeof",
950 TOK
.typedef_
: "typedef",
951 TOK
.unsigned
: "unsigned",
952 TOK
.volatile : "volatile",
953 TOK
._Alignas
: "_Alignas",
954 TOK
._Alignof
: "_Alignof",
955 TOK
._Atomic
: "_Atomic",
957 TOK
._Complex
: "_Complex",
958 TOK
._Generic
: "_Generic",
959 TOK
._Imaginary
: "_Imaginary",
960 TOK
._Noreturn
: "_Noreturn",
961 TOK
._Static_assert
: "_Static_assert",
962 TOK
._Thread_local
: "_Thread_local",
964 // C only extended keywords
965 TOK
.__cdecl
: "__cdecl",
966 TOK
.__declspec
: "__declspec",
967 TOK
.__attribute__
: "__attribute__",
978 int isKeyword() const
980 foreach (kw
; keywords
)
989 * Set to contents of ptr[0..length]
991 * ptr = pointer to string
992 * length = length of string
994 void setString(const(char)* ptr
, size_t length
)
996 auto s
= cast(char*)mem
.xmalloc_noscan(length
+ 1);
997 memcpy(s
, ptr
, length
);
1000 len
= cast(uint)length
;
1005 * Set to contents of buf
1007 * buf = string (not zero terminated)
1009 void setString(const ref OutBuffer buf
)
1011 setString(cast(const(char)*)buf
[].ptr
, buf
.length
);
1015 * Set to empty string
1024 extern (C
++) const(char)* toChars() const
1026 __gshared
char[3 + 3 * floatvalue
.sizeof
+ 1] buffer
;
1027 const(char)* p
= &buffer
[0];
1030 case TOK
.int32Literal
:
1031 sprintf(&buffer
[0], "%d", cast(d_int32
)intvalue
);
1033 case TOK
.uns32Literal
:
1034 case TOK
.charLiteral
:
1035 case TOK
.wcharLiteral
:
1036 case TOK
.dcharLiteral
:
1037 case TOK
.wchar_tLiteral
:
1038 sprintf(&buffer
[0], "%uU", cast(d_uns32
)unsvalue
);
1040 case TOK
.int64Literal
:
1041 sprintf(&buffer
[0], "%lldL", cast(long)intvalue
);
1043 case TOK
.uns64Literal
:
1044 sprintf(&buffer
[0], "%lluUL", cast(ulong)unsvalue
);
1046 case TOK
.float32Literal
:
1047 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1048 strcat(&buffer
[0], "f");
1050 case TOK
.float64Literal
:
1051 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1053 case TOK
.float80Literal
:
1054 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1055 strcat(&buffer
[0], "L");
1057 case TOK
.imaginary32Literal
:
1058 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1059 strcat(&buffer
[0], "fi");
1061 case TOK
.imaginary64Literal
:
1062 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1063 strcat(&buffer
[0], "i");
1065 case TOK
.imaginary80Literal
:
1066 CTFloat
.sprint(&buffer
[0], 'g', floatvalue
);
1067 strcat(&buffer
[0], "Li");
1073 for (size_t i
= 0; i
< len
;)
1076 utf_decodeChar(ustring
[0 .. len
], i
, c
);
1083 buf
.writeByte('\\');
1091 buf
.printf("\\x%02x", c
);
1093 else if (c
<= 0xFFFF)
1094 buf
.printf("\\u%04x", c
);
1096 buf
.printf("\\U%08x", c
);
1103 buf
.writeByte(postfix
);
1105 p
= buf
.extractSlice().ptr
;
1108 case TOK
.hexadecimalString
:
1113 foreach (size_t i
; 0 .. len
)
1117 buf
.printf("%02x", ustring
[i
]);
1121 buf
.writeByte(postfix
);
1123 p
= buf
.extractSlice().ptr
;
1126 case TOK
.identifier
:
1147 case TOK
.imaginary32
:
1148 case TOK
.imaginary64
:
1149 case TOK
.imaginary80
:
1154 p
= ident
.toChars();
1163 static const(char)* toChars(TOK value
)
1165 return toString(value
).ptr
;
1168 static const(char)* toChars(ushort value
)
1170 return toString(cast(TOK
)value
).ptr
;
1173 extern (D
) static string
toString(TOK value
) pure nothrow @nogc @safe
1175 return tochars
[value
];