Parser/parsetok.c

   1
   2 /* Parser-tokenizer link implementation */
   3
   4 #include "pgenheaders.h"
   5 #include "tokenizer.h"
   6 #include "node.h"
   7 #include "grammar.h"
   8 #include "parser.h"
   9 #include "parsetok.h"
  10 #include "errcode.h"
  11
  12 int Py_TabcheckFlag;
  13
  14
  15 /* Forward */
  16 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
  17 static void initerr(perrdetail *err_ret, char* filename);
  18
  19 /* Parse input coming from a string.  Return error code, print some errors. */
  20 node *
  21 PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
  22 {
  23         return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
  24 }
  25
  26 node *
  27 PyParser_ParseStringFlags(char *s, grammar *g, int start,
  28                           perrdetail *err_ret, int flags)
  29 {
  30         return PyParser_ParseStringFlagsFilename(s, NULL,
  31                                                  g, start, err_ret, 0);
  32 }
  33
  34 node *
  35 PyParser_ParseStringFlagsFilename(char *s, char *filename,
  36                           grammar *g, int start,
  37                           perrdetail *err_ret, int flags)
  38 {
  39         struct tok_state *tok;
  40
  41         initerr(err_ret, filename);
  42
  43         if ((tok = PyTokenizer_FromString(s)) == NULL) {
  44                 err_ret->error = E_NOMEM;
  45                 return NULL;
  46         }
  47
  48         if (Py_TabcheckFlag || Py_VerboseFlag) {
  49                 tok->filename = filename ? filename : "<string>";
  50                 tok->altwarning = (tok->filename != NULL);
  51                 if (Py_TabcheckFlag >= 2)
  52                         tok->alterror++;
  53         }
  54
  55         return parsetok(tok, g, start, err_ret, flags);
  56 }
  57
  58
  59 /* Parse input coming from a file.  Return error code, print some errors. */
  60
  61 node *
  62 PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
  63                    char *ps1, char *ps2, perrdetail *err_ret)
  64 {
  65         return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
  66                                        err_ret, 0);
  67 }
  68
  69 node *
  70 PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
  71                         char *ps1, char *ps2, perrdetail *err_ret, int flags)
  72 {
  73         struct tok_state *tok;
  74
  75         initerr(err_ret, filename);
  76
  77         if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
  78                 err_ret->error = E_NOMEM;
  79                 return NULL;
  80         }
  81         if (Py_TabcheckFlag || Py_VerboseFlag) {
  82                 tok->filename = filename;
  83                 tok->altwarning = (filename != NULL);
  84                 if (Py_TabcheckFlag >= 2)
  85                         tok->alterror++;
  86         }
  87
  88
  89         return parsetok(tok, g, start, err_ret, flags);
  90 }
  91
  92 /* Parse input coming from the given tokenizer structure.
  93    Return error code. */
  94
  95 #if 0 /* future keyword */
  96 static char yield_msg[] =
  97 "%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
  98 #endif
  99
 100 static node *
 101 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 102          int flags)
 103 {
 104         parser_state *ps;
 105         node *n;
 106         int started = 0;
 107
 108         if ((ps = PyParser_New(g, start)) == NULL) {
 109                 fprintf(stderr, "no mem for new parser\n");
 110                 err_ret->error = E_NOMEM;
 111                 return NULL;
 112         }
 113 #if 0 /* future keyword */
 114         if (flags & PyPARSE_YIELD_IS_KEYWORD)
 115                 ps->p_generators = 1;
 116 #endif
 117
 118         for (;;) {
 119                 char *a, *b;
 120                 int type;
 121                 size_t len;
 122                 char *str;
 123
 124                 type = PyTokenizer_Get(tok, &a, &b);
 125                 if (type == ERRORTOKEN) {
 126                         err_ret->error = tok->done;
 127                         break;
 128                 }
 129                 if (type == ENDMARKER && started) {
 130                         type = NEWLINE; /* Add an extra newline */
 131                         started = 0;
 132                 }
 133                 else
 134                         started = 1;
 135                 len = b - a; /* XXX this may compute NULL - NULL */
 136                 str = PyMem_NEW(char, len + 1);
 137                 if (str == NULL) {
 138                         fprintf(stderr, "no mem for next token\n");
 139                         err_ret->error = E_NOMEM;
 140                         break;
 141                 }
 142                 if (len > 0)
 143                         strncpy(str, a, len);
 144                 str[len] = '\0';
 145
 146 #if 0 /* future keyword */
 147                 /* Warn about yield as NAME */
 148                 if (type == NAME && !ps->p_generators &&
 149                     len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
 150                         PySys_WriteStderr(yield_msg,
 151                                           err_ret->filename==NULL ?
 152                                           "<string>" : err_ret->filename,
 153                                           tok->lineno);
 154 #endif
 155
 156                 if ((err_ret->error =
 157                      PyParser_AddToken(ps, (int)type, str, tok->lineno,
 158                                        &(err_ret->expected))) != E_OK) {
 159                         if (err_ret->error != E_DONE)
 160                                 PyMem_DEL(str);
 161                         break;
 162                 }
 163         }
 164
 165         if (err_ret->error == E_DONE) {
 166                 n = ps->p_tree;
 167                 ps->p_tree = NULL;
 168         }
 169         else
 170                 n = NULL;
 171
 172         PyParser_Delete(ps);
 173
 174         if (n == NULL) {
 175                 if (tok->lineno <= 1 && tok->done == E_EOF)
 176                         err_ret->error = E_EOF;
 177                 err_ret->lineno = tok->lineno;
 178                 err_ret->offset = tok->cur - tok->buf;
 179                 if (tok->buf != NULL) {
 180                         size_t len = tok->inp - tok->buf;
 181                         err_ret->text = PyMem_NEW(char, len + 1);
 182                         if (err_ret->text != NULL) {
 183                                 if (len > 0)
 184                                         strncpy(err_ret->text, tok->buf, len);
 185                                 err_ret->text[len] = '\0';
 186                         }
 187                 }
 188         }
 189
 190         PyTokenizer_Free(tok);
 191
 192         return n;
 193 }
 194
 195 static void
 196 initerr(perrdetail *err_ret, char* filename)
 197 {
 198         err_ret->error = E_OK;
 199         err_ret->filename = filename;
 200         err_ret->lineno = 0;
 201         err_ret->offset = 0;
 202         err_ret->text = NULL;
 203         err_ret->token = -1;
 204         err_ret->expected = -1;
 205 }