Parser/parsetok.c

   1
   2 /* Parser-tokenizer link implementation */
   3
   4 #include "pgenheaders.h"
   5 #include "tokenizer.h"
   6 #include "node.h"
   7 #include "grammar.h"
   8 #include "parser.h"
   9 #include "parsetok.h"
  10 #include "errcode.h"
  11
  12 int Py_TabcheckFlag;
  13
  14
  15 /* Forward */
  16 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
  17 static void initerr(perrdetail *err_ret, char* filename);
  18
  19 /* Parse input coming from a string.  Return error code, print some errors. */
  20 node *
  21 PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
  22 {
  23         return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
  24 }
  25
  26 node *
  27 PyParser_ParseStringFlags(char *s, grammar *g, int start,
  28                           perrdetail *err_ret, int flags)
  29 {
  30         struct tok_state *tok;
  31
  32         initerr(err_ret, NULL);
  33
  34         if ((tok = PyTokenizer_FromString(s)) == NULL) {
  35                 err_ret->error = E_NOMEM;
  36                 return NULL;
  37         }
  38
  39         if (Py_TabcheckFlag || Py_VerboseFlag) {
  40                 tok->filename = "<string>";
  41                 tok->altwarning = (tok->filename != NULL);
  42                 if (Py_TabcheckFlag >= 2)
  43                         tok->alterror++;
  44         }
  45
  46         return parsetok(tok, g, start, err_ret, flags);
  47 }
  48
  49
  50 /* Parse input coming from a file.  Return error code, print some errors. */
  51
  52 node *
  53 PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
  54                    char *ps1, char *ps2, perrdetail *err_ret)
  55 {
  56         return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
  57                                        err_ret, 0);
  58 }
  59
  60 node *
  61 PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
  62                         char *ps1, char *ps2, perrdetail *err_ret, int flags)
  63 {
  64         struct tok_state *tok;
  65
  66         initerr(err_ret, filename);
  67
  68         if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
  69                 err_ret->error = E_NOMEM;
  70                 return NULL;
  71         }
  72         if (Py_TabcheckFlag || Py_VerboseFlag) {
  73                 tok->filename = filename;
  74                 tok->altwarning = (filename != NULL);
  75                 if (Py_TabcheckFlag >= 2)
  76                         tok->alterror++;
  77         }
  78
  79
  80         return parsetok(tok, g, start, err_ret, flags);
  81 }
  82
  83 /* Parse input coming from the given tokenizer structure.
  84    Return error code. */
  85
  86 static char yield_msg[] =
  87 "%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
  88
  89 static node *
  90 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
  91          int flags)
  92 {
  93         parser_state *ps;
  94         node *n;
  95         int started = 0;
  96
  97         if ((ps = PyParser_New(g, start)) == NULL) {
  98                 fprintf(stderr, "no mem for new parser\n");
  99                 err_ret->error = E_NOMEM;
 100                 return NULL;
 101         }
 102         if (flags & PyPARSE_YIELD_IS_KEYWORD)
 103                 ps->p_generators = 1;
 104
 105         for (;;) {
 106                 char *a, *b;
 107                 int type;
 108                 size_t len;
 109                 char *str;
 110
 111                 type = PyTokenizer_Get(tok, &a, &b);
 112                 if (type == ERRORTOKEN) {
 113                         err_ret->error = tok->done;
 114                         break;
 115                 }
 116                 if (type == ENDMARKER && started) {
 117                         type = NEWLINE; /* Add an extra newline */
 118                         started = 0;
 119                 }
 120                 else
 121                         started = 1;
 122                 len = b - a; /* XXX this may compute NULL - NULL */
 123                 str = PyMem_NEW(char, len + 1);
 124                 if (str == NULL) {
 125                         fprintf(stderr, "no mem for next token\n");
 126                         err_ret->error = E_NOMEM;
 127                         break;
 128                 }
 129                 if (len > 0)
 130                         strncpy(str, a, len);
 131                 str[len] = '\0';
 132
 133                 /* Warn about yield as NAME */
 134                 if (type == NAME && !ps->p_generators &&
 135                     len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
 136                         PySys_WriteStderr(yield_msg,
 137                                           err_ret->filename==NULL ?
 138                                           "<string>" : err_ret->filename,
 139                                           tok->lineno);
 140
 141                 if ((err_ret->error =
 142                      PyParser_AddToken(ps, (int)type, str, tok->lineno,
 143                                        &(err_ret->expected))) != E_OK) {
 144                         if (err_ret->error != E_DONE)
 145                                 PyMem_DEL(str);
 146                         break;
 147                 }
 148         }
 149
 150         if (err_ret->error == E_DONE) {
 151                 n = ps->p_tree;
 152                 ps->p_tree = NULL;
 153         }
 154         else
 155                 n = NULL;
 156
 157         PyParser_Delete(ps);
 158
 159         if (n == NULL) {
 160                 if (tok->lineno <= 1 && tok->done == E_EOF)
 161                         err_ret->error = E_EOF;
 162                 err_ret->lineno = tok->lineno;
 163                 err_ret->offset = tok->cur - tok->buf;
 164                 if (tok->buf != NULL) {
 165                         size_t len = tok->inp - tok->buf;
 166                         err_ret->text = PyMem_NEW(char, len + 1);
 167                         if (err_ret->text != NULL) {
 168                                 if (len > 0)
 169                                         strncpy(err_ret->text, tok->buf, len);
 170                                 err_ret->text[len] = '\0';
 171                         }
 172                 }
 173         }
 174
 175         PyTokenizer_Free(tok);
 176
 177         return n;
 178 }
 179
 180 static void
 181 initerr(perrdetail *err_ret, char* filename)
 182 {
 183         err_ret->error = E_OK;
 184         err_ret->filename = filename;
 185         err_ret->lineno = 0;
 186         err_ret->offset = 0;
 187         err_ret->text = NULL;
 188         err_ret->token = -1;
 189         err_ret->expected = -1;
 190 }