Parser/parsetok.c

   1
   2 /* Parser-tokenizer link implementation */
   3
   4 #include "pgenheaders.h"
   5 #include "tokenizer.h"
   6 #include "node.h"
   7 #include "grammar.h"
   8 #include "parser.h"
   9 #include "parsetok.h"
  10 #include "errcode.h"
  11 #include "graminit.h"
  12
  13 int Py_TabcheckFlag;
  14
  15
  16 /* Forward */
  17 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
  18 static void initerr(perrdetail *err_ret, const char* filename);
  19
  20 /* Parse input coming from a string.  Return error code, print some errors. */
  21 node *
  22 PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
  23 {
  24         return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
  25 }
  26
  27 node *
  28 PyParser_ParseStringFlags(const char *s, grammar *g, int start,
  29                           perrdetail *err_ret, int flags)
  30 {
  31         return PyParser_ParseStringFlagsFilename(s, NULL,
  32                                                  g, start, err_ret, 0);
  33 }
  34
  35 node *
  36 PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
  37                           grammar *g, int start,
  38                           perrdetail *err_ret, int flags)
  39 {
  40         struct tok_state *tok;
  41
  42         initerr(err_ret, filename);
  43
  44         if ((tok = PyTokenizer_FromString(s)) == NULL) {
  45                 err_ret->error = E_NOMEM;
  46                 return NULL;
  47         }
  48
  49         tok->filename = filename ? filename : "<string>";
  50         if (Py_TabcheckFlag || Py_VerboseFlag) {
  51                 tok->altwarning = (tok->filename != NULL);
  52                 if (Py_TabcheckFlag >= 2)
  53                         tok->alterror++;
  54         }
  55
  56         return parsetok(tok, g, start, err_ret, flags);
  57 }
  58
  59
  60 /* Parse input coming from a file.  Return error code, print some errors. */
  61
  62 node *
  63 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
  64                    char *ps1, char *ps2, perrdetail *err_ret)
  65 {
  66         return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
  67                                        err_ret, 0);
  68 }
  69
  70 node *
  71 PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
  72                         char *ps1, char *ps2, perrdetail *err_ret, int flags)
  73 {
  74         struct tok_state *tok;
  75
  76         initerr(err_ret, filename);
  77
  78         if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
  79                 err_ret->error = E_NOMEM;
  80                 return NULL;
  81         }
  82         tok->filename = filename;
  83         if (Py_TabcheckFlag || Py_VerboseFlag) {
  84                 tok->altwarning = (filename != NULL);
  85                 if (Py_TabcheckFlag >= 2)
  86                         tok->alterror++;
  87         }
  88
  89
  90         return parsetok(tok, g, start, err_ret, flags);
  91 }
  92
  93 /* Parse input coming from the given tokenizer structure.
  94    Return error code. */
  95
  96 #if 0 /* future keyword */
  97 static char yield_msg[] =
  98 "%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
  99 #endif
 100
 101 static node *
 102 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 103          int flags)
 104 {
 105         parser_state *ps;
 106         node *n;
 107         int started = 0;
 108
 109         if ((ps = PyParser_New(g, start)) == NULL) {
 110                 fprintf(stderr, "no mem for new parser\n");
 111                 err_ret->error = E_NOMEM;
 112                 return NULL;
 113         }
 114 #if 0 /* future keyword */
 115         if (flags & PyPARSE_YIELD_IS_KEYWORD)
 116                 ps->p_generators = 1;
 117 #endif
 118
 119         for (;;) {
 120                 char *a, *b;
 121                 int type;
 122                 size_t len;
 123                 char *str;
 124
 125                 type = PyTokenizer_Get(tok, &a, &b);
 126                 if (type == ERRORTOKEN) {
 127                         err_ret->error = tok->done;
 128                         break;
 129                 }
 130                 if (type == ENDMARKER && started) {
 131                         type = NEWLINE; /* Add an extra newline */
 132                         started = 0;
 133                 }
 134                 else
 135                         started = 1;
 136                 len = b - a; /* XXX this may compute NULL - NULL */
 137                 str = (char *) PyObject_MALLOC(len + 1);
 138                 if (str == NULL) {
 139                         fprintf(stderr, "no mem for next token\n");
 140                         err_ret->error = E_NOMEM;
 141                         break;
 142                 }
 143                 if (len > 0)
 144                         strncpy(str, a, len);
 145                 str[len] = '\0';
 146
 147 #if 0 /* future keyword */
 148                 /* Warn about yield as NAME */
 149                 if (type == NAME && !ps->p_generators &&
 150                     len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
 151                         PySys_WriteStderr(yield_msg,
 152                                           err_ret->filename==NULL ?
 153                                           "<string>" : err_ret->filename,
 154                                           tok->lineno);
 155 #endif
 156
 157                 if ((err_ret->error =
 158                      PyParser_AddToken(ps, (int)type, str, tok->lineno,
 159                                        &(err_ret->expected))) != E_OK) {
 160                         if (err_ret->error != E_DONE)
 161                                 PyObject_FREE(str);
 162                         break;
 163                 }
 164         }
 165
 166         if (err_ret->error == E_DONE) {
 167                 n = ps->p_tree;
 168                 ps->p_tree = NULL;
 169         }
 170         else
 171                 n = NULL;
 172
 173         PyParser_Delete(ps);
 174
 175         if (n == NULL) {
 176                 if (tok->lineno <= 1 && tok->done == E_EOF)
 177                         err_ret->error = E_EOF;
 178                 err_ret->lineno = tok->lineno;
 179                 err_ret->offset = tok->cur - tok->buf;
 180                 if (tok->buf != NULL) {
 181                         size_t len = tok->inp - tok->buf;
 182                         err_ret->text = (char *) PyObject_MALLOC(len + 1);
 183                         if (err_ret->text != NULL) {
 184                                 if (len > 0)
 185                                         strncpy(err_ret->text, tok->buf, len);
 186                                 err_ret->text[len] = '\0';
 187                         }
 188                 }
 189         } else if (tok->encoding != NULL) {
 190                 node* r = PyNode_New(encoding_decl);
 191                 r->n_str = tok->encoding;
 192                 r->n_nchildren = 1;
 193                 r->n_child = n;
 194                 tok->encoding = NULL;
 195                 n = r;
 196         }
 197
 198         PyTokenizer_Free(tok);
 199
 200         return n;
 201 }
 202
 203 static void
 204 initerr(perrdetail *err_ret, const char* filename)
 205 {
 206         err_ret->error = E_OK;
 207         err_ret->filename = filename;
 208         err_ret->lineno = 0;
 209         err_ret->offset = 0;
 210         err_ret->text = NULL;
 211         err_ret->token = -1;
 212         err_ret->expected = -1;
 213 }