Parser/parsetok.c

   1
   2 /* Parser-tokenizer link implementation */
   3
   4 #include "pgenheaders.h"
   5 #include "tokenizer.h"
   6 #include "node.h"
   7 #include "grammar.h"
   8 #include "parser.h"
   9 #include "parsetok.h"
  10 #include "errcode.h"
  11
  12 int Py_TabcheckFlag;
  13
  14
  15 /* Forward */
  16 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *);
  17
  18 /* Parse input coming from a string.  Return error code, print some errors. */
  19
  20 node *
  21 PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
  22 {
  23         struct tok_state *tok;
  24
  25         err_ret->error = E_OK;
  26         err_ret->filename = NULL;
  27         err_ret->lineno = 0;
  28         err_ret->offset = 0;
  29         err_ret->text = NULL;
  30         err_ret->token = -1;
  31         err_ret->expected = -1;
  32
  33         if ((tok = PyTokenizer_FromString(s)) == NULL) {
  34                 err_ret->error = E_NOMEM;
  35                 return NULL;
  36         }
  37
  38         if (Py_TabcheckFlag || Py_VerboseFlag) {
  39                 tok->filename = "<string>";
  40                 tok->altwarning = (tok->filename != NULL);
  41                 if (Py_TabcheckFlag >= 2)
  42                         tok->alterror++;
  43         }
  44
  45         return parsetok(tok, g, start, err_ret);
  46 }
  47
  48
  49 /* Parse input coming from a file.  Return error code, print some errors. */
  50
  51 node *
  52 PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
  53                    char *ps1, char *ps2, perrdetail *err_ret)
  54 {
  55         struct tok_state *tok;
  56
  57         err_ret->error = E_OK;
  58         err_ret->filename = filename;
  59         err_ret->lineno = 0;
  60         err_ret->offset = 0;
  61         err_ret->text = NULL;
  62
  63         if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
  64                 err_ret->error = E_NOMEM;
  65                 return NULL;
  66         }
  67         if (Py_TabcheckFlag || Py_VerboseFlag) {
  68                 tok->filename = filename;
  69                 tok->altwarning = (filename != NULL);
  70                 if (Py_TabcheckFlag >= 2)
  71                         tok->alterror++;
  72         }
  73
  74
  75         return parsetok(tok, g, start, err_ret);
  76 }
  77
  78 /* Parse input coming from the given tokenizer structure.
  79    Return error code. */
  80
  81 static node *
  82 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
  83 {
  84         parser_state *ps;
  85         node *n;
  86         int started = 0;
  87
  88         if ((ps = PyParser_New(g, start)) == NULL) {
  89                 fprintf(stderr, "no mem for new parser\n");
  90                 err_ret->error = E_NOMEM;
  91                 return NULL;
  92         }
  93
  94         for (;;) {
  95                 char *a, *b;
  96                 int type;
  97                 size_t len;
  98                 char *str;
  99
 100                 type = PyTokenizer_Get(tok, &a, &b);
 101                 if (type == ERRORTOKEN) {
 102                         err_ret->error = tok->done;
 103                         break;
 104                 }
 105                 if (type == ENDMARKER && started) {
 106                         type = NEWLINE; /* Add an extra newline */
 107                         started = 0;
 108                 }
 109                 else
 110                         started = 1;
 111                 len = b - a; /* XXX this may compute NULL - NULL */
 112                 str = PyMem_NEW(char, len + 1);
 113                 if (str == NULL) {
 114                         fprintf(stderr, "no mem for next token\n");
 115                         err_ret->error = E_NOMEM;
 116                         break;
 117                 }
 118                 if (len > 0)
 119                         strncpy(str, a, len);
 120                 str[len] = '\0';
 121                 if ((err_ret->error =
 122                      PyParser_AddToken(ps, (int)type, str, tok->lineno,
 123                                        &(err_ret->expected))) != E_OK) {
 124                         if (err_ret->error != E_DONE)
 125                                 PyMem_DEL(str);
 126                         break;
 127                 }
 128         }
 129
 130         if (err_ret->error == E_DONE) {
 131                 n = ps->p_tree;
 132                 ps->p_tree = NULL;
 133         }
 134         else
 135                 n = NULL;
 136
 137         PyParser_Delete(ps);
 138
 139         if (n == NULL) {
 140                 if (tok->lineno <= 1 && tok->done == E_EOF)
 141                         err_ret->error = E_EOF;
 142                 err_ret->lineno = tok->lineno;
 143                 err_ret->offset = tok->cur - tok->buf;
 144                 if (tok->buf != NULL) {
 145                         size_t len = tok->inp - tok->buf;
 146                         err_ret->text = PyMem_NEW(char, len + 1);
 147                         if (err_ret->text != NULL) {
 148                                 if (len > 0)
 149                                         strncpy(err_ret->text, tok->buf, len);
 150                                 err_ret->text[len] = '\0';
 151                         }
 152                 }
 153         }
 154
 155         PyTokenizer_Free(tok);
 156
 157         return n;
 158 }