git-export-filter.c: avoid extra LFs and grok get-mark

[git-export-filter.git] / git-export-filter.c

/*

git-export-filter.c -- filter/transform git fast-export data streams
Copyright (C) 2013,2014,2019 Kyle J. McKay.  All rights reserved.

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

*/

#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define MAXLINE  2047
#define COPYSIZE 65536

#define SPACETAB " \t"
#define DIGITS   "0123456789"

typedef struct transform_s {
  char *from;
  char *to;
} transform_t;

static const char *const gHelp =
#include "git-export-help.inc"
;

static const char *const gUsage =
"git-export-filter [--authors-file file] [--branches-file file]\n"
"  [--convert-tagger id] [--require-authors] [--trunk-is-master]\n"
"  [--strip-at-suffix] [--expand-renames] [in] > out\n"
"(use git-export-filter -v -h for detailed help)\n";

static const char *const gVersion =
"git-export-filter version 1.5.1\n";

static const char *me = "git-export-filter";

static const char *authorsfile = NULL;
static const char *branchesfile = NULL;
static const char *convertid = NULL;
static int opt_verbose = 0;
static int debug = 0;
static int opt_version = 0;
static int opt_help = 0;
static int opt_require = 0;
static int opt_trunk_is_master = 0;
static int opt_strip_at = 0;
static int opt_no_renames = 0;
static int opt_names = 0;

static char *pushline = NULL;
static char *copybuff;

int (*fout)(FILE *out, const char *fmt, ...);
size_t (*writeout)(const void *ptr, size_t size, size_t nitems, FILE *out);

static void processfile(FILE *in, FILE *out,
                        const transform_t *authors, size_t acount,
                        const transform_t *branches, size_t bcount,
                        const char *convertid);

static void setupme(const char *start)
{
  if (start && *start) {
    const char *last = strrchr(start, '/');
    if (last && last[1])
      me = last+1;
    else
      me = start;
  }
}

static void die(const char *fmt, ...)
{
  va_list args;
  size_t len;

  if (!fmt)
    fmt="error";
  va_start(args, fmt);
  fflush(stdout);
  fprintf(stderr, "%s: ", me);
  vfprintf(stderr, fmt, args);
  va_end(args);
  len = strlen(fmt);
  if (!len || fmt[len-1] != '\n')
    fprintf(stderr, "\n");
  fflush(stderr);
  exit(1);
}

static int cmpxform(const void *_a, const void *_b)
{
  const transform_t *a = (const transform_t *)_a;
  const transform_t *b = (const transform_t *)_b;
  return strcmp(a->from, b->from);
}

static void trimback(char *str, const char *chars)
{
  size_t len;
  if (!str) return;
  len = strlen(str);
  while (strchr(chars, str[--len]))
    str[len] = 0;
}

static int read_transform_file(const char *type, FILE *f, transform_t **ans)
{
  transform_t *xform = NULL;
  int cnt = 0;
  char fmt[16];
  char line[MAXLINE+2];
  sprintf(fmt, "%%%d[^\r\n]", MAXLINE+1);
  for (;;) {
    int e;
    line[0] = 0;
    e = fscanf(f, fmt, line);
    if (e < 0) break;
    if (e == 1) {
      char *str;
      if (strlen(line) > MAXLINE)
        die("%s file line exceeded %d characters", type, MAXLINE);
      str = line + strspn(line, SPACETAB);
      if (*str && *str != '#') {
        size_t idlen = strcspn(str, "=");
        char *from;
        if (!str[idlen])
          die("invalid %s line (no '='): %s", type, str);
        if (!idlen)
          die("invalid %s line (empty before '='): %s", type, str);
        str[idlen] = 0;
        from = str + idlen + 1;
        from += strspn(from, SPACETAB);
        if (!*from)
          die("invalid %s line (empty after '='): %s", type, str);
        trimback(str, SPACETAB);
        trimback(from, SPACETAB);
        if (debug > 1)
          fprintf(stderr, "FROM: %s TO: %s\n", str, from);
        xform = realloc(xform, sizeof(transform_t) * (cnt + 1));
        if (!xform)
          die("out of memory allocating %s array", type);
        xform[cnt].from = (char *)malloc(idlen + 1 + strlen(from) + 1);
        if (!xform[cnt].from)
          die("out of memory allocating %s array", type);
        memcpy(xform[cnt].from, str, idlen + 1);
        xform[cnt].to = xform[cnt].from + idlen + 1;
        strcpy(xform[cnt].to, from);
        ++cnt;
      }
    }
    e = fscanf(f, "%*[\r\n]");
    if (e < 0) break;
  }
  if (feof(f)) {
    qsort(xform, cnt, sizeof(transform_t), cmpxform);
    *ans = xform;
    return cnt;
  }
  if (xform)
    free(xform);
  return -1;
}

static int foutnone(FILE *out, const char *fmt, ...)
{
  (void)out;
  (void)fmt;
  return 0;
}

size_t writeoutnone(const void *ptr, size_t size, size_t nitems, FILE *out)
{
  (void)ptr;
  (void)size;
  (void)out;
  return nitems;
}

int main(int argc, char *argv[])
{
  transform_t *authors = NULL;
  int acount = 0;
  transform_t *branches = NULL;
  int bcount = 0;
  FILE *inbinary = freopen(NULL, "rb", stdin);
  FILE *outbinary = freopen(NULL, "wb", stdout);
  int optind = 1;

  fout = fprintf;
  writeout = fwrite;
  if (argc >= 1)
    setupme(argv[0]);
  copybuff = (char *)malloc(COPYSIZE);
  if (!copybuff)
    die("out of memory allocating copy buffer");
  if (!inbinary)
    die("freopen(NULL, \"rb\", stdin) failed");
  if (!outbinary)
    die("freopen(NULL, \"wb\", stdout) failed");
  for (; optind < argc; ++optind) {
#define A argv[optind]
    if (strcmp(A, "--authors-file") == 0 || strcmp(A, "-A") == 0) {
      if (++optind >= argc || !A || !*A)
        die("--authors-file requires a filename argument");
      authorsfile = A;
      continue;
    }
    if (strncmp(A, "--authors-file=", 15) == 0) {
      const char *arg = A + 15;
      if (!*arg)
        die("--authors-file requires a filename argument");
      authorsfile = arg;
      continue;
    }
    if (strcmp(A, "--branches-file") == 0) {
      if (++optind >= argc || !A || !*A)
        die("--branches-file requires a filename argument");
      branchesfile = A;
      continue;
    }
    if (strncmp(A, "--branches-file=", 16) == 0) {
      const char *arg = A + 16;
      if (!*arg)
        die("--branches-file requires a filename argument");
      branchesfile = arg;
      continue;
    }
    if (strcmp(A, "--convert-tagger") == 0) {
      if (convertid)
        die("--convert-tagger may only be given once");
      if (++optind >= argc || !A || !*A)
        die("--convert-tagger requires an argument");
      convertid = A;
      continue;
    }
    if (strncmp(A, "--convert-tagger=", 17) == 0) {
      const char *arg = A + 17;
      if (convertid)
        die("--convert-tagger may only be given once");
      if (!*arg)
        die("--convert-tagger requires an argument");
      convertid = A;
      continue;
    }
    if (!strcmp(A, "--require-authors")) {
      opt_require = 1;
      continue;
    }
    if (!strcmp(A, "--trunk-is-master")) {
      opt_trunk_is_master = 1;
      continue;
    }
    if (!strcmp(A, "--strip-at-suffix")) {
      opt_strip_at = 1;
      continue;
    }
    if (!strcmp(A, "--expand-renames")) {
      opt_no_renames = 1;
      continue;
    }
    if (!strcmp(A, "--names")) {
      opt_names = 1;
      fout = foutnone;
      writeout = writeoutnone;
      continue;
    }
    if (!strcmp(A, "-V") || !strcmp(A, "--version")) {
      opt_version = 1;
      continue;
    }
    if (!strcmp(A, "-v") || !strcmp(A, "--verbose")) {
      opt_verbose = 1;
      continue;
    }
    if (!strcmp(A, "-h") || !strcmp(A, "--help")) {
      opt_help = 1;
      continue;
    }
    if (!strcmp(A, "-d") || !strcmp(A, "--debug")) {
      ++debug;
      continue;
    }
    if (strcmp(A, "--") == 0) {
      ++optind;
      break;
    }
    if (*A != '-' || !A[1])
      break;
    die("unknown option: %s", A);
#undef A
  }
  if (optind + 1 < argc)
    die("no more than one non-option argument allowed (try -h)");
  if (optind + 1 == argc && strcmp(argv[optind], "-")) {
    inbinary = freopen(argv[optind], "rb", inbinary);
    if (!inbinary)
      die("cannot open file %s", argv[optind]);
  }
  if (opt_version)
    printf("%s", gVersion);
  if (opt_help)
    printf("%s", opt_verbose ? gHelp : gUsage);
  if (opt_version || opt_help)
    exit(0);
  if (opt_require && !authorsfile)
    die("--require-authors requires the --authors-file option");
  if (opt_names && (opt_require || authorsfile || branchesfile || convertid ||
                    opt_trunk_is_master || opt_strip_at || opt_no_renames))
    die("--names may not be used together with any other options");
  if (authorsfile) {
    FILE *af = fopen(authorsfile, "rb");
    if (!af)
      die("cannot open authors file: %s", authorsfile);
    acount = read_transform_file("authors", af, &authors);
    fclose(af);
    if (acount < 0)
      die("invalid authors file format: %s", authorsfile);
    if (debug && acount) {
      int i;
      for (i=0; i<acount; ++i)
        fprintf(stderr, "%s=%s\n", authors[i].from, authors[i].to);
    }
  }
  if (branchesfile) {
    FILE *bf = fopen(branchesfile, "rb");
    if (!bf)
      die("cannot open branches file: %s", branchesfile);
    bcount = read_transform_file("branches", bf, &branches);
    fclose(bf);
    if (bcount < 0)
      die("invalid branches file format: %s", branchesfile);
    if (debug && bcount) {
      int i;
      for (i=0; i<bcount; ++i)
        fprintf(stderr, "%s->%s\n", branches[i].from, branches[i].to);
    }
  }
  if (opt_trunk_is_master) {
    branches = realloc(branches, sizeof(transform_t) * (bcount + 1));
    if (!branches)
      die("out of memory allocating branches array");
    branches[bcount].from = "refs/heads/trunk";
    branches[bcount].to = "refs/heads/master";
    ++bcount;
    qsort(branches, bcount, sizeof(transform_t), cmpxform);
  }

  processfile(inbinary, outbinary, authors, (size_t)acount, branches,
              (size_t)bcount, convertid);

  exit(0);
}

static char *nextline(FILE *in)
{
  static char line[MAXLINE+2];
  char fmt[16];
  int e;

  if (pushline) {
    char *ans = pushline;
    pushline = NULL;
    if (*ans)
      return ans;
  }
  sprintf(fmt, "%%%d[^\r\n]", MAXLINE+1);
  line[0] = 0;
  e = fscanf(in, fmt, line);
  if (e < 0 && !feof(in))
    die("error reading input");
  if (e < 0)
    return NULL;
  if (strlen(line) > MAXLINE)
    die("input line exceeded %d characters", MAXLINE);
  do {
    e = fgetc(in);
  } while (e >= 0 && e != '\n');
  if (e < 0 && !feof(in))
    die("error reading input");
  return line + strspn(line, SPACETAB);
}

static void processblob(FILE *in, FILE *out);
static void processcommit(FILE *in, FILE *out, const char *ref,
                          const transform_t *authors, size_t acount,
                          const transform_t *branches, size_t bcount);
static void processtag(FILE *in, FILE *out, const char *tag,
                       const transform_t *authors, size_t acount,
                       const transform_t *branches, size_t bcount,
                       const char *convertid);
static void processreset(FILE *in, FILE *out, const char *ref,
                         const transform_t *branches, size_t bcount);
 
static void processfile(FILE *in, FILE *out,
                        const transform_t *authors, size_t acount,
                        const transform_t *branches, size_t bcount,
                        const char *convertid)
{
  const char *line;
  for (;;) {
    line = nextline(in);
    if (!line) break;
    if (!*line || *line == '#') continue;
    if (strcmp(line, "blob") == 0)
      processblob(in, out);
    else if (strncmp(line, "commit ", 7) == 0)
      processcommit(in, out, line+7, authors, acount, branches, bcount);
    else if (strncmp(line, "tag ", 4) == 0)
      processtag(in, out, line+4, authors, acount, branches, bcount, convertid);
    else if (strncmp(line, "reset ", 6) == 0)
      processreset(in, out, line+6, branches, bcount);
    else if (!strcmp(line, "checkpoint") || !strcmp(line, "done") ||
             !strncmp(line, "progress ", 9))
      fout(out, "%s\n\n", line);
    else if (!strncmp(line, "get-mark ", 9) || !strncmp(line, "cat-blob ", 9) ||
             !strncmp(line, "ls ", 3) || !strncmp(line, "feature ", 8) ||
             !strncmp(line, "option ", 7))
      fout(out, "%s\n", line);
    else
      die("unrecognized input command: %s", line);
   }
}

static const char *translate(const char *in, const transform_t *x, size_t c)
{
  const transform_t *t;
  transform_t search;
  search.from = (char *)in;
  search.to = NULL;
  t = (transform_t *)bsearch(&search, x, c, sizeof(transform_t), cmpxform);
  return t ? t->to : in;
}

#define translateref(i,x,c) translate((i),(x),(c))

static const char *translateuser(const char *in, const transform_t *x, size_t c)
{
  const transform_t *t;
  transform_t search;
  search.from = (char *)in;
  search.to = NULL;
  t = (transform_t *)bsearch(&search, x, c, sizeof(transform_t), cmpxform);
  if (!t && opt_strip_at && strchr(in, '@')) {
    char user[MAXLINE+1];
    strcpy(user, in);
    *strchr(user, '@') = 0;
    search.from = user;
    t = (transform_t *)bsearch(&search, x, c, sizeof(transform_t), cmpxform);
  }
  return t ? t->to : in;
}

static const char *translatetag(const char *in, const transform_t *x, size_t c)
{
  const transform_t *t;
  transform_t search = {NULL, NULL};
  char *temptag = malloc(10 + strlen(in) + 1);
  if (!temptag)
    die("out of memory");
  sprintf(temptag, "refs/tags/%s", in);
  search.from = temptag;
  t = (transform_t *)bsearch(&search, x, c, sizeof(transform_t), cmpxform);
  if (!t || strncmp(t->to, "refs/tags/", 10) != 0)
    return in;
  return t->to + 10;
}

static int splitauthor(char *line, char **name, char **email, char **when)
{
  char *lt = strchr(line, '<');
  char *gt = strchr(line, '>');
  if (!lt || !gt || gt <= lt)
    return 0;
  *lt = 0;
  *name = line;
  *gt = 0;
  *email = lt+1;
  *when = gt+1;
  if (**when == ' ')
    ++*when;
  return 1;
}

static void copydatapart(FILE *in, FILE *out, const char *data, int nolf)
{
  char *line;
  char lastchar = 0;
  size_t l = strlen(data);
  if (strspn(data, DIGITS) == l) {
    size_t dlen;
    if (!l)
      die("Invalid data line: data %s", data);
    dlen = (size_t)strtol(data, NULL, 10);
    while (dlen) {
      size_t cnt, amnt = COPYSIZE;
      if (amnt > dlen)
        amnt = dlen;
      cnt = fread(copybuff, 1, amnt, in);
      if (!cnt)
        break;
      if (out)
        if (writeout(copybuff, cnt, 1, out) != 1)
          die("failed writing data to output");
      if (cnt == dlen)
        lastchar = copybuff[cnt - 1];
      dlen -= cnt;
    }
    if (dlen)
      die("unexpected EOF reading data %s", data);
    if (out && (!nolf || lastchar != '\n'))
      fout(out, "\n");
  } else if (l < 3 || data[0] != '<' || data[1] != '<')
    die("Invalid data line: data %s", data);
  else {
    fprintf(stderr, "%s: warning: data << not fully supported\n", me);
    for (;;) {
      const char *line = nextline(in);
      if (!line)
        die("unexpected EOF reading data %s", data);
      if (out)
        fout(out, "%s\n", line);
      if (strcmp(line, data+2) == 0) {
        if (out && !nolf)
          fout(out, "\n");
        break;
      }
    }
  }
  line = nextline(in);
  if (line && *line)
    pushline = line;
}

static void copydata(FILE *in, FILE *out, int nolf, const char *err)
{
  const char *line = nextline(in);
  if (!err)
    err = "missing data line";
  if (!line || strncmp(line, "data ", 5) != 0)
    die(err);
  if (out)
    fout(out, "%s\n", line);
  copydatapart(in, out, line+5, nolf);
}

static void processblob(FILE *in, FILE *out)
{
  const char *line = nextline(in);
  if (!line)
    die("error reading blob header");
  fout(out, "blob\n");
  if (strncmp(line, "mark ", 5) == 0) {
    fout(out, "%s\n", line);
    line = nextline(in);
    if (!line)
      die("error reading blob header");
  }
  if (strncmp(line, "data ", 5) != 0)
    die("blob missing data line");
  fout(out, "%s\n", line);
  copydatapart(in, out, line+5, 0);
}

static void processreset(FILE *in, FILE *out, const char *ref,
                         const transform_t *branches, size_t bcount)
{
  char *line;
  const char *newref = translateref(ref, branches, bcount);
  fout(out, "reset %s\n", newref);
  line = nextline(in);
  if (strncmp(line, "from ", 5) == 0)
    fout(out, "%s\n\n", line);
  else
    pushline = line;
}

static void processtag(FILE *in, FILE *out, const char *tag,
                       const transform_t *authors, size_t acount,
                       const transform_t *branches, size_t bcount,
                       const char *convertid)
{
  char *name = NULL, *email = NULL, *when = NULL;
  char *line;
  const char *newref = translatetag(tag, branches, bcount);
  const char *newauth;
  char tagline[MAXLINE+1];
  char fromline[MAXLINE+1];
  sprintf(tagline, "tag %s\n", newref);
  line = nextline(in);
  if (!line || strncmp(line, "from ", 5) != 0)
    die("tag missing from line");
  sprintf(fromline, "%s\n", line);
  line = nextline(in);
  if (!line || strncmp(line, "tagger ", 7) != 0)
    die("tag missing tagger line");
  if (!splitauthor(line+7, &name, &email, &when))
    die("tag has bad tagger line");
  if (opt_names)
    fprintf(out, "%s<%s>\n", name, email);
  if (convertid && strcmp(convertid, email) == 0) {
    fout(out, "reset refs/tags/%s", tagline+4);
    fout(out, "%s\n", fromline);
    out = NULL;
  } else {
    fout(out, "%s%s", tagline, fromline);
    newauth = translateuser(email, authors, acount);
    if (newauth != email)
      fout(out, "tagger %s %s\n", newauth, when);
    else {
      if (opt_require)
        die("missing authors file author: \"%s\"", email);
      fout(out, "tagger %s<%s> %s\n", name, email, when);
    }
  }
  copydata(in, out, 0, "tag missing data line");
}

static const char *find_second_space(const char *line)
{
  if (!line || !*line || *line == ' ' || line[1] != ' ' || line[2] == ' ')
    return NULL;
  if (line[2] != '"')
    return strchr(line + 2, ' ');
  line += 3;
  for (;;) {
    line += strcspn(line, "\\\"");
    if (!*line)
      return NULL;
    if (*line == '\\') {
      if (!line[1])
        return NULL;
      line += 2;
      continue;
    }
    return line[1] == ' ' ? line + 1 : NULL;
  }
}

static int is_inline_modify(const char *line)
{
  if (strncmp(line, "M ", 2) != 0)
    return 0;
  line += 2;
  line += strspn(line, DIGITS);
  return strncmp(line, " inline ", 8) == 0;
}

static void processcommit(FILE *in, FILE *out, const char *ref,
                          const transform_t *authors, size_t acount,
                          const transform_t *branches, size_t bcount)
{
  char *name = NULL, *email = NULL, *when = NULL;
  char *line;
  const char *newref = translateref(ref, branches, bcount);
  const char *newauth;
  fout(out, "commit %s\n", newref);
  line = nextline(in);
  if (!line)
    die("error reading commit header");
  if (strncmp(line, "mark ", 5) == 0) {
    fout(out, "%s\n", line);
    line = nextline(in);
    if (!line)
      die("error reading commit header");
  }
  if (strncmp(line, "author ", 7) == 0) {
    if (!splitauthor(line+7, &name, &email, &when))
      die("commit has bad author line");
    if (opt_names)
      fprintf(out, "%s<%s>\n", name, email);
    newauth = translateuser(email, authors, acount);
    if (newauth != email)
      fout(out, "author %s %s\n", newauth, when);
    else {
      if (opt_require)
        die("missing authors file author: \"%s\"", email);
      fout(out, "author %s<%s> %s\n", name, email, when);
    }
    line = nextline(in);
    if (!line)
      die("error reading commit header");
  }
  if (strncmp(line, "committer ", 10) != 0)
    die("commit missing committer line");
  if (!splitauthor(line+10, &name, &email, &when))
    die("commit has bad committer line");
  if (opt_names)
    fprintf(out, "%s<%s>\n", name, email);
  newauth = translateuser(email, authors, acount);
  if (newauth != email)
    fout(out, "committer %s %s\n", newauth, when);
  else {
    if (opt_require)
      die("missing authors file author: \"%s\"", email);
    fout(out, "committer %s<%s> %s\n", name, email, when);
  }
  copydata(in, out, 1, "commit missing data line");
  line = nextline(in);
  if (!line)
    die("error reading commit header");
  if (strncmp(line, "from ", 5) == 0) {
    fout(out, "%s\n", line);
    line = nextline(in);
    if (!line)
      die("error reading commit header");
  }
  while (strncmp(line, "merge ", 6) == 0) {
    fout(out, "%s\n", line);
    line = nextline(in);
    if (!line)
      die("error reading commit header");
  }
  while (!strcmp(line, "deleteall")     || !strncmp(line, "M ", 2) ||
         !strncmp(line, "D ", 2)        || !strncmp(line, "C ", 2) ||
         !strncmp(line, "R ", 2)        || !strncmp(line, "N ", 2) ||
         !strncmp(line, "cat-blob ", 9) || !strncmp(line, "ls ", 3) ||
         !strncmp(line, "get-mark ", 9)) {
    if (!opt_no_renames || strncmp(line, "R ", 2)) {
      fout(out, "%s\n", line);
    } else {
      /* expand rename into copy + delete */
      const char *space2 = find_second_space(line);
      if (!space2)
        die("error reading 'R' line");
      fout(out, "C %s\n", line+2);
      fout(out, "D %.*s\n", (int)(space2 - line) - 2, line+2);
    }
    if (strncmp(line, "N inline", 8) == 0 || is_inline_modify(line))
      copydata(in, out, 1, "inline N or M missing data line");
    line = nextline(in);
    if (!line)
      die("error reading commit header");
  }
  pushline = line;
  fout(out, "\n");
}