java/pig-galago/src/com/yahoo/pig/impl/mapreduceExec/PigSplit.java

   1 /*
   2  * Copyright (c) 2007 Yahoo! Inc.  All rights reserved.
   3  * See accompanying LICENSE file.
   4  */
   5 package com.yahoo.pig.impl.mapreduceExec;
   6
   7 import java.io.DataInput;
   8 import java.io.DataOutput;
   9 import java.io.IOException;
  10
  11 import org.apache.hadoop.fs.FileSystem;
  12 import org.apache.hadoop.fs.Path;
  13 import org.apache.hadoop.mapred.InputSplit;
  14
  15 import com.yahoo.pig.StorageFunc;
  16 import com.yahoo.pig.builtin.PigStorage;
  17 import com.yahoo.pig.impl.PigContext;
  18 import com.yahoo.pig.impl.logicalLayer.parser.QueryParser;
  19
  20 public class PigSplit implements InputSplit {
  21     Path   file;
  22     long start;
  23     long length;
  24     String groupFunc;
  25     String applyFunc;
  26     int    index;
  27     String parser;
  28     FileSystem fs;
  29
  30     public PigSplit() {}
  31
  32     public PigSplit(FileSystem fs, Path path, String parser, String groupFunc, String mapFuncs, int index, long start, long length) {
  33         this.fs = fs;
  34         this.file = path;
  35         this.start = start;
  36         this.length = length;
  37         this.groupFunc = groupFunc;
  38         this.applyFunc = mapFuncs;
  39         this.index = index;
  40         this.parser = parser;
  41     }
  42
  43     public String getParser() {
  44         return parser;
  45     }
  46
  47     public long getStart() {
  48         return start;
  49     }
  50
  51     public long getLength() {
  52         return length;
  53     }
  54     public String getGroupFunction() {
  55         return groupFunc;
  56     }
  57
  58     public String getApplyFunction() {
  59         return applyFunc;
  60     }
  61
  62     public Path getPath() {
  63         return file;
  64     }
  65
  66     public int getIndex() {
  67         return index;
  68     }
  69
  70     public StorageFunc getLoadFunction() {
  71         StorageFunc loader = null;
  72         if (this.parser == null) {
  73             loader = new PigStorage();
  74         } else {
  75             try {
  76                 loader = (StorageFunc) PigContext.instantiateArgFunc(this.parser);
  77             }catch(Exception exp) {
  78                 exp.printStackTrace();
  79                 loader = new PigStorage();
  80             }
  81         }
  82         return loader;
  83     }
  84
  85
  86     public String[] getLocations() throws IOException {
  87         String hints[][] = fs.getFileCacheHints(file, start, length);
  88         int total = 0;
  89         for(int i = 0; i < hints.length; i++) {
  90             total += hints[i].length;
  91         }
  92         String locations[] = new String[total];
  93         int count = 0;
  94         for(int i = 0; i < hints.length; i++) {
  95             for(int j = 0; j < hints[i].length; j++) {
  96                 locations[count++] = hints[i][j];
  97             }
  98         }
  99         return locations;
 100     }
 101
 102     public void readFields(DataInput is) throws IOException {
 103         file = new Path(is.readUTF());
 104         start = is.readLong();
 105         length = is.readLong();
 106         groupFunc = is.readUTF();
 107         applyFunc = is.readUTF();
 108         index = is.readInt();
 109         parser = is.readUTF();
 110     }
 111
 112     public void write(DataOutput os) throws IOException {
 113         os.writeUTF(file.toString());
 114         os.writeLong(start);
 115         os.writeLong(length);
 116         os.writeUTF(groupFunc);
 117         os.writeUTF(applyFunc);
 118         os.writeInt(index);
 119         os.writeUTF(parser);
 120     }
 121 }