Update README.md

[voldemort.git] / build.gradle

apply plugin: 'java'
apply plugin: 'war'

buildscript {
    repositories { jcenter() }
    dependencies {
        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.4'
    }
}

apply plugin: 'com.github.johnrengelman.shadow'

def String getProjectProperty(String propertyName) {
    String propertyValue = "null"
    if (hasProperty(propertyName)) {
        propertyValue = this.properties[propertyName]
    }
    else {
        throw new GradleScriptException("PropertyName " + propertyName + " is not defined in properties file")
    }
    return propertyValue
}
def projectName = project.name

def sourceDir = getProjectProperty('src.dir')
def distDir = getProjectProperty('dist.dir')
def classesDir = getProjectProperty('classes.dir')
def javaDir = getProjectProperty('java.dir')
def resourcesDir = getProjectProperty('resources.dir')
def javaDocDir = getProjectProperty('javadoc.dir')

def voldTestClassesDir = getProjectProperty('testclasses.dir')

def commonTestSrcDir = getProjectProperty('commontestsrc.dir')
def unitTestSrcDir = getProjectProperty('unittestsrc.dir')
def intTestSrcDir = getProjectProperty('inttestsrc.dir')
def longTestSrcDir = getProjectProperty('longtestsrc.dir')

def voldVersion = getProjectProperty('curr.release')
def javacVersion = getProjectProperty('javac.version')

//This is the javaCompile variable version. Directly defining 'def version' will override this and cause nightmare
version = voldVersion

def archiveDirectoryName = projectName + '-' + version
def archiveDirectoryPath = distDir + "/" + archiveDirectoryName

def javadocEnabled = getProjectProperty('javadoc.enabled').toBoolean()

def deleteDirectoryContents(directory) {
    project.file(directory).deleteDir()
    project.file(directory).mkdirs()
}

def gobblinExcludes = {
    exclude group: 'org.apache.hive'
    exclude group: 'com.google.protobuf'
    exclude group: 'org.apache.avro'
    exclude group: 'com.linkedin.gobblin', module: 'gobblin-hive-registration'
}

allprojects {
    sourceCompatibility = javacVersion
    targetCompatibility = javacVersion
    compileJava.options.debug = true

    repositories {
        mavenCentral()
        maven {
            // For Hadoop dependencies
            url "https://repository.cloudera.com/artifactory/cloudera-repos/"
        }
        maven {
            // For BDB-Je dependencies
            url "http://download.oracle.com/maven/"
        }
    }

    // http://blog.joda.org/2014/02/turning-off-doclint-in-jdk-8-javadoc.html
    if (JavaVersion.current().java8Compatible) {
        tasks.withType(Javadoc) {
            options.addStringOption('Xdoclint:none', '-quiet')
        }
    }

    tasks.withType(Javadoc) {
        enabled = javadocEnabled
    }
}

sourceSets {
    main {
        java { srcDirs = [javaDir] }
        resources {
            srcDirs = [javaDir]
            include '**/*.xsd'
        }
        output.classesDir = classesDir
        output.resourcesDir = resourcesDir
    }
    test {
        java {
            srcDirs = [
                commonTestSrcDir,
                unitTestSrcDir,
                intTestSrcDir,
                longTestSrcDir
            ]
        }
        output.classesDir = voldTestClassesDir
    }
}

compileJava.doLast {
    project.copy {
        from (javaDir) { exclude '**/*.java','**/*.html','**/log4j.properties' }
        into classesDir
    }

    project.copy {
        // Theoretically this block can be replaced by including the log4j.properties in main resources.
        // But that causes the log4j.properties to be present in the voldJar . Not sure what is the
        // implication of this change, so avoiding it for now.
        from (javaDir) { include 'log4j.properties' }
        into resourcesDir
    }
}

javadoc {
    destinationDir = file(javaDocDir)
}

compileTestJava.doLast {
    project.copy {
        from (commonTestSrcDir) { exclude '**/*.java','**/*.html' }
        from (unitTestSrcDir) { exclude '**/*.java','**/*.html' }
        into voldTestClassesDir
    }
}

task testJar(type: Jar) {
    baseName = projectName + "-test"
    from sourceSets.test.output
    destinationDir = project.file(distDir)
}

jar {
    manifest {
        attributes 'Voldemort-Implementation-Version' : version,
        'Implementation-Title': 'Voldemort',
        'Implementation-Version': version,
        'Implementation-Vendor' :'LinkedIn'
    }
    destinationDir = project.file(distDir)
}


task contribJar(type:Jar) {
    baseName = projectName + "-contrib"
    from {subprojects*.sourceSets.main.output}
    destinationDir = project.file(distDir)
}

task srcJar(type: Jar, dependsOn: classes) {
    classifier = 'sources'
    from sourceSets.main.java.srcDirs
    destinationDir = project.file(distDir)
}

task javadocJar(type: Jar) {
    enabled = javadocEnabled
    classifier = 'javadoc'
    from javadoc
    destinationDir = file(distDir)
}

task bnpJar(dependsOn: shadowJar) {
    // Just a nicer more self-explanatory name than "shadowJar"
}

artifacts {
    archives jar
    archives testJar
    archives contribJar
    archives srcJar
    archives javadocJar
}

clean {
    delete(distDir)
    delete('lib')
    doLast { deleteDirectoryContents(javaDocDir) }
}

// Dependencies used by both BnP and Voldemort
// TODO: Decide if we want to do that for all dependencies, even if they're used just in Voldemort...

def depAvro = 'org.apache.avro:avro:1.4.0'
def depProtoBuf = 'com.google.protobuf:protobuf-java:2.3.0'
def depJdom = 'org.jdom:jdom:1.1'
def depAzkaban = 'com.linkedin.azkaban:azkaban:2.5.0'
def depGuava = 'com.google.guava:guava:14.0.1'
def depLog4j = 'log4j:log4j:1.2.15'
def depJacksonMapper = 'org.codehaus.jackson:jackson-mapper-asl:1.9.13'
def depJoda = 'joda-time:joda-time:1.6'
def depTehuti = 'io.tehuti:tehuti:0.7.0'


shadowJar {
    zip64 true  // Required if fatjar has more than 64K files
    classifier "bnp"
    from sourceSets.main.output, sourceSets.test.output, sourceSets.main.resources
    from {subprojects*.sourceSets.main.output}
    from {subprojects*.sourceSets.test.output}
    from project('gobblin').projectDir
    dependsOn 'gobblin:shadowJar'

    // Hadoop dependencies are expected to be provided by Azkaban.
    // If Azkaban is not included in your deployment, you may need to remove the following excludes
    exclude("**/org/apache/hadoop/**")
    exclude("**/org.apache.hadoop**")

    // Required when working in an Hadoop 2.x environment
    dependencies {
        include(dependency(depAvro))
        include(dependency(depProtoBuf))
        include(dependency(depJdom))
        include(dependency(depAzkaban))
        include(dependency(depGuava))
        include(dependency(depLog4j))
        include(dependency(depJacksonMapper))
        include(dependency(depJoda))
        include(dependency(depTehuti))
    }
    relocate 'com.google.protobuf', 'voldemort.shadow.2.3.0.com.google.protobuf'
    relocate 'org.apache.avro', 'voldemort.shadow.1.4.0.org.apache.avro'
    // TODO: find a way to exclude private lib's BDB-JE which gets pulled into the fat jar...
}

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
task protobufJar(type: ShadowJar) {
    baseName = projectName + "-protobuf"

    manifest {
        attributes 'Voldemort-Implementation-Version' : version,
        'Implementation-Title': 'Voldemort',
        'Implementation-Version': version,
        'Implementation-Vendor' :'LinkedIn'
    }

    dependencies {
        include(dependency(depProtoBuf))
        configurations.runtime.each { exclude(dependency(it)) }
    }

    configurations = [ project.configurations.runtime ]

    from sourceSets.main.output
    destinationDir = project.file(distDir)

    relocate 'com.google.protobuf', 'voldemort.shadow.2.3.0.com.google.protobuf'
    // Since BDB is included via compile files option, shadow does not support
    // excluding them. https://github.com/johnrengelman/shadow/issues/142
    // Once shadow supports excluding them, it can be excluded. There are
    // Complex ways to exclude it, but not worth it.
    relocate 'com.sleepycat' , 'voldemort.shadow.5.0.88.com.sleepycat'
}

task copySources (type: Copy) {
    from ('.') { include 'bin/*.sh', 'bin/*.bat' , 'bin/*.py' }
    from ('.') { include  distDir + '/*.jar'}
    from ('.') { exclude distDir + '/**' ,'bin/**' , 'build/**', '.git/**' , '.gradle/**', 'config/**/data/**' }
    into archiveDirectoryPath
}

task copyDeps(type: Copy) {
    // note this only copies the dependencies of the root
    // project into /lib if we start adding compile deps to
    // subprojects will have to rethink
    from { configurations.compile }
    into "lib"
}

task zip (type: Zip) {
    dependsOn copySources, copyDeps, contribJar, protobufJar
    baseName = projectName

    from(distDir) {
        include archiveDirectoryName + '/bin/**'
        fileMode = 0755
    }
    from(distDir) {
        include archiveDirectoryName + '/**'
        exclude archiveDirectoryName + '/bin/**'
    }

    destinationDir = project.file(distDir)
}

task tar (type: Tar) {
    dependsOn copySources, copyDeps, contribJar, protobufJar
    compression = Compression.GZIP
    baseName = projectName
    extension = "tar.gz"

    from(distDir) {
        include archiveDirectoryName + '/bin/**'
        fileMode = 0755
    }
    from(distDir) {
        include archiveDirectoryName + '/**'
        exclude archiveDirectoryName + '/bin/**'
    }

    destinationDir = project.file(distDir)
}

war {
    dependsOn copyDeps
    from sourceSets.main.output
    webXml = project.file('web.xml')
    destinationDir = project.file(distDir)
}

assemble.dependsOn copyDeps

copySources.dependsOn jar

allprojects {
    tasks.withType(Test) {
        maxHeapSize = "8g"
        // If ignoreFailures is not set, then merged reports will not be generated
        // Gradle aborts further tasks on test failure. so if you run junitAll
        // which runs 3 tests, reports task will never be run on failure cases.
        ignoreFailures = true

        useJUnit()

        testLogging {
            events "started", "passed", "skipped", "failed"
            exceptionFormat = 'full'
            // showStandardStreams = true

            doFirst {
                def classesSize = candidateClassFiles.files.size()
                logger.lifecycle("{} starts executing {} test classes {}",
                        path, classesSize, classesSize > 0? "(" + candidateClassFiles.files*.name[0] + ", ...)" : "")
            }

            //Set reasonable defaults for reports location
            reports.html.destination = file("$buildDir/reports/$name")
            reports.junitXml.destination = file("$buildDir/$name-results")
        }
        // Makes sure tests aren't marked "UP-TO-DATE" after running
        outputs.upToDateWhen { false }
    }
}

tasks.withType(Test) {
    // note only the root project's tests fork for very test class
    forkEvery = 1

    // Do not set the max parallelism as there are tests that uses the same port and will
    // run into bind exceptions.

    //ignoreFailures = gradle.startParameter.continueOnFailure

    //all standard error messages from tests will get routed to 'DEBUG' level messages.
    //logging.captureStandardError(LogLevel.DEBUG)
    //all standard output messages from tests will get routed to 'DEBUG' level messages.
    //logging.captureStandardOutput(LogLevel.DEBUG)

    //Set reasonable defaults classpath and classes dir. They can be reconfigured in an individual task.
//    it.testClassesDir = sourceSets.test.output.classesDir
//    classpath = sourceSets.test.runtimeClasspath
}

task resetConfig() {
    doLast {
        def DirsToDelete = [".temp", ".version", "data"]
        def deleteRecursively

        deleteRecursively = { file ->
            file.eachFile() {f ->
                if(f.directory) {
                    if( DirsToDelete.contains(f.getName()) )
                    {
                        println "deleting ${f.getAbsolutePath()}"
                        delete f
                    }
                    else
                    {
                        deleteRecursively(f)
                    }
                }
            }
        }

        deleteRecursively (new File("config"))
    }
}

task junit(dependsOn: test)

Collection<String> testClassesFrom(String dir, String include = '**/*Test.*') {
    //take all *Test.java files found in given dir, make the path relative and replace .java with .class
    fileTree(dir: dir, includes: [include]).collect { it.absolutePath.replace("\\", "/").replaceAll(file(dir).absolutePath.replace("\\", "/") + "/", "").replaceAll(".java\$", ".class")}
}

test {
    description = "Runs acceptance tests"
    include testClassesFrom(unitTestSrcDir)
}

task junitLong(type: Test) {
    description = "Runs long junit tests"
    include testClassesFrom(longTestSrcDir)
}

task junitInt(type: Test) {
    description = "Runs integration tests"
    include testClassesFrom(intTestSrcDir)
}

task junitRebalance(type: Test) {
    include testClassesFrom(unitTestSrcDir, '**/*Rebalance*Test.java')
}

task junitRebalanceLong(type: Test) {
    include testClassesFrom(longTestSrcDir, '**/*Rebalance*Test.java')
}

task contribJunit(type:TestReport) {
    // this populated below by depending on all the test tasks found
    // in the subprojects.
    destinationDir = file("$buildDir/reports/$name")
}

subprojects {
    tasks.withType(Test) {
        // hook up the report to teh contrib task and junitAll
        rootProject.contribJunit.reportOn it
        rootProject.junitAll.reportOn it
    }
}

task junitAll(type: TestReport) {
    reportOn test, junitLong
    destinationDir = file("$project.buildDir/reports/$name")
}


task aggregatedJunit(type: TestReport) {
    destinationDir = file("$project.buildDir/reports/$name")
}

allprojects {
    tasks.withType(Test) {
        finalizedBy rootProject.aggregatedJunit
        doLast { rootProject.aggregatedJunit.reportOn it }
    }
}

task wrapper(type: Wrapper) { gradleVersion = '2.9' }

dependencies {
    // Avro serialization format
    compile depAvro

    // INTERNAL_LIBS azkaban version not found
    // azkaban-common-0.05.jar

    // INTERNAL_LIBS Used for tomcat deployment, not sure if anyone uses it
    // catalina-ant.jar , version not found in maven central

    // coders decoders containing the Base64,binary encoding
    compile 'commons-codec:commons-codec:1.4'

    // TRANSITIVE_DEPENDENCY Contrib jar depends on commons-configuration-1.6.jar
    // commons-configuration instead depends on commons-collection
    //compile 'commons-collections:commons-collections:3.2.1'

    // Used by MySql storage engine classes
    // The jar supports database connection pooling
    compile 'commons-dbcp:commons-dbcp:1.2.2'

    // commons io is used at many places
    // IOUtils, FileUtils and ByteArrayOutputStream
    compile 'commons-io:commons-io:2.1'

    // LZF compression strategy for store and tests.
    compile 'com.ning:compress-lzf:0.9.1'

    // Used all over the place for collections
    compile depGuava

    // used for readonly store hdfs fetcher.
    compile 'org.apache.hadoop:hadoop-auth:2.3.0-cdh5.1.5'

    // used at lots of places. Seems like there is some overlap between httpclient and core, but not clear
    compile 'org.apache.httpcomponents:httpclient:4.1.2'

    // contains both http server and client functionalities. Used for HttpResponse but could be used at more places.
    compile 'org.apache.httpcomponents:httpcore:4.1.2'

    // JSON mapping library from Java Objects to JSON
    compile depJacksonMapper

    // JSON processing library
    compile 'org.codehaus.jackson:jackson-core-asl:1.9.13'

    // Used for reading XML files and Document.
    compile depJdom

    // Jetty is used for HttpService and tests. Jetty Util is used for QueuedThreadPool class.
    compile 'org.mortbay.jetty:jetty-util:6.1.18'
    compile 'org.mortbay.jetty:jetty:6.1.18'

    // A line processing library for command line. No compile time dependency
    // Used by Voldemort shell
    compile 'jline:jline:0.9.94'

    // jna is library for invoking native functions
    // used in the readonly store
    compile 'net.java.dev.jna:jna:3.2.7'

    // joda time is replacement for Java Date and Time
    // used in readonly store code.
    compile depJoda

    // Used for argument command line parsing
    compile 'net.sf.jopt-simple:jopt-simple:4.6'

    // log4j - logger used in almost all files
    compile depLog4j

    // used in readonly store and Co-ordinator
    compile 'javax.mail:mail:1.4.1'

    // Used in co-ordinator and rest services
    compile 'io.netty:netty:3.5.8.Final'

    // TRANSITIVE_DEPENDENCY Paranamer is a library that allows the parameter names of non-private methods and constructors to be accessed at runtime
    // Avro has a dependency on paranamer
    // compile 'com.thoughtworks.paranamer:paranamer:2.1'

    // protobuf is a supported protocol format between voldemort client and server
    compile depProtoBuf

    // Servlet
    compile 'javax.servlet:servlet-api:2.5'

    // slf4j is another logging abstraction framework.
    // It is used by the apache.avro, apache.hadoop and r2 clients
    compile 'org.slf4j:slf4j-api:1.5.6'
    compile 'org.slf4j:slf4j-log4j12:1.5.6'

    // snappy is one of the supported compression strategies in voldemort
    compile 'org.iq80.snappy:snappy:0.2'

    // Velocity is a simple yet powerful Java-based template engine that renders data
    // from plain Java objects to text, xml, email, SQL, Post Script, HTML etc
    // Velocity is used for Http Server GUI
    compile 'org.apache.velocity:velocity:1.6.2'

    // TRANSITIVE_DEPENDENCY Apache XML Parser
    // used by jdom
    // compile 'xerces:xercesImpl:2.9.1'

    // BDB-JE from Oracle
    compile 'com.sleepycat:je:5.0.104'

    // cern library containing high performance Maps for int and double
    // Currently only used in the tests
    testCompile 'colt:colt:1.2.0'

    // Used in resource pool perf testing class
    testCompile 'commons-pool:commons-pool:1.5.2'

    testRuntime 'mysql:mysql-connector-java:5.1.31'

    // Used for unit tests and other automated testing
    testCompile 'junit:junit:4.6'

    // Mockito is written by our beloved friend Szczepan Faber :)
    // Mocking framework used in some tests
    testCompile 'org.mockito:mockito-all:1.8.5'

//    contribCompile sourceSets.main.output
//    contribCompile sourceSets.test.output

    // declaring contribCompile dependencies as compile dependencies
    // otherwise while copying dependencies to lib directory
    // conflict resolution is not done properly across sourceSets
    // and we end up with 2 versions of few jars like ( log4j, servlet etc. )
    compile 'commons-configuration:commons-configuration:1.6'
    compile('org.apache.hadoop:hadoop-core:2.3.0-mr1-cdh5.1.5') {
        exclude group: 'com.google.protobuf'
        exclude group: 'org.apache.avro'
    }
    compile('org.apache.hadoop:hadoop-common:2.3.0-cdh5.1.5') {
        exclude group: 'com.google.protobuf'
        exclude group: 'org.apache.avro'
    }
    compile('org.apache.hadoop:hadoop-hdfs:2.3.0-cdh5.1.5') {
        exclude group: 'com.google.protobuf'
        exclude group: 'org.apache.avro'
    }

    compile 'com.linkedin.pegasus:r2:1.8.3'
    compile 'com.linkedin.pegasus:data:1.8.3'
    compile 'com.linkedin.pegasus:pegasus-common:1.8.3'
    compile depAzkaban

    compile 'com.google.code.typica:typica:1.7.2'
    compile 'com.sna-projects.krati:krati:0.4.9'

    // Metrics
    compile depTehuti
    testCompile 'io.tehuti:tehuti:0.7.0:test'

    // Other libs...
    compile 'org.apache.tomcat:catalina-ant:6.0.43'
    compile 'org.apache.hadoop:libthrift:0.5.0.0'

    // rocksdb from maven
    compile 'org.rocksdb:rocksdbjni:3.13.1'

    // Bouncy Castle Libaray
    compile 'org.bouncycastle:bcprov-jdk15on:1.48'

    // Gobblin
    compile 'com.linkedin.gobblin:gobblin-runtime:0.11.0', gobblinExcludes
    compile 'com.linkedin.gobblin:gobblin-data-management:0.11.0', gobblinExcludes
    compile 'com.linkedin.gobblin:gobblin-throttling-service-client:0.11.0', gobblinExcludes
}

subprojects {
    // this configures all the contrib subprojects
    // note at the moment there dependencies are still
    // declared in the dependency block above.
    apply plugin:'java'
    sourceCompatibility = javacVersion
    targetCompatibility = javacVersion

    sourceSets {
        // note that the contrib projects don't currently have resource
        // directories so below is actually just a to keep idea & gradle from
        // thinking that the test resources dir is src/test/resources
        main {
            java { srcDirs = ['src/java'] }
            resources { srcDirs = ['src/resources'] }
        }
        test {
            java { srcDirs = ['test'] }
            resources { srcDirs = ['testResources'] }
        }
    }
    dependencies {
        compile rootProject
        // Used for unit tests and other automated testing
        testCompile rootProject.sourceSets.test.runtimeClasspath
    }

    tasks.withType(Test) {
        // this is required as the test utils expect the config directory
        // to be at the root of the process working directory.
        workingDir = rootProject.projectDir
    }
}

allprojects {
    apply plugin: 'idea'
    apply plugin: 'eclipse'

    eclipse {
        jdt {
            sourceCompatibility = targetCompatibility = 1.7
        }
        classpath {
            defaultOutputDir = project.file('classes') // overrides the default of /bin which is where our scripts are
            downloadSources = true
            file {
                whenMerged { classpath ->
                    // so for some reason the generated .classpath for the contrib projects includes
                    // two copies of *most* (maybe all) of the libraries from the parent project
                    // the following de-dupes these
                    def duplicateLibs = classpath.entries
                        .findAll { it.kind == 'lib' } // only library entries
                        .groupBy { it.library }       // index by the library
                        .findAll { it.value.size() > 1 } // only where there is more that a single entry

                    duplicateLibs.each { k, v ->
                        // pick one from the list of dupes ..preferring the one without a sourcelib path
                        // else just take the first
                        def toRemove = v.find { !it.sourcePath } ?: v.first()
                        classpath.entries.remove toRemove
                    }
                    // also with for no reason contrib projects gets a source path configured
                    // with the following path.. since this path doesn't exist eclipse complains
                    classpath.entries.removeAll {
                        it.kind == 'lib' && it.library.path.endsWith('build/resources/test')
                    }
                }
            }
        }
    }

    idea {
        module {
            downloadJavadoc = true
            downloadSources = true
        }
    }
}