#!/usr/bin/bash
#############################################################################
#  Copyright (C) 2013-2015 Lawrence Livermore National Security, LLC.
#  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
#  Written by Albert Chu <chu11@llnl.gov>
#  LLNL-CODE-644248
#
#  This file is part of Magpie, scripts for running Hadoop on
#  traditional HPC systems.  For details, see <URL>.
#
#  Magpie is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  Magpie is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with Magpie.  If not, see <http://www.gnu.org/licenses/>.
#############################################################################

# This script sets up configuration files for jobs.  For the most
# part, it shouldn't be editted.  See job submission files for
# configuration details.

if [ "${ZEPPELIN_SETUP}" != "yes" ]
then
    exit 0
fi

source ${MAGPIE_SCRIPTS_HOME}/magpie/exports/magpie-exports-dirs
source ${MAGPIE_SCRIPTS_HOME}/magpie/exports/magpie-exports-user
source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-defaults
source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-node-identification
source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-setup
source ${MAGPIE_SCRIPTS_HOME}/magpie/lib/magpie-lib-version-helper

# zeppelinnoderank set if succeed
if ! Magpie_am_I_a_zeppelin_node
then
    exit 0
fi

# For rest of setup, we will use cluster specific paths
Magpie_make_all_local_dirs_node_specific

# Set up shared location for notebooks, and things like that
zeppelinsharedscratchdirpath="${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/zeppelin/"
if [ ! -d "${zeppelinsharedscratchdirpath}" ]
then
    mkdir -p ${zeppelinsharedscratchdirpath}
    if [ $? -ne 0 ] ; then
        echo "mkdir failed making ${zeppelinsharedscratchdirpath}"
        exit 1
    fi
fi

if [ "${ZEPPELIN_NOTEBOOK_DIR}X" != "X" ]
then
    if [ ! -d ${ZEPPELIN_NOTEBOOK_DIR} ]
    then
        mkdir -p ${ZEPPELIN_NOTEBOOK_DIR}
        if [ $? -ne 0 ] ; then
            echo "mkdir failed making ${ZEPPELIN_NOTEBOOK_DIR}"
            exit 1
        fi
    fi
    zeppelinnotebookdir=${ZEPPELIN_NOTEBOOK_DIR}
else
    zeppelinnotebookdir="${zeppelinsharedscratchdirpath}/notebook"
fi

# Set the default memory if not set
if [ "${ZEPPELIN_MEM}X" == "X" ]
then
    export ZEPPELIN_MEM="-Xmx4096m"
fi

if [ "${ZEPPELIN_JAVA_OPTS}X" != "X" ]
then
    if ! echo ${ZEPPELIN_JAVA_OPTS} | grep -q -E "java.io.tmpdir"
    then
        export ZEPPELIN_JAVA_OPTS="${ZEPPELIN_JAVA_OPTS} -Djava.io.tmpdir=${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/tmp"
    fi
fi
extrazeppelinjavaopts="${extrazeppelinjavaopts}${extrazeppelinjavaopts:+" "}-Djava.io.tmpdir=${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/tmp"

if [ ! -d "${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/tmp" ]
then
    mkdir -p ${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/tmp
    if [ $? -ne 0 ] ; then
        echo "mkdir failed making ${ZEPPELIN_LOCAL_SCRATCHSPACE_DIR}/tmp"
        exit 1
    fi
fi

#
# Get config files for setup
#

# Returns 0 for ==, 1 for $1 > $2, 2 for $1 < $2
Magpie_vercomp ${ZEPPELIN_VERSION} 0.7.0
vercomp_result=$?
if [ "${vercomp_result}" == "0" ] || [ "${vercomp_result}" == "1" ]
then
    zeppelin_jars="${zeppelin_jars}${ZEPPELIN_HOME}/lib/zeppelin-server-${ZEPPELIN_VERSION}.jar"
else
    zeppelin_jars="${zeppelin_jars}${ZEPPELIN_HOME}/zeppelin-server-${ZEPPELIN_VERSION}.jar"
fi

if [ "${HBASE_SETUP}" == "yes" ]
then
    hbase_jars="${HBASE_HOME}/lib/hbase-annotations-${HBASE_VERSION}.jar"
    hbase_jars="${HBASE_HOME}/lib/hbase-client-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-common-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-hadoop2-compat-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-it-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-prefix-tree-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-procedure-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-protocol-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-resource-bundle-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-rest-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-server-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-shell-${HBASE_VERSION}.jar,${hbase_jars}"
    hbase_jars="${HBASE_HOME}/lib/hbase-thrift-${HBASE_VERSION}.jar,${hbase_jars}"
    zeppelin_jars="${hbase_jars},${zeppelin_jars}"
fi

if [ "${KAFKA_SETUP}" == "yes" ]
then
    kafka_jar="${KAFKA_HOME}/libs/kafka_${KAFKA_VERSION}.jar"
    zeppelin_jars="${zeppelin_jars},${kafka_jar}"
fi

if [ "${HIVE_SETUP}" == "yes" ]
then
    hive_jars="${HIVE_HOME}/lib/hive-beeline-${HIVE_VERSION}.jar"
    hive_jars="${HIVE_HOME}/lib/hive-llap-ext-client-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-cli-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-llap-server-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-common-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-llap-tez-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-contrib-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-metastore-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-serde-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-exec-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-service-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-service-rpc-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-shims-0.23-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-shims-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-hplsql-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-shims-common-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-jdbc-${HIVE_VERSION}.jar,${hive_jars}"
    hive_jars="${HIVE_HOME}/lib/hive-shims-scheduler-${HIVE_VERSION}.jar,${hive_jars}"
    zeppelin_jars="${zeppelin_jars},${hive_jars}"

    #  Add Spark's Hive .jars to the Zeppelin jars so it can interact with Hive through the interpreter
    if [ "${HADOOP_SETUP}" == "yes" ] && [ "${SPARK_USE_YARN}" == "yes" ]
    then
        hadoop_yarn=$(ls "${HADOOP_HOME}"/share/hadoop/yarn/*.jar)
        for j in $hadoop_yarn
        do
            zeppelin_jars="$j,${zeppelin_jars}"
        done

        hadoop_common=$(ls "${HADOOP_HOME}"/share/hadoop/common/*.jar)
        for j in $hadoop_common
        do
            zeppelin_jars="$j,${zeppelin_jars}"
        done
    fi

    # sets magpie_sparkmajorminorversion
    Magpie_get_spark_major_minor_version ${SPARK_VERSION}

    # Handle special settings depending on version
    # 0 is =, 1 is >, 2 is <
    Magpie_vercomp ${magpie_sparkmajorminorversion} 2.0
    vercomp_result=$?
    if [ "${vercomp_result}" == "0" ] || [ "${vercomp_result}" == "1" ]
    then
        spark_jars_path="$SPARK_HOME/jars"
    else
        spark_jars_path="$SPARK_HOME/lib"
    fi

    # need the just the spark version number for a couple of the jars
    sparkjarver="$(cut -d '-' -f 1 <<< "${SPARK_VERSION}" )"

    hive_spark_jars="${spark_jars_path}/hive-beeline-1.2.1.spark2.jar"
    hive_spark_jars="${spark_jars_path}/hive-cli-1.2.1.spark2.jar,${hive_spark_jars}"
    hive_spark_jars="${spark_jars_path}/hive-exec-1.2.1.spark2.jar,${hive_spark_jars}"
    hive_spark_jars="${spark_jars_path}/hive-jdbc-1.2.1.spark2.jar,${hive_spark_jars}"
    hive_spark_jars="${spark_jars_path}/hive-metastore-1.2.1.spark2.jar,${hive_spark_jars}"
    hive_spark_jars="${spark_jars_path}/spark-hive_2.11-${sparkjarver}.jar,${hive_spark_jars}"
    hive_spark_jars="${spark_jars_path}/spark-hive-thriftserver_2.11-${sparkjarver}.jar,${hive_spark_jars}"
    zeppelin_jars="${hive_spark_jars},${zeppelin_jars}"
fi

if [ "${SPARK_SETUP}" == "yes" ]; then
    sparkjarver="$(cut -d '-' -f 1 <<< "${SPARK_VERSION}" )"
    # Handle special settings depending on version
    # 0 is =, 1 is >, 2 is <
    Magpie_vercomp ${magpie_sparkmajorminorversion} 2.3
    vercomp_result=$?
    if [ "${vercomp_result}" == "0" ] || [ "${vercomp_result}" == "1" ]
    then
        spark_jars_path="$SPARK_HOME/jars"

        spark_jars="${spark_jars_path}/spark-streaming_2.11-${sparkjarver}.jar"
        zeppelin_jars="${spark_jars},${zeppelin_jars}"
    fi
fi

if [ "${PHOENIX_SETUP}" == "yes" ]
then
    phoenix_jars="${PHOENIX_HOME}/phoenix-${PHOENIX_VERSION}-client.jar"
    phoenix_jars="${PHOENIX_HOME}/phoenix-${PHOENIX_VERSION}-server.jar,$phoenix_jars"
    phoenix_jars="${PHOENIX_HOME}/phoenix-spark-${PHOENIX_VERSION}.jar,$phoenix_jars"
    phoenix_jars="${PHOENIX_HOME}/phoenix-core-${PHOENIX_VERSION}.jar,$phoenix_jars"

    # Returns 0 for ==, 1 for $1 > $2, 2 for $1 < $2
    Magpie_vercomp ${PHOENIX_VERSION} 4.8.0
    vercomp_result=$?
    if [ "${vercomp_result}" == "0" ] || [ "${vercomp_result}" == "1" ]
    then
        phoenix_jars="${PHOENIX_HOME}/phoenix-queryserver-${PHOENIX_VERSION}.jar,$phoenix_jars"
        phoenix_jars="${PHOENIX_HOME}/phoenix-queryserver-client-${PHOENIX_VERSION}.jar,$phoenix_jars"
    else
        phoenix_jars="${PHOENIX_HOME}/phoenix-server-${PHOENIX_VERSION}.jar,$phoenix_jars"
        phoenix_jars="${PHOENIX_HOME}/phoenix-server-client-${PHOENIX_VERSION}.jar,$phoenix_jars"
    fi
    zeppelin_jars="${zeppelin_jars},${phoenix_jars}"
fi

ZEPPELIN_SPARK_SUBMIT_OPTS="${ZEPPELIN_SPARK_SUBMIT_OPTIONS}${ZEPPELIN_SPARK_SUBMIT_OPTIONS:+" "}--jars $zeppelin_jars"

for jar in $(ls ${spark_jars_path}/*.jar)
do
    zeppelinclasspath="${spark_jars_path}/${jar}:${zeppelinclasspath}"
done

for jar in $(ls ${ZEPPELIN_HOME}/lib/*.jar)
do 
    zeppelinclasspath="${ZEPPELIN_HOME}/lib/${jar}:${zeppelinclasspath}"
done

for jar in $(ls ${ZEPPELIN_HOME}/lib/interpreter/*.jar)
do 
    zeppelinclasspath="${ZEPPELIN_HOME}/lib/interpreter/${jar}:${zeppelinclasspath}"
done

# Magpie_find_conffile will set the 'pre' filenames
Magpie_find_conffile "Zeppelin" ${ZEPPELIN_CONF_FILES:-""} "zeppelin-env.sh" "pre_zeppelinenvsh"
Magpie_find_conffile "Zeppelin" ${ZEPPELIN_CONF_FILES:-""} "zeppelin-site.xml" "pre_zeppelinsitexml"
Magpie_find_conffile "Zeppelin" ${ZEPPELIN_CONF_FILES:-""} "zeppelin.log4j.properties" "pre_log4jproperties"
Magpie_find_conffile "Zeppelin" ${ZEPPELIN_CONF_FILES:-""} "shiro.ini" "pre_shiroini"

post_zeppelinenvsh=${ZEPPELIN_CONF_DIR}/zeppelin-env.sh
post_zeppelinsitexml=${ZEPPELIN_CONF_DIR}/zeppelin-site.xml
post_log4jproperties=${ZEPPELIN_CONF_DIR}/log4j.properties
post_shiroini=${ZEPPELIN_CONF_DIR}/shiro.ini

if [ "${SPARK_SETUP_TYPE}" == "YARN" ];
then
    zeppelin_spark_master="yarn-client"
else
    zeppelin_spark_master="spark://${SPARK_MASTER_NODE}:${SPARK_MASTER_PORT}"
fi

javahomesubst=`echo "${JAVA_HOME}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinclasspathsubst=`echo "${zeppelinclasspath}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinhomesubst=`echo "${ZEPPELIN_HOME}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinlogdirsubst=`echo "${ZEPPELIN_LOG_DIR}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinpiddirsubst=`echo "${ZEPPELIN_PID_DIR}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinconfdirsubst=`echo "${ZEPPELIN_CONF_DIR}" | sed "s/\\//\\\\\\\\\//g"`
sparkconfdirsubst=`echo "${SPARK_CONF_DIR}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinsparkhomesubst=`echo "${SPARK_HOME}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinsparkmastersubst=`echo "${zeppelin_spark_master}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinsparksubmitoptionssubst=`echo "${ZEPPELIN_SPARK_SUBMIT_OPTS}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinjavaoptssubst=`echo "${ZEPPELIN_JAVA_OPTS}" | sed "s/\\//\\\\\\\\\//g"`
zeppelinwebappssubst=`echo "${zeppelinsharedscratchdirpath}/webapps" | sed "s/\\//\\\\\\\\\//g"`
zeppelinnotebooksubst=`echo "${zeppelinnotebookdir}" | sed "s/\\//\\\\\\\\\//g"`
zeppelininterpreterdir=`echo "${ZEPPELIN_HOME}/interpreter" | sed "s/\\//\\\\\\\\\//g"`
zeppelinhadoopconfdir=`echo "${HADOOP_CONF_DIR}" | sed "s/\\//\\\\\\\\\//g"`

#
# Setup Zeppelin configuration files and environment files
#
cp ${pre_zeppelinenvsh} ${post_zeppelinenvsh}

sed -i \
    -e "s/ZEPPELIN_JAVA_HOME/${javahomesubst}/g" \
    -e "s/ZEPPELINLOGDIR/${zeppelinlogdirsubst}/g" \
    -e "s/ZEPPELINPIDDIR/${zeppelinpiddirsubst}/g" \
    -e "s/ZEPPELINCONFDIR/${zeppelinconfdirsubst}/g" \
    -e "s/ZEPPELIN_SPARK_CONF/${sparkconfdirsubst}/g" \
    -e "s/ZEPPELIN_SPARK_HOME/${zeppelinsparkhomesubst}/g" \
    -e "s/ZEPPELIN_SPARK_MASTER/${zeppelinsparkmastersubst}/g" \
    -e "s/ZEPPELIN_SPARK_SUBMIT_OPTIONS/${zeppelinsparksubmitoptionssubst}/g" \
    -e "s/ZEPPELINJAVAOPTS/${zeppelinjavaoptssubst}/g" \
    -e "s/ZEPPELINMEM/${ZEPPELIN_MEM}/g" \
    -e "s/ZEPPELINNOTEBOOK/${zeppelinnotebooksubst}/g" \
    -e "s/ZEPPELINCLASSPATH/${zeppelinclasspathsubst}/g" \
    -e "s/ZEPPELIN_HADOOP_CONF_DIR/${zeppelinhadoopconfdir}/g" \
    ${post_zeppelinenvsh}

if [ "${ZEPPELIN_SCALA_PATH}X" != "X" ]
then
    echo "export PATH=${ZEPPELIN_SCALA_PATH}/bin:\$PATH" >> ${post_zeppelinenvsh}
    echo "export LD_LIBRARY_PATH=${ZEPPELIN_SCALA_PATH}/lib:\$LD_LIBRARY_PATH" >> ${post_zeppelinenvsh}
fi

if [ "${MAGPIE_REMOTE_CMD:-ssh}" != "ssh" ]
then
    echo "export ZEPPELIN_SSH_CMD=\"${MAGPIE_REMOTE_CMD}\"" >> ${post_zeppelinenvsh}
fi
if [ "${MAGPIE_REMOTE_CMD_OPTS}X" != "X" ]
then
    echo "export ZEPPELIN_SSH_OPTS=\"${MAGPIE_REMOTE_CMD_OPTS}\"" >> ${post_zeppelinenvsh}
fi

cp ${pre_zeppelinsitexml} ${post_zeppelinsitexml}

sed -i \
    -e "s/ZEPPELINPORT/${default_zeppelin_port}/g" \
    -e "s/ZEPPELINWARTEMPDIR/${zeppelinwebappssubst}/g" \
    -e "s/ZEPPELINNOTEBOOK/${zeppelinnotebooksubst}/g" \
    -e "s/ZEPPELININTERPRETERDIR/${zeppelininterpreterdir}/g" \
    ${post_zeppelinsitexml}

# sets magpie_zeppelinmajorminorversion
Magpie_get_zeppelin_major_minor_version ${ZEPPELIN_VERSION}

# Handle special settings depending on version
# 0 is =, 1 is >, 2 is <
Magpie_vercomp ${magpie_zeppelinmajorminorversion} 0.6
vercomp_result=$?
if [ "${vercomp_result}" == "0" ] || [ "${vercomp_result}" == "1" ]
then
    cp ${pre_shiroini} ${post_shiroini}

    IFS=';'; userlist=(${ZEPPELIN_NOTEBOOK_USERS}); unset IFS;
    for userstring in ${userlist[@]}
    do
        IFS=','; userval=(${userstring}); unset IFS;
        zeppelin_notebook_users="${zeppelin_notebook_users}\\n${userval[0]} = ${userval[1]}, ${userval[2]}"
    done

    if [ "${ZEPPELIN_NOTEBOOK_ENCRYPT_PASSWORD}" == "true" ]
    then
        zeppelin_notebook_encrypted="passwordMatcher = org.apache.shiro.authc.credential.PasswordMatcher\\niniRealm.credentialsMatcher = \$passwordMatcher"
        zeppelin_notebook_users=`echo "${zeppelin_notebook_users}" | sed "s/\\//\\\\\\\\\\\\//g"`
    else
        zeppelin_notebook_encrypted=""
    fi

    sed -i \
        -e "s/ZEPPELINNOTEBOOKUSERS/${zeppelin_notebook_users}/g" \
        -e "s/ZEPPELINNOTEBOOKENCRYPTPASSWORD/${zeppelin_notebook_encrypted}/g" \
        ${post_shiroini}
fi

cp ${pre_log4jproperties} ${post_log4jproperties}

exit 0
