Add docker container for MapR

teradatalabs/mapr52-base/Dockerfile

@@ -0,0 +1,61 @@
+# Copyright 2017 Teradata
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM teradatalabs/centos6-java8-oracle
+MAINTAINER Teradata Docker Team <docker@teradata.com>
+
+# ADD REPO FOR MapR
+ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo
+RUN yum update -y \
+# ... GET MapRGPG KEY
+  && rpm --import http://package.mapr.com/releases/pub/maprgpg.key \
+
+# INSTALL UTILITY SOFTWARE
+  && yum install -y iputils vim openssh-server openssh-clients sudo \ 
+
+# CONFIGURE SSH
+  && chkconfig sshd on \
+  && service sshd start \
+
+# INSTALL MapR 
+  && yum install -y mapr-fileserver mapr-nfs mapr-nodemanager  mapr-cldb \ 
+  && yum install -y mapr-zookeeper mapr-resourcemanager mapr-historyserver \  
+  && yum install -y mapr-webserver  mapr-gateway \
+
+# CONFIGURE ZOOKEEPER'S DATA DIRECTORY
+  && rm -rf /opt/mapr/zkdata \
+  && mkdir /opt/mapr/zkdata \
+  && chmod 777 /opt/mapr/zkdata \
+  && mkdir -p /mapr \
+# THE /root/disk.txt IS READ BY MapR TO DETERMINE WHICH DISKS IT CAN USE AND THE /home/mapr/storagefile IS A FLAT FILE THAT ACTS AS A DISK
+  && echo "/home/mapr/storagefile" > /root/disk.txt \
+
+# INSTALL PYTHON AND SUPERVISORD
+  && yum install -y python-setuptools \
+  && easy_install pip \
+  && pip install supervisor \
+  && mkdir /etc/supervisord.d/ \
+# ... AND ITS MISSING DEPENDENCY
+  && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \
+  && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \
+  && rm python-meld3-0.6.7-1.el6.x86_64.rpm \
+
+# CLEANUP
+  && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
+  && ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa \
+  && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \
+
+  && adduser mapr \
+  && touch /home/mapr \
+  && echo "cd /home/mapr" >> /home/mapr/.bashrc \

teradatalabs/mapr52-base/README.md

@@ -0,0 +1,9 @@
+# mapr52-base
+
+
+Docker image with Hive installed from MapR repositories.
+
+## Oracle license
+
+By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
+[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)

teradatalabs/mapr52-base/files/maprtech.repo

@@ -0,0 +1,13 @@
+[maprtech]
+name=MapR Technologies
+baseurl=http://package.mapr.com/releases/v5.2.0/redhat/
+enabled=1
+gpgcheck=0
+protect=1
+
+[maprecosystem]
+name=MapR Technologies
+baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat
+enabled=1
+gpgcheck=0
+protect=1

teradatalabs/mapr52-hive/Dockerfile

@@ -0,0 +1,52 @@
+# Copyright 2017 Teradata
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM teradatalabs/mapr52-base
+MAINTAINER Teradata Docker Team <docker@teradata.com>
+
+# ADD ALL REQUIRED SCRIPTS AND FILES TO ROOT DIRECTORY
+ADD files/*.sh /root/
+ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml
+ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml
+ADD files/supervisord.conf /etc/supervisord.conf
+COPY files/supervisord.d/* /etc/supervisord.d/
+
+RUN chmod 777 /root/*.sh \
+# INSTALL UTILITY SOFTWARE  
+  && yum install -y openssh \
+  && yum install -y lsof \
+
+# INSTALL MYSQL SERVER
+  && yum install -y mysql-server mysql-connector-java \
+  && yum -y clean all && rm -rf /tmp/* /var/tmp/* \ 
+
+  && /root/setup.sh \
+
+# INSTALL HIVE
+  && yum install -y mapr-hive mapr-hiveserver2 mapr-hivemetastore \
+  && yum -y clean all && rm -rf /tmp/* /var/tmp/* \
+
+# HDFS PORTS
+EXPOSE 5660 6660 5692 5724 5756 6692 6724 6756
+
+# YARN PORTS
+EXPOSE 8030 8020 7222 8088 7220 7221 8443 8031 8032 8033 8040 8041 8042 8088 10020 19888
+
+# HIVE PORT
+EXPOSE 9083 10000
+
+# SOCKS PORT
+EXPOSE 1180
+
+CMD /root/startup.sh

teradatalabs/mapr52-hive/README.md

@@ -0,0 +1,17 @@
+# mapr52-hive 
+
+Docker image with HDFS, YARN and HIVE installed. Please note that running services have lower memory heap size set.
+For more details please check the [hadoop-env.sh](files/conf/hadoop-env.sh) configuration file.
+If you want to work on larger datasets please tune those settings accordingly, the current settings should be optimal
+for general correctness testing.
+
+## Run
+
+```
+$ docker run --privileged -d --name hadoop-master -h hadoop-master teradatalabs/mapr52-hive
+```
+
+## Oracle license
+
+By using this image, you accept the Oracle Binary Code License Agreement for Java SE available here:
+[http://www.oracle.com/technetwork/java/javase/terms/license/index.html](http://www.oracle.com/technetwork/java/javase/terms/license/index.html)

teradatalabs/mapr52-hive/capabilities.txt

@@ -0,0 +1,4 @@
+exposes_hive
+allows_creating_a_table_in_hive
+allows_selecting_from_the_table
+exposes_socks_proxy

teradatalabs/mapr52-hive/files/bootstrap.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# CONFIGURE MapR
+/opt/mapr/server/configure.sh  -N mycluster -Z localhost -C localhost -HS localhost -no-autostart
+
+# SETUP FLAT FILE /home/mapr/storagefile
+dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10
+
+# SETUP DISK FOR MAPR BY RUNNING disksetup
+/opt/mapr/server/disksetup -M -F /root/disk.txt
+
+# CREATE HIVE PROXY USERS
+chmod 755 /opt/mapr/conf/proxy
+
+# START SERVICES
+service mapr-zookeeper start
+service mapr-warden start
+
+# CONFIGURE HIVE
+/opt/mapr/server/configure.sh -R
+
+# WAIT FOR WARDEN TO START ALL THE SERVICES
+sh /root/wardenTracker.sh
+
+# RUN HDFS COMMANDS
+hadoop fs -mkdir /user/root /user/hive /user/hive/warehouse
+hadoop fs -chmod 777 /user/hive /user/hive/warehouse

teradatalabs/mapr52-hive/files/conf/core-site.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>
+  		<name>hadoop.proxyuser.mapr.groups</name>
+  		<value>*</value>
+  	<description>Allow the superuser mapr to impersonate any member of any group</description>
+	</property>
+
+	<property>
+  		<name>hadoop.proxyuser.mapr.hosts</name>
+  		<value>*</value>
+  		<description>The superuser can connect from any host to impersonate a user</description>
+	</property>
+
+</configuration>
+

teradatalabs/mapr52-hive/files/conf/hive-site.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+   <property>
+      <name>datanucleus.schema.autoCreateAll</name>
+      <value>true</value>
+      <description>creates necessary schema on a startup if one doesn't exist. set
+         this to false, after creating it once</description>
+   </property>
+
+   <property>
+  	<name>hive.server2.enable.doAs</name>
+  	<value>true</value>
+  	<description>Set this property to enable impersonation in Hive Server 2</description>
+   </property>
+
+   <property>
+  	<name>hive.metastore.execute.setugi</name>
+  	<value>true</value>
+  	<description>Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored.</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionURL</name>
+      <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
+      <description>JDBC connect string for a JDBC metastore</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionDriverName</name>
+      <value>com.mysql.jdbc.Driver</value>
+      <description>Driver class name for a JDBC metastore</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionUserName</name>
+      <value>root</value>
+      <description>username to use against metastore database</description>
+   </property>
+
+   <property>
+      <name>javax.jdo.option.ConnectionPassword</name>
+      <value>root</value>
+      <description>password to use against metastore database</description>
+   </property>
+
+   <property>
+      <name>hive.metastore.uris</name>
+      <value>thrift://localhost:9083</value>
+   </property>
+
+</configuration>

teradatalabs/mapr52-hive/files/setup.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# SETUP METASTORE
+mysql_install_db
+
+/usr/bin/mysqld_safe &
+sleep 10s
+
+echo "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" | mysql
+echo "CREATE DATABASE metastore; USE metastore; SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.13.0.mysql.sql;" | mysql
+/usr/bin/mysqladmin -u root password 'root'
+
+killall mysqld
+sleep 10s
+mkdir /var/log/mysql/
+chown mysql:mysql /var/log/mysql/

teradatalabs/mapr52-hive/files/socks-proxy.sh

@@ -0,0 +1 @@
+../../../commons/socks-proxy.sh

teradatalabs/mapr52-hive/files/startup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec supervisord -c /etc/supervisord.conf

teradatalabs/mapr52-hive/files/supervisord.conf

@@ -0,0 +1 @@
+../../../commons/supervisord.conf

teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/bootstrap.conf

teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/mysql-metastore.conf

teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/socks-proxy.conf

teradatalabs/mapr52-hive/files/wardenTracker.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+hname=$(hostname)
+
+Services=0
+
+maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+
+# WAIT FOR CLDB TO START
+while [ $Services -ne 2 ]
+do
+if [ $maprcliReady == 1 ]
+then
+   maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+   Services=0
+else
+   Services=$(maprcli service list -node $hname | grep CLDB |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+fi
+done
+
+# WAIT FOR NODEMANAGER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR RESOURCEMANAGER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR HIVE METASTORE TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR HIVESERVER2 TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+# WAIT FOR JOBHISTORYSERVER TO START
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done