Add docker container for MapR

teradatalabs/mapr52-base/Dockerfile

@@ -0,0 +1,55 @@
+FROM teradatalabs/centos6-java8-oracle
+MAINTAINER Teradata Docker Team <docker@teradata.com>
+
+
+#Add repo for mapr
+ADD files/maprtech.repo /etc/yum.repos.d/maprtech.repo
+RUN yum update -y
+
+#install utility softwares
+RUN yum install iputils vim openssh-server openssh-clients sudo -y
+
+#configure sshh
+RUN chkconfig sshd on
+RUN service sshd start
+
+#get maprgpg key
+RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key
+
+#add user and password
+RUN adduser mapr
+RUN adduser hive-user
+RUN usermod --password hive hive-user
+RUN adduser hdfs-user
+RUN usermod --password hdfs hdfs-user
+RUN touch /home/hdfs-user
+RUN touch /home/hive-user
+RUN echo "cd /home/hive-user" >> /home/hive-user/.bashrc
+RUN echo "cd /home/hdfs-user" >> /home/hdfs-user/.bashrc
+
+#install mapr 
+RUN yum install mapr-fileserver mapr-nfs mapr-nodemanager  mapr-cldb mapr-zookeeper mapr-resourcemanager mapr-historyserver  mapr-webserver  mapr-gateway -y
+
+#necessary configuration for mapr
+RUN rm -rf /opt/mapr/zkdata
+RUN mkdir /opt/mapr/zkdata
+RUN chmod 777 /opt/mapr/zkdata
+RUN mkdir -p /mapr
+RUN echo "/home/mapr/storagefile" > /root/disk.txt
+
+
+#install python
+RUN yum install -y python-setuptools \
+  && easy_install pip \
+  && pip install supervisor \
+  && mkdir /etc/supervisord.d/ \
+# ... and its missing dependency
+  && wget http://dl.fedoraproject.org/pub/epel/6/x86_64/python-meld3-0.6.7-1.el6.x86_64.rpm \
+  && rpm -ihv python-meld3-0.6.7-1.el6.x86_64.rpm \
+  && rm python-meld3-0.6.7-1.el6.x86_64.rpm \
+
+# Cleanup
+  && yum -y clean all && rm -rf /tmp/* /var/tmp/*
+
+RUN ssh-keygen -t rsa -b 4096 -C "automation@teradata.com" -N "" -f /root/.ssh/id_rsa
+RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

teradatalabs/mapr52-base/files/maprtech.repo

@@ -0,0 +1,13 @@
+[maprtech]
+name=MapR Technologies
+baseurl=http://package.mapr.com/releases/v5.2.0/redhat/
+enabled=1
+gpgcheck=0
+protect=1
+
+[maprecosystem]
+name=MapR Technologies
+baseurl=http://package.mapr.com/releases/MEP/MEP-1.0/redhat
+enabled=1
+gpgcheck=0
+protect=1

teradatalabs/mapr52-hive/Dockerfile

@@ -0,0 +1,42 @@
+FROM teradatalabs/mapr52-base
+MAINTAINER Teradata Docker Team <docker@teradata.com>
+
+#add all required scripts  to root
+ADD files/*.sh /root/
+ADD files/conf/hive-site.xml /opt/mapr/hive/hive-1.2/conf/hive-site.xml
+ADD files/conf/core-site.xml /opt/mapr/hadoop/hadoop-2.7.0/etc/hadoop/core-site.xml
+ADD files/conf/supervisord.conf /etc/supervisord.conf
+RUN chmod 777 /root/*.sh
+COPY files/supervisord.d/* /etc/supervisord.d/
+RUN yum install openssh -y
+
+#install mysql server and required software
+RUN yum install -y mysql-server mysql-connector-java \
+    && yum -y clean all && rm -rf /tmp/* /var/tmp/* 
+
+#install lsof
+RUN yum install -y lsof
+
+# RUN SETUP script
+RUN /root/setup.sh
+
+
+#install hive
+RUN yum install mapr-hive mapr-hiveserver2 mapr-hivemetastore -y
+RUN yum -y clean all && rm -rf /tmp/* /var/tmp/*
+
+
+# HDFS PORTS
+EXPOSE 1004 1006 8020 50010 50020 50070 50075 50470
+
+# YARN PORTS
+EXPOSE 8030 8031 8032 8033 8040 8041 8042 8088 10020 19888
+
+# HIVE PORT
+EXPOSE 9083 10000
+
+# SOCKS PORT
+EXPOSE 1180
+
+#run startup script
+CMD /root/startup.sh

teradatalabs/mapr52-hive/files/bootstrap.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+#configure mapr
+/opt/mapr/server/configure.sh  -N mycluster -Z localhost -C localhost -HS localhost -no-autostart
+
+#setup disk for mapr
+dd if=/dev/zero of=/home/mapr/storagefile bs=1G count=10
+/opt/mapr/server/disksetup -M -F /root/disk.txt
+
+#create proxy setup for hive users
+chmod 755 /opt/mapr/conf/proxy
+touch /opt/mapr/conf/proxy/hive-user
+touch /opt/mapr/conf/proxy/hdfs-user
+
+#start zookeeper
+service mapr-zookeeper start
+
+#start warden 
+service mapr-warden start
+
+
+#configure hive
+/opt/mapr/server/configure.sh -R
+
+#wait for warden to start all the services
+
+sh /root/wardenTracker.sh
+
+
+#run hdfs commands
+hadoop fs -mkdir /user/hive-user
+hadoop fs -chmod 777 /user/hive-user
+hadoop fs -mkdir /user/hdfs-user
+hadoop fs -chmod 777 /user/hdfs-user
+hadoop fs -mkdir /user/root
+hadoop fs -mkdir /user/hive
+hadoop fs -mkdir /user/hive/warehouse
+hadoop fs -chmod 777 /user/hive
+hadoop fs -chmod 777 /user/hive/warehouse
+
+

teradatalabs/mapr52-hive/files/conf/core-site.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+	<property>
+  		<name>hadoop.proxyuser.mapr.groups</name>
+  		<value>*</value>
+  	<description>Allow the superuser mapr to impersonate any member of any group</description>
+	</property>
+	<property>
+  		<name>hadoop.proxyuser.mapr.hosts</name>
+  		<value>*</value>
+  		<description>The superuser can connect from any host to impersonate a user</description>
+	</property>
+</configuration>
+

teradatalabs/mapr52-hive/files/conf/hive-site.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+   <property>
+      <name>datanucleus.schema.autoCreateAll</name>
+      <value>true</value>
+      <description>creates necessary schema on a startup if one doesn't exist. set
+         this to false, after creating it once</description>
+   </property>
+   <property>
+  	<name>hive.server2.enable.doAs</name>
+  	<value>true</value>
+  	<description>Set this property to enable impersonation in Hive Server 2</description>
+   </property>
+   <property>
+  	<name>hive.metastore.execute.setugi</name>
+  	<value>true</value>
+  	<description>Set this property to enable Hive Metastore service impersonation in unsecure mode. In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. If the client sets it to true and the server sets it to false, the client setting will be ignored.</description>
+   </property>
+
+ <property>
+    <name>javax.jdo.option.ConnectionURL</name>
+    <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
+    <description>JDBC connect string for a JDBC metastore</description>
+</property>
+
+ <property>
+    <name>javax.jdo.option.ConnectionDriverName</name>
+    <value>com.mysql.jdbc.Driver</value>
+    <description>Driver class name for a JDBC metastore</description>
+ </property>
+
+ <property>
+    <name>javax.jdo.option.ConnectionUserName</name>
+    <value>root</value>
+    <description>username to use against metastore database</description>
+ </property>
+
+ <property>
+    <name>javax.jdo.option.ConnectionPassword</name>
+    <value>root</value>
+    <description>password to use against metastore database</description>
+ </property>
+
+ <property>
+    <name>hive.metastore.uris</name>
+    <value>thrift://localhost:9083</value>
+ </property>
+
+</configuration>

teradatalabs/mapr52-hive/files/setup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+mysql_install_db
+/usr/bin/mysqld_safe &
+sleep 20
+/usr/bin/mysqladmin -u root password 'root'
+killall mysqld
+sleep 10
+mkdir /var/log/mysql

teradatalabs/mapr52-hive/files/socks-proxy.sh

@@ -0,0 +1 @@
+../../../commons/socks-proxy.sh

teradatalabs/mapr52-hive/files/startup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec supervisord -c /etc/supervisord.conf

teradatalabs/mapr52-hive/files/supervisord.conf

@@ -0,0 +1,22 @@
+[supervisord]
+logfile = /var/log/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /var/run/supervisord.pid
+nodaemon = true
+directory = /tmp
+strip_ansi = false
+
+[unix_http_server]
+file = /tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+prompt = cdh-pseudo-distributed
+
+[include]
+files = /etc/supervisord.d/*.conf

teradatalabs/mapr52-hive/files/supervisord.d/bootstrap.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/bootstrap.conf

teradatalabs/mapr52-hive/files/supervisord.d/mysql-metastore.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/mysql-metastore.conf

teradatalabs/mapr52-hive/files/supervisord.d/socks-proxy.conf

@@ -0,0 +1 @@
+../../../../commons/supervisord.d/socks-proxy.conf

teradatalabs/mapr52-hive/files/wardenTracker.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+hname=$(hostname)
+
+Services=0
+
+maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+
+#wait for CLDB to start
+while [ $Services -ne 2 ]
+do
+if [ $maprcliReady == 1 ]
+then
+   maprcliReady=$(maprcli service list -node $hname | grep 'ERROR (10009)' | wc -l)
+   Services=0
+else
+   Services=$(maprcli service list -node $hname | grep JobHistoryServer |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+fi
+done
+
+#wait for NodeManager to start
+
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep NodeManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+#wait for ResourceManager to start
+
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep ResourceManager |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+#wait for Hive Metastore to start
+
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveMetastore |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+#wait for HiveServer2 to start
+
+Services=0
+while [ $Services -ne 2 ]
+do
+   Services=$(maprcli service list -node $hname | grep HiveServer2 |awk '{$1=$1};1' | tr ' ' '\n' | tail -1f)
+done
+
+
+