Integrate 1.9.2 release into main, allow integration of user-plugins in an easy way

.github/workflows/anypush.yml

@@ -0,0 +1,37 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Build-Publish-Any
+
+# Controls when the workflow will run
+on:
+  push:
+    branches:
+      - publish
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+        - name: Checkout 
+          uses: actions/checkout@v2
+        - name: Login to Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v1
+        - name: Build and push
+          uses: docker/build-push-action@v2
+          with:
+            context: .
+            file: ./Dockerfile
+            push: true
+            tags: ${{ secrets.DOCKER_USERNAME }}/smarthomeng:develop

.github/workflows/main.yml

@@ -0,0 +1,37 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Build-Publish-Release
+
+# Controls when the workflow will run
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+        - name: Checkout 
+          uses: actions/checkout@v2
+        - name: Login to Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: ${{ secrets.DOCKER_USERNAME }}
+            password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v1
+        - name: Build and push
+          uses: docker/build-push-action@v2
+          with:
+            context: .
+            file: ./Dockerfile
+            push: true
+            tags: ${{ secrets.DOCKER_USERNAME }}/smarthomeng:${{github.ref_name }},${{ secrets.DOCKER_USERNAME }}/smarthomeng:latest

Dockerfile

@@ -1,7 +1,7 @@
 ### dockerfile for smarthomNG flavor "full"
 
 ### select python base image ####################################################
-FROM python:3.8-slim As python-base
+FROM python:3.9-slim As python-base
 
 ### Build Stage 1 - clone smarthome NG from Git #################################
 FROM python-base As stage1
@@ -12,18 +12,20 @@ RUN set -eux; apt-get update; apt-get install -y --no-install-recommends \
   rm -rf /var/lib/apt/lists/*
 
 # prepare clone
-ARG SHNG_VER_CORE="v1.9.1" \
-    SHNG_VER_PLGN="v1.9.1" \
-    PLGN_DEL="gpio"
+ARG SHNG_VER_CORE="v1.9.5" \
+    SHNG_VER_PLGN="v1.9.5" \
+    SHNG_REPO_CORE="https://github.com/smarthomeNG/smarthome.git" \
+    SHNG_REPO_PLGN="https://github.com/smarthomeNG/plugins.git" \
+    PLGN_DEL="gpio zwave"
 
 # clone smarthomeNG from Git
 WORKDIR /usr/local/smarthome
 RUN set -eux; \
 # clone SmarthomeNG
   git -c advice.detachedHead=false clone --single-branch --branch $SHNG_VER_CORE \
-    https://github.com/smarthomeNG/smarthome.git .; \
+    $SHNG_REPO_CORE .; \
   git -c advice.detachedHead=false clone --single-branch --branch $SHNG_VER_PLGN \
-    https://github.com/smarthomeNG/plugins.git plugins; \
+    $SHNG_REPO_PLGN plugins-default; \
 # remove git files - not usefull inside a container
   find . -name ".git*" -print -exec rm -rf {} +; \
   find . -name ".*" -type f -print -exec rm -rf {} +; \
@@ -32,26 +34,30 @@ RUN set -eux; \
   find . -name "*.md" -print -exec rm -rf {} +; \
 # remove plugins if they are not running - for example GPIO is RasPi specific
   if [ "$PLGN_DEL" ]; then \
-    for i in $PLGN_DEL; do rm -rf plugins/$i; done; \
+    for i in $PLGN_DEL; do rm -rf plugins-default/$i; done; \
   fi
 
-### Build Stage 11 - determine requirements for smarthomNG #######################
+### Build Stage 11 - determine requirements for smarthomeNG #######################
 FROM stage1 As stage2
 
 ARG PLGN_CONFLICT="appletv hue2"
 
 WORKDIR /usr/local/smarthome
 RUN set -eux; \
-# remove some plugins to remove there requirements
+# remove some plugins to remove their requirements
   if [ "$PLGN_CONFLICT" ]; then \
-    for i in $PLGN_CONFLICT; do rm -rf plugins/$i; done; \
+    for i in $PLGN_CONFLICT; do rm -rf plugins-default/$i; done; \
   fi; \
+# create links from the default plugins-folder to the to be used one.
+  cp -alr plugins-default plugins; \
 # necessary to run smarthome.py
-  python -m pip install --no-cache-dir ruamel.yaml; \
+  export PIP_ROOT_USER_ACTION=ignore; \
+  /usr/local/bin/pip3 install --upgrade pip; \
+  /usr/local/bin/pip3 install --no-cache-dir ruamel.yaml; \
 # create requirement files
-  python3 bin/smarthome.py --stop
+  python3 bin/smarthome.py --stop --pip3_command /usr/local/bin/pip3
 
-### Build Stage 3 - build requirements for smarthomNG ###########################
+### Build Stage 3 - build requirements for smarthomeNG ###########################
 FROM python-base As stage3
 
 COPY --from=stage2 /usr/local/smarthome/requirements/all.txt /requirements.txt
@@ -60,6 +66,7 @@ COPY --from=stage2 /usr/local/smarthome/requirements/all.txt /requirements.txt
 RUN set -eux; \
   apt-get update; apt-get install -y --no-install-recommends \
     #pyjq
+    python3-ephem \
     automake \
     #pyjq, openzwave
     build-essential \
@@ -68,14 +75,15 @@ RUN set -eux; \
     #rrd
     librrd-dev \
     #pyjq
-    libtool \
+    libtool; \
     #openzwave
-    libudev-dev \
-    openzwave; \
+    ## libudev-dev \
+    ## openzwave; \
   rm -rf /var/lib/apt/lists/*; \
 # fix python requirements
   echo "holidays<0.13" >>/requirements.txt; \
-  #sed -e 's/^\(holidays.*\)/\1,<=0.12;python_version==3.8/g' lib/requirements.txt; \
+# Add pymysql
+  echo "pymysql" >>/requirements.txt; \
 # install python requirements
   python -m pip install --no-cache-dir -r requirements.txt
 
@@ -114,6 +122,7 @@ RUN set -eux; \
   chmod go+rw $PATH_CONF/etc; \
 # prepare data
   mkdir -p $PATH_SHNG/var/run; \
+  chmod go+rw $PATH_SHNG/var; \
   chmod go+rw $PATH_SHNG/var/run; \
   for i in $DIRS_DATA; do \
     mkdir -p $PATH_DATA/$i; \
@@ -134,7 +143,7 @@ RUN set -eux; \
 EXPOSE 2323 2424 8383
 
 # and finalize
-#COPY ./entrypoint.sh ./shng_wrapper.sh /
-COPY * /
+COPY ./entrypoint.sh ./shng_wrapper.sh /
+RUN chmod +x ./entrypoint.sh; chmod +x ./shng_wrapper.sh
 ENTRYPOINT ["/entrypoint.sh"]
 CMD ["--foreground"]

README.md

@@ -5,16 +5,23 @@
 This is a Dockerfile/image to build a container for SmarthomeNG. 
 
 ## Quick Start
-To pull from docker hub:
+### Start from scratch
+
+- You should create a folder to store the files for configuration before starting the container. This folder can stay empty at the beginning, the container will create the structure itself.
+
+#### Docker native
 ```
-docker pull henfri/smarthome-ng
+docker pull smarthomeng/smarthomeng
 ```
 Clone [smartVISU](https://github.com/Martin-Gleiss/smartvisu) to `./volume/html/smartvisu`:
 ```
 git clone https://github.com/Martin-Gleiss/smartvisu.git ./volume/html/smartvisu
 ```
+#### Synology NAS
+
+#### Portainer
 
-### Running
+#### Docker-compose / Portainer Stack
 
 Example with web server docker-compose.yaml:
 ```
@@ -56,6 +63,17 @@ services:
 >
 Don't forget to adjust the driver settings in SmartVISU.
 
+### But I had a container up and running with the previous image (SHNG 1.5.1 - 1.9.1)
+
+While developing a very user-friendly structure for people who start new we haven't forgotten about those who run the Docker infrastructure for years already. Therefor you can run the container like you are used to with folders that are mounted into the container image.
+
+
+#### Migrating to the new folder pattern
+
+
+
+
+
 ## Configuration Parameters
 
 | Name               | Description                                                         |

entrypoint.sh

@@ -5,6 +5,9 @@ SHNG_ARG=$@
 PATH_SHNG=/usr/local/smarthome
 PATH_CONF=/mnt/conf
 PATH_DATA=/mnt/data
+PATH_PLGN_USER=/mnt/plugins
+PATH_PLGN_TRGT=/usr/local/smarthome/plugins
+PATH_PLGN_DFLT=/usr/local/smarthome/plugins-default
 PATH_HTML=/mnt/html
 DIRS_CONF="etc items logics scenes functions"
 DIRS_DATA="backup restore cache db log"
@@ -28,7 +31,7 @@ if [ -f $PATH_SHNG/etc/.not_mounted ]; then
       WARN_MOUNT_CONF="${WARN_MOUNT_CONF# } $i"
     elif [ ! -f $PATH_CONF/$i/.files_created ]; then
       mkdir -p $PATH_CONF/$i
-      cp -vnr $PATH_SHNG/$i/* $PATH_CONF
+      cp -vnr $PATH_SHNG/$i/* $PATH_CONF/$i
       touch $PATH_CONF/$i/.files_created
     fi
   done
@@ -116,6 +119,7 @@ if [ "$USER_SHNG" ]; then
     for i in $DIRS_DATA; do
       chown -R $USER_SHNG $PATH_DATA/$i
     done
+    chown $USER_SHNG $PATH_DATA
   fi
   if [ "$SKIP_CHOWN_HTML" != "1" ]; then
     chown -R $USER_WWW $PATH_HTML
@@ -126,6 +130,61 @@ if [ "$USER_SHNG" ]; then
   fi
 fi
 
+#merge plugins appropriately
+if [ -d $PATH_PLGN_TRGT -a ! -f $PATH_PLGN_TRGT/.was_merge_built ]; then
+  # if Plugin folder is already there, specific plugins were mounted from outside
+  shopt -s extglob nullglob
+  # take all plugin-folders in the custom folder
+  PLUGINS_FROM_DEFAULT=( "$PATH_PLGN_DFLT"/*/ )
+  # remove leading basedir
+  PLUGINS_FROM_DEFAULT=( "${PLUGINS_FROM_DEFAULT[@]#"$PATH_PLGN_DFLT/"}" )
+  # remove trailing slash
+  PLUGINS_FROM_DEFAULT=( "${PLUGINS_FROM_DEFAULT[@]%/}" )
+  for i in "${!PLUGINS_FROM_DEFAULT[@]}"; do
+    if [ -d $PATH_PLGN_TRGT/${PLUGINS_FROM_DEFAULT[i]} ]; then
+      _print INFO Plugin already mounted here ${PLUGINS_FROM_DEFAULT[i]}
+    else
+      cp -alr "$PATH_PLGN_DFLT/${PLUGINS_FROM_DEFAULT[i]}" "$PATH_PLGN_TRGT/${PLUGINS_FROM_DEFAULT[i]}"
+    fi
+  done
+  # copy root files as well
+  if [ ! -f $PATH_PLGN_TRGT/__init__.py ]; then
+    _print INFO __init__.py did not exist in plugin-folder, so it is now put there
+    cp $PATH_PLGN_DFLT/__init__.py $PATH_PLGN_TRGT/
+  fi
+else
+  # if plugin-folder is not yet available, build new one and merge plugins
+  # if we come here, and the folder is present already, we are re-building this whole exercise
+  if [ -d $PATH_PLGN_TRGT ]; then
+    rm -rf $PATH_PLGN_TRGT
+  fi
+  cp -alr $PATH_PLGN_DFLT $PATH_PLGN_TRGT
+  if [ -d $PATH_PLGN_USER ]; then
+    if [ -f $PATH_PLGN_USER/download_plugins.sh ]; then
+      $PATH_PLGN_USER/download_plugins.sh || download_plugins_result=$?
+    fi
+    shopt -s extglob nullglob
+    # take all plugin-folders in the custom folder
+    PLUGINS_FROM_CUSTOM=( "$PATH_PLGN_USER"/*/ )
+    # remove leading basedir
+    PLUGINS_FROM_CUSTOM=( "${PLUGINS_FROM_CUSTOM[@]#"$PATH_PLGN_USER/"}" )
+    # remove trailing slash
+    PLUGINS_FROM_CUSTOM=( "${PLUGINS_FROM_CUSTOM[@]%/}" )
+
+    for i in "${!PLUGINS_FROM_CUSTOM[@]}"; do
+      if [ -d $PATH_PLGN_TRGT/${PLUGINS_FROM_CUSTOM[i]} ]; then
+        _print INFO Overwriting Plugin ${PLUGINS_FROM_CUSTOM[i]}
+        rm -rf $PATH_PLGN_TRGT/${PLUGINS_FROM_CUSTOM[i]}
+      else
+        _print INFO Copying Plugin ${PLUGINS_FROM_CUSTOM[i]}
+      fi
+      cp -vr "$PATH_PLGN_USER/${PLUGINS_FROM_CUSTOM[i]}" "$PATH_PLGN_TRGT/${PLUGINS_FROM_CUSTOM[i]}/"
+      touch $PATH_PLGN_TRGT/${PLUGINS_FROM_CUSTOM[i]}/.from_custom
+    done
+  fi
+  touch $PATH_PLGN_TRGT/.was_merge_built
+fi
+
 # start SmartHomeNG
 cd $PATH_SHNG
 if [ "$USER_SHNG" ]; then