new playbook to backup masters etcd and nodes

backup_restore/README.md

@@ -14,3 +14,5 @@ Script / Playbook Name | Description | Notes
 `ocp-project-backup.sh` | Creates a yaml backup of all projects in the OpenShift cluster. | Must be run on an OCP master node.
 `ocp-etcd3-pods-backup.sh` | Creates a backup of the etcd (data snapshot & config). Also manages backup retention. | Must be run on the OCP nodes running the etcd static Pods (usually masters).
 `master-backup.yaml` | Ansible clone of the bash `ocp-master-cert-backup.sh` script.
+`ocp_backup.yml` | Backup masters and nodes using the Day-2 Guide | Run from a control host
+`./roles` | Roles use in the `ocp_backup.yml` playbook

backup_restore/ocp_backup.yml

@@ -0,0 +1,66 @@
+----
+# This playbook and the roles follows OCP Day-2 Operation Guide
+# Tested on OCP 3.10
+# Commented out - The last tasks send backed up files to S3 bucket
+# Run this playbook from a control host:
+# $ ansible-playbook -i inventory ocp_backup.yml
+#
+- name: backup master hosts
+  hosts: masters
+  roles:
+  - ocp_backup_etcd
+  - ocp_backup_masters
+  - ocp_backup_compress_backup
+
+- name: ocp backup of node hosts
+  hosts: nodes,!masters
+  roles:
+  - ocp_backup_nodes
+  - ocp_backup_compress_backup
+
+- name: Set up directory for fetching tar files
+  hosts: localhost
+  connection: local
+  tasks:
+    - name: Create local temp directory for fetching files 
+      file:
+        path: "/tmp/ocp_backup_{{ ansible_date_time.date }}"
+        state: directory
+      register: local_dir
+
+- name: fetch tar files from all hosts
+  hosts: OSEv3
+  tasks:
+    - name: fetch tar files
+      fetch:
+        src: "/root/ocp_backup/{{ inventory_hostname }}.tar.gz"
+        dest: "{{ hostvars['localhost'].local_dir.path }}/"
+        flat: yes
+
+    - name: remove compressed file from remote dir
+      file:
+        path: "/root/ocp_backup"
+        state: absent
+
+### NOTES ####
+# To enable sync to s3, need the following:
+# yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+# yum -y install python-pip python-wheel
+# pip install boto3
+### END NOTES ####
+### Send to S3 ####
+#- name: prepare files and send backup to S3
+#  hosts: localhost
+#  connection: local
+#  tasks:
+#    - name: sync files to S3
+#      s3_sync:
+#        bucket: my-s3-bucket
+#        key_prefix: "ocp_backup_{{ ansible_date_time.date }}"
+#        file_root: "{{ hostvars['localhost'].local_dir.path }}/"
+#
+#    - name: remove temp files
+#      file:
+#        path: "{{ hostvars['localhost'].local_dir.path }}"
+#        state: absent
+### Send to S3 END ###

backup_restore/roles/ocp_backup_compress_backup/README.md

@@ -0,0 +1,40 @@
+Role Name
+=========
+
+Simple task to compress backed up files and directories
+
+Reference doc:
+
+https://docs.openshift.com/container-platform/3.10/day_two_guide/environment_backup.html#backing-up-node_environment-backup
+
+
+Requirements
+------------
+
+
+Role Variables
+--------------
+
+
+Dependencies
+------------
+
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

backup_restore/roles/ocp_backup_compress_backup/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# defaults file for ocp_backup_compress_backup
+

backup_restore/roles/ocp_backup_compress_backup/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for ocp_backup_compress_backup

backup_restore/roles/ocp_backup_compress_backup/meta/main.yml

@@ -0,0 +1,57 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+
+  min_ansible_version: 1.2
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If Travis integration is configured, only notifications for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
+  #
+  # platforms is a list of platforms, and each platform has a name and a list of versions.
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

backup_restore/roles/ocp_backup_compress_backup/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+# Compress backup
+- name: compress backup directory
+  archive:
+    path: "{{ MYBACKUPDIR }}"
+    dest: "{{ MYBACKUPDIR }}.tar.gz"
+    format: gz
+    remove: no
+
+- name: manually remove data files
+  shell: rm -rf {{ MYBACKUPDIR }}/

backup_restore/roles/ocp_backup_compress_backup/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

backup_restore/roles/ocp_backup_compress_backup/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - ocp_backup_compress_backup

backup_restore/roles/ocp_backup_compress_backup/vars/main.yml

@@ -0,0 +1,4 @@
+---
+# vars file for ocp_backup_compress_backup
+MYBACKUPDIR: "/root/ocp_backup/{{ inventory_hostname }}"
+

backup_restore/roles/ocp_backup_etcd/README.md

@@ -0,0 +1,48 @@
+Role Name
+=========
+
+Backup etcd v3 data and config, of static Pods.
+
+Refererence doc: https://docs.openshift.com/container-platform/3.10/day_two_guide/environment_backup.html#etcd-backup_environment-backup
+
+Documentation notes on restoring etcd:
+
+```
+The etcdctl backup command rewrites some of the metadata contained in
+the backup,specifically, the node ID and cluster ID, which means that in
+the backup,the node loses its former identity. To recreate a cluster from
+the backup, you create a new, single-node cluster, then add the rest of
+the nodes to the cluster. The metadata is rewritten to prevent
+the new node from joining an existing cluster.
+```
+
+Requirements
+------------
+
+
+Role Variables
+--------------
+
+
+Dependencies
+------------
+
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

backup_restore/roles/ocp_backup_etcd/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for ocp_backup_etcd

backup_restore/roles/ocp_backup_etcd/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for ocp_backup_etcd

backup_restore/roles/ocp_backup_etcd/meta/main.yml

@@ -0,0 +1,57 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+
+  min_ansible_version: 1.2
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If Travis integration is configured, only notifications for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
+  #
+  # platforms is a list of platforms, and each platform has a name and a list of versions.
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

backup_restore/roles/ocp_backup_etcd/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+# tasks file for ocp_backup_etcd
+- name: pre_check tasks
+  import_tasks: pre_check.yml
+
+- name: save etcd data
+  import_tasks: save_etcd.yml
+

backup_restore/roles/ocp_backup_etcd/tasks/pre_check.yml

@@ -0,0 +1,28 @@
+---
+- name: backup directory {{ MYBACKUPDIR }} exist
+  file: 
+    name: "{{ MYBACKUPDIR }}"
+    state: directory
+
+- name: check for {{ MASTER_EXEC }} command
+  stat: 
+    path: "{{ MASTER_EXEC }}"
+  register: master_exec
+  failed_when: master_exec.stat.exists == False
+
+- name: check for {{ ETCD_POD_MANIFEST }}
+  stat:
+    path: "{{ ETCD_POD_MANIFEST }}"
+  register: etcd_pod_manifest
+  failed_when: etcd_pod_manifest.stat.exists == False
+
+- name: check {{ MYBACKUPDIR }} destination
+  shell: df -h {{ MYBACKUPDIR }} | grep -v 'Filesystem' | awk '{ print $5}' | cut -d '%' -f 1
+  register: shell_result
+  changed_when: false
+
+- name: fail with custom message
+  fail:
+    msg: "{{ MYBACKUPDIR }} is almost full"
+  when: shell_result.stdout|int > 90
+

backup_restore/roles/ocp_backup_etcd/tasks/save_etcd.yml

@@ -0,0 +1,26 @@
+---
+# Save etcd configuration
+- name: backup dir {{ MYBACKUPDIR }}/etcd-config exists
+  file:
+    path: "{{ MYBACKUPDIR }}/etcd-config"
+    state: directory
+
+- name: save etcd configuration
+  shell: cp -aR /etc/etcd/ {{ MYBACKUPDIR }}/etcd-config/
+
+- name: get etcd endpoint
+  shell: grep https {{ ETCD_POD_MANIFEST }} | cut -d '/' -f3
+  register: etcd_ep
+
+- name: etcd data - save snapshot
+  shell: "{{ MASTER_EXEC }} etcd etcd /bin/bash -c \"ETCDCTL_API=3 /usr/bin/etcdctl --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints {{ etcd_ep.stdout }} snapshot save /var/lib/etcd/snapshot.db\""
+
+- name: etcd data - check snapshot status
+  shell: "{{ MASTER_EXEC }} etcd etcd /bin/bash -c \"ETCDCTL_API=3 /usr/bin/etcdctl --cert /etc/etcd/peer.crt --key /etc/etcd/peer.key --cacert /etc/etcd/ca.crt --endpoints {{ etcd_ep.stdout }} snapshot status /var/lib/etcd/snapshot.db\""
+  register: status_result
+  failed_when: status_result.rc != 0
+
+- name: move snapshot to {{ MYBACKUPDIR }}/etcd-config
+  command: mv /var/lib/etcd/snapshot.db {{ MYBACKUPDIR }}/etcd-config/snapshot.db
+
+

backup_restore/roles/ocp_backup_etcd/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

backup_restore/roles/ocp_backup_etcd/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - ocp_backup_etcd