Add fluentd

incubator/fluentd/examples-elasticsearch-kibana/es.libsonnet

@@ -0,0 +1,155 @@
+local k = import "ksonnet.beta.2/k.libsonnet";
+
+// Destructuring imports for base.
+local container = k.core.v1.replicationController.mixin.spec.template.spec.containersType;
+local containerPort = container.portsType;
+local env = container.envType;
+local rc = k.core.v1.replicationController;
+local service = k.core.v1.service;
+local servicePort = service.mixin.spec.portsType;
+local volume = k.core.v1.replicationController.mixin.spec.template.spec.volumesType;
+local volumeMount = container.volumeMountsType;
+
+// Destructuring RBAC imports.
+local svcAccount = k.core.v1.serviceAccount;
+local clRoleBinding = k.rbac.v1beta1.clusterRoleBinding;
+local clRole = k.rbac.v1beta1.clusterRole;
+local subject = clRoleBinding.subjectsType;
+local rule = clRole.rulesType;
+
+{
+  app:: {
+    new(config)::
+      local dbPortName = "db";
+      local rbacObjs = $.parts.rbac(config.rbac.accountName, config.namespace);
+      rbacObjs + {
+        toArray()::
+          local objs = [self.service, self.controller];
+          if "toArray" in super
+          then super.toArray() + objs
+          else objs,
+        service:: $.parts.service(dbPortName, config),
+        controller:: $.parts.controller(dbPortName, config),
+      },
+  },
+
+  parts:: {
+    local boilerplate = {
+      appName:: "elasticsearch-logging",
+
+      storageName:: "es-persistent-storage",
+
+      controller:: {
+        selector:: {
+          "k8s-app": boilerplate.appName,
+          "version": "v1",
+        },
+
+        labels:: self.selector + {
+          "addonmanager.kubernetes.io/mode": "Reconcile",
+          "kubernetes.io/cluster-service": "true",
+        },
+
+        templateLabels:: self.selector + {
+          "kubernetes.io/cluster-service": "true",
+        },
+      },
+
+      service:: {
+        labels:: {
+          "addonmanager.kubernetes.io/mode": "Reconcile",
+          "k8s-app": boilerplate.appName,
+          "kubernetes.io/cluster-service": "true",
+          "kubernetes.io/name": "Elasticsearch"
+        },
+
+        selector:: {"k8s-app": boilerplate.appName},
+      },
+    },
+
+    container(dbPortName, config)::
+      local dbPort =
+        containerPort.newNamed(dbPortName, 9200) +
+        containerPort.protocol("TCP");
+      local transportPort =
+        containerPort.newNamed("transport", 9300) +
+        containerPort.protocol("TCP");
+      local dataMount = volumeMount.new(boilerplate.storageName, "/data");
+      local resources =
+        container.mixin.resources.limits({cpu: "1000m"}) +
+        container.mixin.resources.requests({cpu: "100m"});
+      container.new(
+        boilerplate.appName,
+        "gcr.io/google_containers/elasticsearch:%s" % config.container.tag) +
+      container.env(env.fromFieldPath("NAMESPACE", "metadata.namespace")) +
+      container.ports([dbPort, transportPort]) +
+      container.volumeMounts(dataMount) +
+      resources,
+
+    controller(dbPortName, config)::
+      local dataVol = volume.fromEmptyDir(boilerplate.storageName, {});
+      rc.new() +
+      rc.mixin.metadata.name("elasticsearch-logging-v1") +
+      rc.mixin.metadata.namespace(config.namespace) +
+      rc.mixin.metadata.labels(boilerplate.controller.labels) +
+      rc.mixin.spec.replicas(2) +
+      rc.mixin.spec.selector(boilerplate.controller.selector) +
+      rc.mixin.spec.template.metadata.labels(boilerplate.controller.templateLabels) +
+      rc.mixin.spec.template.spec.containers($.parts.container(dbPortName, config)) +
+      rc.mixin.spec.template.spec.volumes(dataVol) +
+      rc.mixin.spec.template.spec.serviceAccountName(config.rbac.accountName),
+
+    service(dbPortName, config)::
+      local port =
+        servicePort.new(9200, dbPortName) +
+        servicePort.protocol("TCP");
+      service.new("elasticsearch-logging", boilerplate.service.selector, [port]) +
+      service.mixin.metadata.namespace(config.namespace) +
+      service.mixin.metadata.labels(boilerplate.service.labels) +
+      service.mixin.spec.type("LoadBalancer"),
+
+    // Creates all top-level objects and mixins we need to add RBAC
+    // support to support the ElasticSearch logging infrastructure.
+    rbac(name, namespace)::
+      local metadata = svcAccount.mixin.metadata.name(name) +
+        svcAccount.mixin.metadata.namespace(namespace);
+
+      local hcServiceAccount = svcAccount.new() +
+        metadata;
+
+      local hcClusterRole =
+        clRole.new() +
+        metadata +
+        clRole.rules(
+          rule.new() +
+          rule.apiGroups("*") +
+          rule.resources(["namespaces", "services", "endpoints"]) +
+          rule.verbs(["get"])
+        );
+
+      local hcClusterRoleBinding =
+        clRoleBinding.new() +
+        metadata +
+        clRoleBinding.mixin.roleRef.apiGroup("rbac.authorization.k8s.io") +
+        clRoleBinding.mixin.roleRef.name(name) +
+        clRoleBinding.mixin.roleRef.mixinInstance({kind: "ClusterRole"}) +
+        clRoleBinding.subjects(
+          subject.new() +
+          subject.name(name) +
+          subject.namespace(namespace)
+          {kind: "ServiceAccount"}
+        );
+
+      // Return.
+      {} + {
+        toArray()::
+          local objs = [self.account, self.clusterRole, self.roleBinding];
+          if "toArray" in super
+          then super.toArray() + objs
+          else objs,
+        account:: hcServiceAccount,
+        clusterRole:: hcClusterRole,
+        roleBinding:: hcClusterRoleBinding,
+      },
+  },
+}

incubator/fluentd/examples-elasticsearch-kibana/using-daemonset-builder/es.jsonnet

@@ -0,0 +1,30 @@
+local k = import "ksonnet.beta.2/k.libsonnet";
+
+local es = import "../es.libsonnet";
+
+// Configuration. Specifies how to set up the ElasticSearch app.
+local config = {
+  namespace:: "elasticsearch",
+  rbac:: {
+    accountName:: "elasticsearch-serviceaccount",
+  },
+  container:: {
+    tag:: "v2.4.1-2",
+  },
+};
+
+// TODO: Move the rbac out here, too.
+k.core.v1.list.new(
+  es.app.new(config).toArray() +
+  [
+    {
+      "kind": "Namespace",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": config.namespace,
+        "labels": {
+          "name": config.namespace
+        }
+      }
+    }
+  ])

incubator/fluentd/examples-elasticsearch-kibana/using-daemonset-builder/fluentd-es-ds.jsonnet

@@ -0,0 +1,34 @@
+local k = import "ksonnet.beta.2/k.libsonnet";
+
+local fluentd = import "incubator/fluentd/fluentd.libsonnet";
+
+// Destructuring imports for base.
+local container = k.core.v1.replicationController.mixin.spec.template.spec.containersType;
+local ds = k.extensions.v1beta1.daemonSet;
+local volume = k.core.v1.replicationController.mixin.spec.template.spec.volumesType;
+local volumeMount = container.volumeMountsType;
+
+local config = {
+  namespace:: "elasticsearch",
+  container:: {
+    name:: "fluentd-es",
+    tag:: "1.22",
+  },
+  daemonSet:: {
+    name:: "fluentd-es-v1.22",
+  },
+  rbac:: {
+    accountName:: "fluentd-serviceaccount"
+  },
+};
+
+local ds =
+  fluentd.app.daemonSetBuilder.new(config) +
+  fluentd.app.daemonSetBuilder.configureForPodLogs(config);
+
+local rbacObjs = fluentd.app.admin.rbacForPodLogs(config);
+
+k.core.v1.list.new(
+  ds.toArray() +
+  rbacObjs.toArray()
+)

incubator/fluentd/examples-elasticsearch-kibana/using-daemonset-builder/generated/es.json

@@ -0,0 +1,176 @@
+{
+   "apiVersion": "v1",
+   "items": [
+      {
+         "apiVersion": "v1",
+         "kind": "ServiceAccount",
+         "metadata": {
+            "name": "elasticsearch-serviceaccount",
+            "namespace": "elasticsearch"
+         }
+      },
+      {
+         "apiVersion": "rbac.authorization.k8s.io/v1beta1",
+         "kind": "ClusterRole",
+         "metadata": {
+            "name": "elasticsearch-serviceaccount",
+            "namespace": "elasticsearch"
+         },
+         "rules": [
+            {
+               "apiGroups": [
+                  "*"
+               ],
+               "resources": [
+                  "namespaces",
+                  "services",
+                  "endpoints"
+               ],
+               "verbs": [
+                  "get"
+               ]
+            }
+         ]
+      },
+      {
+         "apiVersion": "rbac.authorization.k8s.io/v1beta1",
+         "kind": "ClusterRoleBinding",
+         "metadata": {
+            "name": "elasticsearch-serviceaccount",
+            "namespace": "elasticsearch"
+         },
+         "roleRef": {
+            "apiGroup": "rbac.authorization.k8s.io",
+            "kind": "ClusterRole",
+            "name": "elasticsearch-serviceaccount"
+         },
+         "subjects": [
+            {
+               "kind": "ServiceAccount",
+               "name": "elasticsearch-serviceaccount",
+               "namespace": "elasticsearch"
+            }
+         ]
+      },
+      {
+         "apiVersion": "v1",
+         "kind": "Service",
+         "metadata": {
+            "labels": {
+               "addonmanager.kubernetes.io/mode": "Reconcile",
+               "k8s-app": "elasticsearch-logging",
+               "kubernetes.io/cluster-service": "true",
+               "kubernetes.io/name": "Elasticsearch"
+            },
+            "name": "elasticsearch-logging",
+            "namespace": "elasticsearch"
+         },
+         "spec": {
+            "ports": [
+               {
+                  "port": 9200,
+                  "protocol": "TCP",
+                  "targetPort": "db"
+               }
+            ],
+            "selector": {
+               "k8s-app": "elasticsearch-logging"
+            },
+            "type": "LoadBalancer"
+         }
+      },
+      {
+         "apiVersion": "v1",
+         "kind": "ReplicationController",
+         "metadata": {
+            "labels": {
+               "addonmanager.kubernetes.io/mode": "Reconcile",
+               "k8s-app": "elasticsearch-logging",
+               "kubernetes.io/cluster-service": "true",
+               "version": "v1"
+            },
+            "name": "elasticsearch-logging-v1",
+            "namespace": "elasticsearch"
+         },
+         "spec": {
+            "replicas": 2,
+            "selector": {
+               "k8s-app": "elasticsearch-logging",
+               "version": "v1"
+            },
+            "template": {
+               "metadata": {
+                  "labels": {
+                     "k8s-app": "elasticsearch-logging",
+                     "kubernetes.io/cluster-service": "true",
+                     "version": "v1"
+                  }
+               },
+               "spec": {
+                  "containers": [
+                     {
+                        "env": [
+                           {
+                              "name": "NAMESPACE",
+                              "valueFrom": {
+                                 "fieldRef": {
+                                    "fieldPath": "metadata.namespace"
+                                 }
+                              }
+                           }
+                        ],
+                        "image": "gcr.io/google_containers/elasticsearch:v2.4.1-2",
+                        "name": "elasticsearch-logging",
+                        "ports": [
+                           {
+                              "containerPort": 9200,
+                              "name": "db",
+                              "protocol": "TCP"
+                           },
+                           {
+                              "containerPort": 9300,
+                              "name": "transport",
+                              "protocol": "TCP"
+                           }
+                        ],
+                        "resources": {
+                           "limits": {
+                              "cpu": "1000m"
+                           },
+                           "requests": {
+                              "cpu": "100m"
+                           }
+                        },
+                        "volumeMounts": [
+                           {
+                              "mountPath": "/data",
+                              "name": "es-persistent-storage",
+                              "readOnly": false
+                           }
+                        ]
+                     }
+                  ],
+                  "serviceAccountName": "elasticsearch-serviceaccount",
+                  "volumes": [
+                     {
+                        "emptyDir": { },
+                        "name": "es-persistent-storage"
+                     }
+                  ]
+               }
+            }
+         }
+      },
+      {
+         "apiVersion": "v1",
+         "kind": "Namespace",
+         "metadata": {
+            "labels": {
+               "name": "elasticsearch"
+            },
+            "name": "elasticsearch"
+         }
+      }
+   ],
+   "kind": "List"
+}