Adding events visualization in kibana

Change-Id: I0d96d66da41bdb3292d827ad5ff63358f54f45f3
Story: 2003458
Task: 24718

README.rst

@@ -24,7 +24,12 @@ member port (default: 5000):
    monasca-kibana-plugin.port: ${keystone_port}
    monasca-kibana-plugin.url: http://${keystone_host}
    monasca-kibana-plugin.enabled: True
+   monasca-kibana-plugin.logs: True
+   monasca-kibana-plugin.events: True
    monasca-kibana-plugin.defaultTimeField: '@timestamp'
+   monasca-kibana-plugin.defaultEventsTimeField: '@timestamp'
+   monasca-kibana-plugin.logsIndexPrefix: 'logs-<project_id>'
+   monasca-kibana-plugin.eventsIndexPrefix: 'events-<project_id>'
 
 Then install using the Kibana plugin manager tool:
 

index.js

@@ -59,7 +59,12 @@ export default (kibana) => {
     return Joi
         .object({
             enabled: Joi.boolean().default(true),
+            events: Joi.boolean().default(false),
+            logs: Joi.boolean().default(true),
             defaultTimeField: Joi.string().default('@timestamp'),
+            defaultEventsTimeField: Joi.string().default('@timestamp'),
+            logsIndexPrefix: Joi.string().default('logs-<project_id>'),
+            eventsIndexPrefix: Joi.string().default('events-<project_id>'),
             cookie: cookie
         })
         .concat(deprecated_keystone)

package.json

@@ -1,6 +1,6 @@
 {
   "name":            "monasca-kibana-plugin",
-  "version":         "1.2.0",
+  "version":         "1.3.0",
   "description":     "Keystone authentication & multitenancy support for Kibana 4.6.x",
   "author":          "OpenStack",
   "license":         "Apache-2.0",
@@ -56,4 +56,4 @@
     "sinon":               "^1.17.3",
     "wreck":               "^8.0.0"
   }
-}
+}

server/__tests__/defaultIndexPattern.spec.js

@@ -27,6 +27,9 @@ describe('plugins/monasca-kibana-plugin', () => {
     configGet = sinon.stub();
     configGet.withArgs('pkg.version').returns('4.4.0');
     configGet.withArgs('monasca-kibana-plugin.defaultTimeField').returns('@timestamp');
+    configGet.withArgs('monasca-kibana-plugin.defaultEventsTimeField').returns('@timestamp');
+    configGet.withArgs('monasca-kibana-plugin.logsIndexPrefix').returns('logs-<project_id>');
+    configGet.withArgs('monasca-kibana-plugin.eventsIndexPrefix').returns('events-<project_id>');
 
     server = {
       log   : sinon.stub(),
@@ -53,44 +56,70 @@ describe('plugins/monasca-kibana-plugin', () => {
   });
 
   describe('defaultIndexPattern_exists', ()=> {
-    it('should return false if default pattern does not exist', (done) => {
+    it('should return false if default logs index-pattern does not exist', (done) => {
-      let exists = require('../mt/kibana/defaultIndexPattern/_exists').default;
+      let patternExists = require('../mt/kibana/defaultIndexPattern/_logs_exists').default;
 
-      let count = sinon.stub();
+      let exists = sinon.stub();
-      count.returns(Promise.resolve({ count: 0 }));
+      exists.returns(Promise.resolve(false));
-      server.plugins.elasticsearch.client.count = count;
+      server.plugins.elasticsearch.client.exists = exists;
 
-      exists(server, indexName)
+      patternExists(server, indexName, userObj)
         .then((resp) => {
           chai.assert.equal(resp, false);
-          chai.assert.isOk(count.calledOnce);
+          chai.assert.isOk(exists.calledOnce);
-          chai.assert.equal(count.args[0][0].index, '.kibana-testdefaultindex');
-          chai.assert.equal(count.args[0][0].type, 'index-pattern');
         })
         .then(done);
     });
 
-    it('should return true if default pattern already exists', (done) => {
+    it('should return true if default logs index-pattern already exists', (done) => {
-      let patternExists = require('../mt/kibana/defaultIndexPattern/_exists').default;
+      let patternExists = require('../mt/kibana/defaultIndexPattern/_logs_exists').default;
 
-      let count = sinon.stub();
+      let exists = sinon.stub();
-      count.returns(Promise.resolve({ count: 1 }));
+      exists.returns(Promise.resolve(true));
-      server.plugins.elasticsearch.client.count = count;
+      server.plugins.elasticsearch.client.exists = exists;
 
-      patternExists(server, indexName)
+      patternExists(server, indexName, userObj)
         .then((resp) => {
           chai.assert.equal(resp, true);
-          chai.assert.isOk(count.calledOnce);
+          chai.assert.isOk(exists.calledOnce);
-          chai.assert.equal(count.args[0][0].index, '.kibana-testdefaultindex');
+        })
-          chai.assert.equal(count.args[0][0].type, 'index-pattern');
+        .then(done);
+    });
+
+    it('should return false if default events index-pattern does not exist', (done) => {
+      let patternExists = require('../mt/kibana/defaultIndexPattern/_events_exists').default;
+
+      let exists = sinon.stub();
+      exists.returns(Promise.resolve(false));
+      server.plugins.elasticsearch.client.exists = exists;
+
+      patternExists(server, indexName, userObj)
+        .then((resp) => {
+          chai.assert.equal(resp, false);
+          chai.assert.isOk(exists.calledOnce);
+        })
+        .then(done);
+    });
+
+    it('should return true if default events index-pattern already exists', (done) => {
+      let patternExists = require('../mt/kibana/defaultIndexPattern/_events_exists').default;
+
+      let exists = sinon.stub();
+      exists.returns(Promise.resolve(true));
+      server.plugins.elasticsearch.client.exists = exists;
+
+      patternExists(server, indexName, userObj)
+        .then((resp) => {
+          chai.assert.equal(resp, true);
+          chai.assert.isOk(exists.calledOnce);
         })
         .then(done);
     });
   });
 
   describe('defaultIndexPattern_create', () => {
-    it('should create pattern with proper value', (done) => {
+    it('should create logs index-pattern with proper value', (done) => {
-      let createPattern = require('../mt/kibana/defaultIndexPattern/_create').default;
+      let createPattern = require('../mt/kibana/defaultIndexPattern/_logs_create').default;
 
       let create = sinon.stub();
       create.returns(Promise.resolve(true));
@@ -109,14 +138,50 @@ describe('plugins/monasca-kibana-plugin', () => {
           chai.assert.isOk(create.calledOnce);
           chai.assert.equal(create.args[0][0].index, '.kibana-testdefaultindex');
           chai.assert.equal(create.args[0][0].type, 'index-pattern');
-          chai.assert.equal(create.args[0][0].id, 'abcdef*');
+          chai.assert.equal(create.args[0][0].id, 'logs-abcdef*');
-          chai.assert.equal(create.args[0][0].body.title, 'abcdef*');
+          chai.assert.equal(create.args[0][0].body.title, 'logs-abcdef*');
           chai.assert.equal(create.args[0][0].body.timeFieldName, '@timestamp');
 
           chai.assert.isOk(update.calledOnce);
           chai.assert.equal(update.args[0][0].index, '.kibana-testdefaultindex');
           chai.assert.equal(update.args[0][0].type, 'config');
-          chai.assert.equal(update.args[0][0].body.doc.defaultIndex, 'abcdef*');
+          chai.assert.equal(update.args[0][0].body.doc.defaultIndex, 'logs-abcdef*');
+
+          chai.assert.isOk(refresh.called);
+          chai.assert.equal(refresh.args[0][0].index, '.kibana-testdefaultindex');
+        })
+        .then(done);
+    });
+
+    it('should create events index-pattern with proper value', (done) => {
+      let createPattern = require('../mt/kibana/defaultIndexPattern/_events_create').default;
+
+      let create = sinon.stub();
+      create.returns(Promise.resolve(true));
+      server.plugins.elasticsearch.client.create = create;
+
+      let update = sinon.stub();
+      update.returns(Promise.resolve(true));
+      server.plugins.elasticsearch.client.update = update;
+
+      let refresh = sinon.stub();
+      refresh.returns(Promise.resolve(true));
+      server.plugins.elasticsearch.client.indices.refresh = refresh;
+
+      createPattern(server, indexName, userObj)
+        .then((resp) => {
+          chai.assert.isOk(create.calledOnce);
+          chai.assert.equal(create.args[0][0].index, '.kibana-testdefaultindex');
+          chai.assert.equal(create.args[0][0].type, 'index-pattern');
+          console.log(create.args[0][0].id);
+          chai.assert.equal(create.args[0][0].id, 'events-abcdef*');
+          chai.assert.equal(create.args[0][0].body.title, 'events-abcdef*');
+          chai.assert.equal(create.args[0][0].body.timeFieldName, '@timestamp');
+
+          chai.assert.isOk(update.calledOnce);
+          chai.assert.equal(update.args[0][0].index, '.kibana-testdefaultindex');
+          chai.assert.equal(update.args[0][0].type, 'config');
+          chai.assert.equal(update.args[0][0].body.doc.defaultIndex, 'events-abcdef*');
 
           chai.assert.isOk(refresh.called);
           chai.assert.equal(refresh.args[0][0].index, '.kibana-testdefaultindex');

server/mt/kibana/defaultIndexPattern/_events_create.js

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2016 FUJITSU LIMITED
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import Promise from 'bluebird';
+
+export default (server, indexName, userObj) => {
+  server.log(['status', 'info', 'keystone'],
+    `Creating default events-index pattern for ${indexName}`);
+
+  const client = server.plugins.elasticsearch.client;
+  const pattern = server.config().get('monasca-kibana-plugin.eventsIndexPrefix')
+                  .replace('<project_id>', `${userObj.project.id}`) + '*';
+
+  return client.create({
+    index: indexName,
+    type : 'index-pattern',
+    body : {
+      title: pattern,
+      timeFieldName : server.config().get('monasca-kibana-plugin.defaultEventsTimeField')
+    },
+    id   : pattern
+  })
+  .then(() => {
+    return client.update({
+      index: indexName,
+      type: 'config',
+      id: server.config().get('pkg.version'),
+      body: {
+        doc: {
+          defaultIndex: pattern
+        }
+      }
+    });
+  })
+  .then(() => {
+    return client.indices.refresh({
+      index: indexName,
+      force: true
+    });
+  })
+  .then((response) => {
+    return Promise.resolve(response);
+  })
+  .catch((err)=> {
+    throw new Error(`Unable to setup events-index pattern, error is ${err}`);
+  });
+};

server/mt/kibana/defaultIndexPattern/_events_delete.js

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2016 FUJITSU LIMITED
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+export default (server, indexName, userObj) => {
+  const client = server.plugins.elasticsearch.client;
+  const options = {
+    index: indexName,
+    type : 'index-pattern',
+    id : server.config().get('monasca-kibana-plugin.eventsIndexPrefix')
+         .replace('<project_id>', `${userObj.project.id}`) + '*',
+  };
+  server.log(['status', 'info', 'keystone'],
+    `Deleting events-index pattern for ${indexName}...`);
+  return client
+  .delete(options)
+  .catch((err)=> {
+    throw new Error(`Deleting events-index pattern for ${indexName} failed, error is ${err}`);
+  });
+};

server/mt/kibana/defaultIndexPattern/_events_exists.js

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2016 FUJITSU LIMITED
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+export default (server, indexName, userObj) => {
+  const client = server.plugins.elasticsearch.client;
+  const options = {
+    index: indexName,
+    type : 'index-pattern',
+    id : server.config().get('monasca-kibana-plugin.eventsIndexPrefix')
+         .replace('<project_id>', `${userObj.project.id}`) + '*',
+  };
+  server.log(['status', 'debug', 'keystone'],
+    `Checking if default events-index pattern for ${indexName} exists...`);
+  return client
+  .exists(options)
+  .then((resp) => {
+    return resp;
+  })
+  .catch((err)=> {
+    throw new Error(`Getting events-index pattern for ${indexName} failed, error is ${err}`);
+  });
+};

server/mt/kibana/defaultIndexPattern/_logs_create.js

@@ -15,11 +15,12 @@
 import Promise from 'bluebird';
 
 export default (server, indexName, userObj) => {
-  server.log(['status', 'info', 'keystone'], `Creating default index pattern for ${indexName}`);
+  server.log(['status', 'info', 'keystone'],
+    `Creating default logs-index pattern for ${indexName}`);
 
   const client = server.plugins.elasticsearch.client;
-  const pattern = `${userObj.project.id}*`;
+  const pattern = server.config().get('monasca-kibana-plugin.logsIndexPrefix')
-
+                  .replace('<project_id>', `${userObj.project.id}`) + '*';
   return client.create({
     index: indexName,
     type : 'index-pattern',
@@ -51,6 +52,6 @@ export default (server, indexName, userObj) => {
     return Promise.resolve(response);
   })
   .catch((err)=> {
-    throw new Error(`Unable to setup index pattern, error is ${err}`);
+    throw new Error(`Unable to setup logs-index pattern, error is ${err}`);
   });
 };

server/mt/kibana/defaultIndexPattern/_logs_delete.js

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2016 FUJITSU LIMITED
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+export default (server, indexName, userObj) => {
+  const client = server.plugins.elasticsearch.client;
+  const options = {
+    index: indexName,
+    type : 'index-pattern',
+    id : server.config().get('monasca-kibana-plugin.logsIndexPrefix')
+         .replace('<project_id>', `${userObj.project.id}`) + '*',
+  };
+  server.log(['status', 'info', 'keystone'],
+    `Deleting logs-index pattern for ${indexName}...`);
+  return client
+  .delete(options)
+  .catch((err)=> {
+    throw new Error(`Deleting logs-index pattern for ${indexName} failed, error is ${err}`);
+  });
+};

server/mt/kibana/defaultIndexPattern/_logs_exists.js

@@ -12,24 +12,22 @@
  * the License.
  */
 
-export default (server, indexName) => {
+export default (server, indexName, userObj) => {
   const client = server.plugins.elasticsearch.client;
   const options = {
     index: indexName,
     type : 'index-pattern',
-    ignoreUnavailable: true
+    id : server.config().get('monasca-kibana-plugin.logsIndexPrefix')
+         .replace('<project_id>', `${userObj.project.id}`) + '*',
   };
-
   server.log(['status', 'debug', 'keystone'],
-    `Checking if default index pattern for ${indexName} exists...`);
+    `Checking if default logs-index pattern for ${indexName} exists...`);
-
   return client
-  .count(options)
+  .exists(options)
   .then((resp) => {
-    return resp.count > 0;
+    return resp;
   })
   .catch((err)=> {
-    throw new Error(`Getting index-pattern for ${indexName} failed, error is ${err}`);
+    throw new Error(`Getting logs-index pattern for ${indexName} failed, error is ${err}`);
   });
-
 };

server/mt/kibana/defaultIndexPattern/index.js

@@ -13,23 +13,71 @@
  */
 
 import Promise from 'bluebird';
-import defaultIndexExists from './_exists';
+import defaultLogsIndexExists from './_logs_exists';
-import createDefaultIndex from './_create';
+import defaultEventsIndexExists from './_events_exists';
+import createLogsDefaultIndex from './_logs_create';
+import createEventsDefaultIndex from './_events_create';
+import deleteLogsDefaultIndex from './_logs_delete';
+import deleteEventsDefaultIndex from './_events_delete';
 import kibanaIndex from '../kibanaIndex';
 
 export default (server, userObj) => {
   return () => {
     const indexName = kibanaIndex(server, userObj);
-    return defaultIndexExists(server, indexName)
+    return defaultLogsIndexExists(server, indexName, userObj)
-      .then((exists) => {
+      .then((logsExists) => {
-        if (!exists) {
+        if (server.config().get('monasca-kibana-plugin.logs')) {
-          server.log(['status', 'warning', 'keystone'],
+          server.log(['status', 'info', 'keystone'],
-            `Default index pattern for ${indexName} does not exist`);
+            `Default logs-index pattern is enabled in kibana config file`);
-          return createDefaultIndex(server, indexName, userObj);
+          if (!logsExists) {
+            server.log(['status', 'warning', 'keystone'],
+              `Default logs-index pattern for ${indexName} does not exist`);
+            return createLogsDefaultIndex(server, indexName, userObj);
+          } else {
+            server.log(['status', 'info', 'keystone'],
+              `Default logs-index pattern for ${indexName} already exists`);
+          }
+
+        } else {
+          server.log(['status', 'info', 'keystone'],
+            `Default logs-index pattern is disabled in kibana config file`);
+          if (logsExists) {
+            server.log(['status', 'warning', 'keystone'],
+              `Default logs-index pattern for ${indexName} exists, but it should not`);
+            return deleteLogsDefaultIndex(server, indexName, userObj);
+          } else {
+            server.log(['status', 'info', 'keystone'],
+              `Default logs-index pattern for ${indexName} does not exist`);
+          }
         }
-        server.log(['status', 'debug', 'keystone'],
+      })
-          `Default index pattern for ${indexName} already exists`);
+      .then(() => {
-        return Promise.resolve();
+        defaultEventsIndexExists(server, indexName, userObj)
+          .then((eventsExists) => {
+            if (server.config().get('monasca-kibana-plugin.events')) {
+              server.log(['status', 'info', 'keystone'],
+                `Default events-index pattern is enabled in kibana config file`);
+              if (!eventsExists) {
+                server.log(['status', 'warning', 'keystone'],
+                  `Default events-index pattern for ${indexName} does not exist`);
+                return createEventsDefaultIndex(server, indexName, userObj);
+              } else {
+                server.log(['status', 'info', 'keystone'],
+                  `Default events-index pattern for ${indexName} already exists`);
+              }
+            } else {
+              server.log(['status', 'info', 'keystone'],
+                `Default events-index pattern is disabled in kibana config file`);
+              if (eventsExists) {
+                server.log(['status', 'warning', 'keystone'],
+                  `Default events-index pattern for ${indexName} exists, but it should not`);
+                return deleteEventsDefaultIndex(server, indexName, userObj);
+              } else {
+                server.log(['status', 'info', 'keystone'],
+                  `Default events-index pattern for ${indexName} does not exist`);
+              }
+            }
+          });
       });
   };
 };

server/mt/routing/routes/paths.js

@@ -41,11 +41,20 @@ export default function (server, method, path) {
     let kibanaIndexRequest = false;
 
     let indexPos = url.findIndex((item) => item === defaultKibanaIndex);
+    let logsIndexPref = server.config().get('monasca-kibana-plugin.logsIndexPrefix');
+    let eventsIndexPref = server.config().get('monasca-kibana-plugin.eventsIndexPrefix');
+    logsIndexPref = logsIndexPref.replace('<project_id>', session[SESSION_USER_KEY].project.id);
+    eventsIndexPref = eventsIndexPref.replace('<project_id>', session[SESSION_USER_KEY].project.id);
+
+    server.log(['status', 'info', 'keystone'],
+      `Allowing only these Index-Prefix ${logsIndexPref}, ${eventsIndexPref}`);
+
     if (indexPos > -1) {
       url[indexPos] = kibanaIndex(server, session[SESSION_USER_KEY]);
       kibanaIndexRequest = true;
     } else if (url.length > logIndexPostionInUrl
-        && !url[logIndexPostionInUrl].startsWith(session[SESSION_USER_KEY].project.id)) {
+        && !(url[logIndexPostionInUrl].startsWith(logsIndexPref)
+        || url[logIndexPostionInUrl].startsWith(eventsIndexPref))) {
       return reply(Boom.unauthorized('User does not have access to this resource'));
     }