[Reporting/Test] move functional tests to apps

x-pack/scripts/functional_tests.js

@@ -14,7 +14,6 @@ const alwaysImportedTests = [
 ];
 const onlyNotInCoverageTests = [
   require.resolve('../test/reporting/configs/chromium_api.js'),
-  require.resolve('../test/reporting/configs/chromium_functional.js'),
   require.resolve('../test/reporting/configs/generate_api.js'),
   require.resolve('../test/api_integration/config_security_basic.js'),
   require.resolve('../test/api_integration/config.js'),

x-pack/test/functional/apps/dashboard/index.ts

@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/dashboard/reporting/README.md

@@ -0,0 +1,17 @@
+## The Reporting Tests
+
+### Reporting baseline snapshots
+
+The functional version of the reporting tests create a few pdf reports and do a snapshot comparison against a couple baselines.  The baseline images are stored in `./functional/reports/baseline`.
+
+#### Updating the baselines
+
+Every now and then visual changes will be made that will require the snapshots to be updated.  This is how you go about updating it.  I will discuss generating snapshots from chromium since that is the way of the future.
+
+1. Load the ES Archive containing the dashboard and data
+  ```
+  node scripts/es_archiver load reporting/ecommerce --config=x-pack/test/reporting/configs/chromium_functional.js
+  node scripts/es_archiver load reporting/ecommerce_kibana --config=x-pack/test/reporting/configs/chromium_functional.js
+  ```
+2. Generate a report of the archived E-commerce dashboard
+3. Download the report and save it in the `reports/baseline` folder.

x-pack/test/functional/apps/dashboard/reporting/index.js

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { checkIfPngsMatch } from './lib/compare_pngs';
+
+const writeFileAsync = promisify(fs.writeFile);
+const mkdirAsync = promisify(fs.mkdir);
+
+const REPORTS_FOLDER = path.resolve(__dirname, 'reports');
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const config = getService('config');
+  const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']);
+
+  describe('Reporting', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PDF Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Print Layout', () => {
+      it('downloads a PDF file', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
+        // function is taking about 15 seconds per comparison in jenkins.
+        this.timeout(300000);
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.checkUsePrintLayout();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+
+    describe('Print PNG button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PNG Dash');
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Preserve Layout', () => {
+      it('matches baseline report', async function() {
+        const writeSessionReport = async (name, rawPdf, reportExt) => {
+          const sessionDirectory = path.resolve(REPORTS_FOLDER, 'session');
+          await mkdirAsync(sessionDirectory, { recursive: true });
+          const sessionReportPath = path.resolve(sessionDirectory, `${name}.${reportExt}`);
+          await writeFileAsync(sessionReportPath, rawPdf);
+          return sessionReportPath;
+        };
+        const getBaselineReportPath = (fileName, reportExt) => {
+          const baselineFolder = path.resolve(REPORTS_FOLDER, 'baseline');
+          const fullPath = path.resolve(baselineFolder, `${fileName}.${reportExt}`);
+          log.debug(`getBaselineReportPath (${fullPath})`);
+          return fullPath;
+        };
+
+        this.timeout(300000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPngReportingPanel();
+        await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 });
+        await PageObjects.reporting.clickGenerateReportButton();
+        await PageObjects.reporting.removeForceSharedItemsContainerSize();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const reportData = await PageObjects.reporting.getRawPdfReportData(url);
+        const reportFileName = 'dashboard_preserve_layout';
+        const sessionReportPath = await writeSessionReport(reportFileName, reportData, 'png');
+        const percentSimilar = await checkIfPngsMatch(
+          sessionReportPath,
+          getBaselineReportPath(reportFileName, 'png'),
+          config.get('screenshots.directory'),
+          log
+        );
+
+        expect(percentSimilar).to.be.lessThan(0.1);
+      });
+    });
+  });
+}

x-pack/test/reporting/functional/lib/compare_pngs.js → x-pack/test/functional/apps/dashboard/reporting/lib/compare_pngs.js

@@ -7,7 +7,7 @@
 import path from 'path';
 import fs from 'fs';
 import { promisify } from 'bluebird';
-import { comparePngs } from '../../../../../test/functional/services/lib/compare_pngs';
+import { comparePngs } from '../../../../../../../test/functional/services/lib/compare_pngs';
 
 const mkdirAsync = promisify(fs.mkdir);
 

x-pack/test/functional/apps/discover/index.ts

@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/discover/reporting.js

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const PageObjects = getPageObjects(['reporting', 'common', 'discover']);
+  const filterBar = getService('filterBar');
+
+  describe('Discover', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+    });
+
+    describe('Generate CSV button', () => {
+      beforeEach(() => PageObjects.common.navigateToApp('discover'));
+
+      it('is not available if new', async () => {
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('becomes available/not available when a saved search is created, changed and saved again', async () => {
+        // create new search, csv export is not available
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+        // add filter, csv export is not available
+        await filterBar.addFilter('currency', 'is', 'EUR');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search again, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('generates a report with data', async () => {
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+
+      it('generates a report with no data', async () => {
+        await PageObjects.reporting.setTimepickerInNoDataRange();
+        await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+    });
+  });
+}

x-pack/test/functional/apps/visualize/index.ts

@@ -15,5 +15,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./hybrid_visualization'));
     loadTestFile(require.resolve('./precalculated_histogram'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/visualize/reporting.js

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const PageObjects = getPageObjects([
+    'reporting',
+    'common',
+    'dashboard',
+    'visualize',
+    'visEditor',
+  ]);
+
+  describe('Visualize', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToUrl('visualize', 'new');
+        await PageObjects.visualize.clickAreaChart();
+        await PageObjects.visualize.clickNewSearch('ecommerce');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.visEditor.clickBucket('X-axis');
+        await PageObjects.visEditor.selectAggregation('Date Histogram');
+        await PageObjects.visEditor.clickGo();
+        await PageObjects.visualize.saveVisualization('my viz');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('downloaded PDF has OK status', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
+        // function is taking about 15 seconds per comparison in jenkins.
+        this.timeout(180000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+  });
+}