Use cypress for visual testing

- Ditch Casper.js / PhantomCSS
- Set up Cypress and cypress-image-snapshot
- Implement visual tests covering most of the same ground as what I had before (and more), but with some caveats
- Some tests are flaky right now due to resource loading, and some have areas blotched out in order to not depend on resource loading
- TODO: set up continuous integration, add more tests, etc.

.gitignore

@@ -1,4 +1,10 @@
 
+# cypress-image-snapshot visual diffs
+__diff_output__
+
+# cypress-image-snapshot images of the whole cypress UI
+*(failed).snap.png
+
 # electron forge output
 out/
 

cypress.json

@@ -0,0 +1,4 @@
+{
+	"baseUrl": "http://localhost:11822",
+	"video": false
+}

cypress/integration/visual-tests.spec.js

@@ -0,0 +1,83 @@
+/// <reference types="Cypress" />
+
+context('visual tests', () => {
+	it('main screenshot', () => {
+		cy.visit('/');
+		cy.matchImageSnapshot();
+	});
+
+	it('brush selected', () => {
+		cy.get('.tool[title="Brush"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('select selected', () => {
+		cy.get('.tool[title="Select"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('magnifier selected', () => {
+		cy.get('.tool[title="Magnifier"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('airbrush selected', () => {
+		cy.get('.tool[title="Airbrush"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('eraser selected', () => {
+		cy.get('.tool[title="Eraser/Color Eraser"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('line selected', () => {
+		cy.get('.tool[title="Line"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+	it('rectangle selected', () => {
+		cy.get('.tool[title="Rectangle"]').click();
+		cy.get('.Tools-component').matchImageSnapshot();
+	});
+
+	it('image attributes window', () => {
+		cy.get('body').type('{ctrl}e');
+		cy.get('.window:visible').matchImageSnapshot();
+		cy.get('.window:visible .window-close-button').click();
+		cy.get('.window').should('not.be.visible');
+	});
+
+	it('flip and rotate window', () => {
+		// TODO: make menus more testable, with IDs
+		cy.get('.menus > .menu-container:nth-child(4) > .menu-button > .menu-hotkey').click();
+		cy.get('.menus > .menu-container:nth-child(4) > .menu-popup > table > tr:nth-child(1)').click();
+		cy.get('.window:visible').matchImageSnapshot();
+		cy.get('.window:visible .window-close-button').click();
+		cy.get('.window').should('not.be.visible');
+	});
+
+	it('stretch and skew window', () => {
+		// TODO: make menus more testable, with IDs
+		cy.get('.menus > .menu-container:nth-child(4) > .menu-button > .menu-hotkey').click();
+		cy.get('.menus > .menu-container:nth-child(4) > .menu-popup > table > tr:nth-child(2)').click();
+		// TODO: wait for images to load and include images?
+		cy.get('.window:visible').matchImageSnapshot({ blackout: ["img"] });
+		cy.get('.window:visible .window-close-button').click();
+		cy.get('.window').should('not.be.visible');
+	});
+
+	it('help window', () => {
+		// TODO: make menus more testable, with IDs
+		cy.get('.menus > .menu-container:nth-child(6) > .menu-button > .menu-hotkey').click();
+		cy.get('.menus > .menu-container:nth-child(6) > .menu-popup > table > tr:nth-child(1)').click();
+		cy.get('.window:visible .folder', {timeout: 10000}); // wait for sidebar contents to load
+		// TODO: wait for iframe to load
+		cy.get('.window:visible').matchImageSnapshot({ blackout: ["iframe"] });
+		cy.get('.window:visible .window-close-button').click();
+		cy.get('.window').should('not.be.visible');
+	});
+
+	it('about window', () => {
+		// TODO: make menus more testable, with IDs
+		cy.get('.menus > .menu-container:nth-child(6) > .menu-button > .menu-hotkey').click();
+		cy.get('.menus > .menu-container:nth-child(6) > .menu-popup > table > tr:nth-child(3)').click();
+		cy.get('.window:visible').matchImageSnapshot({ blackout: ["img"] });
+		cy.get('.window:visible .window-close-button').click();
+		cy.get('.window').should('not.be.visible');
+	});
+});

cypress/plugins/index.js

@@ -0,0 +1,11 @@
+// This function is called when a project is opened or re-opened (e.g. due to
+// the project's config changing)
+
+const {
+	addMatchImageSnapshotPlugin,
+} = require("cypress-image-snapshot/plugin");
+module.exports = (on, config) => {
+	// `on` is used to hook into various events Cypress emits
+	// `config` is the resolved Cypress config
+	addMatchImageSnapshotPlugin(on, config);
+};

cypress/support/commands.js

@@ -0,0 +1,14 @@
+import { addMatchImageSnapshotCommand } from 'cypress-image-snapshot/command';
+addMatchImageSnapshotCommand({
+	failureThreshold: 0.00,
+	failureThresholdType: 'percent',
+	customDiffConfig: { threshold: 0.0 },
+	capture: 'viewport',
+});
+Cypress.Commands.add("setResolution", (size) => {
+	if (Cypress._.isArray(size)) {
+		cy.viewport(size[0], size[1]);
+	} else {
+		cy.viewport(size);
+	}
+})

cypress/support/index.js

@@ -0,0 +1,7 @@
+// ***********************************************************
+// This support/index.js is processed and
+// loaded automatically before your test files.
+//
+// https://on.cypress.io/configuration
+
+import './commands'

package.json

@@ -64,14 +64,15 @@
     "@electron-forge/maker-squirrel": "6.0.0-beta.45",
     "@electron-forge/maker-zip": "6.0.0-beta.45",
     "cross-spawn": "^7.0.0",
+    "cypress": "3.6.0",
+    "cypress-image-snapshot": "3.1.1",
     "devtron": "^1.4.0",
     "electron": "6.0.10",
     "electron-debug": "^3.0.1",
     "eslint": "6.4.0",
     "live-server": "^1.2.1",
-    "npm-run-all": "4.1.5",
-    "phantomcss": "^1.6.0",
-    "serve": "11.2.0"
+    "serve": "11.2.0",
+    "start-server-and-test": "1.10.6"
   },
   "scripts": {
     "start": "electron-forge start",
@@ -80,14 +81,13 @@
     "make": "electron-forge make",
     "publish": "electron-forge publish",
     "lint": "eslint src/",
-    "dev": "live-server",
-    "test": "run-p test:server test:runner",
-    "test-verbose": "run-p test:server test:runner:verbose",
-    "test:server": "serve --listen 11822 --no-clipboard",
-    "test:runner": "casperjs test test.js --verbose --log-level=debug",
-    "test:runner:verbose": "casperjs test test.js --verbose --log-level=debug",
-    "test:runner:firefox": "casperjs test test.js --engine=slimerjs --verbose --log-level=debug",
-    "test:runner:firefox:verbose": "casperjs test test.js --engine=slimerjs --verbose --log-level=debug"
+    "dev": "live-server --ignorePattern=\"(node_modules|cypress|out)[/\\\\]|package\\.json|cypress\\.json\"",
+    "test:start-server": "live-server --port=11822 --no-browser --ignorePattern=\"(node_modules|cypress|out)[/\\\\]|package\\.json|cypress\\.json\"",
+    "cy:open": "cypress open",
+    "cy:run": "cypress run",
+    "cy:accept": "cypress run --env updateSnapshots=true",
+    "test": "start-server-and-test test:start-server http://localhost:11822 cy:run",
+    "accept": "start-server-and-test test:start-server http://localhost:11822 cy:accept"
   },
   "repository": {
     "type": "git",

test.js

@@ -1,112 +0,0 @@
-
-var fs = require('fs');
-var phantomcss = require('phantomcss');
-
-casper.test.begin('jspaint visual tests', function(test){
-	
-	phantomcss.init({
-		rebase: casper.cli.get('rebase'),
-		// SlimerJS needs explicit knowledge of this Casper, and lots of absolute paths
-		casper: casper,
-		libraryRoot: fs.absolute(fs.workingDirectory + '/node_modules/phantomcss'), //module._getFilename('phantomcss'), //require.resolve('phantomcss'),
-		screenshotRoot: fs.absolute(fs.workingDirectory + '/screenshots'),
-		failedComparisonsRoot: fs.absolute(fs.workingDirectory + '/screenshots/failures'),
-		addLabelToFailedImage: false,
-		/*
-		fileNameGetter: function overide_file_naming(){},
-		onPass: function pass_callback(){},
-		onFail: function fail_callback(){},
-		onTimeout: function timeout_callback(){},
-		onComplete: function complete_callback(){},
-		hideElements: '#thing.selector',
-		addLabelToFailedImage: true,
-		outputSettings: {
-			errorColor: {
-				red: 255,
-				green: 255,
-				blue: 0
-			},
-			errorType: 'movement',
-			transparency: 0.3
-		}*/
-	});
-	
-	casper.on('remote.message', function(msg){
-		this.echo("[page] " + msg);
-	});
-	
-	casper.on('error', function(err){
-		this.die("PhantomJS has errored: " + err);
-	});
-	
-	casper.on('resource.error', function(err){
-		casper.log('Resource load error: ' + err, 'warning');
-	});
-	/*
-		The test scenario
-	*/
-	
-	casper.start('http://localhost:11822');
-	
-	casper.viewport(1024, 768);
-	
-	casper.then(function(){
-		phantomcss.screenshot('.jspaint', 'app screen initial');
-		phantomcss.screenshot('.menus', 'menu bar initial');
-		phantomcss.screenshot('.Tools-component', 'toolbox initial');
-		phantomcss.screenshot('.Colors-component', 'color box initial');
-	});
-	
-	var screenshot_and_close_window = function(screenshot_name){
-		// var window_title = "Attributes";
-		// var selector = {
-		// 	type: "xpath",
-		// 	path: "//div[contains(concat(' ', normalize-space(@class), ' '), ' window ')][//span[contains(concat(' ', normalize-space(@class), ' '), ' window-title ')][.='" + window_title + "']]"
-		// };
-		var selector = ".window:not([style*='display: none'])";
-		
-		casper.then(function(){
-			casper.waitUntilVisible(selector,
-				function success(){
-					phantomcss.screenshot(selector, screenshot_name);
-				}
-			);
-		});
-		// casper.thenEvaluate(function(selector){
-		// 	$(selector).find(".window-close-button").trigger("click");
-		// }, selector);
-		casper.then(function close_the_window(){
-			casper.click(selector + " .window-close-button");
-		});
-	};
-	
-	casper.thenEvaluate(function(){
-		image_attributes();
-	});
-	screenshot_and_close_window('attributes window');
-	
-	casper.thenEvaluate(function(){
-		image_flip_and_rotate();
-	});
-	screenshot_and_close_window('flip and rotate window');
-	
-	casper.thenEvaluate(function(){
-		image_stretch_and_skew();
-	});
-	screenshot_and_close_window('stretch and skew window');
-	
-	casper.thenEvaluate(function(){
-		show_help();
-	});
-	screenshot_and_close_window('help window');
-	
-	casper.then(function now_check_the_screenshots(){
-		phantomcss.compareAll();
-	});
-	
-	casper.run(function(){
-		console.log('\nTHE END.');
-		// phantomcss.getExitStatus() // pass or fail?
-		casper.test.done();
-	});
-});