improve the test runner

test/api/getLedger/index.ts

@@ -1,5 +1,5 @@
 import assert from 'assert-diff'
-import { assertResultMatch, TestSuite } from '../utils'
+import { assertResultMatch, TestSuite } from '../../utils'
 import responses from '../../fixtures/responses'
 const { getLedger: RESPONSE_FIXTURES } = responses
 

test/api/getPaths/index.ts

@@ -1,5 +1,5 @@
 import assert from 'assert-diff'
-import { assertResultMatch, assertRejects, TestSuite } from '../utils'
+import { assertResultMatch, assertRejects, TestSuite } from '../../utils'
 import responses from '../../fixtures/responses'
 import requests from '../../fixtures/requests'
 import addresses from '../../fixtures/addresses.json'

test/api/prepareSettings/index.ts

@@ -1,7 +1,7 @@
 import assert from 'assert-diff'
 import requests from '../../fixtures/requests'
 import responses from '../../fixtures/responses'
-import { assertResultMatch, TestSuite } from '../utils'
+import { assertResultMatch, TestSuite } from '../../utils'
 const instructionsWithMaxLedgerVersionOffset = { maxLedgerVersionOffset: 100 }
 
 /**

test/api/xrpToDrops/index.ts

@@ -1,6 +1,6 @@
 import assert from 'assert-diff'
 import BigNumber from 'bignumber.js'
-import { TestSuite } from '../utils'
+import { TestSuite } from '../../utils'
 
 /**
  * Every test suite exports their tests in the default object.

test/ripple-api-test.ts

@@ -1,7 +1,7 @@
-import setupAPI from '../setup-api'
+import setupAPI from './setup-api'
 import { RippleAPI } from 'ripple-api'
-import addresses from '../fixtures/addresses.json'
-import { getAllPublicMethods, loadTestSuite } from './utils'
+import addresses from './fixtures/addresses.json'
+import { getAllPublicMethods, loadTestSuites } from './api/utils'
 
 /**
  * RippleAPI Test Runner
@@ -27,27 +27,17 @@ describe('RippleAPI [Test Runner]', function() {
 
   // Collect all the tests:
   const allPublicMethods = getAllPublicMethods(new RippleAPI())
-  const allTestSuites = allPublicMethods.map(loadTestSuite)
-
-  // TODO: Once migration is complete, remove this filter so that missing tests are reported.
-  const filteredTestSuites = allTestSuites.filter(({ isMissing }) => !isMissing)
+  const allTestSuites = loadTestSuites()
 
   // Run all the tests:
-  for (const { name: suiteName, tests, isMissing } of filteredTestSuites) {
-    describe(suiteName, () => {
-      // Check that tests exist as expected, and report any errors if they don't.
-      it('has valid test suite', () => {
-        if (isMissing) {
-          throw new Error(
-            `Test file not found! Create file "test/api/${suiteName}/index.ts".`
-          )
-        }
-        if (tests.length === 0) {
-          throw new Error(`No tests found! Is your test file set up properly?`)
-        }
-      })
+  for (const {
+    name: methodName,
+    tests,
+    config
+  } of allTestSuites) {
+    describe(`api.${methodName}`, () => {
       // Run each test with the original-style address.
-      describe(`1. Original Address Style`, () => {
+      describe(`[Original Address]`, () => {
         for (const [testName, fn] of tests) {
           it(testName, function() {
             return fn(this.api, addresses.ACCOUNT)
@@ -55,13 +45,27 @@ describe('RippleAPI [Test Runner]', function() {
         }
       })
       // Run each test with the newer, x-address style.
-      describe(`2. X-Address Style`, () => {
-        for (const [testName, fn] of tests) {
-          it(testName, function() {
-            return fn(this.api, addresses.ACCOUNT_X)
-          })
-        }
-      })
+      if (!config.skipXAddress) {
+        describe(`[X-Address]`, () => {
+          for (const [testName, fn] of tests) {
+            it(testName, function() {
+              return fn(this.api, addresses.ACCOUNT_X)
+            })
+          }
+        })
+      }
     })
   }
+
+  const allTestedMethods = new Set(allTestSuites.map(s => s.name));
+  for (const methodName of allPublicMethods) {
+    if (!allTestedMethods.has(methodName)) {
+      // TODO: Once migration is complete, remove this filter so that missing tests are reported.
+      it.skip(`${methodName} - no test suite found`, () => {
+          throw new Error(
+            `Test file not found! Create file "test/api/${methodName}/index.ts".`
+          )
+      })
+    }
+  }
 })

test/utils.ts

@@ -1,4 +1,6 @@
+import net from 'net'
 import _ from 'lodash'
+import fs from 'fs'
 import { RippleAPI } from 'ripple-api'
 import assert from 'assert-diff'
 const { schemaValidator } = RippleAPI._PRIVATE
@@ -29,7 +31,10 @@ export interface TestSuite {
 interface LoadedTestSuite {
   name: string
   tests: [string, TestFn][]
-  isMissing: boolean
+  config: {
+    /** Set to true to skip re-running tests with an X-Address. */
+    skipXAddress?: boolean
+  }
 }
 
 /**
@@ -67,37 +72,71 @@ export function assertResultMatch(
 }
 
 /**
- * Check that the promise rejects with an expected error instance.
+ * Check that the promise rejects with an expected error.
  */
 export async function assertRejects(
   promise: PromiseLike<any>,
-  instanceOf: any
+  instanceOf: any,
+  message?: string | RegExp
 ) {
   try {
     await promise
     assert(false, 'Expected an error to be thrown')
   } catch (error) {
-    assert(error instanceof instanceOf)
+    assert(error instanceof instanceOf, error.message)
+    if (typeof message === 'string') {
+      assert.strictEqual(error.message, message)
+    } else if (message instanceof RegExp) {
+      assert(message.test(error.message))
+    }
   }
 }
 
+// using a free port instead of a constant port enables parallelization
+export function getFreePort() {
+  return new Promise((resolve, reject) => {
+    const server = net.createServer()
+    let port
+    server.on('listening', function() {
+      port = (server.address() as any).port
+      server.close()
+    })
+    server.on('close', function() {
+      resolve(port)
+    })
+    server.on('error', function(error) {
+      reject(error)
+    })
+    server.listen(0)
+  })
+}
+
 export function getAllPublicMethods(api: RippleAPI) {
-  return Object.keys(api).filter(key => !key.startsWith('_'))
+  return Array.from(
+    new Set([
+      ...Object.getOwnPropertyNames(api),
+      ...Object.getOwnPropertyNames(RippleAPI.prototype)
+    ])
+  ).filter(key => !key.startsWith('_'))
 }
 
-export function loadTestSuite(methodName: string): LoadedTestSuite | null {
-  try {
-    const testSuite = require(`./${methodName}`)
-    return {
-      isMissing: false,
-      name: methodName,
-      tests: Object.entries(testSuite.default || {}),
-    }
-  } catch (err) {
-    return {
-      isMissing: true,
-      name: methodName,
-      tests: [],
-    }
-  }
+export function loadTestSuites(): LoadedTestSuite[] {
+  const allTests = fs.readdirSync(__dirname, { encoding: 'utf8' })
+  return allTests
+    .map(methodName => {
+      if (
+        methodName.startsWith('index') ||
+        methodName.startsWith('utils') ||
+        methodName.startsWith('.DS_Store')
+      ) {
+        return null
+      }
+      const testSuite = require(`./${methodName}`)
+      return {
+        name: methodName,
+        config: testSuite.config || {},
+        tests: Object.entries(testSuite.default || {})
+      } as LoadedTestSuite
+    })
+    .filter(Boolean)
 }