Improvements to scons build for beast.

* Common code extracted to Python directories.
* Read ~/.scons file for scons environment defaults.
* Override scons settings with shell environment variables.
* New "tags" for debug, nodebug, optimize, nooptimize builds.
* Universal platform detection.
* Default value of environment variables set through prefix dictionaries.
* Check for correct Boost value and fail otherwise.
* Extract git describe --tags into a preprocesor variable, -DTIP_BRANCH
* More colors - blue for unchanged defaults, green for changed defaults, red for error.
* Contain unit tests for non-obvious stuff.
* Check to see that boost libraries have been built.
* Right now, we accept both .dylib and .a versions but it'd be easy to enforce .a only.

python/beast/util/Boost.py

@@ -0,0 +1,61 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import re
+
+ROOT_ENV_VARIABLE = 'BOOST_ROOT'
+MINIMUM_VERSION = 1, 55, 0
+VERSION_FILE = 'boost', 'version.hpp'
+LIBRARY_PATH_SEGMENT = 'stage', 'lib'
+VERSION_MATCHER = re.compile(r'#define\s+BOOST_VERSION\s+(\d+)')
+
+CANT_OPEN_VERSION_FILE_ERROR = """Unable to open boost version file %s.
+You have set the environment variable BOOST_ROOT to be %s.
+Please check to make sure that this points to a valid installation of boost."""
+
+CANT_UNDERSTAND_VERSION_ERROR = (
+    "Didn't understand version string '%s' from file %s'")
+VERSION_TOO_OLD_ERROR = ('Your version of boost, %s, is older than the minimum '
+                         'required, %s.')
+
+def _text(major, minor, release):
+    return '%d.%02d.%02d' % (major, minor, release)
+
+def _raw_boost_path():
+    try:
+      path = os.environ[ROOT_ENV_VARIABLE]
+      if path:
+        return os.path.normpath(path)
+    except KeyError:
+        pass
+    raise KeyError('%s environment variable is not set.' % ROOT_ENV_VARIABLE)
+
+def _get_version_number(path):
+    version_file = os.path.join(path, *VERSION_FILE)
+    try:
+        with open(version_file) as f:
+            for line in f:
+                match = VERSION_MATCHER.match(line)
+                if match:
+                    version = match.group(1)
+                    try:
+                        return int(version)
+                    except ValueError:
+                        raise Exception(CANT_UNDERSTAND_VERSION_ERROR %
+                                        (version, version_file))
+    except IOError:
+        raise Exception(CANT_OPEN_VERSION_FILE_ERROR % (version_file, path))
+
+def _validate_version(v):
+    version = v // 100000, (v // 100) % 100, v % 100
+    if version < MINIMUM_VERSION:
+        raise Exception(VERSION_TOO_OLD_ERROR % (
+    _text(*version), _text(*MINIMUM_VERSION)))
+
+def _boost_path():
+    path = _raw_boost_path()
+    _validate_version(_get_version_number(path))
+    return path
+
+CPPPATH = _boost_path()
+LIBPATH = os.path.join(CPPPATH, *LIBRARY_PATH_SEGMENT)

python/beast/util/Dict.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+def compose(*dicts):
+    result = {}
+    for d in dicts:
+        result.update(**d)
+    return result
+
+def get_items_with_prefix(key, mapping):
+    """Get all elements from the mapping whose keys are a prefix of the given
+    key, sorted by increasing key length."""
+    for k, v in sorted(mapping.items()):
+        if key.startswith(k):
+            yield v
+
+def compose_prefix_dicts(key, mapping):
+    return compose(*get_items_with_prefix(key, mapping))

python/beast/util/Dict_test.py

@@ -0,0 +1,56 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from unittest import TestCase
+
+from beast.util import Dict
+
+DICT = {
+    '': {
+        'foo': 'foo-default',
+        'bar': 'bar-default',
+      },
+
+    'Darwin': {
+        'foo': 'foo-darwin',
+        'baz': 'baz-darwin',
+      },
+
+    'Darwin.10.8': {
+        'foo': 'foo-darwin-10.8',
+        'bing': 'bing-darwin-10.8',
+      },
+  }
+
+class test_Dict(TestCase):
+    def computeMapValue(self, config, key):
+        return Dict.compose(*Dict.get_items_with_prefix(config, DICT))[key]
+
+    def assertMapValue(self, config, key, result):
+        self.assertEquals(self.computeMapValue(config, key), result)
+
+    def testDefault1(self):
+        self.assertMapValue('', 'foo', 'foo-default')
+
+    def testDefault2(self):
+        self.assertMapValue('Darwin.10.8', 'bar', 'bar-default')
+
+    def testPrefix1(self):
+        self.assertMapValue('Darwin', 'foo', 'foo-darwin')
+
+    def testPrefix2(self):
+        self.assertMapValue('Darwin.10.8', 'foo', 'foo-darwin-10.8')
+
+    def testPrefix3(self):
+        self.assertMapValue('Darwin', 'baz', 'baz-darwin')
+
+    def testPrefix4(self):
+        self.assertMapValue('Darwin.10.8', 'bing', 'bing-darwin-10.8')
+
+    def testFailure1(self):
+        self.assertRaises(KeyError, self.computeMapValue, '', 'baz')
+
+    def testFailure2(self):
+        self.assertRaises(KeyError, self.computeMapValue, '', 'bing')
+
+    def testFailure2(self):
+        self.assertRaises(KeyError, self.computeMapValue, 'Darwin', 'bing')

python/beast/util/Execute.py

@@ -0,0 +1,14 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import subprocess
+
+from beast.util import String
+
+def execute(args, include_errors=True, **kwds):
+    """Execute a shell command and return the value.  If args is a string,
+    it's split on spaces - if some of your arguments contain spaces, args should
+    instead be a list of arguments."""
+    if String.is_string(args):
+        args = args.split()
+    stderr = subprocess.STDOUT if include_errors else None
+    return subprocess.check_output(args, stderr=stderr, **kwds)

python/beast/util/File.py

@@ -0,0 +1,42 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from beast.util import String
+
+import os
+
+LIBRARY_PATTERNS = 'lib%s.a', 'lib%s.dylib'
+
+def first_fields_after_prefix(filename, prefix):
+    with open(filename, 'r') as f:
+        return String.first_fields_after_prefix(prefix, f)
+
+def find_files_with_suffix(base, suffix):
+    for parent, _, files in os.walk(base):
+        for path in files:
+            path = os.path.join(parent, path)
+            if path.endswith(suffix):
+                yield os.path.normpath(path)
+
+def child_files(parent, files):
+    return [os.path.normpath(os.path.join(parent, f)) for f in files]
+
+def sibling_files(path, files):
+    return child_files(os.path.dirname(path), files)
+
+def replace_extension(file, ext):
+    return os.path.splitext(file)[0] + ext
+
+def validate_libraries(path, libraries):
+    bad = []
+    for lib in libraries:
+        found = False
+        for pat in LIBRARY_PATTERNS:
+            libfile = os.path.join(path, pat % lib)
+            if os.path.isfile(libfile):
+                found = True
+                break
+        if not found:
+            bad.append(libfile)
+    if bad:
+        libs = 'library' if len(bad) == 1 else 'libraries'
+        raise Exception('Missing %s: %s' % (libs, ', '.join(bad)))

python/beast/util/Git.py

@@ -0,0 +1,9 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+
+from beast.util import Execute
+from beast.util import String
+
+def describe(**kwds):
+    return String.single_line(Execute.execute('git describe --tags', **kwds))

python/beast/util/Iter.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+def first(condition, sequence):
+    for i in sequence:
+        result = condition(i)
+        if result:
+            return result

python/beast/util/String.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import functools
+
+from beast.util import Iter
+from beast.util.Terminal import warn
+
+def is_string(s):
+    """Is s a string? - in either Python 2.x or 3.x."""
+    return isinstance(s, (str, unicode))
+
+def stringify(item, joiner=''):
+    """If item is not a string, stringify its members and join them."""
+    try:
+        len(item)
+    except:
+        return str(item)
+    if not item or is_string(item):
+        return item or ''
+    else:
+        return joiner.join(str(i) for i in item)
+
+def single_line(line, report_errors=True, joiner='+'):
+    """Force a string to be a single line with no carriage returns, and report
+    a warning if there was more than one line."""
+    lines = line.strip().splitlines()
+    if report_errors and len(lines) > 1:
+      print('multiline result:', lines)
+    return joiner.join(lines)
+
+# Copied from
+# https://github.com/lerugray/pickett/blob/master/pickett/ParseScript.py
+def remove_comment(line):
+    """Remove trailing comments from one line."""
+    start = 0
+    while True:
+        loc = line.find('#', start)
+        if loc == -1:
+            return line.replace('\\#', '#')
+        elif not (loc and line[loc - 1] == '\\'):
+            return line[:loc].replace('\\#', '#')
+        start = loc + 1
+
+def remove_quotes(line, quote='"', print=print):
+    if not line.startswith(quote):
+        return line
+    if line.endswith(quote):
+        return line[1:-1]
+
+    warn('line started with %s but didn\'t end with one:' % quote, print)
+    print(line)
+    return line[1:]
+
+def fields_after_prefix(prefix, line):
+    line = line.strip()
+    return line.startswith(prefix) and line[len(prefix):].split()
+
+def first_fields_after_prefix(prefix, sequence):
+    condition = functools.partial(fields_after_prefix, prefix)
+    return Iter.first(condition, sequence) or []

python/beast/util/String_test.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from unittest import TestCase
+
+from beast.util import String
+from beast.util import Terminal
+
+Terminal.CAN_CHANGE_COLOR = False
+
+class String_test(TestCase):
+  def test_comments(self):
+    self.assertEqual(String.remove_comment(''), '')
+    self.assertEqual(String.remove_comment('#'), '')
+    self.assertEqual(String.remove_comment('# a comment'), '')
+    self.assertEqual(String.remove_comment('hello # a comment'), 'hello ')
+    self.assertEqual(String.remove_comment(
+      r'hello \# not a comment # a comment'),
+      'hello # not a comment ')
+
+  def test_remove_quotes(self):
+    errors = []
+    self.assertEqual(String.remove_quotes('hello', print=errors.append),
+                     'hello')
+    self.assertEqual(String.remove_quotes('"hello"', print=errors.append),
+                     'hello')
+    self.assertEqual(String.remove_quotes('hello"', print=errors.append),
+                     'hello"')
+    self.assertEqual(errors, [])
+
+  def test_remove_quotes_error(self):
+    errors = []
+    self.assertEqual(String.remove_quotes('"hello', print=errors.append),
+                     'hello')
+    self.assertEqual(errors,
+                     ['WARNING: line started with " but didn\'t end with one:',
+                      '"hello'])

python/beast/util/Terminal.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+
+from beast.platform.Platform import PLATFORM
+
+# See https://stackoverflow.com/questions/7445658/how-to-detect-if-the-console-does-support-ansi-escape-codes-in-python
+CAN_CHANGE_COLOR = (
+    hasattr(sys.stderr, "isatty")
+    and sys.stderr.isatty()
+    and not PLATFORM.startswith('Windows'))
+
+
+# See https://en.wikipedia.org/wiki/ANSI_escape_code
+RED = 91
+GREEN = 92
+BLUE = 94
+
+def add_mode(text, *modes):
+    if CAN_CHANGE_COLOR:
+        modes = ';'.join(str(m) for m in modes)
+        return '\033[%sm%s\033[0m' % (modes, text)
+    else:
+        return text
+
+def blue(text):
+    return add_mode(text, BLUE)
+
+def green(text):
+    return add_mode(text, GREEN)
+
+def red(text):
+    return add_mode(text, RED)
+
+def warn(text, print=print):
+    print('%s %s' % (red('WARNING:'), text))

python/beast/util/Tests.py

@@ -0,0 +1,31 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+
+from beast.util import File
+
+LIBS_PREFIX = '// LIBS:'
+MODS_PREFIX = '// MODULES:'
+
+def build_executable(env, path, main_program_file):
+    """Build a stand alone executable that runs
+       all the test suites in one source file."""
+    libs = File.first_fields_after_prefix(path, LIBS_PREFIX)
+    source_modules = File.first_fields_after_prefix(path, MODS_PREFIX)
+    source_modules = File.sibling_files(path, source_modules)
+
+    bin = os.path.basename(os.path.splitext(path)[0])
+    bin = os.path.join('bin', bin)
+
+    # All paths get normalized here, so we can use posix
+    # forward slashes for everything including on Windows
+    srcs = File.child_files('bin', [main_program_file, path] + source_modules)
+    objs = [File.replace_extension(f, '.o') for f in srcs]
+    if libs:
+        env.Append(LIBS=libs)  # DANGER: will append the file over and over.
+    env.Program(bin, srcs)
+
+def run_tests(env, main_program_file, root, suffix):
+    root = os.path.normpath(root)
+    for path in File.find_files_with_suffix(root, suffix):
+        build_executable(env, path, main_program_file)