Merge pull request #2 from hqjenny/configSystem

First draft of configuration system. Not tested at all yet.

.gitignore

@@ -1,2 +1,3 @@
 *.o
 *.a
+cf-config.yaml

deploy/.gitignore

@@ -0,0 +1 @@
+__pycache__

deploy/centrifuge.py

@@ -1,24 +1,21 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # PYTHON_ARGCOMPLETE_OK
 
-# REQUIRES PYTHON2, because fabric requires python2
-
 from __future__ import with_statement, print_function
 import sys
 import os
-import signal
 import argparse
 from time import sleep, strftime, gmtime
 import logging
 import random
 import string  
 import argcomplete
-
+import pathlib
 
 # centrifuge 
-from parseconfig import AccelConfig
+import pkg.util as util 
+import pkg.buildaccel as buildaccel
 
-from util.streamlogger import StreamLogger
 from os.path import dirname as up
 
 def construct_centrifuge_argparser():
@@ -33,61 +30,22 @@ def construct_centrifuge_argparser():
                             'generate_sw',
                             'clean_sw',
                             ])
-    parser.add_argument('-c', '--accelconfigfile', type=str,
+    parser.add_argument('-c', '--accelconfigfile', type=pathlib.Path,
                         help='Path to accelerator SoC config JSON file. Defaults to accel.json',
                         default='accel.json'
                         )
     argcomplete.autocomplete(parser)
     return parser.parse_args()
 
-def main(args):
-    """ Main function for FireSim manager. """
-
-    # load accel.json 
-    accel_config = AccelConfig(args.accelconfigfile, chipyard_dir, centrifuge_dir)
-    
-    # print the info
-    accel_config.info()
-
-    # tasks that have a special config/dispatch setup
-    if args.task == 'generate_hw':
-       pass 
-
-if __name__ == '__main__':
-    # set the program root directory rdir to wherever chipyard is located
-    # this lets you run centrifuge from anywhere, not necessarily centrifuge/deploy/
-    abspath = os.path.abspath(__file__)
-    dname = os.path.dirname(abspath)
-    os.chdir(dname)
-
-    global centrifuge_dir
-    centrifuge_dir = up(up(abspath))
-    
-    global chipyard_dir
-    if os.environ.get('RDIR') is None:
-        chipyard_dir = up(up(up(up(abspath))))
-        os.environ['RDIR'] = chipyard_dir
-    else:
-        chipyard_dir = os.environ.get('RDIR') 
-
-    args = construct_centrifuge_argparser()
-
-    # logging setup
-    def logfilename():
-        """ Construct a unique log file name from: date + 16 char random. """
-        timeline = strftime("%Y-%m-%d--%H-%M-%S", gmtime())
-        randname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
-        return timeline + "-" + args.task + "-" + randname + ".log"
-
+def initLogging():
+    """Set up logging for this run. Assumes that util.initConfig() has been called already."""
     rootLogger = logging.getLogger()
     rootLogger.setLevel(logging.NOTSET) # capture everything
 
     # log to file
-    full_log_filename = "logs/" + logfilename()
+    full_log_filename = util.getOpt("log-dir") / (util.getOpt("run-name") + ".log")
     fileHandler = logging.FileHandler(full_log_filename)
     # formatting for log to file
-    # TODO: filehandler should be handler 0 (firesim_topology_with_passes expects this
-    # to get the filename) - handle this more appropriately later
     logFormatter = logging.Formatter("%(asctime)s [%(funcName)-12.12s] [%(levelname)-5.5s]  %(message)s")
     fileHandler.setFormatter(logFormatter)
     fileHandler.setLevel(logging.NOTSET) # log everything to file
@@ -97,6 +55,33 @@ if __name__ == '__main__':
     consoleHandler = logging.StreamHandler(stream=sys.stdout)
     consoleHandler.setLevel(logging.INFO) # show only INFO and greater in console
     rootLogger.addHandler(consoleHandler)
+    return rootLogger
+
+def main(args):
+    """ Main function for FireSim manager. """
+
+    # load accel.json 
+    accel_config = util.AccelConfig(args.accelconfigfile, util.getOpt('chipyard-dir'), util.getOpt('cf-dir'))
+
+    # print the info
+    accel_config.info()
+
+    # tasks that have a special config/dispatch setup
+    if args.task == 'generate_hw':
+        buildaccel.generate_hw(accel_config)
+    else:
+        print("Command: " + str(args.task) + " not yet implemented")
+
+if __name__ == '__main__':
+
+    args = construct_centrifuge_argparser()
+
+    # Make all paths absolute as early as possible
+    args.accelconfigfile = args.accelconfigfile.resolve()
+
+    ctx = util.initConfig()
+    ctx.setRunName(args.accelconfigfile, args.task)
+    rootLogger = initLogging()
 
     exitcode = 0
     try:
@@ -106,5 +91,5 @@ if __name__ == '__main__':
         rootLogger.exception("Fatal error.")
         exitcode = 1
     finally:
-        rootLogger.info("""The full log of this run is:\n{basedir}/{fulllog}""".format(basedir=dname, fulllog=full_log_filename))
+        rootLogger.info("""The full log of this run is:\n{logdir}/{runname}.log""".format(logdir=util.getOpt('log-dir'), runname=util.getOpt('run-name')))
         exit(exitcode)

deploy/logs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

deploy/pkg/buildaccel/__init__.py

@@ -0,0 +1,2 @@
+from .generate_hw import *
+from .run_hls import *

deploy/pkg/buildaccel/generate_hw.py

@@ -7,10 +7,10 @@ import json
 import os
 import shutil
 import errno    
+from .. import util
 
 rootLogger = logging.getLogger()
 
-
 def mkdir_p(path):
     try:
         os.makedirs(path)
@@ -45,11 +45,9 @@ def init_accel(accel_conf):
         init_proj_dir(accel)
         cp_src(accel)
 
-
 def generate_hw(accel_conf):
     """Generate hardware SoC """
 
+    print("gernerate_hw not implemented. Called with config: ", accel_conf)
     # init project repos
-    init_accel(accel_conf) 
-
-
+    # init_accel(accel_conf) 

deploy/pkg/util/__init__.py

@@ -0,0 +1,7 @@
+"""General-purpose centrifuge utilities"""
+
+# This flattens the util package a bit so we can put things in different files
+# if we want
+from .streamlogger import *
+from .config import *
+from .parseconfig import *

deploy/pkg/util/config.py

@@ -0,0 +1,251 @@
+import pathlib
+import collections
+import yaml
+import re
+import os
+import time
+import random
+import string
+
+# Global configuration (cfCtx set by initialize())
+ctx = None
+
+# These represent all available user-defined options (those set by the
+# environment or config files). See default-config.yaml or the documentation
+# for the meaning of these options.
+userOpts = [
+        'chipyard-dir',
+        'log-dir'
+    ]
+
+# These represent all available derived options (constants and those generated
+# from userOpts, but not directly settable by users)
+derivedOpts = [
+        'cf-dir',           # top of centrifuge directory tree
+        'support-dir',      # Various supporting non-code files (e.g. configs and templates)
+        'genhw-dir',        # where to put generated hardware
+        'run-name'          # A unique name for a single invocation of this command
+    ]
+
+# These options represent userOpts and derivedOpts that should be pathlib paths.
+pathOpts = [
+    'chipyard-dir',
+    'log-dir',
+    'cf-dir',
+    'genhw-dir'
+]
+
+class ConfigurationError(Exception):
+    """Error representing a generic problem with configuration"""
+    def __init__(self, cause):
+        self.cause = cause
+
+    def __str__(self):
+        return "Configuration Error: " + self.cause
+
+class ConfigurationOptionError(ConfigurationError):
+    """Error representing a problem with a specific configuration option."""
+    def __init__(self, opt, cause):
+        self.opt = opt
+        self.cause = cause
+
+    def __str__(self):
+        return "Error with configuration option '" + self.opt + "': " + str(self.cause)
+        
+class ConfigurationFileError(ConfigurationError):
+    """Error representing issues with loading the configuration"""
+    def __init__(self, missingFile, cause):
+        self.missingFile = missingFile
+        self.cause = cause
+
+    def __str__(self):
+        return "Failed to load configuration file: " + str(self.missingFile) + "\n" + \
+                str(self.cause)
+
+def cleanPaths(opts, baseDir=pathlib.Path('.')):
+    """Clean all user-defined paths in an options dictionary by converting them
+    to resolved, absolute, pathlib.Path's. Paths will be interpreted as
+    relative to baseDir."""
+
+    for opt in pathOpts:
+        if opt in opts and opts[opt] is not None:
+            try:
+                path = (baseDir / pathlib.Path(opts[opt])).resolve(strict=True)
+                opts[opt] = path
+            except Exception as e:
+                raise ConfigurationOptionError(opt, "Invalid path: " + str(e))
+
+
+class cfCtx(collections.MutableMapping):
+    """Global Centrifuge context (configuration)."""
+    
+    # Actual internal storage for all options
+    opts = {}
+
+    def __init__(self):
+        """On init, we search for and load all sources of options.
+
+        The order in which options are added here is the order of precidence.
+        
+        Attributes:
+            opts: Dictonary containing all configuration options (static values
+                set by the user or statically derived from those). Option
+                values are documented in the package variables 'derivedOpts' and
+                'userOpts'
+        """
+
+        # These are set early to help with config file search-paths
+        self['cf-dir'] = pathlib.Path(__file__).parent.parent.parent.parent.resolve()
+        self['support-dir'] = self['cf-dir'] / 'deploy' / 'support'
+
+        # This is an exhaustive list of defaults, it always exists and can be
+        # overwritten by other user-defined configs
+        defaultCfg = self['support-dir'] / 'default-config.yaml'
+        self.addPath(defaultCfg)
+        
+        # These are mutually-exlusive search paths (only one will be loaded)
+        cfgSources = [
+            # pwd
+            pathlib.Path('cf-config.yaml'),
+            self['cf-dir'] / 'cf-config.yaml'
+        ]
+        for src in cfgSources:
+            if src.exists():
+                self.addPath(src)
+                break
+
+        self.addEnv()
+
+        # We should have all user-defined options now
+        missingOpts = set(userOpts) - set(self.opts)
+        if len(missingOpts) != 0:
+            raise ConfigurationError("Missing required options: " + str(missingOpts))
+
+        self.deriveOpts()
+
+        # It would be a marshal bug if any of these options are missing
+        missingDOpts = set(derivedOpts) - set(self.opts)
+        if len(missingDOpts) != 0:
+            raise RuntimeError("Internal error: Missing derived options or constants: " + str(missingDOpts))
+
+    def add(self, newOpts):
+        """Add options to this configuration, opts will override any
+        conflicting options.
+        
+        newOpts: dictionary containing new options to add"""
+        
+        self.opts = dict(self.opts, **newOpts)
+        
+    def addPath(self, path):
+        """Add the yaml file at path to the config."""
+
+        try:
+            with open(path, 'r') as newF:
+                newCfg = yaml.full_load(newF)
+        except Exception as e:
+            raise ConfigurationFileError(path, e)
+
+        cleanPaths(newCfg, baseDir=path.parent)
+        self.add(newCfg)
+
+    def addEnv(self):
+        """Find all options in the environment and load them.
+        
+        Environment options take the form CF_OPT where "OPT" will be
+        converted as follows:
+            1) convert to lower-case
+            2) all underscores will be replaced with dashes
+
+        For example CF_GENHW_DIR=../special/generators would add a ('genhw-dir'
+        : '../special/generators') option to the config."""
+
+        reOpt = re.compile("^CF_(\S+)")
+        envCfg = {}
+        for opt,val in os.environ.items():
+            match = reOpt.match(opt)
+            if match:
+                optName = match.group(1).lower().replace('_', '-')
+                envCfg[optName] = val
+
+        cleanPaths(envCfg)
+        self.add(envCfg)
+
+    def deriveOpts(self):
+        """Update or initialize all derived options. This assumes all
+        user-defined options have been set already. See the 'derivedOpts' list
+        above for documentation of these options."""
+        self['genhw-dir'] = self['chipyard-dir'] / 'generators' 
+        self['run-name'] = ""
+
+    def setRunName(self, configPath, operation):
+        """Helper function for formatting a  unique run name. You are free to
+        set the 'run-name' option directly if you don't need the help.
+
+        Args:
+            configPath (pathlike): Config file used for this run
+            operation (str): The operation being performed on this run (e.g. 'build') 
+        """
+
+        if configPath:
+            configName = pathlib.Path(configPath).stem
+        else:
+            configName = ''
+
+        timeline = time.strftime("%Y-%m-%d--%H-%M-%S", time.gmtime())
+        randname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
+
+        runName = configName + \
+                "-" + operation + \
+                "-" + timeline + \
+                "-" +  randname
+
+        self['run-name'] = runName
+
+    # The following methods are needed by MutableMapping
+    def __getitem__(self, key):
+        if key not in self.opts:
+            raise ConfigurationOptionError(key, 'No such option')
+
+        return self.opts[key]
+
+    def __setitem__(self, key, value):
+        self.opts[key] = value
+
+    def __delitem__(self, key):
+        del self.opts[key]
+
+    def __iter__(self):
+        return iter(self.opts)
+
+    def __len__(self):
+        return len(self.opts)
+
+    def __str__(self):
+        return pprint.pformat(self.opts)
+
+    def __repr__(self):
+        return repr(self.opts)
+
+def initConfig():
+    """Initialize the global configuration. You must call this before calling getOpt()."""
+    global ctx
+    ctx = cfCtx()
+    return ctx
+
+# The following two functions exist to work around how python handles globals.
+# Without an explicit getter, other modules that import config.py would only
+# ever get a stale reference to ctx
+def getCtx():
+    """Return the global confguration object (ctx). This is only valid after
+    calling initialize().
+    
+    Returns (cfCtx)
+    """ 
+    return ctx
+
+def getOpt(opt):
+    """Convenience function for reading global configuration options."""
+    if ctx is None:
+        raise RuntimeError("Context not initized")
+    else:
+        return ctx[opt]

deploy/pkg/util/parseconfig.py

@@ -24,7 +24,10 @@ class Accel(object):
         self.scala_dir = os.path.join(src_main_path, 'scala') 
 
     def info(self):
-        rootLogger.info("""\t\taccel_name: {} src_dir: {}""".format(self.name, self.src_dir))
+        rootLogger.info(str(self))
+
+    def __str__(self):
+        return """\t\taccel_name: {} src_dir: {}""".format(self.name, self.src_dir)
 
 
 class RoCCAccel(Accel): 
@@ -61,13 +64,17 @@ class AccelConfig:
         self.tl_accels = []
         self.parse_json_config(self.accel_json)
 
-    def info(self):
-        rootLogger.info("""Accelerator SoC Definition: """)
-        rootLogger.info("Generated SoC Directory: {}".format(self.accel_dir))
+    def __str__(self):
+        s = """Accelerator SoC Definition: \n"""
+        s += "Generated SoC Directory: {}\n".format(self.accel_dir)
         for rocc_accel in self.rocc_accels:
-            rocc_accel.info()
+            s += str(rocc_accel) + "\n"
         for tl_accel in self.tl_accels:
-            tl_accel.info()
+            s += str(tl_accel) + "\n"
+        return s
+
+    def info(self):
+        rootLogger.info(str(self))
 
     def get_default_src_dir(self, pgm): 
         """

deploy/pkg/util/streamlogger.py

@@ -7,7 +7,7 @@ which has no license associated with it.
 """
 import sys
 import logging
-import cStringIO
+import io
 
 
 class StreamLogger(object):
@@ -37,7 +37,7 @@ class StreamLogger(object):
         self.__name = name
         self.__stream = getattr(sys, name)
         self.__logger = logger or logging.getLogger()
-        self.__buffer = cStringIO.StringIO()
+        self.__buffer = io.StringIO()
         self.__unbuffered = unbuffered
         self.__flush_on_new_line = flush_on_new_line
 

deploy/support/default-config.yaml

@@ -0,0 +1,5 @@
+# Path to the chipyard repository to use when generating hardware
+chipyard-dir: '../../../../'
+
+# Directory to store runtime logs in
+log-dir: '../logs'

python-requirements.txt

@@ -0,0 +1,3 @@
+argparse
+argcomplete
+logging