From fc9005c61cad4d51fd40de45640f15d1052ae305 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 30 Dec 2015 21:20:01 -0800 Subject: [PATCH 1/2] Toolchain bash script generator draft --- .gitignore | 5 +- test_toolchain.py | 44 +++++++++ toolchain.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 test_toolchain.py create mode 100644 toolchain.py diff --git a/.gitignore b/.gitignore index 9a504a6..50775a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ build check -source \ No newline at end of file +source + +# python stuff +*.py[ocd] diff --git a/test_toolchain.py b/test_toolchain.py new file mode 100644 index 0000000..c19ed9f --- /dev/null +++ b/test_toolchain.py @@ -0,0 +1,44 @@ +import toolchain + +fake_metadata = { + 'a': { + '0.1': {} + }, + 'b': { + '0.2': { + 'dependencies': ['a=0.1'] + } + }, + 'c': { + '0.3': { + 'dependencies': ['a=0.1', 'b=0.2'] + } + }, + 'd': { + '0.4': { + 'dependencies': ['b=0.2'] + } + }, +} + + +def test_build_ordering(): + libs = ['a=0.1', 'd=0.4'] + builder = toolchain.ScriptBuilder(libs, package_registry=fake_metadata) + result = builder.get_build_script() + expected = toolchain.BUILD_SCRIPT_PREAMBLE + """\ +A_VERSION=0.1 $SOURCE_DIR/source/a/build.sh +A_VERSION=0.1 B_VERSION=0.2 $SOURCE_DIR/source/b/build.sh +B_VERSION=0.2 D_VERSION=0.4 $SOURCE_DIR/source/d/build.sh""" + + assert result == expected + + libs = ['d=0.4'] + builder = toolchain.ScriptBuilder(libs, package_registry=fake_metadata) + result = builder.get_build_script() + expected = toolchain.BUILD_SCRIPT_PREAMBLE + """\ +A_VERSION=0.1 $SOURCE_DIR/source/a/build.sh +A_VERSION=0.1 B_VERSION=0.2 $SOURCE_DIR/source/b/build.sh +B_VERSION=0.2 D_VERSION=0.4 $SOURCE_DIR/source/d/build.sh""" + + assert result == expected diff --git a/toolchain.py b/toolchain.py new file mode 100644 index 0000000..477c741 --- /dev/null +++ b/toolchain.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python +# Copyright 2015 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + +# ===================================================================== +# Package metadata +# ===================================================================== + +PACKAGE_METADATA = {} + + +def add_package_version(pkg_name, version, deps=None): + if deps is None: + deps = [] + if pkg_name not in PACKAGE_METADATA: + PACKAGE_METADATA[pkg_name] = {} + spec = { + 'dependencies': deps + } + PACKAGE_METADATA[pkg_name][version] = spec + +# --------------------------------------------------------------------- +# boost + +add_package_version('boost', '1.57.0') + +# ---------------------------------------------------------------------- +# boost + +add_package_version('python', '2.7.10') + +# --------------------------------------------------------------------- +# cmake + +add_package_version('cmake', '3.2.3') + +# --------------------------------------------------------------------- +# LLVM + +add_package_version('llvm', '3.3-p1') +add_package_version('llvm', '3.7.0') + +# --------------------------------------------------------------------- +# SASL + +add_package_version('cyrus_sasl', '2.1.23') +add_package_version('cyrus_sasl', '2.1.26') + +# --------------------------------------------------------------------- +# libevent + +add_package_version('libevent', '1.4.15') + +# --------------------------------------------------------------------- +# openssl + +add_package_version('openssl', '1.0.1p') + +# --------------------------------------------------------------------- +# zlib + +add_package_version('zlib', '1.2.8') + +# --------------------------------------------------------------------- +# thrift + +_deps = ['libevent=1.4.15', 'boost=1.57.0', 'zlib=1.2.8', 'openssl=1.0.1p'] +add_package_version('thrift', '0.9.0-p2', _deps) +add_package_version('thrift', '0.9.0-p4', _deps) +add_package_version('thrift', '0.9.2-p2', _deps) + +# --------------------------------------------------------------------- +# gflags + +add_package_version('gflags', '2.0') + +# --------------------------------------------------------------------- +# glog + +_deps = ['gflags=2.0'] +add_package_version('glog', '0.3.2-p1', _deps) +add_package_version('glog', '0.3.3-p1', _deps) + +# --------------------------------------------------------------------- +# gperftools + +add_package_version('gperftools', '2.0-p1') +add_package_version('gperftools', '2.3') + +# --------------------------------------------------------------------- +# googletest + +add_package_version('gtest', '1.6.0') +add_package_version('googletest', '20151222') + +# --------------------------------------------------------------------- +# snappy + +add_package_version('snappy', '1.0.5') + +# --------------------------------------------------------------------- +# lz4 + +add_package_version('lz4', 'svn') + +# --------------------------------------------------------------------- +# re2 + +add_package_version('re2', '20130115') +add_package_version('re2', '20130115-p1') + +# --------------------------------------------------------------------- +# openldap + +add_package_version('openldap', '2.4.25') + +# --------------------------------------------------------------------- +# avro-c + +add_package_version('avro', '1.7.4-p3') + +# --------------------------------------------------------------------- +# rapidjson + +add_package_version('rapidjson', '0.11') + +# --------------------------------------------------------------------- +# bzip2 + +add_package_version('bzip2', '1.0.6-p1') + +# --------------------------------------------------------------------- +# gdb + +# --------------------------------------------------------------------- +# libunwind + +add_package_version('libunwind', '1.1') + +# --------------------------------------------------------------------- +# breakpad + +add_package_version('breakpad', '20150612-p1') + +# ====================================================================== +# Install a set of packages +# ====================================================================== + +BUILD_SCRIPT_PREAMBLE = """ +#!/usr/bin/env bash +# Exit on non-true return value +set -e +# Exit on reference to uninitialized variable +set -u +set -o pipefail + +# The init.sh script contains all the necessary logic to setup the environment +# for the build process. This includes setting the right compiler and linker +# flags. +source ./init.sh +""" + + +class ScriptBuilder(object): + + def __init__(self, libraries, package_registry=None): + self.libraries = libraries + self.package_registry = package_registry or PACKAGE_METADATA + + def build(self): + script = self.get_build_script() + proc = subprocess.Popen(script, shell=True) + proc.wait() + + def get_build_script(self): + libraries_ordered = self._get_build_order() + + steps = [] + for libname, version in libraries_ordered: + build_line = self._build_library(libname, version) + steps.append(build_line) + + return BUILD_SCRIPT_PREAMBLE + '\n'.join(steps) + + def _get_build_order(self): + memo = {} + for lib in self.libraries: + name, version = _parse_lib(lib) + self._walk(name, version, memo) + + return [x1 for x0, x1 in + sorted((v, k) for k, v in memo.items())] + + def _walk(self, name, version, memo): + if (name, version) in memo: + return + + for dep in self._get_deps(name, version): + depname, depversion = _parse_lib(dep) + self._walk(depname, depversion, memo) + + memo[name, version] = len(memo) + + def _build_library(self, name, version): + tokens = [_lib_env_decl(*_parse_lib(dep)) + for dep in self._get_deps(name, version)] + + tokens.append(_lib_env_decl(name, version)) + + tokens.append('$SOURCE_DIR/source/{0}/build.sh'.format(name)) + return ' '.join(tokens) + + def _get_deps(self, name, version): + meta = self.package_registry[name] + if version not in meta: + raise KeyError('No {0} build metadata for version: {1}' + .format(name, version)) + return meta[version].get('dependencies', []) + + +def _lib_env_decl(name, version): + return '{0}_VERSION={1}'.format(name.upper(), version) + + +def _parse_lib(lib): + return lib.split('=') From 31b01ebf92849d8606419d501ff88d308c4313be Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 4 Jan 2016 13:19:34 -0800 Subject: [PATCH 2/2] Make $SOURCE_DIR configurable --- init.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/init.sh b/init.sh index 64bc98e..ea74e7f 100644 --- a/init.sh +++ b/init.sh @@ -65,7 +65,9 @@ BUILD_THREADS=$(getconf _NPROCESSORS_ONLN) export BUILD_THREADS # SOURCE DIR for the current script -export SOURCE_DIR="$( cd "$( dirname "$0" )" && pwd )" +if [ -z "$SOURCE_DIR" ]; then + export SOURCE_DIR="$( cd "$( dirname "$0" )" && pwd )" +fi if [[ $DEBUG -eq 1 ]]; then set -x