Mercurial > projects > doodle
view builder.d @ 40:1f97022e5c6d
Checkpoint. Development continues...
author | daveb |
---|---|
date | Mon, 12 Apr 2010 14:01:54 +0930 |
parents | b6c34f1fc7f3 |
children | 3c1b0c4d5847 |
line wrap: on
line source
module builder; private { import std.stream; import std.stdio; import std.string; import std.process; import std.file; import std.path; import std.date; } // // builder is a build tool for D code. It is intended for use during development, // and so supports: // * Building only those targets that are out of date. // * Automated execution and evaluation of regression tests. // * Automatic linking of libraries that are part of the project. // * Enforcement of dependency rules (explained below). // * Enforcement of a directory structure designed to support reusable components. // * Complete separation of source from targets (friendly on checkouts). // // To keep things simple and fast, builder won't build arbitrary code - but it will build // code written to be built with it. Besides having to obey dependency and directory // layout and naming rules, you have to do imports as one of: // public import module_name; // private import module_name; // import module_name; // static import module_name; // static import module_alias = module_name; // specifically, the module-name has to be the last thing on the line. // // // The directory structure employs a hierarchy of: // * Bundle - A collection of products. Has no source code of its own, and does not appear in package paths. // Corresponds to the root of a checkout/repository. The bundle is on the source search path. // * Product - Top-level package, usually to provide namespace, but can also contain code // just like lower-level packages. // * Package - A self-contained set of library code, executable code, docs, tests, etc. // Can contain nested packages, and can refer to other packages. // // Bundles are the mechanism for reusing source code in other repositories. // We do this by lines in a bundle "uses" file that specify where to find a used bundle. // // The directory structure within a bundle is: // // +--README Introductory information // +--configure.d D script to set up a build directory // +--builder.d This file // +--options Compiler options // +--uses Specifies which other bundles to use, with paths relative to this bundle // | // +--package-name(s) A package, containing library source // | // +--doc Restructured-text documents and associated images // | // +--data Data files required by tests and binaries // | // +--test Code for regression tests // | // +--prog Code for binaries // | // +--package-name(s) Nested packages // // The resultant build directory structure is: // // +--build Script to build the system (put there by configure) // | // +--obj Contains object files and libraries in package structure // | // +--test Contains test binaries and results in package structure // | // +--bin Contains the binaries built from prog directories // | // +--doc Contains the package html docs, in package structure // // The dependency rules are: // * A built file depends on its sources, and anything the sources explicitly import. // * A parent depends on its children, transitively, and everything they depend on, transitively. // * Everything depends on the parents and descendants of its explicit imports, // up to but not including the common ancestor, transitively. // // TODO - add doc, data // TODO - add timeout on test execution - say 2 seconds. // TODO - add excludes files in packages to allow files and directories to be ignored. //------------------------------------------------------------------------------- // primitives //------------------------------------------------------------------------------- // // Some global data // abstract class Global { static string buildPath; static string optionsPath; static string[] bundlePaths; static bool[string] products; } // // throw an exception // void error(string text) { writefln("%s", text); throw new Exception(text); } // // return the modification time of the file at path // d_time modified_time(string path) { d_time creation_time, access_time, modified_time; if (!exists(path)) { return 0; } getTimes(path, creation_time, access_time, modified_time); return modified_time; } // // A string formatter that allows progressive composition // class StringFormatter { char[] buffer; int pos; this() { buffer.length = 1024; } void clear() { pos = 0; } // append some formatted text to the internal buffer void format(...) { void put(dchar c) { if (pos+4 >= buffer.length) { buffer.length = buffer.length * 2; } char[4] buf; char[] result = std.utf.toUTF8(buf, c); buffer[pos..pos+4] = buf; pos += result.length; } std.format.doFormat(&put, _arguments, _argptr); } // return a copy of the internal buffer string str() { return buffer[0..pos].idup; } } //--------------------------------------------------------- // parser //--------------------------------------------------------- // FIXME - put in a proper parser that understands about version syntax and all the rest. /// /// Parse file for import statements, returning an array of module names /// string[] parseImports(string path) { string[] result; // // If this is an import declaration line, return the imported module, else null // This isn't perfect by any means (but it is simple and fast), but here we go: // * Strip whitespace from front. // * If first word is private, public, static or import, and // * If we have import already or there is another import later in the line, then // * The last word on the line is a module. // A less-than-perfect parser is ok because it only has to work for code // within the project, so we can write code that it works for. // void parseForImport(string line) { static string[] leaders = ["private ", "public ", "static ", "import "]; bool found; string stripped = stripl(line); if (stripped.length > 8) { foreach (i, leader ; leaders) { if (stripped[0..leader.length] == leader) { // this is a possibility - look for import token scope string[] tokens = split(stripped); if (tokens.length > 1) { if (i == 3) { found = true; } else { foreach (token; tokens[0..$-1]) { if (token == "import") { found = true; break; } } } } if (found) { //writefln("found import of %s", tokens[$-1][0..$-1]); result ~= tokens[$-1][0..$-1].idup; } break; } } } } scope file = new BufferedFile(path); foreach (char[] line; file) { parseForImport(line.idup); } //writefln("imports of %s are %s", file, result); return result; } //------------------------------------------------------------------------------ // Dependency graph //------------------------------------------------------------------------------ // // An item that may depend on things and be depended on in turn // abstract class Item { static Item[string] mItems; // items by path so parent-path items can be found string mKind; // for logging string mPath; // identifies the item long mTime; // modification time, 0 ==> doesn't exist, -1 ==> don't care Item[string] mDepends; // items this one depends on bool mChecking; bool mChecked; bool mDirty; this(string kind, string path) { //writefln("creating %s-item at %s", kind, path); mKind = kind; mPath = path; mTime = modified_time(path); mItems[path] = this; } // add a dependency to this item final void addDepends(Item item) { if (!(item.mPath in mDepends)) { //writefln("%s depends on %s", this.mPath, item.mPath); mDepends[item.mPath] = item; } } // Find out if this item is dirty (needs to be built), returning true if dirty. final bool dirty() { if (mChecking) { error(format("circularity: %s depends on itself", mPath)); } if (!mChecked) { mChecking = true; if (mTime == 0) { //writefln("%s is dirty because it doesn't exist", basename(mPath)); mDirty = true; } foreach (other; mDepends) { if (mDirty) break; if (other.dirty) { //writefln("%s is dirty because %s is dirty", basename(mPath), basename(other.mPath)); mDirty = true; } else if (mTime != -1 && other.mTime != -1 && other.mTime > mTime) { //writefln("%s is dirty because %s is younger", basename(mPath), basename(other.mPath)); //writefln(" other=%s, mine=%s", other.mTime, mTime); mDirty = true; } } mChecking = false; mChecked = true; } return mDirty; } // build just this item protected abstract void build(); // build this item and everything it depends on final void buildChain() { if (dirty) { foreach (other; mDepends) { other.buildChain; } build; if (mTime != -1) { mTime = modified_time(mPath); } mDirty = false; } } string toString() { return mKind ~ "-item-" ~ mPath; } } // // A directory // class DirectoryItem : Item { this(string path) { super("directory", path); mTime = mPath.exists ? 1 : 0; // this directory depends on its parent directory chain string parent_path = path.dirname; if (parent_path.length > 0 && parent_path[0] != '.' ) { Item* parent = parent_path in mItems; if (parent) { assert(cast(DirectoryItem *)parent); addDepends(*parent); } else { // recursively create chain of directories above this, // and depend on it. addDepends(new DirectoryItem(parent_path)); } } } // build this directory override void build() { writefln("Directory %s", mPath); mkdir(mPath); } } /// /// A group Item - used to help manage build order via dependencies /// All FileItems are members of exactly one group, and they all depend on /// all the groups their owning group depends on. /// class GroupItem : Item { static GroupItem[string] mGroups; // all groups Item[] mMembers; this(string path) { super("group", path); mTime = -1; mGroups[mPath] = this; } void addMember(Item member) { addDepends(member); mMembers ~= member; } // add dependencies on other group and its ancestors // Only called AFTER all items have been greated and assigned their group. void dependsOnGroup(GroupItem other) { if (other !is this) { // this group and everything in it (other than source) depends on other group addDepends(other); foreach (member; mMembers) { if (!cast(SourceItem) member) { member.addDepends(other); } } // and all other group's higher-level groups until common ancestor string other_parent_path = other.mPath.dirname; if (other_parent_path != "." && other_parent_path.length < mPath.length && mPath[0..other_parent_path.length] != other_parent_path) { GroupItem* other_parent = other_parent_path in mGroups; if (other_parent) { dependsOnGroup(*other_parent); } } } } // build just this item override void build() { // nothing to do writefln("Group %s", mPath); } } // // a physical file in the filesystem // abstract class FileItem : Item { GroupItem mGroup; this(string kind, string path, GroupItem group) { super(kind, path); mGroup = group; group.addMember(this); // this file depends on on its parent directory string parent_path = path.dirname; Item* parent = parent_path in mItems; if (parent) { assert(cast(DirectoryItem *)parent); addDepends(*parent); } else { addDepends(new DirectoryItem(parent_path)); } } } // // A source file - must exist // class SourceItem : FileItem { SourceItem[string] mImports; // the source items this one imports ObjectItem mObject; // the object item to be built from this source this(string path, GroupItem group) { super("source", path, group); if (!isfile(path)) error(format("source file %s not found", path)); } // set the object item void set_object(ObjectItem obj) { mObject = obj; } // Add an import to this source item. // These are used later to determine all the non-structural item dependencies. void addImport(SourceItem source) { if (!(source.mPath in mImports)) { //writefln("%s imports %s", mPath, source.mPath); mImports[source.mPath] = source; } } override void build() { // nothing to do, and should never be dirty writefln("Source %s", mPath); error("should never need to build Source"); } } // // An object file // class ObjectItem : FileItem { SourceItem mSource; this(string path, SourceItem source, GroupItem group) { super("object", path, group); addDepends(source); mSource = source; } // Use our source's imports to add non-structural dependencies. // The rules are: // * We depend on any sources our source imports, and their objects, transitively. // * Our group depends on the imported source's group, and any of its // ancestral groups that aren't our ancestors. // * We must not trace a chain of imports back to our source. void resolveImports() { void dependsOnImportsOf(SourceItem source) { foreach (s; source.mImports) { if (s is mSource) { error(format("circular chain of imports - last link from %s to %s", source.toString, s.toString)); } if (s.mGroup !is mGroup) { addDepends(s.mGroup); } addDepends(s); addDepends(s.mObject); dependsOnImportsOf(s); } } dependsOnImportsOf(mSource); foreach (s; mSource.mImports) { mGroup.dependsOnGroup(s.mGroup); } } // an object file is built from its source override void build() { writefln("Object %s", mPath); scope cmd = new StringFormatter; cmd.format("dmd -c -g"); foreach (path; Global.bundlePaths) { cmd.format(" -I", path); } cmd.format(" -od%s -of%s %s", dirname(mPath), basename(mPath), mSource.mPath); cmd.format(" @%s", Global.optionsPath); if (std.process.system(cmd.str)) { writefln("%s", cmd.str); error(format("build of %s failed", mPath)); } } } // // A library // class LibraryItem : FileItem { string mName; this(string path, string name, GroupItem group) { super("library", path, group); mName = name; } // a library is built by archiving all of its contributing ObjectFiles override void build() { writefln("Library %s", mPath); scope cmd = new StringFormatter; cmd.format("ar csr %s ", mPath); foreach (item; mDepends) { auto obj = cast(ObjectItem) item; if (obj) { cmd.format(" %s", obj.mPath); } } if (std.process.system(cmd.str)) { writefln("%s", cmd.str); error("command failed"); } } // Add this library and any it depends on to libs, if they aren't there already. // NOTE - they are added with libraries appearing after those they depend on // (reverse of compiler command-line). void addNeeded(ref LibraryItem[] libs) { void add(LibraryItem lib) { foreach (item; libs) { if (lib is item) { return; } } libs ~= lib; } foreach (item; mDepends) { auto lib = cast(LibraryItem) item; if (lib) { lib.addNeeded(libs); } } add(this); } } // // A program // class ProgramItem : FileItem { ObjectItem mObject; this(string path, ObjectItem object, GroupItem group) { super("program", path, group); mObject = object; addDepends(object); } // a program file is built from its object and all the libraries // the object needs, transitively override void build() { writefln("Program %s", mPath); scope cmd = new StringFormatter(); cmd.format("dmd -L-L%s", join(Global.buildPath, "lib")); cmd.format(" -of%s %s", mPath, mObject.mPath); // add the libraries we need LibraryItem[] libs; foreach (item; mDepends) { auto lib = cast(LibraryItem) item; if (lib) { lib.addNeeded(libs); } } foreach_reverse (lib; libs) { cmd.format(" -L-l%s", lib.mName); } cmd.format(" @%s", Global.optionsPath); if (std.process.system(cmd.str)) { writefln("%s", cmd.str); error("command failed"); } } } // // A test result - run the program to generate the result file // class ResultItem : FileItem { ProgramItem mProgram; this(string path, ProgramItem program, GroupItem group) { super("result", path, group); mProgram = program; addDepends(program); } override void build() { writef("Test %s", mPath); if (exists(mPath)) { mPath.remove(); } scope cmd = new StringFormatter(); scope tmpPath = (mPath ~ ".failed").idup; cmd.format("%s > %s 2>&1", mProgram.mPath, tmpPath); if (std.process.system(cmd.str)) { // failed writefln(" failed"); writefln("%s", cmd.str); error("test failed"); } else { tmpPath.rename(mPath); writefln(" passed"); } } } //------------------------------------------------------------------------------- // Tree of modules, packages, products and system, etc - things in the source tree. // When a node explicity depends on another, it (and its parent up to the common ancestor) // implicity depend on that node and its ancestors and descendants, and everything they depend on. // Circularities are not allowed. //------------------------------------------------------------------------------- // // a node in a tree. It depends on explicit mDepends, plus any children, and // anything any of those depend on. The dependencies are tracked to enforce // dependency rules. Actual build-order dependencies are done with Items. // class Node { string mName; Node mParent; Node[] mChildren; string mSlashName; string mDotName; Node[] mDepends; // create the root of the tree this() { } // create a node in the tree this(string name, Node parent) { assert(parent); mName = name; mParent = parent; if (parent.mName.length) { // child of non-root mSlashName = parent.mSlashName ~ sep ~ mName; mDotName = parent.mDotName ~ "." ~ mName; } else { // child of the root mSlashName = mName; mDotName = mName; } parent.mChildren ~= this; } // return a node with the given name-chain, or null if not found. // The root node is not considered part of the chain final Node find(string[] chain) { // return child of node as specified by chain, or null if not found Node locate(Node node, string[] chain) { foreach (child; node.mChildren) { if (child.mName == chain[0]) { if (chain.length > 1) { return locate(child, chain[1..$]); } else { return child; } } } return null; } // ascend to root and recursively descend to specified node assert(chain.length > 0); Node node = this; while (node.mParent) { node = node.mParent; } return locate(node, chain); } } // // A source-code module // class Module : Node { SourceItem mSource; ObjectItem mObject; FileItem mClump; // library or program item this module contributes to this(string name, string path, Node parent, GroupItem group) { super(name, parent); string obj_path = join(Global.buildPath, "obj", mSlashName ~ ".o"); mSource = new SourceItem(path, group); mObject = new ObjectItem(obj_path, mSource, group); mSource.set_object(mObject); //writefln("loaded Module %s from %s", mDotName, mSource.mPath); } void setClump(FileItem clump) { assert(!mClump); mClump = clump; mClump.addDepends(mObject); } // trace direct imports final void trace() { // add an import dependancy (if it is inside our project) void imports(string importing) { string[] chain = split(importing, "."); assert(chain.length > 0); // only depend on internal products if (!(chain[0] in Global.products)) { return; } // find the imported Module's source item and add it to our source item's imports Module other = cast(Module) find(chain); if (!other || !other.mClump) { writefln("import of unknown module " ~ importing ~ " from " ~ mDotName); } else if (cast(LibraryItem)other.mClump) { // the other's mClump is a library // our source imports other's source - only allowed for library source mSource.addImport(other.mSource); if (mClump.mPath != other.mClump.mPath) { // this module's clump depends on other's library (needed for linking) mClump.addDepends(other.mClump); } } else { error(format("import of non-library module %s from %s", importing, mDotName)); } } assert(mClump); foreach (importing; parseImports(mSource.mPath)) { //writefln("%s imports module %s", mDotName, importing); imports(importing); } } } // // An executable program. // class Exe : Module { ProgramItem mProgram; ResultItem mResult; this(string name, string path, Node parent, bool test, GroupItem group) { super(name, path, parent, group); if (test) { mProgram = new ProgramItem(join(Global.buildPath, "test", mSlashName), mObject, group); mResult = new ResultItem(join(Global.buildPath, "test", mSlashName ~ ".result"), mProgram, group); } else { mProgram = new ProgramItem(join(Global.buildPath, "bin", mName), mObject, group); } setClump(mProgram); //writefln("loaded Program %s", mProgram.mPath); } } // // a package // class Package : Node { LibraryItem mLibrary; GroupItem mGroup; // represents the package // load the package this(string name, string path, Node parent) { super(name, parent); mGroup = new GroupItem(mSlashName); string lib_name = replace(mDotName, ".", "_"); // examine all the children of the package's directory foreach (string child; listdir(path)) { string p = join(path, child); if (child[0] != '.') { if (isfile(p)) { if (child.length > 2 && child[$-2..$] == ".d") { // a library module if (!mLibrary) { mLibrary = new LibraryItem(join(Global.buildPath, "lib", "lib" ~ lib_name ~ ".a"), lib_name, mGroup); } Module m = new Module(getName(child), p, this, mGroup); m.setClump(mLibrary); } } else if (isdir(p)) { if (child == "build-tool") { // reserved for build-tool } else if (child == "doc") { // TODO } else if (child == "data") { // TODO } else if (child == "test") { // test programs foreach (string grandchild; listdir(p)) { string p2 = join(p, grandchild); if (grandchild[0] != '.' && isfile(p2) && grandchild.length > 2 && grandchild[$-2..$] == ".d") { Exe exe = new Exe(getName(grandchild), p2, this, true, mGroup); } } } else if (child == "prog") { // deliverable programs foreach (string grandchild; listdir(p)) { string p2 = join(p, grandchild); if (child[0] != '.' && isfile(p2) && grandchild.length > 2 && grandchild[$-2..$] == ".d") { Exe exe = new Exe(getName(grandchild), p2, this, false, mGroup); } } } else { // a child package Package pkg = new Package(child, p, this); mGroup.addDepends(pkg.mGroup); } } } } } // trace dependancies void trace() { foreach (child; mChildren) { Package pkg = cast(Package) child; Module m = cast(Module) child; if (pkg) { pkg.trace(); } else if (m) { m.trace(); } } } } // // A Product - a top-level package // class Product : Package { string mPath; this(string name, string path, Node parent) { writefln("loading Product %s from %s", name, path); mPath = path; super(name, path, parent); } } // // the whole project // class Project : Node { // create the project, loading its bundle and those it uses this(string path) { super(); load(path); } // load the bundle at path void load(string path) { //writefln("loading bundle from %s", path); // add path to Global for use when compiling Global.bundlePaths ~= path; Global.optionsPath = path.join("options"); // // load bundles specified in the uses file - lines are bundle paths relative to path // string uses_path = join(path, "uses"); if (exists(uses_path) && isfile(uses_path)) { //writefln("reading uses file: %s", uses_path); scope file = new BufferedFile(uses_path); foreach (char[] line; file) { load(join(path, line.idup)); } } // // load local products // foreach (string name; listdir(path)) { if (name[0] != '.') { string p = join(path, name); if (isdir(p)) { foreach (node; mChildren) { Product existing = cast(Product)node; if (existing && existing.mName == name) { error(format("product %s has two paths: %s and %s", name, p, existing.mPath)); } } new Product(name, p, this); Global.products[name] = true; } } } // // trace imports, then finish the Item dependency graph // foreach (child; mChildren) { Product product = cast(Product) child; if (product) { product.trace(); } } foreach (item; Item.mItems) { ObjectItem obj = cast(ObjectItem) item; if (obj) { obj.resolveImports; } } // // print a dependency graph // foreach (group; GroupItem.mGroups) { // TODO } // // Build everything that is out of date // foreach (group; GroupItem.mGroups) { group.buildChain; } writefln("all done"); } } //-------------------------------------------------------------------------------------- // main - args[1] is where the project source is, and args[2] is where the build happens // All the paths are set to absolute so that the source files // can be found by an editor or debugger, wherever its current directory is. //-------------------------------------------------------------------------------------- int main(string[] args) { int usage() { writefln("Usage: builder <project-path> <build-path>"); return -1; } if (args.length < 3) return usage; string sourcePath = args[1]; Global.buildPath = args[2]; writefln("building source at %s into %s", sourcePath, Global.buildPath); // do the work scope project = new Project(sourcePath); return 0; }