first draft of cruise control for Moses

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4166 1f5c12ca-751b-0410-a591-d2e778427230

cruise-control/README

@@ -0,0 +1,27 @@
+A simple regular testing of Moses codebase, aka cruise control
+Started by Ondrej Bojar
+2011-08-28
+
+
+Usage:
+
+1. Checkout this directory somewhere on the computer where you want to run the
+   cruise control.
+2. Create as many config files as you wish, an example is ondrej-nb.config
+   ...hardcode all paths to you preferred external tools like LM toolkits.
+3. Run ./test_all_new_commits.sh <CONFIGFILE>
+
+
+
+TODO / KNOWN BUGS
+- regression tests are not run yet
+
+- regression tests always require SRILM, but we need to test all LMs that have
+  been compiled in
+  => add separate regression tests, one for each LM?
+  => modify regression tests to actually loop over all LMs?
+
+- final status is FAIL if any regression test fails, but we should actually
+  allow to expect failures for the given set of ./configure parameters
+  (e.g. regression test requiring irstlm is bound to fail if we're not linking
+  against irstlm)

cruise-control/test_all_new_commits.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+# given a config file runs tests on all untested commits of the scanned branches
+# storing detailed logs to logs/CONFIGNAME/commit
+# and extending the file brief.log
+#
+# A commit is assumed to be tested, if logs/CONFIGNAME/commit exists
+#
+# Ondrej Bojar, 2011
+
+function warn() { echo "$@" >&2; }
+function die() { echo "$@" >&2; exit 1; }
+set -o pipefail  # safer pipes
+
+configf="$1"
+[ -e "$configf" ] || die "usage: $0 configfile"
+configname=$(basename $configf | sed 's/\.config$//')
+
+source "$configf"
+
+[ -z "$MCC_SCAN_BRANCHES" ] \
+  && die "Bad config $configf; does not define MCC_SCAN_BRANCHES"
+
+# use the given tempdir or make subdir tmp here
+USE_TEMPDIR=$MCC_TEMPDIR
+[ -d "$USE_TEMPDIR" ] || USE_TEMPDIR=./tmp
+
+LOGDIR=$MCC_LOGDIR
+[ -d "$LOGDIR" ] || LOGDIR=.
+
+# ensure full path for logdir
+LOGDIR=$(readlink -f "$LOGDIR")
+[ -d "$LOGDIR" ] || die "Fatal: confusing readlink for $LOGDIR"
+
+# this is where moses is cloned into
+WORKDIR=$MCC_WORKDIR
+[ -d "$WORKDIR" ] || WORKDIR=$USE_TEMPDIR/workdir
+
+# this is where moses is taken from
+GITREPO=$MCC_GITREPO
+[ -d "$GITREPO" ] || GITREPO=/home/obo/moses-at-google-code
+
+
+if [ ! -d "$WORKDIR" ]; then
+  mkdir $(dirname "$WORKDIR") || die "Failed to create workdir $WORKDIR"
+  warn "Cloning $GITREPO into $WORKDIR"
+  git clone "$GITREPO" $WORKDIR \
+    || die "Failed to git clone into workdir $WORKDIR"
+else
+  ( cd "$WORKDIR" && git fetch ) \
+    || die "Failed to update our clone at $WORKDIR"
+fi
+
+mkdir -p $LOGDIR/logs/$configname \
+  || die "Failed to create dir $LOGDIR/logs/$configname"
+
+#### How is one test performed
+function run_single_test () {
+  commit=$1
+  longlog="$LOGDIR/logs/$configname/$commit"
+  if [ -e "$longlog" ]; then
+    # Commit already tested
+    return
+  fi
+
+  warn "Testing commit $commit"
+
+  # Get the version of this script
+  ccversion=$(svnversion 2>/dev/null)
+  [ ! -z "$ccversion" ] || ccversion=$(git show 2>&1 | head -n 1)
+  [ ! -z "$ccversion" ] || ccversion="unknown"
+
+  # Create log header with computer details:
+  echo "#### Moses Cruise Control Log for commit $commit" > $longlog
+  date >> $longlog
+  echo "## Cruise Control version" >> $longlog
+  echo $ccversion >> $longlog
+  echo "## Parameters" >> $longlog
+  cat $configf >> $longlog
+  echo "## Envinronment" >> $longlog
+  uname -a >> $longlog
+  env >> $longlog
+
+
+  pushd $WORKDIR 2>/dev/null >/dev/null || die "Failed to chdir to $WORKDIR"
+  git checkout --force $commit 2>/dev/null || die "Failed to checkout commit $commit"
+
+  err=""
+  echo "## regenerate-makefiles.sh" >> $longlog
+  ./regenerate-makefiles.sh >> $longlog 2>&1 || err="regenerate-makefiles"
+
+  echo "## make clean" >> $longlog
+  make clean >> $longlog 2>&1 || warn "make clean failed, suspicious"
+
+  echo "## ./configure $MCC_CONFIGURE_ARGS" >> $longlog
+  [ -z "$err" ] && ./configure $MCC_CONFIGURE_ARGS  >> $longlog 2>&1 \
+    || err="configure"
+
+  echo "## make" >> $longlog
+  [ -z "$err" ] && make  >> $longlog 2>&1 \
+    || err="make"
+
+  cd regression-testing
+  echo "## Not running any regression tests yet." >> $longlog
+
+  echo "## Finished" >> $longlog
+  date >> $longlog
+
+  if [ -z "$err" ]; then
+    status="OK"
+  else
+    status="FAIL:$err"
+  fi
+  echo "## Status: $status" >> $longlog
+  
+  nicedate=$(date +"%Y%m%d-%H%M%S")
+  echo "$commit	$status	$configname	$ccversion	$nicedate" \
+    >> "$LOGDIR/brief.log"
+
+  popd > /dev/null 2> /dev/null
+}
+
+
+#### Main loop over all commits
+( cd "$WORKDIR" && git rev-list $MCC_SCAN_BRANCHES ) \
+| while read commit; do
+  run_single_test $commit || die "Testing failed, stopping the loop."
+done
+

regression-testing/ensure-regression-data-here.perl

@@ -0,0 +1,34 @@
+#!/usr/bin/env perl
+# downloads the regression data
+use strict;
+use MosesRegressionTesting;
+
+my $data_version = MosesRegressionTesting::TESTING_DATA_VERSION;
+
+exit 0 if -d "moses-reg-test-data-$data_version";
+  # data in place
+
+safesystem("wget http://www.statmt.org/moses/reg-testing/moses-reg-test-data-$data_version.tgz")
+  or die "wget failed";
+safesystem("tar xzf moses-reg-test-data-$data_version.tgz")
+  or die "untar failed";
+safesystem("rm moses-reg-test-data-$data_version.tgz");
+
+sub safesystem {
+  # print STDERR "Executing: @_\n";
+  system(@_);
+  if ($? == -1) {
+      print STDERR "Failed to execute: @_\n  $!\n";
+      exit(1);
+  }
+  elsif ($? & 127) {
+      printf STDERR "Execution of: @_\n  died with signal %d, %s coredump\n",
+          ($? & 127),  ($? & 128) ? 'with' : 'without';
+      exit(1);
+  }
+  else {
+    my $exitcode = $? >> 8;
+    print STDERR "Exit code: $exitcode\n" if $exitcode;
+    return ! $exitcode;
+  }
+}