git.iko.soy/typeable/cabal2nix
on v2.8.2 help sign in
typeable/cabal2nix/doc/analyze-hydra-builds.r
54 lines 2.4 kB history raw
1# cabal2nix/doc/analyze-hydra-builds.r
2#
3# Generate the input file "builds.csv" by running
4#
5# sudo -u hydra psql -c "Copy (select b.project, b.jobset, b.job, b.timestamp, b.drvpath, b.system, b.buildstatus, b.size, b.closuresize, bs.type, bs.starttime, bs.stoptime, bs.machine from builds b left join buildsteps bs on b.id=bs.build and b.drvpath=bs.drvpath where finished=1 and iscachedbuild=0) to stdout With CSV HEADER;" >builds.csv
6#
7# on Hydra.
9library(data.table)
11builds <- within(as.data.table(read.csv("builds.csv", header=T, stringsAsFactors=T)), {
12 timestamp <- as.POSIXct(timestamp, origin="1970-01-01")
13 starttime <- as.POSIXct(starttime, origin="1970-01-01")
14 stoptime <- as.POSIXct(stoptime, origin="1970-01-01")
15 runtime <- difftime(stoptime, starttime, units="mins")
16 machine <- sub("^(hydra|root)@", "", machine)
17})
18builds[machine=="", machine:="localhost"]
19setkey(builds, project, jobset, job, timestamp)
21## Determine the average build runtime of all successful Haskell builds that
22## ran on "work".
24hs <- builds[project=="nixpkgs" & jobset=="haskell-updates",]
25hs <- hs[machine=="work" & buildstatus==0 & system == "x86_64-linux" & runtime > 0,]
26hs <- hs[,job:=sub("\\.x86_64-linux$", "", job)]
27hs <- hs[,job:=sub("^haskell(ng)?Packages\\.", "", job)]
28hs <- hs[,job:=sub(".*git-?annex.*", "git-annex", job, ignore.case=T)]
29hs <- hs[,job:=sub("idris_plain$", "idris", job)]
30hs <- hs[!grep("haskell(-ng)?\\.compiler\\.", job),]
31hs <- hs[!grep("^ghc\\.ghc", job),]
32hs <- hs[!grep("ghc[0-9]+$", job),]
33hs <- hs[!duplicated(hs$drvpath)]
34hs <- hs[
35 , list(builds=length(runtime),runtime=sum(runtime),runtime.median=median(runtime),runtime.mean=mean(runtime),runtime.sd=sd(runtime))
36 , by=job
37 ]
38hs <- hs[builds>10,]
39head(hs[order(runtime.mean, decreasing=T)], n=30)
40head(hs[order(builds, decreasing=T)], n=20)
41summary(as.numeric(hs$runtime.mean,units="mins"))
42quantile(as.numeric(hs$runtime.mean,units="mins"), 0.82)
44## Determine the number of builds per day.
46hs <- builds[ project=="nixpkgs" & jobset=="haskell-updates" & starttime >= as.POSIXct("2015-01-01")
47 , list(builds=length(timestamp))
48 , by=list(day=as.Date(timestamp))
49 ]
50hs <- merge(hs, data.table(day=seq(min(hs$day), max(hs$day), by="days"), ignore=0), by="day", all=T)
51hs[is.na(builds)] <- 0
52hs$ignore <- NULL
53summary(hs$builds)
54quantile(hs$builds, .8)