urbit/shrub
1
1
Fork 0
mirror of https://github.com/urbit/shrub.git synced 2024-12-24 20:47:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cc-release' of github.com:urbit/urbit into bs/uterm

Browse Source
This commit is contained in:
Benjamin Summers 2019-05-02 13:14:25 -07:00
commit de04fe53c2
58 changed files with 961 additions and 649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -3,8 +3,9 @@
/result-*
/work
/zod
/bus
/fakezod*
tags
bin/darwin
bin/linux32
bin/linux64
TAGS
cross/
release/

29
.travis.yml
View File

@ -2,15 +2,32 @@ language: nix
nix: 2.1.3
install:
# nix-env -iA cachix -f https://cachix.org/api/v1/install
- nix-env -iA cachix -f https://cachix.org/api/v1/install
before_install:
- git lfs pull
script:
# cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null
# cachix use urbit
- cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null
- cachix use urbit2
- ./sh/cachix
- make
- make test
- sh/update-brass-pill # To verify that this works.
# ./sh/cachix
- make release
- sh/ci-tests
deploy:
- skip_cleanup: true
provider: gcs
access_key_id: GOOGTADOPP55X5ZTH3IKAXQW
secret_access_key:
secure: rSPif0VHX3Q3QpWM9iUt/Z9sicCY8ABuwVFPeT3YUnAAmSXM46PIHKieoGs79kx9IelFyQsM8xS0XWyt4S/haR7VaTJY+zHJjf0AnA1sr5ZIV70t3K0SXzq4+E1ffOZRiC0VmHatPz10wjaIpHxpjcROqQV4M1dBCq2H/rpccIE=
bucket: bootstrap.urbit.org
local-dir: release/
acl: public-read
on:
condition: "-d release/"
repo: urbit/urbit
all_branches: true

13
Makefile
View File

@ -1,8 +1,4 @@
PKGS = minima extrema rusthello prog
################################################################################
.PHONY: build build-all install release test clean
.PHONY: build build-all install cross release test clean
build:
nix-build -A urbit -A herb --no-out-link
@ -13,10 +9,11 @@ build-all:
install:
nix-env -f . -iA urbit -iA urbit-debug -iA herb
cross:
sh/cross
release:
sh/release urbit linux32
sh/release urbit linux64
sh/release urbit darwin
sh/release
test:
sh/test

4
bin/brass.pill
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e85cccf129bc9d03280d6142be58e56e230717c2ca9b8baeab38fb541e91b4f9
size 5280561
oid sha256:a79ceaa44b0c056dc34b15e759757fe3dc470be01530d96a4bc6f44098f73220
size 5503628

3
bin/solid.pill Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:3487ad86985d61dcd3106e7114167d0d6c071ffab645c6c9bfe131ec8bd6dd3c
size 8133778

23
nix/cachix/local.nix Normal file
View File

@ -0,0 +1,23 @@
# All the non-release builds that should be cached in `cachix`.
let
pkgs = import ../pkgs {};
deps = import ../deps {};
# Cache the result of cloning source repos.
repos = {
argon2-src = deps.argon2.src;
ed25519-src = deps.ed25519.src;
h2o-src = deps.h2o.src;
murmur3-src = deps.murmur3.src;
scrypt-src = deps.scrypt.src;
secp256k1-src = deps.secp256k1.src;
sni-src = deps.sni.src;
softfloat3-src = deps.softfloat3.src;
uv-src = deps.uv.src;
};
in
deps // pkgs // repos

11
nix/cachix/release.nix Normal file
View File

@ -0,0 +1,11 @@
let
util = import ./util.nix;
nixcrpkgs = import ../nixcrpkgs.nix;
release = import ../release.nix;
all_releases = util.flattenSetPrefix release;
crosstools = { inherit (nixcrpkgs.native) pkgconf; };
in
crosstools // all_releases

26
nix/cachix/util.nix Normal file
View File

@ -0,0 +1,26 @@
# Some utility functions:
rec {
# The inverse of builtins.listToAttrs
attrsToList = o:
map (a: { name=a; value=builtins.getAttr a o; })
(builtins.attrNames o);
# ∀o,x,y. produce o' such that o'.y == o.x.y (assuming no conflicts)
flattenSet = o:
builtins.foldl' (acc: v: acc // v) {}
(builtins.attrValues o);
prefixSetAttrs = prefix: o:
builtins.listToAttrs
(map ({name, value}: { name=prefix + name; value=value; })
(attrsToList o));
# ∀o,x,y. produce o' such that o'.x-y == o.x.y
flattenSetPrefix = o:
(builtins.foldl' (acc: o: acc // o) {}
(map ({name, value}: prefixSetAttrs name value)
(attrsToList o)));
}

19
nix/dep-repos.nix
View File

@ -1,19 +0,0 @@
# This is a hack to give us access to the fetching of dependency repos
# as nix builds. We use this to cache those Cachix for faster rebuilds
# in CI.
with (import ./deps {});
{
argon2-src = argon2.src;
ed25519-src = ed25519.src;
ent-src = ent.src;
h2o-src = h2o.src;
murmur3-src = murmur3.src;
nodehello-src = nodehello.src;
scrypt-src = scrypt.src;
secp256k1-src = secp256k1.src;
sni-src = sni.src;
softfloat3-src = softfloat3.src;
uv-src = uv.src;
}

2
nix/nixcrpkgs/macos/default.nix
View File

@ -184,7 +184,7 @@ let
global_license_set = { };
# Make it easy to build or refer to the build tools.
inherit clang tapi ld ranlib ar sdk toolchain;
inherit clang tapi ld ranlib ar sdk toolchain strip;
make_derivation = import ../make_derivation.nix crossenv;
};

4
nix/nixcrpkgs/pkgs.nix
View File

@ -20,6 +20,10 @@ rec {
inherit crossenv;
};
lmdb = import ./pkgs/lmdb {
inherit crossenv;
};
ncurses = import ./pkgs/ncurses {
inherit crossenv;
};

16
nix/nixcrpkgs/pkgs/lmdb/builder.sh Normal file
View File

@ -0,0 +1,16 @@
source $setup
cp -r $src ./src
chmod -R u+w ./src
cd src/libraries/liblmdb
sed -i 's/liblmdb.a liblmdb..SOEXT.$/liblmdb.a/' Makefile
sed -i "s/gcc/$host-gcc/" Makefile
sed -i "s/ar/$host-ar/" Makefile
sed -i 's/^CC.*/CC = '"$host-gcc/" Makefile
cat Makefile
make CFLAGS+="-fPIC"
make DESTDIR="$out" prefix=/ install

14
nix/nixcrpkgs/pkgs/lmdb/default.nix Normal file
View File

@ -0,0 +1,14 @@
{ crossenv }:
crossenv.make_derivation rec {
name = "lmdb-${version}";
version = "0.9.23";
builder = ./builder.sh;
src = crossenv.nixpkgs.fetchFromGitHub {
owner = "LMDB";
repo = "lmdb";
rev = "LMDB_${version}";
sha256 = "0ag7l5180ajvm73y59m7sn3p52xm8m972d08cshxhpwgwa4v35k6";
};
}

18
nix/ops/boot-ship.nix Normal file
View File

@ -0,0 +1,18 @@
{ pkgs ? import ../nixpkgs.nix
, debug ? false
, ship ? "zod"
, pill ? ../../bin/solid.pill
}:
let
deps = import ../deps { inherit pkgs; };
tlon = import ../pkgs { inherit pkgs; };
arvo = tlon.arvo;
urbit = tlon.urbit;
in
import ./fakeship {
inherit pkgs tlon deps arvo pill ship debug;
}

21
nix/ops/brass/builder.sh
View File

@ -2,26 +2,21 @@ source $stdenv/setup
set -ex
cp -r $FAKEZOD ./zod
chmod -R u+rw ./zod
cp -r $PIER ./pier
chmod -R u+rw ./pier
urbit -d ./zod
$URBIT -d ./pier
cleanup () {
if [ -e ./zod/.vere.lock ]
then kill $(< ./zod/.vere.lock) || true;
if [ -e ./pier/.vere.lock ]
then kill $(< ./pier/.vere.lock) || true;
fi
}
trap cleanup EXIT
herb ./zod -p hood -d '+hood/autoload |'
herb ./zod -p hood -d "+hood/mount %"
rm -r ./zod/home
cp -r $ARVO ./zod/home
herb ./zod -p hood -d "+hood/commit %home"
herb ./zod -P brass.pill -d '+brass'
herb ./pier -P brass.pill -d '+brass'
mv brass.pill $out
set +x

15
nix/ops/brass/default.nix
View File

@ -1,10 +1,19 @@
{ pkgs, tlon, deps, urbit, fakezod, arvo }:
{ pkgs, tlon, deps, pier, arvo, debug }:
let
urbitExe = if debug
then "${tlon.urbit-debug}/bin/urbit-debug -g"
else "${tlon.urbit}/bin/urbit";
in
pkgs.stdenv.mkDerivation rec {
name = "brass";
builder = ./builder.sh;
buildInputs = [ urbit tlon.herb pkgs.coreutils ];
buildInputs = [ tlon.herb pkgs.coreutils ];
FAKEZOD = fakezod;
URBIT = urbitExe;
PIER = pier;
ARVO = arvo;
}

46
nix/ops/default.nix
View File

@ -1,4 +1,4 @@
{ pkgs ? import ../nixpkgs.nix }:
{ pkgs ? import ../nixpkgs.nix, debug ? false }:
let
@ -7,40 +7,38 @@ let
arvo = tlon.arvo;
urbit = tlon.urbit;
bootbrass = ../../bin/brass.pill;
bootsolid = ../../bin/solid.pill;
zod = import ./fakeship {
inherit pkgs tlon deps arvo debug;
pill = bootsolid;
ship = "zod";
};
bus = import ./fakeship {
inherit pkgs tlon deps arvo debug;
pill = bootsolid;
ship = "bus";
};
in
rec {
bootzod = import ./fakeship {
inherit pkgs tlon deps urbit;
brass = ../../bin/brass.pill;
ship = "zod";
};
bootbus = import ./fakeship {
inherit pkgs tlon deps urbit;
brass = ../../bin/brass.pill;
ship = "bus";
};
test = import ./test {
inherit pkgs tlon deps urbit arvo;
ship = bootzod;
inherit pkgs tlon deps debug;
ship = bus;
};
solid = import ./solid {
inherit arvo pkgs tlon deps urbit;
fakezod = bootzod;
inherit arvo pkgs tlon deps debug;
pier = zod;
};
brass = import ./brass {
inherit arvo pkgs tlon deps urbit;
fakezod = bootzod;
};
fakezod = import ./fakeship {
inherit pkgs tlon deps urbit brass;
ship = "zod";
inherit arvo pkgs tlon deps debug;
pier = zod;
};
}

4
nix/ops/fakeship/builder.sh
View File

@ -2,7 +2,7 @@ source $stdenv/setup
set -ex
urbit -d -F $SHIP -B "$PILL" $out
$URBIT -d -F $SHIP -A "$ARVO" -B "$PILL" $out
check () {
[ 3 -eq "$(herb $out -d 3)" ]
@ -17,3 +17,5 @@ else
kill $(< $out/.vere.lock) || true
exit 1
fi
set +x

18
nix/ops/fakeship/default.nix
View File

@ -1,9 +1,19 @@
{ pkgs, tlon, deps, urbit, brass, ship ? "zod" }:
{ pkgs, tlon, deps, pill, ship, arvo, debug }:
pkgs.stdenv.mkDerivation rec {
let
urbitExe = if debug
then "${tlon.urbit-debug}/bin/urbit-debug -g"
else "${tlon.urbit}/bin/urbit";
in
pkgs.stdenv.mkDerivation {
name = "fake" + ship;
builder = ./builder.sh;
buildInputs = [ urbit tlon.herb ];
PILL = brass;
buildInputs = [ tlon.herb ];
URBIT = urbitExe;
ARVO = arvo;
PILL = pill;
SHIP = ship;
}

25
nix/ops/solid/builder.sh
View File

@ -2,30 +2,21 @@ source $stdenv/setup
set -ex
cp -r $FAKEZOD ./zod
chmod -R u+rw ./zod
cp -r $PIER ./pier
chmod -R u+rw ./pier
urbit -d ./zod
$URBIT -d ./pier
cleanup () {
if [ -e ./zod/.vere.lock ]
then kill $(< ./zod/.vere.lock) || true;
if [ -e ./pier/.vere.lock ]
then kill $(< ./pier/.vere.lock) || true;
fi
}
trap cleanup EXIT
herb ./zod -p hood -d '+hood/autoload |'
herb ./zod -p hood -d "+hood/mount %"
herb ./pier -P solid.pill -d '+solid, =dub &'
rm -r ./zod/home
cp -r $ARVO ./zod/home
mv solid.pill $out
# cp -r $ARVO/sys ./zod/home/
# cp $ARVO/gen/solid.hoon ./zod/home/gen/
# cp $ARVO/lib/pill.hoon ./zod/home/lib/
herb ./zod -p hood -d "+hood/commit %home"
herb ./zod -P urbit.pill -d '+solid, =dub &'
mv urbit.pill $out
set +x

15
nix/ops/solid/default.nix
View File

@ -1,10 +1,19 @@
{ pkgs, tlon, deps, urbit, fakezod, arvo }:
{ pkgs, tlon, deps, pier, arvo, debug }:
let
urbitExe = if debug
then "${tlon.urbit-debug}/bin/urbit-debug -g"
else "${tlon.urbit}/bin/urbit";
in
pkgs.stdenv.mkDerivation rec {
name = "solid";
builder = ./builder.sh;
buildInputs = [ urbit tlon.herb pkgs.coreutils ];
buildInputs = [ tlon.herb ];
FAKEZOD = fakezod;
URBIT = urbitExe;
PIER = pier;
ARVO = arvo;
}

12
nix/ops/test/builder.sh
View File

@ -5,7 +5,7 @@ set -ex
cp -r $SHIP ./ship
chmod -R u+rw ./ship
urbit -d ./ship 2> urbit-output
$URBIT -d ./ship 2> urbit-output
tail -f urbit-output >&2 &
tailproc=$!
@ -20,14 +20,6 @@ shutdown () {
trap shutdown EXIT
herb ./ship -p hood -d '+hood/autoload |'
herb ./ship -p hood -d '+hood/mount %'
rm -r ./ship/home
cp -r $ARVO ./ship/home
herb ./ship -p hood -d '+hood/commit %home'
# Start the test app
herb ./ship -p hood -d '+hood/start %test'
@ -64,3 +56,5 @@ mkdir $out
cp test-renders-output $out/renders
cp test-cores-output $out/cores
cp test-generator-output $out/generator
set +x

14
nix/ops/test/default.nix
View File

@ -1,10 +1,18 @@
{ pkgs, tlon, deps, urbit, ship, arvo }:
{ pkgs, tlon, deps, ship, debug }:
let
urbitExe = if debug
then "${tlon.urbit-debug}/bin/urbit-debug -g"
else "${tlon.urbit}/bin/urbit";
in
pkgs.stdenv.mkDerivation rec {
name = "test";
builder = ./builder.sh;
buildInputs = [ urbit tlon.herb pkgs.coreutils ];
buildInputs = [ tlon.herb ];
URBIT = urbitExe;
SHIP = ship;
ARVO = arvo;
}

2
nix/pkgs/urbit/builder.sh
View File

@ -7,7 +7,7 @@ cd src
bash ./configure
make clean
make urbit urbit-worker -j8
make all -j8
make test
mkdir -p $out/bin

2
nix/pkgs/urbit/default.nix
View File

@ -29,7 +29,7 @@ pkgs.stdenv.mkDerivation {
# See https://github.com/NixOS/nixpkgs/issues/18995
hardeningDisable = if debug then [ "all" ] else [];
CFLAGS = if debug then "-O0 -g" else "-O3";
CFLAGS = if debug then "-O3 -g -Werror" else "-O3 -Werror";
MEMORY_DEBUG = debug;
CPU_DEBUG = debug;
EVENT_TIME_DEBUG = false;

37
nix/release.nix
View File

@ -10,7 +10,6 @@ let
deps = crossdeps env;
};
linux32 = release "linux32" nixcrpkgs.linux32;
linux64 = release "linux64" nixcrpkgs.linux64;
darwin = release "darwin" nixcrpkgs.mac;
@ -21,31 +20,21 @@ let
import ./pkgs/urbit/release.nix env
{ ent = ent env; debug = false; name = "urbit"; };
urbit-debug = env:
import ./pkgs/urbit/release.nix env
{ ent = ent env; debug = true; name = "urbit-debug"; };
builds-for-platform = plat:
plat.deps // {
inherit (plat.env) curl libgmp libsigsegv ncurses openssl zlib lmdb;
inherit (plat.env) cmake_toolchain;
ent = ent plat;
urbit = urbit plat;
};
darwin_extra = {
inherit (darwin.env) ranlib ld sdk ar toolchain tapi strip;
};
in
{
linux32-env = linux32.env;
linux32 = linux32.deps // {
ent = ent linux32;
urbit = urbit linux32;
urbit-debug = urbit-debug linux32;
};
linux64-env = linux64.env;
linux64 = linux64.deps // {
ent = ent linux64;
urbit = urbit linux64;
urbit-debug = urbit-debug linux64;
};
darwin-env = darwin.env;
darwin = darwin.deps // {
ent = ent darwin;
urbit = urbit darwin;
urbit-debug = urbit-debug darwin;
};
linux64 = builds-for-platform linux64;
darwin = darwin_extra // builds-for-platform darwin;
}

2
pkg/arvo

@ -1 +1 @@
Subproject commit 46c92473477ba8f4d14a6e2bb22319c8bf5970a4
Subproject commit 587b4d26df6396a21478a110fc0736df319298a0

6
pkg/urbit/Makefile
View File

@ -17,6 +17,12 @@ all_objs = $(common_objs) $(daemon_objs) $(worker_objs)
all_srcs = $(common) $(daemon) $(worker)
all_exes = ./hash_tests ./hashtable_tests ./urbit ./urbit-worker
# -Werror promotes all warnings that are enabled into errors (this is on)
# -Wall issues all types of errors. This is off (for now)
CFLAGS := $(CFLAGS)
################################################################################
.PHONY: all test clean mkproper

29
pkg/urbit/daemon/main.c
View File

@ -16,6 +16,8 @@
#include <openssl/ssl.h>
#include <openssl/rand.h>
#include <h2o.h>
#include <curl/curl.h>
#include <argon2.h>
#define U3_GLOBAL
#define C3_GLOBAL
@ -441,6 +443,7 @@ u3_ve_sysopt()
static void
report(void)
{
printf("urbit %s\n", URBIT_VERSION);
printf("---------\nLibraries\n---------\n");
printf("gmp: %s\n", gmp_version);
printf("sigsegv: %d.%d\n",
@ -453,6 +456,15 @@ report(void)
H2O_LIBRARY_VERSION_MAJOR,
H2O_LIBRARY_VERSION_MINOR,
H2O_LIBRARY_VERSION_PATCH);
printf("lmdb: %d.%d.%d\n",
MDB_VERSION_MAJOR,
MDB_VERSION_MINOR,
MDB_VERSION_PATCH);
printf("curl: %d.%d.%d\n",
LIBCURL_VERSION_MAJOR,
LIBCURL_VERSION_MINOR,
LIBCURL_VERSION_PATCH);
printf("argon2: 0x%x\n", ARGON2_VERSION_NUMBER);
}
static void
@ -603,7 +615,7 @@ main(c3_i argc,
sigemptyset(&set);
sigaddset(&set, SIGPROF);
if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) {
perror("pthread_sigmask");
u3l_log("boot: thread mask SIGPROF: %s\r\n", strerror(errno));
exit(1);
}
}
@ -717,21 +729,6 @@ main(c3_i argc,
SSL_library_init();
SSL_load_error_strings();
{
c3_i rad;
c3_y buf[4096];
// RAND_status, at least on OS X, never returns true.
// 4096 bytes should be enough entropy for anyone, right?
rad = open("/dev/urandom", O_RDONLY);
if ( 4096 != read(rad, &buf, 4096) ) {
perror("rand-seed");
exit(1);
}
RAND_seed(buf, 4096);
close(rad);
}
u3_daemon_commence();
}
return 0;

18
pkg/urbit/include/config.h.in
View File

@ -1,18 +0,0 @@
#ifndef CONFIG_H
#define CONFIG_H
#mesondefine URBIT_VERSION
#mesondefine U3_OS_linux
#mesondefine U3_OS_bsd
#mesondefine U3_OS_osx
#mesondefine U3_OS_ENDIAN_little
#mesondefine U3_OS_ENDIAN_big
#mesondefine U3_MEMORY_DEBUG
#mesondefine U3_CPU_DEBUG
#mesondefine U3_EVENT_TIME_DEBUG
#mesondefine U3_MEMORY_LOG
#endif /*CONFIG_H*/

26
pkg/urbit/include/noun/vortex.h
View File

@ -4,18 +4,6 @@
*/
/** Data structures.
**/
/* u3_cart: ovum carton.
*/
struct _u3v_arvo;
typedef struct _u3v_cart {
u3_noun vir; // effects of ovum
c3_o did; // cart considered for commit?
c3_o cit; // cart committed?
c3_d ent_d; // event number
u3p(struct _u3v_cart) nex_p;
} u3v_cart;
/* u3v_arvo: modern arvo structure.
*/
typedef struct _u3v_arvo {
@ -28,18 +16,9 @@
u3_noun our; // identity
u3_noun fak; // c3y is fake
u3_noun sac; // space profiling
u3_noun roe; // temporary unsaved events
u3_noun key; // log key, or 0
u3_noun sys; // system pill
u3_noun roc; // kernel core
struct { // ova waiting to process
u3p(u3v_cart) egg_p; // exit of ovum queue
u3p(u3v_cart) geg_p; // entry of ovum queue
} ova;
} u3v_arvo;
/* u3v_home: all internal (within image) state.
@ -145,11 +124,6 @@
void
u3v_plow(u3_noun ova);
/* u3v_hose(): clear initial ovum queue.
*/
void
u3v_hose(void);
/* u3v_mark(): mark arvo kernel.
*/
c3_w

35
pkg/urbit/include/vere/vere.h
View File

@ -603,10 +603,11 @@
struct _u3_pier* pir_u; // backpointer to pier
u3_noun job; // (pair date ovum)
c3_d evt_d; // event number
c3_d rep_d; // replacement count
u3_noun now; // event time
c3_l msc_l; // ms to timeout
c3_l mug_l; // hash before executing
u3_atom mat; // jammed $work, or 0
u3_atom mat; // jammed [mug_l job], or 0
u3_noun act; // action list
struct _u3_writ* nex_u; // next in queue, or 0
} u3_writ;
@ -690,6 +691,7 @@
u3_writ* ent_u; // entry of queue
u3_writ* ext_u; // exit of queue
uv_prepare_t pep_u; // preloop registration
uv_idle_t idl_u; // postloop registration
} u3_pier;
/* u3_king: all executing piers.
@ -1273,6 +1275,10 @@
void
u3_daemon_grab(void* vod_p);
c3_w
u3_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result);
/* Database
*/
/* u3_lmdb_init(): Initializes lmdb inside log_path
@ -1288,11 +1294,29 @@
c3_o u3_lmdb_get_latest_event_number(MDB_env* environment,
c3_d* event_number);
/* u3_lmdb_write_request: opaque write request structures
*/
struct u3_lmdb_write_request;
/* u3_lmdb_build_write_reuqest(): allocates and builds a write request
**
** Reads count sequential writs starting with event_u and creates a
** single write request for all those writs.
*/
struct u3_lmdb_write_request*
u3_lmdb_build_write_request(u3_writ* event_u, c3_d count);
/* u3_lmdb_free_write_request(): frees a write requst
*/
void u3_lmdb_free_write_request(struct u3_lmdb_write_request* request);
/* u3_lmdb_write_event(): Persists an event to the database
*/
void u3_lmdb_write_event(MDB_env* environment,
u3_writ* event_u,
void (*on_complete)(c3_o success, u3_writ*));
u3_pier* pir_u,
struct u3_lmdb_write_request* request_u,
void (*on_complete)(c3_o success, u3_pier*,
c3_d, c3_d));
/* u3_lmdb_read_events(): Reads events back from the database
**
@ -1307,8 +1331,7 @@
c3_d len_d,
c3_o(*on_event_read)(u3_pier* pir_u,
c3_d id,
u3_noun mat,
u3_noun ovo));
u3_noun mat));
/* u3_lmdb_write_identity(): Writes log identity
**
@ -1327,3 +1350,5 @@
u3_noun* who,
u3_noun* is_fake,
u3_noun* life);

2
pkg/urbit/noun/allocate.c
View File

@ -1851,7 +1851,7 @@ u3a_sweep(void)
weq_w += box_u->siz_w;
}
else {
_ca_print_leak("weak", box_u, box_u->eus_w, box_u->use_w);
_ca_print_leak("leak", box_u, box_u->eus_w, box_u->use_w);
leq_w += box_u->siz_w;
}

42
pkg/urbit/noun/events.c
View File

@ -155,7 +155,7 @@ u3e_fault(void* adr_v, c3_i ser_i)
(1 << (u3a_page + 2)),
(PROT_READ | PROT_WRITE)) )
{
perror("mprotect");
u3l_log("loom: fault mprotect: %s\r\n", strerror(errno));
c3_assert(0);
return 0;
}
@ -182,14 +182,14 @@ _ce_image_open(u3e_image* img_u)
snprintf(ful_c, 8192, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c);
if ( -1 == (img_u->fid_i = open(ful_c, mod_i, 0666)) ) {
perror(ful_c);
u3l_log("loom: open %s: %s\r\n", ful_c, strerror(errno));
return c3n;
}
else {
struct stat buf_u;
if ( -1 == fstat(img_u->fid_i, &buf_u) ) {
perror(ful_c);
u3l_log("loom: stat %s: %s\r\n", ful_c, strerror(errno));
c3_assert(0);
return c3n;
}
@ -273,13 +273,13 @@ _ce_patch_create(u3_ce_patch* pat_u)
snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c);
if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) {
perror(ful_c);
u3l_log("loom: patch open control.bin: %s\r\n", strerror(errno));
c3_assert(0);
}
snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c);
if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) {
perror(ful_c);
u3l_log("loom: patch open memory.bin: %s\r\n", strerror(errno));
c3_assert(0);
}
}
@ -311,12 +311,12 @@ _ce_patch_verify(u3_ce_patch* pat_u)
c3_w mem_w[1 << u3a_page];
if ( -1 == lseek(pat_u->mem_i, (i_w << (u3a_page + 2)), SEEK_SET) ) {
perror("seek");
u3l_log("loom: patch seek: %s\r\n", strerror(errno));
c3_assert(0);
return c3n;
}
if ( -1 == read(pat_u->mem_i, mem_w, (1 << (u3a_page + 2))) ) {
perror("read");
u3l_log("loom: patch read: %s\r\n", strerror(errno));
c3_assert(0);
return c3n;
}
@ -633,12 +633,22 @@ _ce_patch_apply(u3_ce_patch* pat_u)
//u3l_log("image: sou_w %d, new %d\r\n", u3P.sou_u.pgs_w, pat_u->con_u->sou_w);
if ( u3P.nor_u.pgs_w > pat_u->con_u->nor_w ) {
ftruncate(u3P.nor_u.fid_i, u3P.nor_u.pgs_w << (u3a_page + 2));
c3_w ret_w;
ret_w = ftruncate(u3P.nor_u.fid_i, u3P.nor_u.pgs_w << (u3a_page + 2));
if (ret_w){
u3l_log("loom: patch apply truncate north: %s\r\n", strerror(errno));
c3_assert(0);
}
}
u3P.nor_u.pgs_w = pat_u->con_u->nor_w;
if ( u3P.sou_u.pgs_w > pat_u->con_u->sou_w ) {
ftruncate(u3P.sou_u.fid_i, u3P.sou_u.pgs_w << (u3a_page + 2));
c3_w ret_w;
ret_w = ftruncate(u3P.sou_u.fid_i, u3P.sou_u.pgs_w << (u3a_page + 2));
if (ret_w){
u3l_log("loom: patch apply truncate south: %s\r\n", strerror(errno));
c3_assert(0);
}
}
u3P.sou_u.pgs_w = pat_u->con_u->sou_w;
@ -646,7 +656,7 @@ _ce_patch_apply(u3_ce_patch* pat_u)
(-1 == lseek(u3P.nor_u.fid_i, 0, SEEK_SET)) ||
(-1 == lseek(u3P.sou_u.fid_i, 0, SEEK_SET)) )
{
perror("apply: seek");
u3l_log("loom: patch apply seek 0: %s\r\n", strerror(errno));
c3_assert(0);
}
@ -666,16 +676,16 @@ _ce_patch_apply(u3_ce_patch* pat_u)
}
if ( -1 == read(pat_u->mem_i, mem_w, (1 << (u3a_page + 2))) ) {
perror("apply: read");
u3l_log("loom: patch apply read: %s\r\n", strerror(errno));
c3_assert(0);
}
else {
if ( -1 == lseek(fid_i, (off_w << (u3a_page + 2)), SEEK_SET) ) {
perror("apply: lseek");
u3l_log("loom: patch apply seek: %s\r\n", strerror(errno));
c3_assert(0);
}
if ( -1 == write(fid_i, mem_w, (1 << (u3a_page + 2))) ) {
perror("apply: write");
u3l_log("loom: patch apply write: %s\r\n", strerror(errno));
c3_assert(0);
}
}
@ -697,7 +707,7 @@ _ce_image_blit(u3e_image* img_u,
lseek(img_u->fid_i, 0, SEEK_SET);
for ( i_w=0; i_w < img_u->pgs_w; i_w++ ) {
if ( -1 == read(img_u->fid_i, ptr_w, (1 << (u3a_page + 2))) ) {
perror("read");
u3l_log("loom: image blit read: %s\r\n", strerror(errno));
c3_assert(0);
}
#if 0
@ -729,7 +739,7 @@ _ce_image_fine(u3e_image* img_u,
c3_w mem_w, fil_w;
if ( -1 == read(img_u->fid_i, buf_w, (1 << (u3a_page + 2))) ) {
perror("read");
u3l_log("loom: image fine read: %s\r\n", strerror(errno));
c3_assert(0);
}
mem_w = u3r_mug_words(ptr_w, (1 << u3a_page));
@ -872,7 +882,7 @@ u3e_live(c3_o nuu_o, c3_c* dir_c)
-(1 << u3a_page));
if ( 0 != mprotect((void *)u3_Loom, u3a_bytes, PROT_READ) ) {
perror("protect");
u3l_log("loom: live mprotect: %s\r\n", strerror(errno));
c3_assert(0);
}

4
pkg/urbit/noun/imprison.c
View File

@ -478,7 +478,7 @@ u3i_list(u3_weak one, ...);
return cut_t ? cut_w : i_w;
}
__attribute__((no_sanitize("address")))
__attribute__((no_sanitize_address))
static u3_noun // transfer
_molt_apply(u3_noun som, // retain
c3_w len_w,
@ -504,7 +504,7 @@ u3i_list(u3_weak one, ...);
}
}
__attribute__((no_sanitize("address")))
__attribute__((no_sanitize_address))
u3_noun
u3i_molt(u3_noun som, ...)
{

30
pkg/urbit/noun/manage.c
View File

@ -94,19 +94,28 @@ _cm_punt(u3_noun tax)
}
#endif
static void _write(int fd, const void *buf, size_t count)
{
if (count != write(fd, buf, count)){
u3l_log("write failed\r\n");
c3_assert(0);
}
}
/* _cm_emergency(): write emergency text to stderr, never failing.
*/
static void
_cm_emergency(c3_c* cap_c, c3_l sig_l)
{
write(2, "\r\n", 2);
write(2, cap_c, strlen(cap_c));
_write(2, "\r\n", 2);
_write(2, cap_c, strlen(cap_c));
if ( sig_l ) {
write(2, ": ", 2);
write(2, &sig_l, 4);
_write(2, ": ", 2);
_write(2, &sig_l, 4);
}
write(2, "\r\n", 2);
_write(2, "\r\n", 2);
}
static void _cm_overflow(void *arg1, void *arg2, void *arg3)
@ -1492,7 +1501,7 @@ _cm_limits(void)
? (65536 << 10)
: rlm.rlim_max;
if ( 0 != setrlimit(RLIMIT_STACK, &rlm) ) {
perror("stack");
u3l_log("boot: stack size: %s\r\n", strerror(errno));
exit(1);
}
}
@ -1504,7 +1513,7 @@ _cm_limits(void)
c3_assert(0 == ret_i);
rlm.rlim_cur = 10240; // default OSX max, not in rlim_max irritatingly
if ( 0 != setrlimit(RLIMIT_NOFILE, &rlm) ) {
// perror("file limit");
u3l_log("boot: open file limit: %s\r\n", strerror(errno));
// no exit, not a critical limit
}
}
@ -1515,7 +1524,7 @@ _cm_limits(void)
getrlimit(RLIMIT_CORE, &rlm);
rlm.rlim_cur = RLIM_INFINITY;
if ( 0 != setrlimit(RLIMIT_CORE, &rlm) ) {
perror("core limit");
u3l_log("boot: core limit: %s\r\n", strerror(errno));
// no exit, not a critical limit
}
}
@ -1527,7 +1536,7 @@ static void
_cm_signals(void)
{
if ( 0 != sigsegv_install_handler(u3e_fault) ) {
u3l_log("sigsegv install failed\n");
u3l_log("boot: sigsegv install failed\n");
exit(1);
}
// signal(SIGINT, _loom_stop);
@ -1543,7 +1552,7 @@ _cm_signals(void)
sigaddset(&set, SIGPROF);
if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) {
perror("pthread_sigmask");
u3l_log("boot: thread mask SIGPROF: %s\r\n", strerror(errno));
exit(1);
}
}
@ -1902,7 +1911,6 @@ u3m_boot_new(c3_c* dir_c)
/* Reactivate jets on old kernel.
*/
if ( !_(nuu_o) ) {
u3v_hose();
u3j_ream();
u3n_ream();

2
pkg/urbit/noun/retrieve.c
View File

@ -161,7 +161,7 @@ u3r_at(u3_atom a, u3_noun b)
}
}
__attribute__((no_sanitize("address")))
__attribute__((no_sanitize_address))
c3_o
u3r_mean(u3_noun som,
...)

6
pkg/urbit/noun/trace.c
View File

@ -167,7 +167,7 @@ _t_samp_process(u3_road* rod_u)
pef = t_pef;
}
// fprintf(stderr, "sample: stack length %d\r\n", u3kb_lent(u3k(pal)));
// u3l_log("sample: stack length %d\r\n", u3kb_lent(u3k(pal)));
return pal;
}
}
@ -536,7 +536,7 @@ u3t_boot(void)
sigemptyset(&set);
sigaddset(&set, SIGPROF);
if ( 0 != pthread_sigmask(SIG_UNBLOCK, &set, NULL) ) {
perror("pthread_sigmask");
u3l_log("trace: thread mask SIGPROF: %s\r\n", strerror(errno));
}
}
@ -569,7 +569,7 @@ u3t_boff(void)
sigemptyset(&set);
sigaddset(&set, SIGPROF);
if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) {
perror("pthread_sigmask");
u3l_log("trace: thread mask SIGPROF: %s\r\n", strerror(errno));
}
}

97
pkg/urbit/noun/vortex.c
View File

@ -159,50 +159,6 @@ u3v_boot_lite(u3_atom lit)
u3z(pru);
}
/* u3v_hose(): clear initial ovum queue.
*/
void
u3v_hose(void)
{
u3p(u3v_cart) egg_p = u3A->ova.egg_p;
while ( egg_p ) {
u3v_cart* egg_u = u3to(u3v_cart, egg_p);
u3p(u3v_cart) nex_p = egg_u->nex_p;
u3a_lose(egg_u->vir);
u3a_free(egg_u);
egg_p = nex_p;
}
u3A->ova.egg_p = u3A->ova.geg_p = 0;
u3z(u3A->roe);
u3A->roe = u3_nul;
}
// XX deprecated, remove
#if 0
/* u3v_start(): start time.
*/
void
u3v_start(u3_noun now)
{
u3v_time(now);
u3v_numb();
u3A->sac = u3_nul;
{
c3_c* wen_c = u3r_string(u3A->wen);
u3l_log("arvo: time: %s\n", wen_c);
free(wen_c);
}
if ( u3C.wag_w & u3o_trace )
u3t_trace_open();
}
#endif
/* u3v_wish(): text expression with cache.
*/
u3_noun
@ -485,56 +441,6 @@ u3v_sway(u3_noun blu, c3_l tab_l, u3_noun tax)
u3z(mok);
}
// XX deprecated, remove
#if 0
/* u3v_plan(): queue ovum (external).
*/
void
u3v_plan(u3_noun pax, u3_noun fav)
{
u3_noun egg = u3nc(pax, fav);
u3A->roe = u3nc(u3nc(u3_nul, egg), u3A->roe);
}
#endif
// XX deprecated, remove
#if 0
/* u3v_plow(): queue multiple ova (external).
*/
void
u3v_plow(u3_noun ova)
{
u3_noun ovi = ova;
while ( u3_nul != ovi ) {
u3_noun ovo=u3h(ovi);
u3v_plan(u3k(u3h(ovo)), u3k(u3t(ovo)));
ovi = u3t(ovi);
}
u3z(ova);
}
#endif
/* _cv_mark_ova(): mark ova queue.
*/
c3_w
_cv_mark_ova(u3p(u3v_cart) egg_p)
{
c3_w tot_w = 0;
while ( egg_p ) {
u3v_cart* egg_u = u3to(u3v_cart, egg_p);
tot_w += u3a_mark_mptr(egg_u);
tot_w += u3a_mark_noun(egg_u->vir);
egg_p = egg_u->nex_p;
}
return tot_w;
}
/* u3v_mark(): mark arvo kernel.
*/
c3_w
@ -549,9 +455,6 @@ u3v_mark(FILE* fil_u)
tot_w += u3a_maid(fil_u, " instance string", u3a_mark_noun(arv_u->sen));
tot_w += u3a_maid(fil_u, " identity", u3a_mark_noun(arv_u->our));
tot_w += u3a_maid(fil_u, " fake", u3a_mark_noun(arv_u->fak));
tot_w += u3a_maid(fil_u, " pending events", u3a_mark_noun(arv_u->roe));
tot_w += u3a_maid(fil_u, " event-log key", u3a_mark_noun(arv_u->key));
tot_w += u3a_maid(fil_u, " kernel", u3a_mark_noun(arv_u->roc));
tot_w += u3a_maid(fil_u, " egg basket", _cv_mark_ova(arv_u->ova.egg_p));
return u3a_maid(fil_u, "total arvo stuff", tot_w);
}

4
pkg/urbit/tests/hashtable_tests.c
View File

@ -6,6 +6,10 @@ static void _test_cache_trimming(void);
static void _test_no_cache(void);
static void _test_skip_slot(void);
// defined in noun/hashtable.c
c3_w _ch_skip_slot(c3_w mug_w, c3_w lef_w);
/* main(): run all test cases.
*/
int

8
pkg/urbit/vere/dawn.c
View File

@ -197,7 +197,7 @@ _dawn_fail(u3_noun who, u3_noun rac, u3_noun sas)
}
}
u3l_log("dawn: invalid keys for %s '%s'\r\n", rac_c, how_c);
u3l_log("boot: invalid keys for %s '%s'\r\n", rac_c, how_c);
// XX deconstruct sas, print helpful error messages
u3m_p("pre-boot error", u3t(sas));
@ -306,20 +306,22 @@ u3_dawn_vent(u3_noun seed)
// load snapshot from file
//
if ( 0 != u3_Host.ops_u.ets_c ) {
u3l_log("boot: loading ethereum snapshot\r\n");
u3l_log("boot: loading azimuth snapshot\r\n");
u3_noun raw_snap = u3ke_cue(u3m_file(u3_Host.ops_u.ets_c));
sap = u3nc(u3_nul, raw_snap);
}
// load snapshot from HTTP URL
//
else if ( 0 != u3_Host.ops_u.sap_c ) {
u3l_log("boot: downloading azimuth snapshot from %s\r\n",
u3_Host.ops_u.sap_c);
u3_noun raw_snap = _dawn_get_jam(u3_Host.ops_u.sap_c);
sap = u3nc(u3_nul, raw_snap);
}
// no snapshot
//
else {
u3l_log("dawn: no ethereum snapshot specified\n");
u3l_log("boot: no azimuth snapshot specified\n");
sap = u3_nul;
}

2
pkg/urbit/vere/http.c
View File

@ -461,7 +461,7 @@ _http_rec_accept(h2o_handler_t* han_u, h2o_req_t* rec_u)
req_u->tim_u = c3_malloc(sizeof(*req_u->tim_u));
req_u->tim_u->data = req_u;
uv_timer_init(u3L, req_u->tim_u);
uv_timer_start(req_u->tim_u, _http_req_timer_cb, 300 * 1000, 0);
uv_timer_start(req_u->tim_u, _http_req_timer_cb, 900 * 1000, 0);
_http_req_dispatch(req_u, req);
}

164
pkg/urbit/vere/lmdb.c
View File

@ -41,9 +41,11 @@ MDB_env* u3_lmdb_init(const char* log_path)
return 0;
}
// TODO: Start with a gigabyte for the event log.
// TODO: Start with forty gigabytes for the maximum event log size. We'll
// need to do something more sophisticated for real in the long term, though.
//
ret_w = mdb_env_set_mapsize(env, 1024 * 1024 * 1024);
const size_t forty_gigabytes = 42949672960;
ret_w = mdb_env_set_mapsize(env, forty_gigabytes);
if (ret_w != 0) {
u3l_log("lmdb: failed to set database size: %s\n", mdb_strerror(ret_w));
return 0;
@ -174,6 +176,67 @@ c3_o _perform_get_on_database_noun(MDB_txn* transaction_u,
return c3y;
}
/* u3_lmdb_write_request: Events to be written together
*/
struct u3_lmdb_write_request {
// The event number of the first event.
c3_d first_event;
// The number of events in this write request. Nonzero.
c3_d event_count;
// An array of serialized event datas. The array size is |event_count|. We
// perform the event serialization on the main thread so we can read the loom
// and write into a malloced structure for the worker thread.
void** malloced_event_data;
// An array of sizes of serialized event datas. We keep track of this for the
// database write.
size_t* malloced_event_data_size;
};
/* u3_lmdb_build_write_request(): Allocates and builds a write request
*/
struct u3_lmdb_write_request*
u3_lmdb_build_write_request(u3_writ* event_u, c3_d count)
{
struct u3_lmdb_write_request* request =
c3_malloc(sizeof(struct u3_lmdb_write_request));
request->first_event = event_u->evt_d;
request->event_count = count;
request->malloced_event_data = c3_malloc(sizeof(void*) * count);
request->malloced_event_data_size = c3_malloc(sizeof(size_t) * count);
for (c3_d i = 0; i < count; ++i) {
// Sanity check that the events in u3_writ are in order.
c3_assert(event_u->evt_d == (request->first_event + i));
// Serialize the jammed event log entry into a malloced buffer we can send
// to the other thread.
c3_w siz_w = u3r_met(3, event_u->mat);
c3_y* data_u = c3_calloc(siz_w);
u3r_bytes(0, siz_w, data_u, event_u->mat);
request->malloced_event_data[i] = data_u;
request->malloced_event_data_size[i] = siz_w;
event_u = event_u->nex_u;
}
return request;
}
/* u3_lmdb_free_write_request(): Frees a write request
*/
void u3_lmdb_free_write_request(struct u3_lmdb_write_request* request) {
for (c3_d i = 0; i < request->event_count; ++i)
free(request->malloced_event_data[i]);
free(request->malloced_event_data);
free(request->malloced_event_data_size);
free(request);
}
/* _write_request_data: callback struct for u3_lmdb_write_event()
*/
struct _write_request_data {
@ -181,27 +244,17 @@ struct _write_request_data {
// the transactions and handles opened from it are explicitly not.
MDB_env* environment;
// The original event. Not to be accessed from the worker thread; only used
// in the callback executed on the main loop thread.
u3_writ* event;
// The pier that we're writing for.
u3_pier* pir_u;
// The event number from event separated out so we can access it on the other
// thread.
c3_d event_number;
// The event serialized out of the loom into a malloced structure accessible
// from the worker thread.
void* malloced_event_data;
// The size of the malloced_event_data. We keep track of this for the
// database write.
size_t malloced_event_data_size;
// The encapsulated request. This may contain multiple event writes.
struct u3_lmdb_write_request* request;
// Whether the write completed successfully.
c3_o success;
// Called on main loop thread on completion.
void (*on_complete)(c3_o, u3_writ*);
void (*on_complete)(c3_o, u3_pier*, c3_d, c3_d);
};
/* _u3_lmdb_write_event_cb(): Implementation of u3_lmdb_write_event()
@ -235,23 +288,42 @@ static void _u3_lmdb_write_event_cb(uv_work_t* req) {
return;
}
// TODO: We need to detect the database being full, making the database
// maxsize larger, and then retrying this transaction.
//
struct u3_lmdb_write_request* request = data->request;
for (c3_d i = 0; i < request->event_count; ++i) {
c3_d event_number = request->first_event + i;
c3_o success = _perform_put_on_database_raw(
transaction_u,
database_u,
MDB_NOOVERWRITE,
&(data->event_number),
&event_number,
sizeof(c3_d),
data->malloced_event_data,
data->malloced_event_data_size);
request->malloced_event_data[i],
request->malloced_event_data_size[i]);
if (success == c3n) {
u3l_log("lmdb: failed to write event %" PRIu64 "\n", event_number);
mdb_txn_abort(transaction_u);
data->success = c3n;
return;
}
}
ret_w = mdb_txn_commit(transaction_u);
if (0 != ret_w) {
if ( request->event_count == 1 ) {
u3l_log("lmdb: failed to commit event %" PRIu64 ": %s\n",
data->event_number,
request->first_event,
mdb_strerror(ret_w));
} else {
c3_d through = request->first_event + request->event_count - 1ULL;
u3l_log("lmdb: failed to commit events %" PRIu64 " through %" PRIu64
": %s\n",
request->first_event,
through,
mdb_strerror(ret_w));
}
data->success = c3n;
return;
}
@ -266,9 +338,12 @@ static void _u3_lmdb_write_event_cb(uv_work_t* req) {
static void _u3_lmdb_write_event_after_cb(uv_work_t* req, int status) {
struct _write_request_data* data = req->data;
data->on_complete(data->success, data->event);
data->on_complete(data->success,
data->pir_u,
data->request->first_event,
data->request->event_count);
free(data->malloced_event_data);
u3_lmdb_free_write_request(data->request);
free(data);
free(req);
}
@ -278,27 +353,17 @@ static void _u3_lmdb_write_event_after_cb(uv_work_t* req, int status) {
** This writes all the passed in events along with log metadata updates to the
** database as a single transaction on a worker thread. Once the transaction
** is completed, it calls the passed in callback on the main loop thread.
**
** TODO: Make this take multiple events in one commit once we have this
** working one at a time.
*/
void u3_lmdb_write_event(MDB_env* environment,
u3_writ* event_u,
void (*on_complete)(c3_o, u3_writ*))
u3_pier* pir_u,
struct u3_lmdb_write_request* request_u,
void (*on_complete)(c3_o, u3_pier*, c3_d, c3_d))
{
// Serialize the jammed $work into a malloced buffer we can send to the other
// thread.
c3_w siz_w = u3r_met(3, event_u->mat);
c3_y* data_u = c3_calloc(siz_w);
u3r_bytes(0, siz_w, data_u, event_u->mat);
// Structure to pass to the worker thread.
struct _write_request_data* data = c3_malloc(sizeof(struct _write_request_data));
data->environment = environment;
data->event = event_u;
data->event_number = event_u->evt_d;
data->malloced_event_data = data_u;
data->malloced_event_data_size = siz_w;
data->pir_u = pir_u;
data->request = request_u;
data->on_complete = on_complete;
data->success = c3n;
@ -324,7 +389,7 @@ c3_o u3_lmdb_read_events(u3_pier* pir_u,
c3_d first_event_d,
c3_d len_d,
c3_o(*on_event_read)(u3_pier* pir_u, c3_d id,
u3_noun mat, u3_noun ovo))
u3_noun mat))
{
// Creates the read transaction.
MDB_txn* transaction_u;
@ -377,24 +442,27 @@ c3_o u3_lmdb_read_events(u3_pier* pir_u,
// As a sanity check, we make sure that there aren't any discontinuities in
// the sequence of loaded events.
c3_d current_id = first_event_d + loaded;
if (key.mv_size != sizeof(c3_d) ||
*(c3_d*)key.mv_data != current_id) {
if (key.mv_size != sizeof(c3_d)) {
u3l_log("lmdb: invalid cursor key\r\n");
return c3n;
}
if (*(c3_d*)key.mv_data != current_id) {
u3l_log("lmdb: missing event in database. Expected %" PRIu64 ", received %"
PRIu64 "\r\n",
current_id,
*(c3_d*)key.mv_data);
return c3n;
}
// Now build the atom version and then the cued version from the raw data
u3_noun mat = u3i_bytes(val.mv_size, val.mv_data);
u3_noun ovo = u3ke_cue(u3k(mat));
if (on_event_read(pir_u, current_id, mat, ovo) == c3n) {
u3z(ovo);
if (on_event_read(pir_u, current_id, mat) == c3n) {
u3z(mat);
u3l_log("lmdb: aborting replay due to error.\r\n");
return c3n;
}
u3z(ovo);
u3z(mat);
ret_w = mdb_cursor_get(cursor_u, &key, &val, MDB_NEXT);

248
pkg/urbit/vere/pier.c
View File

@ -69,6 +69,7 @@ static void _pier_boot_complete(u3_pier* pir_u);
static void _pier_boot_ready(u3_pier* pir_u);
static void _pier_boot_set_ship(u3_pier* pir_u, u3_noun who, u3_noun fak);
static void _pier_exit_done(u3_pier* pir_u);
static void _pier_loop_resume(u3_pier* pir_u);
/* _pier_db_bail(): bail from disk i/o.
*/
@ -89,9 +90,11 @@ _pier_db_shutdown(u3_pier* pir_u)
/* _pier_db_commit_complete(): commit complete.
*/
static void
_pier_db_commit_complete(c3_o success, u3_writ* wit_u)
_pier_db_commit_complete(c3_o success,
u3_pier* pir_u,
c3_d first_event_d,
c3_d event_count_d)
{
u3_pier* pir_u = wit_u->pir_u;
u3_disk* log_u = pir_u->log_u;
if (success == c3n) {
@ -100,45 +103,58 @@ _pier_db_commit_complete(c3_o success, u3_writ* wit_u)
}
#ifdef VERBOSE_EVENTS
u3l_log("pier: (%" PRIu64 "): db commit completed\r\n", wit_u->evt_d);
if (event_count_d != 1) {
u3l_log("pier: (%" PRIu64 "-%" PRIu64 "): db commit: complete\r\n",
first_event_d, first_event_d + event_count_d - 1ULL);
} else {
u3l_log("pier: (%" PRIu64 "): db commit: complete\r\n", first_event_d);
}
#endif
/* advance commit counter
*/
{
c3_assert(wit_u->evt_d == log_u->moc_d);
c3_assert(wit_u->evt_d == (1ULL + log_u->com_d));
log_u->com_d += 1ULL;
c3_assert((first_event_d + event_count_d - 1ULL) == log_u->moc_d);
c3_assert(first_event_d == (1ULL + log_u->com_d));
log_u->com_d += event_count_d;
}
_pier_apply(pir_u);
_pier_loop_resume(pir_u);
}
/* _pier_db_commit_request(): start commit.
*/
static void
_pier_db_commit_request(u3_writ* wit_u)
_pier_db_commit_request(u3_pier* pir_u,
struct u3_lmdb_write_request* request_u,
c3_d first_event_d,
c3_d count_d)
{
u3_pier* pir_u = wit_u->pir_u;
u3_disk* log_u = pir_u->log_u;
#ifdef VERBOSE_EVENTS
u3l_log("pier: (%" PRIu64 "): commit: request\r\n", wit_u->evt_d);
if (count_d != 1) {
u3l_log("pier: (%" PRIu64 "-%" PRIu64 "): db commit: request\r\n",
first_event_d, first_event_d + count_d - 1ULL);
} else {
u3l_log("pier: (%" PRIu64 "): db commit: request\r\n", first_event_d);
}
#endif
/* put it in the database
*/
{
u3_lmdb_write_event(log_u->db_u,
wit_u,
pir_u,
request_u,
_pier_db_commit_complete);
}
/* advance commit-request counter
*/
{
c3_assert(wit_u->evt_d == (1ULL + log_u->moc_d));
log_u->moc_d += 1ULL;
c3_assert(first_event_d == (1ULL + log_u->moc_d));
log_u->moc_d += count_d;
}
}
@ -180,25 +196,30 @@ _pier_db_read_header(u3_pier* pir_u)
static c3_o
_pier_db_on_commit_loaded(u3_pier* pir_u,
c3_d id,
u3_noun mat,
u3_noun ovo)
u3_noun mat)
{
u3_noun evt = u3h(u3t(ovo));
u3_noun job = u3k(u3t(u3t(u3t(ovo))));
c3_d evt_d = u3r_chub(0, evt);
if (evt_d != id) {
_pier_db_bail(0, "pier: load: commit: event order");
return c3n;
}
// Need to grab references to the nouns above.
u3_writ* wit_u = c3_calloc(sizeof(u3_writ));
wit_u->pir_u = pir_u;
wit_u->evt_d = evt_d;
wit_u->job = u3k(job);
wit_u->evt_d = id;
wit_u->mat = u3k(mat);
// Parse the expected mug_l and job out of mat.
u3_noun entry = u3ke_cue(u3k(mat));
u3_noun mug, job;
if ( (c3y != u3du(entry)) ||
(c3n == u3r_cell(entry, &mug, &job)) ||
(c3n == u3ud(mug)) ||
(1 < u3r_met(5, mug)) ) {
u3l_log("pier: load: event %" PRIu64 " malformed.\r\n", id);
return c3n;
}
wit_u->mug_l = u3r_word(0, mug);
wit_u->job = u3k(job);
u3z(entry);
// Insert at queue front since we're loading events in order
if ( !pir_u->ent_u ) {
c3_assert(!pir_u->ext_u);
@ -515,9 +536,7 @@ _pier_work_build(u3_writ* wit_u)
if ( 0 == wit_u->mat ) {
c3_assert(0 != wit_u->job);
wit_u->mat = u3ke_jam(u3nq(c3__work,
u3i_chubs(1, &wit_u->evt_d),
wit_u->mug_l,
wit_u->mat = u3ke_jam(u3nc(wit_u->mug_l,
u3k(wit_u->job)));
}
}
@ -532,7 +551,11 @@ _pier_work_send(u3_writ* wit_u)
c3_assert(0 != wit_u->mat);
u3_newt_write(&god_u->inn_u, u3k(wit_u->mat), wit_u);
u3_noun msg = u3ke_jam(u3nt(c3__work,
u3i_chubs(1, &wit_u->evt_d),
u3k(wit_u->mat)));
u3_newt_write(&god_u->inn_u, msg, wit_u);
}
/* _pier_work_save(): tell worker to save checkpoint.
@ -662,8 +685,7 @@ _pier_work_complete(u3_writ* wit_u,
*/
static void
_pier_work_replace(u3_writ* wit_u,
u3_noun job,
u3_noun mat)
u3_noun job)
{
u3_pier* pir_u = wit_u->pir_u;
u3_controller* god_u = pir_u->god_u;
@ -674,6 +696,12 @@ _pier_work_replace(u3_writ* wit_u,
c3_assert(god_u->sen_d == wit_u->evt_d);
// something has gone very wrong, we should probably stop now
//
if ( wit_u->rep_d >= 3ULL ) {
u3_pier_bail();
}
/* move backward in work processing
*/
{
@ -681,9 +709,12 @@ _pier_work_replace(u3_writ* wit_u,
wit_u->job = job;
u3z(wit_u->mat);
wit_u->mat = mat;
wit_u->mat = u3ke_jam(u3nc(wit_u->mug_l,
u3k(wit_u->job)));
god_u->sen_d -= 1;
wit_u->rep_d += 1ULL;
god_u->sen_d -= 1ULL;
}
if ( wit_u->evt_d > pir_u->lif_d ) {
@ -736,6 +767,7 @@ _pier_work_play(u3_pier* pir_u,
// all events in the worker are complete
//
god_u->rel_d = god_u->dun_d = god_u->sen_d = (lav_d - 1ULL);
god_u->mug_l = mug_l;
_pier_boot_ready(pir_u);
}
@ -873,39 +905,35 @@ _pier_work_poke(void* vod_p,
}
case c3__work: {
if ( (c3n == u3r_qual(jar, 0, &p_jar, &q_jar, &r_jar)) ||
if ( (c3n == u3r_trel(jar, 0, &p_jar, &q_jar)) ||
(c3n == u3ud(p_jar)) ||
(u3r_met(6, p_jar) != 1) ||
(c3n == u3ud(q_jar)) ||
(u3r_met(5, q_jar) > 1) )
(u3r_met(6, p_jar) != 1) )
{
u3l_log("failed to parse replacement atom");
goto error;
}
else {
c3_d evt_d = u3r_chub(0, p_jar);
c3_l mug_l = u3r_word(0, q_jar);
u3_writ* wit_u = _pier_writ_find(pir_u, evt_d);
if ( !wit_u || (mug_l && (mug_l != wit_u->mug_l)) ) {
u3_noun mug, job;
u3_noun entry = u3ke_cue(u3k(q_jar));
if ( (c3y != u3du(entry)) ||
(c3n == u3r_cell(entry, &mug, &job)) ||
(c3n == u3ud(mug)) ||
(1 < u3r_met(5, mug)) ) {
goto error;
}
{
// XX not the right place to print an error!
//
#if 0
u3m_p("wire", u3h(u3t(r_jar)));
u3m_p("oust", u3h(u3t(u3t(wit_u->job))));
u3m_p("with", u3h(u3t(u3t(r_jar))));
if ( c3__crud == u3h(u3t(u3t(r_jar))) ) {
u3_pier_punt(0, u3k(u3t(u3t(u3t(u3t(r_jar))))));
}
#endif
c3_l mug_l = u3r_word(0, mug);
if ( !wit_u || (mug_l && (mug_l != wit_u->mug_l)) ) {
goto error;
}
#ifdef VERBOSE_EVENTS
fprintf(stderr, "pier: replace: %" PRIu64 "\r\n", evt_d);
#endif
_pier_work_replace(wit_u, u3k(r_jar), u3k(mat));
_pier_work_replace(wit_u, u3k(job));
}
break;
}
@ -961,16 +989,14 @@ _pier_work_poke(void* vod_p,
goto error;
}
else {
// XXX: The wit_u pointer will almost always be 0 because of how the
// worker process manages the difference between u3V.evt_d vs
// u3A->ent_d. Either stop communicating the evt_d in the wire protocol
// or fix the worker to keep track of and communicate the correct event
// number.
c3_d evt_d = u3r_chub(0, p_jar);
c3_w pri_w = u3r_word(0, q_jar);
u3_writ* wit_u = _pier_writ_find(pir_u, evt_d);
// Only print this slog if the event is uncommitted.
// skip slog during replay
//
// XX also update the worker to skip *sending* the slog during replay
//
if ( u3_psat_pace != pir_u->sat_e ) {
_pier_work_slog(wit_u, pri_w, u3k(r_jar));
}
@ -980,7 +1006,7 @@ _pier_work_poke(void* vod_p,
}
u3z(jar); u3z(mat);
_pier_apply(pir_u);
_pier_loop_resume(pir_u);
return;
error: {
@ -1078,7 +1104,7 @@ _pier_loop_time(void)
u3v_time(u3_time_in_tv(&tim_tv));
}
/* _pier_loop_prepare():
/* _pier_loop_prepare(): run on every loop iteration before i/o polling.
*/
static void
_pier_loop_prepare(uv_prepare_t* pep_u)
@ -1086,6 +1112,27 @@ _pier_loop_prepare(uv_prepare_t* pep_u)
_pier_loop_time();
}
/* _pier_loop_idle_cb(): run on every loop iteration after i/o polling.
*/
static void
_pier_loop_idle_cb(uv_idle_t* idl_u)
{
u3_pier* pir_u = idl_u->data;
_pier_apply(pir_u);
uv_idle_stop(idl_u);
}
/* _pier_loop_resume(): (re-)activate idle handler
*/
static void
_pier_loop_resume(u3_pier* pir_u)
{
if ( !uv_is_active((uv_handle_t*)&pir_u->idl_u) ) {
uv_idle_start(&pir_u->idl_u, _pier_loop_idle_cb);
}
}
/* _pier_loop_init_pier(): initialize loop handlers.
*/
static void
@ -1139,6 +1186,18 @@ _pier_loop_wake(u3_pier* pir_u)
{
c3_l cod_l;
// inject fresh entropy
//
{
c3_w eny_w[16];
c3_rand(eny_w);
u3_noun wir = u3nt(u3_blip, c3__arvo, u3_nul);
u3_noun car = u3nc(c3__wack, u3i_words(16, eny_w));
u3_pier_work(pir_u, wir, car);
}
cod_l = u3a_lush(c3__unix);
u3_unix_io_talk(pir_u);
u3_unix_ef_bake(pir_u);
@ -1356,8 +1415,6 @@ _pier_boot_vent(u3_boot* bot_u)
// prepend entropy to the module sequence
//
// XX also copy to _pier_loop_wake?
//
{
c3_w eny_w[16];
c3_rand(eny_w);
@ -1616,8 +1673,15 @@ start:
(wit_u->evt_d == (1 + log_u->moc_d)) &&
(wit_u->evt_d == (1 + log_u->com_d)) )
{
// TODO(erg): This is the place where we build up things into a queue.
_pier_db_commit_request(wit_u);
c3_d count = 1 + (god_u->dun_d - wit_u->evt_d);
struct u3_lmdb_write_request* request =
u3_lmdb_build_write_request(wit_u, count);
c3_assert(request != 0);
_pier_db_commit_request(pir_u,
request,
wit_u->evt_d,
count);
act_o = c3y;
}
@ -1633,8 +1697,7 @@ start:
// remove from queue
//
// XX must be done before releasing effects
// which is currently reentrant
// Must be done before releasing effects
//
_pier_writ_unlink(wit_u);
@ -1785,7 +1848,7 @@ u3_pier_discover(u3_pier* pir_u,
u3_noun job)
{
_pier_writ_insert(pir_u, msc_l, job);
_pier_apply(pir_u);
_pier_loop_resume(pir_u);
}
/* u3_pier_send(): modern send with target and path.
@ -1967,6 +2030,29 @@ u3_pier_stub(void)
}
}
/* _pier_init(): initialize pier i/o handles
*/
static void
_pier_init(u3_pier* pir_u)
{
// initialize i/o handlers
//
_pier_loop_init(pir_u);
// initialize pre i/o polling handle
//
uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u);
pir_u->pep_u.data = pir_u;
uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare);
// initialize post i/o polling handle
//
uv_idle_init(u3_Host.lup_u, &pir_u->idl_u);
pir_u->idl_u.data = pir_u;
_pier_loop_resume(pir_u);
}
/* u3_pier_boot(): start the new pier system.
*/
void
@ -1980,6 +2066,12 @@ u3_pier_boot(c3_w wag_w, // config flags
//
u3_pier* pir_u = _pier_create(wag_w, u3r_string(pax));
if ( 0 == pir_u ) {
u3l_log("pier: failed to create\r\n");
u3_daemon_bail();
exit(1);
}
// set boot params
//
{
@ -1988,14 +2080,7 @@ u3_pier_boot(c3_w wag_w, // config flags
_pier_boot_set_ship(pir_u, u3k(who), ( c3__fake == u3h(ven) ) ? c3y : c3n);
}
// initialize i/o handlers
//
_pier_loop_init(pir_u);
// initialize polling handle
//
uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u);
uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare);
_pier_init(pir_u);
u3z(who); u3z(ven); u3z(pil); u3z(pax);
}
@ -2009,14 +2094,13 @@ u3_pier_stay(c3_w wag_w, u3_noun pax)
//
u3_pier* pir_u = _pier_create(wag_w, u3r_string(pax));
// initialize i/o handlers
//
_pier_loop_init(pir_u);
if ( 0 == pir_u ) {
u3l_log("pier: failed to create\r\n");
u3_daemon_bail();
exit(1);
}
// initialize polling handle
//
uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u);
uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare);
_pier_init(pir_u);
u3z(pax);
}

17
pkg/urbit/vere/term.c
View File

@ -27,6 +27,15 @@ static u3_utty* _term_main();
#define _SPIN_RATE_US 250000 // spinner rate (microseconds/frame)
#define _SPIN_IDLE_US 500000 // spinner cools down if stopped this long
static void _write(int fd, const void *buf, size_t count)
{
if (count != write(fd, buf, count)){
u3l_log("write failed\r\n");
c3_assert(0);
}
}
/* _term_msc_out_host(): unix microseconds from current host time.
*/
static c3_d
@ -291,7 +300,7 @@ u3_term_io_exit(void)
if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->cug_i) ) {
c3_assert(!"exit-fcntl");
}
write(uty_u->fid_i, "\r\n", 2);
_write(uty_u->fid_i, "\r\n", 2);
#if 0
if ( uty_u->tat_u.sun.sit_u ) {
@ -765,7 +774,7 @@ _term_try_write_str(u3_utty* uty_u,
{
// c3_i fid_i = uv_fileno(&uty_u->pop_u);
c3_i fid_i = uty_u->pop_u.io_watcher.fd; // XX old libuv
write(fid_i, hun_y, strlen(hun_y));
_write(fid_i, hun_y, strlen(hun_y));
}
/* _term_try_move_left(): move the cursor left (off-thread).
@ -1219,8 +1228,8 @@ u3_term_io_hija(void)
perror("hija-fcntl-0");
c3_assert(!"hija-fcntl");
}
write(uty_u->fid_i, "\r", 1);
write(uty_u->fid_i, uty_u->ufo_u.out.el_y,
_write(uty_u->fid_i, "\r", 1);
_write(uty_u->fid_i, uty_u->ufo_u.out.el_y,
strlen((c3_c*) uty_u->ufo_u.out.el_y));
}
return stdout;

26
pkg/urbit/vere/unix.c
View File

@ -15,6 +15,23 @@
#include "vere/vere.h"
c3_w u3_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result)
{
errno = 0;
struct dirent * tmp_u = readdir(dirp);
if (NULL == tmp_u){
*result = NULL;
return (errno); // either success or error code
} else {
memcpy(entry, tmp_u, sizeof(struct dirent));
*result = entry;
}
return(0);
}
/* _unix_down(): descend path.
*/
static c3_c*
@ -311,9 +328,10 @@ _unix_scan_mount_point(u3_pier *pir_u, u3_umon* mon_u)
struct dirent* out_u;
c3_w err_w;
if ( 0 != (err_w = readdir_r(rid_u, &ent_u, &out_u)) ) {
if ( 0 != (err_w = u3_readdir_r(rid_u, &ent_u, &out_u)) ) {
u3l_log("erroring loading pier directory %s: %s\r\n",
mon_u->dir_u.pax_c, strerror(errno));
c3_assert(0);
}
else if ( !out_u ) {
@ -756,7 +774,8 @@ _unix_update_dir(u3_pier *pir_u, u3_udir* dir_u)
struct dirent* out_u;
c3_w err_w;
if ( (err_w = readdir_r(rid_u, &ent_u, &out_u)) != 0 ) {
if ( (err_w = u3_readdir_r(rid_u, &ent_u, &out_u)) != 0 ) {
u3l_log("error loading directory %s: %s\r\n",
dir_u->pax_c, strerror(err_w));
c3_assert(0);
@ -948,9 +967,10 @@ _unix_initial_update_dir(c3_c* pax_c, c3_c* bas_c)
struct dirent* out_u;
c3_w err_w;
if ( 0 != (err_w = readdir_r(rid_u, &ent_u, &out_u)) ) {
if ( 0 != (err_w = u3_readdir_r(rid_u, &ent_u, &out_u)) ) {
u3l_log("error loading initial directory %s: %s\r\n",
pax_c, strerror(errno));
c3_assert(0);
}
else if ( !out_u ) {

2
pkg/urbit/vere/walk.c
View File

@ -192,7 +192,7 @@ _walk_in(const c3_c* dir_c, c3_w len_w)
struct dirent ent_n;
struct dirent* out_n;
if ( readdir_r(dir_d, &ent_n, &out_n) != 0 ) {
if ( u3_readdir_r(dir_d, &ent_n, &out_n) != 0 ) {
u3l_log("%s: %s\n", dir_c, strerror(errno));
break;
}

239
pkg/urbit/worker/main.c
View File

@ -26,7 +26,9 @@
typedef struct _u3_worker {
c3_w len_w; // boot sequence length
c3_d evt_d; // last event processed
u3_noun roe; // lifecycle formulas
c3_d sen_d; // last event requested
c3_d dun_d; // last event processed
c3_l mug_l; // hash of state
c3_d key_d[4]; // disk key
u3_moat inn_u; // message input
@ -59,7 +61,7 @@
::
$: %done
:: p: event number
:: q: mug of state (or 0)
:: q: mug of kernel
:: r: effects
::
[p=@ q=@ r=(list ovum)]
@ -68,7 +70,7 @@
::
$: %work
:: p: event number
:: q: mug of state (or 0)
:: q: mug of kernel
:: r: replacement event (at date)
::
[p=@ q=@ r=(pair date ovum)]
@ -114,10 +116,9 @@
::
$: %work
:: p: event number
:: q: mug of state (or 0)
:: r: event (at date)
:: q: a jammed noun [mug [date ovum]]
::
[p=@ q=@ r=(pair date ovum)]
[p=@ q=@]
== ==
--
*/
@ -201,9 +202,9 @@ _worker_prof(FILE* fil_u, c3_w den, u3_noun mas)
_worker_print_memory(fil_u, tot_w);
#if 1
/* The basic issue here is that tt_mas is included in
* u3A->sac, so they can't both be roots in the normal
* sense. When we mark u3A->sac later on, we want tt_mas
/* The basic issue here is that tt_mas is included in .sac
* (the whole profile), so they can't both be roots in the
* normal sense. When we mark .sac later on, we want tt_mas
* to appear unmarked, but its children should be already
* marked.
*/
@ -263,7 +264,7 @@ _worker_grab(u3_noun sac, u3_noun ovo, u3_noun vir)
}
}
else {
c3_w usr_w = 0, man_w = 0, sac_w = 0, ova_w = 0, vir_w = 0;
c3_w usr_w = 0, man_w = 0, sac_w = 0, ova_w = 0, roe_w = 0, vir_w = 0;
FILE* fil_u;
@ -309,6 +310,9 @@ _worker_grab(u3_noun sac, u3_noun ovo, u3_noun vir)
ova_w = u3a_mark_noun(ovo);
u3a_print_memory(fil_u, "event", ova_w);
roe_w = u3a_mark_noun(u3V.roe);
u3a_print_memory(fil_u, "lifecycle events", roe_w);
vir_w = u3a_mark_noun(vir);
u3a_print_memory(fil_u, "effects", vir_w);
@ -329,7 +333,7 @@ _worker_grab(u3_noun sac, u3_noun ovo, u3_noun vir)
static void
_worker_fail(void* vod_p, const c3_c* wut_c)
{
u3l_log("worker: fail: %s\r\n", wut_c);
u3l_log("work: fail: %s\r\n", wut_c);
exit(1);
}
@ -344,16 +348,15 @@ _worker_send(u3_noun job)
/* _worker_send_replace(): send replacement job back to daemon.
*/
static void
_worker_send_replace(c3_d evt_d, u3_noun ovo)
_worker_send_replace(c3_d evt_d, u3_noun job)
{
u3l_log("worker_send_replace %" PRIu64 " %s\r\n",
evt_d,
u3r_string(u3h(u3t(ovo))));
u3r_string(u3h(u3t(u3t(job)))));
_worker_send(u3nq(c3__work,
_worker_send(u3nt(c3__work,
u3i_chubs(1, &evt_d),
u3V.mug_l,
u3nc(u3k(u3A->now), ovo)));
u3ke_jam(u3nc(u3V.mug_l, job))));
}
/* _worker_send_complete(): report completion.
@ -362,8 +365,8 @@ static void
_worker_send_complete(u3_noun vir)
{
_worker_send(u3nq(c3__done,
u3i_chubs(1, &u3V.evt_d),
u3r_mug(u3A->roc),
u3i_chubs(1, &u3V.dun_d),
u3V.mug_l,
vir));
}
@ -372,7 +375,7 @@ _worker_send_complete(u3_noun vir)
static void
_worker_send_stdr(c3_c* str_c)
{
_worker_send(u3nt(c3__stdr, u3i_chubs(1, &u3V.evt_d), u3i_string(str_c)));
_worker_send(u3nt(c3__stdr, u3i_chubs(1, &u3V.sen_d), u3i_string(str_c)));
}
/* _worker_send_slog(): send hint output (hod is [priority tank]).
@ -380,18 +383,64 @@ _worker_send_stdr(c3_c* str_c)
static void
_worker_send_slog(u3_noun hod)
{
_worker_send(u3nt(c3__slog, u3i_chubs(1, &u3V.evt_d), hod));
_worker_send(u3nt(c3__slog, u3i_chubs(1, &u3V.sen_d), hod));
}
/* _worker_lame(): event failed, replace with error event.
*/
static void
_worker_lame(c3_d evt_d, u3_noun ovo, u3_noun why, u3_noun tan)
_worker_lame(c3_d evt_d, u3_noun now, u3_noun ovo, u3_noun why, u3_noun tan)
{
// %crud will be sent on the original wire.
u3_noun rep;
u3_noun wir, tag, cad;
u3x_trel(ovo, &wir, &tag, &cad);
// a deterministic error (%exit) in a network packet (%hear)
// generates a negative-acknowlegement attempt (%hole).
//
_worker_send_replace(evt_d, u3nc(u3k(u3h(ovo)), u3nt(c3__crud, why, tan)));
u3z(ovo);
// A comment from the old implementation:
// There should be a separate path for crypto failures,
// to prevent timing attacks, but isn't right now. To deal
// with a crypto failure, just drop the packet.
//
if ( (c3__hear == tag) && (c3__exit == why) ) {
rep = u3nt(u3k(wir), c3__hole, u3k(cad));
}
// failed event notifications (%crud) are replaced with
// an even more generic notifications, on a generic arvo wire.
// N.B this must not be allowed to fail!
//
// [%warn original-event-tag=@tas combined-trace=(list tank)]
//
else if ( c3__crud == tag ) {
u3_noun lef = u3nc(c3__leaf, u3i_tape("crude crashed!"));
u3_noun nat = u3kb_weld(u3k(u3t(cad)), u3nc(lef, u3k(tan)));
rep = u3nc(u3nt(u3_blip, c3__arvo, u3_nul),
u3nt(c3__warn, u3k(u3h(cad)), nat));
}
// failed failure failing fails
//
else if ( c3__warn == tag ) {
_worker_fail(0, "%warn replacement event failed");
c3_assert(0);
}
// failure notifications are sent on the same wire
//
// [%crud event-tag=@tas event-trace=(list tank)]
//
else {
// prepend failure mote to tank
//
u3_noun lef = u3nc(c3__leaf, u3kb_weld(u3i_tape("bail: "),
u3qc_rip(3, why)));
u3_noun nat = u3kb_weld(u3k(tan), u3nc(lef, u3_nul));
rep = u3nc(u3k(wir), u3nt(c3__crud, u3k(tag), nat));
}
_worker_send_replace(evt_d, u3nc(now, rep));
u3z(ovo); u3z(why); u3z(tan);
}
/* _worker_sure(): event succeeded, report completion.
@ -401,6 +450,8 @@ _worker_sure(u3_noun ovo, u3_noun vir, u3_noun cor)
{
u3z(u3A->roc);
u3A->roc = cor;
u3A->ent_d = u3V.dun_d;
u3V.mug_l = u3r_mug(u3A->roc);
u3_noun sac = u3_nul;
@ -453,26 +504,19 @@ _worker_sure(u3_noun ovo, u3_noun vir, u3_noun cor)
/* _worker_work_live(): apply event.
*/
static void
_worker_work_live(c3_d evt_d, // event number
c3_l mug_l, // mug of state
u3_noun job) // event date
_worker_work_live(c3_d evt_d, u3_noun job)
{
u3_noun now, ovo, gon;
u3_noun last_date;
c3_assert(evt_d == u3V.evt_d + 1ULL);
if ( 0 != mug_l ) {
c3_assert(u3r_mug(u3A->roc) == mug_l);
}
c3_assert(evt_d == u3V.dun_d + 1ULL);
u3V.sen_d = evt_d;
u3x_cell(job, &now, &ovo);
u3z(u3A->now);
last_date = u3A->now;
u3A->now = u3k(now);
// XX why is this set before u3v_poke?
//
u3A->ent_d = evt_d;
#ifdef U3_EVENT_TIME_DEBUG
{
struct timeval b4, f2, d0;
@ -481,7 +525,7 @@ _worker_work_live(c3_d evt_d, // event number
if ( c3__belt != u3h(u3t(ovo)) ) {
c3_c* txt_c = u3r_string(u3h(u3t(ovo)));
u3l_log("worker: %s (%" PRIu64 ") live\r\n", txt_c, evt_d);
u3l_log("work: %s (%" PRIu64 ") live\r\n", txt_c, evt_d);
}
}
#endif
@ -510,20 +554,26 @@ _worker_work_live(c3_d evt_d, // event number
if ( u3_blip != u3h(gon) ) {
// event rejected
//
u3V.sen_d = u3V.dun_d;
// restore previous time
//
u3_noun nex = u3A->now;
u3A->now = last_date;
u3_noun why, tan;
u3x_cell(gon, &why, &tan);
u3k(ovo); u3k(why); u3k(tan);
u3z(gon); u3z(job);
_worker_lame(evt_d, ovo, why, tan);
_worker_lame(evt_d, nex, ovo, why, tan);
}
else {
// event accepted
//
// XX reconcile/dedupe with u3A->ent_d
//
u3V.evt_d = evt_d;
u3V.dun_d = u3V.sen_d;
u3z(last_date);
// vir/(list ovum) list of effects
// cor/arvo arvo core
//
@ -535,9 +585,13 @@ _worker_work_live(c3_d evt_d, // event number
_worker_sure(ovo, vir, cor);
// reclaim memory from persistent caches on |reset
// reclaim memory from persistent caches periodically
//
if ( 0 == (u3A->ent_d % 1000ULL) ) {
// XX this is a hack to work around the fact that
// the bytecode caches grow rapidly and are not
// able to be simply capped (due to internal posts).
//
if ( 0 == (evt_d % 1000ULL) ) {
u3m_reclaim();
}
}
@ -562,24 +616,25 @@ _worker_boot_fire(u3_noun eve)
/* _worker_work_boot(): apply initial-stage event.
*/
static void
_worker_work_boot(c3_d evt_d,
c3_l mug_l,
u3_noun job)
_worker_work_boot(c3_d evt_d, u3_noun job)
{
c3_assert(evt_d == u3V.evt_d + 1ULL);
u3V.evt_d = evt_d;
// here we asset on u3V.sen_d, because u3V.dun_d isn't set until
// after u3V.sen_d == u3V.len_w (ie, after the lifecycle evaluation)
//
c3_assert(evt_d == u3V.sen_d + 1ULL);
u3V.sen_d = evt_d;
u3A->roe = u3nc(job, u3A->roe);
u3V.roe = u3nc(job, u3V.roe);
u3l_log("worker: (%" PRIu64 ")| boot\r\n", evt_d);
u3l_log("work: (%" PRIu64 ")| boot\r\n", evt_d);
if ( u3V.len_w == evt_d ) {
u3_noun eve, pru;
eve = u3kb_flop(u3A->roe);
u3A->roe = 0;
eve = u3kb_flop(u3V.roe);
u3V.roe = u3_nul;
u3l_log("worker: (%" PRIu64 ")| pill: %x\r\n", evt_d, u3r_mug(eve));
u3l_log("work: (%" PRIu64 ")| pill: %x\r\n", evt_d, u3r_mug(eve));
pru = u3m_soft(0, _worker_boot_fire, eve);
@ -588,18 +643,25 @@ _worker_work_boot(c3_d evt_d,
exit(1);
}
u3V.dun_d = evt_d;
u3A->ent_d = u3V.dun_d;
u3A->roc = u3k(u3t(pru));
u3V.mug_l = u3r_mug(u3A->roc);
u3l_log("worker: (%" PRIu64 ")| core: %x\r\n", evt_d, u3r_mug(u3A->roc));
u3l_log("work: (%" PRIu64 ")| core: %x\r\n", evt_d, u3V.mug_l);
// XX set u3A->evt_d ?
//
u3z(pru);
}
else {
// prior to the evaluation of the entire lifecycle sequence,
// we simply use the mug of the formula as the kernel mug
//
u3V.mug_l = u3r_mug(job);
}
_worker_send(u3nq(c3__done,
u3i_chubs(1, &evt_d),
0,
u3V.mug_l,
u3_nul));
}
@ -620,12 +682,29 @@ _worker_poke_work(c3_d evt_d, // event number
}
}
// Require mugs to match
//
// We use mugs to enforce that %work is always performed against
// the exact kernel we expect it to be. If it isn't, we have either
// event-log corruption or non-determism on replay, or programmer error
// in normal operation. In either case, we immediately exit.
//
if ( u3V.mug_l != mug_l ) {
u3l_log("work: invalid %%work for event %" PRIu64 ".\r\n", evt_d);
u3l_log("work: computed mug is %x but event %" PRIu64 " expected %x.\r\n",
u3V.mug_l,
evt_d,
mug_l);
_worker_fail(0, "bad jar");
return;
}
if ( evt_d <= u3V.len_w ) {
c3_c lab_c[8];
snprintf(lab_c, 8, "boot: %" PRIu64 "", evt_d);
u3t_event_trace(lab_c, 'B');
_worker_work_boot(evt_d, mug_l, job);
_worker_work_boot(evt_d, job);
u3t_event_trace(lab_c, 'E');
}
else {
@ -637,7 +716,7 @@ _worker_poke_work(c3_d evt_d, // event number
u3m_pretty_path(wir), u3m_pretty(cad));
u3t_event_trace(lab_c, 'B');
_worker_work_live(evt_d, mug_l, job);
_worker_work_live(evt_d, job);
u3t_event_trace(lab_c, 'E');
}
}
@ -707,22 +786,29 @@ _worker_poke(void* vod_p, u3_noun mat)
}
case c3__work: {
u3_noun evt, mug, job;
u3_noun evt, jammed_entry, mug, job;
c3_d evt_d;
c3_l mug_l;
if ( (c3n == u3r_qual(jar, 0, &evt, &mug, &job)) ||
if ( (c3n == u3r_trel(jar, 0, &evt, &jammed_entry)) ||
(c3n == u3ud(evt)) ||
(1 != u3r_met(6, evt)) ||
(c3n == u3ud(mug)) ||
(1 < u3r_met(5, mug)) )
(1 != u3r_met(6, evt)) )
{
goto error;
}
u3_noun entry = u3qe_cue(jammed_entry);
if ( (c3y != u3du(entry)) ||
(c3n == u3r_cell(entry, &mug, &job)) ||
(c3n == u3ud(mug)) ||
(1 < u3r_met(5, mug)) ) {
goto error;
}
evt_d = u3r_chub(0, evt);
mug_l = u3r_word(0, mug);
u3k(job);
u3z(entry);
u3z(jar);
return _worker_poke_work(evt_d, mug_l, job);
@ -756,9 +842,9 @@ _worker_poke(void* vod_p, u3_noun mat)
}
evt_d = u3r_chub(0, evt);
u3z(evt);
u3z(jar);
c3_assert( evt_d == u3V.evt_d );
c3_assert( evt_d == u3V.dun_d );
return u3e_save();
}
@ -780,9 +866,10 @@ u3_worker_boot(void)
u3_noun dat = u3_nul;
if ( u3_none != u3A->our ) {
nex_d = u3A->ent_d + 1ULL;
u3V.mug_l = u3r_mug(u3A->roc);
nex_d = u3V.dun_d + 1ULL;
dat = u3nc(u3_nul, u3nt(u3i_chubs(1, &nex_d),
0, // XX u3r_mug(u3A->roc),
u3V.mug_l,
u3nc(u3k(u3A->our), u3k(u3A->fak))));
// disable hashboard for fake ships
@ -796,7 +883,7 @@ u3_worker_boot(void)
u3V.len_w = 0;
}
u3l_log("worker: play %" PRIu64 "\r\n", nex_d);
u3l_log("work: play %" PRIu64 "\r\n", nex_d);
_worker_send(u3nc(c3__play, dat));
}
@ -813,6 +900,9 @@ main(c3_i argc, c3_c* argv[])
c3_assert(4 == argc);
memset(&u3V, 0, sizeof(u3V));
memset(&u3_Host.tra_u, 0, sizeof(u3_Host.tra_u));
/* load passkey
*/
{
@ -835,19 +925,10 @@ main(c3_i argc, c3_c* argv[])
u3V.dir_c = strdup(dir_c);
}
/* clear tracing struct
*/
{
u3_Host.tra_u.nid_w = 0;
u3_Host.tra_u.fil_u = NULL;
u3_Host.tra_u.con_w = 0;
u3_Host.tra_u.fun_w = 0;
}
/* boot image
*/
{
u3V.evt_d = u3m_boot_new(dir_c);
u3V.sen_d = u3V.dun_d = u3m_boot_new(dir_c);
u3C.stderr_log_f = _worker_send_stdr;
u3C.slog_f = _worker_send_slog;
}

9
sh/mk-fakezod → sh/boot
View File

@ -3,9 +3,10 @@
set -e
case $# in
0) target=./fakezod;;
1) target=$1;;
*) echo "Usage: $0 [target]" >&2
0) ship=zod; target=./zod;;
1) ship=$1; target=./$1;;
2) ship=$1; target=$2;;
*) echo "Usage: $0 [ship] [target-dir]" >&2
exit 1;;
esac
@ -14,7 +15,7 @@ then echo "Target directory '$target' already exists." >&2
exit 1
fi
pkg=$(nix-build nix/ops -A fakezod --no-out-link)
pkg=$(nix-build nix/ops/boot-ship.nix --arg ship "\"$ship\"" --no-out-link)
mkdir -p "$(dirname "$target")"
cp -r $pkg/ $target

19
sh/cachix
View File

@ -5,6 +5,10 @@ fail () {
exit 1
}
if [ -z "$CACHIX_SIGNING_KEY" ]
then fail "The CACHIX_AUTH_TOKEN environment variable needs to be set."
fi
if [ -z "$CACHIX_AUTH_TOKEN" ]
then fail "The CACHIX_AUTH_TOKEN environment variable needs to be set."
fi
@ -15,8 +19,15 @@ cleanup () {
trap cleanup EXIT
set -ex
cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null
cachix use urbit
nix-build nix/dep-repos.nix > .cache.list
nix-build default.nix >> .cache.list
cachix push urbit < .cache.list
cachix use urbit2
nix-build --no-out-link \
--max-jobs 2 \
nix/cachix/local.nix \
nix/cachix/release.nix \
> .cache.list
cachix push urbit2 < .cache.list

21
sh/ci-tests Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -ex
if [ "$TRAVIS_PULL_REQUEST" = false ]
then
sh/test --arg debug false
else
nix-build nix/ops \
-A test \
-A brass \
--max-jobs 2 \
--no-out-link
sh/test --arg debug true
sh/update-brass-pill
fi

31
sh/cross Executable file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -e
err () { echo "$@" >&2; exit 1; }
[ $# = 2 ] || {
sh/cross urbit linux64
sh/cross urbit darwin
exit 0
}
traced () {
echo '$' "$@" >&2; "$@"
}
release () {
traced nix-build nix/release.nix --no-out-link -A "$1"
}
pkg="$1"
env="$2"
res=$(release "$env.$pkg")
mkdir -p ./cross/$env
traced cp -f $res/bin/$pkg ./cross/$env/$pkg
traced cp -f $res/bin/$pkg-worker ./cross/$env/$pkg-worker
traced cp -r $res/bin/$pkg-terminfo ./cross/$env/$pkg-terminfo
chmod -R u+wr ./cross/$env/$pkg-terminfo

21
sh/mk-bootbus
View File

@ -1,21 +0,0 @@
#!/usr/bin/env bash
set -e
case $# in
0) target=./bootbus;;
1) target=$1;;
*) echo "Usage: $0 [target]" >&2
exit 1;;
esac
if [ -e "$target" ]
then echo "Target directory '$target' already exists." >&2
exit 1
fi
pkg=$(nix-build nix/ops -A bootbus --no-out-link)
mkdir -p "$(dirname "$target")"
cp -r $pkg/ $target
chmod -R u+rw $target

21
sh/mk-bootzod
View File

@ -1,21 +0,0 @@
#!/usr/bin/env bash
set -e
case $# in
0) target=./bootzod;;
1) target=$1;;
*) echo "Usage: $0 [target]" >&2
exit 1;;
esac
if [ -e "$target" ]
then echo "Target directory '$target' already exists." >&2
exit 1
fi
pkg=$(nix-build nix/ops -A bootzod --no-out-link)
mkdir -p "$(dirname "$target")"
cp -r $pkg/ $target
chmod -R u+rw $target

36
sh/release
View File

@ -2,26 +2,22 @@
set -e
err () { echo "$@" >&2; exit 1; }
if [ -n "${TRAVIS_TAG-}" ]
then
ver="$TRAVIS_TAG"
elif [ -n "${TRAVIS_COMMIT-}" ]
then
ver="$TRAVIS_COMMIT"
else
ver="$(git rev-parse HEAD)"
fi
[ $# = 2 ] || err "usage: $0 pkg host"
mkdir -p release
traced () {
echo '$' "$@" >&2; "$@"
}
for plat in linux64 darwin
do
sh/cross urbit "$plat"
release () {
traced nix-build nix/release.nix --no-out-link -A "$1"
}
pkg="$1"
env="$2"
res=$(release "$env.$pkg")
mkdir -p ./bin/$env
traced cp -f $res/bin/$pkg ./bin/$env/$pkg
traced cp -f $res/bin/$pkg-worker ./bin/$env/$pkg-worker
traced cp -r $res/bin/$pkg-terminfo ./bin/$env/$pkg-terminfo
chmod -R u+wr ./bin/$env/$pkg-terminfo
echo "packaging release/urbit-$plat-$ver.tgz"
(cd cross/$plat; tar cz .) > release/urbit-$plat-$ver.tgz
done

2
sh/test
View File

@ -2,7 +2,7 @@
set -e
pkg=$(nix-build nix/ops -A test --no-out-link)
pkg=$(nix-build nix/ops -A test --no-out-link "$@")
hdr () {
echo =====$(sed 's/./=/g' <<< "$1")=====

3
sh/update-brass-pill
View File

@ -1,7 +1,6 @@
#!/usr/bin/env bash
set -e
set -ex
pkg=$(nix-build nix/ops -A brass --no-out-link)
cp $pkg bin/brass.pill

6
sh/update-solid-pill Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -ex
pkg=$(nix-build nix/ops -A solid --no-out-link)
cp $pkg bin/solid.pill