From 02549975334bd001c4e232bce68052ea17288e76 Mon Sep 17 00:00:00 2001 From: Patrick O'Sullivan Date: Fri, 4 Nov 2022 16:04:22 -0500 Subject: [PATCH 01/13] Groups/Garden: Add S3 region to s3 settings --- pkg/landscape/app/s3-store.hoon | 33 +++++++++++++++++++--- pkg/landscape/gen/s3-store/set-region.hoon | 10 +++++++ pkg/landscape/lib/s3-json.hoon | 3 ++ pkg/landscape/sur/s3-0.hoon | 27 ++++++++++++++++++ pkg/landscape/sur/s3.hoon | 7 +++++ 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 pkg/landscape/gen/s3-store/set-region.hoon create mode 100644 pkg/landscape/sur/s3-0.hoon diff --git a/pkg/landscape/app/s3-store.hoon b/pkg/landscape/app/s3-store.hoon index 061a35cf1..cee795389 100644 --- a/pkg/landscape/app/s3-store.hoon +++ b/pkg/landscape/app/s3-store.hoon @@ -9,12 +9,14 @@ +$ card card:agent:gall +$ versioned-state $% state-zero + state-one == :: -+$ state-zero [%0 =credentials =configuration] ++$ state-zero [%0 =credentials:zero:past =configuration:zero:past] ++$ state-one [%1 =credentials =configuration] -- :: -=| state-zero +=| state-one =* state - :: %- agent:dbug @@ -28,8 +30,28 @@ ++ on-init on-init:def ++ on-save !>(state) ++ on-load - |= old-vase=vase - [~ this(state !<(state-zero old-vase))] + |= =vase + =/ old !<(versioned-state vase) + |^ + ?- -.old + %1 `this(state old) + %0 `this(state (state-0-to-1 old)) + == + ++ state-0-to-1 + |= zer=state-zero + ^- state-one + :* %1 + credentials.zer + (configuration-0-to-1 configuration.zer) + == + ++ configuration-0-to-1 + |= conf=configuration:zero:past + ^- ^configuration + :* buckets.conf + current-bucket.conf + '' + == + -- :: ++ on-poke ~/ %s3-poke @@ -56,6 +78,9 @@ :: %set-secret-access-key state(secret-access-key.credentials secret-access-key.act) + :: + %set-region + state(region.configuration region.act) :: %set-current-bucket %_ state diff --git a/pkg/landscape/gen/s3-store/set-region.hoon b/pkg/landscape/gen/s3-store/set-region.hoon new file mode 100644 index 000000000..d87b09f93 --- /dev/null +++ b/pkg/landscape/gen/s3-store/set-region.hoon @@ -0,0 +1,10 @@ +:: s3-store|set-current-bucket: set current bucket for S3 +:: +/- *s3 +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[region=@t ~] ~] + == +:- %s3-action +^- action +[%set-region region] diff --git a/pkg/landscape/lib/s3-json.hoon b/pkg/landscape/lib/s3-json.hoon index e4a531daa..04cf848b9 100644 --- a/pkg/landscape/lib/s3-json.hoon +++ b/pkg/landscape/lib/s3-json.hoon @@ -10,6 +10,7 @@ :~ [%set-endpoint so:dejs] [%set-access-key-id so:dejs] [%set-secret-access-key so:dejs] + [%set-region so:dejs] [%add-bucket so:dejs] [%remove-bucket so:dejs] [%set-current-bucket so:dejs] @@ -25,6 +26,7 @@ :~ ?- -.upd %set-current-bucket [%'setCurrentBucket' s+bucket.upd] %add-bucket [%'addBucket' s+bucket.upd] + %set-region [%'setRegion' s+region.upd] %remove-bucket [%'removeBucket' s+bucket.upd] %set-endpoint [%'setEndpoint' s+endpoint.upd] %set-access-key-id [%'setAccessKeyId' s+access-key-id.upd] @@ -44,6 +46,7 @@ %- pairs:enjs :~ [%buckets a+(turn ~(tap in buckets.configuration.upd) |=(a=@t s+a))] [%'currentBucket' s+current-bucket.configuration.upd] + [%'region' s+region.configuration.upd] == == == diff --git a/pkg/landscape/sur/s3-0.hoon b/pkg/landscape/sur/s3-0.hoon new file mode 100644 index 000000000..4f0ca04cb --- /dev/null +++ b/pkg/landscape/sur/s3-0.hoon @@ -0,0 +1,27 @@ +|% ++$ credentials + $: endpoint=@t + access-key-id=@t + secret-access-key=@t + == +:: ++$ configuration + $: buckets=(set @t) + current-bucket=@t + == +:: ++$ action + $% [%set-endpoint endpoint=@t] + [%set-access-key-id access-key-id=@t] + [%set-secret-access-key secret-access-key=@t] + [%add-bucket bucket=@t] + [%remove-bucket bucket=@t] + [%set-current-bucket bucket=@t] + == +:: ++$ update + $% [%credentials =credentials] + [%configuration =configuration] + action + == +-- diff --git a/pkg/landscape/sur/s3.hoon b/pkg/landscape/sur/s3.hoon index 4f0ca04cb..a79ca68ce 100644 --- a/pkg/landscape/sur/s3.hoon +++ b/pkg/landscape/sur/s3.hoon @@ -1,4 +1,9 @@ +/- zer=s3-0 |% +++ past + |% + ++ zero zer + -- +$ credentials $: endpoint=@t access-key-id=@t @@ -8,6 +13,7 @@ +$ configuration $: buckets=(set @t) current-bucket=@t + region=@t == :: +$ action @@ -17,6 +23,7 @@ [%add-bucket bucket=@t] [%remove-bucket bucket=@t] [%set-current-bucket bucket=@t] + [%set-region region=@t] == :: +$ update From 48978bd940ae92c9ee43ede5e6dd176402284cf6 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 14 Dec 2022 12:46:36 -0600 Subject: [PATCH 02/13] group-store: allow rebuilding --- pkg/landscape/app/group-store.hoon | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 7e26a80ff..b373024d6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -140,6 +140,7 @@ ?+ q.vase !! %migrate poke-migrate:gc %export poke-export:gc + %rebuild poke-rebuild:gc == :: ?(%group-update-0 %group-action) @@ -235,6 +236,18 @@ :: |_ bol=bowl:gall +* io ~(. agentio bol) +++ poke-rebuild + ^- (quip card _state) + =/ wex ~(tap by wex.bol) + |- + ?~ wex + `state + =/ [[=wire =ship =term] [acked=? =(pole knot)]] + i.wex + ?. ?=([%gladio ship=@ ~] pole) + $(wex t.wex) + $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) +:: ++ poke-export ^- (quip card _state) :_ state @@ -256,6 +269,8 @@ ^- card [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init] :: + +:: ++ backoff-migrate |= =ship ^- card From 4df4a16881085b8896db1400536731e9f2de5e65 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 14 Dec 2022 13:29:17 -0600 Subject: [PATCH 03/13] group-store: add %rebuild poke --- pkg/landscape/app/group-store.hoon | 41 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index b373024d6..c7c429a55 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -238,15 +238,36 @@ +* io ~(. agentio bol) ++ poke-rebuild ^- (quip card _state) - =/ wex ~(tap by wex.bol) - |- - ?~ wex - `state - =/ [[=wire =ship =term] [acked=? =(pole knot)]] - i.wex - ?. ?=([%gladio ship=@ ~] pole) - $(wex t.wex) - $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) + |^ + =. wait + put-missing + =^ cards state + watch-missing + [cards state] + :: + ++ watch-missing + =/ wait ~(tap in wait) + =| cards=(list card) + |- + ?~ wait + [cards state] + ?: (~(has by wex.bol) [/gladio/(scot %p i.wait) i.wait dap.bol]) + $(wait t.wait) + =. cards + :_(cards (watch-init-migrate i.wait)) + $(wait t.wait) + :: + ++ put-missing + =/ wex ~(tap by wex.bol) + |- + ?~ wex + wait + =/ [[=wire =ship =term] [acked=? =(pole knot)]] + i.wex + ?. ?=([%gladio ship=@ ~] pole) + $(wex t.wex) + $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) + -- :: ++ poke-export ^- (quip card _state) @@ -269,8 +290,6 @@ ^- card [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init] :: - -:: ++ backoff-migrate |= =ship ^- card From eb324b4fa4c2b08c76317601c88708c624f98c70 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Tue, 20 Dec 2022 18:55:39 -0500 Subject: [PATCH 04/13] vere: bumps version --- pkg/urbit/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/urbit/version b/pkg/urbit/version index 9f76d37b7..07fe6f6c9 100644 --- a/pkg/urbit/version +++ b/pkg/urbit/version @@ -1 +1 @@ -1.13 \ No newline at end of file +1.15 \ No newline at end of file From 56ceee500611da066261d95bd5903a6a26f7e071 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Tue, 3 Jan 2023 11:24:54 -0700 Subject: [PATCH 05/13] Revert "Merge pull request #6171 from urbit/jb/release/vere" This reverts commit 21f70231981eab8ed8250254c507dc99b330d857, reversing changes made to 1e09188065f005a5446cc104bc982b1808607a18. --- .github/workflows/tarballs.yml | 9 +- .github/workflows/vere.yml | 12 +- pkg/urbit/Makefile | 3 +- pkg/urbit/bench/ur_bench.c | 4 +- pkg/urbit/c/defs.c | 96 -- pkg/urbit/compat/mingw/compat.c | 57 -- pkg/urbit/compat/mingw/compat.h | 2 - pkg/urbit/compat/mingw/seh_handler.c | 2 +- pkg/urbit/daemon/main.c | 277 +---- pkg/urbit/daemon/whereami.c | 207 +--- pkg/urbit/daemon/whereami.h | 6 +- pkg/urbit/include/all.h | 1 + pkg/urbit/include/c/defs.h | 29 - pkg/urbit/include/noun/allocate.h | 5 - pkg/urbit/include/noun/events.h | 28 +- pkg/urbit/include/noun/manage.h | 22 +- pkg/urbit/include/noun/options.h | 20 +- pkg/urbit/include/noun/urth.h | 2 +- pkg/urbit/include/noun/vortex.h | 5 - pkg/urbit/include/vere/db/lmdb.h | 29 - pkg/urbit/include/vere/mars.h | 22 - pkg/urbit/include/vere/vere.h | 54 +- pkg/urbit/noun/allocate.c | 136 +-- pkg/urbit/noun/events.c | 963 ++++++++---------- pkg/urbit/noun/manage.c | 167 +-- pkg/urbit/noun/trace.c | 17 +- pkg/urbit/noun/urth.c | 67 +- pkg/urbit/noun/vortex.c | 69 +- pkg/urbit/tests/ames_tests.c | 3 +- pkg/urbit/tests/events_tests.c | 251 ----- pkg/urbit/tests/hashtable_tests.c | 3 +- pkg/urbit/tests/jam_tests.c | 4 +- pkg/urbit/tests/jet_tests.c | 3 +- pkg/urbit/tests/mug_tests.c | 3 +- pkg/urbit/tests/newt_tests.c | 3 +- .../tests/{meme_tests.c => nock_tests.c} | 15 +- pkg/urbit/tests/noun_tests.c | 3 +- pkg/urbit/vere/db/lmdb.c | 106 -- pkg/urbit/vere/disk.c | 227 +---- pkg/urbit/vere/io/http.c | 145 +-- pkg/urbit/vere/io/term.c | 58 +- pkg/urbit/vere/io/unix.c | 55 +- pkg/urbit/vere/king.c | 90 +- pkg/urbit/vere/lord.c | 2 +- pkg/urbit/version | 2 +- pkg/urbit/worker/mars.c | 267 ----- pkg/urbit/worker/serf.c | 143 ++- 47 files changed, 986 insertions(+), 2708 deletions(-) delete mode 100644 pkg/urbit/c/defs.c delete mode 100644 pkg/urbit/include/vere/mars.h delete mode 100644 pkg/urbit/tests/events_tests.c rename pkg/urbit/tests/{meme_tests.c => nock_tests.c} (80%) delete mode 100644 pkg/urbit/worker/mars.c diff --git a/.github/workflows/tarballs.yml b/.github/workflows/tarballs.yml index 08bd6c806..9f8456cf6 100644 --- a/.github/workflows/tarballs.yml +++ b/.github/workflows/tarballs.yml @@ -23,13 +23,12 @@ jobs: name: ${{ secrets.CACHIX_NAME }} authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_CREDENTIALS }} - - - uses: google-github-actions/setup-gcloud@v1 + - uses: google-github-actions/setup-gcloud@v0.2.0 with: + version: '290.0.1' + service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} project_id: ${{ secrets.GCS_PROJECT }} + export_default_credentials: true - run: nix-build -A tarball --arg enableStatic true diff --git a/.github/workflows/vere.yml b/.github/workflows/vere.yml index 262bed278..eb94e8ea9 100644 --- a/.github/workflows/vere.yml +++ b/.github/workflows/vere.yml @@ -125,14 +125,13 @@ jobs: echo -n "$version" > ./version-string - name: upload version string artifact - if: inputs.upload && matrix.type == 'linux' + if: matrix.type == 'linux' uses: actions/upload-artifact@v3 with: name: version-string path: version-string - uses: google-github-actions/auth@v1 - if: inputs.upload with: credentials_json: ${{ secrets.GCP_CREDENTIALS }} @@ -245,13 +244,12 @@ jobs: needs: [urbit, mingw] if: inputs.upload steps: - - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCP_CREDENTIALS }} - - - uses: google-github-actions/setup-gcloud@v1 + - uses: google-github-actions/setup-gcloud@v0.2.0 with: + version: '290.0.1' + service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} project_id: ${{ secrets.GCS_PROJECT }} + export_default_credentials: true - name: download version-string uses: actions/download-artifact@v3 diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index f86731275..6b934bc79 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -2,7 +2,6 @@ include config.mk compat_mks := $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.mk)) include $(compat_mks) -c = $(wildcard c/*.c) jets = jets/tree.c $(wildcard jets/*/*.c) noun = $(wildcard noun/*.c) ur = $(wildcard ur/*.c) @@ -14,7 +13,7 @@ bench = $(wildcard bench/*.c) compat := $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.c)) -common = $(c) $(jets) $(noun) $(ur) $(vere) $(compat) +common = $(jets) $(noun) $(ur) $(vere) $(compat) headers = $(shell find include -type f) common_objs = $(shell echo $(common) | sed 's/\.c/.o/g') diff --git a/pkg/urbit/bench/ur_bench.c b/pkg/urbit/bench/ur_bench.c index 91f04bbbf..4c0151357 100644 --- a/pkg/urbit/bench/ur_bench.c +++ b/pkg/urbit/bench/ur_bench.c @@ -7,7 +7,9 @@ static void _setup(void) { - u3m_boot_lite(1 << 24); + u3m_init(1 << 24); + u3m_pave(c3y); + u3e_init(); } /* _ames_writ_ex(): |hi packet from fake ~zod to fake ~nec diff --git a/pkg/urbit/c/defs.c b/pkg/urbit/c/defs.c deleted file mode 100644 index 93a1d9b89..000000000 --- a/pkg/urbit/c/defs.c +++ /dev/null @@ -1,96 +0,0 @@ -/// @file defs.c - -#include "c/defs.h" - -/* c3_pread(): full positioned read(), up to eof, retrying errors. -*/ -ssize_t -c3_pread(c3_i fid_i, void* buf_v, size_t len_i, off_t off_i) -{ - c3_w max_w = 128; - c3_w try_w = 0; - size_t rem_i = len_i; - ssize_t ret_i; - - do { - if ( (0 > (ret_i = pread(fid_i, buf_v, rem_i, off_i))) - && ( (++try_w == max_w) - || ( (EINTR != errno) - && (EAGAIN != errno) - && (EWOULDBLOCK != errno) ))) - { - return -1; - } - else if ( 0 == ret_i ) { - break; - } - else { - buf_v = (void*)((c3_c*)buf_v + ret_i); - rem_i -= ret_i; - off_i += ret_i; - } - - } - while ( rem_i ); - - return len_i - rem_i; -} - -/* c3_pwrite(): full positioned write(), retrying errors. -*/ -ssize_t -c3_pwrite(c3_i fid_i, const void* buf_v, size_t len_i, off_t off_i) -{ - c3_w max_w = 128; - c3_w try_w = 0; - size_t rem_i = len_i; - ssize_t ret_i; - - do { - if ( (0 > (ret_i = pwrite(fid_i, buf_v, rem_i, off_i))) - && ( (++try_w == max_w) - || ( (EINTR != errno) - && (EAGAIN != errno) - && (EWOULDBLOCK != errno) ))) - { - return -1; - } - else { - buf_v = (void*)((c3_c*)buf_v + ret_i); - rem_i -= ret_i; - off_i += ret_i; - } - } - while ( rem_i ); - - return len_i; -} - -/* c3_write(): full write(), retrying errors. -*/ -ssize_t -c3_write(c3_i fid_i, const void* buf_v, size_t len_i) -{ - c3_w max_w = 128; - c3_w try_w = 0; - size_t rem_i = len_i; - ssize_t ret_i; - - do { - if ( (0 > (ret_i = write(fid_i, buf_v, rem_i))) - && ( (++try_w == max_w) - || ( (EINTR != errno) - && (EAGAIN != errno) - && (EWOULDBLOCK != errno) ))) - { - return -1; - } - else { - buf_v = (void*)((c3_c*)buf_v + ret_i); - rem_i -= ret_i; - } - } - while ( rem_i ); - - return len_i; -} diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index 1bd142128..ebef5041f 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -144,63 +144,6 @@ int link(const char *path1, const char *path2) return -1; } -ssize_t pread(int fd, void *buf, size_t count, off_t offset) -{ - DWORD len = 0; - - OVERLAPPED overlapped = {0}; - - overlapped.OffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? - (DWORD)0 : (DWORD)((offset >> 32) & 0xFFFFFFFFL); - overlapped.Offset = (sizeof(off_t) <= sizeof(DWORD)) ? - (DWORD)offset : (DWORD)(offset & 0xFFFFFFFFL); - - HANDLE h = (HANDLE)_get_osfhandle(fd); - - if ( INVALID_HANDLE_VALUE == h ) { - errno = EBADF; - return -1; - } - - if ( !ReadFile(h, buf, count, &len, &overlapped) ) { - DWORD err = GetLastError(); - - if ( ERROR_HANDLE_EOF != err ) { - errno = err_win_to_posix(err); - return -1; - } - } - - return (ssize_t)len; -} - -ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) -{ - DWORD len = 0; - - OVERLAPPED overlapped = {0}; - - overlapped.OffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? - (DWORD)0 : (DWORD)((offset >> 32) & 0xFFFFFFFFL); - overlapped.Offset = (sizeof(off_t) <= sizeof(DWORD)) ? - (DWORD)offset : (DWORD)(offset & 0xFFFFFFFFL); - - HANDLE h = (HANDLE)_get_osfhandle(fd); - - if ( INVALID_HANDLE_VALUE == h ) { - errno = EBADF; - return -1; - } - - if ( !WriteFile(h, buf, count, &len, &overlapped) ) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - return (ssize_t)len; -} - - // from msys2 mingw-packages-dev patches // ----------------------------------------------------------------------- diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h index 80809f54b..56c35e90a 100644 --- a/pkg/urbit/compat/mingw/compat.h +++ b/pkg/urbit/compat/mingw/compat.h @@ -4,8 +4,6 @@ #define mkdir(A, B) mkdir(A) int link(const char *path1, const char *path2); -ssize_t pread(int fd, void *buf, size_t count, off_t offset); -ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); char *realpath(const char *path, char *resolved_path); int fdatasync(int fd); int utimes(const char *path, const struct timeval times[2]); diff --git a/pkg/urbit/compat/mingw/seh_handler.c b/pkg/urbit/compat/mingw/seh_handler.c index 6730435e4..ecf59f0a4 100644 --- a/pkg/urbit/compat/mingw/seh_handler.c +++ b/pkg/urbit/compat/mingw/seh_handler.c @@ -12,7 +12,7 @@ EXCEPTION_DISPOSITION _mingw_exception_filter( { if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && ExceptionRecord->ExceptionInformation[0] == 1 && - u3m_fault((void*)ExceptionRecord->ExceptionInformation[1], 1)) + u3e_fault((void*)ExceptionRecord->ExceptionInformation[1], 1)) { return ExceptionContinueExecution; } diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index d29da9824..1853ed871 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -9,8 +9,6 @@ #include "rsignal.h" #include #include "vere/vere.h" -#include "vere/mars.h" -#include "noun/events.h" #if !defined(U3_OS_mingw) #include #endif @@ -155,15 +153,6 @@ _main_init(void) // u3_Host.ops_u.has = c3y; - // demand paging (ie, file-backed mapping for the loom) - // is not yet supported on windows - // -#ifdef U3_OS_mingw - u3_Host.ops_u.map = c3n; -#else - u3_Host.ops_u.map = c3y; -#endif - u3_Host.ops_u.net = c3y; u3_Host.ops_u.lit = c3n; u3_Host.ops_u.nuu = c3n; @@ -257,7 +246,6 @@ _main_getopt(c3_i argc, c3_c** argv) { "scry-format", required_argument, NULL, 'Z' }, // { "urth-loom", required_argument, NULL, 5 }, - { "no-demand", no_argument, NULL, 6 }, // { NULL, 0, NULL, 0 }, }; @@ -278,10 +266,6 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.lut_y = lut_w; break; } - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - break; - } case 'X': { u3_Host.ops_u.pek_c = strdup(optarg); break; @@ -646,17 +630,14 @@ _cw_usage(c3_c* bin_c) "utilities:\n", " %s cram %.*s jam state:\n", " %s dock %.*s copy binary:\n", - " %s eval %.*s eval hoon:\n", " %s grab %.*s measure memory usage:\n", " %s info %.*s print pier info:\n", " %s meld %.*s deduplicate snapshot:\n", " %s pack %.*s defragment snapshot:\n", - " %s play %.*s recompute events:\n", " %s prep %.*s prepare for upgrade:\n", " %s next %.*s request upgrade:\n", " %s queu %.*s cue state:\n", " %s vere ARGS download binary:\n", - " %s vile %.*s print keyfile:\n", "\n run as a 'serf':\n", " %s serf " #ifdef U3_OS_mingw @@ -1391,8 +1372,7 @@ _cw_cram(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -1400,11 +1380,6 @@ _cw_cram(c3_i argc, c3_c* argv[]) while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -1457,7 +1432,7 @@ _cw_cram(c3_i argc, c3_c* argv[]) // save even on failure, as we just did all the work of deduplication // - u3m_save(); + u3e_save(); u3_disk_exit(log_u); if ( c3n == ret_o ) { @@ -1472,26 +1447,18 @@ _cw_cram(c3_i argc, c3_c* argv[]) static void _cw_queu(c3_i argc, c3_c* argv[]) { - c3_i ch_i, lid_i; - c3_w arg_w; - c3_c* roc_c = 0; + c3_i ch_i, lid_i; + c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, - { "replay-from", required_argument, NULL, 'r' }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; u3_Host.dir_c = _main_pier_run(argv[0]); - while ( -1 != (ch_i=getopt_long(argc, argv, "r:", lop_u, &lid_i)) ) { + while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -1502,10 +1469,6 @@ _cw_queu(c3_i argc, c3_c* argv[]) u3_Host.ops_u.lom_y = lom_w; } break; - case 'r': { - roc_c = strdup(optarg); - } break; - case '?': { fprintf(stderr, "invalid argument\r\n"); exit(1); @@ -1513,13 +1476,9 @@ _cw_queu(c3_i argc, c3_c* argv[]) } } - if ( !roc_c ) { - fprintf(stderr, "invalid command, -r $EVENT required\r\n"); - exit(1); - } - // argv[optind] is always "queu" // + if ( !u3_Host.dir_c ) { if ( optind + 1 < argc ) { u3_Host.dir_c = argv[optind + 1]; @@ -1537,10 +1496,11 @@ _cw_queu(c3_i argc, c3_c* argv[]) exit(1); } - c3_d eve_d; + c3_c* eve_c; + c3_d eve_d; - if ( 1 != sscanf(roc_c, "%" PRIu64 "", &eve_d) ) { - fprintf(stderr, "urbit: queu: invalid number '%s'\r\n", roc_c); + if ( 1 != sscanf(eve_c, "%" PRIu64 "", &eve_d) ) { + fprintf(stderr, "urbit: queu: invalid number '%s'\r\n", eve_c); exit(1); } else { @@ -1559,7 +1519,7 @@ _cw_queu(c3_i argc, c3_c* argv[]) exit(1); } - u3m_save(); + u3e_save(); u3_disk_exit(log_u); fprintf(stderr, "urbit: queu: rock loaded at event %" PRIu64 "\r\n", eve_d); @@ -1576,8 +1536,7 @@ _cw_meld(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -1585,11 +1544,6 @@ _cw_meld(c3_i argc, c3_c* argv[]) while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -1633,9 +1587,11 @@ _cw_meld(c3_i argc, c3_c* argv[]) u3C.wag_w |= u3o_hashless; u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - u3a_print_memory(stderr, "urbit: meld: gained", u3u_meld()); + pre_w = u3a_open(u3R); + u3u_meld(); + u3a_print_memory(stderr, "urbit: meld: gained", (u3a_open(u3R) - pre_w)); - u3m_save(); + u3e_save(); u3_disk_exit(log_u); u3m_stop(); } @@ -1649,9 +1605,8 @@ _cw_next(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "arch", required_argument, NULL, 'a' }, - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "arch", required_argument, NULL, 'a' }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -1663,11 +1618,6 @@ _cw_next(c3_i argc, c3_c* argv[]) u3_Host.arc_c = strdup(optarg); } break; - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -1719,8 +1669,7 @@ _cw_pack(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -1728,11 +1677,6 @@ _cw_pack(c3_i argc, c3_c* argv[]) while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -1775,161 +1719,7 @@ _cw_pack(c3_i argc, c3_c* argv[]) u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); u3a_print_memory(stderr, "urbit: pack: gained", u3m_pack()); - u3m_save(); - u3_disk_exit(log_u); - u3m_stop(); -} - -/* _cw_play_slog(): print during replay. -*/ -static void -_cw_play_slog(u3_noun hod) -{ - u3_pier_tank(0, 0, u3k(u3t(hod))); - u3z(hod); -} - -/* _cw_play_exit(): exit immediately. -*/ -static void -_cw_play_exit(c3_i int_i) -{ - // explicit fprintf to avoid allocation in u3l_log - // - fprintf(stderr, "\r\n[received keyboard stop signal, exiting]\r\n"); - raise(SIGINT); -} - -/* _cw_play(): replay events, but better. -*/ -static void -_cw_play(c3_i argc, c3_c* argv[]) -{ - c3_i ch_i, lid_i; - c3_w arg_w; - c3_o ful_o = c3n; - c3_o mel_o = c3n; - - static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "auto-meld", no_argument, NULL, 4 }, - { "no-demand", no_argument, NULL, 6 }, - { "full", required_argument, NULL, 'f' }, - { "replay-to", no_argument, NULL, 'n' }, - { NULL, 0, NULL, 0 } - }; - - u3_Host.dir_c = _main_pier_run(argv[0]); - - while ( -1 != (ch_i=getopt_long(argc, argv, "fn:", lop_u, &lid_i)) ) { - switch ( ch_i ) { - case 4: { // auto-meld - mel_o = c3y; - } break; - - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - - case c3__loom: { - c3_w lom_w; - c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); - if ( (c3n == res_o) || (lom_w < 20) ) { - fprintf(stderr, "error: --loom must be >= 20 and <= %u\r\n", u3a_bits + 2); - exit(1); - } - u3_Host.ops_u.lom_y = lom_w; - } break; - - case 'f': { - ful_o = c3y; - break; - } - - case 'n': { - u3_Host.ops_u.til_c = strdup(optarg); - break; - } - - case '?': { - fprintf(stderr, "invalid argument\r\n"); - exit(1); - } break; - } - } - - // argv[optind] is always "play" - // - - if ( !u3_Host.dir_c ) { - if ( optind + 1 < argc ) { - u3_Host.dir_c = argv[optind + 1]; - } - else { - fprintf(stderr, "invalid command, pier required\r\n"); - exit(1); - } - - optind++; - } - - if ( optind + 1 != argc ) { - fprintf(stderr, "invalid command\r\n"); - exit(1); - } - - u3_disk* log_u = _cw_disk_init(u3_Host.dir_c); // XX s/b try_aquire lock - -#if !defined(U3_OS_mingw) - // Handle SIGTSTP as if it was SIGINT. - // - // Configured here using signal() so as to be immediately available. - // - signal(SIGTSTP, _cw_play_exit); -#endif - - if ( c3y == mel_o ) { - u3C.wag_w |= u3o_auto_meld; - } - - u3C.wag_w |= u3o_hashless; - - if ( c3y == ful_o ) { - u3l_log("mars: preparing for full replay\r\n"); - u3m_init((size_t)1 << u3_Host.ops_u.lom_y); - u3e_live(u3m_pier(u3_Host.dir_c)); - u3e_yolo(); - u3m_pave(c3y); - u3j_boot(c3y); - u3A->eve_d = 0; - } - else { - u3m_boot(u3_Host.dir_c, (size_t)1 << u3_Host.ops_u.lom_y); - } - - u3C.slog_f = _cw_play_slog; - - { - u3_mars mar_u = { - .log_u = log_u, - .dir_c = u3_Host.dir_c, - .sen_d = u3A->eve_d, - .dun_d = u3A->eve_d, - .mug_l = u3r_mug(u3A->roc) - }; - c3_d eve_d = 0; - c3_c* eve_c = u3_Host.ops_u.til_c; - - if ( u3_Host.ops_u.til_c ) { - if ( 1 != sscanf(eve_c, "%" PRIu64 "", &eve_d) ) { - fprintf(stderr, "mars: replay-to invalid: '%s'\r\n", eve_c); - } - } - - u3_mars_play(&mar_u, eve_d); - } - + u3e_save(); u3_disk_exit(log_u); u3m_stop(); } @@ -1943,8 +1733,7 @@ _cw_prep(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -1952,11 +1741,6 @@ _cw_prep(c3_i argc, c3_c* argv[]) while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -2120,8 +1904,7 @@ _cw_vile(c3_i argc, c3_c* argv[]) c3_w arg_w; static struct option lop_u[] = { - { "loom", required_argument, NULL, c3__loom }, - { "no-demand", no_argument, NULL, 6 }, + { "loom", required_argument, NULL, c3__loom }, { NULL, 0, NULL, 0 } }; @@ -2129,11 +1912,6 @@ _cw_vile(c3_i argc, c3_c* argv[]) while ( -1 != (ch_i=getopt_long(argc, argv, "", lop_u, &lid_i)) ) { switch ( ch_i ) { - case 6: { // no-demand - u3_Host.ops_u.map = c3n; - u3C.wag_w |= u3o_no_demand; - } break; - case c3__loom: { c3_w lom_w; c3_o res_o = _main_readw(optarg, u3a_bits + 3, &lom_w); @@ -2221,13 +1999,11 @@ _cw_utils(c3_i argc, c3_c* argv[]) // $@ ~ :: usage // $% [%cram dir=@t] :: jam state // [%dock dir=@t] :: copy binary - // [%eval ~] :: eval hoon // [?(%grab %mass) dir=@t] :: gc // [%info dir=@t] :: print // [%meld dir=@t] :: deduplicate // [?(%next %upgrade) dir=@t] :: upgrade // [%pack dir=@t] :: defragment - // [%play dir=@t] :: recompute // [%prep dir=@t] :: prep upgrade // [%queu dir=@t eve=@ud] :: cue state // [?(%vere %fetch-vere) dir=@t] :: download vere @@ -2268,7 +2044,6 @@ _cw_utils(c3_i argc, c3_c* argv[]) case c3__meld: _cw_meld(argc, argv); return 1; case c3__next: _cw_next(argc, argv); return 2; // continue on case c3__pack: _cw_pack(argc, argv); return 1; - case c3__play: _cw_play(argc, argv); return 1; case c3__prep: _cw_prep(argc, argv); return 2; // continue on case c3__queu: _cw_queu(argc, argv); return 1; case c3__vere: _cw_vere(argc, argv); return 1; @@ -2406,12 +2181,6 @@ main(c3_i argc, u3C.wag_w |= u3o_debug_ram; } - /* Set no-demand flag. - */ - if ( !_(u3_Host.ops_u.map) ) { - u3C.wag_w |= u3o_no_demand; - } - /* Set profile flag. */ if ( _(u3_Host.ops_u.pro) ) { diff --git a/pkg/urbit/daemon/whereami.c b/pkg/urbit/daemon/whereami.c index 5e31a2490..290005766 100644 --- a/pkg/urbit/daemon/whereami.c +++ b/pkg/urbit/daemon/whereami.c @@ -1,6 +1,4 @@ -// (‑●‑●)> dual licensed under the WTFPL v2 and MIT licenses -// without any warranty. -// by Gregory Pakosz (@gpakosz) +// (‑●‑●)> released under the WTFPL v2 license, by Gregory Pakosz (@gpakosz) // https://github.com/gpakosz/whereami // in case you want to #include "whereami.c" in a larger compilation unit @@ -12,15 +10,6 @@ extern "C" { #endif -#if defined(__linux__) || defined(__CYGWIN__) -#undef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE -#elif defined(__APPLE__) -#undef _DARWIN_C_SOURCE -#define _DARWIN_C_SOURCE -#define _DARWIN_BETTER_REALPATH -#endif - #if !defined(WAI_MALLOC) || !defined(WAI_FREE) || !defined(WAI_REALLOC) #include #endif @@ -57,9 +46,7 @@ extern "C" { #if defined(_WIN32) -#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN -#endif #if defined(_MSC_VER) #pragma warning(push, 3) #endif @@ -68,7 +55,6 @@ extern "C" { #if defined(_MSC_VER) #pragma warning(pop) #endif -#include static int WAI_PREFIX(getModulePath_)(HMODULE module, char* out, int capacity, int* dirname_length) { @@ -76,9 +62,8 @@ static int WAI_PREFIX(getModulePath_)(HMODULE module, char* out, int capacity, i wchar_t buffer2[MAX_PATH]; wchar_t* path = NULL; int length = -1; - bool ok; - for (ok = false; !ok; ok = true) + for (;;) { DWORD size; int length_, length__; @@ -134,12 +119,14 @@ static int WAI_PREFIX(getModulePath_)(HMODULE module, char* out, int capacity, i } length = length__; + + break; } if (path != buffer1) WAI_FREE(path); - return ok ? length : -1; + return length; } WAI_NOINLINE WAI_FUNCSPEC @@ -169,7 +156,7 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) return length; } -#elif defined(__linux__) || defined(__CYGWIN__) || defined(__sun) || defined(WAI_USE_PROC_SELF_EXE) +#elif defined(__linux__) || defined(__CYGWIN__) || defined(__sun) #include #include @@ -183,7 +170,6 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) #define __STDC_FORMAT_MACROS #endif #include -#include #if !defined(WAI_PROC_SELF_EXE) #if defined(__sun) @@ -199,9 +185,8 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) char buffer[PATH_MAX]; char* resolved = NULL; int length = -1; - bool ok; - for (ok = false; !ok; ok = true) + for (;;) { resolved = realpath(WAI_PROC_SELF_EXE, buffer); if (!resolved) @@ -226,9 +211,11 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) } } } + + break; } - return ok ? length : -1; + return length; } #if !defined(WAI_PROC_SELF_MAPS_RETRY) @@ -248,7 +235,6 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) #include #include #endif -#include WAI_NOINLINE WAI_FUNCSPEC int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) @@ -295,24 +281,15 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) &&buffer[length - 4] == '.') { int fd = open(path, O_RDONLY); - if (fd == -1) - { - length = -1; // retry - break; - } + char* begin; + char* p; - char* begin = (char*)mmap(0, offset, PROT_READ, MAP_SHARED, fd, 0); - if (begin == MAP_FAILED) - { - close(fd); - length = -1; // retry - break; - } + begin = (char*)mmap(0, offset, PROT_READ, MAP_SHARED, fd, 0); + p = begin + offset; - char* p = begin + offset - 30; // minimum size of local file header while (p >= begin) // scan backwards { - if (*((uint32_t*)p) == 0x04034b50UL) // local file header signature found + if (*((uint32_t*)p) == 0x04034b50UL) // local file header found { uint16_t length_ = *((uint16_t*)(p + 26)); @@ -326,7 +303,7 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) break; } - --p; + p -= 4; } munmap(begin, offset); @@ -364,17 +341,20 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) break; } + if (maps) + fclose(maps); + return length; } #elif defined(__APPLE__) +#define _DARWIN_BETTER_REALPATH #include #include #include #include #include -#include WAI_FUNCSPEC int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) @@ -384,9 +364,8 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) char* path = buffer1; char* resolved = NULL; int length = -1; - bool ok; - for (ok = false; !ok; ok = true) + for (;;) { uint32_t size = (uint32_t)sizeof(buffer1); if (_NSGetExecutablePath(path, &size) == -1) @@ -419,12 +398,14 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) } } } + + break; } if (path != buffer1) WAI_FREE(path); - return ok ? length : -1; + return length; } WAI_NOINLINE WAI_FUNCSPEC @@ -478,7 +459,6 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) #include #include #include -#include #if !defined(WAI_PROC_SELF_EXE) #define WAI_PROC_SELF_EXE "/proc/self/exefile" @@ -492,9 +472,8 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) char* resolved = NULL; FILE* self_exe = NULL; int length = -1; - bool ok; - for (ok = false; !ok; ok = true) + for (;;) { self_exe = fopen(WAI_PROC_SELF_EXE, "r"); if (!self_exe) @@ -526,11 +505,13 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) } } } + + break; } fclose(self_exe); - return ok ? length : -1; + return length; } WAI_FUNCSPEC @@ -578,7 +559,7 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) } #elif defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__FreeBSD_kernel__) || defined(__NetBSD__) || defined(__OpenBSD__) + defined(__FreeBSD_kernel__) || defined(__NetBSD__) #include #include @@ -586,116 +567,6 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) #include #include #include -#include - -#if defined(__OpenBSD__) - -#include - -WAI_FUNCSPEC -int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) -{ - char buffer1[4096]; - char buffer2[PATH_MAX]; - char buffer3[PATH_MAX]; - char** argv = (char**)buffer1; - char* resolved = NULL; - int length = -1; - bool ok; - - for (ok = false; !ok; ok = true) - { - int mib[4] = { CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV }; - size_t size; - - if (sysctl(mib, 4, NULL, &size, NULL, 0) != 0) - break; - - if (size > sizeof(buffer1)) - { - argv = (char**)WAI_MALLOC(size); - if (!argv) - break; - } - - if (sysctl(mib, 4, argv, &size, NULL, 0) != 0) - break; - - if (strchr(argv[0], '/')) - { - resolved = realpath(argv[0], buffer2); - if (!resolved) - break; - } - else - { - const char* PATH = getenv("PATH"); - if (!PATH) - break; - - size_t argv0_length = strlen(argv[0]); - - const char* begin = PATH; - while (1) - { - const char* separator = strchr(begin, ':'); - const char* end = separator ? separator : begin + strlen(begin); - - if (end - begin > 0) - { - if (*(end -1) == '/') - --end; - - if (((end - begin) + 1 + argv0_length + 1) <= sizeof(buffer2)) - { - memcpy(buffer2, begin, end - begin); - buffer2[end - begin] = '/'; - memcpy(buffer2 + (end - begin) + 1, argv[0], argv0_length + 1); - - resolved = realpath(buffer2, buffer3); - if (resolved) - break; - } - } - - if (!separator) - break; - - begin = ++separator; - } - - if (!resolved) - break; - } - - length = (int)strlen(resolved); - if (length <= capacity) - { - memcpy(out, resolved, length); - - if (dirname_length) - { - int i; - - for (i = length - 1; i >= 0; --i) - { - if (out[i] == '/') - { - *dirname_length = i; - break; - } - } - } - } - } - - if (argv != (char**)buffer1) - WAI_FREE(argv); - - return ok ? length : -1; -} - -#else WAI_FUNCSPEC int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) @@ -705,18 +576,13 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) char* path = buffer1; char* resolved = NULL; int length = -1; - bool ok; - for (ok = false; !ok; ok = true) + for (;;) { -#if defined(__NetBSD__) - int mib[4] = { CTL_KERN, KERN_PROC_ARGS, -1, KERN_PROC_PATHNAME }; -#else int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 }; -#endif size_t size = sizeof(buffer1); - if (sysctl(mib, 4, path, &size, NULL, 0) != 0) + if (sysctl(mib, (u_int)(sizeof(mib) / sizeof(mib[0])), path, &size, NULL, 0) != 0) break; resolved = realpath(path, buffer2); @@ -742,12 +608,15 @@ int WAI_PREFIX(getExecutablePath)(char* out, int capacity, int* dirname_length) } } } + + break; } - return ok ? length : -1; -} + if (path != buffer1) + WAI_FREE(path); -#endif + return length; +} WAI_NOINLINE WAI_FUNCSPEC int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) @@ -801,4 +670,4 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length) #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/pkg/urbit/daemon/whereami.h b/pkg/urbit/daemon/whereami.h index d5edffb8c..6c81af818 100644 --- a/pkg/urbit/daemon/whereami.h +++ b/pkg/urbit/daemon/whereami.h @@ -1,6 +1,4 @@ -// (‑●‑●)> dual licensed under the WTFPL v2 and MIT licenses -// without any warranty. -// by Gregory Pakosz (@gpakosz) +// (‑●‑●)> released under the WTFPL v2 license, by Gregory Pakosz (@gpakosz) // https://github.com/gpakosz/whereami #ifndef WHEREAMI_H @@ -64,4 +62,4 @@ int WAI_PREFIX(getModulePath)(char* out, int capacity, int* dirname_length); } #endif -#endif // #ifndef WHEREAMI_H \ No newline at end of file +#endif // #ifndef WHEREAMI_H diff --git a/pkg/urbit/include/all.h b/pkg/urbit/include/all.h index 402b88583..ae647d865 100644 --- a/pkg/urbit/include/all.h +++ b/pkg/urbit/include/all.h @@ -11,6 +11,7 @@ # include "noun/aliases.h" // general u3 # include "noun/allocate.h" // u3a: allocation +# include "noun/events.h" // u3e: persistence # include "noun/hashtable.h" // u3h: hashtables # include "noun/imprison.h" // u3i: noun construction # include "noun/jets.h" // u3j: jet control diff --git a/pkg/urbit/include/c/defs.h b/pkg/urbit/include/c/defs.h index a667b85a2..b649a8866 100644 --- a/pkg/urbit/include/c/defs.h +++ b/pkg/urbit/include/c/defs.h @@ -1,11 +1,6 @@ #ifndef C3_DEFS_H #define C3_DEFS_H -#include "c/portable.h" -#include "c/types.h" - -#include - /** Loobeans - inverse booleans to match nock. **/ # define c3y 0 @@ -162,28 +157,4 @@ # define c3_fopen(a, b) ({ \ fopen(a, b);}) - /** i/o wrappers - *** - *** these handle partial success and retry ephemeral errors - *** up to hardcoded max try count, either reading/writing fully - *** (up to EOF on read) or returning on error. - *** - *** a wrapper for read() is not provided, as file cursor position - *** is undefined on error. use pread() or loop yourself. - **/ - /* c3_pread(): full positioned read(), up to eof, retrying errors. - */ - ssize_t - c3_pread(c3_i fid_i, void* buf_v, size_t len_i, off_t off_i); - - /* c3_pwrite(): full positioned write(), retrying errors. - */ - ssize_t - c3_pwrite(c3_i fid_i, const void* buf_v, size_t len_i, off_t off_i); - - /* c3_write(): full write(), retrying errors. - */ - ssize_t - c3_write(c3_i fid_i, const void* buf_v, size_t len_i); - #endif /* ifndef C3_DEFS_H */ diff --git a/pkg/urbit/include/noun/allocate.h b/pkg/urbit/include/noun/allocate.h index 03b219c08..964de538b 100644 --- a/pkg/urbit/include/noun/allocate.h +++ b/pkg/urbit/include/noun/allocate.h @@ -645,11 +645,6 @@ void u3a_print_memory(FILE* fil_u, c3_c* cap_c, c3_w wor_w); - /* u3a_prof(): mark/measure/print memory profile. RETAIN. - */ - c3_w - u3a_prof(FILE* fil_u, c3_w den_w, u3_noun mas); - /* u3a_maid(): maybe print memory. */ c3_w diff --git a/pkg/urbit/include/noun/events.h b/pkg/urbit/include/noun/events.h index a8860f53d..c1b95cde8 100644 --- a/pkg/urbit/include/noun/events.h +++ b/pkg/urbit/include/noun/events.h @@ -39,22 +39,13 @@ /* u3e_pool: entire memory system. */ typedef struct _u3e_pool { - c3_c* dir_c; // checkpoint dir + c3_c* dir_c; // path to c3_w dit_w[u3a_pages >> 5]; // touched since last save c3_w pag_w; // number of pages (<= u3a_pages) - c3_w gar_w; // guard page u3e_image nor_u; // north segment u3e_image sou_u; // south segment } u3e_pool; - /* u3e_flaw: loom fault result. - */ - typedef enum { - u3e_flaw_sham = 0, // bogus state - u3e_flaw_base = 1, // vm fail (mprotect) - u3e_flaw_meme = 2, // bail:meme - u3e_flaw_good = 3 // handled - } u3e_flaw; /** Globals. **/ @@ -69,26 +60,31 @@ /** Functions. **/ - /* u3e_fault(): handle a memory fault. + /* u3e_fault(): handle a memory event with libsigsegv protocol. */ - u3e_flaw - u3e_fault(u3_post low_p, u3_post hig_p, u3_post off_p); + c3_i + u3e_fault(void* adr_v, c3_i ser_i); - /* u3e_save(): update the checkpoint. + /* u3e_save(): */ void - u3e_save(u3_post low_p, u3_post hig_p); + u3e_save(void); /* u3e_live(): start the persistence system. Return c3y if no image. */ c3_o - u3e_live(c3_c* dir_c); + u3e_live(c3_o nuu_o, c3_c* dir_c); /* u3e_yolo(): disable dirty page tracking, read/write whole loom. */ c3_o u3e_yolo(void); + /* u3e_foul(): dirty all the pages of the loom. + */ + void + u3e_foul(void); + /* u3e_init(): initialize guard page tracking. */ void diff --git a/pkg/urbit/include/noun/manage.h b/pkg/urbit/include/noun/manage.h index 0ff5b504f..72e29f533 100644 --- a/pkg/urbit/include/noun/manage.h +++ b/pkg/urbit/include/noun/manage.h @@ -8,11 +8,6 @@ c3_d u3m_boot(c3_c* dir_c, size_t len_i); - /* u3m_pier(): make a pier. - */ - c3_c* - u3m_pier(c3_c* dir_c); - /* u3m_boot_lite(): start without checkpointing. */ c3_d @@ -39,21 +34,6 @@ c3_i u3m_bail(c3_m how_m) __attribute__((noreturn)); - /* u3m_fault(): handle a memory event with libsigsegv protocol. - */ - c3_i - u3m_fault(void* adr_v, c3_i ser_i); - - /* u3m_save(): update the checkpoint. - */ - void - u3m_save(void); - - /* u3m_ward(): tend the guard page. - */ - void - u3m_ward(void); - /* u3m_init(): start the environment. */ void @@ -137,7 +117,7 @@ /* u3m_water(): produce high and low watermarks. Asserts u3R == u3H. */ void - u3m_water(u3_post* low_p, u3_post* hig_p); + u3m_water(c3_w *low_w, c3_w *hig_w); /* u3m_pretty(): dumb prettyprint to string. RETAIN. */ diff --git a/pkg/urbit/include/noun/options.h b/pkg/urbit/include/noun/options.h index 26d081f4e..c1fd58c40 100644 --- a/pkg/urbit/include/noun/options.h +++ b/pkg/urbit/include/noun/options.h @@ -22,17 +22,15 @@ ** _check flags are set inside u3 and heard outside it. */ enum u3o_flag { // execution flags - u3o_debug_ram = 1 << 0, // debug: gc - u3o_debug_cpu = 1 << 1, // debug: profile - u3o_check_corrupt = 1 << 2, // check: gc memory - u3o_check_fatal = 1 << 3, // check: unrecoverable - u3o_verbose = 1 << 4, // be remarkably wordy - u3o_dryrun = 1 << 5, // don't touch checkpoint - u3o_quiet = 1 << 6, // disable ~& - u3o_hashless = 1 << 7, // disable hashboard - u3o_trace = 1 << 8, // enables trace dumping - u3o_auto_meld = 1 << 9, // enables meld under pressure - u3o_no_demand = 1 << 10 // disables demand paging + u3o_debug_ram = 0x1, // debug: gc + u3o_debug_cpu = 0x2, // debug: profile + u3o_check_corrupt = 0x4, // check: gc memory + u3o_check_fatal = 0x8, // check: unrecoverable + u3o_verbose = 0x10, // be remarkably wordy + u3o_dryrun = 0x20, // don't touch checkpoint + u3o_quiet = 0x40, // disable ~& + u3o_hashless = 0x80, // disable hashboard + u3o_trace = 0x100 // enables trace dumping }; /** Globals. diff --git a/pkg/urbit/include/noun/urth.h b/pkg/urbit/include/noun/urth.h index 2c6617f4b..3fe29792e 100644 --- a/pkg/urbit/include/noun/urth.h +++ b/pkg/urbit/include/noun/urth.h @@ -5,7 +5,7 @@ **/ /* u3u_meld(): globally deduplicate memory. */ - c3_w + void u3u_meld(void); /* u3u_cram(): globably deduplicate memory, and write a rock to disk. diff --git a/pkg/urbit/include/noun/vortex.h b/pkg/urbit/include/noun/vortex.h index ed24e691e..92320620a 100644 --- a/pkg/urbit/include/noun/vortex.h +++ b/pkg/urbit/include/noun/vortex.h @@ -87,11 +87,6 @@ u3_noun u3v_poke(u3_noun ovo); - /* u3v_poke_sure(): inject an event, saving new state if successful. - */ - c3_o - u3v_poke_sure(c3_w mil_w, u3_noun eve, u3_noun* pro); - /* u3v_tank(): dump single tank. */ void diff --git a/pkg/urbit/include/vere/db/lmdb.h b/pkg/urbit/include/vere/db/lmdb.h index a3ca6a414..573a68fa4 100644 --- a/pkg/urbit/include/vere/db/lmdb.h +++ b/pkg/urbit/include/vere/db/lmdb.h @@ -6,17 +6,6 @@ /* lmdb api wrapper */ - /* u3_lmdb_iter: event iterator - */ - typedef struct _u3_lmdb_walk { - MDB_txn* txn_u; // transaction handle - MDB_dbi mdb_u; // db handle - MDB_cursor* cur_u; // db cursor - c3_o red_o; // have we read from this yet? - c3_d nex_d; // next event number - c3_d las_d; // final event number, inclusive - } u3_lmdb_walk; - /* u3_lmdb_init(): open lmdb at [pax_c], mmap up to [siz_i]. */ MDB_env* @@ -72,22 +61,4 @@ size_t val_i, void* val_p); - /* u3_lmdb_walk_init(): initialize db iterator. - */ - c3_o - u3_lmdb_walk_init(MDB_env* env_u, - u3_lmdb_walk* itr_u, - c3_d nex_d, - c3_d las_d); - - /* u3_lmdb_walk_next(): synchronously read next event from iterator. - */ - c3_o - u3_lmdb_walk_next(u3_lmdb_walk* itr_u, size_t* len_i, void** buf_v); - - /* u3_lmdb_walk_done(): close iterator. - */ - void - u3_lmdb_walk_done(u3_lmdb_walk* itr_u); - #endif /* ifndef U3_VERE_DB_LMDB_H */ diff --git a/pkg/urbit/include/vere/mars.h b/pkg/urbit/include/vere/mars.h deleted file mode 100644 index a445af0ac..000000000 --- a/pkg/urbit/include/vere/mars.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef U3_VERE_MARS_H -#define U3_VERE_MARS_H - - /** Data types. - **/ - /* u3_mars: the urbit state machine. - */ - typedef struct _u3_mars { - c3_d key_d[4]; // disk key - u3_disk* log_u; // event log - c3_c* dir_c; // execution directory (pier) - c3_d sen_d; // last event requested - c3_d dun_d; // last event processed - c3_l mug_l; // hash of state - } u3_mars; - - /* u3_mars_play(): replay logged events up to [eve_d]. - */ - void - u3_mars_play(u3_mars* mar_u, c3_d eve_d); - -#endif /* ifndef U3_VERE_MARS_H */ diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index ed718fd79..d02771343 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -313,7 +313,6 @@ c3_c* puf_c; // -Z, scry result format c3_o con; // run conn c3_o doc; // dock binary in pier - c3_o map; // --no-demand (reversed) } u3_opts; /* u3_host: entire host. @@ -563,10 +562,6 @@ u3_info put_u; // write queue } u3_disk; - /* u3_disk_walk: opaque event log iterator. - */ - typedef struct _u3_disk_walk u3_disk_walk; - /* u3_psat: pier state. */ typedef enum { @@ -947,23 +942,6 @@ u3_disk* u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u); - /* u3_disk_etch(): serialize an event for persistence. RETAIN [eve] - */ - size_t - u3_disk_etch(u3_disk* log_u, - u3_noun eve, - c3_l mug_l, - c3_y** out_y); - - /* u3_disk_sift(): parse a persisted event buffer. - */ - c3_o - u3_disk_sift(u3_disk* log_u, - size_t len_i, - c3_y* dat_y, - c3_l* mug_l, - u3_noun* job); - /* u3_disk_info(): status info as $mass. */ u3_noun @@ -1015,33 +993,6 @@ void u3_disk_plan(u3_disk* log_u, u3_fact* tac_u); - /* u3_disk_read_list(): synchronously read a cons list of events. - */ - u3_weak - u3_disk_read_list(u3_disk* log_u, c3_d eve_d, c3_d len_d, c3_l* mug_l); - - /* u3_disk_walk_init(): init iterator. - */ - u3_disk_walk* - u3_disk_walk_init(u3_disk* log_u, - c3_d eve_d, - c3_d len_d); - - /* u3_disk_walk_live(): check if live. - */ - c3_o - u3_disk_walk_live(u3_disk_walk* wok_u); - - /* u3_disk_walk_live(): get next fact. - */ - c3_o - u3_disk_walk_step(u3_disk_walk* wok_u, u3_fact* tac_u); - - /* u3_disk_walk_done(): close iterator. - */ - void - u3_disk_walk_done(u3_disk_walk* wok_u); - /* u3_lord_init(): start serf. */ u3_lord* @@ -1510,6 +1461,11 @@ void u3_daemon_init(); + /* u3_write_fd(): retry interrupts, continue partial writes, assert errors. + */ + void + u3_write_fd(c3_i fid_i, const void* buf_v, size_t len_i); + c3_w u3_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); diff --git a/pkg/urbit/noun/allocate.c b/pkg/urbit/noun/allocate.c index 093c6fefb..f7ce67958 100644 --- a/pkg/urbit/noun/allocate.c +++ b/pkg/urbit/noun/allocate.c @@ -425,18 +425,8 @@ _ca_willoc(c3_w len_w, c3_w ald_w, c3_w alp_w) alp_w = (alp_w + c3_wiseof(u3a_box)) % ald_w; - /* XX: this logic is totally bizarre, but preserve it. - ** - ** This means we use the next size bigger instead of the "correct" - ** size. For example, a 20 word allocation will be freed into free - ** list 2 but will be allocated from free list 3. - ** - ** This is important to preserve because the sequential search may be - ** very slow. On a real-world task involving many compilations, - ** removing this line made this function appear in ~80% of samples. - ** - ** For reference, this was added in cgyarvin/urbit ffed9e748d8f6c. - */ + // XX: this logic is totally bizarre, but preserve it. + // if ( (sel_w != 0) && (sel_w != u3a_fbox_no - 1) ) { sel_w += 1; } @@ -1627,7 +1617,7 @@ u3a_mark_ptr(void* ptr_v) if ( 0 == box_u->eus_w ) { siz_w = box_u->siz_w; } - else if ( 0xffffffff == box_u->eus_w ) { // see u3a_prof() + else if ( 0xffffffff == box_u->eus_w ) { // see _raft_prof() siz_w = 0xffffffff; box_u->eus_w = 0; } @@ -1645,7 +1635,7 @@ u3a_mark_ptr(void* ptr_v) else { c3_assert(use_ws != 0); - if ( 0x80000000 == (c3_w)use_ws ) { // see u3a_prof() + if ( 0x80000000 == (c3_w)use_ws ) { // see _raft_prof() use_ws = -1; siz_w = 0xffffffff; } @@ -1916,7 +1906,7 @@ u3a_print_memory(FILE* fil_u, c3_c* cap_c, c3_w wor_w) if ( byt_w ) { if ( gib_w ) { fprintf(fil_u, "%s: GB/%d.%03d.%03d.%03d\r\n", - cap_c, gib_w, mib_w, kib_w, bib_w); + cap_c, gib_w, mib_w, kib_w, bib_w); } else if ( mib_w ) { fprintf(fil_u, "%s: MB/%d.%03d.%03d\r\n", cap_c, mib_w, kib_w, bib_w); @@ -1941,122 +1931,6 @@ u3a_maid(FILE* fil_u, c3_c* cap_c, c3_w wor_w) return wor_w; } -/* _ca_print_memory(): un-captioned u3a_print_memory(). -*/ -static void -_ca_print_memory(FILE* fil_u, c3_w wor_w) -{ - c3_w byt_w = (wor_w * 4); - c3_w gib_w = (byt_w / 1000000000); - c3_w mib_w = (byt_w % 1000000000) / 1000000; - c3_w kib_w = (byt_w % 1000000) / 1000; - c3_w bib_w = (byt_w % 1000); - - if ( gib_w ) { - fprintf(fil_u, "GB/%d.%03d.%03d.%03d\r\n", - gib_w, mib_w, kib_w, bib_w); - } - else if ( mib_w ) { - fprintf(fil_u, "MB/%d.%03d.%03d\r\n", mib_w, kib_w, bib_w); - } - else if ( kib_w ) { - fprintf(fil_u, "KB/%d.%03d\r\n", kib_w, bib_w); - } - else { - fprintf(fil_u, "B/%d\r\n", bib_w); - } -} - -/* u3a_prof(): mark/measure/print memory profile. RETAIN. -*/ -c3_w -u3a_prof(FILE* fil_u, c3_w den_w, u3_noun mas) -{ - c3_w tot_w = 0; - u3_noun h_mas, t_mas; - - if ( c3n == u3r_cell(mas, &h_mas, &t_mas) ) { - fprintf(fil_u, "%.*smistyped mass\r\n", den_w, ""); - return tot_w; - } - else if ( _(u3du(h_mas)) ) { - fprintf(fil_u, "%.*smistyped mass head\r\n", den_w, ""); - { - c3_c* lab_c = u3m_pretty(h_mas); - fprintf(fil_u, "h_mas: %s", lab_c); - c3_free(lab_c); - } - return tot_w; - } - else { - { - c3_c* lab_c = u3m_pretty(h_mas); - fprintf(fil_u, "%*s%s: ", den_w, "", lab_c); - c3_free(lab_c); - } - - u3_noun it_mas, tt_mas; - - if ( c3n == u3r_cell(t_mas, &it_mas, &tt_mas) ) { - fprintf(fil_u, "%*smistyped mass tail\r\n", den_w, ""); - return tot_w; - } - else if ( c3y == it_mas ) { - tot_w += u3a_mark_noun(tt_mas); - _ca_print_memory(fil_u, tot_w); - -#if 1 - /* The basic issue here is that tt_mas is included in .sac - * (the whole profile), so they can't both be roots in the - * normal sense. When we mark .sac later on, we want tt_mas - * to appear unmarked, but its children should be already - * marked. - * - * see u3a_mark_ptr(). - */ - if ( _(u3a_is_dog(tt_mas)) ) { - u3a_box* box_u = u3a_botox(u3a_to_ptr(tt_mas)); -#ifdef U3_MEMORY_DEBUG - if ( 1 == box_u->eus_w ) { - box_u->eus_w = 0xffffffff; - } - else { - box_u->eus_w -= 1; - } -#else - if ( -1 == (c3_w)box_u->use_w ) { - box_u->use_w = 0x80000000; - } - else { - box_u->use_w += 1; - } -#endif - } -#endif - - return tot_w; - } - else if ( c3n == it_mas ) { - fprintf(fil_u, "\r\n"); - - while ( _(u3du(tt_mas)) ) { - tot_w += u3a_prof(fil_u, den_w+2, u3h(tt_mas)); - tt_mas = u3t(tt_mas); - } - - fprintf(fil_u, "%*s--", den_w, ""); - _ca_print_memory(fil_u, tot_w); - - return tot_w; - - } - else { - fprintf(fil_u, "%*smistyped (strange) mass tail\r\n", den_w, ""); - return tot_w; - } - } -} - /* u3a_mark_road(): mark ad-hoc persistent road structures. */ c3_w diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index 7b1f78b21..ceec1bd41 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -7,7 +7,7 @@ //! - page: 16KB chunk of the loom. //! - north segment (u3e_image, north.bin): low contiguous loom pages, //! (in practice, the home road heap). indexed from low to high: -//! in-order on disk. in a file-backed mapping by default. +//! in-order on disk. //! - south segment (u3e_image, south.bin): high contiguous loom pages, //! (in practice, the home road stack). indexed from high to low: //! reversed on disk. @@ -20,8 +20,8 @@ //! - with the loom already mapped, all pages are marked dirty in a bitmap. //! - if snapshot is missing or partial, empty segments are created. //! - if a patch is present, it's applied (crash recovery). -//! - snapshot segments are mapped or copied onto the loom; -//! all included pages are marked clean and protected (read-only). +//! - snapshot segments are copied onto the loom; all included pages +//! are marked clean and protected (read-only). //! //! #### page faults (u3e_fault()) //! @@ -47,60 +47,43 @@ //! contiguous free space). //! - patch pages are written to memory.bin, metadata to control.bin. //! - the patch is applied to the snapshot segments, in-place. -//! - memory protections (and file-backed mappings) are re-established. //! - patch files are deleted. //! -//! ### invariants -//! -//! definitions: -//! - a clean page is PROT_READ and 0 in the bitmap -//! - a dirty page is (PROT_READ|PROT_WRITE) and 1 in the bitmap -//! - the guard page is PROT_NONE and 1 in the bitmap -//! -//! assumptions: -//! - all memory access patterns are outside-in, a page at a time -//! - ad-hoc exceptions are supported by calling u3e_ward() -//! -//! - there is a single guard page, between the segments -//! - dirty pages only become clean by being: -//! - loaded from a snapshot during initialization -//! - present in a snapshot after save -//! - clean pages only become dirty by being: -//! - modified (and caught by the fault handler) -//! - orphaned due to segment truncation (explicitly dirtied) -//! - at points of quiescence (initialization, after save) -//! - all pages of the north and south segments are clean -//! - all other pages are dirty -//! //! ### limitations //! //! - loom page size is fixed (16 KB), and must be a multiple of the -//! system page size. -//! - update atomicity is crucial: -//! - patch application must either completely succeed or -//! leave on-disk segments (memory image) intact. -//! - unapplied patches can be discarded (triggering event replay), -//! but once patch application begins it must succeed. -//! - may require integration into the overall signal-handling regime. -//! - many errors are handled with assertions. +//! system page size. (can the size vary at runtime give south.bin's +//! reversed order? alternately, if system page size > ours, the fault +//! handler could dirty N pages at a time.) +//! - update atomicity is suspect: patch application must either +//! completely succeed or leave on-disk segments intact. unapplied +//! patches can be discarded (triggering event replay), but once +//! patch application begins it must succeed (can fail if disk is full). +//! may require integration into the overall signal-handling regime. +//! - any errors are handled with assertions; failed/partial writes are not +//! retried. //! //! ### enhancements //! //! - use platform specific page fault mechanism (mach rpc, userfaultfd, &c). -//! - parallelism (conflicts with demand paging) +//! - implement demand paging / heuristic page-out. +//! - add a guard page in the middle of the loom to reactively handle stack overflow. +//! - parallelism //! #include "all.h" -#include "noun/events.h" #include #include #include +// Base loom offset of the guard page. +static u3p(c3_w) gar_pag_p; + //! Urbit page size in 4-byte words. -#define pag_wiz_i ((size_t)1 << u3a_page) +static const size_t pag_wiz_i = 1 << u3a_page; //! Urbit page size in bytes. -#define pag_siz_i ((size_t)1 << (u3a_page + 2)) +static const size_t pag_siz_i = sizeof(c3_w) * pag_wiz_i; #ifdef U3_SNAPSHOT_VALIDATION /* Image check. @@ -117,7 +100,7 @@ static c3_w _ce_check_page(c3_w pag_w) { c3_w* mem_w = u3_Loom + (pag_w << u3a_page); - c3_w mug_w = u3r_mug_words(mem_w, pag_wiz_i); + c3_w mug_w = u3r_mug_words(mem_w, (1 << u3a_page)); return mug_w; } @@ -135,8 +118,8 @@ u3e_check(c3_c* cap_c) u3m_water(&nwr_w, &swu_w); - nor_w = (nwr_w + (pag_wiz_i - 1)) >> u3a_page; - sou_w = (swu_w + (pag_wiz_i - 1)) >> u3a_page; + nor_w = (nwr_w + ((1 << u3a_page) - 1)) >> u3a_page; + sou_w = (swu_w + ((1 << u3a_page) - 1)) >> u3a_page; } /* Count dirty pages. @@ -162,115 +145,156 @@ u3e_check(c3_c* cap_c) u3l_log("%s: sum %x (%x, %x)\r\n", cap_c, sum_w, nor_w, sou_w); } } + +/* _ce_maplloc(): crude off-loom allocator. +*/ +static void* +_ce_maplloc(c3_w len_w) +{ + void* map_v; + + map_v = mmap(0, + len_w, + (PROT_READ | PROT_WRITE), + (MAP_ANON | MAP_PRIVATE), + -1, 0); + + if ( -1 == (c3_ps)map_v ) { + c3_assert(0); + } + else { + c3_w* map_w = map_v; + + map_w[0] = len_w; + + return map_w + 1; + } +} + +/* _ce_mapfree(): crude off-loom allocator. +*/ +static void +_ce_mapfree(void* map_v) +{ + c3_w* map_w = map_v; + c3_i res_i; + + map_w -= 1; + res_i = munmap(map_w, map_w[0]); + + c3_assert(0 == res_i); +} #endif -/* _ce_flaw_protect(): protect page after fault. -*/ -static inline c3_i -_ce_flaw_protect(c3_w pag_w) -{ - if ( 0 != mprotect((void *)(u3_Loom + (pag_w << u3a_page)), - pag_siz_i, - (PROT_READ | PROT_WRITE)) ) - { - fprintf(stderr, "loom: fault mprotect (%u): %s\r\n", - pag_w, strerror(errno)); - return 1; - } - - return 0; -} - #ifdef U3_GUARD_PAGE -/* _ce_ward_protect(): protect the guard page. -*/ -static inline c3_i -_ce_ward_protect(void) +//! Place a guard page at the (approximate) middle of the free space between +//! the heap and stack of the current road, bailing if memory has been +//! exhausted. +static c3_i +_ce_center_guard_page(void) { - if ( 0 != mprotect((void *)(u3_Loom + (u3P.gar_w << u3a_page)), - pag_siz_i, - PROT_NONE) ) - { - fprintf(stderr, "loom: failed to protect guard page (%u): %s\r\n", - u3P.gar_w, strerror(errno)); - return 1; + u3p(c3_w) bot_p, top_p; + if ( !u3R ) { + top_p = u3a_outa(u3_Loom + u3C.wor_i); + bot_p = u3a_outa(u3_Loom); + } + else if ( c3y == u3a_is_north(u3R) ) { + top_p = c3_rod(u3R->cap_p, pag_wiz_i); + bot_p = c3_rop(u3R->hat_p, pag_wiz_i); + } + else { + top_p = c3_rod(u3R->hat_p, pag_wiz_i); + bot_p = c3_rop(u3R->cap_p, pag_wiz_i); } + if ( top_p < bot_p + pag_wiz_i ) { + fprintf(stderr, + "loom: not enough memory to recenter the guard page\r\n"); + goto bail; + } + const u3p(c3_w) old_gar_p = gar_pag_p; + const c3_w mid_p = (top_p - bot_p) / 2; + gar_pag_p = bot_p + c3_rod(mid_p, pag_wiz_i); + if ( old_gar_p == gar_pag_p ) { + fprintf(stderr, + "loom: can't move the guard page to the same location" + " (base address %p)\r\n", + u3a_into(gar_pag_p)); + goto bail; + } + + if ( -1 == mprotect(u3a_into(gar_pag_p), pag_siz_i, PROT_NONE) ) { + fprintf(stderr, + "loom: failed to protect the guard page " + "(base address %p): %s\r\n", + u3a_into(gar_pag_p), + strerror(errno)); + goto fail; + } + + return 1; + +bail: + u3m_signal(c3__meme); +fail: return 0; } - -/* _ce_ward_post(): set the guard page. -*/ -static inline c3_i -_ce_ward_post(c3_w nop_w, c3_w sop_w) -{ - u3P.gar_w = nop_w + ((sop_w - nop_w) / 2); - return _ce_ward_protect(); -} - -/* _ce_ward_clip(): hit the guard page. -*/ -static inline u3e_flaw -_ce_ward_clip(c3_w nop_w, c3_w sop_w) -{ - c3_w old_w = u3P.gar_w; - - if ( !u3P.gar_w || ((nop_w < u3P.gar_w) && (sop_w > u3P.gar_w)) ) { - fprintf(stderr, "loom: ward bogus (>%u %u %u<)\r\n", - nop_w, u3P.gar_w, sop_w); - return u3e_flaw_sham; - } - - if ( sop_w <= (nop_w + 1) ) { - return u3e_flaw_meme; - } - - if ( _ce_ward_post(nop_w, sop_w) ) { - return u3e_flaw_base; - } - - c3_assert( old_w != u3P.gar_w ); - - return u3e_flaw_good; -} #endif /* ifdef U3_GUARD_PAGE */ -/* u3e_fault(): handle a memory fault. +/* u3e_fault(): handle a memory event with libsigsegv protocol. */ -u3e_flaw -u3e_fault(u3_post low_p, u3_post hig_p, u3_post off_p) +c3_i +u3e_fault(void* adr_v, c3_i ser_i) { - c3_w pag_w = off_p >> u3a_page; - c3_w blk_w = pag_w >> 5; - c3_w bit_w = pag_w & 31; + // Let the stack overflow handler run. + if ( 0 == ser_i ) { + return 0; + } + + // XX u3l_log avoid here, as it can + // cause problems when handling errors + + c3_w* adr_w = (c3_w*) adr_v; + + if ( (adr_w < u3_Loom) || (adr_w >= (u3_Loom + u3C.wor_i)) ) { + fprintf(stderr, "address %p out of loom!\r\n", adr_w); + fprintf(stderr, "loom: [%p : %p)\r\n", u3_Loom, u3_Loom + u3C.wor_i); + c3_assert(0); + return 0; + } + + u3p(c3_w) adr_p = u3a_outa(adr_w); + c3_w pag_w = adr_p >> u3a_page; + c3_w blk_w = (pag_w >> 5); + c3_w bit_w = (pag_w & 31); #ifdef U3_GUARD_PAGE - if ( pag_w == u3P.gar_w ) { - u3e_flaw fal_e = _ce_ward_clip(low_p >> u3a_page, hig_p >> u3a_page); - - if ( u3e_flaw_good != fal_e ) { - return fal_e; - } - - if ( !(u3P.dit_w[blk_w] & (1 << bit_w)) ) { - fprintf(stderr, "loom: strange guard (%d)\r\n", pag_w); - return u3e_flaw_sham; + // The fault happened in the guard page. + if ( gar_pag_p <= adr_p && adr_p < gar_pag_p + pag_wiz_i ) { + if ( 0 == _ce_center_guard_page() ) { + return 0; } } else -#endif - if ( u3P.dit_w[blk_w] & (1 << bit_w) ) { - fprintf(stderr, "loom: strange page (%d): %x\r\n", pag_w, off_p); - return u3e_flaw_sham; +#endif /* ifdef U3_GUARD_PAGE */ + if ( 0 != (u3P.dit_w[blk_w] & (1 << bit_w)) ) { + fprintf(stderr, "strange page: %d, at %p, off %x\r\n", pag_w, adr_w, adr_p); + c3_assert(0); + return 0; } u3P.dit_w[blk_w] |= (1 << bit_w); - if ( _ce_flaw_protect(pag_w) ) { - return u3e_flaw_base; + if ( -1 == mprotect((void *)(u3_Loom + (pag_w << u3a_page)), + pag_siz_i, + (PROT_READ | PROT_WRITE)) ) + { + fprintf(stderr, "loom: fault mprotect: %s\r\n", strerror(errno)); + c3_assert(0); + return 0; } - return u3e_flaw_good; + return 1; } /* _ce_image_open(): open or create image. @@ -281,7 +305,16 @@ _ce_image_open(u3e_image* img_u) c3_i mod_i = O_RDWR | O_CREAT; c3_c ful_c[8193]; - snprintf(ful_c, 8192, "%s/%s.bin", u3P.dir_c, img_u->nam_c); + snprintf(ful_c, 8192, "%s", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb/chk", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c); if ( -1 == (img_u->fid_i = c3_open(ful_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); return c3n; @@ -321,11 +354,17 @@ _ce_image_open(u3e_image* img_u) static void _ce_patch_write_control(u3_ce_patch* pat_u) { - c3_w len_w = sizeof(u3e_control) + - (pat_u->con_u->pgs_w * sizeof(u3e_line)); + ssize_t ret_i; + c3_w len_w = sizeof(u3e_control) + + (pat_u->con_u->pgs_w * sizeof(u3e_line)); - if ( 0 > c3_pwrite(pat_u->ctl_i, pat_u->con_u, len_w, 0) ) { - fprintf(stderr, "loom: patch write: %s\r\n", strerror(errno)); + if ( len_w != (ret_i = write(pat_u->ctl_i, pat_u->con_u, len_w)) ) { + if ( 0 < ret_i ) { + fprintf(stderr, "loom: patch ctl partial write: %zu\r\n", (size_t)ret_i); + } + else { + fprintf(stderr, "loom: patch ctl write: %s\r\n", strerror(errno)); + } c3_assert(0); } } @@ -349,9 +388,8 @@ _ce_patch_read_control(u3_ce_patch* pat_u) } pat_u->con_u = c3_malloc(len_w); - - if ( (len_w != c3_pread(pat_u->ctl_i, pat_u->con_u, len_w, 0)) - || (len_w != sizeof(u3e_control) + + if ( (len_w != read(pat_u->ctl_i, pat_u->con_u, len_w)) || + (len_w != sizeof(u3e_control) + (pat_u->con_u->pgs_w * sizeof(u3e_line))) ) { c3_free(pat_u->con_u); @@ -368,13 +406,19 @@ _ce_patch_create(u3_ce_patch* pat_u) { c3_c ful_c[8193]; - snprintf(ful_c, 8192, "%s/control.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); if ( -1 == (pat_u->ctl_i = c3_open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch c3_open control.bin: %s\r\n", strerror(errno)); c3_assert(0); } - snprintf(ful_c, 8192, "%s/memory.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); if ( -1 == (pat_u->mem_i = c3_open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch c3_open memory.bin: %s\r\n", strerror(errno)); c3_assert(0); @@ -388,13 +432,13 @@ _ce_patch_delete(void) { c3_c ful_c[8193]; - snprintf(ful_c, 8192, "%s/control.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); if ( unlink(ful_c) ) { fprintf(stderr, "loom: failed to delete control.bin: %s\r\n", strerror(errno)); } - snprintf(ful_c, 8192, "%s/memory.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); if ( unlink(ful_c) ) { fprintf(stderr, "loom: failed to remove memory.bin: %s\r\n", strerror(errno)); @@ -406,10 +450,8 @@ _ce_patch_delete(void) static c3_o _ce_patch_verify(u3_ce_patch* pat_u) { - c3_w i_w, pag_w, mug_w; - c3_w mem_w[pag_wiz_i]; - size_t off_i, siz_i = pag_siz_i; - ssize_t ret_i; + ssize_t ret_i; + c3_w i_w; if ( u3e_version != pat_u->con_u->ver_y ) { fprintf(stderr, "loom: patch version mismatch: have %u, need %u\r\n", @@ -419,11 +461,15 @@ _ce_patch_verify(u3_ce_patch* pat_u) } for ( i_w = 0; i_w < pat_u->con_u->pgs_w; i_w++ ) { - pag_w = pat_u->con_u->mem_u[i_w].pag_w; - mug_w = pat_u->con_u->mem_u[i_w].mug_w; - off_i = (size_t)i_w << (u3a_page + 2); + c3_w pag_w = pat_u->con_u->mem_u[i_w].pag_w; + c3_w mug_w = pat_u->con_u->mem_u[i_w].mug_w; + c3_w mem_w[1 << u3a_page]; - if ( siz_i != (ret_i = c3_pread(pat_u->mem_i, mem_w, siz_i, off_i)) ) { + if ( -1 == lseek(pat_u->mem_i, (i_w << (u3a_page + 2)), SEEK_SET) ) { + fprintf(stderr, "loom: patch seek: %s\r\n", strerror(errno)); + return c3n; + } + if ( pag_siz_i != (ret_i = read(pat_u->mem_i, mem_w, pag_siz_i)) ) { if ( 0 < ret_i ) { fprintf(stderr, "loom: patch partial read: %zu\r\n", (size_t)ret_i); } @@ -432,7 +478,6 @@ _ce_patch_verify(u3_ce_patch* pat_u) } return c3n; } - { c3_w nug_w = u3r_mug_words(mem_w, pag_wiz_i); @@ -471,12 +516,18 @@ _ce_patch_open(void) c3_c ful_c[8193]; c3_i ctl_i, mem_i; - snprintf(ful_c, 8192, "%s/control.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb", u3P.dir_c); + c3_mkdir(ful_c, 0700); + + snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); if ( -1 == (ctl_i = c3_open(ful_c, O_RDWR)) ) { return 0; } - snprintf(ful_c, 8192, "%s/memory.bin", u3P.dir_c); + snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); if ( -1 == (mem_i = c3_open(ful_c, O_RDWR)) ) { close(ctl_i); @@ -504,6 +555,32 @@ _ce_patch_open(void) return pat_u; } +/* _ce_patch_write_page(): write a page of patch memory. +*/ +static void +_ce_patch_write_page(u3_ce_patch* pat_u, + c3_w pgc_w, + c3_w* mem_w) +{ + ssize_t ret_i; + + if ( -1 == lseek(pat_u->mem_i, pgc_w * pag_siz_i, SEEK_SET) ) { + fprintf(stderr, "loom: patch page seek: %s\r\n", strerror(errno)); + c3_assert(0); + } + + if ( pag_siz_i != (ret_i = write(pat_u->mem_i, mem_w, pag_siz_i)) ) { + if ( 0 < ret_i ) { + fprintf(stderr, "loom: patch page partial write: %zu\r\n", + (size_t)ret_i); + } + else { + fprintf(stderr, "loom: patch page write: %s\r\n", strerror(errno)); + } + c3_assert(0); + } +} + /* _ce_patch_count_page(): count a page, producing new counter. */ static c3_w @@ -530,17 +607,25 @@ _ce_patch_save_page(u3_ce_patch* pat_u, c3_w bit_w = (pag_w & 31); if ( u3P.dit_w[blk_w] & (1 << bit_w) ) { - c3_w* mem_w = u3_Loom + (pag_w << u3a_page); - size_t off_i = (size_t)pgc_w << (u3a_page + 2); + c3_w* mem_w = u3_Loom + (pag_w << u3a_page); pat_u->con_u->mem_u[pgc_w].pag_w = pag_w; pat_u->con_u->mem_u[pgc_w].mug_w = u3r_mug_words(mem_w, pag_wiz_i); - if ( 0 > c3_pwrite(pat_u->mem_i, mem_w, pag_siz_i, off_i) ) { - fprintf(stderr, "loom: patch save: %s\r\n", strerror(errno)); +#if 0 + u3l_log("protect a: page %d\r\n", pag_w); +#endif + _ce_patch_write_page(pat_u, pgc_w, mem_w); + + if ( -1 == mprotect(u3_Loom + (pag_w << u3a_page), + pag_siz_i, + PROT_READ) ) + { + fprintf(stderr, "loom: patch mprotect: %s\r\n", strerror(errno)); c3_assert(0); } + u3P.dit_w[blk_w] &= ~(1 << bit_w); pgc_w += 1; } return pgc_w; @@ -549,9 +634,25 @@ _ce_patch_save_page(u3_ce_patch* pat_u, /* _ce_patch_compose(): make and write current patch. */ static u3_ce_patch* -_ce_patch_compose(c3_w nor_w, c3_w sou_w) +_ce_patch_compose(void) { c3_w pgs_w = 0; + c3_w nor_w = 0; + c3_w sou_w = 0; + + /* Calculate number of saved pages, north and south. + */ + { + c3_w nwr_w, swu_w; + + u3m_water(&nwr_w, &swu_w); + + nor_w = (nwr_w + (pag_wiz_i - 1)) >> u3a_page; + sou_w = (swu_w + (pag_wiz_i - 1)) >> u3a_page; + + c3_assert( ((gar_pag_p >> u3a_page) >= nor_w) + && ((gar_pag_p >> u3a_page) <= (u3a_pages - (sou_w + 1))) ); + } #ifdef U3_SNAPSHOT_VALIDATION u3K.nor_w = nor_w; @@ -624,7 +725,8 @@ _ce_image_sync(u3e_image* img_u) { if ( -1 == c3_sync(img_u->fid_i) ) { fprintf(stderr, "loom: image (%s) sync failed: %s\r\n", - img_u->nam_c, strerror(errno)); + img_u->nam_c, + strerror(errno)); c3_assert(!"loom: image sync"); } } @@ -634,19 +736,11 @@ _ce_image_sync(u3e_image* img_u) static void _ce_image_resize(u3e_image* img_u, c3_w pgs_w) { - off_t off_i = (off_t)pgs_w << (u3a_page + 2); - if ( img_u->pgs_w > pgs_w ) { - if ( (off_i >> (u3a_page + 2)) != pgs_w ) { - fprintf(stderr, "loom: image (%s) truncate: " - "offset overflow (%" PRId64 ") for page %u\r\n", - img_u->nam_c, (c3_ds)off_i, pgs_w); - c3_assert(0); - } - - if ( ftruncate(img_u->fid_i, off_i) ) { + if ( ftruncate(img_u->fid_i, pgs_w << (u3a_page + 2)) ) { fprintf(stderr, "loom: image (%s) truncate: %s\r\n", - img_u->nam_c, strerror(errno)); + img_u->nam_c, + strerror(errno)); c3_assert(0); } } @@ -659,20 +753,31 @@ _ce_image_resize(u3e_image* img_u, c3_w pgs_w) static void _ce_patch_apply(u3_ce_patch* pat_u) { - c3_w i_w, pag_w, off_w, mem_w[pag_wiz_i]; - c3_i fid_i; - size_t rof_i, wof_i, siz_i = pag_siz_i; - ssize_t ret_i; + ssize_t ret_i; + c3_w i_w; // resize images // _ce_image_resize(&u3P.nor_u, pat_u->con_u->nor_w); _ce_image_resize(&u3P.sou_u, pat_u->con_u->sou_w); + // seek to begining of patch and images + // + if ( (-1 == lseek(pat_u->mem_i, 0, SEEK_SET)) + || (-1 == lseek(u3P.nor_u.fid_i, 0, SEEK_SET)) + || (-1 == lseek(u3P.sou_u.fid_i, 0, SEEK_SET)) ) + { + fprintf(stderr, "loom: patch apply seek 0: %s\r\n", strerror(errno)); + c3_assert(0); + } + // write patch pages into the appropriate image // for ( i_w = 0; i_w < pat_u->con_u->pgs_w; i_w++ ) { - pag_w = pat_u->con_u->mem_u[i_w].pag_w; + c3_w pag_w = pat_u->con_u->mem_u[i_w].pag_w; + c3_w mem_w[pag_wiz_i]; + c3_i fid_i; + c3_w off_w; if ( pag_w < pat_u->con_u->nor_w ) { fid_i = u3P.nor_u.fid_i; @@ -683,10 +788,7 @@ _ce_patch_apply(u3_ce_patch* pat_u) off_w = (u3P.pag_w - (pag_w + 1)); } - rof_i = (size_t)i_w << (u3a_page + 2); - wof_i = (size_t)off_w << (u3a_page + 2); - - if ( siz_i != (ret_i = c3_pread(pat_u->mem_i, mem_w, siz_i, rof_i))) { + if ( pag_siz_i != (ret_i = read(pat_u->mem_i, mem_w, pag_siz_i)) ) { if ( 0 < ret_i ) { fprintf(stderr, "loom: patch apply partial read: %zu\r\n", (size_t)ret_i); @@ -696,10 +798,21 @@ _ce_patch_apply(u3_ce_patch* pat_u) } c3_assert(0); } - - if ( 0 > c3_pwrite(fid_i, mem_w, siz_i, wof_i) ) { - fprintf(stderr, "loom: patch apply write: %s\r\n", strerror(errno)); - c3_assert(0); + else { + if ( -1 == lseek(fid_i, (off_w << (u3a_page + 2)), SEEK_SET) ) { + fprintf(stderr, "loom: patch apply seek: %s\r\n", strerror(errno)); + c3_assert(0); + } + if ( pag_siz_i != (ret_i = write(fid_i, mem_w, pag_siz_i)) ) { + if ( 0 < ret_i ) { + fprintf(stderr, "loom: patch apply partial write: %zu\r\n", + (size_t)ret_i); + } + else { + fprintf(stderr, "loom: patch apply write: %s\r\n", strerror(errno)); + } + c3_assert(0); + } } #if 0 u3l_log("apply: %d, %x\n", pag_w, u3r_mug_words(mem_w, pag_wiz_i)); @@ -707,294 +820,52 @@ _ce_patch_apply(u3_ce_patch* pat_u) } } -/* _ce_loom_track_north(): [pgs_w] clean, followed by [dif_w] dirty. -*/ -void -_ce_loom_track_north(c3_w pgs_w, c3_w dif_w) -{ - c3_w blk_w = pgs_w >> 5; - c3_w bit_w = pgs_w & 31; - c3_w off_w; - - memset((void*)u3P.dit_w, 0, blk_w << 2); - - if ( bit_w ) { - c3_w tib_w = 32 - bit_w; - c3_w dat_w = u3P.dit_w[blk_w]; - - dat_w &= 0xffffffff << bit_w; - - if ( dif_w <= tib_w ) { - dat_w |= ((1 << dif_w) - 1) << bit_w; - dif_w = 0; - } - else { - dat_w |= 0xffffffff << bit_w; - dif_w -= tib_w; - } - - u3P.dit_w[blk_w] = dat_w; - blk_w += 1; - } - - off_w = blk_w; - blk_w = dif_w >> 5; - bit_w = dif_w & 31; - - memset((void*)(u3P.dit_w + off_w), 0xff, blk_w << 2); - - if ( bit_w ) { - u3P.dit_w[off_w + blk_w] |= (1 << bit_w) - 1; - } -} - -/* _ce_loom_track_south(): [pgs_w] clean, preceded by [dif_w] dirty. -*/ -void -_ce_loom_track_south(c3_w pgs_w, c3_w dif_w) -{ - c3_w blk_w = pgs_w >> 5; - c3_w bit_w = pgs_w & 31; - c3_w bas_w = ((u3P.pag_w - pgs_w) + 31) >> 5; - c3_w off_w; - - memset((void*)(u3P.dit_w + bas_w), 0, blk_w << 2); - - // the following index subtractions (bas_w, off) are safe, - // so long as the south segment never includes all pages - // - if ( bit_w ) { - c3_w tib_w = 32 - bit_w; - c3_w dat_w = u3P.dit_w[--bas_w]; - - dat_w &= 0xffffffff >> bit_w; - - if ( dif_w <= tib_w ) { - dat_w |= ((1 << dif_w) - 1) << (tib_w - dif_w); - dif_w = 0; - } - else { - dat_w |= 0xffffffff >> bit_w; - dif_w -= tib_w; - } - - u3P.dit_w[bas_w] = dat_w; - } - - blk_w = dif_w >> 5; - bit_w = dif_w & 31; - off_w = bas_w - blk_w; - - memset((void*)(u3P.dit_w + off_w), 0xff, blk_w << 2); - - if ( bit_w ) { - u3P.dit_w[off_w - 1] |= ((1 << bit_w) - 1) << (32 - bit_w); - } -} - -/* _ce_loom_protect_north(): protect/track pages from the bottom of memory. +/* _ce_image_blit(): apply image to memory. */ static void -_ce_loom_protect_north(c3_w pgs_w, c3_w old_w) +_ce_image_blit(u3e_image* img_u, + c3_w* ptr_w, + c3_ws stp_ws) { - if ( pgs_w ) { - if ( 0 != mprotect((void*)u3_Loom, - (size_t)pgs_w << (u3a_page + 2), - PROT_READ) ) - { - fprintf(stderr, "loom: pure north (%u pages): %s\r\n", - pgs_w, strerror(errno)); - c3_assert(0); - } + if ( 0 == img_u->pgs_w ) { + return; } - if ( old_w > pgs_w ) { - c3_w dif_w = old_w - pgs_w; - - if ( 0 != mprotect((void*)(u3_Loom + (pgs_w << u3a_page)), - (size_t)dif_w << (u3a_page + 2), - (PROT_READ | PROT_WRITE)) ) - { - fprintf(stderr, "loom: foul north (%u pages, %u old): %s\r\n", - pgs_w, old_w, strerror(errno)); - c3_assert(0); - } - -#ifdef U3_GUARD_PAGE - // protect guard page if clobbered - // - // NB: < pgs_w is precluded by assertion in u3e_save() - // - if ( u3P.gar_w < old_w ) { - fprintf(stderr, "loom: guard on reprotect\r\n"); - c3_assert( !_ce_ward_protect() ); - } -#endif - - _ce_loom_track_north(pgs_w, dif_w); - } - else { - _ce_loom_track_north(pgs_w, 0); - } -} - -/* _ce_loom_protect_south(): protect/track pages from the top of memory. -*/ -static void -_ce_loom_protect_south(c3_w pgs_w, c3_w old_w) -{ - if ( pgs_w ) { - if ( 0 != mprotect((void*)(u3_Loom + ((u3P.pag_w - pgs_w) << u3a_page)), - (size_t)pgs_w << (u3a_page + 2), - PROT_READ) ) - { - fprintf(stderr, "loom: pure south (%u pages): %s\r\n", - pgs_w, strerror(errno)); - c3_assert(0); - } - } - - if ( old_w > pgs_w ) { - c3_w dif_w = old_w - pgs_w; - c3_w off_w = u3P.pag_w - old_w; - - if ( 0 != mprotect((void*)(u3_Loom + (off_w << u3a_page)), - (size_t)dif_w << (u3a_page + 2), - (PROT_READ | PROT_WRITE)) ) - { - fprintf(stderr, "loom: foul south (%u pages, %u old): %s\r\n", - pgs_w, old_w, strerror(errno)); - c3_assert(0); - } - -#ifdef U3_GUARD_PAGE - // protect guard page if clobbered - // - // NB: >= pgs_w is precluded by assertion in u3e_save() - // - if ( u3P.gar_w >= off_w ) { - fprintf(stderr, "loom: guard on reprotect\r\n"); - c3_assert( !_ce_ward_protect() ); - } -#endif - - _ce_loom_track_south(pgs_w, dif_w); - } - else { - _ce_loom_track_south(pgs_w, 0); - } -} - -/* _ce_loom_mapf_north(): map [pgs_w] of [fid_i] into the bottom of memory -** (and anonymize [old_w - pgs_w] after if needed). -** -** NB: _ce_loom_mapf_south() is possible, but it would make separate mappings -** for each page since the south segment is reversed on disk. -** in practice, the south segment is a single page (and always dirty); -** a file-backed mapping for it is just not worthwhile. -*/ -static void -_ce_loom_mapf_north(c3_i fid_i, c3_w pgs_w, c3_w old_w) -{ - // XX mingw doesn't support MAP_FIXED clobbering; - // will require explicit unmapping and related bookkeeping. - // - if ( pgs_w ) { - if ( MAP_FAILED == mmap((void*)u3_Loom, - (size_t)pgs_w << (u3a_page + 2), - PROT_READ, - (MAP_FIXED | MAP_PRIVATE), - fid_i, 0) ) - { - fprintf(stderr, "loom: file-backed mmap failed (%u pages): %s\r\n", - pgs_w, strerror(errno)); - c3_assert(0); - } - } - - if ( old_w > pgs_w ) { - c3_w dif_w = old_w - pgs_w; - - if ( MAP_FAILED == mmap((void*)(u3_Loom + (pgs_w << u3a_page)), - (size_t)dif_w << (u3a_page + 2), - (PROT_READ | PROT_WRITE), - (MAP_ANON | MAP_FIXED | MAP_PRIVATE), - -1, 0) ) - { - fprintf(stderr, "loom: anonymous mmap failed (%u pages, %u old): %s\r\n", - pgs_w, old_w, strerror(errno)); - c3_assert(0); - } - -#ifdef U3_GUARD_PAGE - // protect guard page if clobbered - // - // NB: < pgs_w is precluded by assertion in u3e_save() - // - if ( u3P.gar_w < old_w ) { - fprintf(stderr, "loom: guard on remap\r\n"); - c3_assert( !_ce_ward_protect() ); - } -#endif - - _ce_loom_track_north(pgs_w, dif_w); - } - else { - _ce_loom_track_north(pgs_w, 0); - } -} - -/* _ce_loom_blit_north(): apply pages, in order, from the bottom of memory. -*/ -static void -_ce_loom_blit_north(c3_i fid_i, c3_w pgs_w) -{ - size_t len_i = (size_t)pgs_w << (u3a_page + 2); ssize_t ret_i; - - if ( pgs_w ) { - if ( len_i != (ret_i = c3_pread(fid_i, u3_Loom, len_i, 0)) ) { - if ( 0 < ret_i ) { - fprintf(stderr, "loom: blit north partial read: %zu\r\n", - (size_t)ret_i); - } - else { - fprintf(stderr, "loom: blit north read: %s\r\n", strerror(errno)); - } - c3_assert(0); - } - } - - _ce_loom_protect_north(pgs_w, 0); -} - -/* _ce_loom_blit_south(): apply pages, reversed, from the top of memory. -*/ -static void -_ce_loom_blit_south(c3_i fid_i, c3_w pgs_w) -{ c3_w i_w; - c3_w* ptr_w; - size_t off_i; - ssize_t ret_i; + c3_w siz_w = pag_siz_i; - for ( i_w = 0; i_w < pgs_w; i_w++ ) { - off_i = (size_t)i_w << (u3a_page + 2); - ptr_w = u3_Loom + ((u3P.pag_w - (i_w + 1)) << u3a_page); + if ( -1 == lseek(img_u->fid_i, 0, SEEK_SET) ) { + fprintf(stderr, "loom: image (%s) blit seek 0: %s\r\n", + img_u->nam_c, strerror(errno)); + c3_assert(0); + } - if ( pag_siz_i != (ret_i = c3_pread(fid_i, ptr_w, pag_siz_i, off_i)) ) { + for ( i_w = 0; i_w < img_u->pgs_w; i_w++ ) { + if ( siz_w != (ret_i = read(img_u->fid_i, ptr_w, siz_w)) ) { if ( 0 < ret_i ) { - fprintf(stderr, "loom: blit south partial read: %zu\r\n", - (size_t)ret_i); + fprintf(stderr, "loom: image (%s) blit partial read: %zu\r\n", + img_u->nam_c, (size_t)ret_i); } else { - fprintf(stderr, "loom: blit south read: %s\r\n", strerror(errno)); + fprintf(stderr, "loom: image (%s) blit read: %s\r\n", + img_u->nam_c, strerror(errno)); } c3_assert(0); } - } - _ce_loom_protect_south(pgs_w, 0); + if ( 0 != mprotect(ptr_w, siz_w, PROT_READ) ) { + fprintf(stderr, "loom: live mprotect: %s\r\n", strerror(errno)); + c3_assert(0); + } + + c3_w pag_w = u3a_outa(ptr_w) >> u3a_page; + c3_w blk_w = pag_w >> 5; + c3_w bit_w = pag_w & 31; + u3P.dit_w[blk_w] &= ~(1 << bit_w); + + ptr_w += stp_ws; + } } #ifdef U3_SNAPSHOT_VALIDATION @@ -1002,18 +873,22 @@ _ce_loom_blit_south(c3_i fid_i, c3_w pgs_w) */ static void _ce_image_fine(u3e_image* img_u, - c3_w* ptr_w, - c3_ws stp_ws) + c3_w* ptr_w, + c3_ws stp_ws) { - c3_w i_w, mem_w, fil_w; - c3_w buf_w[pag_wiz_i]; - size_t off_i, siz_i = pag_siz_i; - ssize_t ret_i; + ssize_t ret_i; + c3_w i_w; + c3_w buf_w[pag_wiz_i]; + + if ( -1 == lseek(img_u->fid_i, 0, SEEK_SET) ) { + fprintf(stderr, "loom: image fine seek 0: %s\r\n", strerror(errno)); + c3_assert(0); + } for ( i_w=0; i_w < img_u->pgs_w; i_w++ ) { - off_i = (size_t)i_w << (u3a_page + 2); + c3_w mem_w, fil_w; - if ( siz_i != (ret_i = c3_pread(img_u->fid_i, buf_w, siz_i, off_i)) ) { + if ( pag_siz_i != (ret_i = read(img_u->fid_i, buf_w, pag_siz_i)) ) { if ( 0 < ret_i ) { fprintf(stderr, "loom: image (%s) fine partial read: %zu\r\n", img_u->nam_c, (size_t)ret_i); @@ -1024,7 +899,6 @@ _ce_image_fine(u3e_image* img_u, } c3_assert(0); } - mem_w = u3r_mug_words(ptr_w, pag_wiz_i); fil_w = u3r_mug_words(buf_w, pag_wiz_i); @@ -1045,28 +919,36 @@ _ce_image_fine(u3e_image* img_u, } #endif -/* _ce_image_copy(): copy all of [fom_u] to [tou_u] -** -** XX use reflinks a la _king_copy_file()? +/* _ce_image_copy(): */ static c3_o _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) { - c3_w i_w; - c3_w mem_w[pag_wiz_i]; - size_t off_i, siz_i = pag_siz_i; ssize_t ret_i; + c3_w i_w; // resize images // _ce_image_resize(tou_u, fom_u->pgs_w); + // seek to begining of patch and images + // + if ( (-1 == lseek(fom_u->fid_i, 0, SEEK_SET)) + || (-1 == lseek(tou_u->fid_i, 0, SEEK_SET)) ) + { + fprintf(stderr, "loom: image (%s) copy seek: %s\r\n", + fom_u->nam_c, + strerror(errno)); + return c3n; + } + // copy pages into destination image // for ( i_w = 0; i_w < fom_u->pgs_w; i_w++ ) { - off_i = (size_t)i_w << (u3a_page + 2); + c3_w mem_w[pag_wiz_i]; + c3_w off_w = i_w; - if ( siz_i != (ret_i = c3_pread(fom_u->fid_i, mem_w, siz_i, off_i)) ) { + if ( pag_siz_i != (ret_i = read(fom_u->fid_i, mem_w, pag_siz_i)) ) { if ( 0 < ret_i ) { fprintf(stderr, "loom: image (%s) copy partial read: %zu\r\n", fom_u->nam_c, (size_t)ret_i); @@ -1077,30 +959,40 @@ _ce_image_copy(u3e_image* fom_u, u3e_image* tou_u) } return c3n; } - - if ( 0 > c3_pwrite(tou_u->fid_i, mem_w, siz_i, off_i) ) { - fprintf(stderr, "loom: image (%s) copy write: %s\r\n", - tou_u->nam_c, strerror(errno)); - return c3n; + else { + if ( -1 == lseek(tou_u->fid_i, (off_w << (u3a_page + 2)), SEEK_SET) ) { + fprintf(stderr, "loom: image (%s) copy seek: %s\r\n", + tou_u->nam_c, strerror(errno)); + return c3n; + } + if ( pag_siz_i != (ret_i = write(tou_u->fid_i, mem_w, pag_siz_i)) ) { + if ( 0 < ret_i ) { + fprintf(stderr, "loom: image (%s) copy partial write: %zu\r\n", + tou_u->nam_c, (size_t)ret_i); + } + else { + fprintf(stderr, "loom: image (%s) copy write: %s\r\n", + tou_u->nam_c, strerror(errno)); + } + return c3n; + } } } return c3y; } -/* _ce_backup(); copy snapshot to .urb/bhk (if it doesn't exist yet). +/* _ce_backup(); */ static void _ce_backup(void) { u3e_image nop_u = { .nam_c = "north", .pgs_w = 0 }; u3e_image sop_u = { .nam_c = "south", .pgs_w = 0 }; - c3_i mod_i = O_RDWR | O_CREAT; // XX O_TRUNC ? + c3_i mod_i = O_RDWR | O_CREAT; c3_c ful_c[8193]; - // XX move directory creation? - // - snprintf(ful_c, 8192, "%s/.urb/bhk", u3C.dir_c); + snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); if ( c3_mkdir(ful_c, 0700) ) { if ( EEXIST != errno ) { @@ -1109,14 +1001,14 @@ _ce_backup(void) return; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3C.dir_c, nop_u.nam_c); + snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); if ( -1 == (nop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); return; } - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3C.dir_c, sop_u.nam_c); + snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, sop_u.nam_c); if ( -1 == (sop_u.fid_i = c3_open(ful_c, mod_i, 0666)) ) { fprintf(stderr, "loom: c3_open %s: %s\r\n", ful_c, strerror(errno)); @@ -1128,9 +1020,9 @@ _ce_backup(void) { c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3C.dir_c, nop_u.nam_c); + snprintf(ful_c, 8192, "%s/.urb/bhk/%s.bin", u3P.dir_c, nop_u.nam_c); c3_unlink(ful_c); - snprintf(ful_c, 8192, "%s/.urb/bhk", u3C.dir_c); + snprintf(ful_c, 8192, "%s/.urb/bhk", u3P.dir_c); c3_rmdir(ful_c); } @@ -1159,29 +1051,19 @@ _ce_backup(void) before we try to make another snapshot. */ void -u3e_save(u3_post low_p, u3_post hig_p) +u3e_save(void) { u3_ce_patch* pat_u; - c3_w nod_w, sod_w; if ( u3C.wag_w & u3o_dryrun ) { return; } - { - c3_w nop_w = (low_p >> u3a_page); - c3_w nor_w = (low_p + (pag_wiz_i - 1)) >> u3a_page; - c3_w sop_w = hig_p >> u3a_page; - - c3_assert( (u3P.gar_w > nop_w) && (u3P.gar_w < sop_w) ); - - if ( !(pat_u = _ce_patch_compose(nor_w, u3P.pag_w - sop_w)) ) { - return; - } + if ( !(pat_u = _ce_patch_compose()) ) { + return; } - nod_w = u3P.nor_u.pgs_w; - sod_w = u3P.sou_u.pgs_w; + // u3a_print_memory(stderr, "sync: save", 4096 * pat_u->con_u->pgs_w); _ce_patch_sync(pat_u); @@ -1206,15 +1088,6 @@ u3e_save(u3_post low_p, u3_post hig_p) } #endif - if ( u3C.wag_w & u3o_no_demand ) { - _ce_loom_protect_north(u3P.nor_u.pgs_w, nod_w); - } - else { - _ce_loom_mapf_north(u3P.nor_u.fid_i, u3P.nor_u.pgs_w, nod_w); - } - - _ce_loom_protect_south(u3P.sou_u.pgs_w, sod_w); - _ce_image_sync(&u3P.nor_u); _ce_image_sync(&u3P.sou_u); _ce_patch_free(pat_u); @@ -1226,16 +1099,8 @@ u3e_save(u3_post low_p, u3_post hig_p) /* u3e_live(): start the checkpointing system. */ c3_o -u3e_live(c3_c* dir_c) +u3e_live(c3_o nuu_o, c3_c* dir_c) { - c3_o nuu_o = c3n; - - // XX demand paging is not supported on windows - // -#ifdef U3_OS_mingw - u3C.wag_w |= u3o_no_demand; -#endif - // require that our page size is a multiple of the system page size. // { @@ -1271,7 +1136,6 @@ u3e_live(c3_c* dir_c) } else { u3_ce_patch* pat_u; - c3_w nor_w, sou_w; /* Load any patch files; apply them to images. */ @@ -1283,52 +1147,41 @@ u3e_live(c3_c* dir_c) _ce_patch_delete(); } - nor_w = u3P.nor_u.pgs_w; - sou_w = u3P.sou_u.pgs_w; - // detect snapshots from a larger loom // - if ( (nor_w + sou_w + 1) >= u3P.pag_w ) { + if ( (u3P.nor_u.pgs_w + u3P.sou_u.pgs_w + 1) >= u3a_pages ) { fprintf(stderr, "boot: snapshot too big for loom\r\n"); exit(1); } // mark all pages dirty (pages in the snapshot will be marked clean) // - _ce_loom_track_north(0, u3P.pag_w); + u3e_foul(); /* Write image files to memory; reinstate protection. */ { - if ( u3C.wag_w & u3o_no_demand ) { - _ce_loom_blit_north(u3P.nor_u.fid_i, nor_w); - } - else { - _ce_loom_mapf_north(u3P.nor_u.fid_i, nor_w, 0); - } + _ce_image_blit(&u3P.nor_u, + u3_Loom, + pag_wiz_i); - _ce_loom_blit_south(u3P.sou_u.fid_i, sou_w); + _ce_image_blit(&u3P.sou_u, + (u3_Loom + u3C.wor_i) - pag_wiz_i, + -(ssize_t)pag_wiz_i); u3l_log("boot: protected loom\r\n"); } /* If the images were empty, we are logically booting. */ - if ( !nor_w && !sou_w ) { + if ( (0 == u3P.nor_u.pgs_w) && (0 == u3P.sou_u.pgs_w) ) { u3l_log("live: logical boot\r\n"); nuu_o = c3y; } - else if ( u3C.wag_w & u3o_no_demand ) { - u3a_print_memory(stderr, "live: loaded", (nor_w + sou_w) << u3a_page); - } else { - u3a_print_memory(stderr, "live: mapped", nor_w << u3a_page); - u3a_print_memory(stderr, "live: loaded", sou_w << u3a_page); + u3a_print_memory(stderr, "live: loaded", + (u3P.nor_u.pgs_w + u3P.sou_u.pgs_w) << u3a_page); } - -#ifdef U3_GUARD_PAGE - c3_assert( !_ce_ward_post(nor_w, u3P.pag_w - sou_w) ); -#endif } } @@ -1352,17 +1205,23 @@ u3e_yolo(void) return c3n; } -#ifdef U3_GUARD_PAGE - c3_assert( !_ce_ward_protect() ); -#endif - - // mark all pages dirty - // - _ce_loom_track_north(0, u3P.pag_w); + if ( 0 != mprotect(u3a_into(gar_pag_p), pag_siz_i, PROT_NONE) ) { + fprintf(stderr, "loom: failed to protect guard page: %s\r\n", + strerror(errno)); + c3_assert(0); + } return c3y; } +/* u3e_foul(): dirty all the pages of the loom. +*/ +void +u3e_foul(void) +{ + memset((void*)u3P.dit_w, 0xff, sizeof(u3P.dit_w)); +} + /* u3e_init(): initialize guard page tracking. */ void @@ -1371,10 +1230,8 @@ u3e_init(void) u3P.pag_w = u3C.wor_i >> u3a_page; #ifdef U3_GUARD_PAGE - c3_assert( !_ce_ward_post(0, u3P.pag_w) ); + _ce_center_guard_page(); #endif - - _ce_loom_track_north(0, u3P.pag_w); } /* u3e_ward(): reposition guard page if needed. @@ -1383,14 +1240,8 @@ void u3e_ward(u3_post low_p, u3_post hig_p) { #ifdef U3_GUARD_PAGE - c3_w nop_w = low_p >> u3a_page; - c3_w sop_w = hig_p >> u3a_page; - c3_w pag_w = u3P.gar_w; - - if ( !((pag_w > nop_w) && (pag_w < hig_p)) ) { - c3_assert( !_ce_ward_post(nop_w, sop_w) ); - c3_assert( !_ce_flaw_protect(pag_w) ); - c3_assert( u3P.dit_w[pag_w >> 5] & (1 << (pag_w & 31)) ); + if ( (low_p > gar_pag_p) || (hig_p < gar_pag_p) ) { + _ce_center_guard_page(); } #endif } diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index bc868309b..33b57a43a 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -2,7 +2,6 @@ ** */ #include "all.h" -#include "noun/events.h" #include "rsignal.h" #include "vere/vere.h" #include @@ -437,7 +436,7 @@ u3m_file(c3_c* pas_c) u3l_log("Loading file: %s\r\n", pas_c); struct stat buf_b; c3_i fid_i = c3_open(pas_c, O_RDONLY, 0644); - c3_w fln_w; + c3_w fln_w, red_w; c3_y* pad_y; if ( (fid_i < 0) || (fstat(fid_i, &buf_b) < 0) ) { @@ -449,10 +448,10 @@ u3m_file(c3_c* pas_c) pad_y = c3_malloc(buf_b.st_size); - ssize_t red_i = c3_pread(fid_i, pad_y, fln_w, 0); + red_w = read(fid_i, pad_y, fln_w); close(fid_i); - if ( red_i != fln_w ) { + if ( fln_w != red_w ) { c3_free(pad_y); return u3m_bail(c3__fail); } @@ -528,8 +527,6 @@ _pave_north(c3_w* mem_w, c3_w siz_w, c3_w len_w) // the stack starts at the end of the memory segment, // minus space for the road structure [siz_w] // - // NB: the returned road is "above" the stack - // c3_w* rut_w = mem_w; c3_w* mat_w = ((mem_w + len_w) - siz_w); c3_w* cap_w = mat_w; @@ -548,8 +545,6 @@ _pave_south(c3_w* mem_w, c3_w siz_w, c3_w len_w) // the stack starts at the base memory pointer [mem_w], // and ends after the space for the road structure [siz_w] // - // NB: the returned road is *on* the stack - // c3_w* rut_w = (mem_w + len_w); c3_w* mat_w = mem_w; c3_w* cap_w = mat_w + siz_w; @@ -605,7 +600,7 @@ _find_home(void) // this looks risky, but there are no legitimate scenarios // where it's wrong // - u3R->cap_p = u3R->mat_p = u3a_outa(u3H); + u3R->cap_p = u3R->mat_p = u3C.wor_i - c3_wiseof(*u3H); } /* u3m_pave(): instantiate or activate image. @@ -824,6 +819,7 @@ u3m_leap(c3_w pad_w) u3R->cap_p -= len_w; rod_u = _pave_south(u3a_into(bot_p), c3_wiseof(u3a_road), len_w); + u3e_ward(rod_u->cap_p, rod_u->hat_p); #if 0 fprintf(stderr, "leap: from north %p (cap 0x%x), to south %p\r\n", u3R, @@ -836,6 +832,7 @@ u3m_leap(c3_w pad_w) u3R->cap_p += len_w; rod_u = _pave_north(u3a_into(bot_p), c3_wiseof(u3a_road), len_w); + u3e_ward(rod_u->hat_p, rod_u->cap_p); #if 0 fprintf(stderr, "leap: from south %p (cap 0x%x), to north %p\r\n", u3R, @@ -857,7 +854,6 @@ u3m_leap(c3_w pad_w) */ { u3R = rod_u; - u3m_ward(); _pave_parts(); } #ifdef U3_MEMORY_DEBUG @@ -1007,31 +1003,12 @@ u3m_flog(c3_w gof_w) /* u3m_water(): produce watermarks. */ void -u3m_water(u3_post* low_p, u3_post* hig_p) +u3m_water(c3_w* low_w, c3_w* hig_w) { - // allow the segfault handler to fire before the road is set - // - // while not explicitly possible in the codebase, - // compiler optimizations can reorder stores - // - if ( !u3R ) { - *low_p = 0; - *hig_p = u3C.wor_i - 1; - } - // in a north road, hat points to the end of the heap + 1 word, - // while cap points to the top of the stack - // - else if ( c3y == u3a_is_north(u3R) ) { - *low_p = u3R->hat_p - 1; - *hig_p = u3R->cap_p; - } - // in a south road, hat points to the end of the heap, - // while cap points to the top of the stack + 1 word - // - else { - *low_p = u3R->cap_p - 1; - *hig_p = u3R->hat_p; - } + c3_assert(u3R == &u3H->rod_u); + + *low_w = u3a_heap(u3R); + *hig_w = u3a_temp(u3R) + c3_wiseof(u3v_home); } /* u3m_soft_top(): top-level safety wrapper. @@ -1715,80 +1692,6 @@ _cm_limits(void) # endif } -/* u3m_fault(): handle a memory event with libsigsegv protocol. -*/ -c3_i -u3m_fault(void* adr_v, c3_i ser_i) -{ - // let the stack overflow handler run. - // - if ( 0 == ser_i ) { - return 0; - } - - c3_w* adr_w = (c3_w*)adr_v; - u3_post low_p, hig_p; - - if ( (adr_w < u3_Loom) || (adr_w >= (u3_Loom + u3C.wor_i)) ) { - fprintf(stderr, "loom: external fault: %p (%p : %p)\r\n\r\n", - adr_w, u3_Loom, u3_Loom + u3C.wor_i); - c3_assert(0); - return 0; - } - - u3m_water(&low_p, &hig_p); - - switch ( u3e_fault(low_p, hig_p, u3a_outa(adr_w)) ) { - // page tracking invariants violated, fatal - // - case u3e_flaw_sham: { - c3_assert(0); - return 0; - } - - // virtual memory failure (protections), possibly recoverable XX - // - case u3e_flaw_base: { - c3_assert(0); - return 0; - } - - // loom limits exceeded, recoverable - // - case u3e_flaw_meme: { - u3m_signal(c3__meme); // doesn't return - return 1; - } - - case u3e_flaw_good: return 1; - } - - c3_assert(!"unpossible"); -} - -/* u3m_save(): update the checkpoint. -*/ -void -u3m_save(void) -{ - u3_post low_p, hig_p; - u3m_water(&low_p, &hig_p); - - c3_assert(u3R == &u3H->rod_u); - - return u3e_save(low_p, hig_p); -} - -/* u3m_ward(): tend the guard page. -*/ -void -u3m_ward(void) -{ - u3_post low_p, hig_p; - u3m_water(&low_p, &hig_p); - return u3e_ward(low_p, hig_p); -} - /* _cm_signals(): set up interrupts, etc. */ static void @@ -1804,7 +1707,7 @@ _cm_signals(void) // filter (see compat/mingw/seh_handler.c) that handles both memory // access and stack overflow exceptions. It calls u3e_fault directly. # else - if ( 0 != sigsegv_install_handler(u3m_fault) ) { + if ( 0 != sigsegv_install_handler(u3e_fault) ) { u3l_log("boot: sigsegv install failed\n"); exit(1); } @@ -1961,46 +1864,6 @@ void u3m_stop() { u3je_secp_stop(); - - // XX move to jets.c - // - c3_free(u3D.ray_u); -} - -/* u3m_pier(): make a pier. -*/ -c3_c* -u3m_pier(c3_c* dir_c) -{ - c3_c ful_c[8193]; - - u3C.dir_c = dir_c; - - snprintf(ful_c, 8192, "%s", dir_c); - if ( c3_mkdir(ful_c, 0700) ) { - if ( EEXIST != errno ) { - fprintf(stderr, "loom: pier create: %s\r\n", strerror(errno)); - c3_assert(0); - } - } - - snprintf(ful_c, 8192, "%s/.urb", dir_c); - if ( c3_mkdir(ful_c, 0700) ) { - if ( EEXIST != errno ) { - fprintf(stderr, "loom: .urb create: %s\r\n", strerror(errno)); - c3_assert(0); - } - } - - snprintf(ful_c, 8192, "%s/.urb/chk", dir_c); - if ( c3_mkdir(ful_c, 0700) ) { - if ( EEXIST != errno ) { - fprintf(stderr, "loom: .urb/chk create: %s\r\n", strerror(errno)); - c3_assert(0); - } - } - - return strdup(ful_c); } /* u3m_boot(): start the u3 system. return next event, starting from 1. @@ -2016,7 +1879,7 @@ u3m_boot(c3_c* dir_c, size_t len_i) /* Activate the storage system. */ - nuu_o = u3e_live(u3m_pier(dir_c)); + nuu_o = u3e_live(c3n, dir_c); /* Activate tracing. */ @@ -2029,6 +1892,10 @@ u3m_boot(c3_c* dir_c, size_t len_i) */ u3m_pave(nuu_o); + /* Place the guard page. + */ + u3e_init(); + /* Initialize the jet system. */ { diff --git a/pkg/urbit/noun/trace.c b/pkg/urbit/noun/trace.c index 85afcfd46..79d290930 100644 --- a/pkg/urbit/noun/trace.c +++ b/pkg/urbit/noun/trace.c @@ -278,19 +278,11 @@ void u3t_trace_open(c3_c* dir_c) { c3_c fil_c[2048]; - - if ( !dir_c ) { - return; - } - snprintf(fil_c, 2048, "%s/.urb/put/trace", dir_c); struct stat st; - if ( (-1 == stat(fil_c, &st)) - && (-1 == c3_mkdir(fil_c, 0700)) ) - { - fprintf(stderr, "mkdir: %s failed: %s\r\n", fil_c, strerror(errno)); - return; + if ( -1 == stat(fil_c, &st) ) { + c3_mkdir(fil_c, 0700); } c3_c lif_c[2056]; @@ -299,11 +291,6 @@ u3t_trace_open(c3_c* dir_c) u3_Host.tra_u.fil_u = c3_fopen(lif_c, "w"); u3_Host.tra_u.nid_w = (int)getpid(); - if ( !u3_Host.tra_u.fil_u) { - fprintf(stderr, "trace open: %s\r\n", strerror(errno)); - return; - } - fprintf(u3_Host.tra_u.fil_u, "[ "); // We have two "threads", the event processing and the nock stuff. diff --git a/pkg/urbit/noun/urth.c b/pkg/urbit/noun/urth.c index de4e2d560..30ab44817 100644 --- a/pkg/urbit/noun/urth.c +++ b/pkg/urbit/noun/urth.c @@ -2,7 +2,6 @@ ** */ #include "all.h" -#include "noun/events.h" #include "ur/ur.h" #include #include @@ -417,6 +416,10 @@ _cu_realloc(FILE* fil_u, ur_root_t** tor_u, ur_nvec_t* doc_u) // u3A->eve_d = eve_d; + // mark all pages dirty + // + u3e_foul(); + *tor_u = rot_u; *doc_u = cod_u; @@ -426,17 +429,15 @@ _cu_realloc(FILE* fil_u, ur_root_t** tor_u, ur_nvec_t* doc_u) /* u3u_meld(): globally deduplicate memory. */ #ifdef U3_MEMORY_DEBUG -c3_w +void u3u_meld(void) { fprintf(stderr, "u3: unable to meld under U3_MEMORY_DEBUG\r\n"); - return 0; } #else -c3_w +void u3u_meld(void) { - c3_w pre_w = u3a_open(u3R); ur_root_t* rot_u; ur_nvec_t cod_u; @@ -448,8 +449,6 @@ u3u_meld(void) // ur_nvec_free(&cod_u); ur_root_free(rot_u); - - return (u3a_open(u3R) - pre_w); } #endif @@ -565,14 +564,56 @@ _cu_rock_save(c3_c* dir_c, c3_d eve_d, c3_d len_d, c3_y* byt_y) // write jam-buffer into [fid_i] // - ssize_t rit_i = c3_pwrite(fid_i, byt_y, len_d, 0); - if ( rit_i < 0 ) { - fprintf(stderr, "rock: write failed: %s\r\n", strerror(errno)); + // XX deduplicate with _write() wrapper in term.c + // + { + ssize_t ret_i; + + while ( len_d > 0 ) { + c3_w lop_w = 0; + // retry interrupt/async errors + // + do { + // abort pathological retry loop + // + if ( 100 == ++lop_w ) { + fprintf(stderr, "rock: write loop: %s\r\n", strerror(errno)); + close(fid_i); + // XX unlink file? + // + return c3n; + } + + ret_i = write(fid_i, byt_y, len_d); + } + while ( (ret_i < 0) + && ( (errno == EINTR) + || (errno == EAGAIN) + || (errno == EWOULDBLOCK) )); + + // assert on true errors + // + // NB: can't call u3l_log here or we would re-enter _write() + // + if ( ret_i < 0 ) { + fprintf(stderr, "rock: write failed %s\r\n", strerror(errno)); + close(fid_i); + // XX unlink file? + // + return c3n; + } + // continue partial writes + // + else { + len_d -= ret_i; + byt_y += ret_i; + } + } } close(fid_i); - return rit_i < 0 ? c3n : c3y; + return c3y; } /* u3u_cram(): globably deduplicate memory, and write a rock to disk. @@ -848,6 +889,10 @@ u3u_uncram(c3_c* dir_c, c3_d eve_d) // u3A->eve_d = eve_d; + // mark all pages dirty + // + u3e_foul(); + // leave rocks on disk // // if ( 0 != c3_unlink(nam_c) ) { diff --git a/pkg/urbit/noun/vortex.c b/pkg/urbit/noun/vortex.c index a37b76888..016588ea8 100644 --- a/pkg/urbit/noun/vortex.c +++ b/pkg/urbit/noun/vortex.c @@ -27,18 +27,9 @@ u3v_life(u3_noun eve) c3_o u3v_boot(u3_noun eve) { - c3_d len_d; - - { - u3_noun len = u3qb_lent(eve); - c3_assert( c3y == u3r_safe_chub(len, &len_d) ); - u3z(len); - } - // ensure zero-initialized kernel // - u3A->roc = 0; - u3A->eve_d = 0; + u3A->roc = 0; { u3_noun pro = u3m_soft(0, u3v_life, eve); @@ -48,8 +39,7 @@ u3v_boot(u3_noun eve) return c3n; } - u3A->roc = u3k(u3t(pro)); - u3A->eve_d = len_d; + u3A->roc = u3k(u3t(pro)); u3z(pro); } @@ -270,61 +260,6 @@ u3v_poke(u3_noun ovo) return pro; } -/* _cv_poke_eve(): u3v_poke w/out u3A->now XX replace -*/ -static u3_noun -_cv_poke_eve(u3_noun sam) -{ - u3_noun fun = u3n_nock_on(u3k(u3A->roc), u3k(u3x_at(_CVX_POKE, u3A->roc))); - u3_noun pro; - - { -# ifdef U3_MEMORY_DEBUG - c3_w cod_w = u3a_lush(u3h(u3t(u3t(sam)))); -# endif - - pro = u3n_slam_on(fun, sam); - -# ifdef U3_MEMORY_DEBUG - u3a_lop(cod_w); -# endif - } - - return pro; -} - -/* u3v_poke_sure(): inject an event, saving new state if successful. -*/ -c3_o -u3v_poke_sure(c3_w mil_w, u3_noun eve, u3_noun* pro) -{ - u3_noun gon = u3m_soft(mil_w, _cv_poke_eve, eve); - u3_noun tag, dat; - u3x_cell(gon, &tag, &dat); - - // event failed, produce trace - // - if ( u3_blip != tag ) { - *pro = gon; - return c3n; - } - - // event succeeded, persist state and produce effects - // - { - u3_noun vir, cor; - u3x_cell(dat, &vir, &cor); - - u3z(u3A->roc); - u3A->roc = u3k(cor); - u3A->eve_d++; - - *pro = u3k(vir); - u3z(gon); - return c3y; - } -} - /* u3v_tank(): dump single tank. */ void diff --git a/pkg/urbit/tests/ames_tests.c b/pkg/urbit/tests/ames_tests.c index 38581d8ca..0907898d4 100644 --- a/pkg/urbit/tests/ames_tests.c +++ b/pkg/urbit/tests/ames_tests.c @@ -6,7 +6,8 @@ static void _setup(void) { - u3m_boot_lite(1 << 22); + u3m_init(1 << 22); + u3m_pave(c3y); } /* _test_ames(): spot check ames helpers diff --git a/pkg/urbit/tests/events_tests.c b/pkg/urbit/tests/events_tests.c deleted file mode 100644 index 92f1bd2fe..000000000 --- a/pkg/urbit/tests/events_tests.c +++ /dev/null @@ -1,251 +0,0 @@ -#include "all.h" -#include "noun/events.h" - -/* _setup(): prepare for tests. -*/ -static void -_setup(void) -{ - // NB: no loom - // - u3P.pag_w = u3a_pages; -} - -static c3_w -_check_north_clean(void) -{ - c3_w i_w, pag_w, blk_w, bit_w; - - for ( i_w = 0; i_w < u3P.pag_w; i_w++ ) { - pag_w = i_w; - blk_w = pag_w >> 5; - bit_w = pag_w & 31; - - if ( u3P.dit_w[blk_w] & (1 << bit_w) ) { - break; - } - } - - return i_w; -} - -static c3_w -_check_north_dirty(c3_w pgs_w, c3_w max_w) -{ - c3_w i_w, pag_w, blk_w, bit_w; - - for ( i_w = 0; i_w < max_w; i_w++ ) { - pag_w = i_w + pgs_w; - blk_w = pag_w >> 5; - bit_w = pag_w & 31; - - if ( !(u3P.dit_w[blk_w] & (1 << bit_w)) ) { - break; - } - } - - return i_w; -} - -static c3_w -_check_south_clean(void) -{ - c3_w i_w, pag_w, blk_w, bit_w; - - for ( i_w = 0; i_w < u3P.pag_w; i_w++ ) { - pag_w = u3P.pag_w - (i_w + 1); - blk_w = pag_w >> 5; - bit_w = pag_w & 31; - - if ( u3P.dit_w[blk_w] & (1 << bit_w) ) { - break; - } - } - - return i_w; -} - -static c3_w -_check_south_dirty(c3_w pgs_w, c3_w max_w) -{ - c3_w i_w, pag_w, blk_w, bit_w; - - for ( i_w = 0; i_w < max_w; i_w++ ) { - pag_w = u3P.pag_w - (i_w + pgs_w + 1); - blk_w = pag_w >> 5; - bit_w = pag_w & 31; - - if ( !(u3P.dit_w[blk_w] & (1 << bit_w)) ) { - break; - } - } - - return i_w; -} - -void -_ce_loom_track_north(c3_w pgs_w, c3_w dif_w); -void -_ce_loom_track_south(c3_w pgs_w, c3_w dif_w); - -static c3_i -_test_tracking(void) -{ - c3_w ret_w; - - _ce_loom_track_north(0, u3P.pag_w); - - if ( u3P.pag_w != (ret_w = _check_north_dirty(0, u3P.pag_w)) ) { - fprintf(stderr, "test events track north dirty all %u\r\n", ret_w); - return 0; - } - - if ( 0 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north init %u\r\n", ret_w); - return 0; - } - - if ( 0 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south init %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(100, 0); - _ce_loom_track_south(1, 0); - - if ( 100 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean a %u\r\n", ret_w); - return 0; - } - - if ( 1 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean a %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(75, 25); - _ce_loom_track_south(2, 0); - - if ( 75 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean b %u\r\n", ret_w); - return 0; - } - - if ( 25 != (ret_w = _check_north_dirty(75, 25)) ) { - fprintf(stderr, "test events track north dirty b %u\r\n", ret_w); - return 0; - } - - if ( 2 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean b %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(55, 20); - _ce_loom_track_south(1, 1); - - if ( 55 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean c %u\r\n", ret_w); - return 0; - } - - if ( 20 != (ret_w = _check_north_dirty(55, 20)) ) { - fprintf(stderr, "test events track north dirty c %u\r\n", ret_w); - return 0; - } - - if ( 1 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean c %u\r\n", ret_w); - return 0; - } - - if ( 1 != (ret_w = _check_south_dirty(1, 1)) ) { - fprintf(stderr, "test events track north dirty c %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(255, 0); - _ce_loom_track_south(48, 0); - - if ( 255 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean d %u\r\n", ret_w); - return 0; - } - - if ( 48 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean d %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(213, 42); - _ce_loom_track_south(15, 33); - - if ( 213 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean e %u\r\n", ret_w); - return 0; - } - - if ( 42 != (ret_w = _check_north_dirty(213, 42)) ) { - fprintf(stderr, "test events track north dirty e %u\r\n", ret_w); - return 0; - } - - if ( 15 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean e %u\r\n", ret_w); - return 0; - } - - if ( 33 != (ret_w = _check_south_dirty(15, 33)) ) { - fprintf(stderr, "test events track north dirty e %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(200, 13); - _ce_loom_track_south(10, 5); - - if ( 200 != (ret_w = _check_north_clean()) ) { - fprintf(stderr, "test events track north clean f %u\r\n", ret_w); - return 0; - } - - if ( 13 != (ret_w = _check_north_dirty(200, 13)) ) { - fprintf(stderr, "test events track north dirty f %u\r\n", ret_w); - return 0; - } - - if ( 10 != (ret_w = _check_south_clean()) ) { - fprintf(stderr, "test events track south clean f %u\r\n", ret_w); - return 0; - } - - if ( 5 != (ret_w = _check_south_dirty(10, 5)) ) { - fprintf(stderr, "test events track north dirty f %u\r\n", ret_w); - return 0; - } - - _ce_loom_track_north(0, u3P.pag_w); - - if ( u3P.pag_w != (ret_w = _check_north_dirty(0, u3P.pag_w)) ) { - fprintf(stderr, "test events track north dirty all %u\r\n", ret_w); - return 0; - } - - return 1; -} - -/* main(): run all test cases. -*/ -int -main(int argc, char* argv[]) -{ - _setup(); - - if ( !_test_tracking() ) { - fprintf(stderr, "test_events: tracking: failed\r\n"); - exit(1); - } - - fprintf(stderr, "test_events: ok\n"); - - return 0; -} diff --git a/pkg/urbit/tests/hashtable_tests.c b/pkg/urbit/tests/hashtable_tests.c index 051ec709e..3b36dc887 100644 --- a/pkg/urbit/tests/hashtable_tests.c +++ b/pkg/urbit/tests/hashtable_tests.c @@ -9,7 +9,8 @@ c3_w _ch_skip_slot(c3_w mug_w, c3_w lef_w); static void _setup(void) { - u3m_boot_lite(1 << 26); + u3m_init(1 << 26); + u3m_pave(c3y); } /* _test_bit_manipulation(): diff --git a/pkg/urbit/tests/jam_tests.c b/pkg/urbit/tests/jam_tests.c index d32796855..fd416945d 100644 --- a/pkg/urbit/tests/jam_tests.c +++ b/pkg/urbit/tests/jam_tests.c @@ -6,7 +6,9 @@ static void _setup(void) { - u3m_boot_lite(1 << 24); + u3m_init(1 << 24); + u3m_pave(c3y); + u3e_init(); } static void diff --git a/pkg/urbit/tests/jet_tests.c b/pkg/urbit/tests/jet_tests.c index d88d55e93..4904a959d 100644 --- a/pkg/urbit/tests/jet_tests.c +++ b/pkg/urbit/tests/jet_tests.c @@ -5,7 +5,8 @@ static void _setup(void) { - u3m_boot_lite(1 << 20); + u3m_init(1 << 20); + u3m_pave(c3y); } static inline c3_i diff --git a/pkg/urbit/tests/mug_tests.c b/pkg/urbit/tests/mug_tests.c index 1c9dcddbc..ffd051674 100644 --- a/pkg/urbit/tests/mug_tests.c +++ b/pkg/urbit/tests/mug_tests.c @@ -5,7 +5,8 @@ static void _setup(void) { - u3m_boot_lite(1 << 20); + u3m_init(1 << 20); + u3m_pave(c3y); } /* _test_mug(): spot check u3r_mug hashes. diff --git a/pkg/urbit/tests/newt_tests.c b/pkg/urbit/tests/newt_tests.c index 75c0a07b1..6fdbdb759 100644 --- a/pkg/urbit/tests/newt_tests.c +++ b/pkg/urbit/tests/newt_tests.c @@ -6,7 +6,8 @@ static void _setup(void) { - u3m_boot_lite(1 << 20); + u3m_init(1 << 20); + u3m_pave(c3y); } /* _newt_encode(): synchronous serialization into a single buffer, for test purposes diff --git a/pkg/urbit/tests/meme_tests.c b/pkg/urbit/tests/nock_tests.c similarity index 80% rename from pkg/urbit/tests/meme_tests.c rename to pkg/urbit/tests/nock_tests.c index 751abb985..a96ed09bd 100644 --- a/pkg/urbit/tests/meme_tests.c +++ b/pkg/urbit/tests/nock_tests.c @@ -5,7 +5,12 @@ static void _setup(void) { - u3m_boot_lite(1 << 24); + // XX at 1<<24, this succeeds on mac, but bail:exit's on linux. + // investigate possible u3n_prog corruption + // + u3m_init(1 << 25); + u3m_pave(c3y); + u3e_init(); } static u3_noun @@ -48,7 +53,7 @@ _test_nock_meme(void) } static c3_i -_test_meme(void) +_test_nock(void) { c3_i ret_i = 1; @@ -67,8 +72,8 @@ main(int argc, char* argv[]) { _setup(); - if ( !_test_meme() ) { - fprintf(stderr, "test meme: failed\r\n"); + if ( !_test_nock() ) { + fprintf(stderr, "test nock: failed\r\n"); exit(1); } @@ -76,6 +81,6 @@ main(int argc, char* argv[]) // u3m_grab(u3_none); - fprintf(stderr, "test meme: ok\r\n"); + fprintf(stderr, "test nock: ok\r\n"); return 0; } diff --git a/pkg/urbit/tests/noun_tests.c b/pkg/urbit/tests/noun_tests.c index 1f0527bd5..af54cd289 100644 --- a/pkg/urbit/tests/noun_tests.c +++ b/pkg/urbit/tests/noun_tests.c @@ -8,7 +8,8 @@ static void _setup(void) { - u3m_boot_lite(1 << 20); + u3m_init(1 << 20); + u3m_pave(c3y); } /* _test_u3r_chop: "extract bit slices from atom" diff --git a/pkg/urbit/vere/db/lmdb.c b/pkg/urbit/vere/db/lmdb.c index c95cf38c1..443f11525 100644 --- a/pkg/urbit/vere/db/lmdb.c +++ b/pkg/urbit/vere/db/lmdb.c @@ -225,112 +225,6 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) } } -/* u3_lmdb_walk_init(): initialize db iterator. -*/ -c3_o -u3_lmdb_walk_init(MDB_env* env_u, - u3_lmdb_walk* itr_u, - c3_d nex_d, - c3_d las_d) -{ - // XX assumes little-endian - // - MDB_val key_u = { .mv_size = sizeof(c3_d), .mv_data = &nex_d }; - MDB_val val_u; - c3_w ops_w, ret_w; - - itr_u->red_o = c3n; - itr_u->nex_d = nex_d; - itr_u->las_d = las_d; - - // create a read-only transaction. - // - ops_w = MDB_RDONLY; - if ( (ret_w = mdb_txn_begin(env_u, 0, ops_w, &itr_u->txn_u)) ) { - mdb_logerror(stderr, ret_w, "lmdb: read txn_begin fail"); - return c3n; - } - // open the database in the transaction - // - ops_w = MDB_CREATE | MDB_INTEGERKEY; - if ( (ret_w = mdb_dbi_open(itr_u->txn_u, "EVENTS", ops_w, &itr_u->mdb_u)) ) { - mdb_logerror(stderr, ret_w, "lmdb: read: dbi_open fail"); - // XX confirm - // - mdb_txn_abort(itr_u->txn_u); - return c3n; - } - - // creates a cursor to iterate over keys starting at [eve_d] - // - if ( (ret_w = mdb_cursor_open(itr_u->txn_u, itr_u->mdb_u, &itr_u->cur_u)) ) { - mdb_logerror(stderr, ret_w, "lmdb: read: cursor_open fail"); - // XX confirm - // - mdb_txn_abort(itr_u->txn_u); - return c3n; - } - - // set the cursor to the position of [eve_d] - // - ops_w = MDB_SET_KEY; - if ( (ret_w = mdb_cursor_get(itr_u->cur_u, &key_u, &val_u, ops_w)) ) { - mdb_logerror(stderr, ret_w, "lmdb: read: initial cursor_get failed"); - fprintf(stderr, " at %" PRIu64 "\r\n", nex_d); - mdb_cursor_close(itr_u->cur_u); - // XX confirm - // - mdb_txn_abort(itr_u->txn_u); - return c3n; - } - - return c3y; -} - -/* u3_lmdb_walk_next(): synchronously read next event from iterator. -*/ -c3_o -u3_lmdb_walk_next(u3_lmdb_walk* itr_u, size_t* len_i, void** buf_v) -{ - MDB_val key_u, val_u; - c3_w ret_w, ops_w; - - c3_assert( itr_u->nex_d <= itr_u->las_d ); - - ops_w = ( c3y == itr_u->red_o ) ? MDB_NEXT : MDB_GET_CURRENT; - if ( (ret_w = mdb_cursor_get(itr_u->cur_u, &key_u, &val_u, ops_w)) ) { - mdb_logerror(stderr, ret_w, "lmdb: walk error"); - return c3n; - } - - // sanity check: ensure contiguous event numbers - // - if ( *(c3_d*)key_u.mv_data != itr_u->nex_d ) { - fprintf(stderr, "lmdb: read gap: expected %" PRIu64 - ", received %" PRIu64 "\r\n", - itr_u->nex_d, - *(c3_d*)key_u.mv_data); - return c3n; - } - - *len_i = val_u.mv_size; - *buf_v = val_u.mv_data; - - itr_u->nex_d++; - itr_u->red_o = c3y; - - return c3y; -} - -/* u3_lmdb_walk_done(): close iterator. -*/ -void -u3_lmdb_walk_done(u3_lmdb_walk* itr_u) -{ - mdb_cursor_close(itr_u->cur_u); - mdb_txn_abort(itr_u->txn_u); -} - /* u3_lmdb_read(): read [len_d] events starting at [eve_d]. */ c3_o diff --git a/pkg/urbit/vere/disk.c b/pkg/urbit/vere/disk.c index 61fe9f754..eb4a27533 100644 --- a/pkg/urbit/vere/disk.c +++ b/pkg/urbit/vere/disk.c @@ -22,12 +22,6 @@ struct _cd_save { struct _u3_disk* log_u; }; -struct _u3_disk_walk { - u3_lmdb_walk itr_u; - u3_disk* log_u; - c3_o liv_o; -}; - #undef VERBOSE_DISK #undef DISK_TRACE_JAM #undef DISK_TRACE_CUE @@ -155,47 +149,34 @@ _disk_commit_start(struct _cd_save* req_u) _disk_commit_after_cb); } - -/* u3_disk_etch(): serialize an event for persistence. RETAIN [eve] +/* _disk_serialize_v1(): serialize events in format v1. */ -size_t -u3_disk_etch(u3_disk* log_u, - u3_noun eve, - c3_l mug_l, - c3_y** out_y) +static c3_w +_disk_serialize_v1(u3_fact* tac_u, c3_y** out_y) { - size_t len_i; - c3_y* dat_y; - #ifdef DISK_TRACE_JAM - u3t_event_trace("disk etch", 'B'); + u3t_event_trace("king disk jam", 'B'); #endif - // XX check version number in log_u - // XX needs api redesign to limit allocations - // { - u3_atom mat = u3qe_jam(eve); + u3_atom mat = u3qe_jam(tac_u->job); c3_w len_w = u3r_met(3, mat); - - len_i = 4 + len_w; - dat_y = c3_malloc(len_i); - - dat_y[0] = mug_l & 0xff; - dat_y[1] = (mug_l >> 8) & 0xff; - dat_y[2] = (mug_l >> 16) & 0xff; - dat_y[3] = (mug_l >> 24) & 0xff; + c3_y* dat_y = c3_malloc(4 + len_w); + dat_y[0] = tac_u->mug_l & 0xff; + dat_y[1] = (tac_u->mug_l >> 8) & 0xff; + dat_y[2] = (tac_u->mug_l >> 16) & 0xff; + dat_y[3] = (tac_u->mug_l >> 24) & 0xff; u3r_bytes(0, len_w, dat_y + 4, mat); - u3z(mat); - } - #ifdef DISK_TRACE_JAM - u3t_event_trace("disk etch", 'E'); + u3t_event_trace("king disk jam", 'E'); #endif - *out_y = dat_y; - return len_i; + u3z(mat); + + *out_y = dat_y; + return len_w + 4; + } } /* _disk_batch(): create a write batch @@ -219,8 +200,7 @@ _disk_batch(u3_disk* log_u, c3_d len_d) for ( c3_d i_d = 0ULL; i_d < len_d; ++i_d) { c3_assert( (req_u->eve_d + i_d) == tac_u->eve_d ); - req_u->siz_i[i_d] = u3_disk_etch(log_u, tac_u->job, - tac_u->mug_l, &req_u->byt_y[i_d]); + req_u->siz_i[i_d] = _disk_serialize_v1(tac_u, &req_u->byt_y[i_d]); tac_u = tac_u->nex_u; } @@ -378,41 +358,6 @@ _disk_read_done_cb(uv_timer_t* tim_u) _disk_read_close(red_u); } -/* u3_disk_sift(): parse a persisted event buffer. -*/ -c3_o -u3_disk_sift(u3_disk* log_u, - size_t len_i, - c3_y* dat_y, - c3_l* mug_l, - u3_noun* job) -{ - if ( 4 >= len_i ) { - return c3n; - } - -#ifdef DISK_TRACE_CUE - u3t_event_trace("disk sift", 'B'); -#endif - - // XX check version in log_u - // - *mug_l = dat_y[0] - ^ (dat_y[1] << 8) - ^ (dat_y[2] << 16) - ^ (dat_y[3] << 24); - - // XX u3m_soft? - // - *job = u3ke_cue(u3i_bytes(len_i - 4, dat_y + 4)); - -#ifdef DISK_TRACE_CUE - u3t_event_trace("disk sift", 'E'); -#endif - - return c3y; -} - /* _disk_read_one_cb(): lmdb read callback, invoked for each event in order */ static c3_o @@ -422,13 +367,29 @@ _disk_read_one_cb(void* ptr_v, c3_d eve_d, size_t val_i, void* val_p) u3_disk* log_u = red_u->log_u; u3_fact* tac_u; + if ( 4 >= val_i ) { + return c3n; + } + { u3_noun job; - c3_l mug_l; + c3_y* dat_y = val_p; + c3_l mug_l = dat_y[0] + ^ (dat_y[1] << 8) + ^ (dat_y[2] << 16) + ^ (dat_y[3] << 24); - if ( c3n == u3_disk_sift(log_u, val_i, (c3_y*)val_p, &mug_l, &job) ) { - return c3n; - } +#ifdef DISK_TRACE_CUE + u3t_event_trace("king disk cue", 'B'); +#endif + + // XX u3m_soft? + // + job = u3ke_cue(u3i_bytes(val_i - 4, dat_y + 4)); + +#ifdef DISK_TRACE_CUE + u3t_event_trace("king disk cue", 'E'); +#endif tac_u = u3_fact_init(eve_d, mug_l, job); } @@ -500,120 +461,6 @@ u3_disk_read(u3_disk* log_u, c3_d eve_d, c3_d len_d) uv_timer_start(&red_u->tim_u, _disk_read_start_cb, 0, 0); } -struct _cd_list { - u3_disk* log_u; - u3_noun eve; - c3_l mug_l; -}; - -/* _disk_read_list_cb(): lmdb read callback, invoked for each event in order -*/ -static c3_o -_disk_read_list_cb(void* ptr_v, c3_d eve_d, size_t val_i, void* val_p) -{ - struct _cd_list* ven_u = ptr_v; - u3_disk* log_u = ven_u->log_u; - - { - u3_noun job; - c3_l mug_l; - - if ( c3n == u3_disk_sift(log_u, val_i, (c3_y*)val_p, &mug_l, &job) ) { - return c3n; - } - - ven_u->mug_l = mug_l; - ven_u->eve = u3nc(job, ven_u->eve); - } - - return c3y; -} - -/* u3_disk_read_list(): synchronously read a cons list of events. -*/ -u3_weak -u3_disk_read_list(u3_disk* log_u, c3_d eve_d, c3_d len_d, c3_l* mug_l) -{ - struct _cd_list ven_u = { log_u, u3_nul, 0 }; - - if ( c3n == u3_lmdb_read(log_u->mdb_u, &ven_u, - eve_d, len_d, _disk_read_list_cb) ) - { - return u3_none; - } - - *mug_l = ven_u.mug_l; - return u3kb_flop(ven_u.eve); -} - -/* u3_disk_walk_init(): init iterator. -*/ -u3_disk_walk* -u3_disk_walk_init(u3_disk* log_u, - c3_d eve_d, - c3_d len_d) -{ - u3_disk_walk* wok_u = c3_malloc(sizeof(*wok_u)); - c3_d max_d = eve_d + len_d - 1; - - wok_u->log_u = log_u; - wok_u->liv_o = u3_lmdb_walk_init(log_u->mdb_u, - &wok_u->itr_u, - eve_d, - c3_min(max_d, log_u->dun_d)); - - return wok_u; -} - -/* u3_disk_walk_live(): check if live. -*/ -c3_o -u3_disk_walk_live(u3_disk_walk* wok_u) -{ - if ( wok_u->itr_u.nex_d > wok_u->itr_u.las_d ) { - wok_u->liv_o = c3n; - } - - return wok_u->liv_o; -} - -/* u3_disk_walk_step(): get next fact. -*/ -c3_o -u3_disk_walk_step(u3_disk_walk* wok_u, u3_fact* tac_u) -{ - u3_disk* log_u = wok_u->log_u; - size_t len_i; - void* buf_v; - - tac_u->eve_d = wok_u->itr_u.nex_d; - - if ( c3n == u3_lmdb_walk_next(&wok_u->itr_u, &len_i, &buf_v) ) { - fprintf(stderr, "disk: (%" PRIu64 "): read fail\r\n", tac_u->eve_d); - return wok_u->liv_o = c3n; - } - - if ( c3n == u3_disk_sift(log_u, len_i, - (c3_y*)buf_v, - &tac_u->mug_l, - &tac_u->job) ) - { - fprintf(stderr, "disk: (%" PRIu64 "): sift fail\r\n", tac_u->eve_d); - return wok_u->liv_o = c3n; - } - - return c3y; -} - -/* u3_disk_walk_done(): close iterator. -*/ -void -u3_disk_walk_done(u3_disk_walk* wok_u) -{ - u3_lmdb_walk_done(&wok_u->itr_u); - c3_free(wok_u); -} - /* _disk_save_meta(): serialize atom, save as metadata at [key_c]. */ static c3_o diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index 0336cdca2..fa7d778f2 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1504,61 +1504,6 @@ _http_serv_init_h2o(SSL_CTX* tls_u, c3_o log, c3_o red) return h2o_u; } -/* _http_serv_start_err(): handle errors in starting http server. -*/ -static c3_i -_http_serv_start_err(const c3_c* cap_c, u3_http* htp_u, c3_i sas_i) -{ - u3_pier* pir_u = htp_u->htd_u->car_u.pir_u; - - if ( UV_EADDRNOTAVAIL == sas_i ) { - u3l_log("http: unavailable ip address %s\r\n", u3_Host.ops_u.bin_c); - u3_king_bail(); - return 0; - } - - // ports specified, no incrementing/retry - // - if ( c3y == htp_u->dis ) { - u3l_log("http: %s (%" PRIu16 "): %s\n", - cap_c, htp_u->por_s, uv_strerror(sas_i)); - u3_king_bail(); - return 0; - } - - // increment and retry - // - if ( (UV_EADDRINUSE == sas_i) || (UV_EACCES == sas_i) ) { - if ( (c3y == htp_u->sec) && (443 == htp_u->por_s) ) { - htp_u->por_s = 8443; - } - else if ( (c3n == htp_u->sec) && (80 == htp_u->por_s) ) { - htp_u->por_s = 8080; - } - else { - htp_u->por_s++; - // XX - // - if ( c3n == htp_u->lop ) { - if ( c3y == htp_u->sec ) { - pir_u->pes_s = htp_u->por_s; - } - else { - pir_u->per_s = htp_u->por_s; - } - } - } - - return 1; - } - - // total failure XX bail? - // - u3l_log("http: %s: %s\n", cap_c, uv_strerror(sas_i)); - _http_serv_free(htp_u); - return 0; -} - /* _http_serv_start(): start http server. */ static void @@ -1574,12 +1519,10 @@ _http_serv_start(u3_http* htp_u) INADDR_ANY; if ( 0 != u3_Host.ops_u.bin_c && c3n == htp_u->lop ) { - // already validated in arguments parser - // inet_pton(AF_INET, u3_Host.ops_u.bin_c, &adr_u.sin_addr); } - c3_assert( !uv_tcp_init(u3L, &htp_u->wax_u) ); + uv_tcp_init(u3L, &htp_u->wax_u); /* Try ascending ports. */ @@ -1589,25 +1532,46 @@ _http_serv_start(u3_http* htp_u) adr_u.sin_port = htons(htp_u->por_s); if ( 0 != (sas_i = uv_tcp_bind(&htp_u->wax_u, - (const struct sockaddr*)&adr_u, 0)) ) - { - if ( _http_serv_start_err("bind", htp_u, sas_i) ) { - continue; + (const struct sockaddr*)&adr_u, 0)) || + 0 != (sas_i = uv_listen((uv_stream_t*)&htp_u->wax_u, + TCP_BACKLOG, _http_serv_listen_cb)) ) { + if ( UV_EADDRNOTAVAIL == sas_i ) { + u3l_log("http: ip address not available\n"); + u3_king_bail(); } - else { - return; + if ( c3y == htp_u->dis ) { + u3l_log("http: listen (%" PRIu16 "): %s\n", htp_u->por_s, + uv_strerror(sas_i)); + u3_king_bail(); } - } + if ( (UV_EADDRINUSE == sas_i) || (UV_EACCES == sas_i) ) { + if ( (c3y == htp_u->sec) && (443 == htp_u->por_s) ) { + htp_u->por_s = 8443; + } + else if ( (c3n == htp_u->sec) && (80 == htp_u->por_s) ) { + htp_u->por_s = 8080; + } + else { + htp_u->por_s++; + // XX + // + if ( c3n == htp_u->lop ) { + if ( c3y == htp_u->sec ) { + pir_u->pes_s = htp_u->por_s; + } + else { + pir_u->per_s = htp_u->por_s; + } + } + } - if ( 0 != (sas_i = uv_listen((uv_stream_t*)&htp_u->wax_u, - TCP_BACKLOG, _http_serv_listen_cb)) ) - { - if ( _http_serv_start_err("listen", htp_u, sas_i) ) { continue; } - else { - return; - } + + u3l_log("http: listen: %s\n", uv_strerror(sas_i)); + + _http_serv_free(htp_u); + return; } u3l_log("http: %s live on %s://localhost:%d\n", @@ -1713,30 +1677,24 @@ _http_init_tls(uv_buf_t key_u, uv_buf_t cer_u) static void _http_write_ports_file(u3_httd* htd_u, c3_c *pax_c) { + c3_c* nam_c = ".http.ports"; + c3_w len_w = 1 + strlen(pax_c) + 1 + strlen(nam_c); + + c3_c* paf_c = c3_malloc(len_w); + snprintf(paf_c, len_w, "%s/%s", pax_c, nam_c); + + c3_i por_i = c3_open(paf_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); + c3_free(paf_c); + u3_http* htp_u = htd_u->htp_u; u3_pier* pir_u = htd_u->car_u.pir_u; - size_t off_i = 0; - c3_i por_i; - - { - c3_c* nam_c = ".http.ports"; - c3_w len_w = 1 + strlen(pax_c) + 1 + strlen(nam_c); - c3_c* paf_c = c3_malloc(len_w); - snprintf(paf_c, len_w, "%s/%s", pax_c, nam_c); - por_i = c3_open(paf_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); - c3_free(paf_c); - } + c3_c temp[32]; while ( 0 != htp_u ) { if ( 0 < htp_u->por_s ) { - c3_c tmp_c[32]; - c3_i len_i = snprintf(tmp_c, 32, "%u %s %s\n", - htp_u->por_s, - (c3y == htp_u->sec) ? "secure" : "insecure", - (c3y == htp_u->lop) ? "loopback" : "public"); - c3_assert( 0 < len_i); - c3_assert( c3_pwrite(por_i, tmp_c, len_i, off_i) == len_i ); - off_i += len_i; + u3_write_fd(por_i, temp, snprintf(temp, 32, "%u %s %s\n", htp_u->por_s, + (c3y == htp_u->sec) ? "secure" : "insecure", + (c3y == htp_u->lop) ? "loopback" : "public")); } htp_u = htp_u->nex_u; @@ -1752,12 +1710,13 @@ static void _http_release_ports_file(c3_c *pax_c) { c3_c* nam_c = ".http.ports"; - c3_w len_w = 1 + strlen(pax_c) + 1 + strlen(nam_c); + c3_w len_w = 1 + strlen(pax_c) + 1 + strlen(nam_c); c3_c* paf_c = c3_malloc(len_w); c3_i wit_i; wit_i = snprintf(paf_c, len_w, "%s/%s", pax_c, nam_c); - c3_assert( len_w == (c3_w)wit_i + 1 ); + c3_assert(wit_i > 0); + c3_assert(len_w == (c3_w)wit_i + 1); c3_unlink(paf_c); c3_free(paf_c); diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index 22b4cce1e..9994fcfcb 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -17,6 +17,48 @@ static void _term_read_cb(uv_stream_t* tcp_u, ssize_t siz_i, const uv_buf_t* buf_u); +/* u3_write_fd(): retry interrupts, continue partial writes, assert errors. +*/ +void +u3_write_fd(c3_i fid_i, const void* buf_v, size_t len_i) +{ + ssize_t ret_i; + + while ( len_i > 0 ) { + c3_w lop_w = 0; + // retry interrupt/async errors + // + do { + // abort pathological retry loop + // + if ( 100 == ++lop_w ) { + fprintf(stderr, "term: write loop: %s\r\n", strerror(errno)); + return; + } + ret_i = write(fid_i, buf_v, len_i); + } + while ( (ret_i < 0) + && ( (errno == EINTR) + || (errno == EAGAIN) + || (errno == EWOULDBLOCK) )); + + // assert on true errors + // + // NB: can't call u3l_log here or we would re-enter u3_write_fd() + // + if ( ret_i < 0 ) { + fprintf(stderr, "term: write failed %s\r\n", strerror(errno)); + c3_assert(0); + } + // continue partial writes + // + else { + len_i -= ret_i; + buf_v += ret_i; + } + } +} + /* _term_msc_out_host(): unix microseconds from current host time. */ static c3_d @@ -213,7 +255,7 @@ u3_term_log_exit(void) if ( c3n == uty_u->sto_f(uty_u) ) { c3_assert(!"exit-tcsetattr"); } - c3_assert(c3_write(uty_u->fid_i, "\r\n", 2) == 2); + u3_write_fd(uty_u->fid_i, "\r\n", 2); } } @@ -822,7 +864,7 @@ _term_spin_step(u3_utty* uty_u) c3_w i_w; for ( i_w = bac_w; i_w < sol_w; i_w++ ) { - if ( c3_write(fid_i, lef_u.base, lef_u.len) < 0 ) { + if ( lef_u.len != write(fid_i, lef_u.base, lef_u.len) ) { return; } } @@ -832,7 +874,7 @@ _term_spin_step(u3_utty* uty_u) { c3_w len_w = cur_c - buf_c; - if ( c3_write(fid_i, buf_c, len_w) < 0 ) { + if ( len_w != write(fid_i, buf_c, len_w) ) { return; } } @@ -840,7 +882,7 @@ _term_spin_step(u3_utty* uty_u) // Cursor stays on spinner. // while ( sol_w-- ) { - if ( c3_write(fid_i, lef_u.base, lef_u.len) < 0 ) { + if ( lef_u.len != write(fid_i, lef_u.base, lef_u.len) ) { return; } } @@ -1002,9 +1044,7 @@ u3_term_ef_ctlc(void) _term_ovum_plan(uty_u->car_u, wir, cad); } - if ( c3n == u3_Host.ops_u.tem ) { - _term_it_refresh_line(uty_u); - } + _term_it_refresh_line(uty_u); } /* _term_it_put_value(): put numeric color value on lin_w. @@ -1342,10 +1382,10 @@ u3_term_io_hija(void) if ( c3y != uty_u->hij_f(uty_u) ) { c3_assert(!"hija-tcsetattr"); } - c3_assert(c3_write(uty_u->fid_i, "\r", 1) == 1); + u3_write_fd(uty_u->fid_i, "\r", 1); { uv_buf_t* buf_u = &uty_u->ufo_u.out.el_u; - c3_assert(c3_write(uty_u->fid_i, buf_u->base, buf_u->len) == buf_u->len); + u3_write_fd(uty_u->fid_i, buf_u->base, buf_u->len); } } } diff --git a/pkg/urbit/vere/io/unix.c b/pkg/urbit/vere/io/unix.c index ba0145d71..f9e175b5d 100644 --- a/pkg/urbit/vere/io/unix.c +++ b/pkg/urbit/vere/io/unix.c @@ -342,11 +342,11 @@ u3_unix_save(c3_c* pax_c, u3_atom pad) pad_y = c3_malloc(fln_w); u3r_bytes(0, fln_w, pad_y, pad); u3z(pad); - ssize_t rit_i = c3_pwrite(fid_i, pad_y, fln_w, 0); + rit_w = write(fid_i, pad_y, fln_w); close(fid_i); c3_free(pad_y); - if ( rit_i < 0 ) { + if ( rit_w != fln_w ) { u3l_log("%s: %s\n", ful_c, strerror(errno)); c3_free(ful_c); u3m_bail(c3__fail); @@ -427,7 +427,7 @@ static c3_w _unix_write_file_hard(c3_c* pax_c, u3_noun mim) { c3_i fid_i = c3_open(pax_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); - c3_w len_w, siz_w, mug_w = 0; + c3_w len_w, rit_w, siz_w, mug_w = 0; c3_y* dat_y; u3_noun dat = u3t(u3t(mim)); @@ -446,10 +446,11 @@ _unix_write_file_hard(c3_c* pax_c, u3_noun mim) u3r_bytes(0, len_w, dat_y, dat); u3z(mim); - ssize_t rit_i = c3_pwrite(fid_i, dat_y, siz_w, 0); + rit_w = write(fid_i, dat_y, siz_w); - if ( rit_i < 0 ) { - u3l_log("error writing %s: %s\r\n", pax_c, strerror(errno)); + if ( rit_w != siz_w ) { + u3l_log("error writing %s: %s\r\n", + pax_c, strerror(errno)); mug_w = 0; } else { @@ -469,7 +470,7 @@ _unix_write_file_soft(u3_ufil* fil_u, u3_noun mim) { struct stat buf_u; c3_i fid_i = c3_open(fil_u->pax_c, O_RDONLY, 0644); - c3_ws len_ws; + c3_ws len_ws, red_ws; c3_w old_w; c3_y* old_y; @@ -488,21 +489,21 @@ _unix_write_file_soft(u3_ufil* fil_u, u3_noun mim) len_ws = buf_u.st_size; old_y = c3_malloc(len_ws); - ssize_t red_i = c3_pread(fid_i, old_y, len_ws, 0); + red_ws = read(fid_i, old_y, len_ws); if ( close(fid_i) < 0 ) { u3l_log("error closing file (soft) %s: %s\r\n", fil_u->pax_c, strerror(errno)); } - if ( red_i != len_ws ) { - if ( red_i < 0 ) { + if ( len_ws != red_ws ) { + if ( red_ws < 0 ) { u3l_log("error reading file (soft) %s: %s\r\n", - fil_u->pax_c, strerror(-red_i)); + fil_u->pax_c, strerror(errno)); } else { - u3l_log("wrong # of bytes read in file %s: %u %zu\r\n", - fil_u->pax_c, len_ws, red_i); + u3l_log("wrong # of bytes read in file %s: %d %d\r\n", + fil_u->pax_c, len_ws, red_ws); } c3_free(old_y); u3z(mim); @@ -892,7 +893,7 @@ _unix_update_file(u3_unix* unx_u, u3_ufil* fil_u) struct stat buf_u; c3_i fid_i = c3_open(fil_u->pax_c, O_RDONLY, 0644); - c3_ws len_ws; + c3_ws len_ws, red_ws; c3_y* dat_y; if ( fid_i < 0 || fstat(fid_i, &buf_u) < 0 ) { @@ -909,21 +910,21 @@ _unix_update_file(u3_unix* unx_u, u3_ufil* fil_u) len_ws = buf_u.st_size; dat_y = c3_malloc(len_ws); - ssize_t red_i = c3_pread(fid_i, dat_y, len_ws, 0); + red_ws = read(fid_i, dat_y, len_ws); if ( close(fid_i) < 0 ) { u3l_log("error closing file %s: %s\r\n", fil_u->pax_c, strerror(errno)); } - if ( red_i != len_ws ) { - if ( red_i < 0 ) { + if ( len_ws != red_ws ) { + if ( red_ws < 0 ) { u3l_log("error reading file %s: %s\r\n", - fil_u->pax_c, strerror(-red_i)); + fil_u->pax_c, strerror(errno)); } else { - u3l_log("wrong # of bytes read in file %s: %u %zu\r\n", - fil_u->pax_c, len_ws, red_i); + u3l_log("wrong # of bytes read in file %s: %d %d\r\n", + fil_u->pax_c, len_ws, red_ws); } c3_free(dat_y); return u3_nul; @@ -1158,7 +1159,7 @@ _unix_initial_update_file(c3_c* pax_c, c3_c* bas_c) { struct stat buf_u; c3_i fid_i = c3_open(pax_c, O_RDONLY, 0644); - c3_ws len_ws; + c3_ws len_ws, red_ws; c3_y* dat_y; if ( fid_i < 0 || fstat(fid_i, &buf_u) < 0 ) { @@ -1175,21 +1176,21 @@ _unix_initial_update_file(c3_c* pax_c, c3_c* bas_c) len_ws = buf_u.st_size; dat_y = c3_malloc(len_ws); - ssize_t red_i = c3_pread(fid_i, dat_y, len_ws, 0); + red_ws = read(fid_i, dat_y, len_ws); if ( close(fid_i) < 0 ) { u3l_log("error closing initial file %s: %s\r\n", pax_c, strerror(errno)); } - if ( red_i != len_ws ) { - if ( red_i < 0 ) { + if ( len_ws != red_ws ) { + if ( red_ws < 0 ) { u3l_log("error reading initial file %s: %s\r\n", - pax_c, strerror(-red_i)); + pax_c, strerror(errno)); } else { - u3l_log("wrong # of bytes read in initial file %s: %u %zu\r\n", - pax_c, len_ws, red_i); + u3l_log("wrong # of bytes read in initial file %s: %d %d\r\n", + pax_c, len_ws, red_ws); } c3_free(dat_y); return u3_nul; diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index b18d7b441..44a8cd502 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -307,7 +307,7 @@ _king_get_pace(void) { struct stat buf_u; c3_c* pat_c; - c3_w len_w; + c3_w red_w, len_w; c3_i ret_i, fid_i; ret_i = asprintf(&pat_c, "%s/.bin/pace", u3_Host.dir_c); @@ -324,10 +324,10 @@ _king_get_pace(void) len_w = buf_u.st_size; pat_c = c3_malloc(len_w + 1); - ssize_t red_i = c3_pread(fid_i, pat_c, len_w, 0); + red_w = read(fid_i, pat_c, len_w); close(fid_i); - if ( red_i != len_w ) { + if ( len_w != red_w ) { c3_free(pat_c); u3l_log("unable to read pace file, " "falling back to default (\"live\")\n"); @@ -1038,6 +1038,9 @@ _king_make_pace(c3_c* pac_c) return 0; } +static c3_i +_king_write_raw(c3_i fid_i, c3_y* buf_y, size_t len_i); + /* _king_init_pace(): save pace file if not present */ static c3_i @@ -1061,11 +1064,8 @@ _king_init_pace(c3_c* pac_c) } } - size_t len_i = strlen(pac_c); - ssize_t wit_i = c3_pwrite(fid_i, pac_c, len_i, 0); - if ( wit_i != len_i ) { - u3l_log("dock: init pace (%s) write failed: %s\n", - pac_c, strerror(errno)); + if ( _king_write_raw(fid_i, (c3_y*)pac_c, strlen(pac_c)) ) { + u3l_log("dock: init pace (%s): write %s\n", pac_c, strerror(errno)); close(fid_i); c3_free(bin_c); return -1; @@ -1239,24 +1239,62 @@ _king_do_upgrade(c3_c* pac_c, c3_c* ver_c) // XX print restart instructions } -static c3_i -_king_copy_raw(c3_i src_i, c3_i dst_i, c3_y* buf_y, size_t pag_i) +/* _king_read_raw: read (up to) [len_i] from [fid_i] to [buf_y] +*/ +static ssize_t +_king_read_raw(c3_i fid_i, c3_y* buf_y, size_t len_i) { - size_t off_i = 0; ssize_t ret_i; do { - if ( 0 > (ret_i = c3_pread(src_i, buf_y, pag_i, off_i)) ) { - return ret_i; - } - - if ( 0 > (ret_i = c3_pwrite(dst_i, buf_y, (size_t)ret_i, off_i)) ) { - return ret_i; - } - - off_i += (size_t)ret_i; + ret_i = read(fid_i, buf_y, len_i); } - while ( ret_i ); + while ( (ret_i < 0) && (errno == EINTR) ); + + return ret_i; +} + +/* _king_read_raw: write [len_i] from [buf_y] to [fid_i]. +*/ +static c3_i +_king_write_raw(c3_i fid_i, c3_y* buf_y, size_t len_i) +{ + ssize_t ret_i; + + while ( len_i ) { + + do { + ret_i = write(fid_i, buf_y, len_i); + } + while ( (ret_i < 0) && (errno == EINTR) ); + + if ( ret_i < 0 ) { + return -1; + } + else { + len_i -= ret_i; + buf_y += ret_i; + } + } + + return 0; +} + +static c3_i +_king_copy_raw(c3_i src_i, c3_i dst_i, c3_y* buf_y, size_t pag_i) +{ + ssize_t red_i; + + do { + if ( 0 > (red_i = _king_read_raw(src_i, buf_y, pag_i)) ) { + return -1; + } + + if ( _king_write_raw(dst_i, buf_y, (size_t)red_i) ) { + return -1; + } + } + while ( red_i ); return 0; } @@ -1299,8 +1337,6 @@ _king_copy_file(c3_c* src_c, c3_c* dst_c) goto done1; } - // XX O_TRUNC? - // if ( -1 == (dst_i = open(dst_c, O_RDWR | O_CREAT, 0755)) ) { err_i = errno; ret_i = -1; @@ -1367,12 +1403,8 @@ _king_copy_file(c3_c* src_c, c3_c* dst_c) { size_t pag_i = 1 << 14;; c3_y* buf_y = c3_malloc(pag_i); - - if ( 0 > (ret_i = _king_copy_raw(src_i, dst_i, buf_y, pag_i)) ) { - err_i = errno; - ret_i = -1; - } - + ret_i = _king_copy_raw(src_i, dst_i, buf_y, pag_i); + err_i = errno; c3_free(buf_y); } diff --git a/pkg/urbit/vere/lord.c b/pkg/urbit/vere/lord.c index d51945530..205190848 100644 --- a/pkg/urbit/vere/lord.c +++ b/pkg/urbit/vere/lord.c @@ -1051,7 +1051,7 @@ _lord_on_serf_err_cb(uv_stream_t* pyp_u, // serf used to write to 2 directly // this can't be any worse than that // - c3_assert(c3_write(STDERR_FILENO, buf_u->base, siz_i) == siz_i); + u3_write_fd(2, buf_u->base, siz_i); } else { uv_read_stop(pyp_u); diff --git a/pkg/urbit/version b/pkg/urbit/version index 64349e8f7..9f76d37b7 100644 --- a/pkg/urbit/version +++ b/pkg/urbit/version @@ -1 +1 @@ -1.14-rc3 \ No newline at end of file +1.13 \ No newline at end of file diff --git a/pkg/urbit/worker/mars.c b/pkg/urbit/worker/mars.c deleted file mode 100644 index 9e38844cc..000000000 --- a/pkg/urbit/worker/mars.c +++ /dev/null @@ -1,267 +0,0 @@ -/* worker/mars.c -** -** the main loop of a mars process. -*/ -#include "all.h" -#include -#include - -/* _mars_step_trace(): initialize or rotate trace file. -*/ -static void -_mars_step_trace(const c3_c* dir_c) -{ - if ( u3C.wag_w & u3o_trace ) { - if ( u3_Host.tra_u.con_w == 0 && u3_Host.tra_u.fun_w == 0 ) { - u3t_trace_open(dir_c); - } - else if ( u3_Host.tra_u.con_w >= 100000 ) { - u3t_trace_close(); - u3t_trace_open(dir_c); - } - } -} - -/* _mars_poke_play(): replay an event. -*/ -static u3_weak -_mars_poke_play(u3_mars* mar_u, c3_d eve_d, u3_noun job) -{ - u3_noun vir; - - if ( c3n == u3v_poke_sure(0, job, &vir) ) { - return vir; - } - - u3z(vir); - return u3_none; -} - -typedef enum { - _play_yes_e, // success - _play_mem_e, // %meme - _play_int_e, // %intr - _play_log_e, // event log fail - _play_mug_e, // mug mismatch - _play_bad_e // total failure -} _mars_play_e; - -/* _mars_play_batch(): replay a batch of events. -*/ -static _mars_play_e -_mars_play_batch(u3_mars* mar_u, c3_o mug_o, c3_w bat_w) -{ - u3_disk* log_u = mar_u->log_u; - u3_disk_walk* wok_u = u3_disk_walk_init(log_u, mar_u->dun_d + 1, bat_w); - u3_fact tac_u; - u3_noun dud; - - while ( c3y == u3_disk_walk_live(wok_u) ) { - if ( c3n == u3_disk_walk_step(wok_u, &tac_u) ) { - u3_disk_walk_done(wok_u); - return _play_log_e; - } - - c3_assert( ++mar_u->sen_d == tac_u.eve_d ); - - if ( u3_none != (dud = _mars_poke_play(mar_u, tac_u.eve_d, tac_u.job)) ) { - c3_m mot_m; - - mar_u->sen_d = mar_u->dun_d; - u3_disk_walk_done(wok_u); - - c3_assert( c3y == u3r_safe_word(u3h(dud), &mot_m) ); - - switch ( mot_m ) { - case c3__meme: { - fprintf(stderr, "play (%" PRIu64 "): %%meme\r\n", tac_u.eve_d); - u3z(dud); - return _play_mem_e; - } - - case c3__intr: { - fprintf(stderr, "play (%" PRIu64 "): %%intr\r\n", tac_u.eve_d); - u3z(dud); - return _play_int_e; - } - - default: { - fprintf(stderr, "play (%" PRIu64 "): failed\r\n", tac_u.eve_d); - u3_pier_punt_goof("play", dud); - // XX say something uplifting - // - return _play_bad_e; - } - } - } - - mar_u->mug_l = u3r_mug(u3A->roc); - - if ( tac_u.mug_l && (mar_u->mug_l != tac_u.mug_l) ) { - fprintf(stderr, "play (%" PRIu64 "): mug mismatch " - "expected %08x, actual %08x\r\n", - tac_u.eve_d, tac_u.mug_l, mar_u->mug_l); - - if ( c3y == mug_o ) { - mar_u->sen_d = mar_u->dun_d; - u3_disk_walk_done(wok_u); - return _play_mug_e; - } - } - - mar_u->dun_d = mar_u->sen_d; - } - - u3_disk_walk_done(wok_u); - - return _play_yes_e; -} - -static c3_o -_mars_do_boot(u3_disk* log_u, c3_d eve_d) -{ - u3_weak eve; - c3_l mug_l; - - if ( u3_none == (eve = u3_disk_read_list(log_u, 1, eve_d, &mug_l)) ) { - fprintf(stderr, "boot: read failed\r\n"); - return c3n; - } - - u3l_log("--------------- bootstrap starting ----------------\r\n"); - - u3l_log("boot: 1-%u\r\n", u3qb_lent(eve)); - - if ( c3n == u3v_boot(eve) ) { - return c3n; - } - - u3l_log("--------------- bootstrap complete ----------------\r\n"); - return c3y; -} - -/* u3_mars_play(): replay logged events up to [eve_d]. -*/ -void -u3_mars_play(u3_mars* mar_u, c3_d eve_d) -{ - u3_disk* log_u = mar_u->log_u; - - if ( !eve_d ) { - eve_d = log_u->dun_d; - } - else if ( eve_d <= mar_u->dun_d ) { - u3l_log("mars: already computed %" PRIu64 "\r\n", eve_d); - u3l_log(" state=%" PRIu64 ", log=%" PRIu64 "\r\n", - mar_u->dun_d, log_u->dun_d); - return; - } - else { - eve_d = c3_min(eve_d, log_u->dun_d); - } - - if ( !mar_u->dun_d ) { - c3_w lif_w; - - if ( c3n == u3_disk_read_meta(log_u, 0, 0, &lif_w) ) { - fprintf(stderr, "mars: disk read meta fail\r\n"); - // XX exit code, cb - // - exit(1); - } - - if ( c3n == _mars_do_boot(mar_u->log_u, lif_w) ) { - fprintf(stderr, "mars: boot fail\r\n"); - // XX exit code, cb - // - exit(1); - } - - mar_u->sen_d = mar_u->dun_d = lif_w; - } - - if ( mar_u->dun_d == log_u->dun_d ) { - u3l_log("mars: nothing to do!\r\n"); - return; - } - - u3l_log("---------------- playback starting ----------------\r\n"); - - if ( (1ULL + eve_d) == log_u->dun_d ) { - u3l_log("play: event %" PRIu64 "\r\n", log_u->dun_d); - } - else if ( eve_d != log_u->dun_d ) { - u3l_log("play: events %" PRIu64 "-%" PRIu64 " of %" PRIu64 "\r\n", - (c3_d)(1ULL + mar_u->dun_d), - eve_d, - log_u->dun_d); - } - else { - u3l_log("play: events %" PRIu64 "-%" PRIu64 "\r\n", - (c3_d)(1ULL + mar_u->dun_d), - eve_d); - } - - { - c3_d mem_d = 0; // last event to meme - c3_w try_w = 0; // [mem_d] retry count - - while ( mar_u->dun_d < eve_d ) { - _mars_step_trace(mar_u->dir_c); - - // XX get batch from args - // - switch ( _mars_play_batch(mar_u, c3y, 1024) ) { - case _play_yes_e: { - u3l_log("play (%" PRIu64 "): done\r\n", mar_u->dun_d); - u3m_reclaim(); - - // XX save a snapshot every N events? - // - } break; - - case _play_mem_e: { - if ( mem_d != mar_u->dun_d ) { - mem_d = mar_u->dun_d; - try_w = 0; - } - else if ( 3 == ++try_w ) { - fprintf(stderr, "play (%" PRIu64 "): failed\r\n", mar_u->dun_d + 1); - u3m_save(); - // XX check loom size, suggest --loom X - // XX exit code, cb - // - u3_disk_exit(log_u); - exit(1); - } - - // XX pack before meld? - // - if ( u3C.wag_w & u3o_auto_meld ) { - u3a_print_memory(stderr, "mars: meld: gained", u3u_meld()); - } - else { - u3a_print_memory(stderr, "mars: pack: gained", u3m_pack()); - } - } break; - - // XX handle any specifically? - // - case _play_int_e: - case _play_log_e: - case _play_mug_e: - case _play_bad_e: { - fprintf(stderr, "play (%" PRIu64 "): failed\r\n", mar_u->dun_d + 1); - u3m_save(); - // XX exit code, cb - // - u3_disk_exit(log_u); - exit(1); - } - } - } - } - - u3l_log("---------------- playback complete ----------------\r\n"); - u3m_save(); -} diff --git a/pkg/urbit/worker/serf.c b/pkg/urbit/worker/serf.c index 377d26fc0..85420a122 100644 --- a/pkg/urbit/worker/serf.c +++ b/pkg/urbit/worker/serf.c @@ -49,6 +49,137 @@ -- */ +/* _serf_space(): print n spaces. +*/ +static void +_serf_space(FILE* fil_u, c3_w n) +{ + for (; n > 0; n--) + (fprintf(fil_u," ")); +} + +/* _serf_print_memory(): print memory amount. +** +** Helper for _serf_prof(), just an un-captioned u3a_print_memory(). +*/ +static void +_serf_print_memory(FILE* fil_u, c3_w wor_w) +{ + c3_w byt_w = (wor_w * 4); + c3_w gib_w = (byt_w / 1000000000); + c3_w mib_w = (byt_w % 1000000000) / 1000000; + c3_w kib_w = (byt_w % 1000000) / 1000; + c3_w bib_w = (byt_w % 1000); + + if ( gib_w ) { + (fprintf(fil_u, "GB/%d.%03d.%03d.%03d\r\n", + gib_w, mib_w, kib_w, bib_w)); + } + else if ( mib_w ) { + (fprintf(fil_u, "MB/%d.%03d.%03d\r\n", mib_w, kib_w, bib_w)); + } + else if ( kib_w ) { + (fprintf(fil_u, "KB/%d.%03d\r\n", kib_w, bib_w)); + } + else { + (fprintf(fil_u, "B/%d\r\n", bib_w)); + } +} + +/* _serf_prof(): print memory profile. RETAIN. +*/ +c3_w +_serf_prof(FILE* fil_u, c3_w den, u3_noun mas) +{ + c3_w tot_w = 0; + u3_noun h_mas, t_mas; + + if ( c3n == u3r_cell(mas, &h_mas, &t_mas) ) { + _serf_space(fil_u, den); + fprintf(fil_u, "mistyped mass\r\n"); + return tot_w; + } + else if ( _(u3du(h_mas)) ) { + _serf_space(fil_u, den); + fprintf(fil_u, "mistyped mass head\r\n"); + { + c3_c* lab_c = u3m_pretty(h_mas); + fprintf(fil_u, "h_mas: %s", lab_c); + c3_free(lab_c); + } + return tot_w; + } + else { + _serf_space(fil_u, den); + + { + c3_c* lab_c = u3m_pretty(h_mas); + fprintf(fil_u, "%s: ", lab_c); + c3_free(lab_c); + } + + u3_noun it_mas, tt_mas; + + if ( c3n == u3r_cell(t_mas, &it_mas, &tt_mas) ) { + fprintf(fil_u, "mistyped mass tail\r\n"); + return tot_w; + } + else if ( c3y == it_mas ) { + tot_w += u3a_mark_noun(tt_mas); + _serf_print_memory(fil_u, tot_w); + +#if 1 + /* The basic issue here is that tt_mas is included in .sac + * (the whole profile), so they can't both be roots in the + * normal sense. When we mark .sac later on, we want tt_mas + * to appear unmarked, but its children should be already + * marked. + */ + if ( _(u3a_is_dog(tt_mas)) ) { + u3a_box* box_u = u3a_botox(u3a_to_ptr(tt_mas)); +#ifdef U3_MEMORY_DEBUG + if ( 1 == box_u->eus_w ) { + box_u->eus_w = 0xffffffff; + } + else { + box_u->eus_w -= 1; + } +#else + if ( -1 == (c3_w)box_u->use_w ) { + box_u->use_w = 0x80000000; + } + else { + box_u->use_w += 1; + } +#endif + } +#endif + + return tot_w; + } + else if ( c3n == it_mas ) { + fprintf(fil_u, "\r\n"); + + while ( _(u3du(tt_mas)) ) { + tot_w += _serf_prof(fil_u, den+2, u3h(tt_mas)); + tt_mas = u3t(tt_mas); + } + + _serf_space(fil_u, den); + fprintf(fil_u, "--"); + _serf_print_memory(fil_u, tot_w); + + return tot_w; + + } + else { + _serf_space(fil_u, den); + fprintf(fil_u, "mistyped (strange) mass tail\r\n"); + return tot_w; + } + } +} + /* _serf_grab(): garbage collect, checking for profiling. RETAIN. */ static void @@ -69,7 +200,7 @@ _serf_grab(u3_noun sac) c3_c* wen_c = u3r_string(wen); c3_c nam_c[2048]; - snprintf(nam_c, 2048, "%s/.urb/put/mass", u3C.dir_c); + snprintf(nam_c, 2048, "%s/.urb/put/mass", u3P.dir_c); struct stat st; if ( -1 == stat(nam_c, &st) ) { @@ -94,7 +225,7 @@ _serf_grab(u3_noun sac) c3_assert( u3R == &(u3H->rod_u) ); fprintf(fil_u, "\r\n"); - tot_w += u3a_maid(fil_u, "total userspace", u3a_prof(fil_u, 0, sac)); + tot_w += u3a_maid(fil_u, "total userspace", _serf_prof(fil_u, 0, sac)); tot_w += u3m_mark(fil_u); tot_w += u3a_maid(fil_u, "space profile", u3a_mark_noun(sac)); @@ -708,7 +839,7 @@ _serf_writ_live_exit(u3_serf* sef_u, c3_w cod_w) c3_c* wen_c = u3r_string(wen); c3_c nam_c[2048]; - snprintf(nam_c, 2048, "%s/.urb/put/profile", u3C.dir_c); + snprintf(nam_c, 2048, "%s/.urb/put/profile", u3P.dir_c); struct stat st; if ( -1 == stat(nam_c, &st) ) { @@ -752,7 +883,7 @@ _serf_writ_live_save(u3_serf* sef_u, c3_d eve_d) exit(1); } - u3m_save(); + u3e_save(); } /* u3_serf_live(): apply %live command [com], producing *ret on c3y. @@ -823,7 +954,7 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret) return c3n; } - u3m_save(); + u3e_save(); u3_serf_grab(); *ret = u3nc(c3__live, u3_nul); @@ -850,7 +981,7 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret) } else { u3z(com); - u3a_print_memory(stderr, "serf: meld: gained", u3u_meld()); + u3u_meld(); *ret = u3nc(c3__live, u3_nul); return c3y; } From 2f17a17406e2b9cfd40f3ce82fa793e27f041d16 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Tue, 3 Jan 2023 11:37:43 -0700 Subject: [PATCH 06/13] Revert "wip" This reverts commit 0e16d82a46ea569e04326ba497b99f93b7ddc751. --- .husky/post-checkout | 3 --- .husky/post-commit | 3 --- .husky/post-merge | 3 --- .husky/pre-push | 3 --- pkg/urbit/noun/manage.c | 6 ------ pkg/urbit/vere/io/fore.c | 7 ++----- 6 files changed, 2 insertions(+), 23 deletions(-) delete mode 100755 .husky/post-checkout delete mode 100755 .husky/post-commit delete mode 100755 .husky/post-merge delete mode 100755 .husky/pre-push diff --git a/.husky/post-checkout b/.husky/post-checkout deleted file mode 100755 index cab40f264..000000000 --- a/.husky/post-checkout +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; } -git lfs post-checkout "$@" diff --git a/.husky/post-commit b/.husky/post-commit deleted file mode 100755 index 9443f4161..000000000 --- a/.husky/post-commit +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; } -git lfs post-commit "$@" diff --git a/.husky/post-merge b/.husky/post-merge deleted file mode 100755 index 828b70891..000000000 --- a/.husky/post-merge +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; } -git lfs post-merge "$@" diff --git a/.husky/pre-push b/.husky/pre-push deleted file mode 100755 index 81a9cc639..000000000 --- a/.husky/pre-push +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } -git lfs pre-push "$@" diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 33b57a43a..aa33b0328 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -433,7 +433,6 @@ u3m_signal(u3_noun sig_l) u3_noun u3m_file(c3_c* pas_c) { - u3l_log("Loading file: %s\r\n", pas_c); struct stat buf_b; c3_i fid_i = c3_open(pas_c, O_RDONLY, 0644); c3_w fln_w, red_w; @@ -444,8 +443,6 @@ u3m_file(c3_c* pas_c) return u3m_bail(c3__fail); } fln_w = buf_b.st_size; - //u3l_log("file size: %i\r\n", fln_w); - pad_y = c3_malloc(buf_b.st_size); red_w = read(fid_i, pad_y, fln_w); @@ -459,9 +456,6 @@ u3m_file(c3_c* pas_c) u3_noun pad = u3i_bytes(fln_w, (c3_y *)pad_y); c3_free(pad_y); - //u3l_log("size of noun\r\n"); - //u3m_p("size", u3dc("met", 3, u3k(pad))); - return pad; } } diff --git a/pkg/urbit/vere/io/fore.c b/pkg/urbit/vere/io/fore.c index ac3240d6f..7354a8225 100644 --- a/pkg/urbit/vere/io/fore.c +++ b/pkg/urbit/vere/io/fore.c @@ -78,12 +78,9 @@ _fore_inject(u3_auto* car_u, c3_c* pax_c) static void _fore_import(u3_auto* car_u, c3_c* pax_c) { - u3_noun fil = u3m_file(pax_c); - //u3m_p("imp file size", u3dc("met", 3, fil)); - u3_noun imp = u3dt("cat", 3, u3i_string("#import_"), fil); + u3_noun arc = u3ke_cue(u3m_file(pax_c)); + u3_noun imp = u3dt("cat", 3, u3i_string("#import_"), arc); u3_noun siz = u3r_met(3, imp); - u3m_p("imp cue size", siz); - u3_noun dat = u3nt(u3_nul, siz, imp); u3_noun req = u3nt(c3n, From 4e3698f327c6749e5be750d28f2a91ddec598487 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 17:47:26 -0600 Subject: [PATCH 07/13] group-store: instead of only watching missing, restart any migration subscriptions that might be bad --- pkg/landscape/app/group-store.hoon | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 16e66fe35..2eb5f3d9c 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -242,19 +242,22 @@ =. wait put-missing =^ cards state - watch-missing + rewatch [cards state] :: - ++ watch-missing + ++ rewatch =/ wait ~(tap in wait) =| cards=(list card) |- ?~ wait [cards state] - ?: (~(has by wex.bol) [/gladio/(scot %p i.wait) i.wait dap.bol]) - $(wait t.wait) + =/ wir /gladio/(scot %p i.wait) =. cards :_(cards (watch-init-migrate i.wait)) + :: if we have a subscription already, leave first to restart + =? cards + (~(has by wex.bol) [wir i.wait dap.bol]) + :_(cards [%pass wir %agent [our.bol dap.bol] %leave ~]) $(wait t.wait) :: ++ put-missing From 264678a908fa4e913aad28259ee5eff47e6f39a4 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 18:17:02 -0600 Subject: [PATCH 08/13] group-store: correct target --- pkg/landscape/app/group-store.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 2eb5f3d9c..b41645948 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -257,7 +257,7 @@ :: if we have a subscription already, leave first to restart =? cards (~(has by wex.bol) [wir i.wait dap.bol]) - :_(cards [%pass wir %agent [our.bol dap.bol] %leave ~]) + :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) $(wait t.wait) :: ++ put-missing From 936835cd0c2cba5f513c888456d4244b5eb6dc47 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 18:50:05 -0600 Subject: [PATCH 09/13] groups-store: fixing wire check for leaves --- pkg/landscape/app/group-store.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index b41645948..5efa484c6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -256,8 +256,8 @@ :_(cards (watch-init-migrate i.wait)) :: if we have a subscription already, leave first to restart =? cards - (~(has by wex.bol) [wir i.wait dap.bol]) - :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) + (~(has by wex.bol) [wir i.wait %groups]) + :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) $(wait t.wait) :: ++ put-missing From 563aed6b7f0c8eba06174885b275d0054158b551 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 4 Jan 2023 11:17:44 -0600 Subject: [PATCH 10/13] group-store: automatically run rebuild on load --- pkg/landscape/app/group-store.hoon | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 5efa484c6..6b3bfc9f6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -84,7 +84,10 @@ =| cards=(list card) |^ ?- -.old - %3 [(flop cards) this(state old)] + %3 + :_ this(state old) + :_ cards + [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] :: %2 %_ $ From 9316aebec75558203d065f2c3576dbfafe1776d2 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 4 Jan 2023 12:40:18 -0600 Subject: [PATCH 11/13] group-store: only run rebuild once --- pkg/landscape/app/group-store.hoon | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 6b3bfc9f6..bd4dcf982 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -41,6 +41,7 @@ state-one state-two state-three + state-four == :: +$ state-zero @@ -61,9 +62,15 @@ =groups wait=(set ship) == +:: ++$ state-four + $: %4 + =groups + wait=(set ship) + == -- :: -=| state-three +=| state-four =* state - :: %- agent:dbug @@ -84,10 +91,15 @@ =| cards=(list card) |^ ?- -.old - %3 - :_ this(state old) - :_ cards - [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] + %4 [(flop cards) this(state old)] + :: + %3 + %_ $ + old [%4 +.old] + cards + :_ cards + [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] + == :: %2 %_ $ @@ -391,8 +403,8 @@ |= arc=* ^- (quip card _state) |^ - =/ sty=state-three - [%3 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~] + =/ sty=state-four + [%4 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~] :_ sty %+ roll ~(tap by groups.sty) |= [[rid=resource grp=group] out=(list card)] From 00f9908cb6fe616bbf6f510b00bc613d5e2926b5 Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 5 Jan 2023 18:53:44 +0100 Subject: [PATCH 12/13] ci: remove the mingw build flow As discussed internally at the end of last year, we are discontinuing Windows support for vere. The tldr is that we presently don't have the resources to support non-*nix systems at the level of quality we want to deliver. Notably Windows is already very much a second-class citizen (#5822 and others). Windows users may want to run Urbit through WSL instead. As such, the Window build step should be removed from our CI. --- .github/workflows/vere.yml | 79 +------------------------------------- 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/.github/workflows/vere.yml b/.github/workflows/vere.yml index eb94e8ea9..9315d8bf4 100644 --- a/.github/workflows/vere.yml +++ b/.github/workflows/vere.yml @@ -162,86 +162,9 @@ jobs: - if: ${{ matrix.os == 'ubuntu-latest' }} run: nix-build -A docker-image - mingw: - runs-on: windows-latest - defaults: - run: - shell: > - C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c - ". <(cygpath '{0}')" - working-directory: ./pkg/urbit - - steps: - - uses: actions/checkout@v2 - with: - lfs: true - - # echo suppresses pacman prompt - - run: echo|./configure - env: - CACHIX_CACHE: ares - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - - - run: mingw32-make build/urbit - - run: mingw32-make test - - run: > - build/urbit -l -d -B ../../bin/solid.pill -F bus && - curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' - http://localhost:12321 - - - name: confirm binary is mostly static - run: | - if [ -z "$(ldd build/urbit | grep -vi "windows/system32")"]; then - echo "it's mostly static" - exit 0 - else - echo "dynamic links found:" - ldd build/urbit - exit 1 - fi - - - uses: actions/setup-python@v2 - if: inputs.upload - with: - python-version: 3.7 - - - uses: google-github-actions/setup-gcloud@v0.6.0 - if: inputs.upload - env: - # see https://github.com/google-github-actions/setup-gcloud/issues/100 - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - with: - service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} - project_id: ${{ secrets.GCS_PROJECT }} - export_default_credentials: true - - - name: upload binary to bootstrap.urbit.org - if: inputs.upload - env: - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - shell: bash - run: | - if [ "real" == "$VERSION_TYPE" ]; then - version="$(cat ./version)" - else - version="${GITHUB_SHA:0:9}" - fi - - system="x86_64-windows" - target="gs://${UPLOAD_BASE}/${VERE_PACE}/${version}/vere-v${version}-${system}.exe" - - gsutil cp -n ./build/urbit.exe "$target" - exitcode=$? - - test $exitcode -eq 0 && - echo "upload to $target complete." || - echo "upload to $target failed."; - exit $exitcode - - after: runs-on: ubuntu-latest - needs: [urbit, mingw] + needs: [urbit] if: inputs.upload steps: - uses: google-github-actions/setup-gcloud@v0.2.0 From b1a5e109f56fbe6eb0cf55cc63bd96360f75d191 Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 5 Jan 2023 18:56:46 +0100 Subject: [PATCH 13/13] ci: remove frontend-tests flow These are no longer being used as much as they were, most of the stuff it was hitting lives in separate repos now. Can be ripped out as such. --- .github/workflows/frontend-test.yml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .github/workflows/frontend-test.yml diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml deleted file mode 100644 index 89e68662a..000000000 --- a/.github/workflows/frontend-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: frontend-test - -on: - pull_request: - paths: - - 'pkg/interface/**' - - 'pkg/btc-wallet/**' - - 'pkg/npm/**' - -jobs: - frontend-test: - runs-on: ubuntu-latest - name: "Test changed frontend packages" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: git fetch --prune - - name: 'Setup root deps' - run: npm ci - - name: 'Setup dependencies' - run: npm run bootstrap - - name: 'Run tests' - run: npm run test -- --since origin/$GITHUB_BASE_REF --include-dependents