de1a726 · urbit.org · git.iko.soy

git.iko.soy/urbit.org

on master help sign in

Merge branch 'crypto-summary' into encryption

de1a726drbeefsupreme5 years ago

_index.md+2−3

compare with master

unifiedsplit

@@ -51,7 +51,6 @@ between Urbit and Unix.

52The workings of Azimuth, the decentralized ID system utilized by Urbit.

54−### [Landscape (Web Client)](@/docs/landscape/_index.md)

54+### [Userspace](@/docs/userspace/_index.md)

56−Developer guides to Landscape, the web client by which most users interact

56+Developer guides for Gall (the vane for controlling userspace apps), Landscape (the web client by which most users interact with their ship), and Threads (transient apps for IO).

57−with their ship.

anchors.pynew file+135

@@ -0,0 +1,135 @@

1+#!/usr/bin/env python3

3+import os

4+import sys

5+from pathlib import Path

6+from typing import List, Tuple

9+# inspired by weights.sh

10+

11+# NOTE: This script does not prevent you from shooting yourself in the foot.

12+# If you manually set a bogus value for insert_anchor_links it will not fix it for you

13+"""

14+1. get a list of all .md files

15+2. for each md file

16+ a. get front matter

17+ b. get value of key `insert_anchor_links`, default to none if not present

18+ c. output a tuple of (setting, file)

19+3. accumulate all anchor setting values for all md files

20+4. write tuple to 'anchors' file, prompt to edit, open in preferred editor

21+5. once prompt is answered, read back file

22+6. read edited 'anchors' file, overwrite new setting for each file

23+"""

24+

25+ANCHOR_KEY = "insert_anchor_links = "

26+ANCHOR_FILE = "anchors"

27+

28+def nth_occurrence(arr, target_value, wanted_idx):

29+ return [idx for idx, val in enumerate(arr) if val == target_value][wanted_idx - 1]

30+

31+def validate_entry(setting: str, file: str):

32+ if not setting in ["left", "right", "none"]:

33+ sys.exit(f"Invalid setting '{setting}' provided for file '{file}'")

34+

35+

36+def find_all_mds() -> List[str]:

37+ # the `insert_anchor_links` is only applicable on `_index.md` files (i.e. sections)

38+ return [str(p) for p in Path('./').rglob('**/_index.md')]

39+

40+def setting_from_frontmatter(lines: List[str]) -> str:

41+ # 'insert_anchor_links = "none"' -> 'none'

42+ for line in lines:

43+ if line.startswith(ANCHOR_KEY):

44+ # 1. 'insert_anchor_links = "none"\n'

45+ # 2. ['insert_anchor_links = ' '"none"\n'][1] => '"none"\n'

46+ # 3. "none\n"

47+ # 4. "none"

48+ return line \

49+ .split(ANCHOR_KEY)[1] \

50+ .replace('\"', '') \

51+ .replace('\n', '')

52+

53+ return "none" # default setting if key not present

54+

55+def anchors_from_frontmatter(md_files: List[str]) -> List[Tuple[str, str]]:

56+ # returns tuple(setting, file)

57+ anchor_settings = []

58+

59+ for md_file in md_files:

60+ with open(md_file, "r") as f:

61+ lines = f.readlines()

62+ setting = setting_from_frontmatter(lines)

63+ anchor_settings.append((setting, md_file))

64+

65+ return anchor_settings

66+

67+def write_anchor_file(anchor_settings: List[Tuple[str, str]]):

68+ with open(ANCHOR_FILE, "w+") as f:

69+ for setting, file in anchor_settings:

70+ f.write(f"{setting} {file}\n")

71+

72+

73+def prompt_editor():

74+ editor = os.getenv('EDITOR', 'nano')

75+ os.system(f"{editor} {ANCHOR_FILE}")

76+ input("Edit the anchor settings and press enter...")

77+

78+def read_anchor_file() -> List[Tuple[str, str]]:

79+ anchor_settings = []

80+

81+ with open(ANCHOR_FILE, "r") as f:

82+ for line in f:

83+ line = line.replace('\n', '')

84+ setting, file = line.split(' ')

85+ anchor_settings.append((setting, file))

86+

87+ return anchor_settings

88+

89+

90+def write_entry(setting: str, file: str):

91+ validate_entry(setting, file)

92+

93+

94+ # not optimized

95+ with open(file, "r+") as f:

96+ lines = f.readlines()

97+ # lines = list(filter(bool, text.split('\n'))) # split text by lines,

98+ fm_end = nth_occurrence(lines, "+++\n", 2) # get index of end of front matter (signaled by second '+++')

99+

100+ # remove existing setting if it exists

101+ for line in lines[:fm_end]:

102+ if line.startswith(ANCHOR_KEY):

103+ lines.remove(line)

104+ fm_end -= 1 # correct the index for the removal

105+

106+ if setting != "none":

107+ # by default, there is no need to specify "none"

108+ lines.insert(fm_end, ANCHOR_KEY + f'"{setting}"' + '\n') # insert the setting right before the end of front matter

109+ f.seek(0) # go to beginning of file

110+ f.writelines(lines) # overwrite with new contents

111+ f.truncate() # idk why but necessary for bug-free behviour (otherwise it adds excess newlines and stuff)

112+

113+

114+

115+def anchors_to_frontmatter(anchor_settings: List[Tuple[str, str]]):

116+ for setting, file in anchor_settings:

117+ write_entry(setting, file)

118+

119+def cleanup():

120+ os.remove("anchors")

121+

122+

123+def main():

124+ files = find_all_mds()

125+ print(f'found {len(files)} .md files')

126+ write_anchor_file(anchors_from_frontmatter(files))

127+ prompt_editor()

128+ anchors_to_frontmatter(read_anchor_file())

129+ cleanup()

130+ print('done')

131+

132+

133+if __name__ == '__main__':

134+ main()

135+

arvo/_index.md+1−1

@@ -36,7 +36,7 @@ Eyre is our HTTP server. Unix sends HTTP messages to `%eyre`, and `%eyre` produc

37Ford is our build system. It handles resources and publishing.

39−## [Gall](@/docs/arvo/gall/gall.md)

39+## [Gall](@/docs/arvo/gall/overview.md)

41Gall is the vane for controlling userspace apps.

arvo/ames/_index.md+1

@@ -3,4 +3,5 @@ title = "Ames"

3weight = 10

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/ames/ames.md+1−1

@@ -11,7 +11,7 @@ over it. When Unix receives a packet over the correct UDP port, it pipes

11it straight into Ames for handling. Also, all packets sent over the

12Ames network are sent by the Ames vane. Apps and vanes may use

13Ames to directly send messages to other ships. In general, apps use

14−[Gall](@/docs/arvo/gall/gall.md) and [Clay](@/docs/arvo/clay/clay.md)

14+[Gall](@/docs/userspace/gall/gall.md) and [Clay](@/docs/arvo/clay/clay.md)

15to communicate with other ships rather than using Ames

16directly, but this isn't a requirement. Of course, Gall and Clay use

17Ames behind the scenes to communicate across the network. Jael is the only other

arvo/behn/_index.md+1

@@ -3,4 +3,5 @@ title = "Behn"

3weight = 20

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/clay/_index.md+1

@@ -3,6 +3,7 @@ title = "Clay"

3weight = 30

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8[Overview](@/docs/arvo/clay/clay.md)

9[Overview](@/docs/arvo/clay/clay.md)

arvo/clay/architecture.md+82−78

@@ -4,76 +4,79 @@ weight = 2

4template = "doc.html"

5+++

7−Clay is the primary filesystem for the arvo operating system,

7+Clay is the primary filesystem for the [Arvo](@/docs/arvo/overview.md) operating system,

8−which is the [core](/docs/glossary/core/) of an urbit. The architecture of clay is

8+which is the [core](/docs/glossary/core/) of an urbit. The architecture of Clay is

9−intrinsically connected with arvo, but we assume no knowledge of

9+intrinsically connected with Arvo, but for this section we assume no knowledge of

10−either arvo or urbit. We will point out only those features of

10+either Arvo or Urbit. We will point out only those features of

11−arvo that are necessary for an understanding of clay, and we will

11+Arvo that are necessary for an understanding of Clay, and we will

12do so only when they arise.

14−The first relevant feature of arvo is that it is a deterministic

14+The first relevant feature of Arvo is that it is a deterministic

15system where input and output are defined as a series of events

16−and effects. The state of arvo is simply a function of its event

16+and effects. The state of Arvo is simply a pure function of its event

17log. None of the effects from an event are emitted until the

18event is entered in the log and persisted, either to disk or

19another trusted source of persistence, such as a Kafka cluster.

20−Consequently, arvo is a single-level store: everything in its

20+Consequently, Arvo is a single-level store: everything in its

21state is persistent.

23In a more traditional OS, everything in RAM can be erased at any

24time by power failure, and is always erased on reboot. Thus, a

25primary purpose of a filesystem is to ensure files persist across

26−power failures and reboots. In arvo, both power failures and

26+power failures and reboots. In Arvo, both power failures and

27reboots are special cases of suspending computation, which is

28done safely since our event log is already persistent. Therefore,

29−clay is not needed in arvo for persistence. Why, then, do we have a

29+Clay is not needed in Arvo for persistence. Why, then, do we have a

30filesystem? There are two answers to this question.

32−First, clay provides a filesystem tree, which is a convenient

32+First, Clay provides a filesystem tree, which is a convenient

33user interface for some applications. Unix has the useful concept

34of virtual filesystems, which are used for everything from direct

35access to devices, to random number generators, to the /proc

36tree. It is easy and intuitive to read from and write to a

37filesystem tree.

39−Second, clay has a distributed revision-control system baked into

39+Second, Clay has a distributed revision-control system baked into

40it. Traditional filesystems are not revision controlled, so

41userspace software -- such as git -- is written on top of them to

42−do so. clay natively provides the same functionality as modern

42+do so. Clay natively provides the same functionality as modern

43DVCSes, and more.

45−clay has two other unique properties that we'll cover later on:

45+Clay has two other unique properties that we'll cover later on:

46it supports typed data and is referentially transparent.

48### Revision Control

50−Every urbit has one or more "desks", which are independently

50+Every urbit has one or more `desk`s, which are independently

51−revision-controlled branches. Each desk contains its own mark

51+revision-controlled branches. Each `desk` contains its own `mark`

52−definitions, apps, doc, and so forth.

52+definitions, apps, and so forth.

54−Traditionally, an urbit has at least a base and a home desk. The

54+Traditionally, an urbit has at least a `%home` `desk` and `%kids` `desk`. The

55−base desk has all the system software from the distribution. the

55+`%home` `desk` has all the system software from the distribution, along with your

56−home desk is a fork of base with all the stuff specific to the

56+personal files. The `%home` `desk` is a fork of the `%kids` desk of whichever ship

57−user of the urbit.

57+you download [OTAs](@/docs/glossary/ota-updates.md) from - typically your sponsor,

58+but theoretically may be any ship. You will also find references to a `%base`

59+`desk` - which is not a real `desk`, rather it just a hash associated to the

60+`desk` from which your `%home` `desk` was forked.

59−A desk is a series of numbered commits, the most recent of which

62+A `desk` is a series of numbered commits, the most recent of which

60−represents the current state of the desk. A commit is composed of

63+represents the current state of the `desk`. A commit is composed of

61(1) an absolute time when it was created, (2) a list of zero or

64(1) an absolute time when it was created, (2) a list of zero or

62more parents, and (3) a map from paths to data.

65more parents, and (3) a map from paths to data.

64Most commits have exactly one parent, but the initial commit on a

67Most commits have exactly one parent, but the initial commit on a

65−desk may have zero parents, and merge commits have more than one

68+`desk` may have zero parents, and merge commits have more than one

66parent.

69parent.

68−The non-meta data is stored in the map of paths to data. It's

71+The non-metadata is stored as a `map` of `path`s to data. It's

69−worth noting that no constraints are put on this map, so, for

72+worth noting that no constraints are put on this `map`, so, for

70−example, both /a/b and /a/b/c could have data. This is impossible

73+example, both `/a/b` and `/a/b/c` could have data. This is impossible

71−in a traditional Unix filesystem since it means that /a/b is both

74+in a traditional Unix filesystem since it means that `/a/b` is both

72a file and a directory. Conventionally, the final element in the

75a file and a directory. Conventionally, the final element in the

73−path is its mark -- much like a filename extension in Unix. Thus,

76+path is its `mark` -- much like a filename extension in Unix. Thus,

74−/doc/readme.md in Unix is stored as /doc/readme/md in urbit.

77+`/doc/readme.md` in Unix is stored as `/doc/readme/md` in urbit.

76−The data is not stored directly in the map; rather, a hash of the

79+The data is not stored directly in the `map`; rather, a hash of the

77data is stored, and we maintain a master blob store. Thus, if the

80data is stored, and we maintain a master blob store. Thus, if the

78same data is referred to in multiple commits (as, for example,

81same data is referred to in multiple commits (as, for example,

79when a file doesn't change between commits), only the hash is

82when a file doesn't change between commits), only the hash is

@@ -85,8 +88,8 @@ only on the data within and not on whether or not it's stored

85directly, so we may on occasion rearrange the contents of the

88directly, so we may on occasion rearrange the contents of the

86blob store for performance reasons.

89blob store for performance reasons.

88−Recall that a desk is a series of numbered commits. Not every

91+Recall that a `desk` is a series of numbered commits. Not every

89−commit in a desk must be numbered. For example, if the base desk

92+commit in a `desk` must be numbered. For example, if the base `desk`

90has had 50 commits since home was forked from it, then a merge

93has had 50 commits since home was forked from it, then a merge

91from base to home will only add a single revision number to home,

94from base to home will only add a single revision number to home,

92although the full commit history will be accessible by traversing

95although the full commit history will be accessible by traversing

@@ -101,20 +104,20 @@ There are three ways to refer to particular commits in the

101revision history. Firstly, one can use the revision number.

104revision history. Firstly, one can use the revision number.

102Secondly, one can use any absolute time between the one numbered

105Secondly, one can use any absolute time between the one numbered

103commit and the next (inclusive of the first, exclusive of the

106commit and the next (inclusive of the first, exclusive of the

104−second). Thirdly, every desk has a map of labels to revision

107+second). Thirdly, every `desk` has a `map` of labels to revision

105numbers. These labels may be used to refer to specific commits.

108numbers. These labels may be used to refer to specific commits.

106

109

107−Additionally, clay is a global filesystem, so data on other urbit

110+Additionally, Clay is a global filesystem, so data on other urbits

108is easily accessible the same way as data on our local urbit. In

111is easily accessible the same way as data on our local urbit. In

109−general, the path to a particular revision of a desk is

112+general, the path to a particular revision of a `desk` is

110−/~urbit-name/desk-name/revision. Thus, to get /try/readme/md

113+`/~urbit-name/desk-name/revision`. Thus, to get `/try/readme/md`

111−from revision 5 of the home desk on ~sampel-sipnym, we refer to

114+from revision 5 of the home `desk` on `~sampel-sipnym`, we refer to

112−/~sampel-sipnym/home/5/try/readme/md. Clay's namespace is thus

115+`/~sampel-sipnym/home/5/try/readme/md`. Clay's namespace is thus

113global and referentially transparent.

116global and referentially transparent.

114

117

115### A Typed Filesystem

118### A Typed Filesystem

116

119

117−Since clay is a general filesystem for storing data of arbitrary

120+Since Clay is a general filesystem for storing data of arbitrary

118types, in order to revision control correctly it needs to be

121types, in order to revision control correctly it needs to be

119aware of types all the way through. Traditional revision control

122aware of types all the way through. Traditional revision control

120does an excellent job of handling source code, so for source code

123does an excellent job of handling source code, so for source code

@@ -164,33 +167,34 @@ a change. Two diffs consisting of a single added word to

164different sentences may be flagged as a conflict. In general,

167different sentences may be flagged as a conflict. In general,

165prose should be diffed by sentence, not by line.

168prose should be diffed by sentence, not by line.

166

169

167−As far as the author is aware, clay is the first generalized,

170+As far as we are aware, Clay is the first generalized,

168type-aware revision control system. We'll go into the workings

171type-aware revision control system. We'll go into the workings

169of this system in some detail.

172of this system in some detail.

170

173

171### Marks

174### Marks

172

175

173−Central to a typed filesystem is the idea of types. In clay, we

176+Central to a typed filesystem is the idea of types. In Clay, we

174−call these "marks". A mark is a file that defines a type,

177+call these `mark`s. A `mark` is a file that defines a type,

175−conversion routines to and from the mark, and diff, patch, and

178+conversion routines to and from the `mark`, and diff, patch, and

176merge routines.

179merge routines.

177

180

178For example, a `%txt` mark may be a list of lines of text, and it

181For example, a `%txt` mark may be a list of lines of text, and it

179may include conversions to `%mime` to allow it to be serialized

182may include conversions to `%mime` to allow it to be serialized

180−and sent to a browswer or to the unix filesystem. It will also

183+and sent to a browser or to the Unix filesystem. It will also

181include Hunt-McIlroy diff, patch, and merge algorithms.

184include Hunt-McIlroy diff, patch, and merge algorithms.

182

185

183−A `%json` mark would be defined as a json object in the code, and

186+A `%json` `mark` would be defined as a JSON object in the code, and

184it would have a parser to convert from `%txt` and a printer to

187it would have a parser to convert from `%txt` and a printer to

185convert back to `%txt`. The diff, patch, and merge algorithms are

188convert back to `%txt`. The diff, patch, and merge algorithms are

186−fairly straightforward for json, though they're very different

189+fairly straightforward for JSON, though they're very different

187from the text ones.

190from the text ones.

188

191

189−More formally, a mark is a core with three [arms](/docs/glossary/arm/), `++grab`,

192+More formally, a `mark` is a core with three [arms](/docs/glossary/arm/): `+grab`,

190−`++grow`, and `++grad`. In `++grab` is a series of functions to

193+`+grow`, and `+grad`. In `+grab` is a series of functions to

191−convert from other marks to the given mark. In `++grow` is a

194+convert from other `mark`s to the given `mark`. In `+grow` is a

192−series of functions to convert from the given mark to other

195+series of functions to convert from the given `mark` to other

193−marks. In `++grad` is `++diff`, `++pact`, `++join`, and `++mash`.

196+`mark`s. In `+grad` is `+diff`, `+pact`, `+join`, and `+mash`, and

197+`+form`.

194

198

195The types are as follows, in an informal pseudocode:

199The types are as follows, in an informal pseudocode:

196

200

@@ -204,47 +208,47 @@ The types are as follows, in an informal pseudocode:

204 ++ txt: <mark-type> -> <txt>

208 ++ txt: <mark-type> -> <txt>

205 ...

209 ...

206 ++ grad

210 ++ grad

211+ ++ form: <mark-type>

207 ++ diff: (<mark-type>, <mark-type>) -> <diff-type>

212 ++ diff: (<mark-type>, <mark-type>) -> <diff-type>

208 ++ pact: (<mark-type>, <diff-type>) -> <mark-type>

213 ++ pact: (<mark-type>, <diff-type>) -> <mark-type>

209 ++ join: (<diff-type>, <diff-type>) -> <diff-type> or NULL

214 ++ join: (<diff-type>, <diff-type>) -> <diff-type> or NULL

210 ++ mash: (<diff-type>, <diff-type>) -> <diff-type>

215 ++ mash: (<diff-type>, <diff-type>) -> <diff-type>

211```

216```

212

217

213−These types are basically what you would expect. Not every mark

218+These types are basically what you would expect. Not every `mark`

214has each of these functions defined -- all of them are optional

219has each of these functions defined -- all of them are optional

215in the general case.

220in the general case.

216

221

217−In general, for a particular mark, the `++grab` and `++grow` entries

222+In general, for a particular `mark`, the `+grab` and `+grow` entries

218(if they exist) should be inverses of each other.

223(if they exist) should be inverses of each other.

219

224

220−In `++grad`, `++diff` takes two instances of a mark and produces

225+In `+grad`, `+diff` takes two instances of a `mark` and produces a diff of them

221−a diff of them. `++pact` takes an instance of a mark and patches

226+whose `mark` is given by `+form`. `+pact` takes an instance of a `mark` and

222−it with the given diff. `++join` takes two diffs and attempts to

227+patches it with the given diff. `+join` takes two diffs and attempts to merge

223−merge them into a single diff. If there are conflicts, it

228+them into a single diff. If there are conflicts, it produces null. `+mash` takes

224−produces null. `++mash` takes two diffs and forces a merge,

225−annotating any conflicts.

229+two diffs and forces a merge, annotating any conflicts.

226

230

227−In general, if `++diff` called with A and B produces diff D, then

231+In general, if `+diff` called with A and B produces diff D, then

228−`++pact` called with A and D should produce B. Also, if `++join`

232+`+pact` called with A and D should produce B. Also, if `+join`

229−of two diffs does not produce null, then `++mash` of the same

233+of two diffs does not produce null, then `+mash` of the same

230diffs should produce the same result.

234diffs should produce the same result.

231

235

232−Alternately, instead of `++diff`, `++pact`, `++join`, and

236+Alternately, instead of `+diff`, `+pact`, `+join`, and

233−`++mash`, a mark can provide the same functionality by defining

237+`+mash`, a `mark` can provide the same functionality by defining

234−`++sted` to be the name of another mark to which we wish to

238+`+grad` to be the name of another `mark` to which we wish to

235delegate the revision control responsibilities. Then, before

239delegate the revision control responsibilities. Then, before

236−running any of those functions, clay will convert to the other

240+running any of those functions, Clay will convert to the other

237−mark, and convert back afterward. For example, the `%hoon` mark

241+`mark`, and convert back afterward. For example, the `%hoon` `mark`

238−is revision-controlled in the same way as `%txt`, so its `++grad`

242+is revision-controlled in the same way as `%txt`, so its `+grad`

239−is simply `++sted %txt`. Of course, `++txt` must be defined in

243+is simply `++ grad %txt`. Of course, `+txt` must be defined in

240−`++grow` and `++grab` as well.

244+`+grow` and `+grab` as well.

241−

245+

242−Every file in clay has a mark, and that mark must have a

246+Every file in Clay has a `mark`, and that `mark` must have a

243−fully-functioning `++grad`. Marks are used for more than just

247+fully-functioning `+grad`. `Mark`s are used for more than just

244−clay, and other marks don't need a `++grad`, but if a piece of

248+Clay, and other `mark`s don't need a `+grad`, but if a piece of

245−data is to be saved to clay, we must know how to revision-control

249+data is to be saved to Clay, we must know how to revision-control

246it.

250it.

247

251

248−Additionally, if a file is to be synced out to unix, then it must

252+Additionally, if a file is to be synced out to Unix, then it must

249−have conversion routines to and from the `%mime` mark.

253+have conversion routines to and from the `%mime` `mark`.

250

254

arvo/clay/clay.md+14−4

@@ -24,23 +24,33 @@ is the logic for communicating requests to, and receiving requests from,

24foreign ships.

26Clay documentation is extensive, but the reader much tread carefully as portions

27−of it are out of date as of March 2021. We are actively working on updating

27+of it are out of date as of April 2021. We are actively working on updating

28them. If you'd like to help, please reach out to us in the Docs channel on Urbit

29Community or submit a PR at

30[https://github.com/urbit/docs](https://github.com/urbit/docs).

32+### User documentation

33+

34+[Filesystem](@/using/os/filesystem.md)

35+

36+How to interact with the Clay filesystem via Dojo. This includes basics such as

37+mounting to Unix, changing directory, merging, and listing files. (Up to date as

38+of April 2021)

39+

40+### Developer Documentation

41+

32[Architecture](@/docs/arvo/clay/architecture.md)

42[Architecture](@/docs/arvo/clay/architecture.md)

34−A conceptual overview of how Clay was designed.

44+A conceptual overview of how Clay was designed. (Up to date as of April 2021)

36[Using Clay](@/docs/arvo/clay/using.md)

46[Using Clay](@/docs/arvo/clay/using.md)

38A quick overview of how the most common tasks involving Clay are performed:

48A quick overview of how the most common tasks involving Clay are performed:

39−reading and subscribing, syncing to Unix, and merging.

49+reading and subscribing, syncing to Unix, and merging. (Up to date as of April 2021)

41[Data Types](@/docs/arvo/clay/data-types.md)

51[Data Types](@/docs/arvo/clay/data-types.md)

43−Explanations of the many data types found throughout Clay.

53+Explanations of the many data types found throughout Clay. (Up to date as of April 2021)

45[Local Reads and Writes](@/docs/arvo/clay/local-reads.md)

55[Local Reads and Writes](@/docs/arvo/clay/local-reads.md)

arvo/clay/data-types.md+415−487

@@ -5,14 +5,15 @@ template = "doc.html"

5+++

7This section will be reference documentation for the data types used by our

8−filesystem. Parts of it may be inaccurate as of March 2021. As a general guide,

8+filesystem. It is up to date as of April 2021.

9−we recommend reading and attempting to understand the data structures used in

10+As a general guide, we recommend reading and attempting to understand the data

10−any Hoon code before you try to read the code itself. Although complete

11+structures used in any Hoon code before you try to read the code itself.

11−understanding of the data structures is impossible without seeing them used in

12+Although complete understanding of the data structures is impossible without

12−the code, an 80% understanding greatly clarifies the code. As another general

13+seeing them used in the code, an 80% understanding greatly clarifies the code.

13−guide, when reading Hoon, it rarely pays off to understand every line of code

14+As another general guide, when reading Hoon, it rarely pays off to understand

14−when it appears. Try to get the gist of it, and then move on. The next time you

15+every line of code when it appears. Try to get the gist of it, and then move on.

15−come back to it, it'll likely make a lot more sense.

16+The next time you come back to it, it'll likely make a lot more sense.

17### Data Models

18### Data Models

@@ -23,191 +24,275 @@ at least skimming, this so that you get a rough idea of how our state is

23organized.

24organized.

25The types that are certainly worth reading are `++raft`, `++room`,

26The types that are certainly worth reading are `++raft`, `++room`,

26−`++dome:clay`, `++ankh:clay`, `++rung:clay`, `++rang:clay`, `++blob:clay`, `++yaki:clay`, and `++nori:clay`

27+`++dome:clay`, `++ankh:clay`, `++rung:clay`, `++rang:clay`, `++blob:clay`,

27−(possibly in that order). All in all, though, this section isn't too

28+`++yaki:clay`, and `++nori:clay` (possibly in that order). All in all, though,

28−long, so many readers may wish to quickly read through all of it. If you

29+this section isn't too long, so many readers may wish to quickly read through

29−get bored, though, just skip to the next section. You can always come

30+all of it. If you get bored, though, just skip to the next section. You can

30−back when you need to.

31+always come back when you need to.

32−### `++raft`, formal state

33+### `$raft`, formal state

34```hoon

35```hoon

35− ++ raft :: filesystem

36++$ raft :: filesystem

36− $: fat=(map ship room) :: domestic

37+ $: rom=room :: domestic

37− hoy=(map ship rung) :: foreign

38+ hoy=(map ship rung) :: foreign

38− ran=rang :: hashes

39+ ran=rang :: hashes

40+ mon=(map term beam) :: mount points

41+ hez=(unit duct) :: sync duct

42+ cez=(map @ta crew) :: permission groups

43+ pud=(unit [=desk =yoki]) :: pending update

39− == ::

44+ == ::

40```

45```

42−This is the state of our vane. Anything that must be remembered between

47+This is the state of the vane. Anything that must be remembered between

43−calls to clay is stored in this state.

48+calls to Clay is stored in this state.

44−

45−`fat` is the set of domestic servers. This stores all the information

46−that is specfic to a particular ship on this pier. The keys to this map

47−are the ships on the current pier. all the information that is specific

48−to a particular foreign ship. The keys to this map are all the ships

49−whose filesystems we have attempted to access through clay.

51`ran` is the store of all commits and deltas, keyed by hash. The is

50`ran` is the store of all commits and deltas, keyed by hash. The is

52where all the "real" data we know is stored; the rest is "just

51where all the "real" data we know is stored; the rest is "just

53bookkeeping".

52bookkeeping".

54+`rom` is the state for all local desks. It consists of a `duct` to

55+[Dill](@/docs/arvo/dill/dill.md) and a collection of `desk`s.

56+

57+`hoy` is the state for all foreign desks.

58+

59+`ran` is the global, hash-addressed object store. It has maps of commit hashes

60+to commits and content hashes to content.

61+

62+`mon` is a collection of Unix mount points. `term` is the mount point (relative

63+to th pier) and `beam` is a domestic Clay directory.

64+

65+`hez` is the duct used to sync with Unix.

66+

67+`cez` is a collection of named permission groups.

68+

69+`pud` is an update that's waiting on a kernel upgrade.

70+

71+

55−#### `++room`, filesystem per domestic ship

72+#### `$room`, filesystem per domestic ship

57```hoon

74```hoon

58− ++ room :: fs per ship

75++$ room :: fs per ship

59− $: hun=duct :: terminal duct

76+ $: hun=duct :: terminal duct

60− hez=(unit duct) :: sync duch

61− dos=(map desk dojo) :: native desk

77+ dos=(map desk dojo) :: native desk

62− == ::

78+ == ::

63```

79```

65This is the representation of the filesystem of a ship on our pier.

81This is the representation of the filesystem of a ship on our pier.

67−`hun` is the duct we use to send messages to dill to display

83+`hun` is the duct we use to send messages to [Dill](@/docs/arvo/dill/dill.md) to

68−notifications of filesystem changes. Only `%note` gifts should be

84+display notifications of filesystem changes. Only `%note` `%gift`s should be

69−produced along this duct. This is set by the `%init` kiss.

85+produced along this `duct`. This is set by the `%init` `move`.

70−

71−`hez`, if present, is the duct we use to send sync messages to unix so

72−that they end up in the pier unix directory. Only `%ergo` gifts should

73−be producd along this duct. This is set by `%into` and `%invo` kisses.

75`dos` is a well-known operating system released in 1981. It is also the

87`dos` is a well-known operating system released in 1981. It is also the

76−set of desks on this ship, mapped to their data.

88+set of `desk`s on this ship, mapped to their `desk` state.

78−#### `++desk`, filesystem branch

90+#### `$desk`, filesystem branch

80```hoon

92```hoon

81− ++ desk ,@tas :: ship desk case spur

93++$ desk @tas

82```

94```

84−This is the name of a branch of the filesystem. The default desks are

96+This is the name of a branch of the filesystem. The default `desk`s are `%home`

85−"arvo", "main", and "try". More may be created by simply referencing

97+and `%kids`. More may be created by simply referencing them. `desk`s have

86−them. Desks have independent histories and states, and they may be

98+independent histories and states, and they may be

87−merged into each other.

99+[merged](@/docs/arvo/clay/using.md#merging) into each other.

100

89−### `++dojo`, domestic desk state

101+### `$dojo`, domestic desk state

102

91```hoon

103```hoon

104++$ dojo

105+ $: qyx=cult :: subscribers

92− ++ dojo ,[p=cult q=dome] :: domestic desk state

106+ dom=dome :: desk state

107+ per=regs :: read perms per path

108+ pew=regs :: write perms per path

109+ ==

93```

110```

111

95−This is the all the data that is specific to a particular desk on a

112+This is the all the data that is specific to a particular `desk` on a domestic

96−domestic ship. `p` is the set of subscribers to this desk and `q` is the

113+ship. `qyx` is the set of subscribers to this `desk`, and `dom` is the data in

97−data in the desk.

114+the `desk`. `regs` are `(map path rule)`, and so `per` is a `map` of read

115+permissions by `path` and `pew` is a `map` of write permissions by `path`.

116

99−#### `++cult`, subscriptions

117+#### `$cult`, subscriptions

100

118

101```hoon

119```hoon

102− ++ cult (map duct rave) :: subscriptions

120++$ cult (jug wove duct)

103```

121```

104

122

105−This is the set of subscriptions to a particular desk. The keys are the

123+`cult`s keep track of subscribers. `wove`s are associated to requests, and each

106−ducts from where the subscriptions requests came. The results will be

124+`wove` is mapped to a set of `duct`s associated to subscribers who should be

107−produced along these ducts. The values are a description of the

125+notified when the request is filled/updated.

108−requested information.

109

126

110−#### `++rave:clay`, general subscription request

127+#### `$rave:clay`, general subscription request

111

128

112```hoon

129```hoon

113− ++ rave :: general request

130+ +$ rave :: general request

114− $% [& p=mood] :: single request

131+ $% [%sing =mood] :: single request

132+ [%next =mood] :: await next version

133+ [%mult =mool] :: next version of any

115− [| p=moat] :: change range

134+ [%many track=? =moat] :: track range

116− == ::

135+ == ::

117```

136```

118

137

119−This represents a subscription request for a desk. The request can be

138+This represents a subscription request for a `desk`.

120−for either a single item in the desk or else for a range of changes on

121−the desk.

122

139

123−### `++rove`, stored general subscription request

140+A `%sing` request asks for data at single revision.

124

141

125−```hoon

142+A `%next` request asks to be notified the next time there’s a change to the

126− ++ rove (each mood moot) :: stored request

143+specified file.

127−```

128

144

129−When we store a request, we store subscriptions with a little extra

145+A `%mult` request asks to be notified the next time there's a change to a

130−information so that we can determine whether new versions actually

146+specified set of files.

131−affect the path we're subscribed to.

132

147

148+A `%many` request asks to be notified on every change in a `desk` for a range of changes (including into the future).

149+

133−#### `++mood:clay`, single subscription request

150+### `$rove`, stored general subscription request

134

151

135```hoon

152```hoon

136− ++ mood ,[p=care q=case r=path] :: request in desk

153+ +$ rove :: stored request

154+ $% [%sing =mood] :: single request

155+ [%next =mood aeon=(unit aeon) =cach] :: next version of one

156+ $: %mult :: next version of any

157+ =mool :: original request

158+ aeon=(unit aeon) :: checking for change

159+ old-cach=(map [=care =path] cach) :: old version

160+ new-cach=(map [=care =path] cach) :: new version

161+ == ::

162+ [%many track=? =moat lobes=(map path lobe)] :: change range

163+ == ::

137```

164```

138

165

139−This represents a request for the state of the desk at a particular

166+Like a `rave` but with provisions to store current versions for `%next` and `%many`.

140−commit, specfied by `q`. `p` specifies what kind of information is

167+Generally used when we store a request in our state somewhere. This is so that

141−desired, and `r` specifies the path we are requesting.

168+we can determine whether new versions actually affect the path we're subscribed to.

169+

142

170

143−#### `++moat:clay`, range subscription request

171+#### `$mood:clay`, single subscription request

144

172

145```hoon

173```hoon

146− ++ moat ,[p=case q=case r=path] :: change range

174+ +$ mood [=care =case =path] :: request in desk

147```

175```

148

176

149−This represents a request for all changes between `p` and `q` on path

177+This represents a request for data related to the state of the `desk` at a

150−`r`. You will be notified when a change is made to the node referenced

178+particular commit, specfied by `case`. `care` specifies what kind of information

151−by the path or to any of its children.

179+is desired, and `path` specifies the path we are requesting.

152

180

153−### `++moot`, stored range subscription request

181+#### `$moat:clay`, range subscription request

154

182

155```hoon

183```hoon

156− ++ moot ,[p=case q=case r=path s=(map path lobe)] ::

184+ +$ moat [from=case to=case =path] :: change range

157```

185```

158

186

159−This is just a `++moat:clay` plus a map of paths to lobes. This map

187+This represents a request for all changes between `from` and `to` on `path`. You

160−represents the data at the node referenced by the path at case `p`, if

188+will be notified when a change is made to the node referenced by the `path` or to

161−we've gotten to that case (else null). We only send a notification along

189+any of its children.

162−the subscription if the data at a new revision is different than it was.

163

190

164−### `++care:clay`, clay submode

191+### `$care:clay`, Clay submode

165

192

166```hoon

193```hoon

167− ++ care ?(%u %v %w %x %y %z) :: clay submode

194+ +$ care ?(%a %b %c %d %e %f %p %r %s %t %u %v %w %x %y %z) :: clay submode

168```

195```

169

196

197+

170This specifies what type of information is requested in a subscription

198This specifies what type of information is requested in a subscription

171or a scry.

199or a scry.

172

200

173−`%u` requests the `++rang:clay` at the current moment. Because this

201+`%a` build a Hoon file at a `path`.

174−information is not stored for any moment other than the present, we

202+

175−crash if the `++case:clay` is not a `%da` for now.

203+`%b` build a dynamically typed `mark` by name (a `$dais` mark-interface core).

204+

205+`%c` build a dynamically typed `mark` conversion gate (a `$tube`) by "from" and

206+"to" `mark` names.

207+

208+`%d` returns a `(set desk)` of the `desk`s that exist on your ship.

209+

210+`%e` builds a statically typed `mark` by name (a `$nave` mark-interface core).

211+

212+`%f` builds a statically typed mark converstion gate.

213+

214+`%p` produces the permissions for a directory, returned as a `[dict:clay dict:clay]`.

215+

216+`%r` requests the file in the same fashion as `%x`, but wraps the result in a `vase`.

217+

218+`%s` has miscellaneous debug endpoints.

176

219

177−`%v` requests the `++dome:clay` at the specified commit.

220+`%t` produces a `(list path)` of descendent `path`s for a directory within a `yaki`.

178

221

179−`%w` requests the revsion number of the desk.

222+`%u` produces a `?` depending on whether or not the specified file exists. It

223+does not check any of its children.

180

224

181−`%x` requests the file at a specified path at the specified commit. If

225+`%v` requests the entire `dome` for a specified `desk` at a particular `aeon`.

182−there is no node at that path or if the node has no contents (that is,

226+When used on a foreign `desk`, this get us up-to-date to the requested version.

183−if `q:ankh` is null), then this produces null.

184

227

185−`%y` requests a `++arch` of the specfied commit at the specified path.

228+`%w` requests the revision number and date of the specified path, returned as a `cass:clay`.

186

229

187−`%z` requests the `++ankh` of the specified commit at the specfied path.

230+`%x` requests the file at a specified path at the specified commit, returned as

231+an `@`. If there is no node at that path or if the node has no contents (that

232+is, if `fil:ankh` is null), then this crashes.

188

233

234+`%y` requests an `arch` of the specfied commit at the specified path. It will

235+return the bunt of an `arch` if the file or directory is not found.

236+

237+`%z` requests a recursive hash of a node and all its children, returned as a `@uxI`.

238+

189−### `++arch`, shallow filesystem node

239+#### `$ankh`, filesystem node

190

240

191```hoon

241```hoon

192− ++ arch ,[p=@uvI q=(unit ,@uvI) r=(map ,@ta ,~)] :: fundamental node

242++$ ankh :: expanded node

243+ $~ [~ ~]

244+ $: fil=(unit [p=lobe q=cage]) :: file

245+ dir=(map @ta ankh) :: folders

246+ == ::

193```

247```

194

248

195−This is analogous to `++ankh:clay` except that the we have neither our

249+This is a recursive filesystem node type that can describe a whole tree of

196−contents nor the ankhs of our children. The other fields are exactly the

250+folders and files, with subtrees organized by path prefix. In Earth filesystems,

197−same, so `p` is a hash of the associated ankh, `u.q`, if it exists, is a

251+a node is a file xor a directory. On Mars, we're inclusive, so a node is a file

198−hash of the contents of this node, and the keys of `r` are the names of

252+ior a directory.

199−our children. `r` is a map to null rather than a set so that the

253+

200−ordering of the map will be equivalent to that of `r:ankh`, allowing

254+`fil` is the contents of this file, if any. `p.fil` is a hash of the

201−efficient conversion.

255+contents while `q.fil` is the data itself.

202

256

203−#### `++case:clay`, specifying a commit

257+`dir` is the set of children of this node. In the case of a pure file,

258+this is empty. The keys are the names of the children and the values

259+are, recursively, the nodes themselves.

260+

261+

262+### `$arch`, shallow filesystem node

204

263

205```hoon

264```hoon

206− ++ case :: ship desk case spur

265++$ arch (axil @uvI)

266+++ axil

267+ |$ [item]

268+ [fil=(unit item) dir=(map @ta ~)]

269+```

270+

271+An `arch` is a lightweight version of an `ankh` that only contains the hash

272+of the associated file and a `map` of child directories, but not the children of

273+those directories.

274+

275+The child directories are given by a `map` to null rather than a `set` so that the

276+ordering of the `map` will be the same as it is for an `axal`, allowing

277+efficient conversion for when the heavier node is needed.

278+```hoon

279+++ axal

280+ |$ [item]

281+ [fil=(unit item) dir=(map @ta $)]

282+```

283+

284+#### `$case`, specifying a commit

285+

286+```hoon

287++$ case

207− $% [%da p=@da] :: date

288+ $% :: %da: date

208− [%tas p=@tas] :: label

289+ :: %tas: label

290+ :: %ud: sequence

291+ ::

292+ [%da p=@da]

293+ [%tas p=@tas]

209− [%ud p=@ud] :: number

294+ [%ud p=@ud]

210− == ::

295+ ==

211```

296```

212

297

213A commit can be referred to in three ways: `%da` refers to the commit

298A commit can be referred to in three ways: `%da` refers to the commit

@@ -216,91 +301,66 @@ that was at the head on date `p`, `%tas` refers to the commit labeled

216all can be reduced down to a `%ud`, only numbered commits may be

301all can be reduced down to a `%ud`, only numbered commits may be

217referenced with a `++case:clay`.

302referenced with a `++case:clay`.

218

303

219−#### `++dome:clay`, desk data

304+#### `$dome`, desk data

220

305

221```hoon

306```hoon

222− ++ dome :: project state

307++$ dome

223− $: ang=agon :: pedigree

224− ank=ankh :: state

308+ $: ank=ankh :: state

225− let=@ud :: top id

309+ let=aeon :: top id

226− hit=(map ,@ud tako) :: changes by id

310+ hit=(map aeon tako) :: versions by id

227− lab=(map ,@tas ,@ud) :: labels

311+ lab=(map @tas aeon) :: labels

312+ mim=(map path mime) :: mime cache

313+ fod=ford-cache :: ford cache

314+ fer=(unit reef-cache) :: reef cache

228− == ::

315+ == ::

229```

316```

230

317

231−This is the data that is actually stored in a desk.

318+A `dome` is the state of a `desk` and associated data.

232−

233−`ang` is unused and should be removed.

234

319

235`ank` is the current state of the desk. Thus, it is the state of the

320`ank` is the current state of the desk. Thus, it is the state of the

236−filesystem at revison `let`. The head of a desk is always a numbered

321+filesystem at revison `let`. The head of a `desk` is always a numbered

237commit.

322commit.

238

323

239`let` is the number of the most recently numbered commit. This is also

324`let` is the number of the most recently numbered commit. This is also

240the total number of numbered commits.

325the total number of numbered commits.

241

326

242−`hit` is a map of numerical ids to hashes of commits. These hashes are

327+`hit` is a map of numerical ids to commit hashes. These hashes are mapped into

243−mapped into their associated commits in `hut:rang:clay`. In general, the keys

328+their associated commits in `hut.rang` in the `raft` of Clay. In general, the

244−of this map are exactly the numbers from 1 to `let`, with no gaps. Of

329+keys of this map are exactly the numbers from 1 to `let`, with no gaps. Of

245−course, when there are no numbered commits, `let` is 0, so `hit` is

330+course, when there are no numbered commits, `let` is 0, so `hit` is null.

246−null. Additionally, each of the commits is an ancestor of every commit

331+Additionally, each of the commits is an ancestor of every commit numbered

247−numbered greater than this one. Thus, each is a descendant of every

332+greater than this one. Thus, each is a descendant of every commit numbered less

248−commit numbered less than this one. Since it is true that the date in

333+than this one. Since it is true that the date in each commit (`t:yaki`) is no

249−each commit (`t:yaki`) is no earlier than that of each of its parents,

334+earlier than that of each of its parents, the numbered commits are totally

250−the numbered commits are totally ordered in the same way by both

251−pedigree and date. Of course, not every commit is numbered. If that

335+ordered in the same way by both pedigree and date. Of course, not every commit

252−sounds too complicated to you, don't worry about it. It basically

336+is numbered. If that sounds too complicated to you, don't worry about it. It

253−behaves exactly as you would expect.

337+basically behaves exactly as you would expect.

254

338

255`lab` is a map of textual labels to numbered commits. Note that labels

339`lab` is a map of textual labels to numbered commits. Note that labels

256can only be applied to numbered commits. Labels must be unique across a

340can only be applied to numbered commits. Labels must be unique across a

257desk.

341desk.

258

342

259−#### `++ankh`, filesystem node

343+`mim` is a cache of the content in the directories that are mounted to Unix.

260

344

261−```hoon

345+`fod` is the Ford cache, which keeps a cache of the results of builds performed

262− ++ ankh :: fs node (new)

346+at this `desk`'s current revision, including a full transitive closure of

263− $: p=cash :: recursive hash

347+dependencies for each completed build.

264− q=(unit ,[p=cash q=*]) :: file

265− r=(map ,@ta ankh) :: folders

266− == ::

267−```

268−

269−This is a single node in the filesystem. This may be file or a directory

270−or both. In earth filesystems, a node is a file xor a directory. On

271−mars, we're inclusive, so a node is a file ior a directory.

272−

273−`p` is a recursive hash that depends on the contents of the this file or

274−directory and on any children.

275−

276−`q` is the contents of this file, if any. `p.q` is a hash of the

277−contents while `q.q` is the data itself.

278−

279−`r` is the set of children of this node. In the case of a pure file,

280−this is empty. The keys are the names of the children and the values

281−are, recursively, the nodes themselves.

282

348

283−#### `++cash`, ankh hash

349+`fer` is the system file cache, which consists of `vase`s for `hoon.hoon`,

284−

350+`arvo.hoon`, `lull.hoon`, and `zuse.hoon`.

285−```hoon

286− ++ cash ,@uvH :: ankh hash

287−```

288−

289−This is a 128-bit hash of an ankh. These are mostly stored within ankhs

290−themselves, and they are used to check for changes in possibly-deep

291−hierarchies.

292

351

293#### `++rung`, filesystem per neighbor ship

352#### `++rung`, filesystem per neighbor ship

294

353

295```hoon

354```hoon

355++$ rung

296− ++ rung $: rus=(map desk rede) :: neighbor desks

356+ $: rus=(map desk rede) :: neighbor desks

297− == ::

357+ ==

298```

358```

299

359

300−This is the filesystem of a neighbor ship. The keys to this map are all

360+This is the filesystem of a neighbor ship. The keys to this `map` are all

301−the desks we know about on their ship.

361+the `desk`s we know about on their ship.

302

362

303−#### `++rede`, desk state

363+#### `++rede`, generic desk state

304

364

305```hoon

365```hoon

306 ++ rede :: universal project

366 ++ rede :: universal project

@@ -309,342 +369,312 @@ the desks we know about on their ship.

309 ref=(unit rind) :: outgoing requests

369 ref=(unit rind) :: outgoing requests

310 dom=dome :: revision state

370 dom=dome :: revision state

311 == ::

371 == ::

372++$ rede :: universal project

373+ $: lim=@da :: complete to

374+ ref=(unit rind) :: outgoing requests

375+ qyx=cult :: subscribers

376+ dom=dome :: revision state

377+ per=regs :: read perms per path

378+ pew=regs :: write perms per path

379+ == ::

312```

380```

313

381

314This is our knowledge of the state of a desk, either foreign or

382This is our knowledge of the state of a desk, either foreign or

315domestic.

383domestic.

316

384

317−`lim` is the date of the last full update. We only respond to requests

385+`lim` is the most recent `@da` for which we're confident we have all the

318−for stuff before this time.

386+information for. For local `desk`s, this is always `now`. For foriegn `desk`s,

387+this is the last time we got a full update from the foreign ship.

319

388

320−`qyx` is the list of subscribers to this desk. For domestic desks, this

389+`ref` is the request manager for the desk. For domestic `desk`s, this is

321−is simply `p:dojo`, all subscribers to the desk, while in foreign desks

390+null since we handle requests ourselves. For foreign `desk`s, this keeps track

322−this is all the subscribers from our ship to the foreign desk.

391+of all pending foriegn requests plus a cache of the responses to previous requests.

323

392

324−`ref` is the request manager for the desk. For domestic desks, this is

393+`qyx` is the `set` of subscriptions to this desk, with listening `duct`s. These

325−null since we handle requests ourselves.

394+subscriptions exist only until they've been filled. For domestic `desk`s, this

395+is simply `qyx:dojo` - all subscribers to the `desk`. For foreign `desk`s this

396+is all the subscribers from our ship to the foreign `desk`.

326

397

327−`dom` is the actual data in the desk.

398+`dom` is the data in the `desk`.

328

399

400+`regs` are `(map path rule)`, and so `per` is a `map` of read permissions by

401+`path` and `pew` is a `map` of write permissions by `path`.

402+

403+

329−#### `++rind`, request manager

404+#### `$rind`, foreign request manager

330

405

331```hoon

406```hoon

332− ++ rind :: request manager

407++$ rind :: request manager

333− $: nix=@ud :: request index

408+ $: nix=@ud :: request index

334− bom=(map ,@ud ,[p=duct q=rave]) :: outstanding

409+ bom=(map @ud update-state) :: outstanding

335− fod=(map duct ,@ud) :: current requests

410+ fod=(map duct @ud) :: current requests

336− haw=(map mood (unit)) :: simple cache

411+ haw=(map mood (unit cage)) :: simple cache

337− == ::

412+ == ::

338```

413```

339

414

340−This is the request manager for a foreign desk.

415+This is the request manager for a foreign `desk`. When we send a request to a

416+foreign ship, we keep track of it in here.

341

417

342−`nix` is one more than the index of the most recent request. Thus, it is

418+`nix` is one more than the index of the most recent request. Thus, it is the

343−the next available request number.

419+next available request number.

344

420

345−`bom` is the set of outstanding requests. The keys of this map are some

421+`bom` is the set of outstanding requests. The keys of this `map` are some subset

346−subset of the numbers between 0 and one less than `nix`. The members of

422+of the numbers between 0 and one less than `nix`. The members of the `map` are

347−the map are exactly those requests that have not yet been fully

423+exactly those requests that have not yet been fully satisfied.

348−satisfied.

349

424

350−`fod` is the same set as `bom`, but from a different perspective. In

425+`fod` is the same set as `bom`, but from a different perspective. In particular,

351−particular, the values of `fod` are the same as the values of `bom`, and

426+the values of `fod` are the same as the keys of `bom`, and the `duct` of the

352−the `p` out of the values of `bom` are the same as the keys of `fod`.

427+values of `bom` are the same as the keys of `fod`. Thus, we can map `duct`s to

353−Thus, we can map ducts to their associated request number and `++rave:clay`,

428+their associated request number and `update-state`, and we can map request

354−and we can map numbers to their associated duct and `++rave:clay`.

429+numbers to their associated `duct`.

355

430

356−`haw` is a map from simple requests to their values. This acts as a

431+`haw` is a map from `%sing` requests to their values. This acts as a cache for

357−cache for requests that have already been made. Thus, the second request

432+requests that have already been filled.

358−for a particular `++mood:clay` is nearly instantaneous.

359

433

360−#### `++rang:clay`, data store

434+#### `$update-state`, status of outstanding foreign request

361

435

362```hoon

436```hoon

437++$ update-state

438+ $: =duct

439+ =rave

440+ have=(map lobe blob)

441+ need=(list lobe)

442+ nako=(qeu (unit nako))

443+ busy=_|

444+ ==

445+```

446+

447+An `update-state` is used to represent the status of an outstanding request to a

448+foreign `desk`.

449+

450+`duct` and `rave` are the `duct` along which the request was made and the

451+request itself.

452+

453+`have` is a map of hashes thus far acquired in the request to the data

454+associated with those hashes.

455+

456+`need` is a list of hashes yet to be acquired.

457+

458+`nako` is a queue of data yet to be validated.

459+

460+`busy` tracks whether or not the request is currently being fulfilled.

461+

462+#### `$rang:clay`, data repository

463+

464+```hoon

465+ +$ rang :: repository

363− ++ rang $: hut=(map tako yaki) ::

466+ $: hut=(map tako yaki) :: changes

364− lat=(map lobe blob) ::

467+ lat=(map lobe blob) :: data

365− == ::

468+ == ::

366```

469```

367

470

368−This is a set of data keyed by hash. Thus, this is where the "real" data

471+This is a data repository keyed by hash. Thus, this is where the "real" data

369is stored, but it is only meaningful if we know the hash of what we're

472is stored, but it is only meaningful if we know the hash of what we're

370looking for.

473looking for.

371

474

372−`hut` is a map from hashes to commits. We often get the hashes from

475+`hut` is a `map` from commit hashes (`tako`s) to commits (`yaki`s). We often get

373−`hit:dome:clay`, which keys them by logical id. Not every commit has an id.

476+the hashes from `hit:dome`, which keys them by numerical id. Not every commit

477+has an numerical id.

374

478

375−`lat` is a map from hashes to the actual data. We often get the hashes

479+`lat` is a `map` from content hashes (`lobe`s) to the actual content (`blob`s).

376−from a `++yaki`, a commit, which references this map to get the data.

480+We often get the hashes from a `yaki`, which references this `map` to get the

377−There is no `++blob:clay` in any `++yaki:clay`. They are only accessible through

481+data. There is no `blob` in `yaki:clay`. They are only accessible through

378−this map.

482+`lat`.

379

483

380−#### `++tako:clay`, commit reference

484+#### `$tako:clay`, commit reference

381

485

382```hoon

486```hoon

383− ++ tako ,@ :: yaki ref

487+ +$ tako @ :: yaki ref

384```

488```

385

489

386−This is a hash of a `++yaki:clay`, a commit. These are most notably used as

490+This is a hash of a `yaki:clay`, a commit. These are most notably used as the

387−the keys in `hut:rang:clay`, where they are associated with the actual

491+keys in `hut:rang:clay`, where they are associated with the actual `yaki:clay`,

388−`++yaki:clay`, and as the values in `hit:dome:clay`, where sequential ids are

492+and as the values in `hit:dome:clay`, where sequential numerical ids are

389associated with these.

493associated with these.

390

494

391−#### `++yaki:clay`, commit

495+#### `$yaki:clay`, commit

392

496

393```hoon

497```hoon

498+ +$ yaki :: commit

499+ $: p=(list tako) :: parents

394− ++ yaki ,[p=(list tako) q=(map path lobe) r=tako t=@da] :: commit

500+ q=(map path lobe) :: namespace

501+ r=tako :: self-reference

502+ t=@da :: date

503+ == ::

395```

504```

396

505

397This is a single commit.

506This is a single commit.

398

507

399−`p` is a list of the hashes of the parents of this commit. In most

508+`p` is a `list` of the hashes of the parents of this commit. In most

400cases, this will be a single commit, but in a merge there may be more

509cases, this will be a single commit, but in a merge there may be more

401parents. In theory, there may be an arbitrary number of parents, but in

510parents. In theory, there may be an arbitrary number of parents, but in

402practice merges have exactly two parents. This may change in the future.

511practice merges have exactly two parents. This may change in the future.

403For commit 1, there is no parent.

512For commit 1, there is no parent.

404

513

405−`q` is a map of the paths on a desk to the data at that location. If you

514+`q` is a `map` of the `path`s on a desk to the content hashes at that location.

406−understand what a `++lobe:clay` and a `++blob:clay` is, then the type signature

515+If you understand what a `lobe:clay` and a `blob:clay` is, then the type

407−here tells the whole story.

516+signature here tells the whole story.

408

517

409`r` is the hash associated with this commit.

518`r` is the hash associated with this commit.

410

519

411`t` is the date at which this commit was made.

520`t` is the date at which this commit was made.

412

521

413−#### `++lobe:clay`, data reference

522+#### `$lobe:clay`, data reference

414

523

415```hoon

524```hoon

416− ++ lobe ,@ :: blob ref

525+ +$ lobe @uvI :: blob ref

417```

526```

418

527

419−This is a hash of a `++blob:clay`. These are most notably used in `lat:rang:clay`,

528+This is a hash of a `blob:clay`. These are most notably used in `lat:rang:clay`,

420−where they are associated with the actual `++blob:clay`, and as the values in

529+where they are associated with the actual `blob:clay`, and as the values in

421−`q:yaki:clay`, where paths are associated with their data in a commit.

530+`q:yaki:clay`, where `path`s are associated with their content hashes in a commit.

422−

423−#### `++blob:clay`, data

424−

425−```hoon

426− ++ blob $% [%delta p=lobe q=lobe r=udon] :: delta on q

427− [%direct p=lobe q=* r=umph] ::

428− [%indirect p=lobe q=* r=udon s=lobe] ::

429− == ::

430−```

431−

432−This is a node of data. In every case, `p` is the hash of the blob.

433−

434−`%delta` is the case where we define the data by a delta on other data.

435−In practice, the other data is always the previous commit, but nothing

436−depends on this. `q` is the hash of the parent blob, and `r` is the

437−delta.

438

531

439−`%direct` is the case where we simply have the data directly. `q` is the

532+#### `$blob:clay`, data

440−data itself, and `r` is any preprocessing instructions. These almost

441−always come from the creation of a file.

442−

443−`%indirect` is both of the preceding cases at once. `q` is the direct

444−data, `r` is the delta, and `s` is the parent blob. It should always be

445−the case that applying `r` to `s` gives the same data as `q` directly

446−(with the prepreprocessor instructions in `p.r`). This exists purely for

447−performance reasons. This is unused, at the moment, but in general these

448−should be created when there are a long line of changes so that we do

449−not have to traverse the delta chain back to the creation of the file.

450−

451−#### `++udon`, abstract delta

452

533

453```hoon

534```hoon

454− ++ udon :: abstract delta

535+ +$ blob :: fs blob

455− $: p=umph :: preprocessor

536+ $% [%delta p=lobe q=[p=mark q=lobe] r=page] :: delta on q

456− $= q :: patch

537+ [%direct p=lobe q=page] :: immediate

457− $% [%a p=* q=*] :: trivial replace

458− [%b p=udal] :: atomic indel

459− [%c p=(urge)] :: list indel

460− [%d p=upas q=upas] :: tree edit

461− == ::

462− == ::

463```

538```

464

539

465−This is an abstract change to a file. This is a superset of what would

540+This is a node of data. In both cases, `p` is the hash of the blob.

466−normally be called diffs. Diffs usually refer to changes in lines of

467−text while we have the ability to do more interesting deltas on

468−arbitrary data structures.

469−

470−`p` is any preprocessor instructions.

471−

472−`%a` refers to the trival delta of a complete replace of old data with

473−new data.

474

541

475−`%b` refers to changes in an opaque atom on the block level. This has

542+`%delta` is the case where we define the data by a delta on other data. In

476−very limited usefulness, and is not used at the moment.

543+practice, the other data is always the previous commit, but nothing depends on

544+this. `p.q` is the `mark` of the parent blob, `q.q` is the hash of the parent

545+blob, and `r` is the delta.

477

546

478−`%c` refers to changes in a list of data. This is often lines of text,

547+`%direct` is the case where we simply have the data directly. `q` is the data.

479−which is your classic diff. We, however, will work on any list of data.

548+These almost always come from the creation of a file.

480

549

481−`%d` refers to changes in a tree of data. This is general enough to

482−describe changes to any hoon [noun](/docs/glossary/noun/), but often more special-purpose delta

483−should be created for different content types. This is not used at the

484−moment, and may in fact be unimplemented.

485

550

486−#### `++urge`, list change

551+#### `+urge`, list change

487

552

488```hoon

553```hoon

489− ++ urge |*(a=_,* (list (unce a))) :: list change

554+ ++ urge |*(a=mold (list (unce a))) :: list change

490```

555```

491

556

492−This is a parametrized type for list changes. For example, `(urge ,@t)`

557+This is a parametrized type for list changes. For example, `(urge @t)`

493is a list change for lines of text.

558is a list change for lines of text.

494

559

495−#### `++unce`, change part of a list.

560+#### `+unce`, change part of a list.

496

561

497```hoon

562```hoon

498− ++ unce |* a=_,* :: change part

563+ ++ unce :: change part

564+ |* a=mold ::

499− $% [%& p=@ud] :: skip[copy]

565+ $% [%& p=@ud] :: skip[copy]

500− [%| p=(list a) q=(list a)] :: p -> q[chunk]

566+ [%| p=(list a) q=(list a)] :: p -> q[chunk]

501− == ::

567+ == ::

502```

568```

503

569

504This is a single change in a list of elements of type `a`. For example,

570This is a single change in a list of elements of type `a`. For example,

505−`(unce ,@t)` is a single change in a lines of text.

571+`(unce @t)` is a single change in lines of text.

506

572

507`%&` means the next `p` lines are unchanged.

573`%&` means the next `p` lines are unchanged.

508

574

509`%|` means the lines `p` have changed to `q`.

575`%|` means the lines `p` have changed to `q`.

510

576

511−#### `++umph`, preprocessing information

512−

513−```hoon

514− ++ umph :: change filter

515− $| $? %a :: no filter

516− %b :: jamfile

517− %c :: LF text

518− == ::

519− $% [%d p=@ud] :: blocklist

520− == ::

521−```

522−

523−This space intentionally left undocumented. This stuff will change once

524−we get a well-typed clay.

525−

526−#### `++upas`, tree change

527−

528−```hoon

529− ++ upas :: tree change (%d)

530− $& [p=upas q=upas] :: cell

531− $% [%0 p=axis] :: copy old

532− [%1 p=*] :: insert new

533− [%2 p=axis q=udon] :: mutate!

534− == ::

535−```

536−

537−This space intentionally left undocumented. This stuff is not known to

538−work, and will likely change when we get a well-typed clay. Also, this

539−is not a complicated type; it is not difficult to work out the meaning.

540−

541−#### `++nori:clay`, repository action

577+#### `$nori:clay`, repository action

542

578

543```hoon

579```hoon

544− ++ nori :: repository action

580+ +$ nori :: repository action

545− $% [& q=soba] :: delta

581+ $% [%& p=soba] :: delta

546− [| p=@tas] :: label

582+ [%| p=@tas] :: label

547− == ::

583+ == ::

548```

584```

549

585

550−This describes a change that we are asking clay to make to the desk.

586+This describes a change that we are asking Clay to make to the `desk`.

551There are two kinds of changes that may be made: we can modify files or

587There are two kinds of changes that may be made: we can modify files or

552we can apply a label to a commit.

588we can apply a label to a commit.

553

589

554In the `|` case, we will simply label the current commit with the given

590In the `|` case, we will simply label the current commit with the given

555label. In the `&` case, we will apply the given changes.

591label. In the `&` case, we will apply the given changes.

556

592

557−#### `++soba:clay`, delta

593+#### `$soba:clay`, delta

558

594

559```hoon

595```hoon

560− ++ soba ,[p=cart q=(list ,[p=path q=miso])] :: delta

596+ +$ soba (list [p=path q=miso]) :: delta

561```

597```

562

598

563−This describes a set of changes to make to a desk. The `cart` is simply

599+This describes a `list` of changes to make to a `desk`. The `path`s are `path`s

564−a pair of the old hash and the new hash of the desk. The list is a list

565−of changes keyed by the file they're changing. Thus, the paths are paths

566−to files to be changed while `miso` is a description of the change

600+to files to be changed, and the corresponding `miso` value is a description of

567−itself.

601+the change itself.

568

602

569−#### `++miso:clay`, ankh delta

603+#### `$miso:clay`, ankh delta

570

604

571```hoon

605```hoon

572− ++ miso :: ankh delta

606+ +$ miso :: ankh delta

573− $% [%del p=*] :: delete

607+ $% [%del ~] :: delete

574− [%ins p=*] :: insert

608+ [%ins p=cage] :: insert

609+ [%dif p=cage] :: mutate from diff

575− [%mut p=udon] :: mutate

610+ [%mut p=cage] :: mutate from raw

576− == ::

611+ == ::

577```

612```

578

613

579−There are three kinds of changes that may be made to a node in a desk.

614+There are four kinds of changes that may be made to a node in a `desk`.

580−We can insert a file, in which case `p` is the contents of the new file.

581−We can delete a file, in which case `p` is the contents of the old file.

582−Finally, we can mutate that file, in which case the `udon` describes the

583−changes we are applying to the file.

584

615

585−#### `++mizu:clay`, merged state

616+`%del` deletes the node.

586

617

587−```hoon

618+`%ins` inserts a file given by `p`.

588− ++ mizu ,[p=@u q=(map ,@ud tako) r=rang] :: new state

619+

589−```

620+`%dif` is currently unimplemented. This may seem strange, so we remark that

621+diffs for individual files are implemented using `+diff` and `+pact` in `mark`s.

622+So for an `ankh`, which may include both files and directories, `%dif` being

623+unimplemented really just means that we do not yet have a formal concept of

624+changes in directory structure.

590

625

591−This is the input to the `%merg` kiss, which allows us to perform a

626+`%mut` mutates the file using raw data given by `p`.

592−merge. The `p` is the number of the new head commit. The `q` is a map

593−from numbers to commit hashes. This is all the new numbered commits that

594−are to be inserted. The keys to this should always be the numbers from

595−`let.dom` plus one to `p`, inclusive. The `r` is the maps of all the new

596−commits and data. Since these are merged into the current state, no old

597−commits or data need be here.

598

627

599−#### `++riff:clay`, request/desist

628+#### `$riff:clay`, request/desist

600

629

601```hoon

630```hoon

602− ++ riff ,[p=desk q=(unit rave)] :: request/desist

631+ +$ riff [p=desk q=(unit rave)] :: request+desist

603```

632```

604

633

605−This represents a request for data about a particular desk. If `q`

634+This represents a request for data about a particular `desk`. If `q`

606−contains a `rave`, then this opens a subscription to the desk for that

635+contains a `rave`, then this opens a subscription to the `desk` for that

607−data. If `q` is null, then this tells clay to cancel the subscription

636+data. If `q` is null, then this tells Clay to cancel the subscription

608along this duct.

637along this duct.

609

638

610−#### `++riot:clay`, response

639+#### `$riot:clay`, response

611

640

612```hoon

641```hoon

613− ++ riot (unit rant) :: response/complete

642+ +$ riot (unit rant) :: response+complete

614```

643```

615

644

616−A riot is a response to a subscription. If null, the subscription has

645+A `riot` is a response to a subscription. If null, the subscription has

617been completed, and no more responses will be sent. Otherwise, the

646been completed, and no more responses will be sent. Otherwise, the

618`rant` is the produced data.

647`rant` is the produced data.

619

648

620−#### `++rant:clay`, response data

649+#### `$rant:clay`, response data

621

650

622```hoon

651```hoon

623− ++ rant :: namespace binding

652+ +$ rant :: response to request

624− $: p=[p=care q=case r=@tas] :: clade release book

653+ $: p=[p=care q=case r=desk] :: clade release book

625− q=path :: spur

654+ q=path :: spur

626− r=* :: data

655+ r=cage :: data

627− == ::

656+ == ::

628```

657```

629

658

630−This is the data at a particular node in the filesystem. `p.p` specifies

659+This is the data associated to the response to a request. `p.p` specifies

631the type of data that was requested (and is produced). `q.p` gives the

660the type of data that was requested (and is produced). `q.p` gives the

632specific version reported (since a range of versions may be requested in

661specific version reported (since a range of versions may be requested in

633−a subscription). `r.p` is the desk. `q` is the path to the filesystem

662+a subscription). `r.p` is the `desk`. `q` is the path to the filesystem

634node. `r` is the data itself (in the format specified by `p.p`).

663node. `r` is the data itself (in the format specified by `p.p`).

635

664

636−### `++nako`, subscription response data

665+### `$nako`, subscription response data

637

666

638```hoon

667```hoon

668++$ nako :: subscription state

639− ++ nako $: gar=(map ,@ud tako) :: new ids

669+ $: gar=(map aeon tako) :: new ids

640− let=@ud :: next id

670+ let=aeon :: next id

641− lar=(set yaki) :: new commits

671+ lar=(set yaki) :: new commits

642− bar=(set blob) :: new content

672+ bar=(set plop) :: new content

643− == ::

673+ == ::

644```

674```

645

675

646This is the data that is produced by a request for a range of revisions

676This is the data that is produced by a request for a range of revisions

647−of a desk. This allows us to easily keep track of a remote repository --

677+of a `desk`. This allows us to easily keep track of a remote repository --

648all the new information we need is contained in the `nako`.

678all the new information we need is contained in the `nako`.

649

679

650`gar` is a map of the revisions in the range to the hash of the commit

680`gar` is a map of the revisions in the range to the hash of the commit

@@ -656,105 +686,3 @@ revision number, whichever is smaller.

656

686

657`lar` is the set of new commits, and `bar` is the set of new content.

687`lar` is the set of new commits, and `bar` is the set of new content.

658

688

659−## Public Interface

660−

661−As with all vanes, there are exactly two ways to interact with clay.

662−`%clay` exports a namespace accessible through `.^`, which is described

663−above under `++care:clay`. The primary way of interacting with clay, though,

664−is by sending kisses and receiving gifts.

665−

666−```hoon

667− ++ gift :: out result <-$

668− $% [%ergo p=@p q=@tas r=@ud] :: version update

669− [%note p=@tD q=tank] :: debug message

670− [%writ p=riot] :: response

671− == ::

672− ++ kiss :: in request ->$

673− $% [%info p=@p q=@tas r=nori] :: internal edit

674− [%ingo p=@p q=@tas r=nori] :: internal noun edit

675− [%init p=@p] :: report install

676− [%into p=@p q=@tas r=nori] :: external edit

677− [%invo p=@p q=@tas r=nori] :: external noun edit

678− [%merg p=@p q=@tas r=mizu] :: internal change

679− [%wart p=sock q=@tas r=path s=*] :: network request

680− [%warp p=sock q=riff] :: file request

681− == ::

682−```

683−

684−There are only a small number of possible kisses, so it behooves us to

685−describe each in detail.

686−

687−```hoon

688− $% [%info p=@p q=@tas r=nori] :: internal edit

689−

690− [%into p=@p q=@tas r=nori] :: external edit

691−```

692−

693−These two kisses are nearly identical. At a high level, they apply

694−changes to the filesystem. Whenever we add, remove, or edit a file, one

695−of these cards is sent. The `p` is the ship whose filesystem we're

696−trying to change, the `q` is the desk we're changing, and the `r` is the

697−request change. For the format of the requested change, see the

698−documentation for `++nori:clay` above.

699−

700−When a file is changed in the unix filesystem, vere will send a `%into`

701−kiss. This tells clay that the duct over which the kiss was sent is the

702−duct that unix is listening on for changes. From within Arvo, though, we

703−should never send a `%into` kiss. The `%info` kiss is exactly identical

704−except it does not reset the duct.

705−

706−```hoon

707− [%ingo p=@p q=@tas r=nori] :: internal noun edit

708−

709− [%invo p=@p q=@tas r=nori] :: external noun edit

710−```

711−

712−These kisses are currently identical to `%info` and `%into`, though this

713−will not always be the case. The intent is for these kisses to allow

714−typed changes to clay so that we may store typed data. This is currently

715−unimplemented.

716−

717−```hoon

718− [%init p=@p] :: report install

719−```

720−

721−Init is called when a ship is started on our pier. This simply creates a

722−default `room` to go into our `raft`. Essentially, this initializes the

723−filesystem for a ship.

724−

725−```hoon

726− [%merg p=@p q=@tas r=mizu] :: internal change

727−```

728−

729−This is called to perform a merge. This is most visibly called by

730−:update to update the filesystem of the current ship to that of its

731−sein. The `p` and `q` are as in `%info`, and the `r` is the description

732−of the merge. See `++mizu:clay` above.

733−

734−XX

735−`XX [%wake ~] :: timer activate XX`

736−XX\

737−XX This card is sent by unix at the time specified by `++doze`. This

738−time is XX usually the closest time specified in a subscription request.

739−When `%wake` is XX called, we update our subscribers if there have been

740−any changes.

741−

742−```hoon

743− [%wart p=sock q=@tas r=path s=*] :: network request

744−```

745−

746−This is a request that has come across the network for a particular

747−file. When another ship asks for a file from us, that request comes to

748−us in the form of a `%wart` kiss. This is handled by trivially turning

749−it into a `%warp`.

750−

751−```hoon

752− [%warp p=sock q=riff] :: file request

753−```

754−

755−This is a request for information about a particular desk. This is, in

756−its most general form, a subscription, though in many cases it is the

757−trivial case of a subscription -- a read. See `++riff:clay` for the format of

758−the request.

759−

760−

arvo/clay/using.md+78−62

@@ -4,8 +4,7 @@ weight = 3

4template = "doc.html"

5+++

7−This document covers common uses of Clay for developers. Some parts of it may be

8−out of date as of March 2021.

7+This document is up-to-date as of March 2021.

10### Reading and Subscribing

9### Reading and Subscribing

@@ -13,7 +12,7 @@ When reading from Clay, there are three types of requests. A

13`%sing` request asks for data at single revsion. A `%next`

12`%sing` request asks for data at single revsion. A `%next`

14request asks to be notified the next time there's a change to

13request asks to be notified the next time there's a change to

15given file. A `%many` request asks to be notified on every

14given file. A `%many` request asks to be notified on every

16−change in a desk for a range of changes.

15+change in a `desk` for a range of changes.

18For `%sing` and `%next`, there are generally three things to be

17For `%sing` and `%next`, there are generally three things to be

19queried. A `%u` request simply checks for the existence of a

18queried. A `%u` request simply checks for the existence of a

@@ -33,19 +32,19 @@ things in Urbit, aren't guaranteed to return immediately.

33They'll return when they can, and they'll do so in a

32They'll return when they can, and they'll do so in a

34referentially transparent manner.

33referentially transparent manner.

36−A `%next` request checks query at the given revision, and it

35+A `%next` request checks the query at the given revision, and it

37produces the result of the query the next time it changes, along

36produces the result of the query the next time it changes, along

38−with the revsion number when it changes. Thus, a `%next` of a

37+with the revision number when it changes. Thus, a `%next` of a

39`%u` is triggered when a file is added or deleted, a `%next` of a

38`%u` is triggered when a file is added or deleted, a `%next` of a

40`%x` is triggered when a file is added, deleted, or changed, and

39`%x` is triggered when a file is added, deleted, or changed, and

41a `%next` of a `%y` is triggered when a file or any of its

40a `%next` of a `%y` is triggered when a file or any of its

42children is added, deleted, or changed.

41children is added, deleted, or changed.

44−A `%many` request is triggered every time the given desk has a

43+A `%many` request is triggered every time the given `desk` has a

45new revision. Unlike a `%next`, a `%many` has both a start and

44new revision. Unlike a `%next`, a `%many` has both a start and

46−an end revsion, after which it stops returning. For `%next`, a

45+an end revision, after which it stops returning. For `%next`, a

47single change is reported, and if the caller wishes to hear of

46single change is reported, and if the caller wishes to hear of

48−the next change, it must resubscribe. For `%many`, every revsion

47+the next change, it must resubscribe. For `%many`, every revision

49from the start to the end triggers a response. Since a `%many`

48from the start to the end triggers a response. Since a `%many`

50request doesn't ask for any particular data, there aren't `%u`,

49request doesn't ask for any particular data, there aren't `%u`,

51`%x`, and `%y` versions for it.

50`%x`, and `%y` versions for it.

@@ -56,33 +55,43 @@ Requests](@/docs/arvo/clay/foreign.md)

57### Unix sync

56### Unix sync

58+Usage:

59+```

60+|mount %/path/to/directory %mount-point

61+|commit %mount-point

62+|autocommit %mount-point

63+```

64+

59−One of the primary functions of clay is as a convenient user

65+One of the primary functions of Clay is as a convenient user

60−interface. While tools exist to use clay from within urbit, it's

66+interface. While tools exist to use Clay from within Urbit, it's

61−often useful to be able to treat clay like any other filesystem

67+often useful to be able to treat Clay like any other filesystem

62from the Unix perspective -- to "mount" it, as it were.

68from the Unix perspective -- to "mount" it, as it were.

64−From urbit, you can run `|mount /path/to/directory %mount-point`,

70+From the dojo, you can run `|mount %/path/to/directory %mount-point`, and this

65−and this will mount the given clay directory to the mount-point

71+will mount the given Clay directory to the mount-point directory in Unix, with

66−directory in Unix. Every file is converted to `%mime` before it's

72+your pier as the root directory. For example `|mount %/gen %generators` will

67−written to Unix, and converted back when read from Unix. The

73+create a `/generators` directory in your pier, and mount the `/gen` folder at

68−entire directory is watched (a la Dropbox), and every change is

74+that point. If you'd like to use the directory name as the mount point, you can

69−committed once you run `|commit %mount-point`.

75+elide the last argument, e.g. `|mount %/gen` mounts `/gen` to `/gen`.

71−### Merging

77+Every file is converted to `%mime` before it's written to Unix, and converted

78+back when read from Unix. The entire directory is watched (a la Dropbox), and

79+every change is committed once you run `|commit %mount-point`. Every change can

80+be automatically committed with `|autocommit %mount-point`.

73−This section is up to date as of March 2021.

82+### Merging

75Usage:

84Usage:

76```

85```

77−|merge %destination-desk ~source-ship %source-desk

86+|merge %target-desk ~source-ship %source-desk

78−|merge %destination-desk ~source-ship %source-desk, =gem %strategy

87+|merge %target-desk ~source-ship %source-desk, =gem %strategy

79−|merge %destination-desk ~source-ship %source-desk, =cas ud+5

88+|merge %target-desk ~source-ship %source-desk, =cas ud+5

80```

89```

82Clay supports various merge strategies. A "commit" is a snapshot of

91Clay supports various merge strategies. A "commit" is a snapshot of

83the files with a list of parents plus a date. Most commits have

92the files with a list of parents plus a date. Most commits have

84one parent; a "merge" commit is a commit with two parents. The

93one parent; a "merge" commit is a commit with two parents. The

85−%home desk starts with an initial commit with no parents; commits

94+%home `desk` starts with an initial commit with no parents; commits

86with several parents ("octopus merges") are possible but we don't

95with several parents ("octopus merges") are possible but we don't

87generate them right now.

96generate them right now.

@@ -90,67 +99,69 @@ Unless otherwise specified, all of the following create a new commit

90with the source and destination commits as parents.

99with the source and destination commits as parents.

100

92Several strategies need a "merge-base". They find it by identifying

101Several strategies need a "merge-base". They find it by identifying

93−the most recent common ancestor of the two desks. If none, fail

102+the most recent common ancestor of the two `desk`s. If none, fail

94with `%merge-no-merge-base`; if there are two or more, pick one.

103with `%merge-no-merge-base`; if there are two or more, pick one.

104+

105+#### Strategies

106

96−`%init`: the only way to create a desk. Not a true merge, since it

107+**`%init`**: the only way to create a `desk`. Not a true merge, since it

97simply assigns the source commit to the destination.

108simply assigns the source commit to the destination.

109

99−`%fine`: if source or destination are in the ancestry of each other,

110+**`%fine`**: if source or destination are in the ancestry of each other,

100use the newer one; else abort. If the destination is ahead of the

111use the newer one; else abort. If the destination is ahead of the

101source, succeed but do nothing. If the source is ahead of the

112source, succeed but do nothing. If the source is ahead of the

102destination, assign the next revision number to the source commit.

113destination, assign the next revision number to the source commit.

103Some call this "fast-forward".

114Some call this "fast-forward".

104

115

105−`%meet`: combine changes, failing if both sides changed the same file.

116+**`%meet`**: combine changes, failing if both sides changed the same file.

106Specifically, take diff(merge-base,source) and

117Specifically, take diff(merge-base,source) and

107diff(merge-base,destination) and combine them as long as those diffs

118diff(merge-base,destination) and combine them as long as those diffs

108touch different files.

119touch different files.

109

120

110−`%mate`: combine changes, failing if both sides changed the same part

121+**`%mate`**: combine changes, failing if both sides changed the same part

111of a file. Identical to `%meet`, except that some marks, like `%hoon`,

122of a file. Identical to `%meet`, except that some marks, like `%hoon`,

112allow intelligent merge of changes to different parts of a file.

123allow intelligent merge of changes to different parts of a file.

113

124

114−`%meld`: combine changes; if both sides changed the same part of a

125+**`%meld`**: combine changes; if both sides changed the same part of a

115file, use the version of the file in the merge-base.

126file, use the version of the file in the merge-base.

116

127

117−`%only-this`: create a merge commit with exactly the contents of the

128+**`%only-this`**: create a merge commit with exactly the contents of the

118−destination desk.

129+destination `desk`.

119

130

120−`%only-that`: create a merge commit with exactly the contents of the

131+**`%only-that`**: create a merge commit with exactly the contents of the

121source commit.

132source commit.

122

133

123−`%take-this`: create a merge commit with exactly the contents of the

134+**`%take-this`**: create a merge commit with exactly the contents of the

124−destination desk except take any files from the source commit which

135+destination `desk` except take any files from the source commit which

125−are not in the destination desk.

136+are not in the destination `desk`.

126

137

127−`%take-that`: create a merge commit with exactly the contents of the

138+**`%take-that`**: create a merge commit with exactly the contents of the

128−source commit except preserve any files from the destination desk

139+source commit except preserve any files from the destination `desk`

129which are not in the source commit.

140which are not in the source commit.

130

141

131−`%meet-this`: merge as in `%meet`, except if both sides changed the same

142+**`%meet-this`**: merge as in `%meet`, except if both sides changed the same

132−file, use the version in the destination desk.

143+file, use the version in the destination `desk`.

133

144

134−`%meet-that`: merge as in `%meet`, except if both sides changed the same

145+**`%meet-that`**: merge as in `%meet`, except if both sides changed the same

135file, use the version in the source commit.

146file, use the version in the source commit.

136

147

137#### Examples and notes:

148#### Examples and notes:

138

149

139−The most common merge strategy is %mate, which is a normal 3-way

150+The most common merge strategy is `%mate`, which is a normal 3-way

140merge which aborts on conflict.

151merge which aborts on conflict.

141

152

142`%take-that` is useful to "force" an OTA. After running `%take-that`,

153`%take-that` is useful to "force" an OTA. After running `%take-that`,

143you're guaranteed to have exactly the files in the source commit plus

154you're guaranteed to have exactly the files in the source commit plus

144any files you separately added.

155any files you separately added.

145

156

146−We speak of merging into a destination *desk* from a source *commit*

157+We speak of merging into a destination *`desk`* from a source *commit*

147−because while you can only merge on top of a desk, you can merge from

158+because while you can only merge on top of a `desk`, you can merge from

148historical commits. For example,

159historical commits. For example,

149```

160```

150|merge %old our %home, =cas ud+5, =gem %init

161|merge %old our %home, =cas ud+5, =gem %init

151```

162```

152−will create a new desk called `%old` with the 5th commit in `%home`.

163+will create a new `desk` called `%old` with the 5th commit in `%home`.

153−You can revert the contents of a desk to what they were yesterday

164+You can revert the contents of a `desk` to what they were yesterday

154with

165with

155```

166```

156|merge %home our %home, =cas da+(sub now ~d1), =gem %only-that

167|merge %home our %home, =cas da+(sub now ~d1), =gem %only-that

@@ -161,47 +172,52 @@ Note this is a normal `%only-that merge`, which means you're creating a

161

172

162`%meld` is rarely used on its own, however if you specify `%auto` or

173`%meld` is rarely used on its own, however if you specify `%auto` or

163omit the merge strategy, `%kiln` will run a `%meld` merge into a scratch

174omit the merge strategy, `%kiln` will run a `%meld` merge into a scratch

164−desk and then annotate the conflicts there.

175+`desk` and then annotate the conflicts there.

165

176

166If you resolve merge conflicts manually, for example by mounting the

177If you resolve merge conflicts manually, for example by mounting the

167−desks, copying the files in Unix and then running `|commit`, you

178+`desk`s, copying the files in Unix and then running `|commit`, you

168should usually run an `%only-this merge`. This will not change the

179should usually run an `%only-this merge`. This will not change the

169−newly-fixed contents of your desk, but it will record that the merge

180+newly-fixed contents of your `desk`, but it will record that the merge

170happened so that those conflicts don't reappear in later merges.

181happened so that those conflicts don't reappear in later merges.

171

182

172If you get a `%merge-no-merge-base` error, this means you're trying to

183If you get a `%merge-no-merge-base` error, this means you're trying to

173−merge two desks which have no common ancestors. You need to give

184+merge two `desk`s which have no common ancestors. You need to give

174them a common ancestor by choosing a merge strategy which doesn't

185them a common ancestor by choosing a merge strategy which doesn't

175need a merge-base, like `%only-this`, `%only-that`, `%take-this`, or

186need a merge-base, like `%only-this`, `%only-that`, `%take-this`, or

176`%take-that`.

187`%take-that`.

177

188

178`%take-this` could be useful to install 3rd party software, but you

189`%take-this` could be useful to install 3rd party software, but you

179couldn't get subsequent updates this way, since the files would

190couldn't get subsequent updates this way, since the files would

180−already exist in the destination desk. Something like "take only

191+already exist in the destination `desk`. Something like "take only

181the files which aren't in my OTA source or any other 3rd party app"

192the files which aren't in my OTA source or any other 3rd party app"

182would be basically correct. This would require a parameter listing

193would be basically correct. This would require a parameter listing

183−the desks to not conflict with.

194+the `desk`s to not conflict with.

184

195

185`%meet-this` and `%meet-that` imply the existence of `%mate-this` and

196`%meet-this` and `%meet-that` imply the existence of `%mate-this` and

186`%mate-that`, but those don't exist yet.

197`%mate-that`, but those don't exist yet.

187

198

188### Autosync

199### Autosync

189

200

201+Usage:

202+```

203+|sync %target-desk ~source-ship %source-desk

204+```

205+

190−Tracking and staying in sync with another desk is another

206+Tracking and staying in sync with another `desk` is another

191fundamental operation. We call this "autosync". This doesn't mean

207fundamental operation. We call this "autosync". This doesn't mean

192−simply mirroring a desk, since that wouldn't allow local changes.

208+simply mirroring a `desk`, since that wouldn't allow local changes.

193We simply want to apply changes as they are made upstream, as

209We simply want to apply changes as they are made upstream, as

194long as there are no conflicts with local changes.

210long as there are no conflicts with local changes.

195

211

196−This is implemented by watching the other desk, and, when it has

212+This is implemented by subscribing to changes to the other `desk`, with a

197−changes, merging these changes into our desk with the usual merge

213+`%many` request and, when it has changes, merging these changes into our `desk`

198−strategies.

214+with the usual merge strategies.

199

215

200−Note that it's quite reasonable for two desks to be autosynced to

216+Note that it's quite reasonable for two `desk`s to be autosynced to

201−each other. This results in any change on one desk being mirrored

217+each other. This results in any change on one `desk` being mirrored

202to the other and vice versa.

218to the other and vice versa.

203

219

204−Additionally, it's fine to set up an autosync even if one desk,

220+Additionally, it's fine to set up an autosync even if one `desk`,

205−the other desk, or both desks do not exist. The sync will be

221+the other `desk`, or both `desk`s do not exist. The sync will be

206−activated when the upstream desk comes into existence and will

222+activated when the upstream `desk` comes into existence and will

207−create the downstream desk if needed.

223+create the downstream `desk` if needed.

arvo/concepts/_index.md+1−4

@@ -3,14 +3,11 @@ title = "Concepts"

3weight = 100

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8Other concepts related to the way Arvo works.

9Other concepts related to the way Arvo works.

10−## [Input and Output](@/docs/arvo/concepts/io.md)

11−

12−Styles if IO you'll encounter in Urbit.

13−

14## [Subscriptions](@/docs/arvo/concepts/subscriptions.md)

11## [Subscriptions](@/docs/arvo/concepts/subscriptions.md)

16Urbit relies heavily on subscriptions throughout the stack. We strongly prefer reactive data flow compared to querying or polling, which is why for example our equivalent to git natively supports subscriptions to its data.

13Urbit relies heavily on subscriptions throughout the stack. We strongly prefer reactive data flow compared to querying or polling, which is why for example our equivalent to git natively supports subscriptions to its data.

arvo/concepts/io.mddeleted−471

@@ -1,471 +0,0 @@

1−+++

2−title = "Input and Output"

3−weight = 37

4−template = "doc.html"

5−+++

6−

7−Let's talk about IO in Urbit. I won't say much about how events and effects are

8−processed by the runtime; it suffices to note that the runtime is a normal Unix

9−program which (1) listens for various events, (2) passes them to Arvo (Urbit

10−OS), and (3) processes the effects produced by Arvo. Instead, I’ll focus on how

11−IO works from the perspective of Hoon programs, like Arvo or apps running on

12−Arvo. But first, let's review IO paradigms in other languages.

13−

14−All examples are in a made-up language, since it's useful to understand this

15−information even before you're comfortable in Hoon. If you're learning Hoon and

16−want a challenge, try translating them!

17−

18−## The three styles of IO

19−

20−I will call these styles imperative IO, monadic IO, and state machines (note

21−that this classification refers only to the experience of using them and the

22−program structure they dictate). An imperative programming language can use any

23−of these and a functional programming language can use any of them.

24−

25−### Imperative IO

26−

27−Imperative languages (such as C and Python) perform IO by calling a function in

28−the middle of whatever they were doing. Anywhere, in any statement, you could

29−have IO.

30−

31−In this example and others to follow, the task will be to read a filename from

32−the command line, read a URL out of that file, and fetch the contents at that

33−URL.

34−

35− output = fetchUrl(stripWhitespace(readFile(cliInput())))

36−

37−For readability, intermediate variables would be a good idea, but nothing stops

38−you from nesting IO this way.

39−

40−### Monadic IO

41−

42−Languages that use explicit monads for IO (such as Haskell) perform IO by

43−running IO behind a bind operation, which I'll notate with `<-`. Thus, while you

44−can't put IO *anywhere*, the general structure is a sequence of IO operations.

45−

46− fileName <- cliInput()

47− url <- readFile(fileName)

48− output <- fetchUrl(stripWhitespace(url))

49−

50−[Tackling the Awkward

51−Squad](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/07/mark.pdf)

52−is the canonical description of monadic IO, and does a great job of explaining

53−how the Haskell community decided to use this as their primary IO style.

54−

55−### State machines

56−

57−Languages that use explicit state machines for IO (such as Elm) perform IO by

58−accepting events and producing a list of effects.

59−

60− event :: Initialize

61− | CLIInput text

62− | FileInput text

63− | HTTPResponse body

64− effect :: CLIInput

65− | ReadFile fileName

66− | FetchUrl url

67− | Print text

68− main(event) {

69− switch event {

70− Initialize -> [CLIInput ~]

71− CLIinput fileName -> [ReadFile fileName]

72− FileInput url -> [FetchUrl stripWhitespace(url)]

73− HTTPResponse body -> [Print body]

74− }

75− }

76−

77−The effects produced are not functions but data.

78−

79−It's important to note that this example is chosen to highlight a particular

80−kind of IO — sequences of events. Further, this example has no persistent state

81−— each IO operation is a pure function of the output of the previous one.

82−

83−## A stateful, loopy example

84−

85−Let's look at another type of program: a simple HTTP server that you can give

86−text at the CLI, which will respond to all HTTP requests with that text.

87−

88−Imperative:

89−

90− text = "no text entered yet"

91− while (true)

92− cli = readCli()

93− if (cli != Null)

94− text = cli

95− request = readHttp()

96− if (request != Null)

97− respond(request,text)

98−

99−Monadic:

100−

101− loop("no text entered yet")

102−

103− where:

104−

105− loop :: text -> IO Null

106− loop =

107− input <- readInput()

108− switch input

109− CLIInput newText -> loop(newText)

110− HTTPRequest request ->

111− _ <- respond(request,text)

112− loop(text)

113−

114−State machine:

115−

116− text = "no text entered yet"

117− main(text,event) {

118− switch event {

119− CLIInput newText -> [newText, Null]

120− HTTPRequest request -> [text, HTTPResponse text]

121− }

122− }

123−

124−Again, note the response in the state machine is data, not a function.

125−

126−I'm not trying to make a point about which type is better in general. There's a

127−pretty strong argument that in the first example, imperative was the best

128−option, followed by monadic, followed by state machine. There's an equally

129−strong argument that the second example is most naturally represented by the

130−state machine.

131−

132−Look at how state is handled in the monadic example. Formally, bind composes

133−functions, so that every time you see a `<-` there's actually a new function

134−(closure/delimited continuation) being created and stored for when we receive a

135−response. All variables in scope are stored in that closure, so we can

136−conveniently access them later without explicitly storing them. However, the

137−state we preserve over time varies a lot. In this example, it varies between

138−storing `loop`, `[loop input]`, `[loop input newText]`, and `[loop input

139−request]`.

140−

141−By comparison, the state machine does not store intermediate values, it only

142−stores its explicit state. While this is somewhat less flexible, it puts the

143−permanent state front-and-center.

144−

145−I'll reiterate that all of these are possible to implement in purely functional

146−languages, and further they can all be implemented in terms of each other. State

147−machines are trivially implementable -- it's just a function [state event] ->

148−[new-state (list effect)]. Monadic IO can be implemented directly (as in

149−Haskell) or on top of state machines by storing continuations in your state.

150−Imperative IO is somewhat more difficult but should be possible through

151−algebraic effects.

152−

153−When inventing a new system, you should consider which of these you intend to

154−support based on the sort of code you expect to see. For example, a scripting

155−language will almost always benefit from imperative (or at least monadic) style.

156−

157−## In Urbit

158−

159−So much for exploring the space of IO solutions. In terms of deployed code on

160−Earth, the split is about 90% imperative, 5% monadic, and 5% state machine. In

161−Urbit, the split is about 90% state machine and 10% monadic. This is what I

162−intend to explain and defend.

163−

164−Urbit's OS is called Arvo; it has several "kernel modules" called vanes, and it

165−has userspace applications. Colloquially these are called "apps," but when

166−speaking about them abstractly, it's helpful to use their more specific name:

167−["agents"](https://urbit.org/docs/tutorials/arvo/gall/#agents). Arvo, the vanes,

168−and agents are all structured as explicit state machines.

169−

170−In addition, there are

171−["threads"](https://urbit.org/docs/tutorials/arvo/gall/#threads), which are

172−structured as monadic IO. Note there's only a superficial resemblance to Unix

173−threads: they are not executed in parallel and they don't share memory, or at

174−least not any more than agents do. They're simply a computation that takes an

175−argument and produces a result, possibly after doing some IO, structured

176−monadically.

177−

178−As you can see, all the lowest levels of Arvo are explicit state machines, and

179−only the highest layer supports monadic IO. To see why this is the case, let's

180−list advantages and disadvantages to each type. Note that, while I'm speaking

181−specifically of agents and threads in the context of Urbit, these are

182−fundamental properties of monadic and state machine IO.

183−

184−Agents are:

185−

186−- Upgradable. Since the state is explicit, you can upgrade an agent in-place by

187− supplying a new state machine and a function from the old state type to the

188− new state type. This is very similar to a database migration.

189−

190−- Robust. A state machine must accept any input at any time. If it receives

191− unexpected input, it must have had explicit error handling for that input,

192− since it's one of the types in the switch statement. Thus, it's relatively

193− easy to stay in a consistent state.

194−

195−- Permanent. Since it's upgradable and robust, there's no reason to stop its

196− running.

197−

198−- Hard to write long sequences of IO. Long sequences of IO create many possible

199− states, and you must handle any input in any state. This can result in

200− complex, hard-to-follow code with code paths that are rarely if ever taken.

201−

202−Threads are:

203−

204−- Not upgradable in-place. Since the state is implicit, you can only cleanly

205− upgrade by (1) killing the thread and restarting it or (2) letting it resolve

206− to a defined "upgradable state" periodically, usually at the top of a "main

207− loop". If you do (2), then you are in fact structuring it as a state machine,

208− so the other advantages of threads are compromised.

209−

210−- Fragile. If we don't receive expected input, the default behavior is to be

211− stuck waiting for input. You must remember to add any error-handling features

212− you want, and it's easy to end up in an inconsistent state.

213−

214−- Impermanent. Since a thread can't be upgraded and may get stuck in case of

215− unexpected input, a thread can't be expected to last forever. It must be

216− "rebooted" from time to time.

217−

218−- Easy to write long sequences of IO. Just...write the IO one line at a time.

219−

220−Arvo proper and the vanes are permanent pieces of software and, for this reason,

221−state machines are more natural.

222−

223−I'll note that other kernels, like the Unix kernel, use imperative IO. The

224−difference is that Unix isn't permanent in itself — it only lasts until you

225−reboot. Since your permanent state is stored externally, you don't need to be

226−upgradable or permanent. With sufficient discipline you can write robust C code,

227−so it makes sense to use imperative style. Even so, a lot of C code is written

228−in a state machine style; even if it doesn't need to be upgradable, it's worth

229−it for the robustness and concurrency advantages.

230−

231−However, Arvo is a single-level store — all of its state is permanent. This is

232−very convenient and eliminates many long-tail bugs, but it also means we need to

233−code for permanence, and that's much easier when you structure your code with

234−explicit state.

235−

236−Further, Arvo and the vanes rarely perform long sequences of IO. Generally, they

237−do one thing and emit some effects, similar to the HTTP server example above.

238−Sometimes they "pass through" a request. For example, the Eyre vane passes HTTP

239−requests to userspace agents or threads. In this case, there is a sequence of

240−IO, which in monadic code would be:

241−

242− loop()

243−

244− where:

245−

246− loop =

247− request <- receiveRequest()

248− agent = lookupAgent(request)

249− response <- callAgent(agent, request)

250− _ <- sendResponse(request, response)

251− loop()

252−

253−This is clear code to read, but it leaves open the question of how to respond

254−during the callAgent() function. If another request comes in, can we handle it

255−in parallel? If we need to upgrade ourselves before the agent responds, can we

256−make sure to properly handle the response?

257−

258−The naive solution in monadic code is to say that you can only handle one

259−request at a time, and if an upgrade happens you just drop outstanding requests.

260−You can resolve both of these by introducing a sort of main loop, which is just

261−a way of saying "turn it into a state machine".

262−

263−In a state machine, the naive solution is:

264−

265− state :: (map request agent)

266− main(state, event) {

267− switch event {

268− HTTPRequest request ->

269− agent = lookupAgent(request)

270− [put(state, request, agent), CallAgent agent request]

271− AgentResponse request response ->

272− [del(state, request), HTTPRespond request response]

273− }

274− }

275−

276−Upgrading this is trivial, since the state (map of requests to outstanding calls

277−to agents) is explicit. It also trivially handles concurrent requests.

278−

279−I won't claim this is easier or harder to write than the monadic version, but it

280−has the properties we need at this level of the system.

281−

282−Userspace agents similarly are permanent entities that need to not lose data or

283−get stuck. For this reason they’re structured as state machines.

284−

285−A helpful comparison is that explicit state machines are basically Mealy

286−machines, the kind you might have learned about in an introductory digital

287−design class. This shouldn't be surprising; a digital circuit must always be

288−consistent because it can't be manually "rebooted", and their input and output

289−is highly formalized. By contrast, a Rube Goldberg machine doesn't have the same

290−constraints; which is why they're a lot more fun!

291−

292−It's notable also that explicit state machines are considerably more declarative

293−than imperative or monadic IO. Not everyone thinks that's a good thing, but

294−anyone who defends functional programming should see some advantages to that.

295−

296−However, userspace sometimes needs to perform long sequences of IO. Let's look

297−at an example where we want to fetch the front page of a site and the first

298−couple comments on each story. HTTP is unreliable, so on failure we retry five

299−times.

300−

301−Monadic:

302−

303− topStories <- fetch(topStoriesUrl)

304− loop(topStories)

305−

306− where:

307−

308− loop :: topStories -> IO Null

309− if topStories is Null

310− return Null

311− comments <- retryLoop(5,head(topStories))

312− otherComments <- loop(tail(topStories))

313− append(comments,otherComments)

314−

315− retryLoop :: [n story] -> IO Comments

316− comments <- fetchComments(story)

317− if comments is HTTPError:

318− if n == 0:

319− bail

320− else:

321− retryLoop(n-1, story)

322−

323−State machine:

324−

325− state :: Initial

326− | FetchingTop

327− | FetchingStory comments stories retries

328− | Done comments

329−

330− main(state, event) {

331− switch event {

332− Initialize ->

333− assert state == Initial

334− [FetchingTop, Fetch topStoriesUrl]

335− HttpResponse response ->

336− switch state {

337− FetchingTop ->

338− [FetchingStory Null stories 5, FetchComments head(stories)]

339− FetchingStory comments stories retries ->

340− if response is HTTPError:

341− if retries == 0:

342− [Initial, Null]

343− else:

344− [FetchingStory comments stories retries-1,

345− FetchComments head(stories)]

346− else:

347− comments = append(response,comments)

348− if tail(stories) is Null:

349− [Done comments, Null]

350− else:

351− [FetchingStory append(comments,response) tail(stories) 5,

352− FetchComments head(tail(stories))]

353− }

354− }

355− }

356−

357−With practice, the state machine version of this can be written correctly.

358−However, (1) it's verbose, (2) the control flow jumps all over the place, and

359−(3) in practice it's challenging to get exactly right. This is still a small

360−example, and it gets much worse as the length and complexity of the IO sequence

361−grows.

362−

363−The monadic version, on the other hand, flows cleanly down the page. We can

364−factor out generic functionality like "retry this request n times" to make the

365−essence of the computation clear.

366−

367−Suppose you have to upgrade this. In the state machine, you can see exactly

368−which stage of the computation you're in and write specific code to upgrade

369−cleanly.

370−

371−Threads, as we've discussed, can't be upgraded in-place. Let's be real though:

372−we're downloading comment sections, not regulating a nuclear reactor. If we get

373−a better version of this script, just kill the old one and start the new one

374−from scratch. What's a couple extra HTTP requests?

375−

376−This suggests a general division of workloads: code which needs to be permanent,

377−upgradable, and concurrent should generally be a state machine. Code with long

378−or complex sequences of IO and definite termination should generally be a

379−thread.

380−

381−These aren't completely mutually exclusive, but it's surprising how often they

382−are.

383−

384−### Jael: a case study

385−

386−When you need to do something permanent and upgradable, but you also need to do

387−long sequences of IO, the main trick is to factor your problem into both an

388−agent and a thread (or several).

389−

390−As a case study, consider Jael, one of our vanes. This maintains our PKI state,

391−which it downloads over HTTP from an Ethereum node. Downloading from an Ethereum

392−node is a complex sequence that starts with fetching its most recent block

393−number, then fetching all the blocks we haven't seen and scanning for

394−transactions we care about. We also maintain a map of block hashes to block

395−number, and if we see that a block's parent doesn't have the hash we expect,

396−that means a reorganization has occurred, so we must "rewind" one block at a

397−time until we find where the fork happened, then restart going forward.

398−

399−You don't have to understand this sequence of IO; it's sufficient to see that

400−it's complex and must be exactly right. We used to do this in a state machine

401−directly in Jael. This data is, of course, permanent and needs to be accessible

402−to the rest of the system, so Jael must be a state machine.

403−

404−However, we had many small mistakes in this sequence of IO, some of which only

405−appeared during unusual events, such as when an HTTP request failed at the same

406−time as a reorganization happened.

407−

408−We're in a bit of a bind, though: Jael has to be a state machine. The solution

409−is to factor out the act of getting updates into a thread while the main PKI

410−state is stored in the state machine in Jael.

411−

412−This thread has a definite purpose: given the most recent block number we

413−already know about, fetch all the PKI transactions since then. The function

414−signature is:

415−

416− syncEthereum oldBlockNumber -> IO [newBlockNumber, newTxs]

417−

418−Jael runs this thread every five minutes and, if it succeeds, then we update our

419−block number and PKI state. If the thread fails, gets stuck, or we need to

420−upgrade it, we just kill the old thread and start a new one. Again, a few extra

421−HTTP requests don’t matter.

422−

423−From Jael's perspective, it looks something like:

424−

425− state :: State block txs outstandingThread

426− main(state,event) {

427− switch event {

428− Initialize -> [State 0 Null Null, StartTimer (now + five minutes)]

429− TimerFired ->

430− effects = If outstandingThread is Null

431− then Null

432− else [Kill outstandingThread]

433− newId = newThreadId()

434− [State txs newId,

435− append(effects,

436− StartThread newId syncEthereum(block),

437− StartTimer (now + five minutes))]

438− ThreadFinished Failure -> [State block txs Null, Null]

439− ThreadFinished Success newBlockNumber newTxs ->

440− [State newBlockNumber append(txs,newTxs) Null, Null]

441− }

442− }

443−

444−In other words, from Jael's perspective, it's just a single IO event that

445−encapsulates arbitrarily complex IO in the thread. A single IO event is easy

446−enough, and we can maintain all our permanency and robustness guarantees,

447−because we know the thread will have one of three results: success, failure, or

448−it hangs. If we set a timeout, hang becomes failure, so there's only two

449−possible results: success or failure. If we handle both of those correctly,

450−we've handled all the possible IO errors that could have occurred.

451−

452−What about upgrading? We've already said we can handle the thread dying for

453−arbitrary reasons, so if an upgrade comes in, we just kill it. We can upgrade

454−Jael's state machine easily since its state is very simple.

455−

456−## Recap

457−

458−Let's recap what we've covered:

459−

460−- There are three styles of IO: imperative, monadic, and explicit state

461− machines.

462−- Urbit supports explicit state machines (agents) and monadic IO (threads).

463−- Explicit state machines are preferred for permanent, robust, concurrent code.

464−- Monadic IO is preferred for short-lived code with complex IO.

465−- If a state machine needs to perform complex IO, it should encapsulate it in a

466− thread so that, from the perspective of the state machine, it's simple IO.

467−

468−Hopefully it's fairly clear why Urbit uses explicit state machines in most

469−situations. More than anything else, our goal is to build software that can last

470−forever. Explicit state machines are a crucial tool for that.

471−

arvo/dill/_index.md+1

@@ -3,4 +3,5 @@ title = "Dill"

3weight = 40

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/eyre/_index.md+1

@@ -3,6 +3,7 @@ title = "Eyre"

3weight = 50

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8## [Overview](@/docs/arvo/eyre/eyre.md)

9## [Overview](@/docs/arvo/eyre/eyre.md)

arvo/ford/_index.md+1

@@ -3,4 +3,5 @@ title = "Ford"

3weight = 60

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/gall/_index.md+1

@@ -3,4 +3,5 @@ title = "Gall"

3weight = 70

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/gall/overview.mdnew file+11

@@ -0,0 +1,11 @@

1++++

2+title = "Overview"

3+weight = 1

4+template = "doc.html"

5++++

7+**Gall** is the application-management [vane](../vane). Userspace apps –⁠ daemons, really –⁠ are started, stopped, and sandboxed by Gall. Gall provides developers with a consistent interface for connecting their app to [Arvo](../arvo). It allows applications and other vanes to send messages to applications and subscribe to data streams. Messages coming into Gall are routed to the intended application, and the response comes back along the same route. If the intended target is on another [ship](../ship), Gall will route it behind the scenes through [Ames](../ames) to the other ship.

9+Gall is located at `/home/sys/vane/gall.hoon` within Arvo.

10+

11+For more details, see the [userspace](@/docs/userspace/gall/gall.md) section.

arvo/iris/_index.md+1

@@ -3,4 +3,5 @@ title = "Iris"

3weight = 80

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/jael/_index.md+1

@@ -3,4 +3,5 @@ title = "Jael"

3weight = 90

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

arvo/overview.md+1−1

@@ -566,7 +566,7 @@ As of this writing, we have nine vanes, which each provide the following service

566- [Eyre](@/docs/arvo/eyre/eyre.md): an http server. Unix sends http messages to `%eyre`, and `%eyre` produces http messages in response.

567- [Ford](@/docs/arvo/ford/ford.md): a build system also utilized for marks

568 (file types) and debugging.

569−- [Gall](@/docs/arvo/gall/gall.md): manages our userspace applications. `%gall` keeps state and manages subscribers.

569+- [Gall](@/docs/userspace/gall/gall.md): manages our userspace applications. `%gall` keeps state and manages subscribers.

570- `Iris`: an http client.

571- `Jael`: storage for Azimuth information.

572

arvo/tutorials/_index.md+1

@@ -3,6 +3,7 @@ title = "Tutorials"

3weight = 200

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8Tutorials aimed at broadening understanding of Arvo.

9Tutorials aimed at broadening understanding of Arvo.

azimuth/_index.md+1

@@ -3,6 +3,7 @@ title = "Azimuth"

3weight = 700

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

azimuth/azimuth.md+1−1

@@ -6,7 +6,7 @@ aliases = ["/docs/learn/azimuth"]

6+++

7Azimuth is a general-purpose public-key infrastructure (PKI) on the Ethereum blockchain, used as a platform for _Urbit identities_. You need such an identity to use the Arvo network.

9−The primary way to interact with Azimuth is through our [Bridge](https://github.com/urbit/bridge) application and the node libraries that it depends on, [azimuth-js](https://github.com/urbit/azimuth-js) and [urbit-key-generation](https://github.com/urbit/urbit-key-generation). Take a look at the source and play around, or see [Getting Started](@/getting-started.md).

9+The primary way to interact with Azimuth is through our [Bridge](https://github.com/urbit/bridge) application and the node libraries that it depends on, [azimuth-js](https://github.com/urbit/azimuth-js) and [urbit-key-generation](https://github.com/urbit/urbit-key-generation). Take a look at the source and play around, or see [Getting Started](@/getting-started/_index.md).

11## Arvo vs. Azimuth

development/_index.md+1

@@ -3,6 +3,7 @@ title = "Development"

3weight = 100

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8This section covers the development process in general.

9This section covers the development process in general.

development/develop.md+7−9

@@ -13,7 +13,7 @@ contribute to. At a high level, you can:

13- [Build clients](#clients), which use Urbit as their back-end to store data and

14 interact with the network

15- [Develop agents](#agents), or more precisely [*gall*

16− agents](@/docs/arvo/gall/gall.md), which are written in [Hoon](/docs/hoon/)

16+ agents](@/docs/userspace/gall/gall.md), which are written in [Hoon](/docs/hoon/)

17 and can best be thought of as microservices

18- [Contribute to Urbit's core](#core), which involves contributing to Urbit

19 itself — either the operating system

@@ -27,7 +27,7 @@ throughout the docs.

28### Build Clients {#clients}

30−Urbit ships with [Landscape](@/docs/landscape/overview.md), which can be thought

30+Urbit ships with [Landscape](@/docs/userspace/landscape/overview.md), which can be thought

31of as its flagship client. Landscape can be broken into two different technical

32components: there's the collection of [agents](#agents) that utilize the

33Urbit for networking and data persistence, and the JavaScript client, which is

@@ -37,7 +37,7 @@ with [React](https://reactjs.org).

37Additionally, you can [contribute to

38Landscape](https://github.com/urbit/urbit/blob/master/pkg/interface/CONTRIBUTING.md)

39by tackling issues in its [issue

40−tracker](https://github.com/urbit/landscape/issues) and/or running the the

40+tracker](https://github.com/urbit/landscape/issues) and/or running the

41[development stream](https://groups.google.com/a/urbit.org/g/dev/c/r2hv4ajCLwk).

43You can create your own clients that uses Urbit to store data and interact with

@@ -60,17 +60,17 @@ library called `shoe`, which you can learn more about

60Gall agents can be best thought of as individual services with a

61statically-typed interface and built-in database. They're written in

62[Hoon](@/docs/hoon/overview.md) and all conform to a specific

63−[interface](@/docs/arvo/gall/gall-api.md) that's defined by the kernel vane

63+[interface](@/docs/userspace/gall/gall-api.md) that's defined by the kernel vane

64−called [Gall](@/docs/arvo/gall/gall.md).

64+called [Gall](@/docs/userspace/gall/gall.md).

66Applications built on Urbit utilize a collection of different agents, each of

67which handle specific functionality. The [Landscape

68−architecture](@/docs/landscape/overview.md) is a good reference implementation

68+architecture](@/docs/userspace/landscape/overview.md) is a good reference implementation

69to see how this pattern works in practice.

71If you want to learn how to build gall agents, your best bet is to first [learn

72the Hoon programming language](@/docs/hoon/hoon-school/_index.md), and then follow that

73−up with the [Gall guide](@/docs/arvo/gall/tutorial.md).

73+up with the [Gall guide](@/docs/userspace/gall/tutorial.md).

75### Contribute to Urbit Core {#core}

@@ -113,5 +113,3 @@ From here, you'll probably want to learn more about:

113 community-maintained

114- Our development [Precepts](@/docs/development/precepts.md), which outline the

115 philosophies that drive Urbit development

116−

117−

development/grants.md−3

@@ -2,11 +2,8 @@

2title = "Grants Program"

3weight = 20

4template = "doc.html"

5−aliases = ["/docs/using/creating-a-development-ship/", "/docs/getting-started/contributing"]

6+++

5+++

8−## Grants Program {#grants}

9−

10The [Urbit Foundation](https://urbit.org/blog/first-steps-towards-urbit-org/)

7The [Urbit Foundation](https://urbit.org/blog/first-steps-towards-urbit-org/)

11runs a [Grants program](https://grants.urbit.org), where you can earn address

8runs a [Grants program](https://grants.urbit.org), where you can earn address

12space ([stars](@/docs/glossary/star.md)) in exchange for contributing to the

9space ([stars](@/docs/glossary/star.md)) in exchange for contributing to the

glossary/aqua.md+1−1

@@ -13,4 +13,4 @@ Aqua is jetted with the usual [Nock](../nock) interpreter and thus virtual ships

14### Further Reading

16−- [The Gall tutorial](@/docs/arvo/gall/gall.md): A technical guide to the Gall vane.

16+- [The Gall tutorial](@/docs/userspace/gall/gall.md): A technical guide to the Gall vane.

glossary/azimuth.md+1−1

@@ -14,5 +14,5 @@ It should be noted that Azimuth isn't built just for Arvo. It can be used as an

14### Further Reading

16- [The Azimuth concepts page](@/docs/azimuth/advanced-azimuth-tools.md): A more in-depth explanation of Azimuth, including information the storage of Urbit identities.

17−- [Install instructions](@/getting-started.md): Guide to installing Urbit, which includes instructions on booting with your Urbit identity.

17+- [Install instructions](@/getting-started/_index.md): Guide to installing Urbit, which includes instructions on booting with your Urbit identity.

18- [Understanding Urbit ID](@/understanding-urbit/urbit-id.md)

glossary/comet.md+1−1

@@ -12,4 +12,4 @@ Comets have no presence on [Azimuth](../azimuth) and thus have no associated Urb

13### Further reading

15−- [Creating a Comet](@/getting-started.md#comet): A guide to spinning up a comet of your own.

15+- [Creating a Comet](@/getting-started/_index.md#comet): A guide to spinning up a comet of your own.

glossary/gall.md+1−1

@@ -14,4 +14,4 @@ Gall is located at `/home/sys/vane/gall.hoon` within Arvo.

15- [The Hoon Tutorial](@/docs/hoon/hoon-school/_index.md): Our guide to learning the Hoon programming language that will give you the foundation necessary for app development.

16 - [Lesson 2.7: Gall](@/docs/hoon/hoon-school/gall.md): A Hoon Tutorial lesson that explains how to write a simple Gall app.

17−- [The Gall tutorial](@/docs/arvo/gall/gall.md): A technical guide to the Gall vane.

17+- [The Gall tutorial](@/docs/userspace/gall/gall.md): A technical guide to the Gall vane.

glossary/keyfile.md+1−1

@@ -12,4 +12,4 @@ The keyfile is used as an argument in the command line when booting a ship. Duri

13### Further Reading

15−- [Installation Guide](@/getting-started.md): Instructions on using a keyfile to boot a ship.

15+- [Installation Guide](@/getting-started/_index.md): Instructions on using a keyfile to boot a ship.

glossary/moon.md+1

@@ -13,3 +13,4 @@ Unlike planets, moons have no presence on [Azimuth](../azimuth) and have no asso

13### Further Reading

15- [Using Your Ship](@/using/os/getting-started.md#moons): The "Moons" section contains instructions for creating and managing moons.

16+

glossary/ship.md+1−1

@@ -20,7 +20,7 @@ To be able to use the Arvo network, planets, stars, and galaxies must be combine

21### Further Reading

23−- [Using Bridge](@/getting-started.md): A guide to installing Urbit and booting a ship.

23+- [Using Bridge](@/getting-started/_index.md): A guide to installing Urbit and booting a ship.

24- [Ship Troubleshooting](@/using/os/ship-troubleshooting.md): General instructions on getting your ship to work, which includes network connectivity issues.

25- [Understanding Urbit ID](@/understanding-urbit/urbit-id.md)

26- [Pier](../pier): The directory containing a ship's state.

hoon/_index.md+1

@@ -3,6 +3,7 @@ title = "Hoon"

3weight = 400

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8Hoon is Urbit's high-level general-purpose programming language.

9Hoon is Urbit's high-level general-purpose programming language.

hoon/guides/_index.md+1

@@ -3,6 +3,7 @@ title = "Guides"

3weight = 20

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8## [Writing Aqua Tests](@/docs/hoon/guides/aqua.md)

9## [Writing Aqua Tests](@/docs/hoon/guides/aqua.md)

hoon/hoon-school/_index.md+1

@@ -4,6 +4,7 @@ weight = 10

4sort_by = "weight"

5template = "sections/docs/hoon.html"

6aliases = ["/docs/byte/0/", "/docs/byte", "/docs/learn/hoon/hoon-tutorial/"]

7+insert_anchor_links = "right"

7+++

8+++

9This series is designed to teach you Hoon without assuming you have an extensive

10This series is designed to teach you Hoon without assuming you have an extensive

hoon/hoon-school/gall.md+2

@@ -34,6 +34,8 @@ Each `card` is a pair of a tag and a [noun](/docs/glossary/noun/). The tag indic

35Gall applications can have a number of arms that get called depending on the information they are sent.

37+### +on-agent

38+

37This arm is called once when the agent is started. It has no input and

39This arm is called once when the agent is started. It has no input and

38lets you perform any initial IO.

40lets you perform any initial IO.

hoon/hoon-school/iron-polymorphism.md+1−1

@@ -5,7 +5,7 @@ template = "doc.html"

5aliases = ["/docs/learn/hoon/hoon-tutorial/iron-polymorphism/"]

6+++

8−Polymorphism is a programming concept that allows a a piece of code to use different types at different times. It's a common technique in most languages to make code that can be reused for many different situations, and Hoon is no exception. Let's take a look at a [gate](/docs/glossary/gate/) from the Hoon standard library as an example; we'll be passing a few different types in. The code below is an excerpt from `hoon.hoon` and, as such, will not run as-is by itself.

8+Polymorphism is a programming concept that allows a piece of code to use different types at different times. It's a common technique in most languages to make code that can be reused for many different situations, and Hoon is no exception. Let's take a look at a [gate](/docs/glossary/gate/) from the Hoon standard library as an example; we'll be passing a few different types in. The code below is an excerpt from `hoon.hoon` and, as such, will not run as-is by itself.

10`fold` is a _wet_ gate that takes two arguments and produces a _dry_ gate. But what are wet and dry gates?

hoon/hoon-school/setup.md+2−2

@@ -17,7 +17,7 @@ These may not look like numbers, but they are. Each urbit name is written in a b

18You can install Urbit on any Mac or Unix machine; if you're just trying out Urbit or creating a development ship, you can follow the steps for creating a development ship [here](@/docs/development/environment.md). On Windows, make a virtual Linux machine using VirtualBox or a similar tool.

20−Once you're finished, you can [boot your very own ship](@/getting-started.md#comet). While you can develop in Hoon on your own ship on the live network, we strongly suggest developing on a development ship first.

20+Once you're finished, you can [boot your very own ship](@/getting-started/_index.md#comet). While you can develop in Hoon on your own ship on the live network, we strongly suggest developing on a development ship first.

22## Getting started

@@ -67,7 +67,7 @@ You asked Dojo to evaluate `17` and it echoed the number back at you. This value

68Generators are the most straightforward way to write Hoon programs. They are a concept in Arvo, and involve saving Hoon code in a `.hoon` text file. While they aren't strictly part of the Hoon language, we'll be dealing with generators throughout this tutorial.

70−The simplest type of generator is the **naked generator**. All naked generators are `gates`: functions that take an argument and produce an output. Before creating a generator you need to [mount](@/getting-started.md#the-dojo) your `desk` to Unix. To create a generator, all you need to do is write a `gate` and put it into a file in the `/home/gen/` directory of your ship as a `.hoon` file. (Assuming you've already mounted with `|mount /=home=`.) After this, you need to run `|commit %home` in Dojo and the new file will be recognized by your ship. To run a generator named `mygen.hoon`, you would type `+mygen <argument>` in your ship's Dojo.

70+The simplest type of generator is the **naked generator**. All naked generators are `gates`: functions that take an argument and produce an output. Before creating a generator you need to [mount](@/getting-started/_index.md#the-dojo) your `desk` to Unix. To create a generator, all you need to do is write a `gate` and put it into a file in the `/home/gen/` directory of your ship as a `.hoon` file. (Assuming you've already mounted with `|mount /=home=`.) After this, you need to run `|commit %home` in Dojo and the new file will be recognized by your ship. To run a generator named `mygen.hoon`, you would type `+mygen <argument>` in your ship's Dojo.

72If this doesn't make sense yet, that's okay. In the [next lesson](@/docs/hoon/hoon-school/list-of-numbers.md), we will walk you through an example `gate` that is run as a generator.

hoon/reference/_index.md+1

@@ -3,6 +3,7 @@ title = "Reference"

3weight = 40

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8This is where you look for answers to specific technical questions.

9This is where you look for answers to specific technical questions.

hoon/reference/auras.md+28−7

@@ -8,12 +8,9 @@ Auras are system of nested "soft types" on [atoms](@/docs/glossary/atom.md) that

8are used to track metadata about how a particular atom is to be interpreted.

9This is used for type checking as well as pretty printing.

11−A given aura nests under any aura whose name is a substring of the given aura,

11+You can learn more about auras in [Hoon school](@/docs/hoon/hoon-school/atoms-auras-and-simple-cell-types.md).

12−i.e. `@ux` nests under `@u`, and all auras nest under the empty aura `@`. We

13−call auras "soft types" since this nesting behavior can be ignored - see

14−[below](#non-coercive).

16−You can learn more about auras in [Hoon school](@/docs/hoon/hoon-school/atoms-auras-and-simple-types.md).

13+### Table of Auras

18```

15```

19Aura Meaning Example Literal Syntax

16Aura Meaning Example Literal Syntax

@@ -52,6 +49,8 @@ Aura Meaning Example Literal Syntax

52 @ux unsigned hexadecimal 0x5f5.e138

49 @ux unsigned hexadecimal 0x5f5.e138

53```

50```

52+### Bitwidth

53+

55Capital letters at the end of auras indicate the bitwidth in binary powers of

54Capital letters at the end of auras indicate the bitwidth in binary powers of

56two, starting from A.

55two, starting from A.

@@ -63,9 +62,22 @@ two, starting from A.

63 @uvJ unsigned, 512-bit integer (frequently used for entropy)

62 @uvJ unsigned, 512-bit integer (frequently used for entropy)

64```

63```

66−### Non-coercive

65+### Nesting

67+A given aura nests under any aura whose name is a substring or extension of the

68+given aura:

69+```

70+> :- (~(nest ut [%atom %ud ~]) | [%atom %u ~])

71+ (~(nest ut [%atom %ud ~]) | [%atom %udx ~])

72+[%.y %.y]

73+```

74+but does not nest "sideways":

75+```

76+> (~(nest ut [%atom %ud ~]) | [%atom %ux ~])

77+%.n

78+```

79+We call auras "soft types" since this nesting behavior can be ignored. Auras are

68−Auras are non-coercive, but conversions may have to go via the empty aura, e.g.

80+non-coercive, but conversions may have to go via the empty aura, e.g.

69```

81```

70> ^-(@ud ^-(@ 'foo'))

82> ^-(@ud ^-(@ 'foo'))

717.303.014

837.303.014

@@ -75,3 +87,12 @@ This is implicitly done by the irregular form of `^-`.

75> `@ud`'foo'

87> `@ud`'foo'

767.303.014

887.303.014

77```

89```

90+

91+### Bunting

92+

93+The bunt value for all auras is 0 except for `@da`.

94+```

95+> *@da

96+~2000.1.1

97+```

98+A bunt value of 0 is helpful when working with loobeans.

hoon/reference/irregular.md+1

@@ -60,6 +60,7 @@ I:

60[a b c] ==> [a [b c]]

61 a^b^c ==> [a b c]

62 a/b ==> [%a b]

63+ a+b ==> [%a b]

63 `a ==> [~ a]

64 `a ==> [~ a]

64 ~[a b] ==> [a b ~]

65 ~[a b] ==> [a b ~]

65 [a b]~ ==> [[a b] ~]

66 [a b]~ ==> [[a b] ~]

hoon/reference/rune/_index.md+1

@@ -4,6 +4,7 @@ weight = 20

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6aliases = ["docs/reference/hoon-expressions/rune/"]

7+insert_anchor_links = "right"

7+++

8+++

8Runes are a way to form expressions in Hoon.

9Runes are a way to form expressions in Hoon.

hoon/reference/rune/mic.md+74

@@ -59,6 +59,80 @@ Irregular: `:(add a b c)` is `;:(add a b c)`.

5912

60```

62+### `;<` "micgal"

63+

64+`[%mcgl p=spec q=hoon r=hoon s=hoon]`: monadic do notation.

65+

66+##### Syntax

67+

68+Regular: **4-fixed**

69+

70+```hoon

71+;< mold=mold bind=hoon expr1=hoon expr2=hoon

72+```

73+

74+#### Semantics

75+

76+A `;<` is for sequencing two computations, `expr1` and `expr2`, using a provided

77+implementation of monadic bind. This rune takes a gate `bind` which takes a mold

78+`mold` and produces an implementation of monadic bind.

79+

80+##### Desugaring

81+

82+```hoon

83+%+ (bind mold)

84+ expr1

85+|= mold

86+expr2

87+```

88+

89+##### Discussion

90+

91+`;<` is much like Haskell `do` notation. You have a sequence of events you'd

92+like to run such that each past the first depends on the output of the previous

93+one. The output of the previous one may not be of the correct type to use as an

94+input to the next one, and so an adapter such as `+biff` is needed.

95+

96+`;<` differs from [`;~`](#micsig) in that it takes a gate which takes a mold

97+that produces an implementation of monadic bind, rather than taking an

98+implementation of monadic bind directly.

99+

100+`;<` can be used to glue a pipeline together to run an asynchronous function or

101+event. This can be helpful when deferring parts of a computation based on

102+external data.

103+

104+We remark that you can switch binds in the middle of a sequence of `;<`.

105+

106+##### Examples

107+

108+[`+biff`](/docs/hoon/reference/stdlib/2a/#biff) is the unit monad's

109+implementation of monadic bind. That is to say, it takes a unit `a` and a gate

110+`b` that accepts a noun that produces a unit, and extracts the value from `a` to

111+pass as a sample to `b`.

112+

113+We illustrate the usage of `;<` with `+biff` with a `map` of atoms:

114+```

115+~zod:dojo> =m (my ~[[1 3] [2 2] [3 1]])

116+ > (~(get by m) 1)

117+ [~ 3]

118+```

119+A single usage of `;<` only serves to apply the binding function to the output

120+of `expr1`:

121+```

122+~zod:dojo> ;< a=@ _biff (~(get by m) 1)

123+ a

124+ 3

125+```

126+Here we see the result of chaining them together:

127+```

128+~zod:dojo> ;< a=@ _biff (~(get by m) 1)

129+ ;< b=@ _biff (~(get by m) a)

130+ b

131+ 1

132+```

133+

134+

135+

62### `;+` "miclus"

136### `;+` "miclus"

137

64make a single XML node (Sail)

138make a single XML node (Sail)

hoon/reference/stdlib/1a.md+34−34

@@ -12,13 +12,13 @@ Produces the sum of `a` and `b`.

13#### Accepts

15−`a` is an atom.

15+`a` is an [`atom`](@/docs/glossary/atom.md).

17−`b` is an atom.

17+`b` is an [`atom`](@/docs/glossary/atom.md).

19#### Produces

21−An atom.

21+An [`atom`](@/docs/glossary/atom.md).

23#### Source

@@ -65,11 +65,11 @@ Decrements `a` by `1`.

66#### Accepts

68−`a` is an atom.

68+`a` is an [`atom`](@/docs/glossary/atom.md).

70#### Produces

72−An atom.

72+An [`atom`](@/docs/glossary/atom.md).

74#### Source

@@ -116,13 +116,13 @@ Computes `a` divided by `b` without remainder.

116

117#### Accepts

118

119−`a` is an atom.

119+`a` is an [`atom`](@/docs/glossary/atom.md).

120

121−`b` is an atom.

121+`b` is an [`atom`](@/docs/glossary/atom.md).

122

123#### Produces

124

125−An atom.

125+An [`atom`](@/docs/glossary/atom.md).

126

127#### Source

128

@@ -171,13 +171,13 @@ Computes `a` divided by `b`, producing the quotient and the remainder.

171

172#### Accepts

173

174−`a` is an atom.

174+`a` is an [`atom`](@/docs/glossary/atom.md).

175

176−`b` is an atom.

176+`b` is an [`atom`](@/docs/glossary/atom.md).

177

178#### Produces

179

180−A cell of atoms.

180+A cell of [`atoms`](@/docs/glossary/atom.md).

181

182#### Source

183

@@ -221,9 +221,9 @@ Tests whether `a` is greater than or equal to `b`.

221

222#### Accepts

223

224−`a` is an atom.

224+`a` is an [`atom`](@/docs/glossary/atom.md).

225

226−`b` is an atom.

226+`b` is an [`atom`](@/docs/glossary/atom.md).

227

228#### Produces

229

@@ -261,9 +261,9 @@ Tests whether `a` is greater than `b`.

261

262#### Accepts

263

264−`a` is an atom.

264+`a` is an [`atom`](@/docs/glossary/atom.md).

265

266−`b` is an atom.

266+`b` is an [`atom`](@/docs/glossary/atom.md).

267

268#### Produces

269

@@ -302,9 +302,9 @@ Tests whether `a` is less than or equal to `b`.

302

303#### Accepts

304

305−`a` is an atom.

305+`a` is an [`atom`](@/docs/glossary/atom.md).

306

307−`b` is an atom.

307+`b` is an [`atom`](@/docs/glossary/atom.md).

308

309#### Produces

310

@@ -359,9 +359,9 @@ Tests whether `a` is less than `b`.

359

360#### Accepts

361

362−`a` is an atom.

362+`a` is an [`atom`](@/docs/glossary/atom.md).

363

364−`b` is an atom.

364+`b` is an [`atom`](@/docs/glossary/atom.md).

365

366#### Produces

367

@@ -417,13 +417,13 @@ Computes the greater of `a` and `b`.

417

418#### Accepts

419

420−`a` is an atom.

420+`a` is an [`atom`](@/docs/glossary/atom.md).

421

422−`b` is an atom.

422+`b` is an [`atom`](@/docs/glossary/atom.md).

423

424#### Produces

425

426−An atom.

426+An [`atom`](@/docs/glossary/atom.md).

427

428#### Source

429

@@ -473,13 +473,13 @@ Computes the lesser of `a` and `b`.

473

474#### Accepts

475

476−`a` is an atom.

476+`a` is an [`atom`](@/docs/glossary/atom.md).

477

478−`b` is an atom.

478+`b` is an [`atom`](@/docs/glossary/atom.md).

479

480#### Produces

481

482−An atom.

482+An [`atom`](@/docs/glossary/atom.md).

483

484#### Source

485

@@ -527,13 +527,13 @@ Computes the remainder of dividing `a` by `b`.

527

528#### Accepts

529

530−`a` is an atom.

530+`a` is an [`atom`](@/docs/glossary/atom.md).

531

532−`b` is an atom.

532+`b` is an [`atom`](@/docs/glossary/atom.md).

533

534#### Produces

535

536−An atom.

536+An [`atom`](@/docs/glossary/atom.md).

537

538#### Source

539

@@ -578,13 +578,13 @@ Multiplies `a` by `b`.

578

579#### Accepts

580

581−`a` is an atom.

581+`a` is an [`atom`](@/docs/glossary/atom.md).

582

583−`b` is an atom.

583+`b` is an [`atom`](@/docs/glossary/atom.md).

584

585#### Produces

586

587−An atom.

587+An [`atom`](@/docs/glossary/atom.md).

588

589#### Source

590

@@ -629,13 +629,13 @@ Subtracts `b` from `a`.

629

630#### Accepts

631

632−`a` is an atom.

632+`a` is an [`atom`](@/docs/glossary/atom.md).

633

634−`b` is an atom.

634+`b` is an [`atom`](@/docs/glossary/atom.md).

635

636#### Produces

637

638−An atom.

638+An [`atom`](@/docs/glossary/atom.md).

639

640#### Source

641

hoon/reference/stdlib/1b.md+14−14

@@ -11,18 +11,18 @@ information on the tree-addressing system.

12Tree head

14−Tests whether the tree address `a` is in the head or the tail of a noun.

14+Tests whether the tree address `a` is in the head or the tail of a [`noun`](@/docs/glossary/noun.md).

15−Produces the constant atom `%2` if it is within the head (subtree `+2`), or

15+Produces the constant [`atom`](@/docs/glossary/atom.md) `%2` if it is within the head (subtree `+2`), or

16−the constant atom `%3` if it is within the tail (subtree `+3`).

16+the constant [`atom`](@/docs/glossary/atom.md) `%3` if it is within the tail (subtree `+3`).

19#### Accepts

21−`a` is an atom.

21+`a` is an [`atom`](@/docs/glossary/atom.md).

23#### Produces

25−A constant atom.

25+A constant [`atom`](@/docs/glossary/atom.md).

27#### Source

@@ -70,16 +70,16 @@ A constant atom.

71Address within head/tail

73−Computes the tree address of atom `a` within either the head (`+2`) or tail

73+Computes the tree address of [`atom`](@/docs/glossary/atom.md) `a` within either the head (`+2`) or tail

74−(`+3`) of a noun.

74+(`+3`) of a [`noun`](@/docs/glossary/noun.md).

76#### Accepts

78−`a` is an atom.

78+`a` is an [`atom`](@/docs/glossary/atom.md).

80#### Produces

82−An atom.

82+An [`atom`](@/docs/glossary/atom.md).

84#### Source

@@ -144,7 +144,7 @@ An atom.

144 (continues...)

145```

146

147−Running `(mas 7)` in the Dojo will return `3`, because address `+3` is what

147+Running `(mas 7)` in the [`Dojo`](@/docs/glossary/dojo.md) will return `3`, because address `+3` is what

148`+7` now occupies. The tree below helps illustrate the relationship. With

149parentheses are `a` values (if `a` is in subtree `+3`), and without parentheses

150are the values returned with `(mas a)`.

@@ -175,13 +175,13 @@ Computes the absolute address of `b`, a relative address within the subtree

175

176#### Accepts

177

178−`a` is an atom.

178+`a` is an [`atom`](@/docs/glossary/atom.md).

179

180−`b` is an atom.

180+`b` is an [`atom`](@/docs/glossary/atom.md).

181

182#### Produces

183

184−An atom.

184+An [`atom`](@/docs/glossary/atom.md).

185

186#### Source

187

@@ -232,7 +232,7 @@ right (starting with root `+1`, head `+2`, and tail `+3`). Relative address

232`b` is found with respect to `a`, and then its absolute address, within the

233greater tree, is returned.

234

235−Running `(peg 3 4)` in the Dojo, for example, will return `12`. Looking at

235+Running `(peg 3 4)` in the [`Dojo`](@/docs/glossary/dojo.md), for example, will return `12`. Looking at

236a tree diagram makes it easy to see why.

237

238```

hoon/reference/stdlib/1c.md+51−13

@@ -4,11 +4,49 @@ weight = 3

4template = "doc.html"

5+++

7+### `$bite`

9+Atom slice specifier.

10+

11+Either a single atom representing block size, or a cell containing a block size

12+and an atom representing the number of blocks, called `step`. If `step` is not

13+supplied, it defaults to a value of 1.

14+

15+#### Source

16+

17+```hoon

18+ +$ bite $@(bloq [=bloq =step])

19+```

20+

21+#### Examples

22+

23+```

24+ :: ++end gets the least significant bits of an atom.

25+

26+ > (end [1 1] 255)

27+ 3

28+

29+ > (end 1 255)

30+ 3

31+

32+ > (end 3 255)

33+ 255

34+

35+ > (end 3 65.535)

36+ 255

37+

38+ > (end [3 2] 65.535)

39+ 65.535

40+

41+ > (end [3 0] 65.535)

42+ 0

43+```

44+

7−### `++bloq`

45+### `$bloq`

9Blocksize

47Blocksize

11−Atom representing block size. A block of size `a` has a bitwidth of `2^a`.

49+[`Atom`](@/docs/glossary/atom.md) representing block size. A block of size `a` has a bitwidth of `2^a`.

13#### Source

51#### Source

@@ -97,7 +135,7 @@ A mold generator. Produces a discriminated fork between two types, defaulting to

135

98Function

136Function

137

100−A core with one arm, `$`--the empty name--which transforms a sample noun into a

138+A [`core`](@/docs/glossary/core.md) with one arm, `$`--the empty name--which transforms a sample noun into a

101product noun. If used dryly as a type, the subject must have a sample type of

139product noun. If used dryly as a type, the subject must have a sample type of

102`*`.

140`*`.

103

141

@@ -130,7 +168,7 @@ A `gate` is analogous to a function in other programming languages. We created

130this new jargon because other constructs in Urbit are "functions" in the

168this new jargon because other constructs in Urbit are "functions" in the

131mathematical sense. Any `gate` normalizes to an iron `gate`.

169mathematical sense. Any `gate` normalizes to an iron `gate`.

132

170

133−See also: [`++lift`](../2a), [`++cork`](../2n)

171+See also: [`++lift`](@/docs/hoon/reference/stdlib/2a.md#lift), [`++cork`](@/docs/hoon/reference/stdlib/2n.md#cork)

134

172

135---

173---

136### `++list`

174### `++list`

@@ -166,7 +204,7 @@ homogenous type.

166```

204```

167#### Discussion

205#### Discussion

168

206

169−See also: [`++turn`](../2b), [`++snag`](../2b)

207+See also: [`++turn`](@/docs/hoon/reference/stdlib/2b.md#turn), [`++snag`](@/docs/hoon/reference/stdlib/2b.md#snag)

170

208

171---

209---

172### `++lone`

210### `++lone`

@@ -229,9 +267,9 @@ A mold generator. Produces a tuple of two of the types passed in.

229---

267---

230### `++pole`

268### `++pole`

231

269

232−Faceless list

270+Faceless [`list`](@/docs/hoon/reference/stdlib/1c.md#list)

233

271

234−A mold generator. A `list` without the faces `i` and `t`.

272+A mold generator. A [`list`](@/docs/hoon/reference/stdlib/1c.md#list) without the faces `i` and `t`.

235

273

236#### Source

274#### Source

237

275

@@ -280,9 +318,9 @@ A mold generator. Produces a tuple of four of the types passed in.

280---

318---

281### `++quip`

319### `++quip`

282

320

283−Mold of pair of `list` and type

321+Mold of pair of [`list`](@/docs/hoon/reference/stdlib/1c.md#list) and type

284

322

285−A mold generator. Produces a tuple of a `list` of `a` and the mold of `b`.

323+A mold generator. Produces a tuple of a [`list`](@/docs/hoon/reference/stdlib/1c.md#list) of `a` and the mold of `b`.

286

324

287#### Source

325#### Source

288

326

@@ -309,9 +347,9 @@ new state. You'll often see `quip` used in Gall apps.

309---

347---

310### `++trap`

348### `++trap`

311

349

312−Core with one arm `$`

350+[`Core`](@/docs/glossary/core.md) with one arm `$`

313

351

314−A trap is a core with one arm `$`.

352+A trap is a [`core`](@/docs/glossary/core.md) with one arm `$`.

315

353

316#### Source

354#### Source

317

355

@@ -416,6 +454,6 @@ type that was passed in.

416

454

417Using a `unit` allows you to specify something that may not be there.

455Using a `unit` allows you to specify something that may not be there.

418

456

419−See also: [`++bind`](../2b)

457+See also: [`++bind`](@/docs/hoon/reference/stdlib/2a.md#bind)

420

458

421−---

459+---

hoon/reference/stdlib/2a.md+50−50

@@ -6,20 +6,20 @@ template = "doc.html"

7### `++biff`

9−Unit as argument

9+[`Unit`](@/docs/hoon/reference/stdlib/1c.md#unit) as argument

11−Applies a function `b` that produces a unit to the unwrapped value of ++unit

11+Applies a function `b` that produces a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) to the unwrapped value of [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)

12`a` (`u.a`). If `a` is empty, `~` is produced.

14#### Accepts

16−`a` is a unit.

16+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

18−`b` is a function that accepts a noun and produces a unit.

18+`b` is a function that accepts a [`noun`](@/docs/glossary/noun.md) and produces a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

20#### Produces

22−A unit.

22+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

24#### Source

@@ -44,21 +44,21 @@ A unit.

44---

45### `++bind`

47−Non-unit function to unit, producing unit

47+Non-unit function to [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit), producing [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)

49−Applies a function `b` to the value (`u.a`) of a ++unit `a`, producing

49+Applies a function `b` to the value (`u.a`) of a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `a`, producing

50−a unit. Used when you want a function that does not accept or produce a

50+a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit). Used when you want a function that does not accept or produce a

51−unit to both accept and produce a unit.

51+[`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) to both accept and produce a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

53#### Accepts

55−`a` is a unit.

55+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

57`b` is a function.

59#### Produces

61−A unit.

61+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

63#### Source

@@ -87,18 +87,18 @@ A unit.

88Replace null

90−Replaces an empty ++unit `b` with the product of a called trap

90+Replaces an empty [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `b` with the product of a called [`trap`](@/docs/glossary/trap.md)

91−`a`. If the unit is not empty, then the original unit is produced.

91+`a`. If the [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) is not empty, then the original [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) is produced.

93#### Accepts

95−`a` is a trap.

95+`a` is a [`trap`](@/docs/glossary/trap.md).

97−`b` is a unit.

97+`b` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

99#### Produces

100

101−Either the product of `a` or the value inside of unit `b`.

101+Either the product of `a` or the value inside of [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `b`.

102

103#### Source

104

@@ -126,21 +126,21 @@ Either the product of `a` or the value inside of unit `b`.

126---

127### `++both`

128

129−Group unit values into pair

129+Group [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) values into pair

130

131Produces `~` if either `a` or `b` are empty. Otherwise, produces a

132−++unit whose value is a cell of the values of two input units `a` and

132+[`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) whose value is a cell of the values of two input [`units`](@/docs/hoon/reference/stdlib/1c.md#unit) `a` and

133`b`.

134

135#### Accepts

136

137−`a` is a unit.

137+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

138

139−`b` is a unit.

139+`b` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

140

141#### Produces

142

143−A unit of the two initial values.

143+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) of the two initial values.

144

145#### Source

146

@@ -165,22 +165,22 @@ A unit of the two initial values.

165---

166### `++clap`

167

168−Apply function to two units

168+Apply function to two [`units`](@/docs/hoon/reference/stdlib/1c.md#unit)

169

170Applies a binary function `c`--which does not usually accept or produce a

171−`++unit`-- to the values of two units, `a` and `b`, producing a unit.

171+[`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)-- to the values of two [`units`](@/docs/hoon/reference/stdlib/1c.md#unit), `a` and `b`, producing a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

172

173#### Accepts

174

175−`a` is a unit.

175+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

176

177−`b` is a unit.

177+`b` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

178

179`c` is a function that performs a binary operation.

180

181#### Produces

182

183−A unit.

183+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

184

185#### Source

186

@@ -212,13 +212,13 @@ A unit.

212---

213### `++drop`

214

215−Unit to list

215+[`Unit`](@/docs/hoon/reference/stdlib/1c.md#unit) to list

216

217−Makes a ++list of the unwrapped value (`u.a`) of a `++unit` `a`.

217+Makes a [`++list`](@/docs/hoon/reference/stdlib/1c.md#list) of the unwrapped value (`u.a`) of a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `a`.

218

219#### Accepts

220

221−`a` is a unit.

221+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

222

223#### Produces

224

@@ -249,19 +249,19 @@ A list.

249---

250### `++fall`

251

252−Give unit a default value

252+Give [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) a default value

253

254−Produces a default value `b` for a `++unit` `a` in cases where `a` is null.

254+Produces a default value `b` for a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `a` in cases where `a` is null.

255

256#### Accepts

257

258−`a` is a unit.

258+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

259

260−`b` is a noun that's used as the default value.

260+`b` is a [`noun`](@/docs/glossary/noun.md) that's used as the default value.

261

262#### Produces

263

264−Either a noun `b` or the unwrapped value of unit `a`.

264+Either a [`noun`](@/docs/glossary/noun.md) `b` or the unwrapped value of [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) `a`.

265

266#### Source

267

@@ -285,19 +285,19 @@ Either a noun `b` or the unwrapped value of unit `a`.

285

286Curried bind

287

288−Accepts a `++gate` `a` and produces a function that accepts `++unit`

288+Accepts a `++gate` `a` and produces a function that accepts [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)

289`b` to which it applies `a`. Used when you want a function that does not accept

290−or produce a unit to both accept and produce a unit.

290+or produce a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) to both accept and produce a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

291

292#### Accepts

293

294`a` is a gate.

295

296−`b` is a unit.

296+`b` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

297

298#### Produces

299

300−A unit.

300+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

301

302#### Source

303

@@ -324,7 +324,7 @@ A unit.

324

325Choose

326

327−Accepts two units `a` and `b` whose values are expected to be

327+Accepts two [`units`](@/docs/hoon/reference/stdlib/1c.md#unit) `a` and `b` whose values are expected to be

328equivalent. If either is empty, then the value of the other is produced.

329If neither are empty, it asserts that both values are the same and

330produces that value. If the assertion fails, `++mate` crashes with

@@ -332,13 +332,13 @@ produces that value. If the assertion fails, `++mate` crashes with

332

333#### Accepts

334

335−`a` is a unit.

335+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

336

337−`b` is a unit.

337+`b` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

338

339#### Produces

340

341−A unit or crash.

341+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) or crash.

342

343#### Source

344

@@ -374,13 +374,13 @@ A unit or crash.

374---

375### `++need`

376

377−Unwrap unit

377+Unwrap [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)

378

379−Retrieve the value from a `++unit` and crash if the unit is null.

379+Retrieve the value from a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) and crash if the [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) is null.

380

381#### Accepts

382

383−`a` is a unit.

383+`a` is a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

384

385#### Produces

386

@@ -415,17 +415,17 @@ Either the unwrapped value of `a` (`u.a`), or crash.

415---

416### `++some`

417

418−Wrap value in a unit

418+Wrap value in a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit)

419

420−Takes any noun `a` and produces a `++unit` with the value set to `a`.

420+Takes any [`noun`](@/docs/glossary/noun.md) `a` and produces a [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit) with the value set to `a`.

421

422#### Accepts

423

424−`a` is a noun.

424+`a` is a [`noun`](@/docs/glossary/noun.md).

425

426#### Produces

427

428−A unit.

428+A [`unit`](@/docs/hoon/reference/stdlib/1c.md#unit).

429

430#### Source

431

@@ -446,4 +446,4 @@ A unit.

446```

447

448

449−---

449+---

hoon/reference/stdlib/2c.md+165−52

@@ -228,16 +228,16 @@ An atom.

228

229Tail

230

231−Produces an atom by taking the last `b` blocks of size `a` from `c`.

231+Produces an atom by taking the last `step` blocks of size `bloq` from `b`.

232

233#### Accepts

234

235+`a` is an atom slice specifier (see [`++bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

235−`a` is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md)).

236+which is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

237+with optional block count.

236

238

237`b` is an atom.

239`b` is an atom.

238

240

239−`c` is an atom.

240−

241#### Produces

242

243An atom.

@@ -248,44 +248,48 @@ An atom.

248```hoon

249 ++ end

250 ~/ %end

251− |= [a=bloq b=@u c=@]

251+ |= [a=bite b=@]

252+ =/ [=bloq =step] ?^(a a [a *step])

252− (mod c (bex (mul (bex a) b)))

253+ (mod b (bex (mul (bex bloq) step)))

253```

254```

254

255

255#### Examples

256#### Examples

256

257

257```

258```

258− > (end 2 2 255)

259+ > (end [2 2] 255)

260+ 255

261+

262+ > (end [3 1] 255)

259 255

263 255

260

264

261− > (end 3 1 255)

265+ > (end 3 255)

262 255

266 255

263

267

264− > (end 3 1 256)

268+ > (end 3 256)

265 0

269 0

266

270

267 > `@ub`12 :: @ub is the binary aura

271 > `@ub`12 :: @ub is the binary aura

268 0b1100

272 0b1100

269

273

270− > `@ub`(end 0 3 12)

274+ > `@ub`(end [0 3] 12)

271 0b100

275 0b100

272

276

273− > (end 0 3 12)

277+ > (end [0 3] 12)

274 4

278 4

275

279

276− > `@ub`(end 1 3 12)

280+ > `@ub`(end [1 3] 12)

277 0b1100

281 0b1100

278

282

279− > (end 1 3 12)

283+ > (end [1 3] 12)

280 12

284 12

281

285

282 > `@ux`'abc' :: @ux is the hexademical aura

286 > `@ux`'abc' :: @ux is the hexademical aura

283 0x63.6261

287 0x63.6261

284

288

285− > `@ux`(end 3 2 'abc')

289+ > `@ux`(end [3 2] 'abc')

286 0x6261

290 0x6261

287

291

288− > `@t`(end 3 2 'abc') :: @t is the cord aura

292+ > `@t`(end [3 2] 'abc') :: @t is the cord aura

289 'ab'

293 'ab'

290```

294```

291

295

@@ -347,16 +351,16 @@ An atom.

347

351

348Left-shift

352Left-shift

349

353

350−Produces an atom by left-shifting `c` by `b` blocks of size `a`.

354+Produces an atom by left-shifting `b` by `step` blocks of size `bloq`.

351

355

352#### Accepts

356#### Accepts

353

357

358+`a` is an atom slice specifier (see [`++bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

354−`a` is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md)).

359+which is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

360+with optional block count.

355

361

356`b` is an atom.

362`b` is an atom.

357

363

358−`c` is an atom.

359−

360#### Produces

364#### Produces

361

365

362An atom.

366An atom.

@@ -366,8 +370,9 @@ An atom.

366```hoon

370```hoon

367 ++ lsh

371 ++ lsh

368 ~/ %lsh

372 ~/ %lsh

369− |= [a=bloq b=@u c=@]

373+ |= [a=bite b=@]

374+ =/ [=bloq =step] ?^(a a [a *step])

370− (mul (bex (mul (bex a) b)) c)

375+ (mul b (bex (mul (bex bloq) step)))

371```

376```

372

377

373#### Examples

378#### Examples

@@ -376,19 +381,22 @@ An atom.

376 > `@ub`1 :: @ub is the binary aura

381 > `@ub`1 :: @ub is the binary aura

377 0b1

382 0b1

378

383

379− > `@ub`(lsh 0 1 1)

384+ > `@ub`(lsh [0 1] 1)

380 0b10

385 0b10

381

386

382− > (lsh 0 1 1)

387+ > (lsh [0 1] 1)

388+ 2

389+

390+ > (lsh 0 1)

383 2

391 2

384

392

385 > `@ub`255

393 > `@ub`255

386 0b1111.1111

394 0b1111.1111

387

395

388− > `@ub`(lsh 3 1 255)

396+ > `@ub`(lsh [3 1] 255)

389 0b1111.1111.0000.0000

397 0b1111.1111.0000.0000

390

398

391− > (lsh 3 1 255)

399+ > (lsh [3 1] 255)

392 65.280

400 65.280

393```

401```

394

402

@@ -509,7 +517,9 @@ Produces an atom by assembling a list of atoms `b` using block size `a`.

509

517

510#### Accepts

518#### Accepts

511

519

520+`a` is an atom slice specifier (see [`$bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

512−`a` is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md)).

521+which is a block size (see [`$bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

522+with optional block count.

513

523

514`b` is a list of atoms.

524`b` is a list of atoms.

515

525

@@ -522,12 +532,13 @@ An atom.

522```hoon

532```hoon

523 ++ rep

533 ++ rep

524 ~/ %rep

534 ~/ %rep

525− |= [a=bloq b=(list @)]

535+ |= [a=bite b=(list @)]

536+ =/ [=bloq =step] ?^(a a [a *step])

537+ =| i=@ud

526− ^- @

538+ |- ^- @

527− =+ c=0

528− |-

529− ?~ b 0

539+ ?~ b 0

530− (add (lsh a c (end a 1 i.b)) $(c +(c), b t.b))

540+ %+ add $(i +(i), b t.b)

541+ (lsh [bloq (mul step i)] (end [bloq step] i.b))

531```

542```

532

543

533#### Examples

544#### Examples

@@ -548,8 +559,11 @@ An atom.

548 > `@ub`8

559 > `@ub`8

549 0b1000

560 0b1000

550

561

551− > (rep 0 [0 1 0 0 ~])

562+ > `@ub`(rep 0 [1 0 1 0 ~])

552− 2

563+ 0b101

564+

565+ > `@ub`(rep 0 [1 2 3 4 ~])

566+ 0b101

553

567

554 > (rep 0 [0 1 0 1 ~])

568 > (rep 0 [0 1 0 1 ~])

555 10

569 10

@@ -562,6 +576,12 @@ An atom.

562

576

563 > `@ub`(rep 3 [12 166 8 34 ~])

577 > `@ub`(rep 3 [12 166 8 34 ~])

564 0b10.0010.0000.1000.1010.0110.0000.1100

578 0b10.0010.0000.1000.1010.0110.0000.1100

579+

580+ > `*`"abcd"

581+ [97 98 99 100 0]

582+

583+ > `@t`(rep 3 "abcd") :: @t is the text aura

584+ 'abcd'

565```

585```

566

586

567

587

@@ -574,7 +594,9 @@ Produces a list of atoms from the bits of `b` using block size `a`.

574

594

575#### Accepts

595#### Accepts

576

596

597+`a` is an atom slice specifier (see [`$bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

577−`a` is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md)).

598+which is a block size (see [`$bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

599+with optional block count.

578

600

579`b` is an atom.

601`b` is an atom.

580

602

@@ -587,10 +609,10 @@ A list of atoms.

587```hoon

609```hoon

588 ++ rip

610 ++ rip

589 ~/ %rip

611 ~/ %rip

590− |= [a=bloq b=@]

612+ |= [a=bite b=@]

591 ^- (list @)

613 ^- (list @)

592 ?: =(0 b) ~

614 ?: =(0 b) ~

593− [(end a 1 b) $(b (rsh a 1 b))]

615+ [(end a b) $(b (rsh a b))]

594```

616```

595

617

596#### Examples

618#### Examples

@@ -622,6 +644,9 @@ A list of atoms.

622

644

623 > (rip 3 256)

645 > (rip 3 256)

624 ~[0 1]

646 ~[0 1]

647+

648+ > `tape`(rip 3 'abcd')

649+ "abcd"

625```

650```

626

651

627

652

@@ -630,16 +655,16 @@ A list of atoms.

630

655

631Right-shift

656Right-shift

632

657

633−Right-shifts `c` by `b` blocks of size `a`, producing an atom.

658+Right-shifts `b` by `step` blocks of size `bloq`, producing an atom.

634

659

635#### Accepts

660#### Accepts

636

661

662+`a` is an atom slice specifier (see [`$bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

637−`a` is a block size (see [`++bloq`](@/docs/hoon/reference/stdlib/1c.md)).

663+which is a block size (see [`$bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

664+with optional block count.

638

665

639`b` is an atom.

666`b` is an atom.

640

667

641−`c` is an atom.

642−

643#### Produces

668#### Produces

644

669

645An atom.

670An atom.

@@ -649,8 +674,9 @@ An atom.

649```hoon

674```hoon

650 ++ rsh

675 ++ rsh

651 ~/ %rsh

676 ~/ %rsh

652− |= [a=bloq b=@u c=@]

677+ |= [a=bite b=@]

678+ =/ [=bloq =step] ?^(a a [a *step])

653− (div c (bex (mul (bex a) b)))

679+ (div b (bex (mul (bex bloq) step)))

654```

680```

655

681

656#### Examples

682#### Examples

@@ -659,38 +685,125 @@ An atom.

659 > `@ub`145 :: @ub is the binary aura

685 > `@ub`145 :: @ub is the binary aura

660 0b1001.0001

686 0b1001.0001

661

687

662− > `@ub`(rsh 1 1 145)

688+ > `@ub`(rsh [1 1] 145)

663 0b10.0100

689 0b10.0100

664

690

665− > (rsh 1 1 145)

691+ > (rsh [1 1] 145)

666 36

692 36

667

693

694+ > (rsh 1 145)

695+ 36

696+

668− > `@ub`(rsh 2 1 145)

697+ > `@ub`(rsh [2 1] 145)

669 0b1001

698 0b1001

670

699

671− > (rsh 2 1 145)

700+ > (rsh [2 1] 145)

672 9

701 9

673

702

674 > `@ub`10

703 > `@ub`10

675 0b1010

704 0b1010

676

705

677− > `@ub`(rsh 0 1 10)

706+ > `@ub`(rsh [0 1] 10)

678 0b101

707 0b101

679

708

680− > (rsh 0 1 10)

709+ > (rsh [0 1] 10)

681 5

710 5

682

711

683 > `@ux`'abc'

712 > `@ux`'abc'

684 0x63.6261

713 0x63.6261

685

714

686− > `@t`(rsh 3 1 'abc')

715+ > `@t`(rsh [3 1] 'abc')

687 'bc'

716 'bc'

688

717

689− > `@ux`(rsh 3 1 'abc')

718+ > `@ux`(rsh [3 1] 'abc')

690 0x6362

719 0x6362

691```

720```

692

721

693

722

723+---

724+### `++run`

725+

726+`++turn` into atom.

727+

728+Disassembles atom `b` into slices specified by `a`, applies `c` to each slice,

729+and reassembles the results back into an atom.

730+

731+#### Accepts

732+

733+`a` is an atom slice specifier (see [`$bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

734+which is a block size (see [`$bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

735+with optional block count.

736+

737+`b` is an atom.

738+

739+`c` is a gate that accepts an atom and produces an atom.

740+

741+#### Produces

742+

743+An atom.

744+

745+#### Source

746+

747+```hoon

748+ ++ run

749+ ~/ %run

750+ |= [a=bite b=@ c=$-(@ @)]

751+ (rep a (turn (rip a b) c))

752+```

753+

754+#### Examples

755+

756+```

757+ > `@ux`65.535 :: @ux is the hexadecimal aura

758+ 0xffff

759+

760+ > `@ux`(run 2 65.535 dec) :: dec is the decrement gate

761+ 0xeeee

762+```

763+

764+

765+---

766+### `++rut`

767+

768+`++turn` into list.

769+

770+Disassembles atom `b` into slices specified by `a`, applies `c` to each slice,

771+and assembles the results back into a.

772+

773+#### Accepts

774+

775+`a` is an atom slice specifier (see [`$bite`](@/docs/hoon/reference/stdlib/1c.md#bite)),

776+which is a block size (see [`$bloq`](@/docs/hoon/reference/stdlib/1c.md#bloq))

777+with optional block count.

778+

779+`b` is an atom.

780+

781+`c` is a gate that accepts an atom.

782+

783+#### Produces

784+

785+A list.

786+

787+#### Source

788+

789+```hoon

790+ ++ rut

791+ ~/ %rut

792+ |* [a=bite b=@ c=$-(@ *)]

793+ (turn (rip a b) c)

794+```

795+

796+#### Examples

797+

798+```

799+ > `@ux`65.535 :: @ux is the hexadecimal aura

800+ 0xffff

801+

802+ > `(list @ux)`(rut 2 65.535 dec) :: dec is the decrement gate

803+ ~[0xe 0xe 0xe 0xe]

804+```

805+

806+

694---

807---

695### `++swp`

808### `++swp`

696

809

@@ -1248,4 +1361,4 @@ An atom.

1248 44.464

1361 44.464

1249```

1362```

1250

1363

1251−---

1364+---

hoon/reference/stdlib/_index.md+1

@@ -3,6 +3,7 @@ title = "Standard library"

3weight = 40

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8Documentation for the standard library.

9Documentation for the standard library.

hoon/reference/stdlib/table-of-contents.md+4−2

@@ -68,9 +68,10 @@ Can be navigated [alphabetically](#alphabetical) or [by section](#by-section).

68<a class="tooltip" href='/docs/hoon/reference/stdlib/3b/#bit-rh' title="fn to half-precision float"><code>++bit:rh</code></a>

69<a class="tooltip" href='/docs/hoon/reference/stdlib/3b/#bit-rs' title="fn to single-precision float"><code>++bit:rs</code></a>

70<a class="tooltip" href='/docs/hoon/reference/stdlib/3b/#bit-rq' title="fn to quad-precision float"><code>++bit:rq</code></a>

71+<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bite' title="Atom slice specificier"><code>$bite</code></a>

71<a class="tooltip" href='/docs/hoon/reference/stdlib/4j/#bix-ab' title="Parse hex pair"><code>++bix:ab</code></a>

72<a class="tooltip" href='/docs/hoon/reference/stdlib/4j/#bix-ab' title="Parse hex pair"><code>++bix:ab</code></a>

72<a class="tooltip" href='/docs/hoon/reference/stdlib/4l/#bisk-so' title="Parse aura-atom pair"><code>++bisk:so</code></a>

73<a class="tooltip" href='/docs/hoon/reference/stdlib/4l/#bisk-so' title="Parse aura-atom pair"><code>++bisk:so</code></a>

73−<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bloq' title="Blocksize"><code>++bloq</code></a>

74+<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bloq' title="Blocksize"><code>$bloq</code></a>

74<a class="tooltip" href='/docs/hoon/reference/stdlib/2a/#bond' title="Replace null"><code>++bond</code></a>

75<a class="tooltip" href='/docs/hoon/reference/stdlib/2a/#bond' title="Replace null"><code>++bond</code></a>

75<a class="tooltip" href='/docs/hoon/reference/stdlib/4f/#boss' title="Parser modifier: LSB"><code>++boss</code></a>

76<a class="tooltip" href='/docs/hoon/reference/stdlib/4f/#boss' title="Parser modifier: LSB"><code>++boss</code></a>

76<a class="tooltip" href='/docs/hoon/reference/stdlib/2a/#both' title="Group unit values into pair"><code>++both</code></a>

77<a class="tooltip" href='/docs/hoon/reference/stdlib/2a/#both' title="Group unit values into pair"><code>++both</code></a>

@@ -835,7 +836,8 @@ Can be navigated [alphabetically](#alphabetical) or [by section](#by-section).

835

836

836### 1c: molds and mold builders

837### 1c: molds and mold builders

837

838

838−<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bloq' title="Blocksize"><code>++bloq</code></a>

839+<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bite' title="Atom slice specificier"><code>$bite</code></a>

840+<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#bloq' title="Blocksize"><code>$bloq</code></a>

839<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#each' title="Mold of fork between two types"><code>++each</code></a>

841<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#each' title="Mold of fork between two types"><code>++each</code></a>

840<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#gate' title="Function"><code>++gate</code></a>

842<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#gate' title="Function"><code>++gate</code></a>

841<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#list' title="Mold constructor (list)"><code>++list</code></a>

843<a class="tooltip" href='/docs/hoon/reference/stdlib/1c/#list' title="Mold constructor (list)"><code>++list</code></a>

nock/_index.md+1

@@ -4,6 +4,7 @@ weight = 500

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6aliases = ["docs/learn/nock/"]

7+insert_anchor_links = "right"

7+++

8+++

8> Last major revision of this tutorial: 2016. Minor revisions were made in October 2018 to bring it in line with the new Nock 4K spec.

9> Last major revision of this tutorial: 2016. Minor revisions were made in October 2018 to bring it in line with the new Nock 4K spec.

system-overview/_index.md+4

@@ -22,6 +22,10 @@ The Urbit stack is primarily comprised of:

22 virtual machine that mediates between each urbit and the Unix software layer.

23- [Azimuth](@/docs/system-overview/azimuth.md): the Urbit identity layer, built

24 on the Ethereum blockchain.

25+

26+Central to the operation of Urbit are cryptographic methods. We give a

27+high-level overview on the usage of cryptography in Urbit and how it is

28+implemented [here](@/docs/system-overview/cryptography.md).

26## Anatomy of a personal server

30## Anatomy of a personal server

system-overview/arvo.md+1−1

@@ -27,7 +27,7 @@ Arvo modules are also called 'vanes'. Arvo's vanes are:

27 Also includes our typed functional build system.

28- [Dill](@/docs/arvo/dill/dill.md): terminal driver.

29- [Eyre](@/docs/arvo/eyre/eyre.md): HTTP server.

30−- [Gall](@/docs/arvo/gall/gall.md): application sandbox and manager.

30+- [Gall](@/docs/userspace/gall/gall.md): application sandbox and manager.

31- [Iris](@/docs/arvo/iris/iris-api.md): HTTP client.

32- [Jael](@/docs/arvo/jael/jael-api.md): Public and private key storage.

system-overview/cryptography.mdnew file+116

@@ -0,0 +1,116 @@

1++++

2+title = "Cryptography"

3+weight = 50

4+template = "doc.html"

5++++

7+Cryptography is central to the operation of Urbit. Here we give an overview of

8+how it is utilized.

10+There are two categories of keys and five components of the system involved with

11+cryptography on Urbit. We first summarize the two categories of keys and how

12+they are utilized by each ship type, then cover how different parts of Urbit are

13+involved in cryptography.

14+

15+### Types of keys

16+

17+The two categories of keys are your Azimuth/Ethereum keys and your networking

18+keys. In both cases, these are public/private key pairs utilized for public key

19+cryptography.

20+

21+#### Azimuth keys

22+

23+Your Urbit ID exists as an ERC-721 non-fungible token on the Ethereum

24+blockchain, and as such is contained in a wallet whose private key you possess.

25+If you are are utilizing a [master

26+ticket](@/docs/azimuth/azimuth.md#master-ticket), this private key is derived

27+from a seed, which is what you use to login to

28+[Bridge](@/docs/glossary/bridge.md). Otherwise, you have generated the key by

29+some other process, of which there are too many to list here. Besides the

30+private key which unlocks your ownership wallet address, you may have a few

31+other private keys which unlock a wallet that corresponds to your ship's

32+[proxies](@/docs/glossary/proxies.md). We refer collectively to these keys as

33+your _Azimuth keys_.

34+

35+Only [planets](@/docs/glossary/planet.md), [stars](@/docs/glossary/star.md), and

36+[galaxies](@/docs/glossary/galaxy.md) have Azimuth keys.

37+[Moons](@/docs/glossary/moon.md) and [comets](@/docs/glossary/comet.md) do not,

38+as they do not exist on the Ethereum blockchain.

39+

40+It is important to note that no Azimuth keys are stored anywhere within your

41+ship's [pier](@/docs/glossary/pier.md) - Ethereum and Urbit ID are entirely

42+separate entities from Urbit itself, and so you lose access to your Azimuth

43+private keys there is no way to retrieve them somehow from your ship.

44+

45+For more information on the usage of these keys and the associated proxies, see

46+the [Azimuth documentation](@/docs/azimuth/azimuth.md).

47+

48+#### Networking keys

49+

50+All communications in Urbit over the [Ames](@/docs/glossary/ames.md) network

51+are end-to-end encrypted, and thus your ship stores its own public/private pair

52+of _networking keys_ utilized for encryption and authentication. Networking keys

53+for all ship types are stored within the ship's [Jael](@/docs/glossary/jael.md)

54+[vane](@/docs/glossary/vane.md).

55+

56+For planets, stars, and galaxies, your networking public key is configured on

57+the Ethereum blockchain using one of your Azimuth private keys - either the one

58+associated to the wallet which owns the ship, or the one which holds the

59+[management proxy](@/docs/glossary/proxies.md). This is typically accomplished

60+with Bridge. Your networking private key (in the

61+[keyfile](@/docs/glossary/keyfile.md)) is a necessary input for the initial boot

62+sequence of your ship (sometime called its `%dawn`), and this is also provided

63+by Bridge.

64+

65+Public keys for moons are not tracked on the blockchain; instead they are

66+tracked by their parent ship. The networking private key is generated by the

67+parent ship and injected into the `%dawn` sequence for the moon, and its public

68+key is stored by the parent in its Jael vane. Networking keys for moons may be

69+changed by the parent ship, but not by the moon itself.

70+

71+For comets, their 128-bit `@p` name is the hash of their networking public key, and

72+the "mining" process to generate a comet consists of guessing a private key for

73+which the associated public key for which the last two bytes of their `@p`

74+matches one of the stars on the comet sponsors list downloaded during boot.

75+Thus, comets cannot change their networking keys - to get a new private

76+networking key, a new comet must be generated. For a comet to perform an initial

77+handshake with another ship, it utilizes its networking private key stored in

78+Jael to sign an unencrypted attestation packet to verify that it is the owner of

79+the associated public key. Because of this, it is currently impossible for a

80+ship to initiate communication with a comet - the comet must always be the

81+initiator. This means that ultimately two comets cannot communicate with one

82+another unless they have somehow verified each others' public keys via some

83+other method. This is merely a technical limitation imposed by the design of the

84+system, not an intentional handicapping of comet abilities. A workaround to this

85+limitation is slated to be implemented as of May 2021.

86+

87+### System components

88+

89+[Ames](@/docs/arvo/ames/ames.md) is Arvo's networking vane. All packets sent by

90+Ames are encrypted utilizing a cryptosuite found in `zuse`. The only exception

91+to this are comet self-attestation packets utilized to transmit authentication

92+of ownership of the private networking key associated to their public key. Ames

93+is responsible for encryption, decryption, and authentication of all packets. By

94+default, this utilizes AES symmetric key encryption, whose shared private key is

95+got by elliptic curve Diffie-Hellman key exchange of the ships' networking keys.

96+

97+[Jael](@/docs/arvo/jael/jael-api.md) is primarily utilized for the safe storage

98+of private networking keys and retrieval of public networking keys utilized by

99+Ames. The Jael vane of planets, stars, and galaxies are responsible for

100+distributing the public keys of their moons (ultimately via Ames).

101+

102+`zuse` is the Hoon standard library. It contains cryptographic functions which

103+are utilized by Ames. All cryptographic primitives are

104+[jetted](@/docs/vere/jetting.md) in Vere with standard vetted implementations of

105+cryptographic libraries.

106+

107+[Vere](@/docs/vere/_index.md) is Urbit's Nock runtime system, written in C. All

108+cryptographic functions implemented in Hoon are hinted to the interpreter,

109+causing it to utilize the [jet system](@/docs/vere/jetting.md) to run standard vetted cryptographic

110+libraries.

111+

112+[Azimuth](@/docs/azimuth/_index.md) is an Ethereum-based public key

113+infrastructure utilized by Urbit. `azimuth-tracker` obtains networking public

114+keys for planets, stars, and galaxies from this store, which are then stored in

115+Jael and utilized by Ames for end-to-end encrypted communication.

116+

userspace/_index.mdnew file+30

@@ -0,0 +1,30 @@

1++++

2+title = "Userspace"

3+weight = 350

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+Urbit's userspace is managed by Arvo's Gall vane.

10+

11+## [Overview](@/docs/userspace/overview.md)

12+

13+An overview of userspace in Urbit.

14+

15+## [Gall](@/docs/userspace/gall/gall.md)

16+

17+Resources on Gall which manages userspace applications.

18+

19+## [Landscape](@/docs/userspace/landscape/_index.md)

20+

21+Resources on the flagship web client Landcape, through which most users interact with their ship.

22+

23+## [Threads](@/docs/userspace/threads/_index.md)

24+

25+Resources on threads - monadic functions useful for complex IO.

26+

27+## [Graph Store](@/docs/userspace/graph-store/_index.md)

28+

29+Resources on Graph Store, a non-relational database specialized for social media applications.

30+

userspace/gall/_index.mdnew file+21

@@ -0,0 +1,21 @@

1++++

2+title = "Gall"

3+weight = 40

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+Gall is the Arvo vane that manages userspace applications.

10+

11+## [Overview](@/docs/userspace/gall/gall.md)

12+

13+Overview of the Gall vane.

14+

15+## [Tutorial](@/docs/userspace/gall/tutorial.md)

16+

17+Tutorial for writing Gall agents.

18+

19+## [API Reference](@/docs/userspace/gall/gall-api.md)

20+

21+Reference information.

arvo/gall/gall-api.md → userspace/gall/gall-api.mdrenamed

The content is the same.

arvo/gall/gall.md → userspace/gall/gall.mdrenamed+2−237

@@ -5,8 +5,7 @@ template = "doc.html"

5aliases = ["/docs/learn/arvo/gall/"]

6+++

8−This document describes Gall, the Arvo userspace vane. It is split into two main

8+This document describes Gall, the Arvo userspace vane.

9−sections: Agents and [Threads](#threads).

11# Agents

10# Agents

@@ -15,7 +14,7 @@ Urbit code lives in the following basic categories:

15- Runtime (Nock interpreter, persistence engine, IO drivers, jets)

14- Runtime (Nock interpreter, persistence engine, IO drivers, jets)

16- Kernel vanes (managed by Arvo)

15- Kernel vanes (managed by Arvo)

17- Userspace agents (managed by Gall, permanent state)

16- Userspace agents (managed by Gall, permanent state)

18−- Userspace imps (managed by Spider, transient state)

17+- Userspace threads (managed by Spider, transient state)

20This lays out the framework for the third category: Agents.

19This lays out the framework for the third category: Agents.

@@ -558,237 +557,3 @@ If an error happens in `+on-poke`, the crash report goes into the

558`+on-subscription`, the crash report goes into the `%watch-ack`

557`+on-subscription`, the crash report goes into the `%watch-ack`

559response. If a crash happens in any of the other handlers, the report

558response. If a crash happens in any of the other handlers, the report

560is passed into this arm.

559is passed into this arm.

561−

562−

563−# Threads

564−

565−As discussed in [Agents](#agents), Urbit code lives in the following basic

566−categories:

567−

568−- Runtime (Nock interpreter, persistence engine, IO drivers, jets)

569−- Kernel vanes (managed by Arvo)

570−- Userspace agents (managed by Gall, permanent state)

571−- Userspace threads (managed by Spider, transient state)

572−

573−This describes the last category: Threads.

574−

575−A thread is a monadic function that takes arguments and produces a

576−result. It may perform input and output while running, so it is not a

577−pure function. It may fail.

578−

579−An agent's strength is that it's permanent and bulletproof. All state

580−transitions are defined, and each action it performs is a transaction.

581−Code upgrades preserve existing state.

582−

583−An agent's weakness is complex input and output. Since each state

584−transition must be explicitly handled, the complexity of an agent

585−explodes with the amount of IO it handles. At best, this results in

586−long and complex code; at worst, unexpected states are mishandled,

587−corrupting permanent state.

588−

589−A thread's strength is that it can easily perform complex IO operations.

590−It uses what's often called the IO monad (plus the exception monad) to

591−provide a natural framework for IO.

592−

593−A thread's weakness is that it's impermanent and may fail unexpectedly.

594−In most of its intermediate states, it expects only a small number of

595−events (usually one), so if it receives anything it didn't expect, it

596−fails. When code is upgraded, it's impossible to upgrade a running

597−thread, so it fails.

598−

599−Thus, for anything that needs to be permament, use an agent. When you

600−need to do a long or complex sequence of IO operations, reduce that to a

601−single logical IO operation by spinning it out into a thread. If you

602−only change your agent's state in response to success of the thread, an

603−IO failure will never result in partially applied state changes.

604−

605−A thread may also be run from the dojo by prefixing its name with `-`

606−and giving it any arguments it requires. If alone, any result will be

607−printed to the screen; else the output may be piped into an agent or

608−other sinks.

609−

610−## Specification

611−

612−```hoon

613−+$ thread $-(vase _*form:(strand ,vase))

614−```

615−

616−A thread is a running strand. It receives an argument as a vase and may

617−prodduce a vase of a result.

618−

619−```hoon

620−+$ tid @tasession

621−+$ start-args

622− [parent=(unit tid) use=(unit tid) file=term =vase]

623−```

624−

625−To run a thread, poke Spider with mark `%spider-start` and a value of

626−type `start-args`. `parent` is an optional parent, `use` is an optional

627−id, `file` is the filename, and `vase` is the argument. If the parent

628−is specified (usually to the thread that spawned this thread), then

629−when the parent ends, the child will end too.

630−

631−A thread may succeed or fail. You may recieve the return value or be

632−notified of failure by subscribing to Spider at `/thread-result/[tid]`,

633−where `tid` is the same one you passed in `use`. Take care to create a

634−`tid` that will be unique from any other that may be created, for

635−example with:

636−

637−```hoon

638−(scot %ta (cat 3 'my-agent_' (scot %uv (sham eny))))`

639−```

640−

641−The `%fact` that comes back on this subscription will have mark either

642−`%thread-fail` or `%thread-done`. If it failed, the content will have

643−type `[term tang]` with an explanation of the error. If it succeeded,

644−the content will be whatever the thread produced.

645−

646−```hoon

647−+$ input [=tid =cage]

648−```

649−

650−You may poke a thread by poking Spider with `%spider-input`. `tid` is

651−the id of the thread you wish to poke, and `cage` is the data to poke it

652−with.

653−

654−You may subscribe to a thread by subscribing to Spider at

655−`/thread/[tid]/path`.

656−

657−You may stop a thread by poking Spider with `%spider-stop` and data

658−`[tid ?]`, where `tid` is the thread you wish to stop and the loobean

659−decides whether to quit nicely (suppresses printfs).

660−

661−## Writing threads

662−

663−A thread is just a function from a vase of arguments to a strand that

664−produces a vase of the result. We say "just" not because it's obvious

665−yet what that means but because it means we only need to explain how to

666−write strands, and threads are just a particular case.

667−

668−The type of a strand is defined in lib/strand, but you will rarely touch

669−those types directly. Most of the time, you will use two functions

670−`pure` and `bind` combined with any of the functions defined in

671−lib/strandio.

672−

673−### Strand fundamentals

674−

675−When writing strands, you almost always want to start by specializing

676−`strand` to your return type. A strand can produce any type, but you

677−must specify up front what type it will produce. By convention, we use

678−the variable `m` for this. For example, if you are going to produce a

679−`@ud`, use:

680−

681−```hoon

682−=/ m (strand ,@ud)

683−^- form:m

684−...

685−```

686−

687−Note that `m` is a core that has three arms:

688−

689−`+form` is the mold of the strand, suitable for `^-`.

690−

691−`+pure` produces a strand that does nothing except return a value. So,

692−`(pure:(strand ,@tas) %foo)` is a strand that produces `%foo` without

693−doing any IO. Thus, the most trivial thread is:

694−

695−```hoon

696−/- spider

697−/+ strandio

698−=, strand=strand:spider

699−^- thread:spider

700−|= vase

701−=/ m (strand ,vase)

702−^- form:m

703−(pure:m *vase)

704−```

705−

706−`+bind`, used with `;<`, takes two strands and runs one after the other,

707−letting you use the result of the first strand in the second. For

708−example:

709−

710−```hoon

711−;< num=@ud bind:m (pure:m 5)

712−(pure:m +(num))

713−```

714−

715−Has the same result as:

716−

717−```hoon

718−(pure:m 6)

719−```

720−

721−### Other useful functions

722−

723−There are many useful strand functions in lib/strandio and some other

724−libraries. For example:

725−

726−`+sleep` waits for the specified time.

727−

728−`+get-time` gets the current time.

729−

730−`+poke` pokes an agent.

731−

732−`+watch` subscribes to an agent.

733−

734−`+fetch-json` produces the JSON at a particular URL.

735−

736−`+retry` tries a strand repeatedly with exponential backoff until it

737−succeeeds.

738−

739−`+start-thread` starts another thread.

740−

741−`send-raw-card` sends any card.

742−

743−## Examples

744−

745−```hoon

746−/- spider

747−/+ strandio

748−=, strand=strand:spider

749−^- thread:spider

750−|= arg=vase

751−=/ m (strand ,vase)

752−^- form:m

753−=+ !<([arg=@dr ~] arg)

754−;< now-1=@da bind:m get-time:strandio

755−;< ~ bind:m (sleep:strandio arg)

756−;< now-2=@da bind:m get-time:strandio

757−(pure:m !>(`@dr`(sub now-2 now-1)))

758−```

759−

760−The first seven lines are the same boilerplate we saw before. After

761−that, it `!<`'s the argument to its expected type. It saves the current

762−time in `now-1`, sleeps how long the argument says to, then gets the

763−time again and produces the difference between the two times. This

764−number should be close to the argument you passed in, plus a little for

765−computation time. You can run this at the dojo with `-time ~s1`.

766−

767−```hoon

768−/- spider

769−/+ *ph-io

770−=, strand=strand:spider

771−^- thread:spider

772−|= args=vase

773−=/ m (strand ,vase)

774−;< ~ bind:m start-simple

775−;< ~ bind:m (raw-ship ~bud ~)

776−;< ~ bind:m (dojo ~bud "[%test-result (add 2 3)]")

777−;< ~ bind:m (wait-for-output ~bud "[%test-result 5]")

778−;< ~ bind:m end-simple

779−(pure:m *vase)

780−```

781−

782−This is uses `lib/ph/io` to run a pH test. Aqua and pH are described

783−[elsewhere](@/docs/hoon/guides/aqua.md), but if you've initialized Aqua by running `|start %aqua` and

784−`:aqua +solid`, then you can run `-ph-add` to execute this thread.

785−

786−It starts by running some virtualization initialization with

787−`start-simple`. Then, it boots a guest ship called ~bud, types a line

788−into the dojo, and waits for it to printed the expected output.

789−

790−It's easy to see how this building blocks could be combined to perform

791−complex tests. For example, you could start several ships and have them

792−`|hi` each other, sync from each other, or do anything else. Check out

793−`lib/ph/io` and `lib/ph/azimuth` for other useful functions in constructing

794−pH tests.

arvo/gall/tutorial.md → userspace/gall/tutorial.mdrenamed

The content is the same.

userspace/graph-store/_index.mdnew file+23

@@ -0,0 +1,23 @@

1++++

2+title = "Graph Store"

3+weight = 50

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+## [Overview](@/docs/userspace/graph-store/overview.md)

10+

11+Basic concepts and architectural overview of Graph Store

12+

13+## [Data Structure Overview](@/docs/userspace/graph-store/data-structure-overview.md)

14+

15+A walkthrough of the various data structures and types used in Graph Store.

16+

17+## [Validator Walkthrough](@/docs/userspace/graph-store/validator-walkthrough.md)

18+

19+A walkthrough of existing applications built using Graph Store

20+

21+## [Advanced Information](@/docs/userspace/graph-store/advanced-info.md)

22+

23+Advanced information and selected implementation details about Graph Store.

userspace/graph-store/advanced-info.mdnew file+64

@@ -0,0 +1,64 @@

1++++

2+title = "Advanced Info"

3+weight = 50

4+sort_by = "weight"

5+template = "doc.html"

6+insert_anchor_links = "right"

7++++

9+This section is not required, but does shed light on some implicit assumptions that Graph Store makes.

10+

11+

12+### Miscellaneous

13+- Only nodes that successfully typecheck under the validator will be added to the graph

14+- Graphs are validated recursively (see [`app/graph-store.hoon#L598-L616`](https://github.com/urbit/urbit/blob/5cb6af0433a65fb28b4bc957be10cb436781392d/pkg/arvo/app/graph-store.hoon#L598-L616))

15+ - First, validate all top level nodes of a graph.

16+ - Then, recursively validate all children nodes of those nodes.

17+ - Base case: empty children is always valid

18+- Nodes are also validated against the graph's mark when inserted individually (see [`app/graph-store.hoon#L380`](https://github.com/urbit/urbit/blob/e2ad6e3e9219c8bfad62f27f05c7cac94c9effa8/pkg/arvo/app/graph-store.hoon#L380))

19+- Only the root level graphs get validated with marks. There is only one mark / validator per graph. All child graphs get validated with the same mark as the root/top-level one.

20+- The way in which Graph Store works is by mirroring all data from a given social media channel. Thus, anything you see is your copy of it, and anything you do is sent as a request to the hosting ship.

21+- Unmanaged graphs (a graph that isn't associated to a group) actually still have a unmanaged / hidden group. If you want to construct a `reference` to content in an unmanaged graph, it is still possible; the group in that case is then the same resource id as the graph itself

22+

23+

24+

25+### What happens when you add or remove a node?

26+

27+When adding nodes, Graph Store takes in a `(map index node)`. That is, a map of `node`s to add keyed by an `index` representing where in the graph the node should be inserted. However, there are some limitations. For instance, Graph Store does not allow for a `node` which has non-existent ancestors to be added, meaning that missing ancestor nodes are not automatically created. Graph Store requires that every node up until the 2nd to last level of nesting is created, meaning that every ancestor except the parent must be created before being added. The ancestors must either already exist in Graph Store or must be included in the `graph-update`. To ensure consistent behavior, all nodes are added shallowest-index first, which ensures that no child is added before it's parent (if it exists). This is why any non-leaf node must have its parent exist, either already within the graph or in the update.

28+

29+Also, Graph Store merges the flat list of nodes from an `%add-nodes` update into the fully connected graph structure. This means that when sending an update to `graph-store`, children nodes do not have to be nested within the parent's data structure in the update. They simply have to exist somewhere in the update.

30+

31+Another constraint to be aware of is that in a map of indexed nodes: for each node, the index in map entry has to be the same index as the in that `node`'s post index.

32+

33+To see how Graph Store handles add-nodes, take a look at [`app/graph-store.hoon#L370-L417`](https://github.com/urbit/urbit/blob/e2ad6e3e9219c8bfad62f27f05c7cac94c9effa8/pkg/arvo/app/graph-store.hoon#L370-L417).

34+

35+

36+### Permissions internals

37+Internally, the permissions part of a mark (the grow arm) are built and then watched for changes by Graph Store. See [`app/graph-push-hook.hoon#L154`](https://github.com/urbit/urbit/blob/ac096d85ae847fcfe8786b51039c92c69abc006e/pkg/arvo/app/graph-push-hook.hoon#L154) for more info.

38+

39+It is important to note that the push hook and permissions system is bespoke, and doesn't necessarily fit all permissions schemes out there. In addition, the permissions structure is likely to change, so it can make sense to implement your own permissioning structure (if you are not using `graph-push-hook`), especially in the case where you are making your own resource control system.

40+The `graph-push-hook` permissions system is not the definite way to implement permissions, and is simply one pre-existing way to do it.

41+The validator mark doesn't need any permissions to compile; it just needs a grow arm with at least `noun`.

42+The way to implement your own permissioning structure is in the form of your own grow arm definitions in the validator.

43+There's nothing special about the `graph-permission-add` arms; they are just constants, arms which are known to push-hook.

44+As stated before, Graph Store proper (`app/graph-store.hoon`) doesn't know anything about the permissions.

45+

46+

47+

48+

49+### Push Hook Details

50+

51+As a result of the hook architecture, all hooks have the ability to transform incoming data before it is accepted and trusted as a local input.

52+Graph Push Hook is no exception. This is why the `transform-add-nodes` is required for every validator; Graph Store calls this function before adding any nodes to itself.

53+Right now, the only thing that `transform-add-nodes` is used for is to replace untrusted foreign timestamps with trusted local timestamps.

54+

55+Here is a stubbed out example of a `transform-add-nodes` to show its inputs and outputs:

56+

57+```hoon

58+|= [=index =post =atom was-parent-modified=?]

59+^- [^index ^post]

60+!!

61+```

62+

63+The simplest transformer can simply return the exact same indexed post, which would mean that it would trust all foreign `index`es and `post`s by default.

64+The atom is always `now.bowl`, i.e. what the datetime on us, the receiving side is.

userspace/graph-store/data-structure-overview.mdnew file+338

@@ -0,0 +1,338 @@

1++++

2+title = "Data Structure Overview"

3+weight = 2

4+sort_by = "weight"

5+template = "doc.html"

6+insert_anchor_links = "right"

7++++

10+Let's go through the type definitions of some of the most used types when working with Graph Store.

11+

12+## Post

13+Here's `sur/post.hoon`

14+```hoon

15++$ index (list atom)

16++$ uid [=resource =index]

17+::

18+:: +sham (half sha-256) hash of +validated-portion

19++$ hash @ux

20+::

21++$ signature [p=@ux q=ship r=life]

22++$ signatures (set signature)

23++$ post

24+ $: author=ship

25+ =index

26+ time-sent=time

27+ contents=(list content)

28+ hash=(unit hash)

29+ =signatures

30+ ==

31+::

32++$ indexed-post [a=atom p=post]

33+::

34++$ validated-portion

35+ $: parent-hash=(unit hash)

36+ author=ship

37+ time-sent=time

38+ contents=(list content)

39+ ==

40+::

41++$ reference

42+ $% [%graph group=resource =uid]

43+ [%group group=resource]

44+ ==

45+::

46++$ content

47+ $% [%text text=cord]

48+ [%mention =ship]

49+ [%url url=cord]

50+ [%code expression=cord output=(list tank)]

51+ [%reference =reference]

52+ ==

53+--

54+```

55+

56+

57+### Index

58+```hoon

59++$ index (list atom)

60++$ uid [=resource =index]

61+```

62+

63+`index` is a list of atoms (big integers). It represents a path to a specific node on a graph that is nested arbitrarily deep.

64+

65+`uid` is simply a pair of a `resource` and an `index`. These two pieces of information combine to form an unambiguous way to identify and reference a node. It is used in the `content` type to model a reference.

66+

67+An index fragment is not an explicitly defined type, but since an index is `(list atom)`, it follows that the type of an index fragment is `atom`. This is what gives the developer the flexibility to use more than just numbers in an index: any type that is representable as an atom can be used in the index.

68+

69+### Hashing (Part 1)

70+```hoon

71+:: +sham (half sha-256) hash of +validated-portion

72++$ hash @ux

73+::

74++$ signature [p=@ux q=ship r=life]

75++$ signatures (set signature)

76+```

77+

78+These types are used to cryptographically sign a given post, so that the host of some content cannot act as an imposter, and post content impersonating as someone else. `signature` represents a triple of signed message of hash, author, and author's life at time of posting. These can be used to cryptographically attest to a message. The implementation is a form of asymmetric/public-key encryption, where `q` and `r` are data necessary to look up a ship's public key on azimuth, which can be used to verify the validity of the message.

79+

80+### Content Types

81+```hoon

82++$ reference

83+ $% [%graph group=resource =uid]

84+ [%group group=resource]

85+ ==

86+::

87++$ content

88+ $% [%text text=cord]

89+ [%mention =ship]

90+ [%url url=cord]

91+ [%code expression=cord output=(list tank)]

92+ [%reference =reference]

93+ ==

94+::

95+```

96+

97+`reference` is a type that models a reference to a graph object. Currently, the valid types of references are either a reference to a specific node in a graph, or a specific group.

98+

99+`content` enumerates all the possible content types that a post can have. The possible content types can be:

100+

101+- **Text** - representing plain text content

102+- **Url** - specific data type for urls

103+- **Mention** - mentioning another ship

104+- **Code** - a pair of a piece of code that was executed and its result (static data, no execution takes place inside of Graph Store)

105+- **Reference** - a reference to another post, using the `reference` type

106+

107+Currently, these are the only content types supported by Graph Store.

108+

109+### Post

110+```hoon

111++$ post

112+ $: author=ship

113+ =index

114+ time-sent=time

115+ contents=(list content)

116+ hash=(unit hash)

117+ =signatures

118+ ==

119+::

120++$ indexed-post [a=atom p=post]

121+```

122+

123+Post is a fundamental type that represents what we normally think of as a post on social media. Most of the rest of the types are self-explanatory. `hash` is the optional hash of the post, and `signatures` is the (potentially empty) set of `signature`s if the post is cryptographically signed.

124+

125+An `indexed-post` is a post with an associated index fragment that can be used to validate a post's index with an index fragment that is expected at the end of the index list.

126+

127+### Hashing (Part 2)

128+```hoon

129++$ validated-portion

130+ $: parent-hash=(unit hash)

131+ author=ship

132+ time-sent=time

133+ contents=(list content)

134+ ==

135+::

136+```

137+

138+The parts of a `post` that are actually hashed to obtain a value of type the earlier type `hash`.

139+

140+

141+## Graph Store

142+

143+Here's `sur/graph-store.hoon`

144+

145+```hoon

146++$ graph ((mop atom node) gth)

147++$ marked-graph [p=graph q=(unit mark)]

148+::

149++$ node [=post children=internal-graph]

150++$ graphs (map resource marked-graph)

151+::

152++$ tag-queries (jug term resource)

153+::

154++$ update-log ((mop time logged-update) gth)

155++$ update-logs (map resource update-log)

156+::

157+::

158++$ internal-graph

159+ $~ [%empty ~]

160+ $% [%graph p=graph]

161+ [%empty ~]

162+ ==

163+::

164++$ network

165+ $: =graphs

166+ =tag-queries

167+ =update-logs

168+ archive=graphs

169+ validators=(set mark)

170+ ==

171+::

172++$ update [p=time q=action]

173+::

174++$ logged-update [p=time q=logged-action]

175+

176+::

177++$ logged-action

178+ $% [%add-graph =resource =graph mark=(unit mark) overwrite=?]

179+ [%add-nodes =resource nodes=(map index node)]

180+ [%remove-nodes =resource indices=(set index)]

181+ [%add-signatures =uid =signatures]

182+ [%remove-signatures =uid =signatures]

183+ ==

184+::

185++$ action

186+ $% logged-action

187+ [%remove-graph =resource]

188+ ::

189+ [%add-tag =term =resource]

190+ [%remove-tag =term =resource]

191+ ::

192+ [%archive-graph =resource]

193+ [%unarchive-graph =resource]

194+ [%run-updates =resource =update-log]

195+ ::

196+ :: NOTE: cannot be sent as pokes

197+ ::

198+ [%keys =resources]

199+ [%tags tags=(set term)]

200+ [%tag-queries =tag-queries]

201+ ==

202+--

203++$ permissions

204+ [admin=permission-level writer=permission-level reader=permission-level]

205+::

206+:: $permission-level: levels of permissions in increasing order

207+::

208+:: %no: May not add/remove node

209+:: %self: May only nodes beneath nodes that were added by

210+:: the same pilot, may remove nodes that the pilot 'owns'

211+:: %yes: May add a node or remove node

212++$ permission-level

213+ ?(%no %self %yes)

214+::

215+```

216+

217+### Graph, Node, and Related Objects

218+```hoon

219++$ graph ((mop atom node) gth)

220++$ marked-graph [p=graph q=(unit mark)]

221+::

222++$ node [=post children=internal-graph]

223++$ graphs (map resource marked-graph)

224+::

225++$ internal-graph

226+ $~ [%empty ~]

227+ $% [%graph p=graph]

228+ [%empty ~]

229+ ==

230+::

231+

232++$ network

233+ $: =graphs

234+ =tag-queries

235+ =update-logs

236+ archive=graphs

237+ validators=(set mark)

238+ ==

239+::

240+```

241+A graph is a loosely interconnected set of data which can reference each other and be arbitrarily nested and interconnected. This concept is based off of the mathematical concept of a graph: a collection of nodes, and connections between nodes called edges.

242+

243+`graph` is a `mop`, a map which maintains ordering on its keys based on a sorting function also known as an ordered map. A graph's keys are `atom`s representing a node's index fragment and whose values are `node`s, where entries are sorted by largest valued keys first, using `gth` as the sorting function. This is the fundamental data structure used in Graph Store.

244+

245+Here are some helpful Wikipedia pages for more info on what this data type represents:

246+- [Graph (abstract data type)](https://en.wikipedia.org/wiki/Graph_(abstract_data_type))

247+- [Graph database](https://en.wikipedia.org/wiki/Graph_database#Background)

248+- [Graph (discrete mathematics)](https://en.wikipedia.org/wiki/Graph_(discrete_mathematics))

249+

250+`node` represents a pair of a `post` and all of its children, which is an `internal-graph`.

251+

252+`internal-graph` is a tagged union representing the state that children can be in. Either a `node` has children in the form of a `graph`, or does not have any and is labeled as `%empty`.

253+

254+`marked-graph` is the pair of a `graph` and an optionally present `mark`, which is used by Graph Store to validate a graph against the provided validator.

255+

256+`graphs` is a mapping between `resource`s and`marked-graph`s

257+

258+`network` is the highest level data structure used by Graph Store to represent all the information that the agent is aware of.

259+

260+

261+

262+### Tag Queries

263+

264+```hoon

265++$ tag-queries (jug term resource)

266+```

267+

268+`tag-queries` is a mapping where the keys are `term`s and the values are a `set` of `resource`s (this pattern is called a `jug`). It is a simple tagging system that allows for various ad-hoc collections, similar to filesystem tags being used to sort different files/folders. While the type's name is `tag-queries`, there is no complex querying system as of now. Currently, you can add term/resources pairs into the tag queries, get a list of all terms in tag-queries, and get the whole `jug` out of Graph Store.

269+

270+### Update (Part 1)

271+```hoon

272++$ update [p=time q=action]

273+::

274++$ logged-action

275+ $% [%add-graph =resource =graph mark=(unit mark) overwrite=?]

276+ [%add-nodes =resource nodes=(map index node)]

277+ [%remove-nodes =resource indices=(set index)]

278+ [%add-signatures =uid =signatures]

279+ [%remove-signatures =uid =signatures]

280+ ==

281+::

282++$ action

283+ $% logged-action

284+ [%remove-graph =resource]

285+ ::

286+ [%add-tag =term =resource]

287+ [%remove-tag =term =resource]

288+ ::

289+ [%archive-graph =resource]

290+ [%unarchive-graph =resource]

291+ [%run-updates =resource =update-log]

292+ ::

293+ :: NOTE: cannot be sent as pokes

294+ ::

295+ [%keys =resources]

296+ [%tags tags=(set term)]

297+ [%tag-queries =tag-queries]

298+ ==

299+::

300+```

301+

302+The `update` type is what is used to interact with Graph Store. It is used both to update subscribers with data (outgoing data) and to write to Graph Store itself (incoming data). All actions are meant to be sent as pokes to Graph Store in the form of a `action` (encapsulated by a `graph-update`), except for the last three actions ,`%keys`, `%tags`, and `%tag-queries`, which must be sent scry requests. All actions defined here allow you to create/read/update/delete various objects in a running Graph Store gall agent. An `action` encapsulates all `logged-action`; any `logged-action` is an `action`, but not necessarily the other way around.

303+

304+If you want to check out a relevant code listing to see how graph store handles these pokes, check out the `graph-update` arm in the poke handler at [`app/graph-store.hoon`](https://github.com/urbit/urbit/blob/e2ad6e3e9219c8bfad62f27f05c7cac94c9effa8/pkg/arvo/app/graph-store.hoon#L221-L227)

305+

306+### Update (Part 2)

307+```hoon

308++$ update-log ((mop time logged-update) gth)

309++$ update-logs (map resource update-log)

310+::

311+::

312++$ logged-update [p=time q=logged-action]

313+::

314+```

315+

316+`update-log` is an ordered map where the keys are a timestamp (time is an alias for `@da`, an absolute datetime) and the values are `logged-update`s, where entries are sorted with the most recent timestamp first. It represents a history of updates applied to a graph. `update-logs` is a mapping where keys are resources and values are `update-log`s. This is the data structure used by Graph Store to store the history of actions associated with all graphs that it knows about, where each graph has a unique resource that identifies it. A `logged-update` is a data structure that holds any `logged-update-0` along with a time identifying when the update happened. It follows a versioning pattern similar to the versioned state of a gall agent.

317+

318+It is important to note that the source of truth is actually the `update-logs`, and not the `graphs`. Similar to the Urbit event log, Graph Store also stores all updates that it receives, so that it can rebuild its current state on demand. The current state of the database is more of a product of the event log, like a checkpoint, or a materialized db view, and is not the source of truth. As a result, the Graph Store database is immutable in nature, where all data is preserved and deleted data is only inaccessible in the current view or checkpoint, and is still present and recoverable by replaying the log.

319+

320+The reason for having the main CRUD actions being `logged-update`s is so that Graph Store knows which order to process the log entries in when it is rebuilding its current state. The time associated with the `logged-update` is a way of specifying the canonical order to process operations. All other actions that aren't part of logged-update stand on their own and don't need a timestamp in order to properly apply them.

321+

322+### Permissions

323+```hoon

324++$ permissions

325+ [admin=permission-level writer=permission-level reader=permission-level]

326+::

327+:: $permission-level: levels of permissions in increasing order

328+::

329+:: %no: May not add/remove node

330+:: %self: May only nodes beneath nodes that were added by

331+:: the same pilot, may remove nodes that the pilot 'owns'

332+:: %yes: May add a node or remove node

333++$ permission-level

334+ ?(%no %self %yes)

335+```

336+

337+These are the types from the permissioning system explained earlier. `permissions` is just a length-3 cell of `permissions-level`s for Admins, Writers, and Readers respectively.

338+

userspace/graph-store/overview.mdnew file+109

@@ -0,0 +1,109 @@

1++++

2+title = "Graph Store Overview"

3+weight = 1

4+sort_by = "weight"

5+template = "doc.html"

6+insert_anchor_links = "right"

7++++

10+## Intro

11+Graph store is a non-relational database suitable for use in building social media applications. It is primarily designed for storing text-based content and data that is generally threaded and nested. It is not suitable as a general purpose database for highly structured or pure binary data.

12+

13+

14+ <img src="https://media.urbit.org/docs/userspace/graph-store/1_graph_store_overview.png"/>

15+

16+

17+Graph Store is mainly in charge of two things: data storage and retrieval, and schema validation. There are two related tools: Graph Push Hook, which provides permissioning support to Graph Store and acts as a proxy layer to Graph Store for outside ships to access, and Graph Pull Hook, which can be used to request Graph Store data from other ships. Importantly, Graph Store itself does not have a built-in permissions system (or rather, is permissions-agnostic), and acts in a purely trusted manner, and assumes that all inputs are trusted. However, it only allows local, authenticated connections to communicate with it, meaning that by default, no one else on the network may modify the state of your Graph Store agent. This is why Graph Push Hook exists: to mediate untrusted requests from outside ships to your instance of Graph Store, rejecting invalid ones where a ship should not be able to modify the data.

18+

19+## Data Structures

20+### Posts

21+

22+

23+ <img src="https://media.urbit.org/docs/userspace/graph-store/2_post_diagram.png"/>

24+

25+

26+Above, we can see a representation of a post on the left, along with its table form on the right.

27+A post is the most basic building block of a graph.

28+Every post is made up of an:

29+- **Author** - who created the post

30+- **Index** - the unique path/identifier of the post on the graph (more on this later)

31+- **Time Sent** - when the post was created

32+- **Contents** - data that the user intended to post

33+

34+Currently, Graph Store supports 5 data types for content that is to be stored within a graph:

35+- **Text** - plain text data

36+- **Url** - specific data type for urls

37+- **Mention** - mentioning another ship

38+- **Code** - a pair of a piece of code that was executed and its result (this is static data, and is not used to execute code on other's ships)

39+- **Reference** - a reference to another post

40+

41+

42+### Graphs and Nodes

43+

44+

45+ <img src="https://media.urbit.org/docs/userspace/graph-store/3_graph_node_diagram_a.png"/>

46+

47+

48+

49+ <img src="https://media.urbit.org/docs/userspace/graph-store/4_graph_node_diagram_b.png"/>

50+

51+

52+A graph is a flat, ordered map of nodes, where each node can have a child graph, which is itself a flat ordered map of nodes. Nodes contain a post and a child graph. In the above diagram, we can see an example of a basic graph on the top, along with the underlying structure of the data in table form underneath.

53+

54+A few vocab terms:

55+- **Root graph** refers to the outermost graph, G in this case

56+- **Top level** is a loose term usually used to refer to a node that exists in the root graph. **A** and **D** would be considered nodes at the top level, but **B** and all its descendants would not qualify.

57+- **Sibling nodes** are two nodes that reside next to each other, meaning that they reside in the same `graph`. **A** and **D** are an example of sibling nodes.

58+- We say a node is a **child** of another node if it directly resides within the child graph of that node

59+- A node is a **parent** to another node if it directly contains the node within its children. An example is the relationship between **B** and **C**: **C** is a child of **B**, while **B** is the parent of **C**

60+- Another way of saying a node is a child is by saying that node **B** is nested within node **A** Strictly speaking, **C** is not a child of **A**, but we do say that **A** is an **ancestor** of **C**, while **C** is a **descendant** of **A**.

61+- **Leaf nodes** refer to nodes that do not have children. **C** and **D** are both examples of leaf nodes

62+

63+### Index

64+

65+

66+ <img src="https://media.urbit.org/docs/userspace/graph-store/5_index_diagram.png"/>

67+

68+

69+Indexes are a way of uniquely identifying a node within a graph. You can think of `index`es as similar to file paths, although they aren't exactly the same. Roughly, a file path is a unique reference to a file or folder located in the filesystem. Similarly, an index is a unique reference to a node nested within a graph. The written syntax for a full index is very similar to file paths. It consists of every index fragment in order separated by a slash. A node's **level of nesting** refers to how deeply it is nested within the context of the root graph. The level of nesting directly corresponds to the number of items in the index. An index fragment is the atom by which a node is uniquely identified within it's graph, and roughly corresponds to a specific name of a directory along a path. In the diagrams that follow, we'll use the index fragment instead of the index to avoid repeating redundant information, but please note that internally graph-store still store the full index in every node's post.

70+

71+In the above diagram, we would say that nodes **A** and **D** are nested 1 level deep, while **B** would be at the 2nd level of nesting, and **C** would be nested 3 levels deep.

72+

73+Indexes are usually numbers and are stored as such in the database. Most commonly, they can represent:

74+- Date or time of posting

75+- A sequence of numbers starting from 1 increasing

76+- Structural/constant value: values which are associated with a specific meaning in the context of the schema of an application.

77+

78+However, there is no strict requirement for them to be numbers; they can be cords as well as other data types. As we'll see in the later sections, it is up to the app developer to decide this when creating their application.

79+

80+

81+### Structural Nodes vs. Content-Centric Nodes

82+

83+

84+ <img src="https://media.urbit.org/docs/userspace/graph-store/6_structural_vs_content_centric_diagram.png"/>

85+

86+

87+When using Graph Store, there is a notion of **structural nodes** vs. **content-centric nodes**. In the example diagram, we've color coded the different nodes based on what type of node they are. **Content-centric nodes** represent data created or consumed directly by the user. **Structural nodes**, on the other hand do not directly represent user data, and instead represent a higher level relationship between different user data. Structural nodes are used to implement the structure of the schema that is being implemented. In other words, they exist away from the eyes of the user, solely for organizational purposes.

88+

89+Note that this differentiation between content-centric vs. structural nodes is purely developer-facing, and not encoded within the actual system. Although these patterns aren't hard or fast rules, we'll see how they can be used in practice in the validator walkthrough section.

90+

91+## Validator Overview - Schema and Permissions

92+

93+Every social application has a minimum amount of information it needs to function along with the structure that the information must follow. We'll call this the application's schema. This is enforced by a **validator**. Given a schema, a validator's primary function is to encode its constraints and validate arbitrary data against it. Graph Store uses clay marks to represent validators. Validators are a special case of a mark, so the terms will be used interchangeably.

94+

95+In addition, validators can also encode structural permissions. **Structural permissions** govern who is allowed to add or remove a given node (and by extension modify its children) based on the node's characteristics.

96+

97+There are 3 different classes of users:

98+- **Admin** - An owner of a resource or someone who's been delegated similar privileges

99+- **Writer** - Someone who can create and modify their own content but cannot modify others'

100+- **Reader** - Someone who is only given permission to access but not create or modify, except in special cases

101+

102+There are two different types of privileges:

103+- **Add privileges** - permission to create a given node and create any child nodes

104+- **Remove privileges** - permission to remove a given node and remove any of its children

105+

106+There are also three different levels of access a given permission level can have:

107+- **%no** - user does not have add or remove privileges for this node

108+- **%yes** - user has add or remove privileges for this node, whether or not they authored the parent node

109+- **%self** - user has add or remove privileges for child nodes only if they authored the parent node (determined by `author` of post, i.e., they are the author of the post)

userspace/graph-store/validator-walkthrough.mdnew file+487

@@ -0,0 +1,487 @@

1++++

2+title = "Validator Walkthrough"

3+weight = 3

4+sort_by = "weight"

5+template = "doc.html"

6+insert_anchor_links = "right"

7++++

10+## Schema and Permissioning Implementation

11+In this section, we will analyze various applications and see how they implement their schema and permissions.

12+

13+

14+### Current State of Permissioning

15+Please note that as it stands today, the current permissioning system is likely subject to change. It is by no means the only manner in which to set up permissionings. For many applications, writing a small and bespoke permissioning library may be suitable as an alternative to the `graph-push-hook` permissioning scheme which is what will be covered in the following examples.

16+

17+Also, please be aware that the current set of applications use a special type known as `vip-metadata`, which stands for "variation in permission".

18+It is extra metadata attached to a post that is available to the permissioning arms that is mainly used to specify whether reader comments are enabled or disabled.

19+Here's the source if you want to explore: [`sur/metadata-store.hoon#L20-L30`](https://github.com/urbit/urbit/blob/ac096d85ae847fcfe8786b51039c92c69abc006e/pkg/arvo/sur/metadata-store.hoon#L20-L30)

20+

21+

22+### Chat

23+#### Schema

24+Here's what the schema of chat looks like:

25+

26+

27+ <img src="https://media.urbit.org/docs/userspace/graph-store/7_chat_schema_diagram.png"/>

28+

29+

30+A chat is a flat graph, where all chat messages are nodes appended to the root of the graph.

31+The graph represents a chat channel and contains all chat messages in order. A chat message is a child node of the root graph.

32+

33+

34+Here's the definition of the schema in the chat validator mark:

35+File: [`mar/graph/validator/chat.hoon`](https://github.com/urbit/urbit/blob/fbd85abf4e41d580654606a6defb764f6a97256d/pkg/arvo/mar/graph/validator/chat.hoon#L33-L40)

36+```hoon

37+++ grab

38+ |%

39+ ++ noun

40+ |= p=* :: 1

41+ =/ ip ;;(indexed-post p) :: 2

42+ ?> ?=([@ ~] index.p.ip) :: 3

43+ ip :: 4

44+ --

45+::

46+```

47+

48+Here are the steps:

49+1. Given a noun (we expect an indexed-post)

50+1. Try to coerce p to an indexed-post, crash if it doesn't cast

51+1. Assert that the index of the post of the indexed post is only a single atom, i.e., that it is only nested one level deep

52+1. Return the indexed post

53+

54+Notably, under this set of rules, there is no nesting allowed. Put another way, no node is allowed to have any children. Nodes can only be added to the root graph. Step 3 is what enforces the flat hierarchy. If someone were to manually try to submit a node with children, Graph Store would reject it, preventing them from sending an invalid chat message.

55+

56+Since the schema of the chat application is simple enough, it has no need for structural nodes at all.

57+

58+#### Permissioning

59+

60+

61+ <img src="https://media.urbit.org/docs/userspace/graph-store/8_chat_permissions_diagram.png"/>

62+

63+

64+Let's take a look at the permissions table in the diagram.

65+

66+Chat Message `[@ ~]`

67+- Add Privileges

68+ - Admins and Writers have `%yes` add permissions for all nodes at the top level, meaning that they have the ability to post chat messages, even if they did not create the chat channel

69+ - Readers have `%no` add privileges for any nodes at the root level, so they do not have the ability to post chat messages

70+- Remove Privileges

71+ - Admins and Writers have `%self` remove privileges, meaning that they may only delete chat messages that they posted, not anyone else's

72+ - Readers have `%no` remove privileges for any nodes, meaning they cannot delete any chat messages

73+

74+This follows our general intuition of how permissions for chat messages should be structured.

75+For example, it wouldn't make sense to give readers `%self`, because they do not have the ability to create nodes in the first place, so they will never be in a position to delete any nodes.

76+

77+Let's see how this permissioning system is implemented in the validator code.

78+

79+Here is the `grow` arm of [`mar/validator/chat.hoon`](https://github.com/urbit/urbit/blob/fbd85abf4e41d580654606a6defb764f6a97256d/pkg/arvo/mar/graph/validator/chat.hoon#L7-L20)

80+```hoon

81+|_ i=indexed-post :: A

82+++ grow

83+ |%

84+ ++ graph-permissions-add

85+ |= vip=vip-metadata:met :: 1

86+ ?+ index.p.i !! :: 2

87+ [@ ~] [%yes %yes %no] :: 3

88+ ==

89+ ::

90+ ++ graph-permissions-remove

91+ |= vip=vip-metadata:met :: 4

92+ ?+ index.p.i !! :: 5

93+ [@ ~] [%self %self %no] :: 6

94+ ==

95+ ::

96+ --

97+::

98+```

99+

100+In line A, we accept an `indexed-post` that is used in the rest of the `grow` arm.

101+

102+`graph-permissions-add`

103+1. Accept a noun `vip` of type `vip-metadata`

104+2. Switch on the index of the post found in `i`, crashing if no successful matches occur

105+3. If the index is nested one level deep

106+4. Return a `permissions` noun defined as: [admin: %yes, writer: %yes, reader: %no]

107+

108+

109+ `graph-permissions-remove`

110+

111+5. Accept a noun `vip` of type vip-metadata

112+6. Switch on the index, crashing if no successful matches occur

113+7. If the index is nested one level deep

114+8. Return a `permissions` noun defined as: [admin: %self, writer: %self, reader: %no]

115+

116+In this example, a switch statement is used to determine the `permissions` values based on the index of the post.

117+

118+

119+### Links

120+#### Schema

121+

122+

123+ <img src="https://media.urbit.org/docs/userspace/graph-store/9_link_schema_diagram.png"/>

124+

125+

126+The root graph represents the whole Links collection. Every Links entry is a child node of this graph. Every Links entry is made up of:

127+- The link and its description

128+- A comments section

129+

130+

131+The comments section holds all individual comment nodes, but comments are not simple leaf nodes. An individual comment is actually a structural node that acts as a revision container, storing the comment's full edit history by storing each edit as a child node. The front-end is responsible for properly displaying the latest revision of the comment.

132+

133+

134+Here's the validator, located at [`mar/graph/validator/link.hoon`](https://github.com/urbit/urbit/blob/fbd85abf4e41d580654606a6defb764f6a97256d/pkg/arvo/mar/graph/validator/link.hoon#L49-L73):

135+```hoon

136+++ grab

137+ |%

138+ ++ noun

139+ |= p=* :: 1

140+ =/ ip ;;(indexed-post p) :: 2

141+ ?+ index.p.ip ~|(index+index.p.ip !!) :: 3

142+ :: top-level link post; title and url

143+ ::

144+ [@ ~] :: 4

145+ ?> ?=([[%text @] [%url @] ~] contents.p.ip) :: 4a

146+ ip

147+ ::

148+ :: comment on link post; container structure

149+ ::

150+ [@ @ ~] :: 5

151+ ?> ?=(~ contents.p.ip) :: 5a

152+ ip

153+ ::

154+ :: comment on link post; comment text

155+ ::

156+ [@ @ @ ~] :: 6

157+ ?> ?=(^ contents.p.ip) :: 6a

158+ ip

159+ ==

160+ --

161+```

162+

163+

164+<ol>

165+ <li>Get the post as a noun</li>

166+ <li>Force cast to indexed post</li>

167+ <li>

168+ Switch on index of post, crash if no match occurs

169+ <ol type="a">

170+ <li>If the node is nested one level deep, (if the index is made up of a single atom)</li>

171+ <li>Ensure that it is a cell that has two pieces of data, whose content types are text and url</li>

172+ <li>If the node is nested two levels deep</li>

173+ <li>Ensure that it is empty, this is the structural node for holding comment revisions; it should not contain any content</li>

174+ </ol>

175+ </li>

176+ <li>

177+ If the node is nested three levels deep

178+ <ol type="a">

179+ <li>Ensure that it is a cell, this is a specific revision of a comment under a revision container</li>

180+ </ol>

181+ </li>

182+</ol>

183+

184+

185+It is important to note that you cannot directly edit the url or link afterwards, only the whole link entry itself. This is because you can only add or remove nodes to a graph, not modify them. Comments, however, can be “edited”. This is possible by using a technique called a revision container. Instead of having a single leaf node holding the comment text in its contents, a node with empty contents is created. All revisions of the comment are added as children to this note, and the frontend simply shows the most recent one only.

186+

187+#### Permissioning

188+

189+

190+ <img src="https://media.urbit.org/docs/userspace/graph-store/10_links_permissions_diagram_a.png"/>

191+

192+

193+

194+ <img src="https://media.urbit.org/docs/userspace/graph-store/11_links_permissions_diagram_b.png"/>

195+

196+

197+Let's analyze the permissions structure.

198+

199+**TODO** *stubbed out awaiting potential changes in source*

200+

201+Here's how it is implemented [(source)](https://github.com/urbit/urbit/blob/master/pkg/arvo/mar/graph/validator/link.hoon#L2-L27):

202+

203+```hoon

204+|_ i=indexed-post

205+++ grow

206+ |%

207+ ++ noun i

208+ ::

209+ ++ graph-permissions-add

210+ |= vip=vip-metadata:met :: 1

211+ =/ reader :: 2

212+ ?=(%reader-comments vip)

213+ ?+ index.p.i !! :: 3

214+ [@ ~] [%yes %yes %no] :: 3a

215+ [@ @ ~] [%yes %yes ?:(reader %yes %no)] :: 3b

216+ [@ @ @ ~] [%self %self %self] :: 3c

217+ ==

218+ ::

219+ ++ graph-permissions-remove

220+ |= vip=vip-metadata:met :: 4

221+ =/ reader :: 5

222+ ?=(%reader-comments vip)

223+ ?+ index.p.i !! :: 6

224+ [@ ~] [%yes %self %self] :: 6a

225+ [@ @ ~] [%yes %self %self] ::

226+ [@ @ @ ~] [%yes %self %self] ::

227+ ==

228+ ::

229+```

230+

231+`graph-permissions-add`

232+<ol>

233+ <li>Accept a noun <code>vip</code> of type vip-metadata

234+ <ul>

235+ <li>Declare a variable <code>reader</code>, a flag which is true if reader comments are enabled, false otherwise</li>

236+ </ul>

237+ </li>

238+ <li>Switch on the index of the post found in <code>i</code>, crashing if no successful matches occur</li>

239+ <li>If the index is nested one level deep, return a <code>permissions</code> noun defined as:

240+ <ul>

241+ <li>Admin - <code>%yes</code></li>

242+ <li>Writer - <code>%yes</code></li>

243+ <li>Reader: <code>%no</code></li>

244+ </ul>

245+ </li>

246+ <li>If the index is nested two levels deep, return a <code>permissions</code> noun defined as:

247+ <ul>

248+ <li>Admin - <code>%yes</code></li>

249+ <li>Writer - <code>%yes</code></li>

250+ <li>Reader: <code>%yes</code> if reader comments are enabled, else <code>%no</code></li>

251+ </ul>

252+ </li>

253+ <li>If the index is nested three levels deep, return a <code>permissions</code> noun defined as:

254+ <ul>

255+ <li>Admin - <code>%yes</code></li>

256+ <li>Writer - <code>%yes</code></li>

257+ <li>Reader: <code>%yes</code> if reader comments are enabled, else <code>%no</code></li>

258+ </ul>

259+ </li>

260+</ol>

261+

262+

263+`graph-permissions-remove`

264+<ol>

265+ <li>Accept a noun <code>vip</code> of type vip-metadata</li>

266+ <li>Declare a variable <code>reader</code>, a flag which is true if reader comments are enabled, false otherwise</li>

267+ <li>Switch on the index of the post found in <code>i</code>, crashing if no successful matches occur</li>

268+ <li>If the index is nested one level deep, two levels deep, or three levels deep, return a <code>permissions</code> noun defined as:

269+ <ul>

270+ <li>Admin - <code>%yes</code></li>

271+ <li>Writer - <code>%self</code></li>

272+ <li>Reader: <code>%self</code></li>

273+ </ul>

274+ </li>

275+</ol>

276+

277+

278+

279+### Publish

280+

281+#### Schema

282+

283+

284+ <img src="https://media.urbit.org/docs/userspace/graph-store/12_publish_schema_diagram.png"/>

285+

286+

287+Here, a notebook, which is a collection of blog posts (called notes), is represented by the root graph. All data associated with the blog post is represented by the top level node, which is the note itself along with the associated comments. One level deeper, we see two container structures. The first one is the post revision container; it holds the edit history of your blog post. Every child node of this corresponds to the actual title and text of your blog post. The second one is the comments container. This represents the comment section of your blog post. Every child node of this is not a comment, but a comment revision container, which, as before, contains the edit history of your comment.

288+

289+Here's its validator, located at [`mar/graph/validator.hoon`](https://github.com/urbit/urbit/blob/master/pkg/arvo/mar/graph/validator/publish.hoon#L56-L97)

290+```hoon

291+ ++ noun

292+ |= p=* :: 1

293+ =/ ip ;;(indexed-post p) :: 2

294+ ?+ index.p.ip !! :: 3

295+ :: top level post must have no content

296+ [@ ~] :: 4

297+ ?> ?=(~ contents.p.ip) :: 4a

298+ ip

299+ :: container for revisions

300+ ::

301+ [@ %1 ~] :: 5

302+ ?> ?=(~ contents.p.ip) :: 5a

303+ ip

304+ :: specific revision

305+ :: first content is the title

306+ :: revisions are numbered by the revision count

307+ :: starting at one

308+ [@ %1 @ ~] :: 6

309+ ?> ?=([* * *] contents.p.ip) :: 6a

310+ ?> ?=(%text -.i.contents.p.ip) :: 6b

311+ ip

312+ :: container for comments

313+ ::

314+ [@ %2 ~] :: 7

315+ ?> ?=(~ contents.p.ip) :: 7a

316+ ip

317+ :: container for comment revisions

318+ ::

319+ [@ %2 @ ~] :: 8

320+ ?> ?=(~ contents.p.ip) :: 8a

321+ ip

322+ :: specific comment revision

323+ ::

324+ [@ %2 @ @ ~] :: 9

325+ ?> ?=(^ contents.p.ip) :: 9a

326+ ip

327+ ==

328+ --

329+```

330+

331+Walkthrough

332+

333+

334+<ol>

335+ <li>Get the post as a noun</li>

336+ <li>Force cast to indexed post</li>

337+ <li>Switch on index of post, crashing (reject) if no matches found</li>

338+ <li>If the node is nested one level deep

339+ <ol type="a"><li>Ensure that its contents are empty. The top level node is a structural node containing all of the post and associated data.</li></ol>

340+ </li>

341+ <li>If it is nested 2 levels deep and contains a 1 as it's last index fragment

342+ <ol type="a"><li>Ensure that its contents are empty. This is a structural node for holding revisions to the blog post.</li></ol>

343+ </li>

344+ <li>If the node is nested three levels deep, and has a 1 as it's second index fragment

345+ <ol type="a">

346+ <li>Ensure that its contents is a list of at least 2 elements</li>

347+ <li>Ensure that the first element of contents has a content type of text</li>

348+ </ol>

349+ </li>

350+ <li>If the node is nested two levels deep, and has a 2 as it's last index fragment

351+ <ol type="a"><li>Ensure that its contents are empty. This is a structural node for holding comments.</li></ol>

352+ </li>

353+ <li>If the node is nested three levels deep, and has a 2 as it's second index fragment

354+ <ol type="a"><li>Ensure that its contents are empty. This is a structural node for holding revisions of a specific comment.</li></ol>

355+ </li>

356+ <li>If the node is nested four levels deep, and has a 2 as it's second index fragment

357+ <ol type="a"><li>Ensure that contents has type `cell`. This is a specific revision of a comment</li></ol>

358+ </li>

359+</ol>

360+

361+Items 1-3 are setting up the validator

362+Items 4-6 are enforcing the schema for the post in general

363+Items 7-9 are enforcing the schema for comments specifically

364+

365+Notably, the revision container for the blog post itself allows the post to be edited, unlike the link entry in the previous example. In addition, in step six, the reason that the validator is made this way is because the first element of the contents is interpreted as the title of the post, and the rest of the elements are interpreted as the body of the post. Otherwise, the structure is unchanged from the Links example.

366+

367+#### Permissioning

368+

369+

370+ <img src="https://media.urbit.org/docs/userspace/graph-store/13_publish_permissions_diagram_a.png"/>

371+

372+

373+

374+ <img src="https://media.urbit.org/docs/userspace/graph-store/14_publish_permissions_diagram_b.png"/>

375+

376+

377+Let's take a look at the permissioning structure for Publish.

378+

379+**TODO** *stubbed out awaiting potential changes in source*

380+

381+[(source)](https://github.com/urbit/urbit/blob/fbd85abf4e41d580654606a6defb764f6a97256d/pkg/arvo/mar/graph/validator/publish.hoon#L6-L24)

382+

383+```hoon

384+|_ i=indexed-post

385+++ grow

386+ |%

387+ ++ noun i

388+ ++ graph-permissions-add

389+ |= vip=vip-metadata:met :: 1

390+ ?+ index.p.i !! :: 2

391+ [@ ~] [%yes %yes %no] :: new note :: 2a

392+ [@ %1 @ ~] [%self %self %no] :: 2b

393+ [@ %2 @ ~] [%yes %yes ?:(?=(%reader-comments vip) %yes %no)] :: 2c

394+ [@ %2 @ @ ~] [%self %self %self] :: 2d

395+ ==

396+ ::

397+ ++ graph-permissions-remove

398+ |= vip=vip-metadata:met :: 3

399+ ?+ index.p.i !! :: 4

400+ [@ ~] [%yes %self %self] :: 4a

401+ [@ %1 @ @ ~] [%yes %self %self] ::

402+ [@ %2 @ ~] [%yes %self %self] ::

403+ [@ %2 @ @ ~] [%yes %self %self] ::

404+ ==

405+::

406+```

407+

408+`graph-permissions-add`

409+

410+ <ol>

411+ <li>Accept a noun <code>vip</code> of type vip-metadata</li>

412+ <li>Switch on the index of the post found in <code>i</code>, crashing if no successful matches occur</li>

413+ <li>If the index is nested one level deep, return a <code>permissions</code> noun defined as:

414+ <ul>

415+ <li>Admin - <code>%yes</code></li>

416+ <li>Writer - <code>%yes</code></li>

417+ <li>Reader: <code>%no</code></li>

418+ </ul>

419+ </li>

420+ <li>If the index is nested three levels deep and has a 1 as its 2nd index fragment, return a <code>permissions</code> noun defined as:

421+ <ul>

422+ <li>Admin - <code>%self</code></li>

423+ <li>Writer - <code>%self</code></li>

424+ <li>Reader: <code>%no</code></li>

425+ </ul>

426+ </li>

427+ <li>If the index is nested three levels deep and has a 2 as its 2nd index fragment, return a <code>permissions</code> noun defined as:

428+ <ul>

429+ <li>Admin - <code>%yes</code></li>

430+ <li>Writer - <code>%yes</code></li>

431+ <li>Reader: <code>%yes</code> if reader comments are enabled, else <code>%no</code></li>

432+ </ul>

433+ </li>

434+ <li>If the index is nested four levels deep and has a 2 as its 2nd index fragment, return a <code>permissions</code> noun defined as:

435+ <ul>

436+ <li>Admin - <code>%self</code></li>

437+ <li>Writer - <code>%self</code></li>

438+ <li>Reader: <code>%self</code></li>

439+ </ul>

440+ </li>

441+</ol>

442+

443+

444+`graph-permissions-remove`

445+<ol>

446+ <li>Accept a noun <code>vip</code> of type <code>vip-metadata</code></li>

447+ <li>Declare a variable <code>reader</code>, a flag which is true if reader comments are enabled, false otherwise</li>

448+ <li>Switch on the index of the post found in <code>i</code>, crashing if no successful matches occur</li>

449+ <li>If the index is nested one level deep, two levels deep, or three levels deep, return a <code>permissions</code> noun defined as:

450+ <ul>

451+ <li>Admin - <code>%yes</code></li>

452+ <li>Writer - <code>%self</code></li>

453+ <li>Reader: <code>%self</code></li>

454+ </ul>

455+ </li>

456+</ol>

457+

458+## General Patterns and Best Practices

459+### General Patterns

460+#### Schemas

461+The general pattern for enforcing a schema in a validator is to:

462+1. Cast the incoming data as an `indexed-post`

463+1. Switch on the index, matching by its depth and structure

464+1. Validate the structure of the contents of the post based on what we expect to see at that level of nesting. This logic can definitely be more involved than simply presence or absence of data.

465+

466+When designing your schema, it may help to decide first what data the user will be directly consuming or producing, then try to think of what extra information and structure is needed to support that. For instance, if you may want a user to be able to enter a comment, then later realize that you need a structural node that contains the whole comments section. One pattern that we can see is that all structural nodes tend to have empty contents. Another pattern we can see is that all leaf nodes have their contents set.

467+

468+In general, validators can be as robust and expressive as desired, because the mark system already sets proper limitations of what you can and cannot do, with the main restriction being no side effects can be produced by a mark.

469+

470+

471+#### Permissions

472+In general, you want to first have your schema finalized, then at every node ask the question: who should be able to modify (add/remove) this node, and its children if present in your schema. You may find that you don't need to set permission for every single type of node afterwards, although being thorough can help to find bugs in permissioning early on. Writing out the permissions in plain words and bullet points, sketching them out in the form of a table, then splitting it up into the code can also make the design process easier.

473+

474+### Earth/Mars Interface Details

475+There are a few general ways to talk to a Graph Store gall agent. You can either interact with the agent in the form of pokes/scries/spiders either directly from the dojo, or through HTTP requests via Eyre.

476+

477+- All graph store applications are going to have a mark, which are applied to data going in or coming out

478+- Eyre lets you choose the mark you scry for by specifying the desired format in the url while making the HTTP request

479+- In the case of data going in, you pass in a `marked-graph` which the Graph Store agent uses to validate the whole `graph` with. In the case of adding a node to a graph, the node will simply be checked for validity using the existing mark since the mark is already set on a per-graph basis

480+- In the case of data coming out, Eyre forcibly tries to convert the hoon noun into json, and silently fails if no json conversion exists. This process is handled by Graph Store under the hood because all graphs have a well-defined en-json/de-json format, so you never have to worry about making your own de/serialization arms.

481+

482+

483+Code References

484+- [`interface/src/logic/api/base.ts#L62`](https://github.com/urbit/urbit/blob/master/pkg/interface/src/logic/api/base.ts#L62)

485+- [`mar/graph/update.hoon`](https://github.com/urbit/urbit/blob/e2ad6e3e9219c8bfad62f27f05c7cac94c9effa8/pkg/arvo/mar/graph/update.hoon)

486+- [`sys/vane/eyre.hoon#L1617-L1625`](https://github.com/urbit/urbit/blob/ac096d85ae847fcfe8786b51039c92c69abc006e/pkg/arvo/sys/vane/eyre.hoon#L1617-L1625) shows how Eyre applies marks

487+

landscape/_index.md → userspace/landscape/_index.mdrenamed+3−2

@@ -3,15 +3,16 @@ title = "Landscape"

3weight = 800

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8While not technically part of Urbit, Landscape serves as a reference

9While not technically part of Urbit, Landscape serves as a reference

9implementation for how to build applications in user space.

10implementation for how to build applications in user space.

11−## [Landscape Architecture](@/docs/landscape/overview.md)

12+## [Landscape Architecture](@/docs/userspace/landscape/overview.md)

13An overview of how Landscape, our flagship client, is structured.

14An overview of how Landscape, our flagship client, is structured.

15−## [API Reference](@/docs/landscape/reference/_index.md)

16+## [API Reference](@/docs/userspace/landscape/reference/_index.md)

17API documentation about specific agents that comprise Landscape.

18API documentation about specific agents that comprise Landscape.

landscape/overview.md → userspace/landscape/overview.mdrenamed+2−2

@@ -37,7 +37,7 @@ Landscape is composed of two primary parts: its backend, which runs as a suite o

37### Backend

39The Landscape backend is implemented as a collection of long-running

40−microservices known as [Gall agents](@/docs/arvo/gall/gall.md). They can be divided into the following,

40+microservices known as [Gall agents](@/docs/userspace/gall/gall.md). They can be divided into the following,

41though this is a simplification:

43- **Graph**: handles storing messages / posts in a non-relational database

@@ -152,7 +152,7 @@ Pinterest equivalent, a more private version of Facebook, a decentralized

152Reddit, or an even more distributed version of Mastodon. You can read more about

153graph-store by [viewing the

154source](https://github.com/urbit/urbit/blob/ac096d85ae847fcfe8786b51039c92c69abc006e/pkg/arvo/app/graph-store.hoon)

155−or reading our [API Reference](@/docs/landscape/reference/graph-store.md) documentation.

155+or reading our [API Reference](@/docs/userspace/landscape/reference/graph-store.md) documentation.

156

157The `%graph-store` may be accessed from foreign ships through the use of the

158`%graph-pull-hook` and `%graph-push-hook`.

landscape/reference/_index.md → userspace/landscape/reference/_index.mdrenamed+2−1

@@ -3,8 +3,9 @@ title = "API Reference"

3weight = 100

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

6+++

7+++

8−## [Graph Store](@/docs/landscape/reference/graph-store.md)

9+## [Graph Store](@/docs/userspace/landscape/reference/graph-store.md)

10The primary data backend utilized by Landscape.

11The primary data backend utilized by Landscape.

landscape/reference/graph-store.md → userspace/landscape/reference/graph-store.mdrenamed

The content is the same.

userspace/overview.mdnew file+19

@@ -0,0 +1,19 @@

1++++

2+title = "Overview"

3+weight = 2

4+template = "doc.html"

5++++

7+Urbit's userspace is managed by Arvo's Gall vane.

9+## [Gall](@/docs/userspace/gall/gall.md)

10+

11+Resources on Gall which manages userspace applications.

12+

13+## [Landscape](@/docs/userspace/landscape/_index.md)

14+

15+Resources on the flagship web client Landcape, through which most users interact with their ship.

16+

17+## [Threads](@/docs/userspace/threads/_index.md)

18+

19+Resources on threads - monadic functions useful for complex IO.

userspace/threads/_index.mdnew file+27

@@ -0,0 +1,27 @@

1++++

2+title = "Threads"

3+weight = 50

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+## [Overview](@/docs/userspace/threads/overview.md)

10+

11+Overview of threads.

12+

13+## [Basics](@/docs/userspace/threads/basics/_index.md)

14+

15+Tutorial and explanation of the basics of writing threads.

16+

17+## [Gall](@/docs/userspace/threads/gall/_index.md)

18+

19+Tutorial and explanation of interacting with threads from agents.

20+

21+## [Examples](@/docs/userspace/threads/examples/_index.md)

22+

23+A collection of example threads and how-tos.

24+

25+## [Reference](@/docs/userspace/threads/reference.md)

26+

27+Basic reference for interacting with threads via the Gall agent Spider.

userspace/threads/basics/_index.mdnew file+29

@@ -0,0 +1,29 @@

1++++

2+title = "Basics"

3+weight = 10

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+Tutorial and explanation of the basics of writing threads.

10+

11+## [Fundamentals](@/docs/userspace/threads/basics/fundamentals.md)

12+

13+Basic explanation of threads and usage of the strand arms `form` and `pure`.

14+

15+## [Bind](@/docs/userspace/threads/basics/bind.md)

16+

17+Using micgal (`;<`) and the strand arm `bind` to chain strands together.

18+

19+## [Input](@/docs/userspace/threads/basics/input.md)

20+

21+Information on what a strand takes.

22+

23+## [Output](@/docs/userspace/threads/basics/output.md)

24+

25+Information on what a strand produces.

26+

27+## [Summary](@/docs/userspace/threads/basics/summary.md)

28+

29+Summary of this information.

userspace/threads/basics/bind.mdnew file+97

@@ -0,0 +1,97 @@

1++++

2+title = "Bind"

3+weight = 2

4+template = "doc.html"

5++++

8+Having looked at `form` and `pure`, we'll now look at the last `strand` arm `bind`. Bind is typically used in combination with micgal (`;<`).

10+## Micgal

11+

12+Micgal takes four arguments like `spec hoon hoon hoon`. Given `;< a b c d`, it composes them like `((b ,a) c |=(a d))`. So, for example, these two expressions are equivalent:

13+

14+```hoon

15+;< ~ bind:m (sleep:strandio ~s2)

16+(pure:m !>(~))

17+```

18+

19+and

20+

21+```hoon

22+((bind:m ,~) (sleep:strandio ~s2) |=(~ (pure:m !>(~))))

23+```

24+

25+Micgal exists simply for readability. The above isn't too bad, but consider this:

26+

27+```hoon

28+;< a b c

29+;< d e f

30+;< g h i

31+j

32+```

33+...as opposed to this monstrosity: `((b ,a) c |=(a ((e ,d) f |=(d ((h ,g) i |=(g j))))))`

34+

35+## bind

36+

37+Bind by itself must be specialised like `(bind:m ,<type>)` and it takes two arguments:

38+

39+- The first argument is a function that returns the `form` of a strand which produces `<type>`.

40+- The second argument is a gate whose sample is `<type>` and which returns a `form`.

41+

42+Since you'll invariably use it in conjunction with micgal, the `<type>` in `;< <type> bind:m ...` will both specialise `bind` and specify the gate's sample.

43+

44+Bind calls the first function then, if it succeeded, calls the second gate with the result of the first as its sample. If the first function failed, it will instead just return an error message and not bother calling the next gate. So it's essentially "strand A then strand B".

45+

46+Since the second gate may itself contain another `;< <type> bind:m ...`, you can see how this allows you to glue together an arbitrarily large pipeline, where subsequent gates depend on the previous ones.

47+

48+## strandio

49+

50+`/lib/strandio/hoon` contains a large collection of useful, ready-made functions for use in threads. For example:

51+

52+- `sleep` waits for the specified time.

53+- `get-time` gets the current time.

54+- `poke` pokes an agent.

55+- `watch` subscribes to an agent.

56+- `fetch-json` produces the JSON at a particular URL.

57+- `retry` tries a strand repeatedly with exponential backoff until it succeeds.

58+- `start-thread` starts another thread.

59+- `send-raw-card` sends any card.

60+

61+...and many more.

62+

63+## Putting it together

64+

65+Here's a simple thread with a couple of `strandio` functions:

66+

67+```hoon

68+/- spider

69+/+ strandio

70+=, strand=strand:spider

71+^- thread:spider

72+|= arg=vase

73+=/ m (strand ,vase)

74+^- form:m

75+;< t=@da bind:m get-time:strandio

76+;< s=ship bind:m get-our:strandio

77+(pure:m !>([s t]))

78+```

79+

80+Save it as `/ted/mythread.hoon`, `|commit` it and run it with `-mythread`. You should see something like:

81+

82+```

83+> -mythread

84+[~zod ~2021.3.8..14.52.15..bdfe]

85+```

86+

87+## Analysis

88+

89+To use `strandio` functions we've imported the library with `/+ strandio`.

90+

91+`get-time` and `get-our` get the current time & ship from the bowl in `strand-input`. We'll discuss `strand-input` in more detail later.

92+

93+Note how we've specified the face and return type of each strand like `t=@da`, etc.

94+

95+You can see how `pure` has access to the results of previous strands in the pipeline. Note how we've wrapped `pure`'s argument in a `!>` because the thread must produce a `vase`.

96+

97+Next we'll look at `strand-input` in more detail.

userspace/threads/basics/fundamentals.mdnew file+169

@@ -0,0 +1,169 @@

1++++

2+title = "Fundamentals"

3+weight = 1

4+template = "doc.html"

5++++

7+## Introduction

9+A thread is like a transient gall agent. Unlike an agent, it can end and it can fail. The primary uses for threads are:

10+

11+1. Complex IO, like making a bunch of external API calls where each call depends on the last. Doing this in an agent significantly increases its complexity and the risk of a mishandled intermediary state corrupting permanent state. If you spin the IO out into a thread, your agent only has to make one call to the thread and receive one response.

12+2. Testing - threads are very useful for writing complex tests for your agents.

13+

14+Threads are managed by the gall agent called `spider`.

15+

16+## Thread location

17+

18+Threads live in the `ted` directory of your desk:

19+

20+```

21+%

22+├──app

23+├──gen

24+├──lib

25+├──mar

26+├──sur

27+├──sys

28+└──ted <-

29+ ├──foo

30+ │ └──bar.hoon

31+ └──baz.hoon

32+```

33+

34+From the dojo, `ted/baz.hoon` can be run with `-baz`, and `ted/foo/bar.hoon` with `-foo-bar`.

35+

36+**NOTE:** When the dojo sees the `-` prefix it automatically handles creating a thread ID, composing the argument, poking the `spider` gall agent and subscribing for the result. Running a thread from another context (eg. a gall agent) requires doing these things explicitly and is outside the scope of this particular tutorial.

37+

38+## Libraries and Structures

39+

40+There are three files that matter:

41+

42+- `/sur/spider/hoon` - this contains a few simple structures used by spider. It's not terribly useful except it imports libstrand, so you'll typically get `strand` from `spider`.

43+

44+- `/lib/strand/hoon` - this contains all the main functions and structures for strands (a thread is a running strand), and you'll refer to this fairly frequently.

45+

46+- `/lib/strandio/hoon` - this contains a large collection of ready-made functions for use in threads. You'll likely use many of these when you write threads, so it's very useful.

47+

48+## Thread definition

49+

50+`/sur/spider/hoon` defines a thread as:

51+

52+```hoon

53++$ thread $-(vase _*form:(strand ,vase))

54+```

55+

56+That is, a gate which takes a `vase` and returns the `form` of a `strand` that produces a `vase`. This is a little confusing and we'll look at each part in detail later. For now, note that the thread doesn't just produce a result, it actually produces a strand that takes input and produces output from which a result can be extracted. It works something like this:

57+

58+![thread diagram](https://storage.googleapis.com/media.urbit.org/site/thread-diagram.png "diagram of a thread")

59+

60+This is because threads typically do a bunch of I/O so it can't just immediately produce a result and end. Instead the strand will get some input, produce output, get some new input, produce new output, and so forth, until they eventually produce a `%done` with the actual final result.

61+

62+## Strands

63+

64+Strands are the building blocks of threads. A thread will typically compose multiple strands.

65+

66+A strand is a function of `strand-input:strand -> output:strand` and is defined in `/lib/strand/hoon`. You can see the details of `strand-input` [here](https://github.com/urbit/urbit/blob/master/pkg/arvo/lib/strand.hoon#L2-L21) and `output:strand` [here](https://github.com/urbit/urbit/blob/master/pkg/arvo/lib/strand.hoon#L23-L48). At this stage you don't need to know the nitty-gritty but it's helpful to have a quick look through. We'll discuss these things in more detail later.

67+

68+A strand is a core that has three important arms:

69+- `form` - the mold of the strand

70+- `pure` - produces a strand that does nothing except return a value

71+- `bind` - monadic bind, like `then` in javascript promises

72+

73+We'll discuss each of these arms later.

74+

75+A strand must be specialised to produce a particular type like `(strand ,<type>)`. As previously mentioned, a `thread` produces a `vase` so is specialised like `(strand ,vase)`. Within your thread you'll likely compose multiple strands which produce different types like `(strand ,@ud)`, `(strand ,[path cage])`, etc, but the thread itself will always come back to a `(strand ,vase)`.

76+

77+Strands are conventionally given the face `m` like:

78+

79+```hoon

80+=/ m (strand ,vase)

81+...

82+```

83+

84+**NOTE:** a comma prefix as in `,vase` is the irregular form of `^:` which is a gate that returns the sample value if it's of the correct type, but crashes otherwise.

85+

86+## Form and Pure

87+

88+### `form`

89+

90+The `form` arm is the mold of the strand, suitable for casting. The two other arms produce `form`s so you'll cast everything to this like:

91+

92+```hoon

93+=/ m (strand ,@ud)

94+^- form:m

95+...

96+```

97+

98+### `pure`

99+

100+Pure produces a strand that does nothing except return a value. So, `(pure:(strand ,@tas) %foo)` is a strand that produces `%foo` without doing any IO.

101+

102+We'll cover `bind` later.

103+

104+## A trivial thread

105+

106+```hoon

107+/- spider

108+=, strand=strand:spider

109+^- thread:spider

110+|= arg=vase

111+=/ m (strand ,vase)

112+^- form:m

113+(pure:m arg)

114+```

115+

116+The above code is a simple thread that just returns its argument, and it's a good boilerplate to start from.

117+

118+Save the above code as a file in `ted/mythread.hoon` and `|commit` it. Run it with `-mythread 'foo'`, you should see the following:

119+

120+

121+```

122+> -mythread 'foo'

123+[~ 'foo']

124+```

125+

126+**NOTE:** The dojo wraps arguments in a unit so that's why it's `[~ 'foo']` rather than just `foo`.

127+

128+## Analysis

129+

130+We'll go through it line-by line.

131+

132+```hoon

133+/- spider

134+=, strand=strand:spider

135+```

136+

137+First we import `/sur/spider/hoon` which includes `/lib/strand/hoon` and give give the latter the face `strand` for convenience.

138+

139+```hoon

140+^- thread:spider

141+```

142+

143+We make it a thread by casting it to `thread:spider`

144+

145+```hoon

146+|= arg=vase

147+```

148+

149+We create a gate that takes a vase, the first part of the previously mentioned thread definition.

150+

151+```hoon

152+=/ m (strand ,vase)

153+```

154+

155+Inside the gate we create our `strand` specialised to produce a `vase` and give it the canonical face `m`.

156+

157+```hoon

158+^- form:m

159+```

160+

161+We cast the output to `form` - the mold of the strand we created.

162+

163+```hoon

164+(pure:m arg)

165+```

166+

167+Finally we call `pure` with the gate input `arg` as its argument. Since `arg` is a `vase` it will return the `form` of a `strand` which produces a `vase`. Thus we've created a thread in accordance with its type definition.

168+

169+Next we'll look at the third arm of a strand: `bind`.

userspace/threads/basics/input.mdnew file+145

@@ -0,0 +1,145 @@

1++++

2+title = "Input"

3+weight = 3

4+template = "doc.html"

5++++

7+The input to a `strand` is defined in `/lib/strand/hoon` as:

9+```hoon

10++$ strand-input [=bowl in=(unit input)]

11+```

12+

13+When a thread is first started, spider will populate the `bowl` and provide it along with an `input` of `~`. If/when new input comes in (such as a poke, sign or watch) it will provide a new updated bowl along with the new input.

14+

15+For example, here's a thread that gets the time from the bowl, runs an IO-less function that takes one or two seconds to compute, and then gets the time again:

16+

17+```hoon

18+ /- spider

19+ /+ *strandio

20+ =, strand=strand:spider

21+ |%

22+ ++ ackermann

23+ |= [m=@ n=@]

24+ ?: =(m 0) +(n)

25+ ?: =(n 0) $(m (dec m), n 1)

26+ $(m (dec m), n $(n (dec n)))

27+ --

28+ ^- thread:spider

29+ |= arg=vase

30+ =/ m (strand ,vase)

31+ ^- form:m

32+ ;< t1=@da bind:m get-time

33+ =/ ack (ackermann 3 8)

34+ ;< t2=@da bind:m get-time

35+ (pure:m !>([t1 t2]))

36+```

37+

38+Since it never does any IO, `t1` and `t2` are the same: `[~2021.3.17..07.47.39..e186 ~2021.3.17..07.47.39..e186]`. However, if we replace the ackermann function with a 2 second `sleep` from strandio:

39+

40+```hoon

41+/- spider

42+/+ *strandio

43+=, strand=strand:spider

44+^- thread:spider

45+|= arg=vase

46+=/ m (strand ,vase)

47+^- form:m

48+;< t1=@da bind:m get-time

49+;< ~ bind:m (sleep ~s2)

50+;< t2=@da bind:m get-time

51+(pure:m !>([t1 t2]))

52+```

53+

54+...and run it again we get different values for `t1` and `t2`: `[~2021.3.17..07.50.28..8a5d ~2021.3.17..07.50.30..8a66]`. This is because `sleep` gets a `%wake` sign back from `behn`, so spider updates the time in the bowl along with it.

55+

56+Now let's look at the contents of `bowl` and `input` in detail:

57+

58+## bowl

59+

60+`bowl` is the following:

61+

62+```hoon

63++$ bowl

64+ $: our=ship

65+ src=ship

66+ tid=tid

67+ mom=(unit tid)

68+ wex=boat:gall

69+ sup=bitt:gall

70+ eny=@uvJ

71+ now=@da

72+ byk=beak

73+ ==

74+```

75+

76+- `our` - our ship

77+- `src` - ship where input is coming from

78+- `tid` - ID of this thread

79+- `mom` - parent thread if this is a child thread

80+- `wex` - outgoing subscriptions

81+- `sup` - incoming subscriptions

82+- `eny` - entropy

83+- `now` - current datetime

84+- `byk` - `[p=ship q=desk r=case]` path prefix

85+

86+There are a number of functions in `strandio` to access the `bowl` contents like `get-bowl`, `get-beak`, `get-time`, `get-our` and `get-entropy`.

87+

88+You can also write a function with a gate whose sample is `strand-input:strand` and access the bowl that way like:

89+

90+```hoon

91+/- spider

92+/+ strandio

93+=, strand=strand:spider

94+=>

95+|%

96+++ bowl-stuff

97+ =/ m (strand ,[boat:gall bitt:gall])

98+ ^- form:m

99+ |= tin=strand-input:strand

100+ `[%done [wex.bowl.tin sup.bowl.tin]]

101+--

102+^- thread:spider

103+|= arg=vase

104+=/ m (strand ,vase)

105+^- form:m

106+;< res=[boat:gall bitt:gall] bind:m bowl-stuff

107+(pure:m !>(res))

108+```

109+

110+## input

111+

112+`input` is defined in libstrand as:

113+

114+```hoon

115++$ input

116+ $% [%poke =cage]

117+ [%sign =wire =sign-arvo]

118+ [%agent =wire =sign:agent:gall]

119+ [%watch =path]

120+ ==

121+```

122+

123+- `%poke` incoming poke

124+- `%sign` incoming sign from arvo

125+- `%agent` incoming sign from a gall agent

126+- `%watch` incoming subscription

127+

128+Various functions in `strandio` will check `input` and conditionally do things based on its contents. For example, `sleep` sets a `behn` timer and then calls `take-wake` to wait for a `%wake` sign from behn:

129+

130+```hoon

131+++ take-wake

132+ |= until=(unit @da)

133+ =/ m (strand ,~)

134+ ^- form:m

135+ |= tin=strand-input:strand

136+ ?+ in.tin `[%skip ~]

137+ ~ `[%wait ~]

138+ [~ %sign [%wait @ ~] %behn %wake *]

139+ ?. |(?=(~ until) =(`u.until (slaw %da i.t.wire.u.in.tin)))

140+ `[%skip ~]

141+ ?~ error.sign-arvo.u.in.tin

142+ `[%done ~]

143+ `[%fail %timer-error u.error.sign-arvo.u.in.tin]

144+ ==

145+```

userspace/threads/basics/output.mdnew file+119

@@ -0,0 +1,119 @@

1++++

2+title = "Output"

3+weight = 4

4+template = "doc.html"

5++++

7+A strand produces a `[(list card) <response>]`. The first part is a list of cards to be sent off immediately, and `<response>` is one of:

9+- `[%wait ~]`

10+- `[%skip ~]`

11+- `[%cont self=(strand-form-raw a)]`

12+- `[%fail err=(pair term tang)]`

13+- `[%done value=a]`

14+

15+So, for example, if you feed `2 2` into the following function:

16+

17+```hoon

18+ |= [a=@ud b=@ud]

19+ =/ m (strand ,vase)

20+ ^- form:m

21+ =/ res !>(`@ud`(add a b))

22+ (pure:m res)

23+```

24+

25+The resulting strand won't just produce `[#t/@ud q=4]`, but rather `[~ %done [#t/@ud q=4]]`.

26+

27+**Note:** that spider doesn't actually return the codes themselves to thread subscribers, they're only used internally to manage the flow of the thread.

28+

29+Since a strand is a function from the previously discussed `strand-input` to the output discussed here, you can compose a valid strand like:

30+

31+```hoon

32+|= strand-input:strand

33+[~ %done 'foo']

34+```

35+

36+So this is a valid thread:

37+

38+```hoon

39+/- spider

40+=, strand=strand:spider

41+^- thread:spider

42+|= arg=vase

43+=/ m (strand ,vase)

44+^- form:m

45+|= strand-input:strand

46+[~ %done arg]

47+```

48+

49+As is this:

50+

51+```hoon

52+/- spider

53+=, strand=strand:spider

54+|%

55+++ my-function

56+ =/ m (strand ,@t)

57+ ^- form:m

58+ |= strand-input:strand

59+ [~ %done 'foo']

60+--

61+^- thread:spider

62+|= arg=vase

63+=/ m (strand ,vase)

64+^- form:m

65+;< msg=@t bind:m my-function

66+(pure:m !>(msg))

67+```

68+

69+As is this:

70+

71+```hoon

72+/- spider

73+=, strand=strand:spider

74+^- thread:spider

75+|= arg=vase

76+=/ m (strand ,vase)

77+^- form:m

78+|= strand-input:strand

79+=/ umsg !< (unit @tas) arg

80+?~ umsg

81+[~ %fail %no-arg ~]

82+=/ msg=@tas u.umsg

83+?. =(msg %foo)

84+[~ %fail %not-foo ~]

85+[~ %done arg]

86+```

87+

88+Which works like:

89+

90+```

91+> -mythread

92+thread failed: %no-arg

93+> -mythread %bar

94+thread failed: %not-foo

95+> -mythread %foo

96+[~ %foo]

97+```

98+

99+Now let's look at the meaning of each of the response codes.

100+

101+### wait

102+

103+Wait tells spider not to move on from the current strand, and to wait for some new input. For example, `sleep:strandio` will return a `[%wait ~]` along with a card to start a behn timer. Spider passes the card to behn, and when behn sends a wake back to spider, the new input will be given back to `sleep` as a `%sign`. Sleep will then issue `[~ %done ~]` and (assuming it's in a `bind`) `bind` will proceed to the next strand.

104+

105+### skip

106+

107+Spider will normally treat a `%skip` the same as a `%wait` and just wait for some new input. When used inside a `main-loop:strandio`, however, it will instead tell `main-loop` to skip this function and try the next one with the same input. This is very useful when you want to call different functions depending on the mark of a poke or some other condition.

108+

109+### cont

110+

111+Cont means continue computation. When a `%cont` is issued, the issuing gate will be called again with the new value provided. Therefore `%cont` essentially creates a loop.

112+

113+### fail

114+

115+Fail says to end the thread here and don't call any subsequent strands. It includes an error message and optional traceback. When spider gets a `%fail` it will send a fact with mark `%thread-fail` containing the error and traceback to its subscribers, and then end the thread.

116+

117+### done

118+

119+Done means the computation was completed successfully and includes the result. When `spider` recieves a `%done` it will send the result it contains in a fact with a mark of `%thread-done` to subscribers and end the thread. When `bind` receives a `%done` it will extract the result and call the next gate with it.

userspace/threads/basics/summary.mdnew file+75

@@ -0,0 +1,75 @@

1++++

2+title = "Summary"

3+weight = 5

4+template = "doc.html"

5++++

7+That's basically all you need to know to write threads. The best way to get a good handle on them is just to experiment with some `strandio` functions. For information on running threads from gall agents, see [here](../index.md#gall) and for some examples see [here](../index.md#how-tos--examples).

9+Now here's a quick recap of the main points covered:

10+

11+## Spider

12+

13+- is the gall agent that manages threads.

14+- Details of interacting with threads via spider can be seen [here](../reference.md).

15+

16+## Threads

17+- are like transient gall agents

18+- are used mostly to chain a series of IO operations

19+- can be used by gall agents to spin out IO operations

20+- live in the `ted` directory

21+- are managed by the gall agent `spider`

22+- take a `vase` and produce a `strand` which produces a `vase`

23+

24+#### Example

25+

26+```hoon

27+/- spider

28+=, strand=strand:spider

29+^- thread:spider

30+|= arg=vase

31+=/ m (strand ,vase)

32+^- form:m

33+(pure:m arg)

34+```

35+

36+## Strands

37+

38+- are the building blocks of threads

39+- take [this](https://github.com/urbit/urbit/blob/master/pkg/arvo/lib/strand.hoon#L2-L21) input and produce [this](https://github.com/urbit/urbit/blob/master/pkg/arvo/lib/strand.hoon#L23-L48) output.

40+- must be specialised to produce a particular type like `(strand ,@ud)`.

41+- are conventionally given the face `m`.

42+- are a core that has three main arms - `form`, `pure` and `bind`:

43+

44+#### form

45+- is the mold of the strand suitable for casting

46+- is the type returned by the other arms

47+#### pure

48+- simply returns the `form` of a `strand` that produces pure's argument without doing any IO

49+#### bind

50+- is used to chain strands together like javascript promises

51+- is used in conjunction with micgal (`;<`)

52+- must be specialised to a type like `;< <type> bind:m ...`

53+- takes two arguments. The first is a function that returns the `form` of a `strand` that produces `<type>`. The second is a gate whose sample is `<type>` and which returns a `form`.

54+- calls the first and then, if it succeeded, calls the second with the result of the first as its sample.

55+

56+## Strand input

57+

58+- looks like `[=bowl in=(unit input)]`

59+- `bowl` has things like `our`, `now`, `eny` and so forth

60+- `bowl` is populated once when the thread is first called and then every time it receives new input

61+- `input` contains any incoming pokes, signs and watches.

62+

63+## Strand output

64+

65+- contains `[cards=(list card:agent:gall) <response>]`

66+- `cards` are any cards to be sent immediately

67+- `<response>` is something like `[%done value]`, `[%fail err]`, etc.

68+- `%done` will contain the result

69+- responses are only used internally to manage the flow of the thread and are not returned to subscribers.

70+

71+## Strandio

72+

73+- is located in `/lib/strandio/hoon`

74+- contains a collection of ready-made functions for use in threads

75+- eg. `sleep`, `get-bowl`, `take-watch`, `poke`, `fetch-json`, etc.

userspace/threads/examples/_index.mdnew file+33

@@ -0,0 +1,33 @@

1++++

2+title = "Examples"

3+weight = 30

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+Collection of how-tos and examples for threads.

10+

11+## [Fetch JSON](@/docs/userspace/threads/examples/get-json.md)

12+

13+Get some JSON from an external website.

14+

15+## [Child Thread](@/docs/userspace/threads/examples/child-thread.md)

16+

17+Spawn and manage threads from within threads.

18+

19+## [Main-loop](@/docs/userspace/threads/examples/main-loop.md)

20+

21+Create a loop - useful for long-running threads and for trying the same input against multiple functions.

22+

23+## [Poke Agent](@/docs/userspace/threads/examples/poke-agent.md)

24+

25+Poke an agent from a thread.

26+

27+## [Scry](@/docs/userspace/threads/examples/scry.md)

28+

29+Scry example.

30+

31+## [Take Fact](@/docs/userspace/threads/examples/take-fact.md)

32+

33+Take a fact from arvo or an agent.

userspace/threads/examples/child-thread.mdnew file+237

@@ -0,0 +1,237 @@

1++++

2+title = "Child Thread"

3+weight = 2

4+template = "doc.html"

5++++

7+Here's a simple example of a thread that starts another thread:

9+### parent.hoon

10+

11+```hoon

12+/- spider

13+/+ *strandio

14+=, strand=strand:spider

15+^- thread:spider

16+|= arg=vase

17+=/ m (strand ,vase)

18+^- form:m

19+;< tid=tid:spider bind:m (start-thread %child)

20+(pure:m !>(~))

21+```

22+

23+### child.hoon

24+

25+```hoon

26+/- spider

27+/+ *strandio

28+=, strand=strand:spider

29+^- thread:spider

30+|= arg=vase

31+=/ m (strand ,vase)

32+^- form:m

33+%- (slog leaf+"foo" ~)

34+(pure:m !>(~))

35+```

36+

37+Save `parent.hoon` and `child.hoon` in `/ted`, `|commit %home` and run `-parent`. You should see something like:

38+

39+```

40+foo

41+> -parent

42+~

43+```

44+

45+`parent.hoon` just uses the `strandio` function `start-thread` to start `child.hoon`, and `child.hoon` just prints `foo` to the dojo. Since we got `foo` we can tell the second thread did, in fact, run.

46+

47+```hoon

48+;< tid=tid:spider bind:m (start-thread %child)

49+```

50+

51+See here how we gave `start-thread` the name of the thread to run. It returns the `tid` of the thread, which we could then use to poke it or whatever.

52+

53+`start-thread` handles creating the `tid` for the thread so is quite convenient.

54+

55+Note that threads we start this way will be a child of the thread that started them, and so will be killed when the parent thread ends.

56+

57+## Start thread and get its result

58+

59+If we want to actually get the result of the thread we started, it's slightly more complicated:

60+

61+### parent.hoon

62+

63+```hoon

64+/- spider

65+/+ *strandio

66+=, strand=strand:spider

67+^- thread:spider

68+|= arg=vase

69+=/ m (strand ,vase)

70+^- form:m

71+;< =bowl:spider bind:m get-bowl

72+=/ tid `@ta`(cat 3 'strand_' (scot %uv (sham %child eny.bowl)))

73+;< ~ bind:m (watch-our /awaiting/[tid] %spider /thread-result/[tid])

74+;< ~ bind:m %- poke-our

75+ :* %spider

76+ %spider-start

77+ !>([`tid.bowl `tid %child !>(~)])

78+ ==

79+;< =cage bind:m (take-fact /awaiting/[tid])

80+;< ~ bind:m (take-kick /awaiting/[tid])

81+?+ p.cage ~|([%strange-thread-result p.cage %child tid] !!)

82+ %thread-done (pure:m q.cage)

83+ %thread-fail (strand-fail !<([term tang] q.cage))

84+==

85+```

86+

87+### child.hoon

88+

89+```hoon

90+/- spider

91+/+ *strandio

92+=, strand=strand:spider

93+=>

94+|%

95+++ url "https://www.whatsthelatestbasehash.com/"

96+--

97+^- thread:spider

98+|= arg=vase

99+=/ m (strand ,vase)

100+^- form:m

101+;< =cord bind:m (fetch-cord url)

102+=/ hash-as-cord `@t`(end [3 (sub (met 3 cord) 1)] cord)

103+=/ hash `@uv`(slav %uv hash-as-cord)

104+(pure:m !>(hash))

105+```

106+

107+`child.hoon` simply grabs the latest base hash from https://www.whatsthelatestbasehash.com/ and returns it.

108+

109+`parent.hoon` is a bit more complicated so we'll look at it line-by-line

110+

111+```hoon

112+;< =bowl:spider bind:m get-bowl

113+```

114+

115+First we grab the bowl

116+

117+```hoon

118+=/ tid `@ta`(cat 3 'strand_' (scot %uv (sham %child eny.bowl)))

119+```

120+

121+Then we generate a `tid` (thread ID) for the thread we're gonna start

122+

123+```hoon

124+;< ~ bind:m (watch-our /awaiting/[tid] %spider /thread-result/[tid])

125+```

126+

127+We pre-emptively subscribe for the result. Spider sends the result at `/thread-result/<tid>` so that's where we subscribe.

128+

129+```hoon

130+;< ~ bind:m %- poke-our

131+ :* %spider

132+ %spider-start

133+ !>([`tid.bowl `tid %child !>(~)])

134+ ==

135+```

136+

137+Spider takes a poke with a mark %spider-start and a vase containing `[parent=(unit tid) use=(unit tid) file=term =vase]` to start a thread, where:

138+

139+* `parent` is an optional parent thread. In this case we say the parent is our tid. Specifying a parent means the child will be killed if the parent ends.

140+* `use` is the thread ID for the thread we're creating

141+* `file` is the filename of the thread we want to start

142+* `vase` is the vase it will be given as an argument when it's started

143+

144+```hoon

145+;< =cage bind:m (take-fact /awaiting/[tid])

146+```

147+

148+We wait for a fact which will be the result of the thread.

149+

150+```hoon

151+;< ~ bind:m (take-kick /awaiting/[tid])

152+```

153+

154+Spider will kick us from the subscription when it ends the thread so we also take that kick.

155+

156+```hoon

157+?+ p.cage ~|([%strange-thread-result p.cage %child tid] !!)

158+ %thread-done (pure:m q.cage)

159+ %thread-fail (strand-fail !<([term tang] q.cage))

160+==

161+```

162+

163+Finally we test whether the thread produced a `%thread-done` or a `%thread-fail`. These are the two possible marks produced by spider when it returns the results of a thread. A `%thread-done` will contain a vase with the result, and a `%thread-fail` will contain an error message and traceback, so we see which it is and then either produce the result with `pure` or trigger a `%thread-fail` with the error we got from the child.

164+

165+## Stop a thread

166+

167+### parent.hoon

168+

169+```hoon

170+/- spider

171+/+ *strandio

172+=, strand=strand:spider

173+^- thread:spider

174+|= arg=vase

175+=/ m (strand ,vase)

176+^- form:m

177+;< =bowl:spider bind:m get-bowl

178+=/ tid `@ta`(cat 3 'strand_' (scot %uv (sham %child eny.bowl)))

179+%- (slog leaf+"Starting child thread..." ~)

180+;< ~ bind:m %- poke-our

181+ :* %spider

182+ %spider-start

183+ !>([`tid.bowl `tid %child !>(~)])

184+ ==

185+;< ~ bind:m (sleep ~s5)

186+%- (slog leaf+"Stopping child thread..." ~)

187+;< ~ bind:m %- poke-our

188+ :* %spider

189+ %spider-stop

190+ !>([tid %.y])

191+ ==

192+;< ~ bind:m (sleep ~s2)

193+(pure:m !>("Done"))

194+```

195+

196+### child.hoon

197+

198+```hoon

199+/- spider

200+/+ *strandio

201+=, strand=strand:spider

202+=>

203+|%

204+++ looper

205+ =/ m (strand ,~)

206+ ^- form:m

207+ %- (main-loop ,~)

208+ :~ |= ~

209+ ^- form:m

210+ ;< ~ bind:m (sleep `@dr`(div ~s1 2))

211+ %- (slog leaf+"child thread" ~)

212+ (pure:m ~)

213+ ==

214+--

215+^- thread:spider

216+|= arg=vase

217+=/ m (strand ,vase)

218+^- form:m

219+;< ~ bind:m looper

220+(pure:m !>(~))

221+```

222+

223+`child.hoon` just prints to the dojo in a loop.

224+

225+`parent.hoon` starts `child.hoon`, and then pokes spider like:

226+

227+```hoon

228+;< ~ bind:m %- poke-our

229+ :* %spider

230+ %spider-stop

231+ !>([tid %.y])

232+ ==

233+```

234+

235+* `%spider-stop` is the mark that tells spider to kill a thread.

236+* `tid` is the tid of the thread to kill

237+* `%.y` tells spider to suppress traceback in the result of the killed thread. If you give it `%.n` it will include the traceback.

userspace/threads/examples/get-json.mdnew file+126

@@ -0,0 +1,126 @@

1++++

2+title = "Fetch JSON"

3+weight = 1

4+template = "doc.html"

5++++

7+Grabbing JSON from some url is very easy.

9+`strandio` includes the `fetch-json` function which will handle the http request and response parsing and just return unformatted json data.

10+

11+Here's a simple thread that will:

12+- grab JSON of the latest base-hash from https://www.whatsthelatestbasehash.com/latesthash.json

13+- grab JSON of a list of all base-hashes with version info from https://www.whatsthelatestbasehash.com/all.json

14+- format and cast them

15+- check `all.json` includes the latest base-hash

16+- print the details of the latest base-hash to the dojo

17+

18+### latest-hash.hoon

19+

20+```hoon

21+/- spider

22+/+ *strandio

23+=, strand=strand:spider

24+=>

25+|%

26+++ url-1 "https://www.whatsthelatestbasehash.com/latesthash.json"

27+++ url-2 "https://www.whatsthelatestbasehash.com/all.json"

28+++ latest-format

29+ =, dejs:format

30+ %- ot

31+ :~ ['hash' so]

32+ ['timestamp' so]

33+ ['last_five' so]

34+ ==

35+++ latest-sur

36+ $: hash=@t

37+ timestamp=@t

38+ last-five=@t

39+ ==

40+++ all-format

41+ =, dejs:format

42+ %- ar

43+ %- ot

44+ :~ ['hash' so]

45+ ['pill' bo]

46+ ['version' so]

47+ ['published' so]

48+ ['release' so]

49+ ==

50+++ all-sur

51+ $: hash=@t

52+ pill=?

53+ version=@t

54+ published=@t

55+ release=@t

56+ ==

57+--

58+^- thread:spider

59+|= arg=vase

60+=/ m (strand ,vase)

61+^- form:m

62+;< js-latest=json bind:m (fetch-json url-1)

63+;< js-all=json bind:m (fetch-json url-2)

64+=/ latest `latest-sur`(latest-format js-latest)

65+=/ all `(list all-sur)`(all-format js-all)

66+=/ res (skim all |=(a=all-sur =(hash.a hash.latest)))

67+?~ res

68+ %- (slog leaf+"Not Found" ~) (pure:m !>(~))

69+%- (slog leaf+"Hash: {(trip hash.i.res)}" ~)

70+%- (slog leaf+"Pill? {?:(pill.i.res "Yes" "No")}" ~)

71+%- (slog leaf+"Version: {(trip version.i.res)}" ~)

72+%- (slog leaf+"Published: {(trip published.i.res)}" ~)

73+%- (slog leaf+"Release: {(trip release.i.res)}" ~)

74+(pure:m !>(~))

75+```

76+

77+Save it as `/ted/latest-hash.hoon`, `|commit %home` and run it with `-latest-hash`. You should see something like:

78+

79+```

80+> -latest-hash

81+Hash: 0vj.dqqil.77m3k.a7v8d.mvrmi.8mavk.0a2nl.te6i6.cstfa.1ko6e.872gd

82+Pill? No

83+Version: v2.39

84+Published: ~2021.3.10..02.20.48

85+Release: https://github.com/urbit/urbit/releases/tag/urbit-os-v2.39

86+~

87+```

88+

89+## Analysis

90+

91+`fetch-json` takes a url as a tape, so we've added a core with the urls like:

92+

93+```hoon

94+=>

95+|%

96+++ url-1 "https://www.whatsthelatestbasehash.com/latesthash.json"

97+++ url-2 "https://www.whatsthelatestbasehash.com/all.json"

98+...

99+```

100+

101+...then just called `fetch-json` like:

102+

103+```hoon

104+;< js-latest=json bind:m (fetch-json url-1)

105+```

106+

107+We've added formatting and structure arms like:

108+

109+```hoon

110+++ latest-format

111+ =, dejs:format

112+ %- ot

113+ :~ ['hash' so]

114+ ['timestamp' so]

115+ ['last_five' so]

116+ ==

117+++ latest-sur

118+ $: hash=@t

119+ timestamp=@t

120+ last-five=@t

121+ ==

122+```

123+

124+...to turn it from `json` to a noun with faces. You can have a look at the JSON formatting functions under `dejs:format` in `zuse.hoon` for how to deal with the various json types.

125+

126+After that it's just some ordinary list logic and printing.

userspace/threads/examples/main-loop.mdnew file+178

@@ -0,0 +1,178 @@

1++++

2+title = "Main-loop"

3+weight = 3

4+template = "doc.html"

5++++

7+`main-loop` is a useful function included in `strandio` that:

9+1. lets you create a loop

10+2. lets you try the same input against multiple functions

11+3. queues input on `%skip` and then dequeues from the beginning on `%done`

12+

13+`main-loop` takes a list of functions as its argument but only moves to the next item in the list on a `[%fail %ignore ~]` (whose usage we'll describe in the second example). In other cases it restarts from the top, so providing multiple functions is only useful for trying the same input against multiple functions.

14+

15+## Create a loop

16+

17+This is useful if you want to (for example) take an arbitrary number of facts.

18+

19+Here's an example of a thread that subscribes to `graph-store` for updates and nicely prints the messages (an extremely basic chat reader):

20+

21+#### chat-watch.hoon

22+

23+```hoon

24+/- spider

25+/+ *strandio, *graph-store

26+=, strand=strand:spider

27+=>

28+|%

29+++ watcher

30+ =/ m (strand ,~)

31+ ^- form:m

32+ %- (main-loop ,~)

33+ :~ |= ~

34+ ^- form:m

35+ ;< =cage bind:m (take-fact /graph-store)

36+ =/ up=update !< update q.cage

37+ ?. ?=(%add-nodes -.q.up)

38+ (pure:m ~)

39+ =/ res=tape "{(scow %p entity.resource.q.up)}/{(scow %tas name.resource.q.up)}"

40+ =/ node-list `(list (pair index node))`~(tap by nodes.q.up)

41+ ?~ node-list

42+ (pure:m ~)

43+ ?: (gth (lent node-list) 1)

44+ %- (slog leaf+"{res}: <multi-node update skipped>" ~)

45+ (pure:m ~)

46+ =/ from=tape (scow %p author.post.q.i.node-list)

47+ =/ conts `(list content)`contents.post.q.i.node-list

48+ ?~ conts

49+ (pure:m ~)

50+ ?: (gth (lent conts) 1)

51+ %- (slog leaf+"{res}: [{from}] <mixed-type message skipped>" ~)

52+ (pure:m ~)

53+ ?. ?=(%text -.i.conts)

54+ %- (slog leaf+"{res}: [{from}] <non-text message skipped>" ~)

55+ (pure:m ~)

56+ =/ msg=tape (trip text.i.conts)

57+ %- (slog leaf+"{res}: [{from}] {msg}" ~)

58+ (pure:m ~)

59+ ==

60+--

61+^- thread:spider

62+|= arg=vase

63+=/ m (strand ,vase)

64+^- form:m

65+;< now=@da bind:m get-time

66+;< ~ bind:m (watch-our /graph-store %graph-store /updates)

67+;< ~ bind:m watcher

68+(pure:m !>(~))

69+```

70+

71+Save it in `/ted`, `|commit %home`, and run it with `-chat-watch`. Now try typing some messages in the chat and you should see them printed like:

72+

73+```

74+~zod/test-8488: [~zod] x

75+~zod/test-8488: [~zod] blah blah blah

76+~zod/test-8488: [~zod] foo

77+~zod/test-8488: [~zod] some text

78+~zod/test-8488: [~zod] .

79+```

80+

81+To stop this hit backspace and then run `:spider|kill`.

82+

83+First we subscribe to graph-store for updates with `watch-our`, then we call the watcher arm of the core we have added. Watcher just calls `main-loop`:

84+

85+```hoon

86+=/ m (strand ,~)

87+^- form:m

88+%- (main-loop ,~)

89+...

90+```

91+

92+...with a list of functions. In this case we've just given it one. Our function first calls `take-fact`:

93+

94+```hoon

95+;< =cage bind:m (take-fact /graph-store)

96+```

97+

98+...to receive the fact and then the rest is just processing & printing logic which isn't too important.

99+

100+Once this is done, main-loop will just call the same function again which will again wait for a fact and so on. So you see how it creates a loop. The only way to exit the loop is with a `%fail` or else by poking spider with a `%spider-stop` and the thread's `tid`.

101+

102+## Try input against multiple functions

103+

104+To try the same input against multiple function you must use another `strandio` function `handle`. Handle converts a `%skip` into a `[fail %ignore ~]`. When `main-loop` sees a `[fail %ignore ~]` it tries the next function in its list with the same input.

105+

106+Here are two files: `tester.hoon` and `tested.hoon`. Save them both to `/ted`, `|commit %home` and run `-tester`. You should see:

107+

108+```

109+> -tester

110+baz

111+~

112+```

113+

114+#### tester.hoon

115+

116+```hoon

117+/- spider

118+/+ *strandio

119+=, strand=strand:spider

120+^- thread:spider

121+|= arg=vase

122+=/ m (strand ,vase)

123+^- form:m

124+;< tid=tid:spider bind:m (start-thread %tested)

125+;< our=ship bind:m get-our

126+;< ~ bind:m %- poke

127+ :- [our %spider]

128+ [%spider-input !>([tid `cage`[%baz !>("baz")]])]

129+(pure:m !>(~))

130+```

131+

132+#### tested.hoon

133+

134+```hoon

135+/- spider

136+/+ *strandio

137+=, strand=strand:spider

138+=>

139+|%

140+++ looper

141+ =/ m (strand ,~)

142+ ^- form:m

143+ %- (main-loop ,~)

144+ :~ |= ~

145+ ^- form:m

146+ ;< =vase bind:m ((handle ,vase) (take-poke %foo))

147+ =/ msg=tape !<(tape vase)

148+ %- (slog leaf+"{msg}" ~)

149+ (pure:m ~)

150+ ::

151+ |= ~

152+ ^- form:m

153+ ;< =vase bind:m ((handle ,vase) (take-poke %bar))

154+ =/ msg=tape !<(tape vase)

155+ %- (slog leaf+"{msg}" ~)

156+ (pure:m ~)

157+ ::

158+ |= ~

159+ ^- form:m

160+ ;< =vase bind:m ((handle ,vase) (take-poke %baz))

161+ =/ msg=tape !<(tape vase)

162+ %- (slog leaf+"{msg}" ~)

163+ (pure:m ~)

164+ ==

165+--

166+^- thread:spider

167+|= arg=vase

168+=/ m (strand ,vase)

169+^- form:m

170+;< ~ bind:m looper

171+(pure:m !>(~))

172+```

173+

174+The first thread (tester.hoon) just starts the second thread and pokes it with a vase containing `"baz"` and the mark `%baz`.

175+

176+The second thread (tested.hoon) has a `main-loop` with a list of three `take-poke` strands. As you can see it's the third one expecting a mark of `%baz` but yet it still successfully prints the message. This is because it tried the previous two which each saw the wrong mark and said `%skip`.

177+

178+Notice we've wrapped the `take-poke`s in `handle` to convert the `%skip`s into `[%fail %ignore ~]`s, which `main-loop` takes to mean it should try the next function with the same input.

userspace/threads/examples/poke-agent.mdnew file+60

@@ -0,0 +1,60 @@

1++++

2+title = "Poke Agent"

3+weight = 4

4+template = "doc.html"

5++++

7+Here's a thread that lets you post a message to a chat in graph-store:

9+#### post-msg.hoon

10+

11+```hoon

12+/- spider

13+/+ *strandio, *graph-store, *resource

14+=, strand=strand:spider

15+=>

16+|%

17+++ make-post

18+ |= [our=ship now=@da res=resource msg=@t]

19+ ^- cage

20+ ::

21+ =/ =post *post

22+ =: author.post our

23+ index.post ~[now]

24+ time-sent.post now

25+ contents.post ~[[%text msg]]

26+ ==

27+ ::

28+ :- %graph-update

29+ !> ^- update

30+ :+ %0 now

31+ :+ %add-nodes res

32+ %- ~(gas by *(map index node))

33+ ~[[~[now] [post ~[%empty]]]]

34+--

35+^- thread:spider

36+|= arg=vase

37+=/ m (strand ,vase)

38+=/ uarg !< (unit (pair resource @t)) arg

39+?~ uarg

40+ (strand-fail %no-arg ~)

41+=/ res p.u.uarg

42+=/ msg q.u.uarg

43+^- form:m

44+;< our=@p bind:m get-our

45+;< now=@da bind:m get-time

46+;< ~ bind:m (poke [our %graph-push-hook] (make-post our now res msg))

47+(pure:m !>(~))

48+```

49+

50+Save it in `/ted`, `|commit`, and run it like:

51+

52+```

53+-post-msg [~zod %foo-9955] 'some message'

54+```

55+

56+(obviously change the channel name to whatever you have)

57+

58+## Analysis

59+

60+Pretty simple, just use `on-poke` with an argument of `[ship term] cage` where `term` is the agent and `cage` is whatever the particular agent expects.

userspace/threads/examples/scry.mdnew file+70

@@ -0,0 +1,70 @@

1++++

2+title = "Scry"

3+weight = 5

4+template = "doc.html"

5++++

7+Here's an example of a thread that scries ames for the IP address & port of a ship and nicely prints it:

9+#### get-ip.hoon

10+

11+```hoon

12+/- spider

13+/+ strandio

14+=, strand=strand:spider

15+=, strand-fail=strand-fail:libstrand:spider

16+|%

17+++ process-lanes

18+ |= [target=@p lanes=(list lane:ames)]

19+ =/ m (strand ,~)

20+ ^- form:m

21+ ?~ `(list lane:ames)`lanes

22+ %- (slog leaf+"No route for {(scow %p target)}." ~)

23+ (pure:m ~)

24+ =/ lroute (skip lanes |=(a=lane:ames -.a))

25+ ?~ lroute

26+ %- (slog leaf+"No direct route for {(scow %p target)}." ~)

27+ (pure:m ~)

28+ =/ ip +:(scow %if p.i.lroute)

29+ =/ port (skip (scow %ud (cut 5 [1 1] p.i.lroute)) |=(a=@tD =(a '.')))

30+ %- (slog leaf+"{ip}:{port}" ~)

31+ (pure:m ~)

32+--

33+^- thread:spider

34+|= arg=vase

35+=/ m (strand ,vase)

36+^- form:m

37+=/ utarget !< (unit @p) arg

38+?~ utarget

39+ (strand-fail %no-arg ~)

40+=/ target u.utarget

41+;< lanes=(list lane:ames) bind:m (scry:strandio (list lane:ames) /ax//peers/(scot %p target)/forward-lane)

42+;< ~ bind:m (process-lanes target lanes)

43+(pure:m !>(~))

44+```

45+

46+**Note:** Pretty useless on a fake ship.

47+

48+Save as `ted/get-ip.hoon`, `|commit %home`, and run it with `-get-ip ~bitbet-bolbel`. You should see something like:

49+

50+```

51+34.83.113.220:60659

52+```

53+

54+## Analysis

55+

56+Here we use the `strandio` function `scry` which takes an argument of `[mold path]` where:

57+

58+- `mold` is the return type of the scry

59+- `path` is the scry path formatted like:

60+ 1. vane letter and care

61+ 2. desk if scrying arvo or agent if scrying a gall agent

62+ 3. rest of path

63+

64+In our case the mold is `(list lane:ames)` and the path is `/ax//peers/(scot %p target)/forward-lane` like:

65+

66+```hoon

67+;< lanes=(list lane:ames) bind:m (scry:strandio (list lane:ames) /ax//peers/(scot %p target)/forward-lane)

68+```

69+

70+After that we just process the result in `++ process-lanes` and print it.

userspace/threads/examples/take-fact.mdnew file+65

@@ -0,0 +1,65 @@

1++++

2+title = "Take Fact"

3+weight = 6

4+template = "doc.html"

5++++

7+Taking a fact from an agent, arvo or whatever is easy. First you subscribe using `watch:strandio` or `watch-our:strandio`, then you use `take-fact:strandio` to receive the fact. Here's an example that takes an update from `graph-store` and prints the message to the dojo:

9+#### print-msg.hoon

10+

11+```hoon

12+/- spider

13+/+ *strandio, *graph-store

14+=, strand=strand:spider

15+=>

16+|%

17+++ take-update

18+ =/ m (strand ,~)

19+ ^- form:m

20+ ;< =cage bind:m (take-fact /graph-store)

21+ =/ =update !< update q.cage

22+ ?. ?=(%add-nodes -.q.update)

23+ (pure:m ~)

24+ =/ nodes=(list [=index =node]) ~(tap by nodes.q.update)

25+ ?~ nodes

26+ (pure:m ~)

27+ =/ contents=(list content) contents.post.node.i.nodes

28+ ?~ contents

29+ (pure:m ~)

30+ ?. ?=(%text -.i.contents)

31+ (pure:m ~)

32+ =/ msg (trip text.i.contents)

33+ %- (slog leaf+msg ~)

34+ (pure:m ~)

35+--

36+^- thread:spider

37+|= arg=vase

38+=/ m (strand ,vase)

39+^- form:m

40+;< ~ bind:m (watch-our /graph-store %graph-store /updates)

41+;< ~ bind:m take-update

42+(pure:m !>(~))

43+```

44+

45+Create a chat on your fake zod if you don't have one already, then save it in `/ted`, `|commit %home`, and run `-print-msg`. Next, type some message in your chat and you'll see it printed in the dojo.

46+

47+## Analysis

48+

49+First we call `watch-our` to subscribe:

50+

51+```hoon

52+;< ~ bind:m (watch-our /graph-store %graph-store /updates)

53+```

54+

55+We've spun the next part out into its own core, but it's just a `take-fact` to receive the update:

56+

57+```hoon

58+ ;< =cage bind:m (take-fact /graph-store)

59+```

60+

61+The rest of the code is just to pull the message out of the complicate data structure returned by graph-store and isn't important.

62+

63+Spider will automatically leave the subscription once the thread finishes.

64+

65+Note that `take-fact` only takes a single fact, so you'd need one for each message you're expecting. Alternatively you can use `main-loop` to take an arbitrary number of facts.

userspace/threads/gall/_index.mdnew file+29

@@ -0,0 +1,29 @@

1++++

2+title = "Gall"

3+weight = 20

4+sort_by = "weight"

5+template = "sections/docs/chapters.html"

6+insert_anchor_links = "right"

7++++

9+Tutorial and explanation of interacting with threads from a Gall agent.

10+

11+## [Start Thread](@/docs/userspace/threads/gall/start-thread.md)

12+

13+Start a thread from an agent.

14+

15+## [Take Result](@/docs/userspace/threads/gall/take-result.md)

16+

17+Subscribe for thread result from an agent.

18+

19+## [Take Facts](@/docs/userspace/threads/gall/take-facts.md)

20+

21+Subscribe for facts sent from a running thread.

22+

23+## [Stop Thread](@/docs/userspace/threads/gall/stop-thread.md)

24+

25+Stop a thread from an agent.

26+

27+## [Poke Thread](@/docs/userspace/threads/gall/poke-thread.md)

28+

29+Poke a running thread from an agent.

userspace/threads/gall/poke-thread.mdnew file+116

@@ -0,0 +1,116 @@

1++++

2+title = "Poke Thread"

3+weight = 5

4+template = "doc.html"

5++++

7+Here's a modified agent that pokes our thread. I've replaced some off the previous stuff because it was getting a little unwieldly.

9+#### thread-starter.hoon

10+

11+```hoon

12+/+ default-agent, dbug

13+=* card card:agent:gall

14+%- agent:dbug

15+^- agent:gall

16+|_ =bowl:gall

17++* this .

18+ def ~(. (default-agent this %|) bowl)

19+::

20+++ on-init on-init:def

21+++ on-save on-save:def

22+++ on-load on-load:def

23+++ on-poke

24+ |= [=mark =vase]

25+ ^- (quip card _this)

26+ ?+ mark (on-poke:def mark vase)

27+ %noun

28+ ?+ q.vase (on-poke:def mark vase)

29+ (pair term term)

30+ =/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

31+ =/ ta-now `@ta`(scot %da now.bowl)

32+ =/ start-args [~ `tid p.q.vase !>(q.q.vase)]

33+ :_ this

34+ :~

35+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %watch /thread-result/[tid]]

36+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

37+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-input !>([tid %foo !>(q.q.vase)])]

38+ ==

39+ ==

40+ ==

41+++ on-watch on-watch:def

42+++ on-leave on-leave:def

43+++ on-peek on-peek:def

44+++ on-agent

45+ |= [=wire =sign:agent:gall]

46+ ^- (quip card _this)

47+ ?+ -.wire (on-agent:def wire sign)

48+ %thread

49+ ?+ -.sign (on-agent:def wire sign)

50+ %fact

51+ ?+ p.cage.sign (on-agent:def wire sign)

52+ %thread-fail

53+ =/ err !< (pair term tang) q.cage.sign

54+ %- (slog leaf+"Thread failed: {(trip p.err)}" q.err)

55+ `this

56+ %thread-done

57+ ?: =(q.cage.sign *vase)

58+ %- (slog leaf+"Thread cancelled nicely" ~)

59+ `this

60+ =/ res (trip !<(term q.cage.sign))

61+ %- (slog leaf+"Result: {res}" ~)

62+ `this

63+ ==

64+ ==

65+ ==

66+++ on-arvo on-arvo:def

67+++ on-fail on-fail:def

68+--

69+```

70+

71+And here we've modified the thread to take the poke and return it as the result:

72+

73+#### test-thread.hoon

74+

75+```hoon

76+/- spider

77+/+ *strandio

78+=, strand=strand:spider

79+^- thread:spider

80+|= arg=vase

81+=/ m (strand ,vase)

82+^- form:m

83+;< vmsg=vase bind:m (take-poke %foo)

84+(pure:m vmsg)

85+```

86+

87+Save them, `|commit` and run it like `:thread-starter [%test-thread %blah]`. You should see:

88+

89+

90+```

91+Result: blah

92+> :thread-starter [%test-thread %blah]

93+```

94+

95+## Analysis

96+

97+In our agent we've added this card:

98+

99+```hoon

100+[%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-input !>([tid %foo !>(q.q.vase)])]

101+```

102+

103+To poke a particular thread you poke `%spider` with a mark of `%spider-input` and a vase of `[tid cage]` where:

104+

105+- `tid` is the thread you want to poke

106+- `cage` has whatever mark and vase of data you want to give the thread

107+

108+In our case we've given it a mark of `%foo` and a vase of whatever `term` we poked our agent with.

109+

110+In our thread we've added:

111+

112+```hoon

113+;< vmsg=vase bind:m (take-poke %foo)

114+```

115+

116+`take-poke` is a `strandio` function that just waits for a poke with the given mark and skips everything else. In this case we've specified a mark of `%foo`. Once our thread gets a poke with this mark it returns it as a result with `(pure:m vmsg)`. When our agent gets that it just prints it.

userspace/threads/gall/start-thread.mdnew file+145

@@ -0,0 +1,145 @@

1++++

2+title = "Start Thread"

3+weight = 1

4+template = "doc.html"

5++++

7+Here's an example of a barebones gall agent that just starts a thread:

9+#### thread-starter.hoon

10+

11+```hoon

12+/+ default-agent, dbug

13+=* card card:agent:gall

14+%- agent:dbug

15+^- agent:gall

16+|_ =bowl:gall

17++* this .

18+ def ~(. (default-agent this %|) bowl)

19+::

20+++ on-init on-init:def

21+++ on-save on-save:def

22+++ on-load on-load:def

23+++ on-poke

24+ |= [=mark =vase]

25+ ^- (quip card _this)

26+ ?+ mark (on-poke:def mark vase)

27+ %noun

28+ ?+ q.vase (on-poke:def mark vase)

29+ (pair term term)

30+ =/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

31+ =/ ta-now `@ta`(scot %da now.bowl)

32+ =/ start-args [~ `tid p.q.vase !>(q.q.vase)]

33+ :_ this

34+ :~

35+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

36+ ==

37+ ==

38+ ==

39+++ on-watch on-watch:def

40+++ on-leave on-leave:def

41+++ on-peek on-peek:def

42+++ on-agent

43+ |= [=wire =sign:agent:gall]

44+ ^- (quip card _this)

45+ ?+ -.wire (on-agent:def wire sign)

46+ %thread

47+ ?+ -.sign (on-agent:def wire sign)

48+ %poke-ack

49+ ?~ p.sign

50+ %- (slog leaf+"Thread started successfully" ~)

51+ `this

52+ %- (slog leaf+"Thread failed to start" u.p.sign)

53+ `this

54+ ==

55+ ==

56+++ on-arvo on-arvo:def

57+++ on-fail on-fail:def

58+--

59+```

60+

61+And here's a minimal thread to test it with:

62+

63+#### test-thread.hoon

64+

65+```hoon

66+/- spider

67+=, strand=strand:spider

68+^- thread:spider

69+|= arg=vase

70+=/ m (strand ,vase)

71+^- form:m

72+|= strand-input:strand

73+?+ q.arg [~ %fail %not-foo ~]

74+ %foo

75+ [~ %done arg]

76+==

77+```

78+

79+Save them as `/app/thread-starter.hoon` and `/ted/test-thread.hoon` respectively, `|commit %home`, and start the app with `|start %thread-starter`.

80+

81+Now you can poke it with a pair of thread name and argument like:

82+

83+```

84+:thread-starter [%test-thread %foo]

85+```

86+

87+You should see `Thread started successfully`.

88+

89+Now try poking it with `[%fake-thread %foo]`, you should see something like:

90+

91+```

92+Thread failed to start

93+/app/spider/hoon:<[355 5].[355 60]>

94+[%no-file-for-thread %fake-thread]

95+/app/spider/hoon:<[354 5].[355 60]>

96+/app/spider/hoon:<[353 3].[359 19]>

97+/app/spider/hoon:<[350 3].[359 19]>

98+/app/spider/hoon:<[346 3].[359 19]>

99+/app/spider/hoon:<[343 3].[359 19]>

100+/app/spider/hoon:<[341 3].[359 19]>

101+/app/spider/hoon:<[340 3].[359 19]>

102+/app/spider/hoon:<[336 3].[359 19]>

103+/app/spider/hoon:<[335 3].[359 19]>

104+/app/spider/hoon:<[202 24].[202 68]>

105+/app/spider/hoon:<[200 7].[207 9]>

106+/app/spider/hoon:<[199 5].[208 17]>

107+/app/spider/hoon:<[197 5].[208 17]>

108+/app/spider/hoon:<[196 5].[208 17]>

109+/sys/vane/gall/hoon:<[1.370 9].[1.370 37]>

110+```

111+

112+## Analysis

113+

114+We can ignore the input logic, here's the important part:

115+

116+```hoon

117+=/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

118+=/ ta-now `@ta`(scot %da now.bowl)

119+=/ start-args [~ `tid p.q.vase !>(q.q.vase)]

120+:_ this

121+:~

122+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

123+==

124+```

125+

126+You can generate a tid any way you like, just make sure it's unique. Here we just use the hash of some entropy prefixed with `thread_`.

127+

128+Then it's just a poke to `%spider` with the mark `%spider-start` and a vase containing [start-args](../reference.md#start-thread). Spider will then respond with a `%poke-ack` with a `(unit tang)` which will be `~` if it started successfully or else contain an error and a traceback if it failed. Here we test for this and print the result:

129+

130+```hoon

131+++ on-agent

132+ |= [=wire =sign:agent:gall]

133+ ^- (quip card _this)

134+ ?+ -.wire (on-agent:def wire sign)

135+ %thread

136+ ?+ -.sign (on-agent:def wire sign)

137+ %poke-ack

138+ ?~ p.sign

139+ %- (slog leaf+"Thread started successfully" ~)

140+ `this

141+ %- (slog leaf+"Thread failed to start" u.p.sign)

142+ `this

143+ ==

144+ ==

145+```

userspace/threads/gall/stop-thread.mdnew file+152

@@ -0,0 +1,152 @@

1++++

2+title = "Stop Thread"

3+weight = 4

4+template = "doc.html"

5++++

7+Here we've added one last card to `on-poke` to stop the thread and a little extra to `on-agent` to print things for demonstrative purposes.

9+#### thread-starter.hoon

10+

11+```hoon

12+/+ default-agent, dbug

13+=* card card:agent:gall

14+%- agent:dbug

15+^- agent:gall

16+|_ =bowl:gall

17++* this .

18+ def ~(. (default-agent this %|) bowl)

19+::

20+++ on-init on-init:def

21+++ on-save on-save:def

22+++ on-load on-load:def

23+++ on-poke

24+ |= [=mark =vase]

25+ ^- (quip card _this)

26+ ?+ mark (on-poke:def mark vase)

27+ %noun

28+ ?+ q.vase (on-poke:def mark vase)

29+ (pair term term)

30+ =/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

31+ =/ ta-now `@ta`(scot %da now.bowl)

32+ =/ start-args [~ `tid p.q.vase !>(q.q.vase)]

33+ :_ this

34+ :~

35+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %watch /thread-result/[tid]]

36+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

37+ [%pass /thread/updates/[ta-now] %agent [our.bowl %spider] %watch /thread/[tid]/updates]

38+ [%pass /thread-stop/[ta-now] %agent [our.bowl %spider] %poke %spider-stop !>([tid %.y])]

39+ ==

40+ ==

41+ ==

42+++ on-watch on-watch:def

43+++ on-leave on-leave:def

44+++ on-peek on-peek:def

45+++ on-agent

46+ |= [=wire =sign:agent:gall]

47+ ^- (quip card _this)

48+ ?+ -.wire (on-agent:def wire sign)

49+ %thread

50+ ?+ -.sign (on-agent:def wire sign)

51+ %poke-ack

52+ ?~ p.sign

53+ %- (slog leaf+"Thread started successfully" ~)

54+ `this

55+ %- (slog leaf+"Thread failed to start" u.p.sign)

56+ `this

57+ ::

58+ %fact

59+ ?+ p.cage.sign (on-agent:def wire sign)

60+ %thread-fail

61+ =/ err !< (pair term tang) q.cage.sign

62+ %- (slog leaf+"Thread failed: {(trip p.err)}" q.err)

63+ `this

64+ %thread-done

65+ ?: =(q.cage.sign *vase)

66+ %- (slog leaf+"Thread cancelled nicely" ~)

67+ `this

68+ =/ res (trip !<(term q.cage.sign))

69+ %- (slog leaf+"Result: {res}" ~)

70+ `this

71+ %update

72+ =/ msg !< tape q.cage.sign

73+ %- (slog leaf+msg ~)

74+ `this

75+ ==

76+ ==

77+ %thread-stop

78+ ?+ -.sign (on-agent:def wire sign)

79+ %poke-ack

80+ ?~ p.sign

81+ %- (slog leaf+"Thread cancelled successfully" ~)

82+ `this

83+ %- (slog leaf+"Thread failed to stop" u.p.sign)

84+ `this

85+ ==

86+ ==

87+++ on-arvo on-arvo:def

88+++ on-fail on-fail:def

89+--

90+```

91+

92+We've also added a `sleep` to the thread to keep it running for demonstration.

93+

94+#### test-thread.hoon

95+

96+```hoon

97+/- spider

98+/+ *strandio

99+=, strand=strand:spider

100+^- thread:spider

101+|= arg=vase

102+=/ m (strand ,vase)

103+^- form:m

104+;< =path bind:m take-watch

105+;< ~ bind:m (send-raw-card [%give %fact ~[path] %update !>("message 1")])

106+;< ~ bind:m %- send-raw-cards

107+ :~ [%give %fact ~[path] %update !>("message 2")]

108+ [%give %fact ~[path] %update !>("message 3")]

109+ [%give %fact ~[path] %update !>("message 4")]

110+ ==

111+;< ~ bind:m (send-raw-card [%give %kick ~[path] ~])

112+;< ~ bind:m (sleep ~m1)

113+|= strand-input:strand

114+?+ q.arg [~ %fail %not-foo ~]

115+ %foo

116+ [~ %done arg]

117+==

118+```

119+

120+Save these, `|commit`, and run with `:thread-starter [%test-thread %foo]`. You should see:

121+

122+```

123+Thread started successfully

124+message 1

125+message 2

126+message 3

127+message 4

128+Thread cancelled successfully

129+Thread cancelled nicely

130+```

131+

132+Now, try changing the vase in our new card from `!>([tid %.y])` to `!>([tid %.n])`. Save, `|commit`, and run again. You should see:

133+

134+```

135+Thread started successfully

136+message 1

137+message 2

138+message 3

139+message 4

140+Thread cancelled successfully

141+Thread failed: cancelled

142+```

143+

144+## Analysis

145+

146+The card we've added to our agent:

147+

148+```hoon

149+[%pass /thread-stop/[ta-now] %agent [our.bowl %spider] %poke %spider-stop !>([tid %.y])]

150+```

151+

152+...pokes spider with mark `%spider-stop` and a vase containing the tid of the thread we want to stop and a `?`. The `?` specifies whether to end it nicely or not. If `%.y` it will end with `%thread-done` and a `*vase` bunted vase. If `%.n` it will end with `%thread-fail` and a vase containing `[term tang]` where `term` is `%cancelled` and `tang` is `~`. You can see the difference in our tests above.

userspace/threads/gall/take-facts.mdnew file+184

@@ -0,0 +1,184 @@

1++++

2+title = "Take Facts"

3+weight = 3

4+template = "doc.html"

5++++

7+Most of the time you'll just want the final result like how we did previously. Sometimes, though, you might want to send out facts while the thread runs rather than just at the end.

9+Here we've added another card to subscribe for any facts sent by the thread and some small changes to `on-agent`:

10+

11+#### thread-starter.hoon

12+

13+```hoon

14+/+ default-agent, dbug

15+=* card card:agent:gall

16+%- agent:dbug

17+^- agent:gall

18+|_ =bowl:gall

19++* this .

20+ def ~(. (default-agent this %|) bowl)

21+::

22+++ on-init on-init:def

23+++ on-save on-save:def

24+++ on-load on-load:def

25+++ on-poke

26+ |= [=mark =vase]

27+ ^- (quip card _this)

28+ ?+ mark (on-poke:def mark vase)

29+ %noun

30+ ?+ q.vase (on-poke:def mark vase)

31+ (pair term term)

32+ =/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

33+ =/ ta-now `@ta`(scot %da now.bowl)

34+ =/ start-args [~ `tid p.q.vase !>(q.q.vase)]

35+ :_ this

36+ :~

37+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %watch /thread-result/[tid]]

38+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

39+ [%pass /thread/updates/[ta-now] %agent [our.bowl %spider] %watch /thread/[tid]/updates]

40+ ==

41+ ==

42+ ==

43+++ on-watch on-watch:def

44+++ on-leave on-leave:def

45+++ on-peek on-peek:def

46+++ on-agent

47+ |= [=wire =sign:agent:gall]

48+ ^- (quip card _this)

49+ ?+ -.wire (on-agent:def wire sign)

50+ %thread

51+ ?+ -.sign (on-agent:def wire sign)

52+ %poke-ack

53+ ?~ p.sign

54+ %- (slog leaf+"Thread started successfully" ~)

55+ `this

56+ %- (slog leaf+"Thread failed to start" u.p.sign)

57+ `this

58+ ::

59+ %fact

60+ ?+ p.cage.sign (on-agent:def wire sign)

61+ %thread-fail

62+ =/ err !< (pair term tang) q.cage.sign

63+ %- (slog leaf+"Thread failed: {(trip p.err)}" q.err)

64+ `this

65+ %thread-done

66+ =/ res (trip !<(term q.cage.sign))

67+ %- (slog leaf+"Result: {res}" ~)

68+ `this

69+ %update

70+ =/ msg !< tape q.cage.sign

71+ %- (slog leaf+msg ~)

72+ `this

73+ ==

74+ ==

75+ ==

76+++ on-arvo on-arvo:def

77+++ on-fail on-fail:def

78+--

79+```

80+

81+We've also made some changes to the thread:

82+

83+#### test-thread.hoon

84+

85+```hoon

86+/- spider

87+/+ *strandio

88+=, strand=strand:spider

89+^- thread:spider

90+|= arg=vase

91+=/ m (strand ,vase)

92+^- form:m

93+;< =path bind:m take-watch

94+;< ~ bind:m (send-raw-card [%give %fact ~[path] %update !>("message 1")])

95+;< ~ bind:m %- send-raw-cards

96+ :~ [%give %fact ~[path] %update !>("message 2")]

97+ [%give %fact ~[path] %update !>("message 3")]

98+ [%give %fact ~[path] %update !>("message 4")]

99+ ==

100+;< ~ bind:m (send-raw-card [%give %kick ~[path] ~])

101+|= strand-input:strand

102+?+ q.arg [~ %fail %not-foo ~]

103+ %foo

104+ [~ %done arg]

105+==

106+```

107+

108+Save & `|commit`, then run `:thread-starter [%test-thread %foo]`. You should see:

109+

110+```

111+Thread started successfully

112+message 1

113+message 2

114+message 3

115+message 4

116+Result: foo

117+```

118+

119+Now try `:thread-starter [%test-thread %bar]`. You should see:

120+

121+```

122+Thread started successfully

123+message 1

124+message 2

125+message 3

126+message 4

127+Thread failed: not-foo

128+```

129+

130+## Analysis

131+

132+In our agent's `on-poke` arm we've added another card to subscribe to `/thread/[tid]/updates`:

133+

134+```hoon

135+[%pass /thread/updates/[ta-now] %agent [our.bowl %spider] %watch /thread/[tid]/updates]

136+```

137+

138+**Note:** In practice you'll want to include some kind of tag in the wire so you can differentiate particular threads and subscriptions and test for it in `on-agent`.

139+

140+Threads always send facts on `/thread/[tid]/some-path`. The thread itself will see the incoming subscription on `/some-path` though, not the full thing.

141+

142+In the thread we've first added:

143+

144+```hoon

145+;< =path bind:m take-watch

146+```

147+

148+...to take the subscription. Without something like this to handle the incoming `%watch`, spider will reject the subscription.

149+

150+Then we've added:

151+

152+```hoon

153+;< ~ bind:m (send-raw-card [%give %fact ~[path] %update !>("message 1")])

154+;< ~ bind:m %- send-raw-cards

155+ :~ [%give %fact ~[path] %update !>("message 2")]

156+ [%give %fact ~[path] %update !>("message 3")]

157+ [%give %fact ~[path] %update !>("message 4")]

158+ ==

159+```

160+

161+...to send some facts out to subscribers. Here we've used both `send-raw-card` and `send-raw-cards` to demonstrate both ways.

162+

163+**Note:** in practice you'd probably want to send facts on a predetermined path and just test the path of the incoming subscription rather than just accepting anything.

164+

165+Finally we've added:

166+

167+```hoon

168+;< ~ bind:m (send-raw-card [%give %kick ~[path] ~])

169+```

170+

171+...to kick subscribers. This is important because, unlike on `/thread-result`, spider will not automatically kick subscribers when the thread ends. You have to do it explicitly so your agent doesn't accumulate wires with repeated executions.

172+

173+Back in our agent: In the `on-agent` arm we've added:

174+

175+```hoon

176+ %update

177+=/ msg !< tape q.cage.sign

178+%- (slog leaf+msg ~)

179+`this

180+```

181+

182+... to take the facts and print them.

183+

184+One last thing: Notice how when we gave the thread an argument of `%bar` and made it fail, we still got the facts we sent. This is because such cards are sent immediately as the thread runs, they don't depend on the final result.

userspace/threads/gall/take-result.mdnew file+132

@@ -0,0 +1,132 @@

1++++

2+title = "Take Result"

3+weight = 2

4+template = "doc.html"

5++++

7+Here we've added an extra card to subscribe for the result and a couple of lines in on-agent to test if it succeeded:

9+#### thread-starter.hoon

10+

11+```hoon

12+/+ default-agent, dbug

13+=* card card:agent:gall

14+%- agent:dbug

15+^- agent:gall

16+|_ =bowl:gall

17++* this .

18+ def ~(. (default-agent this %|) bowl)

19+::

20+++ on-init on-init:def

21+++ on-save on-save:def

22+++ on-load on-load:def

23+++ on-poke

24+ |= [=mark =vase]

25+ ^- (quip card _this)

26+ ?+ mark (on-poke:def mark vase)

27+ %noun

28+ ?+ q.vase (on-poke:def mark vase)

29+ (pair term term)

30+ =/ tid `@ta`(cat 3 'thread_' (scot %uv (sham eny.bowl)))

31+ =/ ta-now `@ta`(scot %da now.bowl)

32+ =/ start-args [~ `tid p.q.vase !>(q.q.vase)]

33+ :_ this

34+ :~

35+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %watch /thread-result/[tid]]

36+ [%pass /thread/[ta-now] %agent [our.bowl %spider] %poke %spider-start !>(start-args)]

37+ ==

38+ ==

39+ ==

40+++ on-watch on-watch:def

41+++ on-leave on-leave:def

42+++ on-peek on-peek:def

43+++ on-agent

44+ |= [=wire =sign:agent:gall]

45+ ^- (quip card _this)

46+ ?+ -.wire (on-agent:def wire sign)

47+ %thread

48+ ?+ -.sign (on-agent:def wire sign)

49+ %poke-ack

50+ ?~ p.sign

51+ %- (slog leaf+"Thread started successfully" ~)

52+ `this

53+ %- (slog leaf+"Thread failed to start" u.p.sign)

54+ `this

55+ ::

56+ %fact

57+ ?+ p.cage.sign (on-agent:def wire sign)

58+ %thread-fail

59+ =/ err !< (pair term tang) q.cage.sign

60+ %- (slog leaf+"Thread failed: {(trip p.err)}" q.err)

61+ `this

62+ %thread-done

63+ =/ res (trip !<(term q.cage.sign))

64+ %- (slog leaf+"Result: {res}" ~)

65+ `this

66+ ==

67+ ==

68+ ==

69+++ on-arvo on-arvo:def

70+++ on-fail on-fail:def

71+--

72+```

73+

74+#### test-thread.hoon

75+

76+```hoon

77+/- spider

78+=, strand=strand:spider

79+^- thread:spider

80+|= arg=vase

81+=/ m (strand ,vase)

82+^- form:m

83+|= strand-input:strand

84+?+ q.arg [~ %fail %not-foo ~]

85+ %foo

86+ [~ %done arg]

87+==

88+```

89+

90+Save these, `|commit` and then poke the app with `:thread-starter [%test-thread %foo]`. You should see:

91+

92+```

93+Thread started successfully

94+Result: foo

95+```

96+

97+Now try `:thread-starter [%test-thread %bar]`. You should see:

98+

99+```

100+Thread started successfully

101+Thread failed: not-foo

102+```

103+

104+## Analysis

105+

106+In `on-poke` we've added an extra card *before* the `%spider-start` poke to subscribe for the result:

107+

108+```hoon

109+[%pass /thread/[ta-now] %agent [our.bowl %spider] %watch /thread-result/[tid]]

110+```

111+

112+If successful the thread will return a cage with a mark of `%thread-done` and a vase containing the result.

113+

114+If the thread failed it will return a cage with a mark of `%thread-fail` and a vase containing `[term tang]` where `term` is an error message and `tang` is a traceback. In our case our thread fails with error `%not-foo` when its argument is not `%foo`.

115+

116+Note that spider will automatically `%kick` us from the subscription after ending the thread and returning the result.

117+

118+```hoon

119+ %fact

120+?+ p.cage.sign (on-agent:def wire sign)

121+ %thread-fail

122+ =/ err !< (pair term tang) q.cage.sign

123+ %- (slog leaf+"Thread failed: {(trip p.err)}" q.err)

124+ `this

125+ %thread-done

126+ =/ res (trip !<(term q.cage.sign))

127+ %- (slog leaf+"Result: {res}" ~)

128+ `this

129+==

130+```

131+

132+Here in on-agent we've added a test for `%thread-fail` or `%thread-done` and print the appropriate result.

userspace/threads/overview.mdnew file+81

@@ -0,0 +1,81 @@

1++++

2+title = "Overview"

3+weight = 1

4+template = "doc.html"

5++++

7+As discussed in the [Gall](@/docs/userspace/gall/gall.md) section, Urbit code lives in the following basic

8+categories:

10+- Runtime (Nock interpreter, persistence engine, IO drivers, jets)

11+- Kernel vanes (managed by Arvo)

12+- Userspace agents (managed by Gall, permanent state)

13+- Userspace threads (managed by Spider, transient state)

14+

15+This describes the last category: Threads.

16+

17+A thread is a monadic function that takes arguments and produces a

18+result. It may perform input and output while running, so it is not a

19+pure function. It may fail.

20+

21+An agent's strength is that it's permanent and bulletproof. All state

22+transitions are defined, and each action it performs is a transaction.

23+Code upgrades preserve existing state.

24+

25+An agent's weakness is complex input and output. Since each state

26+transition must be explicitly handled, the complexity of an agent

27+explodes with the amount of IO it handles. At best, this results in

28+long and complex code; at worst, unexpected states are mishandled,

29+corrupting permanent state.

30+

31+A thread's strength is that it can easily perform complex IO operations.

32+It uses what's often called the IO monad (plus the exception monad) to

33+provide a natural framework for IO.

34+

35+A thread's weakness is that it's impermanent and may fail unexpectedly.

36+In most of its intermediate states, it expects only a small number of

37+events (usually one), so if it receives anything it didn't expect, it

38+fails. When code is upgraded, it's impossible to upgrade a running

39+thread, so it fails.

40+

41+Thus, for anything that needs to be permament, use an agent. When you

42+need to do a long or complex sequence of IO operations, reduce that to a

43+single logical IO operation by spinning it out into a thread. If you

44+only change your agent's state in response to success of the thread, an

45+IO failure will never result in partially applied state changes.

46+

47+A thread may also be run from the dojo by prefixing its name with `-`

48+and giving it any arguments it requires. If alone, any result will be

49+printed to the screen; else the output may be piped into an agent or

50+other sinks.

51+

52+## Thread basics

53+

54+These docs walk through the fundamental things you need to know to write threads. They're focused on basic thread composition so don't touch on interacting with threads from gall agents and such. The included examples can all just be run from the dojo.

55+

56+1. [Thread Fundamentals](@/docs/userspace/threads/basics/fundamentals.md) - Basic information and overview of threads, strands, `form` & `pure`.

57+2. [Micgal and Bind](@/docs/userspace/threads/basics/bind.md) - Covers using micgal and `bind` to chain strands.

58+3. [Strand Input](@/docs/userspace/threads/basics/input.md) - What strands receive as input

59+4. [Strand Output](@/docs/userspace/threads/basics/output.md) - What strands produce

60+5. [Summary](@/docs/userspace/threads/basics/summary.md)

61+

62+## Gall

63+These docs walk through the basics of interacting with threads from gall agents.

64+

65+1. [Start a thread](@/docs/userspace/threads/gall/start-thread.md)

66+2. [Subscribe for result](@/docs/userspace/threads/gall/take-result.md)

67+3. [Subscribe for facts](@/docs/userspace/threads/gall/take-facts.md)

68+4. [Stop a thread](@/docs/userspace/threads/gall/stop-thread.md)

69+5. [Poke a thread](@/docs/userspace/threads/gall/poke-thread.md)

70+

71+## How-tos & Examples

72+

73+- [Grab some JSON from a URL](@/docs/userspace/threads/examples/get-json.md) - Here's an example of chaining a couple of external http requests for JSON.

74+- [Start a child thread](@/docs/userspace/threads/examples/child-thread.md) - Starting and managing child threads.

75+- [Main Loop](@/docs/userspace/threads/examples/main-loop.md) - Some notes and examples of the `strandio` function `main-loop`.

76+- [Poke an agent](@/docs/userspace/threads/examples/poke-agent.md) - Example of poking an agent from a thread.

77+- [Scry](@/docs/userspace/threads/examples/scry.md) - Scry arvo or an agent.

78+- [Take a fact](@/docs/userspace/threads/examples/take-fact.md) - Subscribe to an agent and receive a fact.

79+

80+## [Reference](@/docs/userspace/threads/reference.md)

81+Basic reference information. For usage of particular `strandio` functions just refer directly to `/lib/strandio/hoon` since they're largely self-explanatory.

userspace/threads/reference.mdnew file+86

@@ -0,0 +1,86 @@

1++++

2+title = "Reference"

3+weight = 50

4+template = "doc.html"

5++++

7+## Spider

9+- [Start a thread](#start-thread)

10+- [Stop a thread](#stop-thread)

11+- [Subscribe to thread result](#subscribe-to-thread-result)

12+- [Subscribe to thread](#subscribe-to-thread)

13+- [Poke a thread](#poke-thread)

14+

15+### Start thread

16+

17+Poke `spider` with mark `%spider-start` and a vase containing `start-args`:

18+

19+```hoon

20++$ start-args

21+ [parent=(unit tid) use=(unit tid) file=term =vase]

22+```

23+

24+Where:

25+

26+- `parent` - optional `tid` of parent thread if the thread is a child. If specified, the child thread will be killed with the parent thread ends.

27+- `use` - `tid` (thread ID) to give the new thread. Can be generated with something like `(scot %ta (cat 3 'my-agent_' (scot %uv (sham eny))))`. However you do it, make sure it's unique.

28+- `file` - name of the thread file in `/ted`. For example, if the thread you want to start is `/ted/foo/hoon` you'd specify `%foo`.

29+- `vase` - `vase` to be given to the thread when it's started. Can be whatever or just `!>(~)` if it doesn't need any args.

30+

31+#### Example

32+

33+```hoon

34+[%pass /some-path %agent [our.bowl %spider] %poke %spider-start !>([~ `tid %foo !>(~)])]

35+```

36+

37+### Stop thread

38+

39+Poke `spider` with mark `%spider-stop` and a vase containing `[tid ?]`, where:

40+

41+- `tid` - the `tid` of the thread you want to stop

42+- `?` - whether thread should end nicely. If `%.y` it'll end with mark `%thread-done` and the bunt value of a vase. If `%.n` it'll end with mark `%thread-fail` and a `[term tang]` where `term` is `%cancelled` and `tang` is `~`.

43+

44+#### Example

45+

46+```hoon

47+[%pass /some-path %agent [our.bowl %spider] %poke %spider-stop !>([tid %.y)]

48+```

49+

50+### Subscribe to thread result

51+

52+Spider will send the result on `/thread-result/[tid]` so you can subscribe there for the result. You should subscribe before starting the thread.

53+

54+The result will have a mark of either `%thread-fail` or `%thread-done`.

55+

56+- `%thread-fail` - has a vase containing a `[term tang]` where `term` is an error message and `tang` is a traceback.

57+- `%thread-done` - has a vase of the result of the thread.

58+

59+#### Example

60+

61+```hoon

62+[%pass /some-path %agent [our.bowl %spider] %watch /thread-result/[tid]]

63+```

64+

65+### Subscribe to thread

66+

67+You can subscribe to a thread on `/thread/[tid]/path`. Note this is for facts sent off by the thread while it's running, not the final result. The path depends on the particular thread.

68+

69+#### Example

70+

71+```hoon

72+[%pass /some-path %agent [our.bowl %spider] %watch /thread/[tid]/thread-path]

73+```

74+

75+### Poke Thread

76+

77+To poke a thread you poke spider with a mark of `%spider-input` and a vase of `[tid cage]`.

78+

79+- `tid` is the tid of the thread you want to poke

80+- `cage` is whatever mark and vase you want to poke it with

81+

82+#### Example

83+

84+```hoon

85+[%pass /some-path %agent [our.bowl %spider] %poke %spider-input !>([tid %foo !>('foooo')])]

86+```

vere/_index.md+1

@@ -4,6 +4,7 @@ weight = 600

4sort_by = "weight"

5template = "sections/docs/chapters.html"

6aliases = ["/docs/learn/vere/"]

7+insert_anchor_links = "right"

7+++

8+++

8The Nock runtime system, written in C.

9The Nock runtime system, written in C.