2016-04-01 10:46:28 +03:00
|
|
|
// Copyright 2016-present Facebook. All Rights Reserved.
|
|
|
|
//
|
|
|
|
// node.h: declarations for representing a node in a tree. for internal use
|
|
|
|
// only.
|
2016-04-01 21:19:53 +03:00
|
|
|
//
|
|
|
|
// no-check-code
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
#ifndef __FASTMANIFEST_NODE_H__
|
|
|
|
#define __FASTMANIFEST_NODE_H__
|
|
|
|
|
|
|
|
#include <assert.h>
|
|
|
|
#include <stdbool.h>
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <stdint.h>
|
2016-04-25 21:58:38 +03:00
|
|
|
#include <stdlib.h>
|
2016-04-01 10:46:28 +03:00
|
|
|
#include <string.h>
|
|
|
|
|
|
|
|
#include "internal_result.h"
|
|
|
|
|
|
|
|
#define STORAGE_INCREMENT_PERCENTAGE 20
|
|
|
|
#define MIN_STORAGE_INCREMENT 10
|
|
|
|
#define MAX_STORAGE_INCREMENT 100
|
|
|
|
|
|
|
|
#define CHECKSUM_BYTES 21
|
|
|
|
#define SHA1_BYTES 20
|
|
|
|
|
|
|
|
#define PTR_ALIGN_MASK (~ ((ptrdiff_t) (sizeof(ptrdiff_t) - 1)))
|
|
|
|
|
|
|
|
#define TYPE_UNDEFINED 0
|
|
|
|
#define TYPE_IMPLICIT 1
|
|
|
|
#define TYPE_LEAF 2
|
2016-04-05 02:24:22 +03:00
|
|
|
#define TYPE_ROOT 3
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
// the start of each of these nodes must be 32-bit aligned.
|
|
|
|
typedef struct _node_t {
|
|
|
|
uint32_t block_sz;
|
|
|
|
uint32_t num_children;
|
|
|
|
uint16_t name_sz;
|
|
|
|
uint8_t checksum[CHECKSUM_BYTES];
|
|
|
|
uint8_t checksum_sz;
|
|
|
|
uint8_t flags;
|
|
|
|
bool in_use : 1;
|
|
|
|
unsigned int type : 2;
|
|
|
|
bool checksum_valid : 1;
|
[fastmanifest] persistence layer for trees
Summary:
API to write trees to disk and read trees from disk.
Some interesting things to keep in mind:
* To keep valgrind happy, we have to clear all the memory we allocate but never initialize. That can be padding between fields in structs, fields that don't make sense to initialize, the extra byte in checksums that doesn't get used normally, or portions of bitfields that are never used. All this logic is kept compartmentalized in `initialize_unused_bytes(..)`. Normal runtime does not invoke this code, though I suspect the cost is probably not significant.
* A significant chunk of the write path is made up of the `CHECKED_WRITE_XXX` macros. They're calls to ensure that writes succeed. If not, they jump to a fixed exit point.
* The write path is not very optimized. We can use a bottom-up traversal, similar to tree_convert, to make the write less costly. However, our design doesn't require a fast write, so tentatively, I'm going to reuse `tree_copy(..)` to compact a tree.
Depends on D3255048
Test Plan: pass unit tests!
Reviewers: lcharignon, wez
Reviewed By: wez
Subscribers: mitrandir, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3255656
Tasks: 10589048
Signature: t1:3255656:1462342423:c28d32c610b2351614d7648c03f35f931370a770
2016-05-05 00:11:55 +03:00
|
|
|
unsigned int unused : 4;
|
2016-04-01 10:46:28 +03:00
|
|
|
char name[0];
|
|
|
|
// padding to the nearest ptrdiff_t boundary.
|
|
|
|
// then a series of ptrdiff_t-sized pointers to the children.
|
|
|
|
} node_t;
|
|
|
|
|
2016-04-25 21:58:38 +03:00
|
|
|
/**
|
|
|
|
* Define some macros for users to test if their values are within the
|
|
|
|
* restrictions of our node implementation.
|
|
|
|
*/
|
|
|
|
#define VERIFY_BLOCK_SZ(block_sz) ((uintmax_t) (block_sz) < UINT32_MAX)
|
|
|
|
#define VERIFY_NAME_SZ(name_sz) ((uintmax_t) (name_sz) < UINT16_MAX)
|
|
|
|
#define VERIFY_CHILD_NUM(child_num) ((uintmax_t) (child_num) < UINT32_MAX)
|
|
|
|
|
|
|
|
#define block_sz_t uint32_t
|
|
|
|
#define name_sz_t uint16_t
|
|
|
|
#define child_num_t uint32_t
|
|
|
|
|
2016-04-01 10:46:28 +03:00
|
|
|
/**
|
|
|
|
* Returns <0 if (`name`, `name_sz`) is lexicographically less than the name in
|
|
|
|
* node.
|
|
|
|
*
|
|
|
|
* Returns =0 if (`name`, `name_sz`) is lexicographically equal to the name in
|
|
|
|
* node.
|
|
|
|
*
|
|
|
|
* Returns >0 if (`name`, `name_sz`) is lexicographically greater than the name
|
|
|
|
* in node.
|
|
|
|
*/
|
|
|
|
static inline int name_compare(
|
2016-04-09 08:33:07 +03:00
|
|
|
const char *name,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint16_t name_sz,
|
2016-04-09 08:33:07 +03:00
|
|
|
const node_t *node) {
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t min_sz = (name_sz < node->name_sz) ? name_sz : node->name_sz;
|
|
|
|
int sz_compare = name_sz - node->name_sz;
|
|
|
|
|
|
|
|
int cmp = strncmp(name, node->name, min_sz);
|
|
|
|
if (cmp) {
|
|
|
|
return cmp;
|
|
|
|
} else {
|
|
|
|
return sz_compare;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Returns the offset of the first child pointer, given a node with name size
|
|
|
|
* `name_sz`.
|
|
|
|
*/
|
|
|
|
static inline ptrdiff_t get_child_ptr_base_offset(
|
|
|
|
uint16_t name_sz) {
|
2016-05-05 08:20:17 +03:00
|
|
|
intptr_t ptr = offsetof(node_t, name);
|
|
|
|
ptr += name_sz;
|
|
|
|
|
|
|
|
// this aligns to ptrdiff_t, since some platforms do not support unaligned
|
|
|
|
// loads.
|
2016-04-01 10:46:28 +03:00
|
|
|
ptr = (ptr + sizeof(intptr_t) - 1) & PTR_ALIGN_MASK;
|
|
|
|
|
|
|
|
return (ptrdiff_t) ptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Returns the address of the first child pointer. Since a child pointer is an
|
|
|
|
* ptrdiff_t, the type returned is an ptrdiff_t. Note that this is *not* the
|
|
|
|
* value of the first child pointer.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline ptrdiff_t *get_child_ptr_base(node_t *node) {
|
2016-04-01 10:46:28 +03:00
|
|
|
assert(node->in_use);
|
|
|
|
|
|
|
|
intptr_t address = (intptr_t) node;
|
|
|
|
ptrdiff_t offset = get_child_ptr_base_offset(node->name_sz);
|
2016-04-09 08:33:07 +03:00
|
|
|
return (ptrdiff_t *) (address + offset);
|
2016-04-01 10:46:28 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Const version of get_child_ptr_base
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline const ptrdiff_t *get_child_ptr_base_const(const node_t *node) {
|
|
|
|
return get_child_ptr_base((node_t *) node);
|
2016-04-01 10:46:28 +03:00
|
|
|
}
|
|
|
|
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline uint32_t max_children(const node_t *node) {
|
2016-04-01 10:46:28 +03:00
|
|
|
ptrdiff_t bytes_avail = node->block_sz;
|
2016-04-09 08:33:07 +03:00
|
|
|
bytes_avail -=
|
|
|
|
((intptr_t) get_child_ptr_base_const(node)) - ((intptr_t) node);
|
2016-04-25 21:58:38 +03:00
|
|
|
|
|
|
|
// if it requires > 32b, then we're kind of hosed.
|
|
|
|
if (!VERIFY_CHILD_NUM(bytes_avail)) {
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
return ((uint32_t) (bytes_avail / sizeof(intptr_t)));
|
2016-04-01 10:46:28 +03:00
|
|
|
}
|
|
|
|
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline node_t *get_child_by_index(
|
|
|
|
const node_t *node,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t child_num) {
|
|
|
|
assert(node->in_use);
|
2016-04-05 02:24:22 +03:00
|
|
|
assert(node->type == TYPE_IMPLICIT || node->type == TYPE_ROOT);
|
2016-04-01 10:46:28 +03:00
|
|
|
assert(child_num < node->num_children);
|
|
|
|
|
|
|
|
intptr_t address = (intptr_t) get_child_ptr_base_const(node);
|
|
|
|
address += sizeof(ptrdiff_t) * child_num;
|
|
|
|
|
|
|
|
intptr_t base = (intptr_t) node;
|
2016-04-09 08:33:07 +03:00
|
|
|
ptrdiff_t offset = *((ptrdiff_t *) address);
|
2016-04-01 10:46:28 +03:00
|
|
|
base += offset;
|
2016-04-09 08:33:07 +03:00
|
|
|
return (node_t *) base;
|
2016-04-01 10:46:28 +03:00
|
|
|
}
|
|
|
|
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline node_t *get_child_from_diff(const node_t *node, ptrdiff_t diff) {
|
2016-04-01 10:46:28 +03:00
|
|
|
assert(node->in_use);
|
2016-04-05 02:24:22 +03:00
|
|
|
assert(node->type == TYPE_IMPLICIT || node->type == TYPE_ROOT);
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
intptr_t base = (intptr_t) node;
|
|
|
|
base += diff;
|
2016-04-09 08:33:07 +03:00
|
|
|
return (node_t *) base;
|
2016-04-01 10:46:28 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void set_child_by_index(
|
|
|
|
node_t *node,
|
|
|
|
size_t child_num,
|
|
|
|
const node_t *child) {
|
|
|
|
assert(node->in_use);
|
2016-04-05 02:24:22 +03:00
|
|
|
assert(node->type == TYPE_IMPLICIT || node->type == TYPE_ROOT);
|
2016-04-01 10:46:28 +03:00
|
|
|
assert(child_num < node->num_children);
|
|
|
|
assert(child->in_use);
|
|
|
|
|
2016-04-09 08:33:07 +03:00
|
|
|
ptrdiff_t *base = get_child_ptr_base(node);
|
2016-04-01 10:46:28 +03:00
|
|
|
ptrdiff_t delta = ((intptr_t) child) - ((intptr_t) node);
|
|
|
|
base[child_num] = delta;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Allocate a node on the heap suitably sized for a given name and a given
|
|
|
|
* number of children. Initialize the node as unused, but copy the name to the
|
|
|
|
* node.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
extern node_t *alloc_node(
|
|
|
|
const char *name, uint16_t name_sz,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t max_children
|
2016-04-09 08:33:07 +03:00
|
|
|
);
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Given a block of memory, attempt to place a node at the start of the block.
|
|
|
|
* The node will suitably sized for a given name and a given number of children.
|
|
|
|
* Initialize the node as unused, but copy the name to the node.
|
|
|
|
*
|
|
|
|
* Returns the address following the end of the node if the block is large
|
|
|
|
* enough to accommodate the node, or NULL if the block is too small.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
extern void *setup_node(
|
|
|
|
void *ptr, size_t ptr_size_limit,
|
|
|
|
const char *name, uint16_t name_sz,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t max_children);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Clone a node and increase the storage capacity by
|
|
|
|
* STORAGE_INCREMENT_PERCENTAGE, but by at least MIN_STORAGE_INCREMENT and no
|
|
|
|
* more than MAX_STORAGE_INCREMENT.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
extern node_t *clone_node(const node_t *node);
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Adds a child to the node. A child with the same name must not already exist.
|
|
|
|
*
|
|
|
|
* The caller is responsible for going up the chain and updating metadata, such
|
|
|
|
* as the total number of leaf nodes in tree_t and marking the checksum bit
|
|
|
|
* dirty recursively up the tree.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
extern node_add_child_result_t add_child(node_t *node, const node_t *child);
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Remove a child of a node, given a child index.
|
|
|
|
*
|
|
|
|
* The caller is responsible for going up the chain and updating metadata, such
|
|
|
|
* as the total number of leaf nodes in tree_t and marking the checksum bit
|
|
|
|
* dirty recursively up the tree.
|
|
|
|
*/
|
|
|
|
extern node_remove_child_result_t remove_child(
|
2016-04-09 08:33:07 +03:00
|
|
|
node_t *node,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t child_num);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Enlarge a child of a node, given a child index. By itself, this operation
|
|
|
|
* should not affect things like the total number of leaf nodes in the tree and
|
|
|
|
* the freshness of the checksums. However, it may affect total allocation.
|
|
|
|
*/
|
|
|
|
extern node_enlarge_child_capacity_result_t enlarge_child_capacity(
|
2016-04-09 08:33:07 +03:00
|
|
|
node_t *node,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint32_t child_num);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Find the index of a child given a name. Returns true iff the child was
|
|
|
|
* found.
|
|
|
|
*
|
|
|
|
* If the child was found, return the index and the pointer to the child.
|
|
|
|
*/
|
|
|
|
extern node_search_children_result_t search_children(
|
2016-04-09 08:33:07 +03:00
|
|
|
const node_t *node,
|
|
|
|
const char *name,
|
2016-04-01 10:46:28 +03:00
|
|
|
const uint16_t name_sz);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Find the index of a child given a node. If the node is found, return its
|
|
|
|
* index. Otherwise return UINT32_MAX.
|
|
|
|
*/
|
|
|
|
extern uint32_t get_child_index(
|
2016-04-09 08:33:07 +03:00
|
|
|
const node_t *const parent,
|
|
|
|
const node_t *const child);
|
2016-04-01 10:46:28 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Convenience function just to find a child.
|
|
|
|
*/
|
2016-04-09 08:33:07 +03:00
|
|
|
static inline node_t *get_child_by_name(
|
|
|
|
const node_t *node,
|
|
|
|
const char *name,
|
2016-04-01 10:46:28 +03:00
|
|
|
uint16_t name_sz) {
|
|
|
|
node_search_children_result_t result = search_children(node, name, name_sz);
|
|
|
|
|
|
|
|
return result.child;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* #ifndef __FASTMANIFEST_NODE_H__ */
|