diff --git a/CMakeLists.txt b/CMakeLists.txt index 359284499e..efd838bbf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ add_library(fastmanifest clib/convert.h cfastmanifest/internal_result.h cfastmanifest/node.c + cfastmanifest/path_buffer.h cfastmanifest/result.h cfastmanifest/tree.c cfastmanifest/tree.h @@ -62,7 +63,8 @@ add_library(fastmanifest cfastmanifest/tree_iterator.c cfastmanifest/tree_iterator.h cfastmanifest/tree_path.c - cfastmanifest/tree_path.h) + cfastmanifest/tree_path.h + ) target_include_directories(fastmanifest PUBLIC /opt/local/include clib) target_link_libraries(fastmanifest PUBLIC ${OPENSSL}) @@ -170,12 +172,12 @@ target_include_directories(cdatapack PUBLIC ${PYTHON_INCLUDE_DIRS}) target_link_libraries(cdatapack PUBLIC ${PYTHON_LIBRARIES}) add_library(cdatapack_lib - remotefilelog/cdatapack/buffer.h + clib/buffer.h remotefilelog/cdatapack/cdatapack.c remotefilelog/cdatapack/cdatapack.h - remotefilelog/cdatapack/convert.h) + clib/convert.h) -target_include_directories(cdatapack_lib PUBLIC /opt/local/include) +target_include_directories(cdatapack_lib PUBLIC /opt/local/include clib) target_link_libraries(cdatapack_lib PUBLIC ${LZ4}) add_executable(cdatapack_dump remotefilelog/cdatapack/cdatapack_dump.c) diff --git a/cfastmanifest/path_buffer.h b/cfastmanifest/path_buffer.h new file mode 100644 index 0000000000..0e6ca5cf38 --- /dev/null +++ b/cfastmanifest/path_buffer.h @@ -0,0 +1,25 @@ +// Copyright 2016-present Facebook. All Rights Reserved. +// +// path_buffer.h: macros for managing a path buffer. +// +// no-check-code + +#ifndef CFASTMANIFEST_PATH_BUFFER_H +#define CFASTMANIFEST_PATH_BUFFER_H + +#include "buffer.h" + +// a common usage pattern for this module is to store a path. the path can +// be of any length, theoretically, so we have to support expansion. +#define DEFAULT_PATH_BUFFER_SZ 16384 +#define PATH_BUFFER_GROWTH_FACTOR 1.2 +#define PATH_BUFFER_MINIMUM_GROWTH 65536 +#define PATH_BUFFER_MAXIMUM_GROWTH (1024 * 1024) + +#define PATH_APPEND(buffer, buffer_idx, buffer_sz, input, input_sz) \ + buffer_append(buffer, buffer_idx, buffer_sz, input, input_sz, \ + PATH_BUFFER_GROWTH_FACTOR, \ + PATH_BUFFER_MINIMUM_GROWTH, \ + PATH_BUFFER_MAXIMUM_GROWTH) + +#endif //CFASTMANIFEST_PATH_BUFFER_H diff --git a/cfastmanifest/tree_convert.c b/cfastmanifest/tree_convert.c index 9078a9ab35..86c9fae531 100644 --- a/cfastmanifest/tree_convert.c +++ b/cfastmanifest/tree_convert.c @@ -9,6 +9,7 @@ #include "buffer.h" #include "convert.h" +#include "path_buffer.h" #include "tree.h" #include "tree_arena.h" @@ -20,7 +21,8 @@ #define BUFFER_MAXIMUM_GROWTH (32 * 1024 * 1024) #define CONVERT_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \ - expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz, \ + expand_to_fit((void **) buffer, buffer_idx, buffer_sz, input_sz, \ + sizeof(char), \ BUFFER_GROWTH_FACTOR, \ BUFFER_MINIMUM_GROWTH, \ BUFFER_MAXIMUM_GROWTH) @@ -460,7 +462,7 @@ static convert_to_flat_code_t convert_to_flat_iterator( if (CONVERT_EXPAND_TO_FIT( &state->output_buffer, - &state->output_buffer_idx, + state->output_buffer_idx, &state->output_buffer_sz, space_needed) == false) { return CONVERT_TO_FLAT_OOM; diff --git a/cfastmanifest/tree_copy.c b/cfastmanifest/tree_copy.c index 55f3ee41df..442cea05d4 100644 --- a/cfastmanifest/tree_copy.c +++ b/cfastmanifest/tree_copy.c @@ -5,11 +5,11 @@ // // no-check-code -#include "buffer.h" #include "internal_result.h" #include "node.h" #include "tree.h" #include "tree_arena.h" +#include "path_buffer.h" typedef enum { COPY_OK, diff --git a/cfastmanifest/tree_diff.c b/cfastmanifest/tree_diff.c index 1fb41b6803..ce5a4eceac 100644 --- a/cfastmanifest/tree_diff.c +++ b/cfastmanifest/tree_diff.c @@ -17,9 +17,10 @@ #define BUFFER_MINIMUM_GROWTH 16384 #define BUFFER_MAXIMUM_GROWTH 65536 -#define DIFF_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \ - expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz, \ +#define DIFF_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \ + expand_to_fit((void **) buffer, buffer_idx, buffer_sz, input_sz, \ BUFFER_GROWTH_FACTOR, \ + sizeof(char), \ BUFFER_MINIMUM_GROWTH, \ BUFFER_MAXIMUM_GROWTH) @@ -111,7 +112,7 @@ consider_children_result_t consider_children( if (DIFF_EXPAND_TO_FIT( &diff_context->path_build_buffer, - &diff_context->path_build_buffer_idx, + diff_context->path_build_buffer_idx, &diff_context->path_build_buffer_sz, name_sz) == false) { return CONSIDER_PROCESSED_OOM; diff --git a/cfastmanifest/tree_iterator.c b/cfastmanifest/tree_iterator.c index 702786350b..f26b50f311 100644 --- a/cfastmanifest/tree_iterator.c +++ b/cfastmanifest/tree_iterator.c @@ -7,10 +7,10 @@ #include -#include "buffer.h" #include "node.h" #include "tree.h" #include "tree_iterator.h" +#include "path_buffer.h" #define DEFAULT_PATH_RECORDS_SZ 1024 diff --git a/clib/buffer.c b/clib/buffer.c index 721cf92cd8..b5e5c884e9 100644 --- a/clib/buffer.c +++ b/clib/buffer.c @@ -16,8 +16,8 @@ bool buffer_append( const float factor, const size_t min_increment, const size_t max_increment) { - if (expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz, - factor, min_increment, max_increment) == false) { + if (expand_to_fit((void **) buffer, *buffer_idx, buffer_sz, input_sz, + sizeof(char), factor, min_increment, max_increment) == false) { return false; } diff --git a/clib/buffer.h b/clib/buffer.h index 1b112af287..3cbc4b5469 100644 --- a/clib/buffer.h +++ b/clib/buffer.h @@ -12,46 +12,33 @@ #include #include -// a common usage pattern for this module is to store a path. the path can -// be of any length, theoretically, so we have to support expansion. -#define DEFAULT_PATH_BUFFER_SZ 16384 -#define PATH_BUFFER_GROWTH_FACTOR 1.2 -#define PATH_BUFFER_MINIMUM_GROWTH 65536 -#define PATH_BUFFER_MAXIMUM_GROWTH (1024 * 1024) - -#define PATH_APPEND(buffer, buffer_idx, buffer_sz, input, input_sz) \ - buffer_append(buffer, buffer_idx, buffer_sz, input, input_sz, \ - PATH_BUFFER_GROWTH_FACTOR, \ - PATH_BUFFER_MINIMUM_GROWTH, \ - PATH_BUFFER_MAXIMUM_GROWTH) - static inline bool expand_to_fit( - char **buffer, size_t *buffer_idx, size_t *buffer_sz, - size_t input_sz, + void **buffer, size_t num_slots_used, size_t *num_slots_total, + size_t input_count, size_t item_sz, const float factor, const size_t min_increment, const size_t max_increment) { - size_t remaining = *buffer_sz - *buffer_idx; - if (input_sz > remaining) { + size_t remaining = *num_slots_total - num_slots_used; + if (input_count > remaining) { // need realloc - size_t new_sz = factor * ((float) *buffer_sz); - if (new_sz < min_increment + *buffer_sz) { - new_sz = min_increment + *buffer_sz; + size_t new_slots_total = factor * ((float) *num_slots_total); + if (new_slots_total < min_increment + *num_slots_total) { + new_slots_total = min_increment + *num_slots_total; } - if (new_sz > max_increment + *buffer_sz) { - new_sz = max_increment + *buffer_sz; + if (new_slots_total > max_increment + *num_slots_total) { + new_slots_total = max_increment + *num_slots_total; } - if (new_sz < input_sz + *buffer_sz) { - new_sz = input_sz + *buffer_sz; + if (new_slots_total < input_count + *num_slots_total) { + new_slots_total = input_count + *num_slots_total; } - void *newbuffer = realloc(*buffer, new_sz); + void *newbuffer = realloc(*buffer, item_sz * new_slots_total); if (newbuffer == NULL) { return false; } *buffer = newbuffer; - *buffer_sz = new_sz; + *num_slots_total = new_slots_total; } return true; diff --git a/remotefilelog/cdatapack/buffer.h b/remotefilelog/cdatapack/buffer.h deleted file mode 100644 index dc91a3d7cf..0000000000 --- a/remotefilelog/cdatapack/buffer.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2016-present Facebook. All Rights Reserved. -// -// buffer.c: declarations for a generic mechanism to expand a heap-allocated -// buffer. this is for internal use only. -// -// no-check-code - -#ifndef __FASTMANIFEST_BUFFER_H__ -#define __FASTMANIFEST_BUFFER_H__ - -#include -#include -#include - -static inline bool expand_to_fit( - void **buffer, size_t num_slots_used, size_t *num_slots_total, - size_t input_count, size_t item_sz, - const float factor, - const size_t min_increment, - const size_t max_increment) { - size_t remaining = *num_slots_total - num_slots_used; - if (input_count > remaining) { - // need realloc - size_t new_slots_total = factor * ((float) *num_slots_total); - if (new_slots_total < min_increment + *num_slots_total) { - new_slots_total = min_increment + *num_slots_total; - } - if (new_slots_total > max_increment + *num_slots_total) { - new_slots_total = max_increment + *num_slots_total; - } - if (new_slots_total < input_count + *num_slots_total) { - new_slots_total = input_count + *num_slots_total; - } - - void *newbuffer = realloc(*buffer, item_sz * new_slots_total); - if (newbuffer == NULL) { - return false; - } - - *buffer = newbuffer; - *num_slots_total = new_slots_total; - } - - return true; -} - -#endif /* __FASTMANIFEST_BUFFER_H__ */ diff --git a/remotefilelog/cdatapack/convert.h b/remotefilelog/cdatapack/convert.h deleted file mode 100644 index 00b69b6eda..0000000000 --- a/remotefilelog/cdatapack/convert.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2016-present Facebook. All Rights Reserved. -// -// convert.h: hex-string conversions -// -// no-check-code - -#ifndef __FASTMANIFEST_CONVERT_H__ -#define __FASTMANIFEST_CONVERT_H__ - -#include -#include -#include - -static int8_t hextable[256] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, /* 0-9 */ - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* A-F */ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* a-f */ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 -}; - -static char chartable[16] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' -}; - -/* - * Turn a hex-encoded string into binary. Returns false on failure. - */ -static inline bool unhexlify(const char *input, int len, uint8_t *dst) { - if (len % 2 != 0) { - // wtf. - return false; - } - - for (size_t ix = 0; ix < len; ix += 2, dst++) { - int hi = hextable[(unsigned char) input[ix]]; - int lo = hextable[(unsigned char) input[ix + 1]]; - - if (hi < 0 || lo < 0) { - return false; - } - *dst = (hi << 4) | lo; - } - - return true; -} - -/* - * Turn binary data into a hex-encoded string. - */ -static inline void hexlify(const uint8_t *input, int len, char *dst) { - for (size_t ix = 0; ix < len; ix++, dst += 2) { - unsigned char ch = (unsigned char) input[ix]; - char hi = chartable[ch >> 4]; - char lo = chartable[ch & 0xf]; - - *dst = hi; - *(dst + 1) = lo; - } -} - - -#endif /* #ifndef __FASTMANIFEST_CONVERT_H__ */ diff --git a/setup.py b/setup.py index 6b90b2cc87..ce70d80c59 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( 'remotefilelog/cdatapack/cdatapack.c', ], include_dirs=[ + 'clib', 'remotefilelog/cdatapack', '/usr/local/include', '/opt/local/include', diff --git a/tests/test-check-code-hg.t b/tests/test-check-code-hg.t index 1747fbe0da..fb90a72181 100644 --- a/tests/test-check-code-hg.t +++ b/tests/test-check-code-hg.t @@ -20,6 +20,7 @@ New errors are not allowed. Warnings are strongly discouraged. Skipping cfastmanifest/node.c it has no-che?k-code (glob) Skipping cfastmanifest/node.h it has no-che?k-code (glob) Skipping cfastmanifest/node_test.c it has no-che?k-code (glob) + Skipping cfastmanifest/path_buffer.h it has no-che?k-code (glob) Skipping cfastmanifest/result.h it has no-che?k-code (glob) Skipping cfastmanifest/tests.c it has no-che?k-code (glob) Skipping cfastmanifest/tests.h it has no-che?k-code (glob) @@ -48,12 +49,10 @@ New errors are not allowed. Warnings are strongly discouraged. Skipping clib/buffer.h it has no-che?k-code (glob) Skipping clib/convert.h it has no-che?k-code (glob) Skipping clib/null_test.c it has no-che?k-code (glob) - Skipping remotefilelog/cdatapack/buffer.h it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/cdatapack.c it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/cdatapack.h it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/cdatapack_dump.c it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/cdatapack_get.c it has no-che?k-code (glob) - Skipping remotefilelog/cdatapack/convert.h it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/null_test.c it has no-che?k-code (glob) Skipping remotefilelog/cdatapack/py-cdatapack.c it has no-che?k-code (glob) Skipping remotefilelog/ctreemanifest/convert.h it has no-che?k-code (glob)