facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-07 07:17:55 +03:00
Code Issues Projects Releases Wiki Activity

port upgrades of buffer.h into clib

Browse Source 
Summary:
buffer.h gained the ability to deal with non-char-sized buffers when I built cdatapack.  We need to update the callers in ctreemanifest to be aware of this.  Most of this is done with macro magic.

Some functionality was dropped from cdatapack's buffer.h (macro definitions to deal with paths).  Those are moved to path_buffer.h

Test Plan:
make local && clion build.
pass cfastmanifest unit tests.

Reviewers: #fastmanifest, durham

Reviewed By: durham

Subscribers: mitrandir, mjpieters

Differential Revision: https://phabricator.intern.facebook.com/D3780767

Signature: t1:3780767:1472255278:40a19edfd171df5804e9cdfa4444d5c6386f00e8
This commit is contained in:
Tony Tung 2016-08-26 17:14:52 -07:00
parent 2f5e85ccb8
commit 2b3e7ac198
12 changed files with 58 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -50,6 +50,7 @@ add_library(fastmanifest
clib/convert.h
cfastmanifest/internal_result.h
cfastmanifest/node.c
cfastmanifest/path_buffer.h
cfastmanifest/result.h
cfastmanifest/tree.c
cfastmanifest/tree.h
@ -62,7 +63,8 @@ add_library(fastmanifest
cfastmanifest/tree_iterator.c
cfastmanifest/tree_iterator.h
cfastmanifest/tree_path.c
cfastmanifest/tree_path.h)
cfastmanifest/tree_path.h
)
target_include_directories(fastmanifest PUBLIC /opt/local/include clib)
target_link_libraries(fastmanifest PUBLIC ${OPENSSL})
@ -170,12 +172,12 @@ target_include_directories(cdatapack PUBLIC ${PYTHON_INCLUDE_DIRS})
target_link_libraries(cdatapack PUBLIC ${PYTHON_LIBRARIES})
add_library(cdatapack_lib
remotefilelog/cdatapack/buffer.h
clib/buffer.h
remotefilelog/cdatapack/cdatapack.c
remotefilelog/cdatapack/cdatapack.h
remotefilelog/cdatapack/convert.h)
clib/convert.h)
target_include_directories(cdatapack_lib PUBLIC /opt/local/include)
target_include_directories(cdatapack_lib PUBLIC /opt/local/include clib)
target_link_libraries(cdatapack_lib PUBLIC ${LZ4})
add_executable(cdatapack_dump remotefilelog/cdatapack/cdatapack_dump.c)

25
cfastmanifest/path_buffer.h Normal file
View File

@ -0,0 +1,25 @@
// Copyright 2016-present Facebook. All Rights Reserved.
//
// path_buffer.h: macros for managing a path buffer.
//
// no-check-code
#ifndef CFASTMANIFEST_PATH_BUFFER_H
#define CFASTMANIFEST_PATH_BUFFER_H
#include "buffer.h"
// a common usage pattern for this module is to store a path. the path can
// be of any length, theoretically, so we have to support expansion.
#define DEFAULT_PATH_BUFFER_SZ 16384
#define PATH_BUFFER_GROWTH_FACTOR 1.2
#define PATH_BUFFER_MINIMUM_GROWTH 65536
#define PATH_BUFFER_MAXIMUM_GROWTH (1024 * 1024)
#define PATH_APPEND(buffer, buffer_idx, buffer_sz, input, input_sz) \
buffer_append(buffer, buffer_idx, buffer_sz, input, input_sz, \
PATH_BUFFER_GROWTH_FACTOR, \
PATH_BUFFER_MINIMUM_GROWTH, \
PATH_BUFFER_MAXIMUM_GROWTH)
#endif //CFASTMANIFEST_PATH_BUFFER_H

6
cfastmanifest/tree_convert.c
View File

@ -9,6 +9,7 @@
#include "buffer.h"
#include "convert.h"
#include "path_buffer.h"
#include "tree.h"
#include "tree_arena.h"
@ -20,7 +21,8 @@
#define BUFFER_MAXIMUM_GROWTH (32 * 1024 * 1024)
#define CONVERT_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \
expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz, \
expand_to_fit((void **) buffer, buffer_idx, buffer_sz, input_sz, \
sizeof(char), \
BUFFER_GROWTH_FACTOR, \
BUFFER_MINIMUM_GROWTH, \
BUFFER_MAXIMUM_GROWTH)
@ -460,7 +462,7 @@ static convert_to_flat_code_t convert_to_flat_iterator(
if (CONVERT_EXPAND_TO_FIT(
&state->output_buffer,
&state->output_buffer_idx,
state->output_buffer_idx,
&state->output_buffer_sz,
space_needed) == false) {
return CONVERT_TO_FLAT_OOM;

2
cfastmanifest/tree_copy.c
View File

@ -5,11 +5,11 @@
//
// no-check-code
#include "buffer.h"
#include "internal_result.h"
#include "node.h"
#include "tree.h"
#include "tree_arena.h"
#include "path_buffer.h"
typedef enum {
COPY_OK,

7
cfastmanifest/tree_diff.c
View File

@ -17,9 +17,10 @@
#define BUFFER_MINIMUM_GROWTH 16384
#define BUFFER_MAXIMUM_GROWTH 65536
#define DIFF_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \
expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz, \
#define DIFF_EXPAND_TO_FIT(buffer, buffer_idx, buffer_sz, input_sz) \
expand_to_fit((void **) buffer, buffer_idx, buffer_sz, input_sz, \
BUFFER_GROWTH_FACTOR, \
sizeof(char), \
BUFFER_MINIMUM_GROWTH, \
BUFFER_MAXIMUM_GROWTH)
@ -111,7 +112,7 @@ consider_children_result_t consider_children(
if (DIFF_EXPAND_TO_FIT(
&diff_context->path_build_buffer,
&diff_context->path_build_buffer_idx,
diff_context->path_build_buffer_idx,
&diff_context->path_build_buffer_sz,
name_sz) == false) {
return CONSIDER_PROCESSED_OOM;

2
cfastmanifest/tree_iterator.c
View File

@ -7,10 +7,10 @@
#include <stdlib.h>
#include "buffer.h"
#include "node.h"
#include "tree.h"
#include "tree_iterator.h"
#include "path_buffer.h"
#define DEFAULT_PATH_RECORDS_SZ 1024

4
clib/buffer.c
View File

@ -16,8 +16,8 @@ bool buffer_append(
const float factor,
const size_t min_increment,
const size_t max_increment) {
if (expand_to_fit(buffer, buffer_idx, buffer_sz, input_sz,
factor, min_increment, max_increment) == false) {
if (expand_to_fit((void **) buffer, *buffer_idx, buffer_sz, input_sz,
sizeof(char), factor, min_increment, max_increment) == false) {
return false;
}

39
clib/buffer.h
View File

@ -12,46 +12,33 @@
#include <stddef.h>
#include <stdlib.h>
// a common usage pattern for this module is to store a path. the path can
// be of any length, theoretically, so we have to support expansion.
#define DEFAULT_PATH_BUFFER_SZ 16384
#define PATH_BUFFER_GROWTH_FACTOR 1.2
#define PATH_BUFFER_MINIMUM_GROWTH 65536
#define PATH_BUFFER_MAXIMUM_GROWTH (1024 * 1024)
#define PATH_APPEND(buffer, buffer_idx, buffer_sz, input, input_sz) \
buffer_append(buffer, buffer_idx, buffer_sz, input, input_sz, \
PATH_BUFFER_GROWTH_FACTOR, \
PATH_BUFFER_MINIMUM_GROWTH, \
PATH_BUFFER_MAXIMUM_GROWTH)
static inline bool expand_to_fit(
char **buffer, size_t *buffer_idx, size_t *buffer_sz,
size_t input_sz,
void **buffer, size_t num_slots_used, size_t *num_slots_total,
size_t input_count, size_t item_sz,
const float factor,
const size_t min_increment,
const size_t max_increment) {
size_t remaining = *buffer_sz - *buffer_idx;
if (input_sz > remaining) {
size_t remaining = *num_slots_total - num_slots_used;
if (input_count > remaining) {
// need realloc
size_t new_sz = factor * ((float) *buffer_sz);
if (new_sz < min_increment + *buffer_sz) {
new_sz = min_increment + *buffer_sz;
size_t new_slots_total = factor * ((float) *num_slots_total);
if (new_slots_total < min_increment + *num_slots_total) {
new_slots_total = min_increment + *num_slots_total;
}
if (new_sz > max_increment + *buffer_sz) {
new_sz = max_increment + *buffer_sz;
if (new_slots_total > max_increment + *num_slots_total) {
new_slots_total = max_increment + *num_slots_total;
}
if (new_sz < input_sz + *buffer_sz) {
new_sz = input_sz + *buffer_sz;
if (new_slots_total < input_count + *num_slots_total) {
new_slots_total = input_count + *num_slots_total;
}
void *newbuffer = realloc(*buffer, new_sz);
void *newbuffer = realloc(*buffer, item_sz * new_slots_total);
if (newbuffer == NULL) {
return false;
}
*buffer = newbuffer;
*buffer_sz = new_sz;
*num_slots_total = new_slots_total;
}
return true;

47
remotefilelog/cdatapack/buffer.h
View File

@ -1,47 +0,0 @@
// Copyright 2016-present Facebook. All Rights Reserved.
//
// buffer.c: declarations for a generic mechanism to expand a heap-allocated
// buffer. this is for internal use only.
//
// no-check-code
#ifndef __FASTMANIFEST_BUFFER_H__
#define __FASTMANIFEST_BUFFER_H__
#include <stdbool.h>
#include <stddef.h>
#include <stdlib.h>
static inline bool expand_to_fit(
void **buffer, size_t num_slots_used, size_t *num_slots_total,
size_t input_count, size_t item_sz,
const float factor,
const size_t min_increment,
const size_t max_increment) {
size_t remaining = *num_slots_total - num_slots_used;
if (input_count > remaining) {
// need realloc
size_t new_slots_total = factor * ((float) *num_slots_total);
if (new_slots_total < min_increment + *num_slots_total) {
new_slots_total = min_increment + *num_slots_total;
}
if (new_slots_total > max_increment + *num_slots_total) {
new_slots_total = max_increment + *num_slots_total;
}
if (new_slots_total < input_count + *num_slots_total) {
new_slots_total = input_count + *num_slots_total;
}
void *newbuffer = realloc(*buffer, item_sz * new_slots_total);
if (newbuffer == NULL) {
return false;
}
*buffer = newbuffer;
*num_slots_total = new_slots_total;
}
return true;
}
#endif /* __FASTMANIFEST_BUFFER_H__ */

75
remotefilelog/cdatapack/convert.h
View File

@ -1,75 +0,0 @@
// Copyright 2016-present Facebook. All Rights Reserved.
//
// convert.h: hex-string conversions
//
// no-check-code
#ifndef __FASTMANIFEST_CONVERT_H__
#define __FASTMANIFEST_CONVERT_H__
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
static int8_t hextable[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, /* 0-9 */
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* A-F */
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* a-f */
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
static char chartable[16] = {
'0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
};
/*
* Turn a hex-encoded string into binary. Returns false on failure.
*/
static inline bool unhexlify(const char *input, int len, uint8_t *dst) {
if (len % 2 != 0) {
// wtf.
return false;
}
for (size_t ix = 0; ix < len; ix += 2, dst++) {
int hi = hextable[(unsigned char) input[ix]];
int lo = hextable[(unsigned char) input[ix + 1]];
if (hi < 0 || lo < 0) {
return false;
}
*dst = (hi << 4) | lo;
}
return true;
}
/*
* Turn binary data into a hex-encoded string.
*/
static inline void hexlify(const uint8_t *input, int len, char *dst) {
for (size_t ix = 0; ix < len; ix++, dst += 2) {
unsigned char ch = (unsigned char) input[ix];
char hi = chartable[ch >> 4];
char lo = chartable[ch & 0xf];
*dst = hi;
*(dst + 1) = lo;
}
}
#endif /* #ifndef __FASTMANIFEST_CONVERT_H__ */

1
setup.py
View File

@ -41,6 +41,7 @@ setup(
'remotefilelog/cdatapack/cdatapack.c',
],
include_dirs=[
'clib',
'remotefilelog/cdatapack',
'/usr/local/include',
'/opt/local/include',

3
tests/test-check-code-hg.t
View File

@ -20,6 +20,7 @@ New errors are not allowed. Warnings are strongly discouraged.
Skipping cfastmanifest/node.c it has no-che?k-code (glob)
Skipping cfastmanifest/node.h it has no-che?k-code (glob)
Skipping cfastmanifest/node_test.c it has no-che?k-code (glob)
Skipping cfastmanifest/path_buffer.h it has no-che?k-code (glob)
Skipping cfastmanifest/result.h it has no-che?k-code (glob)
Skipping cfastmanifest/tests.c it has no-che?k-code (glob)
Skipping cfastmanifest/tests.h it has no-che?k-code (glob)
@ -48,12 +49,10 @@ New errors are not allowed. Warnings are strongly discouraged.
Skipping clib/buffer.h it has no-che?k-code (glob)
Skipping clib/convert.h it has no-che?k-code (glob)
Skipping clib/null_test.c it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/buffer.h it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/cdatapack.c it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/cdatapack.h it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/cdatapack_dump.c it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/cdatapack_get.c it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/convert.h it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/null_test.c it has no-che?k-code (glob)
Skipping remotefilelog/cdatapack/py-cdatapack.c it has no-che?k-code (glob)
Skipping remotefilelog/ctreemanifest/convert.h it has no-che?k-code (glob)