2017-08-26 02:46:07 +03:00
|
|
|
// Copyright (c) 2004-present, Facebook, Inc.
|
|
|
|
// All Rights Reserved.
|
2016-08-02 00:05:37 +03:00
|
|
|
//
|
2017-08-26 02:46:07 +03:00
|
|
|
// This software may be used and distributed according to the terms of the
|
|
|
|
// GNU General Public License version 2 or any later version.
|
|
|
|
|
2017-04-06 19:33:34 +03:00
|
|
|
// cdatapack:
|
2016-08-02 00:05:37 +03:00
|
|
|
// no-check-code
|
|
|
|
|
2017-08-26 02:46:07 +03:00
|
|
|
#ifndef FBHGEXT_CDATAPACK_CDATAPACK_H
|
|
|
|
#define FBHGEXT_CDATAPACK_CDATAPACK_H
|
2016-08-02 00:05:37 +03:00
|
|
|
|
|
|
|
#include <stdbool.h>
|
2017-08-26 02:46:07 +03:00
|
|
|
#include <stddef.h>
|
|
|
|
#include <stdint.h>
|
2016-11-29 01:24:45 +03:00
|
|
|
#include <sys/types.h>
|
2016-08-02 00:05:37 +03:00
|
|
|
|
2017-08-26 02:46:07 +03:00
|
|
|
#include "clib/portability/portability.h"
|
2017-04-06 19:33:34 +03:00
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
#define NODE_SZ 20
|
|
|
|
|
2017-02-24 01:03:03 +03:00
|
|
|
#define PACKSUFFIX ".datapack"
|
|
|
|
#define PACKSUFFIXLEN 9
|
|
|
|
#define INDEXSUFFIX ".dataidx"
|
|
|
|
#define INDEXSUFFIXLEN 8
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
typedef uint32_t index_offset_t;
|
|
|
|
#define ntoh_index_offset ntohl
|
|
|
|
#define FULLTEXTINDEXMARK ((index_offset_t) -1)
|
|
|
|
#define NOBASEINDEXMARK ((index_offset_t) -2)
|
|
|
|
typedef uint64_t data_offset_t;
|
|
|
|
|
|
|
|
struct _disk_index_entry_t;
|
|
|
|
struct _fanout_table_entry_t;
|
|
|
|
|
2016-08-04 23:48:32 +03:00
|
|
|
/**
|
|
|
|
* This is a post-processed index entry. The node pointer is valid only if
|
|
|
|
* the handle that generated this entry hasn't been closed.
|
|
|
|
*
|
|
|
|
* This is the counterpart of disk_index_entry_t.
|
|
|
|
*/
|
|
|
|
typedef struct _pack_index_entry_t {
|
|
|
|
const uint8_t *node;
|
|
|
|
|
|
|
|
// offset and size of this current element in the delta chain in the data
|
|
|
|
// file.
|
|
|
|
data_offset_t data_offset;
|
|
|
|
data_offset_t data_sz;
|
|
|
|
|
|
|
|
// offset of the next element in the delta chain in the index file
|
|
|
|
index_offset_t deltabase_index_offset;
|
|
|
|
} pack_index_entry_t;
|
|
|
|
|
2016-08-30 07:49:26 +03:00
|
|
|
typedef enum {
|
|
|
|
DATAPACK_HANDLE_OK,
|
|
|
|
DATAPACK_HANDLE_OOM,
|
|
|
|
DATAPACK_HANDLE_IO_ERROR,
|
|
|
|
DATAPACK_HANDLE_MMAP_ERROR,
|
|
|
|
DATAPACK_HANDLE_CORRUPT,
|
|
|
|
DATAPACK_HANDLE_VERSION_MISMATCH,
|
|
|
|
} datapack_handle_status_t;
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
typedef struct _datapack_handle_t {
|
2016-08-30 07:49:26 +03:00
|
|
|
datapack_handle_status_t status;
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
void* index_mmap;
|
|
|
|
void* data_mmap;
|
|
|
|
off_t index_file_sz;
|
|
|
|
off_t data_file_sz;
|
|
|
|
|
|
|
|
bool large_fanout;
|
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
uint8_t version;
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
// this is the computed fanout table.
|
|
|
|
struct _fanout_table_entry_t *fanout_table;
|
|
|
|
|
|
|
|
// this points to the first index entry.
|
|
|
|
struct _disk_index_entry_t* index_table;
|
2016-08-31 02:21:24 +03:00
|
|
|
|
|
|
|
size_t paged_in_datapack_memory;
|
2016-08-02 00:05:37 +03:00
|
|
|
} datapack_handle_t;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This represents a single entry in a delta chain.
|
|
|
|
*/
|
|
|
|
typedef struct _delta_chain_link_t {
|
|
|
|
uint16_t filename_sz;
|
|
|
|
const char *filename;
|
|
|
|
const uint8_t *node;
|
|
|
|
const uint8_t *deltabase_node;
|
|
|
|
|
2017-05-01 23:03:38 +03:00
|
|
|
data_offset_t compressed_sz;
|
|
|
|
const uint8_t *compressed_buf;
|
|
|
|
|
|
|
|
/* delta is (lazily) uncompressed from compressed_buf
|
|
|
|
* allocated by uncompressdeltachainlink, and freed by caller */
|
2016-08-02 00:05:37 +03:00
|
|
|
data_offset_t delta_sz;
|
|
|
|
const uint8_t *delta;
|
2017-04-27 05:50:36 +03:00
|
|
|
|
|
|
|
uint32_t meta_sz;
|
|
|
|
const uint8_t *meta;
|
2016-08-02 00:05:37 +03:00
|
|
|
} delta_chain_link_t;
|
|
|
|
|
2016-08-30 07:46:43 +03:00
|
|
|
typedef enum {
|
|
|
|
GET_DELTA_CHAIN_OK,
|
|
|
|
GET_DELTA_CHAIN_OOM,
|
|
|
|
GET_DELTA_CHAIN_NOT_FOUND,
|
|
|
|
GET_DELTA_CHAIN_CORRUPT,
|
|
|
|
} get_delta_chain_code_t;
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
/**
|
|
|
|
* This represents an entire delta chain.
|
|
|
|
*/
|
|
|
|
typedef struct _delta_chain_t {
|
2016-08-30 07:46:43 +03:00
|
|
|
get_delta_chain_code_t code;
|
2016-08-02 00:05:37 +03:00
|
|
|
delta_chain_link_t *delta_chain_links;
|
|
|
|
size_t links_count;
|
|
|
|
} delta_chain_t;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Open a datapack + index file. The fanout table is read and processed at
|
|
|
|
* this point.
|
|
|
|
*
|
|
|
|
* Returns a handle for subsequent operations.
|
|
|
|
*/
|
|
|
|
extern datapack_handle_t *open_datapack(
|
|
|
|
char *indexfp, size_t indexfp_sz,
|
|
|
|
char *datafp, size_t datafp_sz);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Release a datapack + index file handle.
|
|
|
|
*/
|
|
|
|
extern void close_datapack(datapack_handle_t *);
|
|
|
|
|
2016-08-04 23:48:32 +03:00
|
|
|
/**
|
|
|
|
* Finds a node using the index, and fills out the packindex pointer.
|
|
|
|
* Returns true iff the node is found.
|
|
|
|
*/
|
2017-02-24 01:03:03 +03:00
|
|
|
extern bool find(
|
2016-08-04 23:48:32 +03:00
|
|
|
const datapack_handle_t *handle,
|
|
|
|
const uint8_t node[NODE_SZ],
|
|
|
|
pack_index_entry_t *packindex);
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
/**
|
|
|
|
* Retrieves a delta chain for a given node.
|
|
|
|
*/
|
2016-08-30 07:46:43 +03:00
|
|
|
extern delta_chain_t getdeltachain(
|
2016-08-31 02:21:24 +03:00
|
|
|
datapack_handle_t *handle,
|
2016-08-04 23:50:00 +03:00
|
|
|
const uint8_t node[NODE_SZ]);
|
2016-08-02 00:05:37 +03:00
|
|
|
|
2016-08-30 07:46:43 +03:00
|
|
|
extern void freedeltachain(delta_chain_t chain);
|
2016-08-02 00:11:16 +03:00
|
|
|
|
2016-08-30 07:51:22 +03:00
|
|
|
typedef enum {
|
|
|
|
GET_DELTA_CHAIN_LINK_OK,
|
|
|
|
GET_DELTA_CHAIN_LINK_OOM,
|
|
|
|
GET_DELTA_CHAIN_LINK_CORRUPT,
|
|
|
|
} get_delta_chain_link_code_t;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This represents an entire delta chain.
|
|
|
|
*/
|
|
|
|
typedef struct _get_delta_chain_link_result_t {
|
|
|
|
get_delta_chain_link_code_t code;
|
|
|
|
const uint8_t *ptr;
|
|
|
|
} get_delta_chain_link_result_t;
|
|
|
|
|
2016-08-02 00:05:37 +03:00
|
|
|
// this should really be private, but we need it for the cdatapack_dump tool.
|
2016-08-30 07:51:22 +03:00
|
|
|
extern const get_delta_chain_link_result_t getdeltachainlink(
|
2017-04-27 05:50:36 +03:00
|
|
|
const datapack_handle_t *handle,
|
2016-08-02 00:05:37 +03:00
|
|
|
const uint8_t *ptr, delta_chain_link_t *link);
|
2017-05-01 23:03:38 +03:00
|
|
|
// caller is responsible for freeing link->delta.
|
|
|
|
extern bool uncompressdeltachainlink(delta_chain_link_t *link);
|
2016-08-02 00:05:37 +03:00
|
|
|
|
2017-08-26 02:46:07 +03:00
|
|
|
#endif // FBHGEXT_CDATAPACK_CDATAPACK_H
|