facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-06 14:58:03 +03:00
Code Issues Projects Releases Wiki Activity

fsmonitor: dependencies for new experimental extension

Browse Source 
In preparation for the filesystem monitor extension, include the pywatchman
library. The fbmonitor extension relies on this library to communicate with
the Watchman service. The library is BSD licensed and is taken from
https://github.com/facebook/watchman/tree/master/python.

This package has not been updated to mercurial code standards.
This commit is contained in:
Martijn Pieters 2016-03-02 16:25:12 +00:00
parent b4b94546a3
commit 64a4b2e8cf
7 changed files with 2167 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

779
hgext/fsmonitor/pywatchman/__init__.py Normal file
View File

@ -0,0 +1,779 @@
# Copyright 2014-present Facebook, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name Facebook nor the names of its contributors may be used to
# endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
import errno
import math
import socket
import subprocess
import time
# Sometimes it's really hard to get Python extensions to compile,
# so fall back to a pure Python implementation.
try:
import bser
except ImportError:
import pybser as bser
import capabilities
if os.name == 'nt':
import ctypes
import ctypes.wintypes
wintypes = ctypes.wintypes
GENERIC_READ = 0x80000000
GENERIC_WRITE = 0x40000000
FILE_FLAG_OVERLAPPED = 0x40000000
OPEN_EXISTING = 3
INVALID_HANDLE_VALUE = -1
FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100
FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
WAIT_TIMEOUT = 0x00000102
WAIT_OBJECT_0 = 0x00000000
ERROR_IO_PENDING = 997
class OVERLAPPED(ctypes.Structure):
_fields_ = [
("Internal", wintypes.ULONG), ("InternalHigh", wintypes.ULONG),
("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD),
("hEvent", wintypes.HANDLE)
]
def __init__(self):
self.Offset = 0
self.OffsetHigh = 0
self.hEvent = 0
LPDWORD = ctypes.POINTER(wintypes.DWORD)
CreateFile = ctypes.windll.kernel32.CreateFileA
CreateFile.argtypes = [wintypes.LPSTR, wintypes.DWORD, wintypes.DWORD,
wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
wintypes.HANDLE]
CreateFile.restype = wintypes.HANDLE
CloseHandle = ctypes.windll.kernel32.CloseHandle
CloseHandle.argtypes = [wintypes.HANDLE]
CloseHandle.restype = wintypes.BOOL
ReadFile = ctypes.windll.kernel32.ReadFile
ReadFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
LPDWORD, ctypes.POINTER(OVERLAPPED)]
ReadFile.restype = wintypes.BOOL
WriteFile = ctypes.windll.kernel32.WriteFile
WriteFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
LPDWORD, ctypes.POINTER(OVERLAPPED)]
WriteFile.restype = wintypes.BOOL
GetLastError = ctypes.windll.kernel32.GetLastError
GetLastError.argtypes = []
GetLastError.restype = wintypes.DWORD
FormatMessage = ctypes.windll.kernel32.FormatMessageA
FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR),
wintypes.DWORD, wintypes.LPVOID]
FormatMessage.restype = wintypes.DWORD
LocalFree = ctypes.windll.kernel32.LocalFree
GetOverlappedResultEx = ctypes.windll.kernel32.GetOverlappedResultEx
GetOverlappedResultEx.argtypes = [wintypes.HANDLE,
ctypes.POINTER(OVERLAPPED), LPDWORD,
wintypes.DWORD, wintypes.BOOL]
GetOverlappedResultEx.restype = wintypes.BOOL
CancelIoEx = ctypes.windll.kernel32.CancelIoEx
CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)]
CancelIoEx.restype = wintypes.BOOL
# 2 bytes marker, 1 byte int size, 8 bytes int64 value
sniff_len = 13
# This is a helper for debugging the client.
_debugging = False
if _debugging:
def log(fmt, *args):
print('[%s] %s' %
(time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
fmt % args[:]))
else:
def log(fmt, *args):
pass
class WatchmanError(Exception):
pass
class SocketTimeout(WatchmanError):
"""A specialized exception raised for socket timeouts during communication to/from watchman.
This makes it easier to implement non-blocking loops as callers can easily distinguish
between a routine timeout and an actual error condition.
Note that catching WatchmanError will also catch this as it is a super-class, so backwards
compatibility in exception handling is preserved.
"""
class CommandError(WatchmanError):
"""error returned by watchman
self.msg is the message returned by watchman.
"""
def __init__(self, msg, cmd=None):
self.msg = msg
self.cmd = cmd
super(CommandError, self).__init__('watchman command error: %s' % msg)
def setCommand(self, cmd):
self.cmd = cmd
def __str__(self):
if self.cmd:
return '%s, while executing %s' % (self.msg, self.cmd)
return self.msg
class Transport(object):
""" communication transport to the watchman server """
buf = None
def close(self):
""" tear it down """
raise NotImplementedError()
def readBytes(self, size):
""" read size bytes """
raise NotImplementedError()
def write(self, buf):
""" write some data """
raise NotImplementedError()
def setTimeout(self, value):
pass
def readLine(self):
""" read a line
Maintains its own buffer, callers of the transport should not mix
calls to readBytes and readLine.
"""
if self.buf is None:
self.buf = []
# Buffer may already have a line if we've received unilateral
# response(s) from the server
if len(self.buf) == 1 and "\n" in self.buf[0]:
(line, b) = self.buf[0].split("\n", 1)
self.buf = [b]
return line
while True:
b = self.readBytes(4096)
if "\n" in b:
result = ''.join(self.buf)
(line, b) = b.split("\n", 1)
self.buf = [b]
return result + line
self.buf.append(b)
class Codec(object):
""" communication encoding for the watchman server """
transport = None
def __init__(self, transport):
self.transport = transport
def receive(self):
raise NotImplementedError()
def send(self, *args):
raise NotImplementedError()
def setTimeout(self, value):
self.transport.setTimeout(value)
class UnixSocketTransport(Transport):
""" local unix domain socket transport """
sock = None
def __init__(self, sockpath, timeout):
self.sockpath = sockpath
self.timeout = timeout
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.settimeout(self.timeout)
sock.connect(self.sockpath)
self.sock = sock
except socket.error as e:
raise WatchmanError('unable to connect to %s: %s' %
(self.sockpath, e))
def close(self):
self.sock.close()
self.sock = None
def setTimeout(self, value):
self.timeout = value
self.sock.settimeout(self.timeout)
def readBytes(self, size):
try:
buf = [self.sock.recv(size)]
if not buf[0]:
raise WatchmanError('empty watchman response')
return buf[0]
except socket.timeout:
raise SocketTimeout('timed out waiting for response')
def write(self, data):
try:
self.sock.sendall(data)
except socket.timeout:
raise SocketTimeout('timed out sending query command')
class WindowsNamedPipeTransport(Transport):
""" connect to a named pipe """
def __init__(self, sockpath, timeout):
self.sockpath = sockpath
self.timeout = int(math.ceil(timeout * 1000))
self._iobuf = None
self.pipe = CreateFile(sockpath, GENERIC_READ | GENERIC_WRITE, 0, None,
OPEN_EXISTING, FILE_FLAG_OVERLAPPED, None)
if self.pipe == INVALID_HANDLE_VALUE:
self.pipe = None
self._raise_win_err('failed to open pipe %s' % sockpath,
GetLastError())
def _win32_strerror(self, err):
""" expand a win32 error code into a human readable message """
# FormatMessage will allocate memory and assign it here
buf = ctypes.c_char_p()
FormatMessage(
FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER
| FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None)
try:
return buf.value
finally:
LocalFree(buf)
def _raise_win_err(self, msg, err):
raise IOError('%s win32 error code: %d %s' %
(msg, err, self._win32_strerror(err)))
def close(self):
if self.pipe:
CloseHandle(self.pipe)
self.pipe = None
def readBytes(self, size):
""" A read can block for an unbounded amount of time, even if the
kernel reports that the pipe handle is signalled, so we need to
always perform our reads asynchronously
"""
# try to satisfy the read from any buffered data
if self._iobuf:
if size >= len(self._iobuf):
res = self._iobuf
self.buf = None
return res
res = self._iobuf[:size]
self._iobuf = self._iobuf[size:]
return res
# We need to initiate a read
buf = ctypes.create_string_buffer(size)
olap = OVERLAPPED()
log('made read buff of size %d', size)
# ReadFile docs warn against sending in the nread parameter for async
# operations, so we always collect it via GetOverlappedResultEx
immediate = ReadFile(self.pipe, buf, size, None, olap)
if not immediate:
err = GetLastError()
if err != ERROR_IO_PENDING:
self._raise_win_err('failed to read %d bytes' % size,
GetLastError())
nread = wintypes.DWORD()
if not GetOverlappedResultEx(self.pipe, olap, nread,
0 if immediate else self.timeout, True):
err = GetLastError()
CancelIoEx(self.pipe, olap)
if err == WAIT_TIMEOUT:
log('GetOverlappedResultEx timedout')
raise SocketTimeout('timed out after waiting %dms for read' %
self.timeout)
log('GetOverlappedResultEx reports error %d', err)
self._raise_win_err('error while waiting for read', err)
nread = nread.value
if nread == 0:
# Docs say that named pipes return 0 byte when the other end did
# a zero byte write. Since we don't ever do that, the only
# other way this shows up is if the client has gotten in a weird
# state, so let's bail out
CancelIoEx(self.pipe, olap)
raise IOError('Async read yielded 0 bytes; unpossible!')
# Holds precisely the bytes that we read from the prior request
buf = buf[:nread]
returned_size = min(nread, size)
if returned_size == nread:
return buf
# keep any left-overs around for a later read to consume
self._iobuf = buf[returned_size:]
return buf[:returned_size]
def write(self, data):
olap = OVERLAPPED()
immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data),
None, olap)
if not immediate:
err = GetLastError()
if err != ERROR_IO_PENDING:
self._raise_win_err('failed to write %d bytes' % len(data),
GetLastError())
# Obtain results, waiting if needed
nwrote = wintypes.DWORD()
if GetOverlappedResultEx(self.pipe, olap, nwrote, 0 if immediate else
self.timeout, True):
return nwrote.value
err = GetLastError()
# It's potentially unsafe to allow the write to continue after
# we unwind, so let's make a best effort to avoid that happening
CancelIoEx(self.pipe, olap)
if err == WAIT_TIMEOUT:
raise SocketTimeout('timed out after waiting %dms for write' %
self.timeout)
self._raise_win_err('error while waiting for write of %d bytes' %
len(data), err)
class CLIProcessTransport(Transport):
""" open a pipe to the cli to talk to the service
This intended to be used only in the test harness!
The CLI is an oddball because we only support JSON input
and cannot send multiple commands through the same instance,
so we spawn a new process for each command.
We disable server spawning for this implementation, again, because
it is intended to be used only in our test harness. You really
should not need to use the CLI transport for anything real.
While the CLI can output in BSER, our Transport interface doesn't
support telling this instance that it should do so. That effectively
limits this implementation to JSON input and output only at this time.
It is the responsibility of the caller to set the send and
receive codecs appropriately.
"""
proc = None
closed = True
def __init__(self, sockpath, timeout):
self.sockpath = sockpath
self.timeout = timeout
def close(self):
if self.proc:
self.proc.kill()
self.proc = None
def _connect(self):
if self.proc:
return self.proc
args = [
'watchman',
'--sockname={}'.format(self.sockpath),
'--logfile=/BOGUS',
'--statefile=/BOGUS',
'--no-spawn',
'--no-local',
'--no-pretty',
'-j',
]
self.proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
return self.proc
def readBytes(self, size):
self._connect()
res = self.proc.stdout.read(size)
if res == '':
raise WatchmanError('EOF on CLI process transport')
return res
def write(self, data):
if self.closed:
self.closed = False
self.proc = None
self._connect()
res = self.proc.stdin.write(data)
self.proc.stdin.close()
self.closed = True
return res
class BserCodec(Codec):
""" use the BSER encoding. This is the default, preferred codec """
def _loads(self, response):
return bser.loads(response)
def receive(self):
buf = [self.transport.readBytes(sniff_len)]
if not buf[0]:
raise WatchmanError('empty watchman response')
elen = bser.pdu_len(buf[0])
rlen = len(buf[0])
while elen > rlen:
buf.append(self.transport.readBytes(elen - rlen))
rlen += len(buf[-1])
response = ''.join(buf)
try:
res = self._loads(response)
return res
except ValueError as e:
raise WatchmanError('watchman response decode error: %s' % e)
def send(self, *args):
cmd = bser.dumps(*args)
self.transport.write(cmd)
class ImmutableBserCodec(BserCodec):
""" use the BSER encoding, decoding values using the newer
immutable object support """
def _loads(self, response):
return bser.loads(response, False)
class JsonCodec(Codec):
""" Use json codec. This is here primarily for testing purposes """
json = None
def __init__(self, transport):
super(JsonCodec, self).__init__(transport)
# optional dep on json, only if JsonCodec is used
import json
self.json = json
def receive(self):
line = self.transport.readLine()
try:
return self.json.loads(line)
except Exception as e:
print(e, line)
raise
def send(self, *args):
cmd = self.json.dumps(*args)
self.transport.write(cmd + "\n")
class client(object):
""" Handles the communication with the watchman service """
sockpath = None
transport = None
sendCodec = None
recvCodec = None
sendConn = None
recvConn = None
subs = {} # Keyed by subscription name
sub_by_root = {} # Keyed by root, then by subscription name
logs = [] # When log level is raised
unilateral = ['log', 'subscription']
tport = None
useImmutableBser = None
def __init__(self,
sockpath=None,
timeout=1.0,
transport=None,
sendEncoding=None,
recvEncoding=None,
useImmutableBser=False):
self.sockpath = sockpath
self.timeout = timeout
self.useImmutableBser = useImmutableBser
transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local'
if transport == 'local' and os.name == 'nt':
self.transport = WindowsNamedPipeTransport
elif transport == 'local':
self.transport = UnixSocketTransport
elif transport == 'cli':
self.transport = CLIProcessTransport
if sendEncoding is None:
sendEncoding = 'json'
if recvEncoding is None:
recvEncoding = sendEncoding
else:
raise WatchmanError('invalid transport %s' % transport)
sendEncoding = sendEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser'
recvEncoding = recvEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser'
self.recvCodec = self._parseEncoding(recvEncoding)
self.sendCodec = self._parseEncoding(sendEncoding)
def _parseEncoding(self, enc):
if enc == 'bser':
if self.useImmutableBser:
return ImmutableBserCodec
return BserCodec
elif enc == 'json':
return JsonCodec
else:
raise WatchmanError('invalid encoding %s' % enc)
def _hasprop(self, result, name):
if self.useImmutableBser:
return hasattr(result, name)
return name in result
def _resolvesockname(self):
# if invoked via a trigger, watchman will set this env var; we
# should use it unless explicitly set otherwise
path = os.getenv('WATCHMAN_SOCK')
if path:
return path
cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
try:
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=os.name != 'nt')
except OSError as e:
raise WatchmanError('"watchman" executable not in PATH (%s)', e)
stdout, stderr = p.communicate()
exitcode = p.poll()
if exitcode:
raise WatchmanError("watchman exited with code %d" % exitcode)
result = bser.loads(stdout)
if 'error' in result:
raise WatchmanError('get-sockname error: %s' % result['error'])
return result['sockname']
def _connect(self):
""" establish transport connection """
if self.recvConn:
return
if self.sockpath is None:
self.sockpath = self._resolvesockname()
self.tport = self.transport(self.sockpath, self.timeout)
self.sendConn = self.sendCodec(self.tport)
self.recvConn = self.recvCodec(self.tport)
def __del__(self):
self.close()
def close(self):
if self.tport:
self.tport.close()
self.tport = None
self.recvConn = None
self.sendConn = None
def receive(self):
""" receive the next PDU from the watchman service
If the client has activated subscriptions or logs then
this PDU may be a unilateral PDU sent by the service to
inform the client of a log event or subscription change.
It may also simply be the response portion of a request
initiated by query.
There are clients in production that subscribe and call
this in a loop to retrieve all subscription responses,
so care should be taken when making changes here.
"""
self._connect()
result = self.recvConn.receive()
if self._hasprop(result, 'error'):
raise CommandError(result['error'])
if self._hasprop(result, 'log'):
self.logs.append(result['log'])
if self._hasprop(result, 'subscription'):
sub = result['subscription']
if not (sub in self.subs):
self.subs[sub] = []
self.subs[sub].append(result)
# also accumulate in {root,sub} keyed store
root = os.path.normcase(result['root'])
if not root in self.sub_by_root:
self.sub_by_root[root] = {}
if not sub in self.sub_by_root[root]:
self.sub_by_root[root][sub] = []
self.sub_by_root[root][sub].append(result)
return result
def isUnilateralResponse(self, res):
for k in self.unilateral:
if k in res:
return True
return False
def getLog(self, remove=True):
""" Retrieve buffered log data
If remove is true the data will be removed from the buffer.
Otherwise it will be left in the buffer
"""
res = self.logs
if remove:
self.logs = []
return res
def getSubscription(self, name, remove=True, root=None):
""" Retrieve the data associated with a named subscription
If remove is True (the default), the subscription data is removed
from the buffer. Otherwise the data is returned but left in
the buffer.
Returns None if there is no data associated with `name`
If root is not None, then only return the subscription
data that matches both root and name. When used in this way,
remove processing impacts both the unscoped and scoped stores
for the subscription data.
"""
if root is not None:
if not root in self.sub_by_root:
return None
if not name in self.sub_by_root[root]:
return None
sub = self.sub_by_root[root][name]
if remove:
del self.sub_by_root[root][name]
# don't let this grow unbounded
if name in self.subs:
del self.subs[name]
return sub
if not (name in self.subs):
return None
sub = self.subs[name]
if remove:
del self.subs[name]
return sub
def query(self, *args):
""" Send a query to the watchman service and return the response
This call will block until the response is returned.
If any unilateral responses are sent by the service in between
the request-response they will be buffered up in the client object
and NOT returned via this method.
"""
log('calling client.query')
self._connect()
try:
self.sendConn.send(args)
res = self.receive()
while self.isUnilateralResponse(res):
res = self.receive()
return res
except CommandError as ex:
ex.setCommand(args)
raise ex
def capabilityCheck(self, optional=None, required=None):
""" Perform a server capability check """
res = self.query('version', {
'optional': optional or [],
'required': required or []
})
if not self._hasprop(res, 'capabilities'):
# Server doesn't support capabilities, so we need to
# synthesize the results based on the version
capabilities.synthesize(res, optional)
if 'error' in res:
raise CommandError(res['error'])
return res
def setTimeout(self, value):
self.recvConn.setTimeout(value)
self.sendConn.setTimeout(value)
# no-check-code -- this is a 3rd party library

950
hgext/fsmonitor/pywatchman/bser.c Normal file
View File

@ -0,0 +1,950 @@
/*
Copyright (c) 2013-2015, Facebook, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <Python.h>
#ifdef _MSC_VER
#define inline __inline
#include <stdint.h>
#endif
/* Return the smallest size int that can store the value */
#define INT_SIZE(x) (((x) == ((int8_t)x)) ? 1 : \
((x) == ((int16_t)x)) ? 2 : \
((x) == ((int32_t)x)) ? 4 : 8)
#define BSER_ARRAY 0x00
#define BSER_OBJECT 0x01
#define BSER_STRING 0x02
#define BSER_INT8 0x03
#define BSER_INT16 0x04
#define BSER_INT32 0x05
#define BSER_INT64 0x06
#define BSER_REAL 0x07
#define BSER_TRUE 0x08
#define BSER_FALSE 0x09
#define BSER_NULL 0x0a
#define BSER_TEMPLATE 0x0b
#define BSER_SKIP 0x0c
// An immutable object representation of BSER_OBJECT.
// Rather than build a hash table, key -> value are obtained
// by walking the list of keys to determine the offset into
// the values array. The assumption is that the number of
// array elements will be typically small (~6 for the top
// level query result and typically 3-5 for the file entries)
// so that the time overhead for this is small compared to
// using a proper hash table. Even with this simplistic
// approach, this is still faster for the mercurial use case
// as it helps to eliminate creating N other objects to
// represent the stat information in the hgwatchman extension
typedef struct {
PyObject_HEAD
PyObject *keys; // tuple of field names
PyObject *values; // tuple of values
} bserObject;
static Py_ssize_t bserobj_tuple_length(PyObject *o) {
bserObject *obj = (bserObject*)o;
return PySequence_Length(obj->keys);
}
static PyObject *bserobj_tuple_item(PyObject *o, Py_ssize_t i) {
bserObject *obj = (bserObject*)o;
return PySequence_GetItem(obj->values, i);
}
static PySequenceMethods bserobj_sq = {
bserobj_tuple_length, /* sq_length */
0, /* sq_concat */
0, /* sq_repeat */
bserobj_tuple_item, /* sq_item */
0, /* sq_ass_item */
0, /* sq_contains */
0, /* sq_inplace_concat */
0 /* sq_inplace_repeat */
};
static void bserobj_dealloc(PyObject *o) {
bserObject *obj = (bserObject*)o;
Py_CLEAR(obj->keys);
Py_CLEAR(obj->values);
PyObject_Del(o);
}
static PyObject *bserobj_getattrro(PyObject *o, PyObject *name) {
bserObject *obj = (bserObject*)o;
Py_ssize_t i, n;
const char *namestr;
if (PyIndex_Check(name)) {
i = PyNumber_AsSsize_t(name, PyExc_IndexError);
if (i == -1 && PyErr_Occurred()) {
return NULL;
}
return PySequence_GetItem(obj->values, i);
}
// hack^Wfeature to allow mercurial to use "st_size" to reference "size"
namestr = PyString_AsString(name);
if (!strncmp(namestr, "st_", 3)) {
namestr += 3;
}
n = PyTuple_GET_SIZE(obj->keys);
for (i = 0; i < n; i++) {
const char *item_name = NULL;
PyObject *key = PyTuple_GET_ITEM(obj->keys, i);
item_name = PyString_AsString(key);
if (!strcmp(item_name, namestr)) {
return PySequence_GetItem(obj->values, i);
}
}
PyErr_Format(PyExc_AttributeError,
"bserobject has no attribute '%.400s'", namestr);
return NULL;
}
static PyMappingMethods bserobj_map = {
bserobj_tuple_length, /* mp_length */
bserobj_getattrro, /* mp_subscript */
0 /* mp_ass_subscript */
};
PyTypeObject bserObjectType = {
PyVarObject_HEAD_INIT(NULL, 0)
"bserobj_tuple", /* tp_name */
sizeof(bserObject), /* tp_basicsize */
0, /* tp_itemsize */
bserobj_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
&bserobj_sq, /* tp_as_sequence */
&bserobj_map, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
bserobj_getattrro, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
"bserobj tuple", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
};
static PyObject *bser_loads_recursive(const char **ptr, const char *end,
int mutable);
static const char bser_true = BSER_TRUE;
static const char bser_false = BSER_FALSE;
static const char bser_null = BSER_NULL;
static const char bser_string_hdr = BSER_STRING;
static const char bser_array_hdr = BSER_ARRAY;
static const char bser_object_hdr = BSER_OBJECT;
static inline uint32_t next_power_2(uint32_t n)
{
n |= (n >> 16);
n |= (n >> 8);
n |= (n >> 4);
n |= (n >> 2);
n |= (n >> 1);
return n + 1;
}
// A buffer we use for building up the serialized result
struct bser_buffer {
char *buf;
int wpos, allocd;
};
typedef struct bser_buffer bser_t;
static int bser_append(bser_t *bser, const char *data, uint32_t len)
{
int newlen = next_power_2(bser->wpos + len);
if (newlen > bser->allocd) {
char *nbuf = realloc(bser->buf, newlen);
if (!nbuf) {
return 0;
}
bser->buf = nbuf;
bser->allocd = newlen;
}
memcpy(bser->buf + bser->wpos, data, len);
bser->wpos += len;
return 1;
}
static int bser_init(bser_t *bser)
{
bser->allocd = 8192;
bser->wpos = 0;
bser->buf = malloc(bser->allocd);
if (!bser->buf) {
return 0;
}
// Leave room for the serialization header, which includes
// our overall length. To make things simpler, we'll use an
// int32 for the header
#define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00"
bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER)-1);
return 1;
}
static void bser_dtor(bser_t *bser)
{
free(bser->buf);
bser->buf = NULL;
}
static int bser_long(bser_t *bser, int64_t val)
{
int8_t i8;
int16_t i16;
int32_t i32;
int64_t i64;
char sz;
int size = INT_SIZE(val);
char *iptr;
switch (size) {
case 1:
sz = BSER_INT8;
i8 = (int8_t)val;
iptr = (char*)&i8;
break;
case 2:
sz = BSER_INT16;
i16 = (int16_t)val;
iptr = (char*)&i16;
break;
case 4:
sz = BSER_INT32;
i32 = (int32_t)val;
iptr = (char*)&i32;
break;
case 8:
sz = BSER_INT64;
i64 = (int64_t)val;
iptr = (char*)&i64;
break;
default:
PyErr_SetString(PyExc_RuntimeError,
"Cannot represent this long value!?");
return 0;
}
if (!bser_append(bser, &sz, sizeof(sz))) {
return 0;
}
return bser_append(bser, iptr, size);
}
static int bser_string(bser_t *bser, PyObject *sval)
{
char *buf = NULL;
Py_ssize_t len;
int res;
PyObject *utf = NULL;
if (PyUnicode_Check(sval)) {
utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore");
sval = utf;
}
res = PyString_AsStringAndSize(sval, &buf, &len);
if (res == -1) {
res = 0;
goto out;
}
if (!bser_append(bser, &bser_string_hdr, sizeof(bser_string_hdr))) {
res = 0;
goto out;
}
if (!bser_long(bser, len)) {
res = 0;
goto out;
}
if (len > UINT32_MAX) {
PyErr_Format(PyExc_ValueError, "string too big");
res = 0;
goto out;
}
res = bser_append(bser, buf, (uint32_t)len);
out:
if (utf) {
Py_DECREF(utf);
}
return res;
}
static int bser_recursive(bser_t *bser, PyObject *val)
{
if (PyBool_Check(val)) {
if (val == Py_True) {
return bser_append(bser, &bser_true, sizeof(bser_true));
}
return bser_append(bser, &bser_false, sizeof(bser_false));
}
if (val == Py_None) {
return bser_append(bser, &bser_null, sizeof(bser_null));
}
if (PyInt_Check(val)) {
return bser_long(bser, PyInt_AS_LONG(val));
}
if (PyLong_Check(val)) {
return bser_long(bser, PyLong_AsLongLong(val));
}
if (PyString_Check(val) || PyUnicode_Check(val)) {
return bser_string(bser, val);
}
if (PyFloat_Check(val)) {
double dval = PyFloat_AS_DOUBLE(val);
char sz = BSER_REAL;
if (!bser_append(bser, &sz, sizeof(sz))) {
return 0;
}
return bser_append(bser, (char*)&dval, sizeof(dval));
}
if (PyList_Check(val)) {
Py_ssize_t i, len = PyList_GET_SIZE(val);
if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
return 0;
}
if (!bser_long(bser, len)) {
return 0;
}
for (i = 0; i < len; i++) {
PyObject *ele = PyList_GET_ITEM(val, i);
if (!bser_recursive(bser, ele)) {
return 0;
}
}
return 1;
}
if (PyTuple_Check(val)) {
Py_ssize_t i, len = PyTuple_GET_SIZE(val);
if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
return 0;
}
if (!bser_long(bser, len)) {
return 0;
}
for (i = 0; i < len; i++) {
PyObject *ele = PyTuple_GET_ITEM(val, i);
if (!bser_recursive(bser, ele)) {
return 0;
}
}
return 1;
}
if (PyMapping_Check(val)) {
Py_ssize_t len = PyMapping_Length(val);
Py_ssize_t pos = 0;
PyObject *key, *ele;
if (!bser_append(bser, &bser_object_hdr, sizeof(bser_object_hdr))) {
return 0;
}
if (!bser_long(bser, len)) {
return 0;
}
while (PyDict_Next(val, &pos, &key, &ele)) {
if (!bser_string(bser, key)) {
return 0;
}
if (!bser_recursive(bser, ele)) {
return 0;
}
}
return 1;
}
PyErr_SetString(PyExc_ValueError, "Unsupported value type");
return 0;
}
static PyObject *bser_dumps(PyObject *self, PyObject *args)
{
PyObject *val = NULL, *res;
bser_t bser;
uint32_t len;
if (!PyArg_ParseTuple(args, "O", &val)) {
return NULL;
}
if (!bser_init(&bser)) {
return PyErr_NoMemory();
}
if (!bser_recursive(&bser, val)) {
bser_dtor(&bser);
if (errno == ENOMEM) {
return PyErr_NoMemory();
}
// otherwise, we've already set the error to something reasonable
return NULL;
}
// Now fill in the overall length
len = bser.wpos - (sizeof(EMPTY_HEADER) - 1);
memcpy(bser.buf + 3, &len, sizeof(len));
res = PyString_FromStringAndSize(bser.buf, bser.wpos);
bser_dtor(&bser);
return res;
}
int bunser_int(const char **ptr, const char *end, int64_t *val)
{
int needed;
const char *buf = *ptr;
int8_t i8;
int16_t i16;
int32_t i32;
int64_t i64;
switch (buf[0]) {
case BSER_INT8:
needed = 2;
break;
case BSER_INT16:
needed = 3;
break;
case BSER_INT32:
needed = 5;
break;
case BSER_INT64:
needed = 9;
break;
default:
PyErr_Format(PyExc_ValueError,
"invalid bser int encoding 0x%02x", buf[0]);
return 0;
}
if (end - buf < needed) {
PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding");
return 0;
}
*ptr = buf + needed;
switch (buf[0]) {
case BSER_INT8:
memcpy(&i8, buf + 1, sizeof(i8));
*val = i8;
return 1;
case BSER_INT16:
memcpy(&i16, buf + 1, sizeof(i16));
*val = i16;
return 1;
case BSER_INT32:
memcpy(&i32, buf + 1, sizeof(i32));
*val = i32;
return 1;
case BSER_INT64:
memcpy(&i64, buf + 1, sizeof(i64));
*val = i64;
return 1;
default:
return 0;
}
}
static int bunser_string(const char **ptr, const char *end,
const char **start, int64_t *len)
{
const char *buf = *ptr;
// skip string marker
buf++;
if (!bunser_int(&buf, end, len)) {
return 0;
}
if (buf + *len > end) {
PyErr_Format(PyExc_ValueError, "invalid string length in bser data");
return 0;
}
*ptr = buf + *len;
*start = buf;
return 1;
}
static PyObject *bunser_array(const char **ptr, const char *end, int mutable)
{
const char *buf = *ptr;
int64_t nitems, i;
PyObject *res;
// skip array header
buf++;
if (!bunser_int(&buf, end, &nitems)) {
return 0;
}
*ptr = buf;
if (nitems > LONG_MAX) {
PyErr_Format(PyExc_ValueError, "too many items for python array");
return NULL;
}
if (mutable) {
res = PyList_New((Py_ssize_t)nitems);
} else {
res = PyTuple_New((Py_ssize_t)nitems);
}
for (i = 0; i < nitems; i++) {
PyObject *ele = bser_loads_recursive(ptr, end, mutable);
if (!ele) {
Py_DECREF(res);
return NULL;
}
if (mutable) {
PyList_SET_ITEM(res, i, ele);
} else {
PyTuple_SET_ITEM(res, i, ele);
}
// DECREF(ele) not required as SET_ITEM steals the ref
}
return res;
}
static PyObject *bunser_object(const char **ptr, const char *end,
int mutable)
{
const char *buf = *ptr;
int64_t nitems, i;
PyObject *res;
bserObject *obj;
// skip array header
buf++;
if (!bunser_int(&buf, end, &nitems)) {
return 0;
}
*ptr = buf;
if (mutable) {
res = PyDict_New();
} else {
obj = PyObject_New(bserObject, &bserObjectType);
obj->keys = PyTuple_New((Py_ssize_t)nitems);
obj->values = PyTuple_New((Py_ssize_t)nitems);
res = (PyObject*)obj;
}
for (i = 0; i < nitems; i++) {
const char *keystr;
int64_t keylen;
PyObject *key;
PyObject *ele;
if (!bunser_string(ptr, end, &keystr, &keylen)) {
Py_DECREF(res);
return NULL;
}
if (keylen > LONG_MAX) {
PyErr_Format(PyExc_ValueError, "string too big for python");
Py_DECREF(res);
return NULL;
}
key = PyString_FromStringAndSize(keystr, (Py_ssize_t)keylen);
if (!key) {
Py_DECREF(res);
return NULL;
}
ele = bser_loads_recursive(ptr, end, mutable);
if (!ele) {
Py_DECREF(key);
Py_DECREF(res);
return NULL;
}
if (mutable) {
PyDict_SetItem(res, key, ele);
Py_DECREF(key);
Py_DECREF(ele);
} else {
/* PyTuple_SET_ITEM steals ele, key */
PyTuple_SET_ITEM(obj->values, i, ele);
PyTuple_SET_ITEM(obj->keys, i, key);
}
}
return res;
}
static PyObject *bunser_template(const char **ptr, const char *end,
int mutable)
{
const char *buf = *ptr;
int64_t nitems, i;
PyObject *arrval;
PyObject *keys;
Py_ssize_t numkeys, keyidx;
if (buf[1] != BSER_ARRAY) {
PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE");
return NULL;
}
// skip header
buf++;
*ptr = buf;
// Load template keys
keys = bunser_array(ptr, end, mutable);
if (!keys) {
return NULL;
}
numkeys = PySequence_Length(keys);
// Load number of array elements
if (!bunser_int(ptr, end, &nitems)) {
Py_DECREF(keys);
return 0;
}
if (nitems > LONG_MAX) {
PyErr_Format(PyExc_ValueError, "Too many items for python");
Py_DECREF(keys);
return NULL;
}
arrval = PyList_New((Py_ssize_t)nitems);
if (!arrval) {
Py_DECREF(keys);
return NULL;
}
for (i = 0; i < nitems; i++) {
PyObject *dict = NULL;
bserObject *obj = NULL;
if (mutable) {
dict = PyDict_New();
} else {
obj = PyObject_New(bserObject, &bserObjectType);
if (obj) {
obj->keys = keys;
Py_INCREF(obj->keys);
obj->values = PyTuple_New(numkeys);
}
dict = (PyObject*)obj;
}
if (!dict) {
fail:
Py_DECREF(keys);
Py_DECREF(arrval);
return NULL;
}
for (keyidx = 0; keyidx < numkeys; keyidx++) {
PyObject *key;
PyObject *ele;
if (**ptr == BSER_SKIP) {
*ptr = *ptr + 1;
ele = Py_None;
Py_INCREF(ele);
} else {
ele = bser_loads_recursive(ptr, end, mutable);
}
if (!ele) {
goto fail;
}
if (mutable) {
key = PyList_GET_ITEM(keys, keyidx);
PyDict_SetItem(dict, key, ele);
Py_DECREF(ele);
} else {
PyTuple_SET_ITEM(obj->values, keyidx, ele);
// DECREF(ele) not required as SET_ITEM steals the ref
}
}
PyList_SET_ITEM(arrval, i, dict);
// DECREF(obj) not required as SET_ITEM steals the ref
}
Py_DECREF(keys);
return arrval;
}
static PyObject *bser_loads_recursive(const char **ptr, const char *end,
int mutable)
{
const char *buf = *ptr;
switch (buf[0]) {
case BSER_INT8:
case BSER_INT16:
case BSER_INT32:
case BSER_INT64:
{
int64_t ival;
if (!bunser_int(ptr, end, &ival)) {
return NULL;
}
if (ival < LONG_MIN || ival > LONG_MAX) {
return PyLong_FromLongLong(ival);
}
return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t));
}
case BSER_REAL:
{
double dval;
memcpy(&dval, buf + 1, sizeof(dval));
*ptr = buf + 1 + sizeof(double);
return PyFloat_FromDouble(dval);
}
case BSER_TRUE:
*ptr = buf + 1;
Py_INCREF(Py_True);
return Py_True;
case BSER_FALSE:
*ptr = buf + 1;
Py_INCREF(Py_False);
return Py_False;
case BSER_NULL:
*ptr = buf + 1;
Py_INCREF(Py_None);
return Py_None;
case BSER_STRING:
{
const char *start;
int64_t len;
if (!bunser_string(ptr, end, &start, &len)) {
return NULL;
}
if (len > LONG_MAX) {
PyErr_Format(PyExc_ValueError, "string too long for python");
return NULL;
}
return PyString_FromStringAndSize(start, (long)len);
}
case BSER_ARRAY:
return bunser_array(ptr, end, mutable);
case BSER_OBJECT:
return bunser_object(ptr, end, mutable);
case BSER_TEMPLATE:
return bunser_template(ptr, end, mutable);
default:
PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]);
}
return NULL;
}
// Expected use case is to read a packet from the socket and
// then call bser.pdu_len on the packet. It returns the total
// length of the entire response that the peer is sending,
// including the bytes already received. This allows the client
// to compute the data size it needs to read before it can
// decode the data
static PyObject *bser_pdu_len(PyObject *self, PyObject *args)
{
const char *start = NULL;
const char *data = NULL;
int datalen = 0;
const char *end;
int64_t expected_len, total_len;
if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) {
return NULL;
}
data = start;
end = data + datalen;
// Validate the header and length
if (memcmp(data, EMPTY_HEADER, 2) != 0) {
PyErr_SetString(PyExc_ValueError, "invalid bser header");
return NULL;
}
data += 2;
// Expect an integer telling us how big the rest of the data
// should be
if (!bunser_int(&data, end, &expected_len)) {
return NULL;
}
total_len = expected_len + (data - start);
if (total_len > LONG_MAX) {
return PyLong_FromLongLong(total_len);
}
return PyInt_FromLong((long)total_len);
}
static PyObject *bser_loads(PyObject *self, PyObject *args)
{
const char *data = NULL;
int datalen = 0;
const char *end;
int64_t expected_len;
int mutable = 1;
PyObject *mutable_obj = NULL;
if (!PyArg_ParseTuple(args, "s#|O:loads", &data, &datalen, &mutable_obj)) {
return NULL;
}
if (mutable_obj) {
mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0;
}
end = data + datalen;
// Validate the header and length
if (memcmp(data, EMPTY_HEADER, 2) != 0) {
PyErr_SetString(PyExc_ValueError, "invalid bser header");
return NULL;
}
data += 2;
// Expect an integer telling us how big the rest of the data
// should be
if (!bunser_int(&data, end, &expected_len)) {
return NULL;
}
// Verify
if (expected_len + data != end) {
PyErr_SetString(PyExc_ValueError, "bser data len != header len");
return NULL;
}
return bser_loads_recursive(&data, end, mutable);
}
static PyMethodDef bser_methods[] = {
{"loads", bser_loads, METH_VARARGS, "Deserialize string."},
{"pdu_len", bser_pdu_len, METH_VARARGS, "Extract PDU length."},
{"dumps", bser_dumps, METH_VARARGS, "Serialize string."},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC initbser(void)
{
(void)Py_InitModule("bser", bser_methods);
PyType_Ready(&bserObjectType);
}
/* vim:ts=2:sw=2:et:
*/
// no-check-code -- this is a 3rd party library

69
hgext/fsmonitor/pywatchman/capabilities.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2015 Facebook, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name Facebook nor the names of its contributors may be used to
# endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import re
def parse_version(vstr):
res = 0
for n in vstr.split('.'):
res = res * 1000
res = res + int(n)
return res
cap_versions = {
"cmd-watch-del-all": "3.1.1",
"cmd-watch-project": "3.1",
"relative_root": "3.3",
"term-dirname": "3.1",
"term-idirname": "3.1",
"wildmatch": "3.7",
}
def check(version, name):
if name in cap_versions:
return version >= parse_version(cap_versions[name])
return False
def synthesize(vers, opts):
""" Synthesize a capability enabled version response
This is a very limited emulation for relatively recent feature sets
"""
parsed_version = parse_version(vers['version'])
vers['capabilities'] = {}
for name in opts['optional']:
vers['capabilities'][name] = check(parsed_version, name)
failed = False
for name in opts['required']:
have = check(parsed_version, name)
vers['capabilities'][name] = have
if not have:
vers['error'] = 'client required capability `' + name + \
'` is not supported by this server'
return vers
# no-check-code -- this is a 3rd party library

359
hgext/fsmonitor/pywatchman/pybser.py Normal file
View File

@ -0,0 +1,359 @@
# Copyright 2015 Facebook, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name Facebook nor the names of its contributors may be used to
# endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import collections
import ctypes
import struct
import sys
BSER_ARRAY = '\x00'
BSER_OBJECT = '\x01'
BSER_STRING = '\x02'
BSER_INT8 = '\x03'
BSER_INT16 = '\x04'
BSER_INT32 = '\x05'
BSER_INT64 = '\x06'
BSER_REAL = '\x07'
BSER_TRUE = '\x08'
BSER_FALSE = '\x09'
BSER_NULL = '\x0a'
BSER_TEMPLATE = '\x0b'
BSER_SKIP = '\x0c'
# Leave room for the serialization header, which includes
# our overall length. To make things simpler, we'll use an
# int32 for the header
EMPTY_HEADER = "\x00\x01\x05\x00\x00\x00\x00"
# Python 3 conditional for supporting Python 2's int/long types
if sys.version_info > (3,):
long = int
def _int_size(x):
"""Return the smallest size int that can store the value"""
if -0x80 <= x <= 0x7F:
return 1
elif -0x8000 <= x <= 0x7FFF:
return 2
elif -0x80000000 <= x <= 0x7FFFFFFF:
return 4
elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF):
return 8
else:
raise RuntimeError('Cannot represent value: ' + str(x))
class _bser_buffer(object):
def __init__(self):
self.buf = ctypes.create_string_buffer(8192)
struct.pack_into(str(len(EMPTY_HEADER)) + 's', self.buf, 0, EMPTY_HEADER)
self.wpos = len(EMPTY_HEADER)
def ensure_size(self, size):
while ctypes.sizeof(self.buf) - self.wpos < size:
ctypes.resize(self.buf, ctypes.sizeof(self.buf) * 2)
def append_long(self, val):
size = _int_size(val)
to_write = size + 1
self.ensure_size(to_write)
if size == 1:
struct.pack_into('=cb', self.buf, self.wpos, BSER_INT8, val)
elif size == 2:
struct.pack_into('=ch', self.buf, self.wpos, BSER_INT16, val)
elif size == 4:
struct.pack_into('=ci', self.buf, self.wpos, BSER_INT32, val)
elif size == 8:
struct.pack_into('=cq', self.buf, self.wpos, BSER_INT64, val)
else:
raise RuntimeError('Cannot represent this long value')
self.wpos += to_write
def append_string(self, s):
if isinstance(s, unicode):
s = s.encode('utf-8')
s_len = len(s)
size = _int_size(s_len)
to_write = 2 + size + s_len
self.ensure_size(to_write)
if size == 1:
struct.pack_into('=ccb' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT8, s_len, s)
elif size == 2:
struct.pack_into('=cch' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT16, s_len, s)
elif size == 4:
struct.pack_into('=cci' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT32, s_len, s)
elif size == 8:
struct.pack_into('=ccq' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT64, s_len, s)
else:
raise RuntimeError('Cannot represent this string value')
self.wpos += to_write
def append_recursive(self, val):
if isinstance(val, bool):
needed = 1
self.ensure_size(needed)
if val:
to_encode = BSER_TRUE
else:
to_encode = BSER_FALSE
struct.pack_into('=c', self.buf, self.wpos, to_encode)
self.wpos += needed
elif val is None:
needed = 1
self.ensure_size(needed)
struct.pack_into('=c', self.buf, self.wpos, BSER_NULL)
self.wpos += needed
elif isinstance(val, (int, long)):
self.append_long(val)
elif isinstance(val, (str, unicode)):
self.append_string(val)
elif isinstance(val, float):
needed = 9
self.ensure_size(needed)
struct.pack_into('=cd', self.buf, self.wpos, BSER_REAL, val)
self.wpos += needed
elif isinstance(val, collections.Mapping) and isinstance(val, collections.Sized):
val_len = len(val)
size = _int_size(val_len)
needed = 2 + size
self.ensure_size(needed)
if size == 1:
struct.pack_into('=ccb', self.buf, self.wpos, BSER_OBJECT, BSER_INT8, val_len)
elif size == 2:
struct.pack_into('=cch', self.buf, self.wpos, BSER_OBJECT, BSER_INT16, val_len)
elif size == 4:
struct.pack_into('=cci', self.buf, self.wpos, BSER_OBJECT, BSER_INT32, val_len)
elif size == 8:
struct.pack_into('=ccq', self.buf, self.wpos, BSER_OBJECT, BSER_INT64, val_len)
else:
raise RuntimeError('Cannot represent this mapping value')
self.wpos += needed
for k, v in val.iteritems():
self.append_string(k)
self.append_recursive(v)
elif isinstance(val, collections.Iterable) and isinstance(val, collections.Sized):
val_len = len(val)
size = _int_size(val_len)
needed = 2 + size
self.ensure_size(needed)
if size == 1:
struct.pack_into('=ccb', self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len)
elif size == 2:
struct.pack_into('=cch', self.buf, self.wpos, BSER_ARRAY, BSER_INT16, val_len)
elif size == 4:
struct.pack_into('=cci', self.buf, self.wpos, BSER_ARRAY, BSER_INT32, val_len)
elif size == 8:
struct.pack_into('=ccq', self.buf, self.wpos, BSER_ARRAY, BSER_INT64, val_len)
else:
raise RuntimeError('Cannot represent this sequence value')
self.wpos += needed
for v in val:
self.append_recursive(v)
else:
raise RuntimeError('Cannot represent unknown value type')
def dumps(obj):
bser_buf = _bser_buffer()
bser_buf.append_recursive(obj)
# Now fill in the overall length
obj_len = bser_buf.wpos - len(EMPTY_HEADER)
struct.pack_into('=i', bser_buf.buf, 3, obj_len)
return bser_buf.buf.raw[:bser_buf.wpos]
def _bunser_int(buf, pos):
try:
int_type = buf[pos]
except IndexError:
raise ValueError('Invalid bser int encoding, pos out of range')
if int_type == BSER_INT8:
needed = 2
fmt = '=b'
elif int_type == BSER_INT16:
needed = 3
fmt = '=h'
elif int_type == BSER_INT32:
needed = 5
fmt = '=i'
elif int_type == BSER_INT64:
needed = 9
fmt = '=q'
else:
raise ValueError('Invalid bser int encoding 0x%02x' % int(int_type))
int_val = struct.unpack_from(fmt, buf, pos + 1)[0]
return (int_val, pos + needed)
def _bunser_string(buf, pos):
str_len, pos = _bunser_int(buf, pos + 1)
str_val = struct.unpack_from(str(str_len) + 's', buf, pos)[0]
return (str_val, pos + str_len)
def _bunser_array(buf, pos, mutable=True):
arr_len, pos = _bunser_int(buf, pos + 1)
arr = []
for i in range(arr_len):
arr_item, pos = _bser_loads_recursive(buf, pos, mutable)
arr.append(arr_item)
if not mutable:
arr = tuple(arr)
return arr, pos
# This is a quack-alike with the bserObjectType in bser.c
# It provides by getattr accessors and getitem for both index
# and name.
class _BunserDict(object):
__slots__ = ('_keys', '_values')
def __init__(self, keys, values):
self._keys = keys
self._values = values
def __getattr__(self, name):
return self.__getitem__(name)
def __getitem__(self, key):
if isinstance(key, (int, long)):
return self._values[key]
elif key.startswith('st_'):
# hack^Wfeature to allow mercurial to use "st_size" to
# reference "size"
key = key[3:]
try:
return self._values[self._keys.index(key)]
except ValueError as ex:
raise KeyError('_BunserDict has no key %s' % key)
def __len__(self):
return len(self._keys)
def _bunser_object(buf, pos, mutable=True):
obj_len, pos = _bunser_int(buf, pos + 1)
if mutable:
obj = {}
else:
keys = []
vals = []
for i in range(obj_len):
key, pos = _bunser_string(buf, pos)
val, pos = _bser_loads_recursive(buf, pos, mutable)
if mutable:
obj[key] = val
else:
keys.append(key)
vals.append(val)
if not mutable:
obj = _BunserDict(keys, vals)
return obj, pos
def _bunser_template(buf, pos, mutable=True):
if buf[pos + 1] != BSER_ARRAY:
raise RuntimeError('Expect ARRAY to follow TEMPLATE')
keys, pos = _bunser_array(buf, pos + 1)
nitems, pos = _bunser_int(buf, pos)
arr = []
for i in range(nitems):
if mutable:
obj = {}
else:
vals = []
for keyidx in range(len(keys)):
if buf[pos] == BSER_SKIP:
pos += 1
ele = None
else:
ele, pos = _bser_loads_recursive(buf, pos, mutable)
if mutable:
key = keys[keyidx]
obj[key] = ele
else:
vals.append(ele)
if not mutable:
obj = _BunserDict(keys, vals)
arr.append(obj)
return arr, pos
def _bser_loads_recursive(buf, pos, mutable=True):
val_type = buf[pos]
if (val_type == BSER_INT8 or val_type == BSER_INT16 or
val_type == BSER_INT32 or val_type == BSER_INT64):
return _bunser_int(buf, pos)
elif val_type == BSER_REAL:
val = struct.unpack_from('=d', buf, pos + 1)[0]
return (val, pos + 9)
elif val_type == BSER_TRUE:
return (True, pos + 1)
elif val_type == BSER_FALSE:
return (False, pos + 1)
elif val_type == BSER_NULL:
return (None, pos + 1)
elif val_type == BSER_STRING:
return _bunser_string(buf, pos)
elif val_type == BSER_ARRAY:
return _bunser_array(buf, pos, mutable)
elif val_type == BSER_OBJECT:
return _bunser_object(buf, pos, mutable)
elif val_type == BSER_TEMPLATE:
return _bunser_template(buf, pos, mutable)
else:
raise RuntimeError('unhandled bser opcode 0x%02x' % (val_type,))
def pdu_len(buf):
if buf[0:2] != EMPTY_HEADER[0:2]:
raise RuntimeError('Invalid BSER header')
expected_len, pos = _bunser_int(buf, 2)
return expected_len + pos
def loads(buf, mutable=True):
if buf[0:2] != EMPTY_HEADER[0:2]:
raise RuntimeError('Invalid BSER header')
expected_len, pos = _bunser_int(buf, 2)
if len(buf) != expected_len + pos:
raise RuntimeError('bser data len != header len')
return _bser_loads_recursive(buf, pos, mutable)[0]
# no-check-code -- this is a 3rd party library

2
setup.py
View File

@ -532,6 +532,8 @@ extmodules = [
Extension('mercurial.osutil', ['mercurial/osutil.c'],
extra_link_args=osutil_ldflags,
depends=common_depends),
Extension('hgext.fsmonitor.pywatchman.bser',
['hgext/fsmonitor/pywatchman/bser.c']),
]
try:

4
tests/test-check-code.t
View File

@ -8,6 +8,10 @@ New errors are not allowed. Warnings are strongly discouraged.
$ hg locate | sed 's-\\-/-g' |
> xargs "$check_code" --warnings --per-file=0 || false
Skipping hgext/fsmonitor/pywatchman/__init__.py it has no-che?k-code (glob)
Skipping hgext/fsmonitor/pywatchman/bser.c it has no-che?k-code (glob)
Skipping hgext/fsmonitor/pywatchman/capabilities.py it has no-che?k-code (glob)
Skipping hgext/fsmonitor/pywatchman/pybser.py it has no-che?k-code (glob)
Skipping i18n/polib.py it has no-che?k-code (glob)
Skipping mercurial/httpclient/__init__.py it has no-che?k-code (glob)
Skipping mercurial/httpclient/_readers.py it has no-che?k-code (glob)

4
tests/test-check-py3-compat.t
View File

@ -25,6 +25,10 @@
hgext/extdiff.py not using absolute_import
hgext/factotum.py not using absolute_import
hgext/fetch.py not using absolute_import
hgext/fsmonitor/pywatchman/__init__.py not using absolute_import
hgext/fsmonitor/pywatchman/__init__.py requires print_function
hgext/fsmonitor/pywatchman/capabilities.py not using absolute_import
hgext/fsmonitor/pywatchman/pybser.py not using absolute_import
hgext/gpg.py not using absolute_import
hgext/graphlog.py not using absolute_import
hgext/hgcia.py not using absolute_import