2018-08-08 15:24:56 +03:00
|
|
|
// Copyright (c) 2004-present, Facebook, Inc.
|
|
|
|
// All Rights Reserved.
|
|
|
|
//
|
|
|
|
// This software may be used and distributed according to the terms of the
|
|
|
|
// GNU General Public License version 2 or any later version.
|
|
|
|
|
|
|
|
#![deny(warnings)]
|
|
|
|
|
2018-10-22 15:20:40 +03:00
|
|
|
use super::HookResults;
|
2018-08-08 15:24:56 +03:00
|
|
|
use blobrepo::BlobRepo;
|
2019-05-20 17:54:53 +03:00
|
|
|
use bookmarks::BookmarkName;
|
2019-04-08 12:49:27 +03:00
|
|
|
use cloned::cloned;
|
2018-11-30 21:11:37 +03:00
|
|
|
use context::CoreContext;
|
2019-09-12 08:00:24 +03:00
|
|
|
use failure_ext::{err_msg, Error, Fail, Result};
|
2018-08-08 15:24:56 +03:00
|
|
|
use futures::{Future, Stream};
|
2018-10-22 15:20:40 +03:00
|
|
|
use futures_ext::{spawn_future, BoxFuture, FutureExt};
|
2019-01-31 11:31:12 +03:00
|
|
|
use hooks::{hook_loader::load_hooks, HookManager};
|
|
|
|
use hooks_content_stores::{BlobRepoChangesetStore, BlobRepoFileContentStore};
|
2018-08-08 15:24:56 +03:00
|
|
|
use manifold::{ManifoldHttpClient, PayloadRange};
|
2019-03-15 00:16:15 +03:00
|
|
|
use mercurial_types::HgChangesetId;
|
2019-01-31 11:31:11 +03:00
|
|
|
use metaconfig_types::RepoConfig;
|
2018-09-26 14:46:41 +03:00
|
|
|
use mononoke_types::ChangesetId;
|
2018-08-08 15:24:56 +03:00
|
|
|
use revset::AncestorsNodeStream;
|
2019-04-08 12:49:27 +03:00
|
|
|
use slog::{debug, info, Logger};
|
2019-04-17 17:56:57 +03:00
|
|
|
use std::collections::HashSet;
|
2018-08-08 15:24:56 +03:00
|
|
|
use std::sync::Arc;
|
|
|
|
|
|
|
|
pub struct Tailer {
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx: CoreContext,
|
2019-01-09 14:49:56 +03:00
|
|
|
repo: BlobRepo,
|
2018-08-08 15:24:56 +03:00
|
|
|
hook_manager: Arc<HookManager>,
|
2019-05-20 17:54:53 +03:00
|
|
|
bookmark: BookmarkName,
|
2018-08-08 15:24:56 +03:00
|
|
|
last_rev_key: String,
|
|
|
|
manifold_client: ManifoldHttpClient,
|
|
|
|
logger: Logger,
|
2019-04-17 17:56:57 +03:00
|
|
|
excludes: HashSet<ChangesetId>,
|
2018-08-08 15:24:56 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Tailer {
|
|
|
|
pub fn new(
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx: CoreContext,
|
2019-01-09 14:49:56 +03:00
|
|
|
repo: BlobRepo,
|
2018-08-08 15:24:56 +03:00
|
|
|
config: RepoConfig,
|
2019-05-20 17:54:53 +03:00
|
|
|
bookmark: BookmarkName,
|
2018-08-08 15:24:56 +03:00
|
|
|
manifold_client: ManifoldHttpClient,
|
|
|
|
logger: Logger,
|
2019-04-17 17:56:57 +03:00
|
|
|
excludes: HashSet<ChangesetId>,
|
2019-07-11 16:38:11 +03:00
|
|
|
disabled_hooks: &HashSet<String>,
|
2018-08-08 15:24:56 +03:00
|
|
|
) -> Result<Tailer> {
|
2019-01-09 14:49:56 +03:00
|
|
|
let changeset_store = BlobRepoChangesetStore::new(repo.clone());
|
|
|
|
let content_store = BlobRepoFileContentStore::new(repo.clone());
|
2018-08-08 15:24:56 +03:00
|
|
|
|
|
|
|
let mut hook_manager = HookManager::new(
|
2018-12-04 22:28:47 +03:00
|
|
|
ctx.clone(),
|
2018-08-08 15:24:56 +03:00
|
|
|
Box::new(changeset_store),
|
|
|
|
Arc::new(content_store),
|
2018-11-22 14:28:28 +03:00
|
|
|
Default::default(),
|
2018-08-08 15:24:56 +03:00
|
|
|
logger.clone(),
|
|
|
|
);
|
|
|
|
|
2019-07-11 16:38:11 +03:00
|
|
|
load_hooks(&mut hook_manager, config, disabled_hooks)?;
|
2018-08-08 15:24:56 +03:00
|
|
|
|
|
|
|
let repo_id = repo.get_repoid().id();
|
|
|
|
let last_rev_key = format!("{}{}", "__mononoke_hook_tailer_last_rev.", repo_id).to_string();
|
|
|
|
|
|
|
|
Ok(Tailer {
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx,
|
2018-08-08 15:24:56 +03:00
|
|
|
repo,
|
|
|
|
hook_manager: Arc::new(hook_manager),
|
|
|
|
bookmark,
|
|
|
|
last_rev_key,
|
|
|
|
manifold_client,
|
|
|
|
logger,
|
2019-04-17 17:56:57 +03:00
|
|
|
excludes,
|
2018-08-08 15:24:56 +03:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_last_rev_key(&self) -> String {
|
|
|
|
self.last_rev_key.clone()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn run_in_range0(
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx: CoreContext,
|
2019-01-09 14:49:56 +03:00
|
|
|
repo: BlobRepo,
|
2018-08-08 15:24:56 +03:00
|
|
|
hm: Arc<HookManager>,
|
2019-03-15 00:16:15 +03:00
|
|
|
last_rev: HgChangesetId,
|
|
|
|
end_rev: HgChangesetId,
|
2019-05-20 17:54:53 +03:00
|
|
|
bm: BookmarkName,
|
2018-08-08 15:24:56 +03:00
|
|
|
logger: Logger,
|
2019-04-17 17:56:57 +03:00
|
|
|
excludes: HashSet<ChangesetId>,
|
2018-10-22 15:20:40 +03:00
|
|
|
) -> BoxFuture<Vec<HookResults>, Error> {
|
2018-08-08 15:24:56 +03:00
|
|
|
debug!(logger, "Running in range {} to {}", last_rev, end_rev);
|
2018-10-17 18:03:39 +03:00
|
|
|
cloned!(logger);
|
2018-11-30 21:11:37 +03:00
|
|
|
nodehash_to_bonsai(ctx.clone(), &repo, end_rev)
|
2018-09-26 14:46:41 +03:00
|
|
|
.and_then(move |end_rev| {
|
2018-11-30 21:11:37 +03:00
|
|
|
AncestorsNodeStream::new(ctx.clone(), &repo.get_changeset_fetcher(), end_rev)
|
2018-10-17 18:03:39 +03:00
|
|
|
.take(1000) // Limit number so we don't process too many
|
2019-04-17 17:56:57 +03:00
|
|
|
.filter(move |cs| !excludes.contains(cs))
|
2018-10-17 18:03:39 +03:00
|
|
|
.map({
|
|
|
|
move |cs| {
|
2018-11-30 21:11:37 +03:00
|
|
|
cloned!(ctx, bm, hm, logger, repo);
|
|
|
|
run_hooks_for_changeset(ctx, repo, hm, bm, cs, logger)
|
2018-10-17 18:03:39 +03:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.map(spawn_future)
|
|
|
|
.buffered(100)
|
2018-10-22 15:20:40 +03:00
|
|
|
.take_while(move |(hg_cs, _)| {
|
2019-03-15 00:16:15 +03:00
|
|
|
Ok(*hg_cs != last_rev)
|
2018-10-17 18:03:39 +03:00
|
|
|
})
|
2018-10-22 15:20:40 +03:00
|
|
|
.map(|(_, res)| res)
|
2018-10-17 18:03:39 +03:00
|
|
|
.collect()
|
2018-09-26 14:46:41 +03:00
|
|
|
})
|
2018-08-08 15:24:56 +03:00
|
|
|
.boxify()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn run_in_range(
|
|
|
|
&self,
|
2019-03-15 00:16:15 +03:00
|
|
|
last_rev: HgChangesetId,
|
|
|
|
end_rev: HgChangesetId,
|
2018-10-22 15:20:40 +03:00
|
|
|
) -> BoxFuture<Vec<HookResults>, Error> {
|
2019-04-17 17:56:57 +03:00
|
|
|
cloned!(
|
|
|
|
self.ctx,
|
|
|
|
self.repo,
|
|
|
|
self.hook_manager,
|
|
|
|
self.bookmark,
|
|
|
|
self.excludes
|
|
|
|
);
|
2018-11-30 21:11:37 +03:00
|
|
|
Tailer::run_in_range0(
|
|
|
|
ctx,
|
|
|
|
repo,
|
|
|
|
hook_manager,
|
|
|
|
last_rev,
|
|
|
|
end_rev,
|
|
|
|
bookmark,
|
|
|
|
self.logger.clone(),
|
2019-04-17 17:56:57 +03:00
|
|
|
excludes,
|
2018-11-30 21:11:37 +03:00
|
|
|
)
|
2018-08-08 15:24:56 +03:00
|
|
|
}
|
|
|
|
|
2019-04-09 18:36:33 +03:00
|
|
|
pub fn run_single_changeset(
|
|
|
|
&self,
|
|
|
|
changeset: HgChangesetId,
|
|
|
|
) -> BoxFuture<Vec<HookResults>, Error> {
|
|
|
|
cloned!(
|
|
|
|
self.ctx,
|
|
|
|
self.repo,
|
|
|
|
self.hook_manager,
|
|
|
|
self.bookmark,
|
|
|
|
self.logger
|
|
|
|
);
|
|
|
|
repo.get_bonsai_from_hg(ctx, changeset)
|
|
|
|
.and_then(move |maybe_bonsai| {
|
|
|
|
maybe_bonsai.ok_or(err_msg(format!(
|
|
|
|
"changeset does not exist {}",
|
|
|
|
changeset.to_string()
|
|
|
|
)))
|
|
|
|
})
|
|
|
|
.and_then({
|
|
|
|
cloned!(self.ctx);
|
|
|
|
move |bonsai| {
|
|
|
|
run_hooks_for_changeset(ctx, repo, hook_manager, bookmark, bonsai, logger)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.map(|(_, result)| vec![result])
|
|
|
|
.boxify()
|
|
|
|
}
|
|
|
|
|
2018-10-22 15:20:40 +03:00
|
|
|
pub fn run_with_limit(&self, limit: u64) -> BoxFuture<Vec<HookResults>, Error> {
|
2018-11-30 21:11:37 +03:00
|
|
|
let ctx = self.ctx.clone();
|
2018-10-22 15:20:40 +03:00
|
|
|
let bm = self.bookmark.clone();
|
|
|
|
let hm = self.hook_manager.clone();
|
2019-04-17 17:56:57 +03:00
|
|
|
let excludes = self.excludes.clone();
|
2018-10-22 15:20:40 +03:00
|
|
|
|
2019-01-14 20:29:33 +03:00
|
|
|
let bm_rev = self
|
|
|
|
.repo
|
2018-11-30 21:11:37 +03:00
|
|
|
.get_bookmark(ctx.clone(), &bm)
|
2018-10-22 15:20:40 +03:00
|
|
|
.and_then({
|
|
|
|
cloned!(bm);
|
|
|
|
|opt| opt.ok_or(ErrorKind::NoSuchBookmark(bm).into())
|
|
|
|
})
|
|
|
|
.and_then({
|
2018-11-30 21:11:37 +03:00
|
|
|
cloned!(ctx, self.repo);
|
2019-03-15 00:16:15 +03:00
|
|
|
move |bm_rev| nodehash_to_bonsai(ctx, &repo, bm_rev)
|
2018-10-22 15:20:40 +03:00
|
|
|
});
|
|
|
|
|
2018-11-30 21:11:37 +03:00
|
|
|
cloned!(self.ctx, self.repo, self.logger);
|
2018-10-22 15:20:40 +03:00
|
|
|
bm_rev
|
|
|
|
.and_then(move |bm_rev| {
|
2018-11-30 21:11:37 +03:00
|
|
|
AncestorsNodeStream::new(ctx.clone(), &repo.get_changeset_fetcher(), bm_rev)
|
2018-10-22 15:20:40 +03:00
|
|
|
.take(limit)
|
2019-04-17 17:56:57 +03:00
|
|
|
.filter(move |cs| !excludes.contains(cs))
|
2018-10-22 15:20:40 +03:00
|
|
|
.map({
|
|
|
|
move |cs| {
|
2018-11-30 21:11:37 +03:00
|
|
|
cloned!(ctx, bm, hm, logger, repo);
|
|
|
|
run_hooks_for_changeset(ctx, repo, hm, bm, cs, logger)
|
2018-10-22 15:20:40 +03:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.map(spawn_future)
|
|
|
|
.buffered(100)
|
|
|
|
.map(|(_, res)| res)
|
|
|
|
.collect()
|
|
|
|
})
|
|
|
|
.boxify()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn run(&self) -> BoxFuture<Vec<HookResults>, Error> {
|
2019-04-17 17:56:57 +03:00
|
|
|
info!(
|
|
|
|
self.logger,
|
|
|
|
"Running tailer on bookmark {}",
|
|
|
|
self.bookmark.clone()
|
|
|
|
);
|
2018-08-08 15:24:56 +03:00
|
|
|
|
|
|
|
self.repo
|
2019-04-17 17:56:57 +03:00
|
|
|
.get_bookmark(self.ctx.clone(), &self.bookmark.clone())
|
|
|
|
.and_then({
|
|
|
|
cloned!(self.bookmark);
|
|
|
|
|opt| opt.ok_or(ErrorKind::NoSuchBookmark(bookmark).into())
|
|
|
|
})
|
|
|
|
.and_then({
|
|
|
|
cloned!(self.last_rev_key, self.manifold_client);
|
|
|
|
move |current_bm_cs| {
|
|
|
|
manifold_client
|
|
|
|
.read(last_rev_key, PayloadRange::Full)
|
|
|
|
.map(move |opt| (current_bm_cs, opt))
|
|
|
|
}
|
2018-08-08 15:24:56 +03:00
|
|
|
})
|
|
|
|
.and_then(|(current_bm_cs, opt)| match opt {
|
|
|
|
Some(last_rev_bytes) => Ok((current_bm_cs, last_rev_bytes)),
|
|
|
|
None => Err(ErrorKind::NoLastRevision.into()),
|
|
|
|
})
|
|
|
|
.and_then(|(current_bm_cs, last_rev_bytes)| {
|
2019-03-15 00:16:15 +03:00
|
|
|
let node_hash = HgChangesetId::from_bytes(&*last_rev_bytes.payload.payload)?;
|
2018-08-08 15:24:56 +03:00
|
|
|
Ok((current_bm_cs, node_hash))
|
|
|
|
})
|
2019-04-17 17:56:57 +03:00
|
|
|
.and_then({
|
|
|
|
cloned!(
|
|
|
|
self.logger,
|
|
|
|
self.bookmark,
|
|
|
|
self.excludes,
|
|
|
|
self.hook_manager,
|
|
|
|
self.repo,
|
|
|
|
self.ctx
|
2018-08-08 15:24:56 +03:00
|
|
|
);
|
2019-04-17 17:56:57 +03:00
|
|
|
move |(current_bm_cs, last_rev)| {
|
|
|
|
let end_rev = current_bm_cs;
|
|
|
|
info!(
|
|
|
|
logger,
|
|
|
|
"Bookmark is currently at {}, last processed revision is {}",
|
|
|
|
end_rev,
|
|
|
|
last_rev
|
|
|
|
);
|
|
|
|
if last_rev == end_rev {
|
|
|
|
info!(logger, "Nothing to do");
|
|
|
|
}
|
|
|
|
Tailer::run_in_range0(
|
|
|
|
ctx,
|
|
|
|
repo,
|
|
|
|
hook_manager,
|
|
|
|
last_rev,
|
|
|
|
end_rev,
|
|
|
|
bookmark,
|
|
|
|
logger,
|
|
|
|
excludes,
|
|
|
|
)
|
2018-08-08 15:24:56 +03:00
|
|
|
.map(move |res| (end_rev, res))
|
2019-04-17 17:56:57 +03:00
|
|
|
}
|
2018-08-08 15:24:56 +03:00
|
|
|
})
|
2019-04-17 17:56:57 +03:00
|
|
|
.and_then({
|
|
|
|
cloned!(self.last_rev_key, self.logger, self.manifold_client);
|
|
|
|
move |(end_rev, res)| {
|
|
|
|
info!(logger, "Setting last processed revision to {:?}", end_rev);
|
|
|
|
let bytes = end_rev.as_bytes().into();
|
|
|
|
manifold_client.write(last_rev_key, bytes).map(|()| res)
|
|
|
|
}
|
2018-08-08 15:24:56 +03:00
|
|
|
})
|
|
|
|
.boxify()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-26 14:46:41 +03:00
|
|
|
fn nodehash_to_bonsai(
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx: CoreContext,
|
2019-01-09 14:49:56 +03:00
|
|
|
repo: &BlobRepo,
|
2019-03-15 00:16:15 +03:00
|
|
|
hg_cs: HgChangesetId,
|
2018-09-26 14:46:41 +03:00
|
|
|
) -> impl Future<Item = ChangesetId, Error = Error> {
|
2019-02-07 01:58:25 +03:00
|
|
|
repo.get_bonsai_from_hg(ctx, hg_cs)
|
2018-09-26 14:46:41 +03:00
|
|
|
.and_then(move |maybe_node| maybe_node.ok_or(ErrorKind::BonsaiNotFound(hg_cs).into()))
|
|
|
|
}
|
|
|
|
|
2018-10-17 18:03:39 +03:00
|
|
|
fn run_hooks_for_changeset(
|
2018-11-30 21:11:37 +03:00
|
|
|
ctx: CoreContext,
|
2019-01-09 14:49:56 +03:00
|
|
|
repo: BlobRepo,
|
2018-10-17 18:03:39 +03:00
|
|
|
hm: Arc<HookManager>,
|
2019-05-20 17:54:53 +03:00
|
|
|
bm: BookmarkName,
|
2018-10-17 18:03:39 +03:00
|
|
|
cs: ChangesetId,
|
|
|
|
logger: Logger,
|
2018-10-22 15:20:40 +03:00
|
|
|
) -> impl Future<Item = (HgChangesetId, HookResults), Error = Error> {
|
2018-12-04 22:28:47 +03:00
|
|
|
repo.get_hg_from_bonsai_changeset(ctx.clone(), cs)
|
2018-10-17 18:03:39 +03:00
|
|
|
.and_then(move |hg_cs| {
|
|
|
|
debug!(logger, "Running file hooks for changeset {:?}", hg_cs);
|
2018-12-04 22:28:47 +03:00
|
|
|
hm.run_file_hooks_for_bookmark(ctx.clone(), hg_cs.clone(), &bm, None)
|
2018-10-17 18:03:39 +03:00
|
|
|
.map(move |res| (hg_cs, res))
|
|
|
|
.and_then(move |(hg_cs, file_res)| {
|
|
|
|
let hg_cs = hg_cs.clone();
|
|
|
|
debug!(logger, "Running changeset hooks for changeset {:?}", hg_cs);
|
2018-12-04 22:28:47 +03:00
|
|
|
hm.run_changeset_hooks_for_bookmark(ctx, hg_cs.clone(), &bm, None)
|
2018-10-22 15:20:40 +03:00
|
|
|
.map(move |res| {
|
|
|
|
let hook_results = HookResults {
|
|
|
|
file_hooks_results: file_res,
|
|
|
|
cs_hooks_result: res,
|
|
|
|
};
|
|
|
|
(hg_cs, hook_results)
|
|
|
|
})
|
2018-10-17 18:03:39 +03:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-08-08 15:24:56 +03:00
|
|
|
#[derive(Debug, Fail)]
|
|
|
|
pub enum ErrorKind {
|
2019-01-14 20:29:33 +03:00
|
|
|
#[fail(display = "No such bookmark '{}'", _0)]
|
2019-05-20 17:54:53 +03:00
|
|
|
NoSuchBookmark(BookmarkName),
|
2019-01-14 20:29:33 +03:00
|
|
|
#[fail(display = "Cannot find last revision in blobstore")]
|
|
|
|
NoLastRevision,
|
|
|
|
#[fail(display = "Cannot find bonsai for {}", _0)]
|
|
|
|
BonsaiNotFound(HgChangesetId),
|
2018-08-08 15:24:56 +03:00
|
|
|
}
|