diff --git a/eden/mononoke/edenapi_service/src/handlers/commit.rs b/eden/mononoke/edenapi_service/src/handlers/commit.rs index 175f9abbb6..6951a1943e 100644 --- a/eden/mononoke/edenapi_service/src/handlers/commit.rs +++ b/eden/mononoke/edenapi_service/src/handlers/commit.rs @@ -24,7 +24,9 @@ use types::HgId; use crate::context::ServerContext; use crate::errors::ErrorKind; use crate::middleware::RequestContext; -use crate::utils::{cbor_stream, get_repo, parse_cbor_request, parse_wire_request}; +use crate::utils::{ + cbor_stream, get_repo, parse_cbor_request, parse_wire_request, simple_cbor_stream, +}; use super::{EdenApiMethod, HandlerInfo}; @@ -76,18 +78,21 @@ pub async fn hash_to_location(state: &mut State) -> Result, hg_cs_ids: Vec, - ) -> Result>, Error> { + ) -> impl Stream { let hgcsid_to_location = hg_repo_ctx .many_changeset_ids_to_locations(master_heads, hg_cs_ids.clone()) - .await?; - let responses = hgcsid_to_location.into_iter().map(|(hgcsid, location)| { - let response = CommitHashToLocationResponse { + .await; + let responses = hg_cs_ids.into_iter().map(move |hgcsid| { + let result = hgcsid_to_location + .as_ref() + .map(|hsh| hsh.get(&hgcsid).map(|l| l.map_descendant(|x| x.into()))) + .map_err(|e| (&*e).into()); + CommitHashToLocationResponse { hgid: hgcsid.into(), - location: location.map_descendant(|x| x.into()), - }; - Ok(response) + result, + } }); - Ok(stream::iter(responses)) + stream::iter(responses) } let params = HashToLocationParams::take_from(state); @@ -103,6 +108,7 @@ pub async fn hash_to_location(state: &mut State) -> Result(state).await?; + let unfiltered = batch.unfiltered; let master_heads = batch .master_heads .into_iter() @@ -117,9 +123,22 @@ pub async fn hash_to_location(state: &mut State) -> Result true, + _ => false, + } + }; + futures::future::ready(to_keep) + }) + .map(|response| response.to_wire()); + Ok(simple_cbor_stream(rctx, response)) } pub async fn revlog_data(state: &mut State) -> Result { diff --git a/eden/mononoke/edenapi_service/src/utils/cbor.rs b/eden/mononoke/edenapi_service/src/utils/cbor.rs index 4cf2ccbd0f..c89e31601b 100644 --- a/eden/mononoke/edenapi_service/src/utils/cbor.rs +++ b/eden/mononoke/edenapi_service/src/utils/cbor.rs @@ -55,6 +55,17 @@ where StreamBody::new(content_stream, cbor_mime()) } +pub fn simple_cbor_stream(rctx: RequestContext, stream: S) -> impl TryIntoResponse +where + S: Stream + Send + 'static, + T: Serialize + Send + 'static, +{ + let byte_stream = stream.then(|item| async { to_cbor_bytes(item) }); + let content_stream = ContentStream::new(byte_stream).forward_err(rctx.error_tx); + + StreamBody::new(content_stream, cbor_mime()) +} + pub async fn parse_cbor_request(state: &mut State) -> Result { let body = get_request_body(state).await?; serde_cbor::from_slice(&body) diff --git a/eden/mononoke/edenapi_service/src/utils/mod.rs b/eden/mononoke/edenapi_service/src/utils/mod.rs index 094e16d521..af63823fd6 100644 --- a/eden/mononoke/edenapi_service/src/utils/mod.rs +++ b/eden/mononoke/edenapi_service/src/utils/mod.rs @@ -22,7 +22,10 @@ use crate::middleware::RequestContext; pub mod cbor; pub mod convert; -pub use cbor::{cbor_mime, cbor_stream, parse_cbor_request, parse_wire_request, to_cbor_bytes}; +pub use cbor::{ + cbor_mime, cbor_stream, parse_cbor_request, parse_wire_request, simple_cbor_stream, + to_cbor_bytes, +}; pub use convert::{to_hg_path, to_mononoke_path, to_mpath}; pub async fn get_repo( diff --git a/eden/mononoke/tests/integration/test-edenapi-server-commit-hash-to-location.t b/eden/mononoke/tests/integration/test-edenapi-server-commit-hash-to-location.t index 958b5764fa..11fbd73d01 100644 --- a/eden/mononoke/tests/integration/test-edenapi-server-commit-hash-to-location.t +++ b/eden/mononoke/tests/integration/test-edenapi-server-commit-hash-to-location.t @@ -51,6 +51,7 @@ Populate test repo o 426bada5c67598ca65036d57d9e4b64b0c1ce7a0 A + Blobimport test repo. $ cd .. $ blobimport repo-hg/.hg repo @@ -61,6 +62,70 @@ Start up EdenAPI server. $ wait_for_mononoke Create and send file data request. + $ edenapi_make_req commit-hash-to-location > req.cbor < { + > "master_heads": [ + > "$G" + > ], + > "hgids": [ + > "$F", + > "$D", + > "$C", + > "$B", + > "$A", + > "$F", + > "$H", + > "000000000000000000000000000000123456789a" + > ], + > "unfiltered": true + > } + > EOF + Reading from stdin + Generated request: WireCommitHashToLocationRequestBatch { + client_head: Some( + WireHgId("1b794c59b583e47686701d0142848e90a3a94a7d"), + ), + hgids: [ + WireHgId("bb56d4161ee371c720dbc8b504810c62a22fe314"), + WireHgId("f585351a92f85104bff7c284233c338b10eb1df7"), + WireHgId("26805aba1e600a82e93661149f2313866a221a7b"), + WireHgId("112478962961147124edd43549aedd1a335e44bf"), + WireHgId("426bada5c67598ca65036d57d9e4b64b0c1ce7a0"), + WireHgId("bb56d4161ee371c720dbc8b504810c62a22fe314"), + WireHgId("06383dd46c9bcbca9300252b4b6cddad88f8af21"), + WireHgId("000000000000000000000000000000123456789a"), + ], + master_heads: [ + WireHgId("1b794c59b583e47686701d0142848e90a3a94a7d"), + ], + unfiltered: Some( + true, + ), + } + + $ sslcurl -s "https://localhost:$MONONOKE_SOCKET/edenapi/repo/commit/hash_to_location" --data-binary @req.cbor > res.cbor + +Check files in response. + $ edenapi_read_res commit-hash-to-location res.cbor + Reading from file: "res.cbor" + 000000000000000000000000000000123456789a => + Ok(None) + 06383dd46c9bcbca9300252b4b6cddad88f8af21 => + Ok(None) + 112478962961147124edd43549aedd1a335e44bf => + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=1))) + 26805aba1e600a82e93661149f2313866a221a7b => + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=1))) + 426bada5c67598ca65036d57d9e4b64b0c1ce7a0 => + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=2))) + bb56d4161ee371c720dbc8b504810c62a22fe314 => + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) + bb56d4161ee371c720dbc8b504810c62a22fe314 => + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) + f585351a92f85104bff7c284233c338b10eb1df7 => + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=0))) + +And now again but this time with the default value for unfiltered $ edenapi_make_req commit-hash-to-location > req.cbor < { > "master_heads": [ @@ -96,6 +161,7 @@ Create and send file data request. master_heads: [ WireHgId("1b794c59b583e47686701d0142848e90a3a94a7d"), ], + unfiltered: None, } $ sslcurl -s "https://localhost:$MONONOKE_SOCKET/edenapi/repo/commit/hash_to_location" --data-binary @req.cbor > res.cbor @@ -104,12 +170,14 @@ Check files in response. $ edenapi_read_res commit-hash-to-location res.cbor Reading from file: "res.cbor" 112478962961147124edd43549aedd1a335e44bf => - Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=1) + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=1))) 26805aba1e600a82e93661149f2313866a221a7b => - Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=1) + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=1))) 426bada5c67598ca65036d57d9e4b64b0c1ce7a0 => - Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=2) + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=2))) bb56d4161ee371c720dbc8b504810c62a22fe314 => - Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1) + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) + bb56d4161ee371c720dbc8b504810c62a22fe314 => + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) f585351a92f85104bff7c284233c338b10eb1df7 => - Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=0) + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=0))) diff --git a/eden/mononoke/tests/integration/test-edenapi-server-segmented-changelog-setup.t b/eden/mononoke/tests/integration/test-edenapi-server-segmented-changelog-setup.t index 2d0059c140..c31855b21e 100644 --- a/eden/mononoke/tests/integration/test-edenapi-server-segmented-changelog-setup.t +++ b/eden/mononoke/tests/integration/test-edenapi-server-segmented-changelog-setup.t @@ -104,6 +104,7 @@ Create and send file data request. master_heads: [ WireHgId("1b794c59b583e47686701d0142848e90a3a94a7d"), ], + unfiltered: None, } $ sslcurl -s "https://localhost:$MONONOKE_SOCKET/edenapi/repo/commit/hash_to_location" --data-binary @req.cbor > res.cbor @@ -112,12 +113,14 @@ Check files in response. $ edenapi_read_res commit-hash-to-location res.cbor Reading from file: "res.cbor" 112478962961147124edd43549aedd1a335e44bf => - Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=1) + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=1))) 26805aba1e600a82e93661149f2313866a221a7b => - Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=1) + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=1))) 426bada5c67598ca65036d57d9e4b64b0c1ce7a0 => - Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=2) + Ok(Some(Location(descendant=49cb92066bfd0763fff729c354345650b7428554, dist=2))) bb56d4161ee371c720dbc8b504810c62a22fe314 => - Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1) + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) + bb56d4161ee371c720dbc8b504810c62a22fe314 => + Ok(Some(Location(descendant=1b794c59b583e47686701d0142848e90a3a94a7d, dist=1))) f585351a92f85104bff7c284233c338b10eb1df7 => - Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=0) + Ok(Some(Location(descendant=f585351a92f85104bff7c284233c338b10eb1df7, dist=0))) diff --git a/eden/scm/lib/edenapi/src/client.rs b/eden/scm/lib/edenapi/src/client.rs index 8d7f1d8862..2a2829909c 100644 --- a/eden/scm/lib/edenapi/src/client.rs +++ b/eden/scm/lib/edenapi/src/client.rs @@ -570,6 +570,7 @@ impl EdenApi for Client { let batch = CommitHashToLocationRequestBatch { master_heads: master_heads.clone(), hgids, + unfiltered: Some(true), }; self.log_request(&batch, "commit_hash_to_location"); batch.to_wire() diff --git a/eden/scm/lib/edenapi/tools/read_res/src/main.rs b/eden/scm/lib/edenapi/tools/read_res/src/main.rs index 6f1275838e..4459726f0f 100644 --- a/eden/scm/lib/edenapi/tools/read_res/src/main.rs +++ b/eden/scm/lib/edenapi/tools/read_res/src/main.rs @@ -442,14 +442,24 @@ fn cmd_commit_hash_to_location(args: CommitHashToLocationArgs) -> Result<()> { let mut response_list: Vec = read_input(args.input, None)?; response_list.sort_by_key(|r| r.hgid); let iter = response_list - .iter() + .into_iter() .skip(args.start.unwrap_or(0)) .take(args.limit.unwrap_or(usize::MAX)); for response in iter { - println!( - "{} =>\n Location(descendant={}, dist={})", - response.hgid, response.location.descendant, response.location.distance - ); + if let Some(result) = response.result { + let s = match result { + Ok(Some(l)) => format!( + "Ok(Some(Location(descendant={}, dist={})))", + l.descendant, l.distance + ), + Ok(None) => String::from("Ok(None)"), + Err(e) => format!( + "Err({})", + e.message.unwrap_or_else(|| String::from("error")) + ), + }; + println!("{} =>\n {}", response.hgid, s); + } } Ok(()) } diff --git a/eden/scm/lib/edenapi/types/src/commit.rs b/eden/scm/lib/edenapi/types/src/commit.rs index bbfa85f6e4..629c298f02 100644 --- a/eden/scm/lib/edenapi/types/src/commit.rs +++ b/eden/scm/lib/edenapi/types/src/commit.rs @@ -5,6 +5,7 @@ * GNU General Public License version 2. */ +use anyhow::Result; use bytes::Bytes; #[cfg(any(test, feature = "for-tests"))] use quickcheck::Arbitrary; @@ -13,6 +14,8 @@ use serde_derive::{Deserialize, Serialize}; use dag_types::Location; use types::hgid::HgId; +use crate::ServerError; + /// Given a graph location, return `count` hashes following first parent links. /// /// Example: @@ -83,13 +86,14 @@ impl Arbitrary for CommitLocationToHashRequestBatch { pub struct CommitHashToLocationRequestBatch { pub master_heads: Vec, pub hgids: Vec, + pub unfiltered: Option, } -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] -#[derive(Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Serialize)] // used to convert to Python pub struct CommitHashToLocationResponse { pub hgid: HgId, - pub location: Location, + pub result: Result>, ServerError>, } #[cfg(any(test, feature = "for-tests"))] @@ -98,6 +102,7 @@ impl Arbitrary for CommitHashToLocationRequestBatch { CommitHashToLocationRequestBatch { master_heads: Arbitrary::arbitrary(g), hgids: Arbitrary::arbitrary(g), + unfiltered: Arbitrary::arbitrary(g), } } } @@ -107,7 +112,7 @@ impl Arbitrary for CommitHashToLocationResponse { fn arbitrary(g: &mut G) -> Self { CommitHashToLocationResponse { hgid: Arbitrary::arbitrary(g), - location: Arbitrary::arbitrary(g), + result: Arbitrary::arbitrary(g), } } } diff --git a/eden/scm/lib/edenapi/types/src/json.rs b/eden/scm/lib/edenapi/types/src/json.rs index 5444deb26b..e9f2982fa6 100644 --- a/eden/scm/lib/edenapi/types/src/json.rs +++ b/eden/scm/lib/edenapi/types/src/json.rs @@ -154,9 +154,11 @@ pub fn parse_commit_hash_to_location_req(json: &Value) -> Result, #[serde(rename = "3", default)] pub master_heads: Vec, + #[serde(rename = "4")] + pub unfiltered: Option, } #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] @@ -189,7 +191,9 @@ pub struct WireCommitHashToLocationResponse { #[serde(rename = "1")] pub hgid: WireHgId, #[serde(rename = "2")] - pub location: WireCommitLocation, + pub location: Option, + #[serde(rename = "3")] + pub result: Option>>, } impl ToWire for CommitHashToLocationRequestBatch { @@ -201,6 +205,7 @@ impl ToWire for CommitHashToLocationRequestBatch { client_head, hgids: self.hgids.to_wire(), master_heads: self.master_heads.to_wire(), + unfiltered: self.unfiltered, } } } @@ -220,6 +225,7 @@ impl ToApi for WireCommitHashToLocationRequestBatch { let api = Self::Api { master_heads, hgids: self.hgids.to_api()?, + unfiltered: self.unfiltered, }; Ok(api) } @@ -236,9 +242,14 @@ impl ToWire for CommitHashToLocationResponse { type Wire = WireCommitHashToLocationResponse; fn to_wire(self) -> Self::Wire { + let location = match self.result { + Ok(Some(x)) => Some(x.to_wire()), + _ => None, + }; Self::Wire { hgid: self.hgid.to_wire(), - location: self.location.to_wire(), + location, + result: Some(self.result.to_wire()), } } } @@ -248,9 +259,16 @@ impl ToApi for WireCommitHashToLocationResponse { type Error = WireToApiConversionError; fn to_api(self) -> Result { + let result = match self.result { + Some(x) => x.to_api()?, + None => match self.location { + None => Ok(None), + Some(l) => Ok(Some(l.to_api()?)), + }, + }; let api = Self::Api { hgid: self.hgid.to_api()?, - location: self.location.to_api()?, + result, }; Ok(api) } diff --git a/eden/scm/lib/hgcommits/src/hybrid.rs b/eden/scm/lib/hgcommits/src/hybrid.rs index 8c29066bd4..c445796003 100644 --- a/eden/scm/lib/hgcommits/src/hybrid.rs +++ b/eden/scm/lib/hgcommits/src/hybrid.rs @@ -93,13 +93,15 @@ impl RemoteIdConvertProtocol for EdenApiProtocol { }; while let Some(response) = response.entries.next().await { let response = response.map_err(to_dag_error)?; - let path = AncestorPath { - x: Vertex::copy_from(response.location.descendant.as_ref()), - n: response.location.distance, - batch_size: 1, - }; - let name = Vertex::copy_from(response.hgid.as_ref()); - pairs.push((path, vec![name])); + if let Some(location) = response.result.map_err(to_dag_error)? { + let path = AncestorPath { + x: Vertex::copy_from(location.descendant.as_ref()), + n: location.distance, + batch_size: 1, + }; + let name = Vertex::copy_from(response.hgid.as_ref()); + pairs.push((path, vec![name])); + } } Ok(pairs) }