git.iko.soy/rowtype-yoga/purescript-protobuf
on master help sign in
CONFORMANCE SUITE FAILED: 193 successes, 646 skipped, 0 expected failures, 1 unexpected failures.
commite993d1afe9bf5d4bae6e4e9ac00b9fc10686bedd authorJames Brock <jamesbrock@gmail.com> dateThu 17 Sep 2020, 23:37 (6 years ago) parent269e7d0
10 files changed+48−128
conformance.nix−19 conformance/Main.purs+4−41 conformance/README.md+6−20 shell.nix−6 spago.dhall+8−8 src/Protobuf/Decode.purs+18−26 src/Protobuf/Decode32.purs+2−3 src/Protobuf/Decode64.purs+3−3 src/Protobuf/Runtime.purs+3−1 test.dhall+4−1
conformance.nix−19
@@ -1,40 +1,21 @@
11{ pkgs ? import ./nix/pkgs.nix { } }:
22let
33 protobuf = import ./nix/protobuf.nix { inherit pkgs; };
4− # protoc = protobuf.protoc;
5− # conformance = protobuf.conformance;
6− # conformance-purescript = pkgs.writeScriptBin "conformance-purescript" ''
7− # exec node -e "require('./Conformance.Main/index.js').main()"
8− # '';
9−
104in
115pkgs.mkShell {
126 nativeBuildInputs = [
137 pkgs.easy-ps.purs-0_13_8
148 pkgs.easy-ps.spago
159 pkgs.nodejs-13_x
16− # pkgs.easy-ps.pulp
17− # pkgs.protobuf3_9
18− # pkgs.nodePackages.bower
19− # pkgs.easy-ps.psc-package
20− # pkgs.dhall
21− # pkgs.dhall-json
22− # conformance.conformance
2310 protobuf
2411 ];
25− # conformance-test-runner --enforce_recommended ${conformance-purescript}
26− # buildInputs = [ conformance-purescript ];
2712 shellHook = ''
2813 export PATH="./bin:$PATH" # PATH to protoc-gen-purescript
29− # export NODE_PATH="$PWD/output:$PWD/node_modules:$NODE_PATH" # PATH to protoc-gen-purescript
3014 set -e
31− # npm install
3215 spago build
3316 protoc --purescript_out=./conformance/generated --proto_path=${protobuf}/src --proto_path=${protobuf}/conformance ${protobuf}/conformance/conformance.proto
3417 protoc --purescript_out=./conformance/generated --proto_path=${protobuf}/src --proto_path=${protobuf}/conformance ${protobuf}/src/google/protobuf/test_messages_proto3.proto
3518 spago -x conformance.dhall build
36− # conformance-test-runner --enforce_recommended $(node -e "require('./output/Conformance.Main/index.js').main()")
37− # conformance-test-runner --enforce_recommended node -e "require('./output/Conformance.Main/index.js').main()"
3819 conformance-test-runner --enforce_recommended bin/conformance-purescript
3920 '';
4021 LC_ALL = "C.UTF-8"; # https://github.com/purescript/spago/issues/507
conformance/Main.purs+4−41
@@ -1,20 +1,4 @@
1−-- | Entry point for the code-generating executable `protoc` plugin. See the
2−-- | package README for instructions on how to run the code generator.
3−-- |
4−-- | The funny thing about writing a `protoc` compiler plugin codec is that it
5−-- | bootstraps itself. We just have to write enough of the compiler plugin codec
6−-- | that it can handle the `plugin.proto` and `descriptor.proto` files, and
7−-- | then we call the compiler plugin on these `.proto` files and the compiler
8−-- | plugin codec generates the rest of itself.
9−-- |
10−-- | Then we can delete the hand-written code and generate code to replace it
11−-- | with this command.
12−-- |
13−-- | protoc --purescript_out=./src/ProtocPlugin google/protobuf/compiler/plugin.proto
14−-- |
15−-- | See
16−-- | * https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.compiler.plugin.pb
17−-- | * https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.descriptor.pb
1+-- | Entry point for the conformance test runner.
182module Conformance.Main (main) where
193
204import Prelude
@@ -25,15 +9,13 @@ import Control.Monad.Writer (tell)
259import Data.ArrayBuffer.Builder (execPut, subBuilder, putInt32le)
2610import Data.ArrayBuffer.Builder as Builder
2711import Data.ArrayBuffer.DataView as DV
28−import Data.ArrayBuffer.Types (ArrayBuffer)
2912import Data.Either (Either(..))
30−import Data.Generic.Rep.Show (genericShow)
3113import Data.Maybe (Maybe(..))
3214import Effect (Effect)
3315import Node.Buffer (toArrayBuffer, fromArrayBuffer)
3416import Node.Encoding (Encoding(..))
3517import Node.Process (stdin, stdout, stderr, exit)
36−import Node.Stream (onFinish, onReadable, read, write, writeString)
18+import Node.Stream (onReadable, read, write, writeString)
3719import ProtobufTestMessages.Proto3.TestMessagesProto3 as T3
3820import Text.Parsing.Parser (runParserT)
3921import Text.Parsing.Parser.DataView (anyInt32le)
@@ -42,7 +24,6 @@ bailOutMaybe :: forall a. String -> Effect (Maybe a) -> Effect a
4224bailOutMaybe err thing =
4325 thing >>= case _ of
4426 Nothing -> do
45− -- writeString stderr UTF8 err (void $ exit 1)
4627 void $ writeString stderr UTF8 err (pure unit)
4728 exit 1
4829 Just x -> pure x
@@ -58,28 +39,11 @@ bailOutEither thing = thing >>= case _ of
5839-- https://github.com/protocolbuffers/protobuf/blob/master/conformance/conformance_test_runner.cc
5940main :: Effect Unit
6041main = do
61− -- void $ writeString stderr UTF8 "Beginning Conformance Check" (pure unit)
6242 onReadable stdin $ do
63− -- void $ writeString stderr UTF8 "stdin readable" (pure unit)
64− -- msglenbuf <- bailOutMaybe "No message length" $ read stdin (Just 4)
65− -- msglendv <- DV.whole <$> toArrayBuffer msglenbuf
6643 msglendv <- map DV.whole $ toArrayBuffer =<< (bailOutMaybe "No message length" $ read stdin (Just 4))
6744 msglen <- bailOutEither $ runParserT msglendv $ anyInt32le
68− -- void $ writeString stderr UTF8 (show msglen) (pure unit)
69−
7045 stdinview <- map DV.whole $ toArrayBuffer =<< (bailOutMaybe "No stdin" $ read stdin (Just msglen))
71− -- stdinbufMay <- read stdin Nothing
72− -- case stdinbufMay of
73− -- Nothing -> pure unit
74− -- Just stdinbuf -> do
7546 request <- bailOutEither $ runParserT stdinview $ parseConformanceRequest $ DV.byteLength stdinview
76−
77− -- case requestParsed of
78− -- Left err -> void $ writeString stderr UTF8 (show err) (pure unit)
79− -- Right request -> do
80− -- Uncomment this line to write the parsed declarations to stderr.
81− -- void $ writeString stderr UTF8 (show request) (pure unit)
82−
8347 response <- reply request
8448
8549 case response of
@@ -97,7 +61,7 @@ main = do
9761reply :: ConformanceRequest -> Effect ConformanceResponse
9862reply (ConformanceRequest
9963 { requested_output_format: Just C.WireFormat_PROTOBUF
100− , message_type: Just "protobuf_test_messages.proto3.TestAllTypesProto3" -- :: Maybe.Maybe String
64+ , message_type: Just "protobuf_test_messages.proto3.TestAllTypesProto3"
10165 , test_category: _ -- :: Maybe.Maybe TestCategory
10266 , jspb_encoding_options: _ -- :: Maybe.Maybe JspbEncodingConfig
10367 , print_unknown_fields: _ -- :: Maybe.Maybe Boolean
@@ -117,13 +81,12 @@ reply (ConformanceRequest
11781 }
11882reply (ConformanceRequest
11983 { requested_output_format: _ -- :: Maybe.Maybe WireFormat
120− , message_type: Just "protobuf_test_messages.proto2.TestAllTypesProto2" -- :: Maybe.Maybe String
84+ , message_type: Just "protobuf_test_messages.proto2.TestAllTypesProto2"
12185 , test_category: _ -- :: Maybe.Maybe TestCategory
12286 , jspb_encoding_options: _ -- :: Maybe.Maybe JspbEncodingConfig
12387 , print_unknown_fields: _ -- :: Maybe.Maybe Boolean
12488 , payload: _ -- Just (C.ConformanceRequest_Payload_Protobuf_payload arraybuffer)
12589 }) = pure $ ConformanceResponse
126− -- { result: Just $ ConformanceResponse_Result_Runtime_error $ "runtime " <> message_type
12790 { result: Just $ ConformanceResponse_Result_Skipped "Skipped proto2"
12891 }
12992
conformance/README.md+6−20
@@ -2,28 +2,14 @@
22
33[Conformance README](https://github.com/protocolbuffers/protobuf/tree/master/conformance)
44
5−The `protobuf.nix` file has a derivation attribute `conformance`.
5+The `nix/protobuf.nix` derivation will build protobuf along with its
6+conformance test runners.
67
7−Build the conformance test runner.
8+## Running the conformance test
89
9−```
10−nix-build -A conformance protobuf.nix
11−```
12−
13−From the `protobuf/conformance/Makefile.am`, here is an example of
14−[how to run the conformance test runner](https://github.com/protocolbuffers/protobuf/blob/f4aa17b28af168cb3168f029de796d5994c321f6/conformance/Makefile.am#L334-L336) for C++.
15−
16−> ```
17−> ./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp
18−> ```
19−
20−We can do
10+From the top directory of this repo, run
2111
2212```
23−./result/bin/conformance-test-runner --enforce_recommended ./result/bin/conformance-cpp
13+nix-shell --command 'npm install'
14+nix-shell nix/conformance.nix
2415```
25−
26−## Writing a conformance test program
27−
28−See `./result/conformance/conformance.proto`.
29−
shell.nix−6
@@ -13,7 +13,6 @@ pkgs.mkShell {
1313 pkgs.easy-ps.psc-package
1414 pkgs.dhall
1515 pkgs.dhall-json
16− # conformance.conformance
1716 ];
1817 shellHook = ''
1918 export PATH="./bin:$PATH" # PATH to protoc-gen-purescript
@@ -33,11 +32,6 @@ pkgs.mkShell {
3332 echo ""
3433 echo " protoc --purescript_out=path_to_output *.proto"
3534 echo ""
36− echo "To run the Google conformance test on purescript-protobuf, run:"
37− echo ""
38− echo " protoc --purescript_out=./generate-conformance --proto_path=conformance/result/src/ --proto_path=conformance/result/conformance/ ./conformance/result/src/google/protobuf/test_messages_proto3.proto"
39− echo " spago -x conformance.dhall build"
40− echo " spago -x conformance.dhall build"
4135 '';
4236 LC_ALL = "C.UTF-8"; # https://github.com/purescript/spago/issues/507
4337}
spago.dhall+8−8
@@ -2,18 +2,18 @@
22-}
33{ name = "protobuf-library"
44, dependencies =
5− [ "parsing"
6− , "parsing-dataview"
7− , "arraybuffer-types"
8− , "arraybuffer"
5+ [ "arraybuffer"
96 , "arraybuffer-builder"
10− , "uint"
11− , "text-encoding"
7+ , "arraybuffer-types"
128 , "longs"
13− , "node-streams"
14− , "node-process"
159 , "node-buffer"
1610 , "node-path"
11+ , "node-process"
12+ , "node-streams"
13+ , "parsing"
14+ , "parsing-dataview"
15+ , "text-encoding"
16+ , "uint"
1717 ]
1818, packages = ./packages.dhall
1919, sources = [ "src/**/*.purs" ]
src/Protobuf/Decode.purs+18−26
@@ -23,25 +23,24 @@ module Protobuf.Decode
2323) where
2424
2525import Prelude
26−import Effect (Effect)
27−import Effect.Class (class MonadEffect, liftEffect)
28−import Data.Maybe (Maybe(..))
29−import Data.Either (Either(..))
26+
3027import Control.Monad.Trans.Class (lift)
31−import Text.Parsing.Parser (ParserT, fail)
32−import Text.Parsing.Parser.DataView as Parse
33−import Data.UInt (UInt)
34−import Data.UInt as UInt
35−import Data.Long.Internal (Long, Signed, Unsigned, fromLowHighBits, unsignedToSigned)
36−import Data.Long as SLong
37−import Data.Float32 (Float32)
38−import Data.ArrayBuffer.Types (ArrayBuffer, DataView, Uint8Array)
39−import Data.ArrayBuffer.Typed as AT
4028import Data.ArrayBuffer.ArrayBuffer as AB
4129import Data.ArrayBuffer.DataView as DV
30+import Data.ArrayBuffer.Typed as AT
31+import Data.ArrayBuffer.Types (ArrayBuffer, DataView, Uint8Array)
32+import Data.Either (Either(..))
33+import Data.Float32 (Float32)
34+import Data.Long.Internal (Long, Signed, Unsigned, fromLowHighBits, unsignedToSigned, lowBits)
4235import Data.TextDecoding (decodeUtf8)
36+import Data.UInt (UInt)
37+import Data.UInt as UInt
38+import Effect (Effect)
39+import Effect.Class (class MonadEffect, liftEffect)
4340import Protobuf.Decode32 (varint32, zigzag32, tag32)
4441import Protobuf.Decode64 (varint64, zigzag64)
42+import Text.Parsing.Parser (ParserT, fail)
43+import Text.Parsing.Parser.DataView as Parse
4544
4645-- | __double__
4746-- | [Scalar Value Type](https://developers.google.com/protocol-buffers/docs/proto3#scalar)
@@ -58,9 +57,7 @@ float = Parse.anyFloat32le
5857int32 :: forall m. MonadEffect m => ParserT DataView m Int
5958int32 = do
6059 n <- varint64
61− case SLong.toInt (unsignedToSigned n) of
62− Just x -> pure x
63− Nothing -> fail "int32 overflow. Decoded varint was more than 32 bits."
60+ pure $ lowBits n
6461 -- But this is a problem with the Protobuf spec?
6562 -- “If you use int32 or int64 as the type for a negative number, the resulting
6663 -- varint is always ten bytes long”
@@ -68,6 +65,10 @@ int32 = do
6865 -- So what are we supposed to do if the field type is int32 but the
6966 -- decoded varint is too big? There is no guarantee that either negative
7067 -- or positive numbers encoded as varints will be in range.
68+ -- I suppose we could check that the high bits are all 0
69+ -- for positive or all 1 for negative.
70+ -- The conformance tests seem to like it if we just throw away
71+ -- the hight bits. So that's what we'll do.
7172
7273-- | __int64__
7374-- | [Scalar Value Type](https://developers.google.com/protocol-buffers/docs/proto3#scalar)
@@ -79,16 +80,7 @@ int64 = unsignedToSigned <$> varint64
7980uint32 :: forall m. MonadEffect m => ParserT DataView m UInt
8081uint32 = do
8182 n <- varint64
82− case SLong.toInt (unsignedToSigned n) of
83− Just x -> pure $ UInt.fromInt x
84− Nothing -> fail "uint32 overflow. Decoded varint was more than 32 bits."
85− -- But this is a problem with the Protobuf spec?
86− -- “If you use int32 or int64 as the type for a negative number, the resulting
87− -- varint is always ten bytes long”
88− -- https://developers.google.com/protocol-buffers/docs/encoding#signed_integers
89− -- So what are we supposed to do if the field type is uint32 but the
90− -- decoded varint is too big? There is no guarantee that either negative
91− -- or positive numbers encoded as varints will be in range.
83+ pure $ UInt.fromInt $ lowBits n
9284
9385-- | __uint64__
9486-- | [Scalar Value Type](https://developers.google.com/protocol-buffers/docs/proto3#scalar)
src/Protobuf/Decode32.purs+2−3
@@ -24,17 +24,16 @@ import Data.Enum (toEnum)
2424import Data.Maybe (Maybe(..))
2525import Text.Parsing.Parser (ParserT, fail)
2626import Text.Parsing.Parser.DataView as Parse
27−import Data.UInt (UInt, fromInt, toInt, (.&.), (.|.), (.^.), shr, shl)
27+import Data.UInt (UInt, fromInt, toInt, (.&.), (.|.), (.^.), zshr, shr, shl)
2828import Data.ArrayBuffer.Types (DataView)
2929import Protobuf.Common (FieldNumber, WireType)
3030
3131-- | https://stackoverflow.com/questions/2210923/zig-zag-decoding
3232zigzag32 :: UInt -> Int
33−zigzag32 n = toInt $ (n `shr` (fromInt 1)) .^. (unegate (n .&. (fromInt 1)))
33+zigzag32 n = toInt $ (n `zshr` (fromInt 1)) .^. (unegate (n .&. (fromInt 1)))
3434 where
3535 unegate :: UInt -> UInt
3636 unegate = fromInt <<< negate <<< toInt
37− -- unegate x = complement x + (fromInt 1) -- TODO switch to this definition?
3837
3938-- | Parse the field number and wire type of the next field.
4039-- | https://developers.google.com/protocol-buffers/docs/encoding#structure
src/Protobuf/Decode64.purs+3−3
@@ -23,8 +23,8 @@ import Data.Long.Internal
2323 , or
2424 , xor
2525 , complement
26− , shr
2726 , shl
27+ , zshr
2828 )
2929import Data.UInt (UInt)
3030import Data.UInt as UInt
@@ -44,7 +44,7 @@ fromInt = signedToUnsigned <<< signedLongFromInt <<< UInt.toInt
4444
4545-- | https://stackoverflow.com/questions/2210923/zig-zag-decoding
4646zigzag64 :: Long Unsigned -> Long Signed
47−zigzag64 n = let n' = unsignedToSigned n in (n' `shr` u1) .^. (lnegate (n' .&. u1))
47+zigzag64 n = let n' = unsignedToSigned n in (n' `zshr` u1) .^. (lnegate (n' .&. u1))
4848 where
4949 lnegate x = complement x + u1
5050 u1 = unsafeFromInt 1
@@ -100,7 +100,7 @@ varint64 = do
100100 n_9 <- fromInt <$> Parse.anyUint8
101101 if n_9 < u0x02
102102 then pure $ acc_8 .|. (n_9 `shl` u63)
103− else fail "varint64 overflow. Possibly there is an encoding error in the input stream. Please report issues at https://github.com/xc-jp/purescript-protobuf-library/issues"
103+ else fail "varint64 overflow. Possibly there is an encoding error in the input stream."
104104 where
105105 u7 = unsafeFromInt 7
106106 u14 = unsafeFromInt 14
src/Protobuf/Runtime.purs+3−1
@@ -74,7 +74,9 @@ parseMessage construct default parseField length = do
7474 where
7575 applyParser = do
7676 Tuple fieldNumber wireType <- Decode.tag32
77− parseField (UInt.toInt fieldNumber) wireType
77+ if fieldNumber == UInt.fromInt 0
78+ then fail "Field number 0 not allowed." -- Conformance tests require this
79+ else parseField (UInt.toInt fieldNumber) wireType
7880
7981-- | Parse position, zero-based, unlike Text.Parsing.Parser.Position which is one-based.
8082type Pos = Int
test.dhall+4−1
@@ -12,5 +12,8 @@ let conf = ./spago.dhall
1212
1313in conf //
1414 { sources = [ "test/**/*.purs", "src/Protobuf/*.purs" ]
15− , dependencies = conf.dependencies # [ "assert" ]
15+ , dependencies = conf.dependencies #
16+ [ "assert"
17+ , "psci-support"
18+ ]
1619 }