proto：无法解析无效的有线格式数据

Question

我是 protobufs 的新手，目前正在编写一个从 NATS 服务器读取数据的客户端。从 NATS 服务器发送的数据是 protobuf。

我正在写的客户端是用 Go 编写的。这是我编写的 .proto 文件：

syntax = "proto3";

package execution;

option go_package = "./protos/execution";

enum OrderStatus {
  Working = 0;
  Rejected = 1;
  Cancelled = 2;
  Completed = 3;
}

enum OrderType {
  Limit = 0;
  Market = 1;
  StopLimit = 2;
  StopMarket = 3;
}

enum OrderSide {
  Buy = 0;
  Sell = 1;
}

enum RejectReason {
  NoRejection = 0;
  InstrumentNotFound = 1;
  OrderNotFound = 2;
  InvalidOrderType = 3;
  InvalidAccount = 4;
  InvalidSide = 5;
  InvalidAmount = 6;
  InvalidLimitPrice = 7;
  InvalidQuoteLimit = 8;
  InvalidActivationPrice = 9;
  InvalidTimeInForce = 10;
  MarketHalted = 11;
  MarketPaused = 12;
  NoCounterOrders = 13;
  MissingExpirationTime = 14;
  IncorrectExpirationTime = 15;
  InternalError = 16;
  IllegalStatusSwitch = 17;
  OrderAlreadyExists = 18;
  InstrumentNotReady = 19;
  ExternalSystemError = 20;
}

enum ReportCause {
  NONE = 0;
  NewOrder = 1;
  CancelOrder = 2;
  MassCancel = 3;
  Expiration = 4;
  Trigger = 5;
  MarketStatusChange = 6;
}

enum TimeInForce {
  GoodTillCancel = 0;
  ImmediateOrCancel = 1;
  FillOrKill = 2;
}

enum CancelReason {
  NotCancelled = 0;
  CancelledByTrader = 1;
  CancelledBySystem = 2;
  SelfMatchPrevention = 3;
  OrderTimeInForce = 4;
  Liquidation = 100;
}


message TradeData {
  int64 TradeId = 1;
  string Amount = 4;
  string ExecutionPrice = 5;
  OrderStatus OrderStatus = 7;
  int64 AccountId = 11;
  string MatchedOrderExternalId = 14;
  int64 MatchedOrderId = 16;
  string RemainingAmount = 17;
}

message Execution {
  string Origin = 4;
  OrderSide Side = 7;
  string RequestedPrice = 8;
  string RequestedAmount = 9;
  string RemainingAmount = 10;
  int64 ExecutedAt = 13;
  OrderStatus OrderStatus = 14;
  repeated TradeData Trades = 16;
  OrderType OrderType = 20;
  int64 Version = 22;
  int64 AccountId = 23;
  RejectReason RejectReason = 25;
  ReportCause ReportCause = 26;
  string InstructionId = 27;
  string ExternalOrderId = 28;
  int32 ExecutionEngineMarketId = 29;
  int64 OrderId = 30;
  CancelReason CancelReason = 31;
  int64 TxId = 32;
  TimeInForce TimeInForce = 34;
  string CancelledBy = 35;
}

发布服务器是用 C# 编写的，其原始消息的代码如下：

[ProtoContract]
    public class ExecutionReport : IMarketResponse, IInstructionMessage, IOrderMatcherResponse
    {
        [ProtoIgnore]
        FeedMessageType IFeedMessage.Type => FeedMessageType.ExecutionReport;

        // ReSharper disable FieldCanBeMadeReadOnly.Global
        [ProtoMember(4)] public string Origin;
        [ProtoMember(7)] public OrderSide Side;
        [ProtoMember(8)] public decimal RequestedPrice;
        [ProtoMember(9)] public decimal RequestedAmount;
        [ProtoMember(10)] public decimal RemainingAmount;
        [ProtoMember(13)] public long ExecutedAt;
        [ProtoMember(14)] public OrderStatus OrderStatus;
        [ProtoMember(16)] public List<TradeData> Trades = new List<TradeData>();
        [ProtoMember(20)] public OrderType OrderType;
        [ProtoMember(22)] public long Version { get; set; }
        [ProtoMember(23)] public long AccountId;
        [ProtoMember(25)] public RejectReason RejectReason;
        [ProtoMember(26)] public ReportCause ReportCause;
        [ProtoMember(27)] public Guid InstructionId { get; set; }
        [ProtoMember(28)] public Guid ExternalOrderId;
        [ProtoMember(29)] public int ExecutionEngineMarketId { get; set; }
        [ProtoMember(30)] public long OrderId;
        [ProtoMember(31)] public CancelReason CancelReason;
        [ProtoMember(32)] public long TxId;
        [ProtoMember(34)] public TimeInForce TimeInForce;
        [ProtoMember(35)] public string CancelledBy;
    }

[ProtoContract]
    [StructLayout(LayoutKind.Sequential)]
    public struct TradeData
    {
        [ProtoMember(1)] public long TradeId;
        [ProtoMember(4)] public decimal Amount;
        [ProtoMember(5)] public decimal ExecutionPrice;
        [ProtoMember(7)] public OrderStatus OrderStatus;
        [ProtoMember(11)] public long AccountId;
        [ProtoMember(14)] public Guid MatchedOrderExternalId;
        [ProtoMember(16)] public long MatchedOrderId;
        [ProtoMember(17)] public decimal RemainingAmount;
    }

在尝试解组数据时，我收到此错误

proto: cannot parse invalid wire-format data

这就是我解析数据的方式：

_, err = sc.Subscribe("EXEC", func(m *stan.Msg) {
varr := &protos.Execution{}
err = proto.Unmarshal(m.Data, varr)
if err != nil {
    fmt.Printf("Err unmarshalling!: %v\n\n", err.Error())
} else {
    fmt.Printf("Received a message: %+v\n", varr)
}

我从服务器接收到的示例字节数据：

[5 85 0 0 0 56 1 66 3 8 144 78 74 2 8 1 82 2 8 1 104 197 192 132 194 159 143 219 237 8 176 1 25 184 1 11 208 1 1 218 1 18 9 133 66 138 247 239 67 93 77 17 176 192 189 75 170 203 186 145 226 1 18 9 133 66 138 247 239 67 93 77 17 176 192 189 75 170 203 186 145 232 1 1 240 1 25 128 2 25]

添加更多细节：

这就是 C# 发送数据的方式：

public async Task SendAsync(IFeedMessage msg)
{
    var subject = FeedSubject.ForMessage(msg);
    var data = msg.SerializeToArray();
    using (_metrics.FeedSendLatency.Start(new MetricTags("subject", subject.Value)))
    {
        await _connection.PublishAsync(subject, data);
    }
}

这是FeedMessage的结构（ExecutionReport也是间接继承的）

public interface IFeedMessage
{
    FeedMessageType Type { get; }
    IFeedMessage Clone();
    void Reset();
}

这是如何SerializeToArray()工作的：

public static ArraySegment<byte> SerializeToArray(this IFeedMessage message)
{
    return message.SerializeToMemory(new MemoryStream());
}

public static ArraySegment<byte> SerializeToMemory(this IFeedMessage message, MemoryStream stream)
{
    var start = stream.Position;
    message.Serialize(stream);
    return new ArraySegment<byte>(stream.GetBuffer(), (int)start, (int)(stream.Position - start));
}

public static void Serialize(this IFeedMessage message, Stream stream)
{
    stream.WriteByte((byte)message.Type);
    RuntimeTypeModel.Default.SerializeWithLengthPrefix(stream, message, message.GetType(), PrefixStyle.Fixed32, 0);
}

我不确定确切的原因是什么。但我写的proto文件似乎是错误的。我浏览了几篇面临相同错误的帖子，但大多数都没有解决相同的问题。如果需要任何其他详细信息，请告诉我。

请在这件事上给予我帮助。

Answer 1

根据评论中的讨论，我成功地整理了数据。

\n

注意事项：

\n

\n
1. 数据以 5 个字节为前缀（这是完全没有必要的）：\n
   \n
   • 1 个字节表示消息类型
   \n
   • 4字节数据长度
   \n
   \n
\n
2. C# 实现使用C# 特有的decimal和数据类型。Guid（正如 中所评论的bcl.proto，跨平台代码通常应该完全避免它们）。
\n

\n

这是文件夹结构：

\n

\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 bcl.proto\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 execution.proto\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 go.mod\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 go.sum\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 main.go\n\xe2\x94\x94\xe2\x94\x80\xe2\x94\x80 protos\n    \xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 bcl.pb.go\n    \xe2\x94\x94\xe2\x94\x80\xe2\x94\x80 execution.pb.go\n

\n

bcl.proto：

\n

该文件是从github.com/protobuf-net/protobuf-net复制的。这是必需的，因为 .NET 实现使用此原始文件中的Decimal和。Guid

\n

// The types in here indicate how protobuf-net represents certain types when using protobuf-net specific\n// library features. Note that it is not *required* to use any of these types, and cross-platform code\n// should usually avoid them completely (ideally starting from a .proto schema)\n\n// Some of these are ugly, sorry. The TimeSpan / DateTime dates here pre-date the introduction of Timestamp\n// and Duration, and the "well known" types should be preferred when possible. Guids are particularly\n// awkward - it turns out that there are multiple guid representations, and I accidentally used one that\n// I can only call... "crazy-endian". Just make sure you check the order!\n\n// It should not be necessary to use bcl.proto from code that uses protobuf-net\n\nsyntax = "proto3";\n\noption csharp_namespace = "ProtoBuf.Bcl";\noption go_package = "./protos";\n\npackage bcl;\n\nmessage TimeSpan {\n  sint64 value = 1; // the size of the timespan (in units of the selected scale)\n  TimeSpanScale scale = 2; // the scale of the timespan [default = DAYS]\n  enum TimeSpanScale {\n    DAYS = 0;\n    HOURS = 1;\n    MINUTES = 2;\n    SECONDS = 3;\n    MILLISECONDS = 4;\n    TICKS = 5;\n\n    MINMAX = 15; // dubious\n  }\n}\n\nmessage DateTime {\n  sint64 value = 1; // the offset (in units of the selected scale) from 1970/01/01\n  TimeSpanScale scale = 2; // the scale of the timespan [default = DAYS]\n  DateTimeKind kind = 3; // the kind of date/time being represented [default = UNSPECIFIED]\n  enum TimeSpanScale {\n    DAYS = 0;\n    HOURS = 1;\n    MINUTES = 2;\n    SECONDS = 3;\n    MILLISECONDS = 4;\n    TICKS = 5;\n\n    MINMAX = 15; // dubious\n  }\n  enum DateTimeKind\n  {\n     // The time represented is not specified as either local time or Coordinated Universal Time (UTC).\n     UNSPECIFIED = 0;\n     // The time represented is UTC.\n     UTC = 1;\n     // The time represented is local time.\n     LOCAL = 2;\n   }\n}\n\nmessage NetObjectProxy {\n  int32 existingObjectKey = 1; // for a tracked object, the key of the **first** time this object was seen\n  int32 newObjectKey = 2; // for a tracked object, a **new** key, the first time this object is seen\n  int32 existingTypeKey = 3; // for dynamic typing, the key of the **first** time this type was seen\n  int32 newTypeKey = 4; // for dynamic typing, a **new** key, the first time this type is seen\n  string typeName = 8; // for dynamic typing, the name of the type (only present along with newTypeKey)\n  bytes payload = 10; // the new string/value (only present along with newObjectKey)\n}\n\nmessage Guid {\n  fixed64 lo = 1; // the first 8 bytes of the guid (note:crazy-endian)\n  fixed64 hi = 2; // the second 8 bytes of the guid (note:crazy-endian)\n}\n\nmessage Decimal {\n  uint64 lo = 1; // the first 64 bits of the underlying value\n  uint32 hi = 2; // the last 32 bis of the underlying value\n  uint32 signScale = 3; // the number of decimal digits (bits 1-16), and the sign (bit 0)\n}\n

\n

执行.proto

\n

syntax = "proto3";\n\npackage execution;\n\noption go_package = "./protos";\n\nimport "bcl.proto";\n\nenum OrderStatus {\n  Working = 0;\n  Rejected = 1;\n  Cancelled = 2;\n  Completed = 3;\n}\n\nenum OrderType {\n  Limit = 0;\n  Market = 1;\n  StopLimit = 2;\n  StopMarket = 3;\n}\n\nenum OrderSide {\n  Buy = 0;\n  Sell = 1;\n}\n\nenum RejectReason {\n  NoRejection = 0;\n  InstrumentNotFound = 1;\n  OrderNotFound = 2;\n  InvalidOrderType = 3;\n  InvalidAccount = 4;\n  InvalidSide = 5;\n  InvalidAmount = 6;\n  InvalidLimitPrice = 7;\n  InvalidQuoteLimit = 8;\n  InvalidActivationPrice = 9;\n  InvalidTimeInForce = 10;\n  MarketHalted = 11;\n  MarketPaused = 12;\n  NoCounterOrders = 13;\n  MissingExpirationTime = 14;\n  IncorrectExpirationTime = 15;\n  InternalError = 16;\n  IllegalStatusSwitch = 17;\n  OrderAlreadyExists = 18;\n  InstrumentNotReady = 19;\n  ExternalSystemError = 20;\n}\n\nenum ReportCause {\n  NONE = 0;\n  NewOrder = 1;\n  CancelOrder = 2;\n  MassCancel = 3;\n  Expiration = 4;\n  Trigger = 5;\n  MarketStatusChange = 6;\n}\n\nenum TimeInForce {\n  GoodTillCancel = 0;\n  ImmediateOrCancel = 1;\n  FillOrKill = 2;\n}\n\nenum CancelReason {\n  NotCancelled = 0;\n  CancelledByTrader = 1;\n  CancelledBySystem = 2;\n  SelfMatchPrevention = 3;\n  OrderTimeInForce = 4;\n  Liquidation = 100;\n}\n\n\nmessage TradeData {\n  int64 TradeId = 1;\n  bcl.Decimal Amount = 4;\n  bcl.Decimal ExecutionPrice = 5;\n  OrderStatus OrderStatus = 7;\n  int64 AccountId = 11;\n  bcl.Guid MatchedOrderExternalId = 14;\n  int64 MatchedOrderId = 16;\n  bcl.Decimal RemainingAmount = 17;\n}\n\nmessage Execution {\n  bytes Origin = 4;\n  OrderSide Side = 7;\n  bcl.Decimal RequestedPrice = 8;\n  bcl.Decimal RequestedAmount = 9;\n  bcl.Decimal RemainingAmount = 10;\n  int64 ExecutedAt = 13;\n  OrderStatus OrderStatus = 14;\n  repeated TradeData Trades = 16;\n  OrderType OrderType = 20;\n  int64 Version = 22;\n  int64 AccountId = 23;\n  RejectReason RejectReason = 25;\n  ReportCause ReportCause = 26;\n  bcl.Guid InstructionId = 27;\n  bcl.Guid ExternalOrderId = 28;\n  int32 ExecutionEngineMarketId = 29;\n  int64 OrderId = 30;\n  CancelReason CancelReason = 31;\n  int64 TxId = 32;\n  TimeInForce TimeInForce = 34;\n  string CancelledBy = 35;\n}\n

\n

原型/

\n

此文件夹中的文件是使用以下命令从 proto 文件生成的：

\n

protoc --go_out=protos --go_opt=paths=source_relative bcl.proto execution.proto\n

\n

go.mod

\n

module mymodule.local\n\ngo 1.20\n\nrequire google.golang.org/protobuf v1.30.0\n

\n

主程序

\n

\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 bcl.proto\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 execution.proto\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 go.mod\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 go.sum\n\xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 main.go\n\xe2\x94\x94\xe2\x94\x80\xe2\x94\x80 protos\n    \xe2\x94\x9c\xe2\x94\x80\xe2\x94\x80 bcl.pb.go\n    \xe2\x94\x94\xe2\x94\x80\xe2\x94\x80 execution.pb.go\n

\n

问题中提供的数据的输出：

\n

2023/06/15 17:50:58 message type: 5, message: Side:Sell  RequestedPrice:{lo:10000}  RequestedAmount:{lo:1}  RemainingAmount:{lo:1}  ExecutedAt:638223043314917445  Version:25  AccountId:11  ReportCause:NewOrder  InstructionId:{lo:5574686611683820165  hi:10500929413443338416}  ExternalOrderId:{lo:5574686611683820165  hi:10500929413443338416}  ExecutionEngineMarketId:1  OrderId:25  TxId:25\n

\n