b02acec10d140c07a80616c5829575afe874984d
angie
  Thu Feb 11 16:20:29 2021 -0800
Tolerate undefined fields in protobuf for forwards-compatibility with newer versions of protobuf spec (e.g. newer versions of UShER).

diff --git src/hg/lib/protobuf.c src/hg/lib/protobuf.c
index 22fa60c..90b704a 100644
--- src/hg/lib/protobuf.c
+++ src/hg/lib/protobuf.c
@@ -142,82 +142,109 @@
 {
 if (*pBytesLeft < length)
     errAbort("pbParseString: called with length (%llu) > bytesLeft (%lld)", length, *pBytesLeft);
 char *string = needMem(length+1);
 bits64 i;
 for (i = 0;  i < length;  i++)
     {
     bits8 val;
     if (!pbNextByte(stream, &val, pBytesLeft))
         errAbort("Expecting string length %llu but got EOF at byte %llu", length, i);
     string[i] = val;
     }
 return string;
 }
 
+static void pbSkipBytes(FILE *stream, bits64 length, long long *pBytesLeft)
+/* Skip <length> bytes from stream.  Inefficient but I don't expect to do this much. */
+{
+if (*pBytesLeft < length)
+    errAbort("pbSkipBytes: called with length (%llu) > bytesLeft (%lld)", length, *pBytesLeft);
+bits64 i;
+for (i = 0;  i < length;  i++)
+    {
+    bits8 val;
+    if (!pbNextByte(stream, &val, pBytesLeft))
+        errAbort("Expecting to skip %llu bytes but got EOF at byte %llu", length, i);
+    }
+}
+
 static struct protobufFieldDef *protobufDefForField(struct protobufDef *proto, bits32 fieldNum)
 {
 struct protobufFieldDef *field;
 for (field = proto->fields;  field != NULL;  field = field->next)
     if (field->fieldNum == fieldNum)
         return field;
-errAbort("protobufDefForField: no field in message '%s' has number %d", proto->name, fieldNum);
 return NULL;
 }
 
 static struct protobufField *pbParseField(FILE *stream, struct protobufDef *proto,
                                           long long *pBytesLeft)
 /* If stream is at EOF, return NULL; otherwise parse and return the next message from stream. */
 {
 bits8 val;
 if (!pbNextByte(stream, &val, pBytesLeft))
     return NULL;
 struct protobufField *field;
 AllocVar(field);
 enum pbWireType wireType = pbParseWireType(val);
 bits8 fieldNumB1 = pbParseFieldNumberFirstByte(val);
 bits32 fieldNum = pbFinishVarint(stream, fieldNumB1, pBytesLeft);
 field->def = protobufDefForField(proto, fieldNum);
-enum pbDataType dataType = field->def->dataType;
+// Forwards compatibility: when a newer protobuf spec has a new field, and we don't know what
+// to do with that field, just ignore the field.
+enum pbDataType dataType = pbdtInvalid;
+if (field->def)
+    {
+    dataType = field->def->dataType;
     verbose(2, "pbParseField: wire type %d, fieldNumber %u, field name '%s', data type %d, "
             "bytesLeft %lld\n",
             wireType, fieldNum, field->def->name, dataType, *pBytesLeft);
+    }
+else
+    {
+    verbose(2, "pbParseField: wire type %d, fieldNumber %u, no definition for field, "
+            "bytesLeft %lld\n",
+            wireType, fieldNum, *pBytesLeft);
+    }
 bits64 varint;
 bits32 val32b;
 int i;
 switch (wireType)
     {
     case pbwtVarint:
         varint = pbParseVarint(stream, pBytesLeft);
         if (dataType == pbdtInt32)
             field->value.vInt32 = varint;
-        else
+        else if (field->def)
             errAbort("Sorry, data type %d not yet supported", dataType);
         break;
     case pbwt64b:
         varint = pbParse64b(stream, pBytesLeft);
         if (dataType == pbdtUint64)
             field->value.vUint64 = varint;
-        else
+        else if (field->def)
             errAbort("Sorry, data type %d not yet supported", dataType);
         break;
     case pbwtLengthDelim:
         {
         long long emByteLength = pbParseVarint(stream, pBytesLeft);
         if (*pBytesLeft < emByteLength)
             errAbort("Embedded byte length (%lld) > *pBytesLeft (%lld)",
                      emByteLength, *pBytesLeft);
+        if (field->def)
+            {
             verbose(2, "Embedded with byte length %lld, data type %d, bytesLeft %lld\n",
                     emByteLength, dataType, *pBytesLeft);
             if (emByteLength > 0)
                 {
                 long long emBytesLeft = emByteLength;
                 if (dataType == pbdtString)
                     {
                     field->value.vString = pbParseString(stream, emByteLength, &emBytesLeft);
                     field->length = emByteLength;
                     }
                 else if (dataType == pbdtInt32)
                     {
                     AllocArray(field->value.vPacked, emByteLength);
                     for (i = 0;  emBytesLeft > 0;  i++)
                         {
@@ -232,55 +259,66 @@
                     verbose(2, "Expecting %lld bytes of embedded '%s' message.\n",
                             emByteLength, field->def->embedded->name);
                     if (emByteLength > 0)
                         {
                         AllocArray(field->value.vEmbedded, emByteLength);
                         for (i = 0;  emBytesLeft > 0;  i++)
                             field->value.vEmbedded[i] = protobufParse(stream, field->def->embedded,
                                                                       &emBytesLeft);
                         field->length = i;
                         }
                     }
                 else
                     errAbort("Sorry, data type %d not yet supported", dataType);
                 *pBytesLeft -= emByteLength;
                 }
+            else
+                {
+                verbose(2, "Skipping %lld bytes of undefined field\n", emByteLength);
+                pbSkipBytes(stream, emByteLength, pBytesLeft);
+                }
+            }
         }
         break;
     case pbwt32b:
                 val32b = pbParse32b(stream, pBytesLeft);
         if (dataType == pbdtFixed32)
             field->value.vUint32 = val32b;
-        else
+        else if (field->def)
             errAbort("Sorry, data type %d not yet supported", dataType);
         break;
     default:
         errAbort("Unsupported wire type %d", wireType);
     }
 return field;
 }
 
 struct protobuf *protobufParse(FILE *stream, struct protobufDef *proto, long long *pBytesLeft)
 /* If stream is at EOF, return NULL; otherwise parse and return the next message from stream,
  * consuming up to *pBytesLeft bytes from stream.. */
 {
 verbose(2, "protobufParse: expecting message '%s' (or EOF), bytes left %lld\n",
         proto->name, *pBytesLeft);
 struct protobufField *field = pbParseField(stream, proto, pBytesLeft);
 if (field == NULL)
     return NULL;
 struct protobuf *pb;
 AllocVar(pb);
 pb->def = proto;
 pb->fields = field;
 verbose(2, "protobufParse: got field %s, %lld bytes left\n", field->def->name, *pBytesLeft);
 while (*pBytesLeft > 0)
     {
     field = pbParseField(stream, proto, pBytesLeft);
     if (field == NULL)
         break;
+    if (field->def)
+        {
         slAddHead(&pb->fields, field);
         verbose(2, "protobufParse: got field %s, %lld bytes left\n", field->def->name, *pBytesLeft);
         }
+    else
+        verbose(2, "protobufParse: skipped undefined field, %lld bytes left\n", *pBytesLeft);
+    }
 slReverse(&pb->fields);
 return pb;
 }