Extract EPS macro and use it in fast decoder

This allows eliding null checks on the hot path

PiperOrigin-RevId: 919794585

Author: protobuf-github-bot

upb/wire/decode.c

@@ -89,12 +89,6 @@ enum {
 #define OP_FIXPCK_LG2(n) (n + 5) /* n in [2, 3] => op in [7, 8] */
 #define OP_VARPCK_LG2(n) (n + 9) /* n in [0, 2, 3] => op in [9, 11, 12] */
 
-static void _upb_Decoder_AssumeEpsHasErrorHandler(upb_Decoder* d) {
-  UPB_ASSUME(upb_EpsCopyInputStream_HasErrorHandler(&d->input));
-}
-
-#define EPS(d) (_upb_Decoder_AssumeEpsHasErrorHandler(d), &(d)->input)
-
 static bool _upb_Decoder_Reserve(upb_Decoder* d, upb_Array* arr, size_t elem) {
   bool need_realloc =
       arr->UPB_PRIVATE(capacity) - arr->UPB_PRIVATE(size) < elem;

upb/wire/decode_fast/cardinality.c

@@ -1,15 +1,15 @@
 
 #include "upb/wire/decode_fast/cardinality.h"
 
-#include "upb/wire/eps_copy_input_stream.h"
 #include "upb/wire/internal/decoder.h"
+#include "upb/wire/internal/eps_copy_input_stream.h"
 
 UPB_PRESERVE_MOST
 const char* upb_DecodeFast_IsDoneFallback(upb_Decoder* d, const char* ptr) {
   int overrun;
   upb_IsDoneStatus status = UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneStatus)(
       &d->input, ptr, &overrun);
   UPB_ASSERT(status == kUpb_IsDoneStatus_NeedFallback);
-  return UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneFallback)(&d->input, ptr,
+  return UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneFallback)(EPS(d), ptr,
                                                             overrun);
 }

upb/wire/decode_fast/cardinality.h

@@ -470,12 +470,7 @@ bool upb_DecodeFast_Delimited(upb_Decoder* d, const char** ptr,
                                                    ctx)) {
     if (UPB_UNLIKELY(p == NULL)) goto fail;
   } else {
-    ptrdiff_t delta = upb_EpsCopyInputStream_PushLimit(&d->input, p, size);
-    if (UPB_UNLIKELY(delta < 0)) {
-      // Corrupt wire format: invalid limit.
-      *ptr = NULL;
-      return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, ret);
-    }
+    ptrdiff_t delta = upb_EpsCopyInputStream_PushLimit(EPS(d), p, size);
     p = func(&d->input, p, size, ctx);
     if (UPB_UNLIKELY(p == NULL)) goto fail;
     upb_EpsCopyInputStream_PopLimit(&d->input, p, delta);

upb/wire/decode_fast/dispatch.c

@@ -34,7 +34,7 @@ upb_DecodeFast_MessageIsDoneFallback(UPB_PARSE_PARAMS) {
     }
     case kUpb_IsDoneStatus_NeedFallback:
       // We've reached end-of-buffer.  Refresh the buffer.
-      ptr = UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneFallback)(&d->input, ptr,
+      ptr = UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneFallback)(EPS(d), ptr,
                                                                overrun);
 
       // We successfully refreshed the buffer (otherwise the function above

upb/wire/decode_fast/field_mismatch.c

@@ -98,7 +98,7 @@ UPB_PRESERVE_NONE upb_FastDecoder_Return _upb_FastDecoder_DecodeCheckMiniTable(
   uint32_t field_num = data;
 #ifndef NDEBUG
   uint32_t check;
-  const char* read = upb_WireReader_ReadTag(ptr, &check, &d->input);
+  const char* read = upb_WireReader_ReadTag(ptr, &check, EPS(d));
   UPB_PRIVATE(upb_EpsCopyInputStream_BoundsChecked)(&d->input);
   UPB_ASSERT(upb_WireReader_GetFieldNumber(check) == field_num);
   UPB_ASSERT(ptr + upb_DecodeFastData2_GetTagLen(data2) == read);
@@ -127,7 +127,7 @@ _upb_FastDecoder_DecodeCheckExtRegMiniTable(struct upb_Decoder* d,
 #ifndef NDEBUG
     UPB_PRIVATE(upb_EpsCopyInputStream_BoundsChecked)(&d->input);
     uint32_t check;
-    const char* read = upb_WireReader_ReadTag(ptr, &check, &d->input);
+    const char* read = upb_WireReader_ReadTag(ptr, &check, EPS(d));
 
     UPB_ASSERT(upb_WireReader_GetFieldNumber(check) == field_num);
     UPB_ASSERT(ptr + upb_DecodeFastData2_GetTagLen(data2) == read);

upb/wire/decode_fast/field_string.c

@@ -32,16 +32,12 @@ bool upb_DecodeFast_SingleStringAlias(upb_Decoder* d, const char** ptr,
 
   if (!upb_DecodeFast_DecodeSize(d, ptr, &size, next)) return false;
 
-  const char* p = *ptr;
-  if (!upb_EpsCopyInputStream_ReadStringAlwaysAlias(&d->input, p, size, sv)) {
-    return UPB_DECODEFAST_EXIT(kUpb_DecodeFastNext_FallbackToMiniTable, next);
-  }
+  *ptr = upb_EpsCopyInputStream_ReadStringAlwaysAlias(EPS(d), *ptr, size, sv);
 
   if (validate_utf8 && !utf8_range_IsValid(sv->data, sv->size)) {
     return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_BadUtf8, next);
   }
 
-  *ptr = p + size;
   return true;
 }
 

upb/wire/decode_fast/field_unknown.c

@@ -125,10 +125,7 @@ UPB_FORCEINLINE bool _upb_FastDecoder_DoDecodeUnknown(
   while (true) {
     switch (wire_type) {
       case kUpb_WireType_Varint:
-        *ptr = upb_WireReader_SkipVarint(*ptr, &d->input);
-        if (UPB_UNLIKELY(!*ptr)) {
-          return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, ret);
-        }
+        *ptr = upb_WireReader_SkipVarint(*ptr, EPS(d));
         break;
       case kUpb_WireType_32Bit:
         UPB_PRIVATE(upb_EpsCopyInputStream_ConsumeBytes)(&d->input, 4);
@@ -140,9 +137,8 @@ UPB_FORCEINLINE bool _upb_FastDecoder_DoDecodeUnknown(
         break;
       case kUpb_WireType_Delimited: {
         int size;
-        const char* p = upb_WireReader_ReadSize(*ptr, &size, &d->input);
-        if (UPB_UNLIKELY(
-                !p || !upb_EpsCopyInputStream_CheckSize(&d->input, p, size))) {
+        const char* p = upb_WireReader_ReadSize(*ptr, &size, EPS(d));
+        if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(EPS(d), p, size))) {
           return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, ret);
         }
         *ptr = p + size;
@@ -205,9 +201,7 @@ UPB_FORCEINLINE bool _upb_FastDecoder_DoDecodeUnknown(
   }
 
   upb_StringView sv;
-  if (UPB_UNLIKELY(!upb_EpsCopyCapture_End(&capture, &d->input, *ptr, &sv))) {
-    return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, ret);
-  }
+  upb_EpsCopyCapture_End(&capture, EPS(d), *ptr, &sv);
 
   bool handled_fast =
       // Check AddUnknown mode is AliasAllowMerge.

upb/wire/decode_fast/field_varint.c

@@ -52,10 +52,7 @@ static bool upb_DecodeFast_SingleVarint(upb_Decoder* d, const char** ptr,
   const char* p = *ptr;
   uint64_t val;
 
-  p = upb_WireReader_ReadVarint(p, &val, &d->input);
-  if (!p) {
-    return UPB_DECODEFAST_ERROR(d, kUpb_DecodeStatus_Malformed, next);
-  }
+  p = upb_WireReader_ReadVarint(p, &val, EPS(d));
 
   switch (type) {
     case kUpb_DecodeFast_Bool:

upb/wire/eps_copy_input_stream.c

@@ -16,11 +16,10 @@
 // Must be last.
 #include "upb/port/def.inc"
 
-const char* UPB_PRIVATE(upb_EpsCopyInputStream_ReturnError)(
-    upb_EpsCopyInputStream* e) {
+UPB_NORETURN UPB_NOINLINE void UPB_PRIVATE(
+    upb_EpsCopyInputStream_ThrowMalformed)(upb_EpsCopyInputStream* e) {
   e->error = true;
-  if (e->err) upb_ErrorHandler_ThrowError(e->err, kUpb_ErrorCode_Malformed);
-  return NULL;
+  upb_ErrorHandler_ThrowError(e->err, kUpb_ErrorCode_Malformed);
 }
 
 const char* UPB_PRIVATE(upb_EpsCopyInputStream_IsDoneFallback)(

upb/wire/internal/decoder.h

@@ -69,6 +69,12 @@ typedef struct upb_Decoder {
 #endif
 } upb_Decoder;
 
+UPB_INLINE void _upb_Decoder_AssumeEpsHasErrorHandler(upb_Decoder* d) {
+  UPB_ASSUME(upb_EpsCopyInputStream_HasErrorHandler(&d->input));
+}
+
+#define EPS(d) (_upb_Decoder_AssumeEpsHasErrorHandler(d), &(d)->input)
+
 UPB_INLINE const char* upb_Decoder_Init(upb_Decoder* d, const char* buf,
                                         size_t size,
                                         const upb_ExtensionRegistry* extreg,