fix(core): bound unpackValue container nesting depth

msgpack-core/src/main/java/org/msgpack/core/MessagePack.java

@@ -518,6 +518,8 @@ public static class UnpackerConfig
 
         private int stringSizeLimit = Integer.MAX_VALUE;
 
+        private int maxNestingDepth = 512;
+
         private int bufferSize = 8192;
 
         private int stringDecoderBufferSize = 8192;
@@ -533,6 +535,7 @@ private UnpackerConfig(UnpackerConfig copy)
             this.actionOnMalformedString = copy.actionOnMalformedString;
             this.actionOnUnmappableString = copy.actionOnUnmappableString;
             this.stringSizeLimit = copy.stringSizeLimit;
+            this.maxNestingDepth = copy.maxNestingDepth;
             this.bufferSize = copy.bufferSize;
         }
 
@@ -550,6 +553,7 @@ public int hashCode()
             result = 31 * result + (actionOnMalformedString != null ? actionOnMalformedString.hashCode() : 0);
             result = 31 * result + (actionOnUnmappableString != null ? actionOnUnmappableString.hashCode() : 0);
             result = 31 * result + stringSizeLimit;
+            result = 31 * result + maxNestingDepth;
             result = 31 * result + bufferSize;
             result = 31 * result + stringDecoderBufferSize;
             return result;
@@ -567,6 +571,7 @@ public boolean equals(Object obj)
                     && this.actionOnMalformedString == o.actionOnMalformedString
                     && this.actionOnUnmappableString == o.actionOnUnmappableString
                     && this.stringSizeLimit == o.stringSizeLimit
+                    && this.maxNestingDepth == o.maxNestingDepth
                     && this.stringDecoderBufferSize == o.stringDecoderBufferSize
                     && this.bufferSize == o.bufferSize;
         }
@@ -728,6 +733,21 @@ public int getStringSizeLimit()
             return stringSizeLimit;
         }
 
+        /**
+         * Maximum container nesting depth for {@link MessageUnpacker#unpackValue()} (default: 512).
+         */
+        public UnpackerConfig withMaxNestingDepth(int depth)
+        {
+            UnpackerConfig copy = clone();
+            copy.maxNestingDepth = depth;
+            return copy;
+        }
+
+        public int getMaxNestingDepth()
+        {
+            return maxNestingDepth;
+        }
+
         /**
          *
          */

msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java

@@ -165,6 +165,7 @@ public class MessageUnpacker
     private final CodingErrorAction actionOnMalformedString;
     private final CodingErrorAction actionOnUnmappableString;
     private final int stringSizeLimit;
+    private final int maxNestingDepth;
     private final int stringDecoderBufferSize;
 
     private MessageBufferInput in;
@@ -224,6 +225,7 @@ protected MessageUnpacker(MessageBufferInput in, MessagePack.UnpackerConfig conf
         this.actionOnMalformedString = config.getActionOnMalformedString();
         this.actionOnUnmappableString = config.getActionOnUnmappableString();
         this.stringSizeLimit = config.getStringSizeLimit();
+        this.maxNestingDepth = config.getMaxNestingDepth();
         this.stringDecoderBufferSize = config.getStringDecoderBufferSize();
     }
 
@@ -616,6 +618,17 @@ private static MessagePackException unexpectedExtension(String expected, int exp
     public ImmutableValue unpackValue()
             throws IOException
     {
+        return unpackValue(0);
+    }
+
+    private ImmutableValue unpackValue(int depth)
+            throws IOException
+    {
+        if (depth > maxNestingDepth) {
+            throw new MessageSizeException(
+                    String.format("cannot unpack a value with nesting depth larger than %,d: %,d", maxNestingDepth, depth),
+                    depth);
+        }
         MessageFormat mf = getNextFormat();
         switch (mf.getValueType()) {
             case NIL:
@@ -646,18 +659,20 @@ public ImmutableValue unpackValue()
             case ARRAY: {
                 int size = unpackArrayHeader();
                 Value[] array = new Value[size];
+                int childDepth = depth + 1;
                 for (int i = 0; i < size; i++) {
-                    array[i] = unpackValue();
+                    array[i] = unpackValue(childDepth);
                 }
                 return ValueFactory.newArray(array, true);
             }
             case MAP: {
                 int size = unpackMapHeader();
                 Value[] kvs = new Value[size * 2];
+                int childDepth = depth + 1;
                 for (int i = 0; i < size * 2; ) {
-                    kvs[i] = unpackValue();
+                    kvs[i] = unpackValue(childDepth);
                     i++;
-                    kvs[i] = unpackValue();
+                    kvs[i] = unpackValue(childDepth);
                     i++;
                 }
                 return ValueFactory.newMap(kvs, true);
@@ -679,6 +694,17 @@ public ImmutableValue unpackValue()
     public Variable unpackValue(Variable var)
             throws IOException
     {
+        return unpackValue(var, 0);
+    }
+
+    private Variable unpackValue(Variable var, int depth)
+            throws IOException
+    {
+        if (depth > maxNestingDepth) {
+            throw new MessageSizeException(
+                    String.format("cannot unpack a value with nesting depth larger than %,d: %,d", maxNestingDepth, depth),
+                    depth);
+        }
         MessageFormat mf = getNextFormat();
         switch (mf.getValueType()) {
             case NIL:
@@ -716,19 +742,21 @@ public Variable unpackValue(Variable var)
             case ARRAY: {
                 int size = unpackArrayHeader();
                 Value[] kvs = new Value[size];
+                int childDepth = depth + 1;
                 for (int i = 0; i < size; i++) {
-                    kvs[i] = unpackValue();
+                    kvs[i] = unpackValue(childDepth);
                 }
                 var.setArrayValue(kvs);
                 return var;
             }
             case MAP: {
                 int size = unpackMapHeader();
                 Value[] kvs = new Value[size * 2];
+                int childDepth = depth + 1;
                 for (int i = 0; i < size * 2; ) {
-                    kvs[i] = unpackValue();
+                    kvs[i] = unpackValue(childDepth);
                     i++;
-                    kvs[i] = unpackValue();
+                    kvs[i] = unpackValue(childDepth);
                     i++;
                 }
                 var.setMapValue(kvs);

msgpack-core/src/test/scala/org/msgpack/core/MessagePackTest.scala

@@ -713,6 +713,7 @@ class MessagePackTest extends AirSpec with PropertyCheck with Benchmark:
       a.withActionOnMalformedString(CodingErrorAction.REPORT).equals(b) shouldBe false
       a.withActionOnUnmappableString(CodingErrorAction.REPORT).equals(b) shouldBe false
       a.withStringSizeLimit(32).equals(b) shouldBe false
+      a.withMaxNestingDepth(32).equals(b) shouldBe false
       a.withStringDecoderBufferSize(32).equals(b) shouldBe false
     }
   }

msgpack-core/src/test/scala/org/msgpack/core/NestingDepthLimitTest.scala

@@ -0,0 +1,44 @@
+package org.msgpack.core
+
+import org.msgpack.core.MessagePack.UnpackerConfig
+import org.msgpack.value.Variable
+import wvlet.airspec.AirSpec
+
+class NestingDepthLimitTest extends AirSpec:
+
+  private def nestedFixArrayPayload(arrayNesting: Int): Array[Byte] =
+    val payload = new Array[Byte](arrayNesting + 1)
+    var i       = 0
+    while i < arrayNesting do
+      payload(i) = 0x91.toByte
+      i += 1
+    payload(arrayNesting) = 0xc0.toByte
+    payload
+
+  test("throws when unpackValue nesting exceeds the configured limit") {
+    val limit = 10
+    val msgpack = nestedFixArrayPayload(limit + 1)
+
+    test("unpackValue") {
+      val unpacker = new UnpackerConfig().withMaxNestingDepth(limit).newUnpacker(msgpack)
+      intercept[MessageSizeException] {
+        unpacker.unpackValue()
+      }
+    }
+
+    test("unpackValue(var)") {
+      val unpacker = new UnpackerConfig().withMaxNestingDepth(limit).newUnpacker(msgpack)
+      intercept[MessageSizeException] {
+        unpacker.unpackValue(new Variable())
+      }
+    }
+  }
+
+  test("unpackValue succeeds at the configured nesting limit") {
+    val limit   = 10
+    val msgpack = nestedFixArrayPayload(limit)
+    val unpacker = new UnpackerConfig().withMaxNestingDepth(limit).newUnpacker(msgpack)
+    unpacker.unpackValue()
+  }
+
+end NestingDepthLimitTest