compiler: prevent recursive class glob cycles

ci/release/changelogs/next.md

@@ -5,6 +5,7 @@
 #### Bugfixes ⛑️
 
 - exports: honor `D2_TIMEOUT` during PNG and GIF rendering in Playwright
+- compiler: keep recursive globs out of class and variable definitions and report class reference cycles instead of overflowing the stack
 
 ---
 

d2compiler/compile.go

@@ -256,7 +256,8 @@ func _findFieldAST(ast *d2ast.Map, path []string) *d2ast.Map {
 }
 
 type compiler struct {
-	err *d2parser.ParseError
+	err             *d2parser.ParseError
+	activeClassMaps map[*d2ir.Map]struct{}
 }
 
 func (c *compiler) errorf(n d2ast.Node, f string, v ...interface{}) {
@@ -291,7 +292,10 @@ func (c *compiler) compileMap(obj *d2graph.Object, m *d2ir.Map) {
 		for _, className := range classNames {
 			classMap := m.GetClassMap(className)
 			if classMap != nil {
-				c.compileMap(obj, classMap)
+				if c.beginClass(class, className, classMap) {
+					c.compileMap(obj, classMap)
+					c.endClass(classMap)
+				}
 			} else {
 				if strings.Contains(className, ",") {
 					split := strings.Split(className, ",")
@@ -342,6 +346,22 @@ func (c *compiler) compileMap(obj *d2graph.Object, m *d2ir.Map) {
 	}
 }
 
+func (c *compiler) beginClass(class *d2ir.Field, className string, classMap *d2ir.Map) bool {
+	if c.activeClassMaps == nil {
+		c.activeClassMaps = make(map[*d2ir.Map]struct{})
+	}
+	if _, ok := c.activeClassMaps[classMap]; ok {
+		c.errorf(class.LastRef().AST(), `class %q forms a reference cycle`, className)
+		return false
+	}
+	c.activeClassMaps[classMap] = struct{}{}
+	return true
+}
+
+func (c *compiler) endClass(classMap *d2ir.Map) {
+	delete(c.activeClassMaps, classMap)
+}
+
 func (c *compiler) compileField(obj *d2graph.Object, f *d2ir.Field) {
 	keyword := strings.ToLower(f.Name.ScalarString())
 	_, isStyleReserved := d2ast.StyleKeywords[keyword]
@@ -905,7 +925,10 @@ func (c *compiler) compileEdgeMap(edge *d2graph.Edge, m *d2ir.Map) {
 		for _, className := range classNames {
 			classMap := m.GetClassMap(className)
 			if classMap != nil {
-				c.compileEdgeMap(edge, classMap)
+				if c.beginClass(class, className, classMap) {
+					c.compileEdgeMap(edge, classMap)
+					c.endClass(classMap)
+				}
 			}
 		}
 	}

d2compiler/compile_test.go

@@ -46,6 +46,18 @@ func TestOpacityValidation(t *testing.T) {
 	}
 }
 
+func TestClassReferenceCycle(t *testing.T) {
+	t.Parallel()
+
+	_, _, err := d2compiler.Compile(
+		"class-cycle.d2",
+		strings.NewReader("classes: { x: { class: x } }\na.class: x"),
+		nil,
+	)
+	assert.ErrorString(t, err, `class-cycle.d2:1:17: "class" cannot appear within "classes"
+class-cycle.d2:1:17: class "x" forms a reference cycle`)
+}
+
 func TestCompile(t *testing.T) {
 	t.Parallel()
 
@@ -3330,6 +3342,31 @@ nostar -> 1star: { class: path }
 				tassert.Equal(t, "then", g.Edges[0].Label.Value)
 			},
 		},
+		{
+			name: "recursive-glob-skips-definitions",
+			text: `classes: {
+  container: {
+    style.fill: red
+  }
+}
+vars: {
+  cont_variable: {
+    label: cont variable
+  }
+}
+**: {
+  &label: cont*
+  class: container
+}
+cont_target
+`,
+			assertions: func(t *testing.T, g *d2graph.Graph) {
+				tassert.Equal(t, 1, len(g.Objects))
+				tassert.Equal(t, "cont_target", g.Objects[0].AbsID())
+				tassert.Equal(t, []string{"container"}, g.Objects[0].Classes)
+				tassert.Equal(t, "red", g.Objects[0].Style.Fill.Value)
+			},
+		},
 		{
 			name: "array-classes",
 			text: `classes: {

d2ir/pattern.go

@@ -76,7 +76,7 @@ func _doubleGlobField(f *Field, matches *[]*Field) {
 	}
 	name := f.Name.ScalarString()
 	if _, reserved := d2ast.ReservedKeywords[name]; reserved && f.Name.IsUnquoted() {
-		if _, board := d2ast.BoardKeywords[name]; board {
+		if skipDoubleGlobSubtree(name) {
 			return
 		}
 		if f.Map() != nil {
@@ -92,6 +92,15 @@ func _doubleGlobField(f *Field, matches *[]*Field) {
 	}
 }
 
+func skipDoubleGlobSubtree(name string) bool {
+	if _, board := d2ast.BoardKeywords[name]; board {
+		return true
+	}
+	// Classes and variables are definitions, not diagram objects. Applying
+	// recursive globs inside either map can mutate definitions before use.
+	return name == "classes" || name == "vars"
+}
+
 func _tripleGlobField(f *Field, matches *[]*Field) {
 	if f == nil || f.Name == nil {
 		return
@@ -119,8 +128,9 @@ func (m *Map) _doubleGlob(fa *[]*Field) {
 		if f.Name == nil {
 			continue
 		}
-		if _, ok := d2ast.ReservedKeywords[f.Name.ScalarString()]; ok && f.Name.IsUnquoted() {
-			if _, ok := d2ast.BoardKeywords[f.Name.ScalarString()]; ok {
+		name := f.Name.ScalarString()
+		if _, ok := d2ast.ReservedKeywords[name]; ok && f.Name.IsUnquoted() {
+			if skipDoubleGlobSubtree(name) {
 				continue
 			}
 			if f.Map() != nil {