New - add support for profiling flag

custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessor.java

@@ -25,6 +25,7 @@
 import com.solarwinds.joboe.logging.Logger;
 import com.solarwinds.joboe.logging.LoggerFactory;
 import com.solarwinds.joboe.sampling.Metadata;
+import com.solarwinds.joboe.sampling.TraceDecisionUtil;
 import com.solarwinds.opentelemetry.core.Util;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.SpanContext;
@@ -75,13 +76,16 @@ public boolean isEndRequired() {
   private boolean isProfilingEnabled() {
     final ProfilerSetting profilerSetting =
         (ProfilerSetting) ConfigManager.getConfig(ConfigProperty.PROFILER);
-    return profilerSetting != null && profilerSetting.isEnabled();
+    if (profilerSetting == null || !profilerSetting.isEnabled()) {
+      return false;
+    }
+    return TraceDecisionUtil.isProfilingEnabledByFlags();
   }
 
   @Override
   public void onEnding(ReadWriteSpan readWriteSpan) {
     SpanContext spanContext = readWriteSpan.getSpanContext();
-    if (spanContext.isSampled() && isProfilingEnabled()) { // only profile on sampled spans
+    if (spanContext.isSampled() && Profiler.hasActiveProfiles()) { // only profile on sampled spans
       SpanContext parentSpanContext = readWriteSpan.toSpanData().getParentSpanContext();
       if (!parentSpanContext.isValid()
           || parentSpanContext.isRemote()) { // then a root span of this service

custom/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessorTest.java

@@ -38,6 +38,7 @@
 import com.solarwinds.joboe.core.profiler.ProfilerSetting;
 import com.solarwinds.joboe.sampling.Metadata;
 import com.solarwinds.joboe.sampling.SamplingConfiguration;
+import com.solarwinds.joboe.sampling.TraceDecisionUtil;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.SpanContext;
 import io.opentelemetry.api.trace.TraceFlags;
@@ -74,6 +75,8 @@ class SolarwindsProfilingSpanProcessorTest {
 
   private MockedStatic<Profiler> profilerMock;
 
+  private MockedStatic<TraceDecisionUtil> traceDecisionUtilMock;
+
   private static final String traceId = "0123456789abcdef0123456789abcdef";
 
   private static final String spanId = "0123456789abcdef";
@@ -83,12 +86,14 @@ void setup() {
     Metadata.setup(SamplingConfiguration.builder().build());
     processor = new SolarwindsProfilingSpanProcessor();
     spanMock = mockStatic(Span.class);
-
     profilerMock = mockStatic(Profiler.class);
+    traceDecisionUtilMock = mockStatic(TraceDecisionUtil.class);
+    traceDecisionUtilMock.when(TraceDecisionUtil::isProfilingEnabledByFlags).thenReturn(true);
   }
 
   @AfterEach
   void teardown() {
+    traceDecisionUtilMock.close();
     spanMock.close();
     profilerMock.close();
   }
@@ -234,6 +239,7 @@ void onEndingWhenParentSpanValidAndLocalShouldNotCallStopProfile() throws Invali
     ConfigManager.setConfig(ConfigProperty.PROFILER, new ProfilerSetting(true, 1));
     when(mockSpan.getSpanContext()).thenReturn(mockSpanContext);
     when(mockSpanContext.isSampled()).thenReturn(true);
+    profilerMock.when(Profiler::hasActiveProfiles).thenReturn(true);
 
     when(mockSpan.toSpanData()).thenReturn(mockSpanData);
     when(mockSpanData.getParentSpanContext()).thenReturn(mockParentSpanContext);
@@ -254,6 +260,7 @@ void onEndingShouldSetAttributeToOneWhenProfileHasSample() throws InvalidConfigE
     when(mockSpanContext.isSampled()).thenReturn(true);
     when(mockSpanContext.getTraceId()).thenReturn(traceId);
     when(mockSpanContext.getSpanId()).thenReturn(spanId);
+    profilerMock.when(Profiler::hasActiveProfiles).thenReturn(true);
 
     when(mockSpan.toSpanData()).thenReturn(mockSpanData);
     when(mockSpanData.getParentSpanContext()).thenReturn(mockParentSpanContext);
@@ -273,6 +280,7 @@ void onEndingShouldSetAttributeToZeroWhenProfileHasNoSample() throws InvalidConf
     ConfigManager.setConfig(ConfigProperty.PROFILER, new ProfilerSetting(true, 1));
     when(mockSpan.getSpanContext()).thenReturn(mockSpanContext);
     when(mockSpanContext.isSampled()).thenReturn(true);
+    profilerMock.when(Profiler::hasActiveProfiles).thenReturn(true);
 
     when(mockSpanContext.getTraceId()).thenReturn(traceId);
     when(mockSpanContext.getSpanId()).thenReturn(spanId);
@@ -293,6 +301,7 @@ void onEndingShouldNotThrowWhenProfileIsNull() throws InvalidConfigException {
     ConfigManager.setConfig(ConfigProperty.PROFILER, new ProfilerSetting(true, 1));
     when(mockSpan.getSpanContext()).thenReturn(mockSpanContext);
     when(mockSpanContext.isSampled()).thenReturn(true);
+    profilerMock.when(Profiler::hasActiveProfiles).thenReturn(true);
 
     when(mockSpanContext.getTraceId()).thenReturn(traceId);
     when(mockSpan.toSpanData()).thenReturn(mockSpanData);
@@ -320,4 +329,39 @@ void isEndRequiredShouldReturnFalse() {
   void isOnEndingRequiredShouldReturnTrue() {
     assertTrue(processor.isOnEndingRequired());
   }
+
+  @Test
+  @DisplayName(
+      "onStart should set attribute to -1 on root span when profiling disabled by remote flags")
+  void onStartWhenProfilingDisabledByFlagsShouldSetAttributeToMinusOne()
+      throws InvalidConfigException {
+    ConfigManager.setConfig(ConfigProperty.PROFILER, new ProfilerSetting(true, 1));
+    traceDecisionUtilMock.when(TraceDecisionUtil::isProfilingEnabledByFlags).thenReturn(false);
+
+    when(mockSpan.getSpanContext()).thenReturn(mockSpanContext);
+    when(mockSpanContext.isSampled()).thenReturn(true);
+
+    Span mockParentSpan = mock(Span.class);
+    spanMock.when(() -> Span.fromContext(mockParentContext)).thenReturn(mockParentSpan);
+    when(mockParentSpan.getSpanContext()).thenReturn(mockParentSpanContext);
+    when(mockParentSpanContext.isValid()).thenReturn(false);
+
+    processor.onStart(mockParentContext, mockSpan);
+    verify(mockSpan).setAttribute(SW_KEY_PREFIX + "profile.spans", -1);
+    profilerMock.verify(() -> Profiler.addProfiledThread(any(), any(), any()), never());
+  }
+
+  @Test
+  @DisplayName("onEnding should not profile when profiling disabled by remote flags")
+  void onEndingWhenProfilingDisabledByFlagsShouldNotProfile() throws InvalidConfigException {
+    ConfigManager.setConfig(ConfigProperty.PROFILER, new ProfilerSetting(true, 1));
+    traceDecisionUtilMock.when(TraceDecisionUtil::isProfilingEnabledByFlags).thenReturn(false);
+
+    when(mockSpan.getSpanContext()).thenReturn(mockSpanContext);
+    when(mockSpanContext.isSampled()).thenReturn(true);
+
+    processor.onEnding(mockSpan);
+    verify(mockSpan, never()).setAttribute(anyString(), anyInt());
+    profilerMock.verify(() -> Profiler.stopProfile(any()), never());
+  }
 }

custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegateTest.java

@@ -60,7 +60,7 @@ class HttpSettingsReaderDelegateTest {
   private static final String TEST_JSON_RESPONSE =
       "{\n"
           + "    \"value\": 1000000,\n"
-          + "    \"flags\": \"SAMPLE_START,SAMPLE_THROUGH_ALWAYS,SAMPLE_BUCKET_ENABLED,TRIGGER_TRACE\",\n"
+          + "    \"flags\": \"SAMPLE_START,SAMPLE_THROUGH_ALWAYS,PROFILING,TRIGGER_TRACE\",\n"
           + "    \"timestamp\": 1741301987,\n"
           + "    \"ttl\": 120,\n"
           + "    \"arguments\": {\n"

libs/core/src/main/java/com/solarwinds/joboe/core/profiler/Profiler.java

@@ -330,25 +330,8 @@ public static Profile stopProfile(String traceId) {
     return profile;
   }
 
-  /**
-   * Stops profiling on a particular thread
-   *
-   * @param profiledThread
-   * @param traceId
-   */
-  public static void removeProfiledThread(Thread profiledThread, String traceId) {
-    Profile profile = profileByTraceId.get(traceId);
-    if (profile != null) {
-      if (profile.stopProfilingOnThread(profiledThread)) {
-        logger.debug(
-            "Stopped profiling on Thread id: "
-                + profiledThread.getId()
-                + " name: "
-                + profiledThread.getName()
-                + " for trace "
-                + traceId);
-      }
-    }
+  public static boolean hasActiveProfiles() {
+    return !profileByTraceId.isEmpty();
   }
 
   /**

libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcSettings.java

@@ -109,8 +109,8 @@ public static short convertFlagsFromStringToShort(String stringFlags) {
         flags |= OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS;
       } else if ("TRIGGER_TRACE".equals(flagToken)) {
         flags |= OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED;
-      } else if ("SAMPLE_BUCKET_ENABLED".equals(flagToken)) { // not used anymore
-        flags |= OBOE_SETTINGS_FLAG_SAMPLE_BUCKET_ENABLED;
+      } else if ("PROFILING".equals(flagToken)) {
+        flags |= OBOE_SETTINGS_FLAG_PROFILING;
       } else {
         logger.debug("Unknown flag found from settings: " + flagToken);
       }

libs/core/src/test/java/com/solarwinds/joboe/core/profiler/ProfileTest.java

@@ -232,6 +232,25 @@ private void assertNewFrames(
     }
   }
 
+  @Test
+  void hasActiveProfilesReturnsFalseWhenNoProfilesExist() {
+    assertFalse(Profiler.hasActiveProfiles());
+  }
+
+  @Test
+  void hasActiveProfilesReturnsTrueAfterAddingProfiledThread() throws SamplingException {
+    Metadata.setup(SamplingConfiguration.builder().build());
+    Thread thread = Thread.currentThread();
+    String traceId = "970026c88092a447d3b2bba3be3be2fc";
+
+    Profiler.addProfiledThread(
+        thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01"), traceId);
+    assertTrue(Profiler.hasActiveProfiles());
+
+    Profiler.stopProfile(traceId);
+    assertFalse(Profiler.hasActiveProfiles());
+  }
+
   private StackTraceElement[] simulateStackChange(Profile profile) {
     Thread thread = Thread.currentThread();
     StackTraceElement[] stackTrace = thread.getStackTrace();

libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcSettingsTest.java

@@ -0,0 +1,77 @@
+/*
+ * © SolarWinds Worldwide, LLC. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.solarwinds.joboe.core.rpc;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.solarwinds.joboe.sampling.Settings;
+import org.junit.jupiter.api.Test;
+
+class RpcSettingsTest {
+
+  @Test
+  void convertFlagsFromStringToShortParsesProfiling() {
+    short flags = RpcSettings.convertFlagsFromStringToShort("PROFILING");
+    assertEquals(Settings.OBOE_SETTINGS_FLAG_PROFILING, flags);
+  }
+
+  @Test
+  void convertFlagsFromStringToShortParsesProfilingWithOtherFlags() {
+    short flags = RpcSettings.convertFlagsFromStringToShort("OVERRIDE,SAMPLE_START,PROFILING");
+    short expected =
+        (short)
+            (Settings.OBOE_SETTINGS_FLAG_OVERRIDE
+                | Settings.OBOE_SETTINGS_FLAG_SAMPLE_START
+                | Settings.OBOE_SETTINGS_FLAG_PROFILING);
+    assertEquals(expected, flags);
+  }
+
+  @Test
+  void convertFlagsFromStringToShortParsesAllKnownFlags() {
+    short flags =
+        RpcSettings.convertFlagsFromStringToShort(
+            "OVERRIDE,SAMPLE_START,SAMPLE_THROUGH,SAMPLE_THROUGH_ALWAYS,TRIGGER_TRACE,PROFILING");
+    short expected =
+        (short)
+            (Settings.OBOE_SETTINGS_FLAG_OVERRIDE
+                | Settings.OBOE_SETTINGS_FLAG_SAMPLE_START
+                | Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH
+                | Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS
+                | Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED
+                | Settings.OBOE_SETTINGS_FLAG_PROFILING);
+    assertEquals(expected, flags);
+  }
+
+  @Test
+  void convertFlagsFromStringToShortWithoutProfilingDoesNotSetBit() {
+    short flags = RpcSettings.convertFlagsFromStringToShort("OVERRIDE,SAMPLE_START,TRIGGER_TRACE");
+    assertEquals(
+        0, flags & Settings.OBOE_SETTINGS_FLAG_PROFILING, "PROFILING bit should not be set");
+  }
+
+  @Test
+  void convertFlagsFromStringToShortIgnoresUnknownFlags() {
+    short flags = RpcSettings.convertFlagsFromStringToShort("PROFILING,UNKNOWN_FLAG");
+    assertEquals(Settings.OBOE_SETTINGS_FLAG_PROFILING, flags);
+  }
+
+  @Test
+  void convertFlagsFromStringToShortEmptyStringReturnsZero() {
+    short flags = RpcSettings.convertFlagsFromStringToShort("");
+    assertEquals(0, flags);
+  }
+}

libs/lambda/src/test/resources/solarwinds-apm-settings.json

@@ -9,7 +9,7 @@
             "TriggerStrictBucketCapacity": 6,
             "TriggerStrictBucketRate": 0.1
         },
-        "flags": "SAMPLE_START,SAMPLE_THROUGH_ALWAYS,SAMPLE_BUCKET_ENABLED,TRIGGER_TRACE",
+        "flags": "SAMPLE_START,SAMPLE_THROUGH_ALWAYS,PROFILING,TRIGGER_TRACE",
         "layer": "",
         "timestamp": 1698176407,
         "ttl": 120,

libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Settings.java

@@ -28,11 +28,7 @@ public abstract class Settings {
       OBOE_SETTINGS_FLAG_SAMPLE_THROUGH = 0x8,
       OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS = 0x10,
       OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED = 0x20,
-      OBOE_SETTINGS_FLAG_SAMPLE_BUCKET_ENABLED =
-          0x40; // NOT USED This flag is to indicates whether the args position in settings contains
-  // valid bucket rate and bucket capacity in order to avoid errors reading old
-  // settings. It does not directly control whether token bucket check should be
-  // enforced
+      OBOE_SETTINGS_FLAG_PROFILING = 0x40;
   public static final short OBOE_SETTINGS_TYPE_SKIP = 0,
       OBOE_SETTINGS_TYPE_STOP = 1,
       OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE = 2,

libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfig.java

@@ -97,8 +97,14 @@ public boolean isMetricsEnabled() {
   }
 
   public boolean hasSampleTriggerTraceFlag() {
-    return (flags & Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED)
-        == Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED;
+    return flags != null
+        && (flags & Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED)
+            == Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED;
+  }
+
+  public boolean hasProfilingFlag() {
+    return flags != null
+        && (flags & Settings.OBOE_SETTINGS_FLAG_PROFILING) == Settings.OBOE_SETTINGS_FLAG_PROFILING;
   }
 
   short getFlags() {

libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecisionUtil.java

@@ -396,6 +396,24 @@ static TraceConfig computeTraceConfig(
         remoteConfig.getBucketRates()); // token bucket parameters are always from remote config
   }
 
+  /**
+   * Callers must check local profiling config before calling this method. Returns true when remote
+   * settings are unavailable, assuming the caller already confirmed local profiling is enabled.
+   */
+  public static boolean isProfilingEnabledByFlags() {
+    Settings settings = SettingsManager.getSettings();
+    if (settings == null) {
+      return true;
+    }
+
+    short flags = settings.getFlags();
+    if ((flags & Settings.OBOE_SETTINGS_FLAG_OVERRIDE) == Settings.OBOE_SETTINGS_FLAG_OVERRIDE) {
+      return (flags & Settings.OBOE_SETTINGS_FLAG_PROFILING)
+          == Settings.OBOE_SETTINGS_FLAG_PROFILING;
+    }
+    return true;
+  }
+
   private static boolean sampled(int sampleRate) {
     return (sampleRate == SAMPLE_RESOLUTION
         || (sampleRate < SAMPLE_RESOLUTION && rand.nextInt(SAMPLE_RESOLUTION) <= sampleRate));

libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsStub.java

@@ -98,7 +98,21 @@ public SettingsStubBuilder withFlags(
         boolean isThroughAlways,
         boolean isTriggerTraceEnabled,
         boolean isOverride) {
-      flags = getFlags(isStart, isThrough, isThroughAlways, isTriggerTraceEnabled, isOverride);
+      flags =
+          getFlags(isStart, isThrough, isThroughAlways, isTriggerTraceEnabled, isOverride, false);
+      return this;
+    }
+
+    public SettingsStubBuilder withFlags(
+        boolean isStart,
+        boolean isThrough,
+        boolean isThroughAlways,
+        boolean isTriggerTraceEnabled,
+        boolean isOverride,
+        boolean isProfiling) {
+      flags =
+          getFlags(
+              isStart, isThrough, isThroughAlways, isTriggerTraceEnabled, isOverride, isProfiling);
       return this;
     }
 
@@ -137,8 +151,9 @@ private static short getFlags(
         boolean isThrough,
         boolean isThroughAlways,
         boolean isTriggerTraceEnabled,
-        boolean isOverride) {
-      byte flags = 0;
+        boolean isOverride,
+        boolean isProfiling) {
+      short flags = 0;
       if (isOverride) {
         flags |= Settings.OBOE_SETTINGS_FLAG_OVERRIDE;
       }
@@ -159,6 +174,10 @@ private static short getFlags(
         flags |= Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED;
       }
 
+      if (isProfiling) {
+        flags |= Settings.OBOE_SETTINGS_FLAG_PROFILING;
+      }
+
       return flags;
     }
   }