NR-367975: Pre filter and trim request body to 500kb if it is greater

instrumentation-security/grpc-1.22.0/src/main/java/com/newrelic/agent/security/instrumentation/grpc1220/client/GrpcClient.java

@@ -204,7 +204,7 @@ private static Object customServerStream(ManagedChannel channel, FuzzRequestBean
 
     public static Object customBiDiStream(ManagedChannel channel, FuzzRequestBean requestBean, List<String> payloads) throws InterruptedException {
         GrpcStubs.CustomStub stub = GrpcStubs.newStub(channel);
-        StringBuilder body = requestBean.getBody();
+//        StringBuilder body = requestBean.getBody().getSb();
         String[] methodSplitData = requestBean.getMethod().split("/");
         String serviceName = methodSplitData[0];
         String methodName = methodSplitData[1];

instrumentation-security/grpc-1.4.0/src/main/java/com/newrelic/agent/security/instrumentation/grpc140/client/GrpcClient.java

@@ -204,7 +204,7 @@ private static Object customServerStream(ManagedChannel channel, FuzzRequestBean
 
     private static Object customBiDiStream(ManagedChannel channel, FuzzRequestBean requestBean, List<String> payloads) throws InterruptedException {
         GrpcStubs.CustomStub stub = GrpcStubs.newStub(channel);
-        StringBuilder body = requestBean.getBody();
+//        StringBuilder body = requestBean.getBody().getSb();
         String[] methodSplitData = requestBean.getMethod().split("/");
         String serviceName = methodSplitData[0];
         String methodName = methodSplitData[1];

instrumentation-security/grpc-1.40.0/src/main/java/com/newrelic/agent/security/instrumentation/grpc1400/client/GrpcClient.java

@@ -206,7 +206,7 @@ private static Object customServerStream(ManagedChannel channel, FuzzRequestBean
 
     public static Object customBiDiStream(ManagedChannel channel, FuzzRequestBean requestBean, List<String> payloads) throws InterruptedException {
         GrpcStubs.CustomStub stub = GrpcStubs.newStub(channel);
-        StringBuilder body = requestBean.getBody();
+//        StringBuilder body = requestBean.getBody().getSb();
         String[] methodSplitData = requestBean.getMethod().split("/");
         String serviceName = methodSplitData[0];
         String methodName = methodSplitData[1];

newrelic-security-agent/src/main/java/com/newrelic/agent/security/instrumentator/dispatcher/Dispatcher.java

@@ -70,9 +70,9 @@ public class Dispatcher implements Callable {
     private ExitEventBean exitEventBean;
     private AbstractOperation operation;
     private SecurityMetaData securityMetaData;
-    private Map<String, Object> extraInfo = new HashMap<String, Object>();
-    private boolean isNRCode = false;
-    private static AtomicBoolean firstEventSent = new AtomicBoolean(false);
+    private final Map<String, Object> extraInfo = new HashMap<String, Object>();
+    private final boolean isNRCode = false;
+    private static final AtomicBoolean firstEventSent = new AtomicBoolean(false);
     private final String SQL_STORED_PROCEDURE ="SQL_STORED_PROCEDURE";
 
     public ExitEventBean getExitEventBean() {
@@ -87,7 +87,7 @@ public SecurityMetaData getSecurityMetaData() {
         return securityMetaData;
     }
 
-    private static Gson GsonUtil = new Gson();
+    private static final Gson GsonUtil = new Gson();
 
     public Dispatcher(AbstractOperation operation, SecurityMetaData securityMetaData) {
         this.securityMetaData = securityMetaData;
@@ -348,7 +348,7 @@ private void processReflectedXSSEvent(JavaAgentEventBean eventBean) {
             return;
         }
         Set<String> xssConstructs = CallbackUtils.checkForReflectedXSS(securityMetaData.getRequest(), securityMetaData.getResponse());
-        if ((!xssConstructs.isEmpty() && !actuallyEmpty(xssConstructs) && StringUtils.isNotBlank(securityMetaData.getResponse().getBody().getSb())) ||
+        if ((!xssConstructs.isEmpty() && !actuallyEmpty(xssConstructs) && StringUtils.isNotBlank(securityMetaData.getResponse().getBody().toString())) ||
                 (AgentUtils.getInstance().getAgentPolicy().getVulnerabilityScan().getEnabled()
                         && AgentUtils.getInstance().getAgentPolicy().getVulnerabilityScan().getIastScan().getEnabled())) {
             JSONArray params = new JSONArray();

newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/apache/httpclient/ApacheHttpClientWrapper.java

@@ -247,7 +247,7 @@ private HttpUriRequest buildIastFuzzRequest(HttpRequest httpRequest, String endp
             requestBuilder.setHeader(NR_CSEC_JAVA_HEAD_REQUEST, "true");
         }
 
-        if (httpRequest.getBody() != null && StringUtils.isNotBlank(httpRequest.getBody())) {
+        if (httpRequest.getBody() != null && StringUtils.isNotBlank(httpRequest.getBody().toString())) {
             requestBuilder.setEntity(new StringEntity(httpRequest.getBody().toString()));
         }
 

newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/serializers/StringBuilderLimitSerializer.java

@@ -0,0 +1,16 @@
+package com.newrelic.agent.security.intcodeagent.serializers;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.newrelic.agent.security.instrumentator.utils.AgentUtils;
+import com.newrelic.api.agent.security.schema.StringBuilderLimit;
+
+import java.io.IOException;
+
+public class StringBuilderLimitSerializer extends JsonSerializer<StringBuilderLimit> {
+    @Override
+    public void serialize(StringBuilderLimit stringBuilderLimit, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
+        jsonGenerator.writeString(stringBuilderLimit.toString());
+    }
+}

newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/utils/RestrictionUtility.java

@@ -159,7 +159,7 @@ private static void parseHttpRequestParameters(HttpRequest request) {
         request.setQueryParameters(parseQueryParameters(request.getUrl()));
         request.setRequestHeaderParameters(parseRequestHeaders(request.getHeaders()));
         try {
-            request.setRequestBodyParameters(parseRequestBody(request.getBody(), request.getContentType(), request.getRequestBodyParameters()));
+            request.setRequestBodyParameters(parseRequestBody(request.getBody().toString(), request.getContentType(), request.getRequestBodyParameters()));
         } catch (RestrictionModeException e) {
             logger.log(LogLevel.WARNING, String.format("Request Body parsing failed reason %s", e.getMessage()), RestrictionUtility.class.getName());
         }
@@ -191,8 +191,8 @@ private static Map<String, Set<String>> parseRequestParameterMap(Map<String, Str
         return requestBodyParameters;
     }
 
-    private static Map<String, Set<String>> parseRequestBody(StringBuilder body, String contentType, Map<String, Set<String>> requestBodyParameters) throws RestrictionModeException {
-        if(StringUtils.isBlank(body.toString())) {
+    private static Map<String, Set<String>> parseRequestBody(String body, String contentType, Map<String, Set<String>> requestBodyParameters) throws RestrictionModeException {
+        if(StringUtils.isBlank(body)) {
             return requestBodyParameters;
         }
 
@@ -203,14 +203,14 @@ private static Map<String, Set<String>> parseRequestBody(StringBuilder body, Str
         switch (contentType) {
             case CONTENT_TYPE_APPLICATION_JSON:
             case CONTENT_TYPE_TEXT_JSON:
-                requestBodyParameters.putAll(parseJsonRequestBody(body.toString()));
+                requestBodyParameters.putAll(parseJsonRequestBody(body));
                 break;
             case CONTENT_TYPE_APPLICATION_XML:
             case CONTENT_TYPE_TEXT_XML:
-                requestBodyParameters.putAll(parseXmlRequestBody(body.toString()));
+                requestBodyParameters.putAll(parseXmlRequestBody(body));
                 break;
             case CONTENT_TYPE_APPLICATION_X_WWW_FORM_URLENCODED:
-                requestBodyParameters.putAll(queryParamKeyValueGenerator(body.toString(),new HashMap<>()));
+                requestBodyParameters.putAll(queryParamKeyValueGenerator(body,new HashMap<>()));
                 break;
             default:
                 break;

newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/utils/TransactionUtils.java

@@ -4,9 +4,9 @@
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
 import com.newrelic.agent.security.intcodeagent.logging.IAgentConstants;
 import com.newrelic.agent.security.intcodeagent.models.javaagent.HttpResponseEvent;
-import com.newrelic.agent.security.intcodeagent.websocket.EventSendPool;
 import com.newrelic.api.agent.NewRelic;
 import com.newrelic.api.agent.security.NewRelicSecurity;
+import com.newrelic.api.agent.security.schema.StringBuilderLimit;
 import com.newrelic.api.agent.security.schema.HttpResponse;
 import com.newrelic.api.agent.security.schema.SecurityMetaData;
 import com.newrelic.api.agent.security.schema.operation.SecureCookieOperationSet;
@@ -16,7 +16,7 @@
 
 public class TransactionUtils {
 
-    private static FileLoggerThreadPool logger = FileLoggerThreadPool.getInstance();
+    private static final FileLoggerThreadPool logger = FileLoggerThreadPool.getInstance();
 
     public static void reportHttpResponse() {
         if(!NewRelicSecurity.isHookProcessingActive()) {
@@ -48,10 +48,10 @@ public static void reportHttpResponse() {
     }
 
     public static boolean trimResponseBody(HttpResponse response) {
-        if(response.getBody().getSb().length() > HttpResponse.MAX_ALLOWED_RESPONSE_BODY_LENGTH) {
-            response.setBody(new StringBuilder(response.getBody().getSb().substring(0, HttpResponse.MAX_ALLOWED_RESPONSE_BODY_LENGTH)));
+        if(response.getBody().toString().length() > StringBuilderLimit.MAX_ALLOWED_BODY_LENGTH) {
+            response.setBody(new StringBuilder(response.getBody().toString().substring(0, StringBuilderLimit.MAX_ALLOWED_BODY_LENGTH)));
             response.setBody(new StringBuilder(response.getBody().append("...")));
-            response.setDataTruncated(true);
+            response.getBody().setDataTruncated(true);
             return true;
         }
         return false;

newrelic-security-agent/src/main/java/com/newrelic/agent/security/intcodeagent/websocket/JsonConverter.java

@@ -10,9 +10,11 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.newrelic.agent.security.instrumentator.utils.AgentUtils;
 import com.newrelic.agent.security.intcodeagent.filelogging.FileLoggerThreadPool;
-import com.newrelic.api.agent.security.utils.logging.LogLevel;
 import com.newrelic.agent.security.intcodeagent.serializers.K2StackTraceSerializer;
+import com.newrelic.agent.security.intcodeagent.serializers.StringBuilderLimitSerializer;
+import com.newrelic.api.agent.security.schema.StringBuilderLimit;
 import com.newrelic.api.agent.security.schema.annotations.JsonIgnore;
+import com.newrelic.api.agent.security.utils.logging.LogLevel;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.text.StringEscapeUtils;
 import org.json.simple.JSONArray;
@@ -41,14 +43,15 @@ public class JsonConverter {
     private static final String STR_END_CUELY_BRACKET = "}";
     private static final String STR_START_CUELY_BRACKET = "{";
 
-    private static ObjectMapper mapper;
+    private static final ObjectMapper mapper;
 
-    private static String serializerSelection = System.getenv().getOrDefault("K2_JSON_SERIALIZER", "Jackson");
+    private static final String serializerSelection = System.getenv().getOrDefault("K2_JSON_SERIALIZER", "Jackson");
 
     static {
         ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         SimpleModule module = new SimpleModule();
         module.addSerializer(StackTraceElement.class, new K2StackTraceSerializer());
+        module.addSerializer(StringBuilderLimit.class, new StringBuilderLimitSerializer());
         objectMapper = objectMapper.registerModule(module);
         objectMapper = objectMapper.setAnnotationIntrospector(new JacksonAnnotationIntrospector() {
 
@@ -153,7 +156,7 @@ private static String getFieldsAsJsonString(List<Field> fields, Object obj) {
                             mapField.putAll(processMap((Map) value));
                             jsonString.append(mapField);
                         } else {
-                            jsonString.append(value.toString());
+                            jsonString.append(value);
                         }
                         jsonString.append(STR_COMMA);
                     }

newrelic-security-agent/src/main/java/com/newrelic/api/agent/security/Agent.java

@@ -8,7 +8,6 @@
 import com.newrelic.agent.security.instrumentator.httpclient.RestRequestThreadPool;
 import com.newrelic.agent.security.instrumentator.os.OsVariablesInstance;
 import com.newrelic.agent.security.instrumentator.utils.*;
-import com.newrelic.agent.security.intcodeagent.apache.httpclient.SecurityClient;
 import com.newrelic.agent.security.intcodeagent.communication.ConnectionFactory;
 import com.newrelic.agent.security.intcodeagent.constants.AgentServices;
 import com.newrelic.agent.security.intcodeagent.constants.HttpStatusCodes;
@@ -43,8 +42,6 @@
 import java.io.IOException;
 import java.lang.instrument.Instrumentation;
 import java.lang.instrument.UnmodifiableClassException;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.text.ParseException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -556,17 +553,17 @@ private int jumpRoute(List<RouteSegment> value, int i1, List<String> uriSegments
     }
 
     private static boolean isRequestBodyDataExccedsAllowedLimit(SecurityMetaData securityMetaData) {
-        if(securityMetaData != null && StringUtils.length(securityMetaData.getRequest().getBody()) > HttpRequest.MAX_ALLOWED_REQUEST_BODY_LENGTH) {
-            securityMetaData.getRequest().setDataTruncated(true);
+        if(securityMetaData != null && StringUtils.length(securityMetaData.getRequest().getBody().toString()) > StringBuilderLimit.MAX_ALLOWED_BODY_LENGTH) {
+            securityMetaData.getRequest().getBody().setDataTruncated(true);
             securityMetaData.getRequest().setBody(new StringBuilder());
             return true;
             //TODO send IASTScanFailure for body truncation and drop event.
         }
         if(!securityMetaData.getRequest().getParameterMap().isEmpty()) {
             boolean parameterTruncated = false;
             for (String[] requestParam : securityMetaData.getRequest().getParameterMap().values()) {
-                if(requestParam.length > HttpRequest.MAX_ALLOWED_REQUEST_BODY_LENGTH) {
-                    securityMetaData.getRequest().setDataTruncated(true);
+                if(requestParam.length > StringBuilderLimit.MAX_ALLOWED_BODY_LENGTH) {
+                    securityMetaData.getRequest().getBody().setDataTruncated(true);
                     parameterTruncated = true;
                 }
             }

newrelic-security-api/src/main/java/com/newrelic/api/agent/security/schema/HttpRequest.java

@@ -1,5 +1,6 @@
 package com.newrelic.api.agent.security.schema;
 
+import com.newrelic.api.agent.security.schema.StringBuilderLimit;
 import com.newrelic.api.agent.security.schema.annotations.JsonIgnore;
 import java.nio.file.Paths;
 import java.util.*;
@@ -8,14 +9,10 @@
 public class HttpRequest {
     public static final String HTTP = "http";
 
-    @JsonIgnore
-    public static final int MAX_ALLOWED_REQUEST_BODY_LENGTH = 500000;
     @JsonIgnore
     public static final String QUESTION_MARK = "?";
 
-    private StringBuilder body;
-
-    private boolean dataTruncated;
+    private final StringBuilderLimit body;
 
     private String method;
 
@@ -63,8 +60,7 @@ public class HttpRequest {
 
     public HttpRequest() {
         this.clientIP = StringUtils.EMPTY;
-        this.body = new StringBuilder();
-        this.dataTruncated = false;
+        this.body = new StringBuilderLimit();
         this.method = StringUtils.EMPTY;
         this.url = StringUtils.EMPTY;
         this.headers = new ConcurrentHashMap<>();
@@ -82,13 +78,11 @@ public HttpRequest() {
         this.queryParameters = new HashMap<>();
         this.requestHeaderParameters = new HashMap<>();
         this.requestBodyParameters = new HashMap<>();
-        this.pathParameters = new HashSet<>();
     }
 
     public HttpRequest(HttpRequest servletInfo) {
         this.clientIP = servletInfo.clientIP.trim();
-        this.body = new StringBuilder(servletInfo.getBody());
-        this.dataTruncated = servletInfo.isDataTruncated();
+        this.body = servletInfo.body;
         this.method = servletInfo.getMethod().trim();
         this.url = servletInfo.getUrl().trim();
         this.headers = new ConcurrentHashMap<>(servletInfo.getHeaders());
@@ -107,6 +101,7 @@ public HttpRequest(HttpRequest servletInfo) {
         this.requestBodyParameters = servletInfo.requestBodyParameters;
         this.isRequestParametersParsed = servletInfo.isRequestParametersParsed;
         this.customDataType = servletInfo.customDataType;
+        this.pathParameters = servletInfo.pathParameters;
     }
 
     public String getMethod() {
@@ -137,7 +132,7 @@ public void setHeaders(Map<String, String> headers) {
     /**
      * @return the body
      */
-    public StringBuilder getBody() {
+    public StringBuilderLimit getBody() {
         return this.body;
     }
 
@@ -157,20 +152,6 @@ public void setParameterMap(Map<String, String[]> parameterMap) {
         this.parameterMap = parameterMap;
     }
 
-    /**
-     * @return the dataTruncated
-     */
-    public boolean isDataTruncated() {
-        return this.dataTruncated;
-    }
-
-    /**
-     * @param dataTruncated the dataTruncated to set
-     */
-    public void setDataTruncated(boolean dataTruncated) {
-        this.dataTruncated = dataTruncated;
-    }
-
     /**
      * @return the clientIP
      */
@@ -186,7 +167,7 @@ public void setClientIP(String clientIP) {
     }
 
     public void setBody(StringBuilder body) {
-        this.body = body;
+        this.body.sb = body;
     }
 
     public String getContentType() {
@@ -324,8 +305,7 @@ public Map<String, String> getCustomDataType() {
     }
 
     public void clean () {
-        this.body = new StringBuilder();
-        this.dataTruncated = false;
+        this.body.clean();
         this.headers.clear();
         this.serverPort = -1;
         this.parameterMap.clear();