From 725247de9739bfc4aa064abc96aef8c882878a06 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Wed, 21 Dec 2016 16:52:13 +0100 Subject: [PATCH 01/11] If result field is JSON, add it as a branch to the output JSON, instead of escaping. --- bin/run-search-job-result-dumper.sh | 2 +- .../searchjob/SearchJobResultDumper.java | 34 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/bin/run-search-job-result-dumper.sh b/bin/run-search-job-result-dumper.sh index 5eed129..b9a692e 100755 --- a/bin/run-search-job-result-dumper.sh +++ b/bin/run-search-job-result-dumper.sh @@ -1 +1 @@ -java -cp "target/sumo-java-client-1.1-SNAPSHOT-jar-with-dependencies.jar" -server -Xmx128m com.sumologic.client.searchjob.SearchJobResultDumper $* +java -cp "target/sumo-java-client-2.3-SNAPSHOT-jar-with-dependencies.jar" -server -Xmx128m com.sumologic.client.searchjob.SearchJobResultDumper $* diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index d5cd3ad..b984b9d 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -9,15 +9,14 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import au.com.bytecode.opencsv.CSVWriter; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -705,6 +704,7 @@ private static int getMessages(CSVWriter csvWriter, searchJobId, messageOffset, messageLength); messageOffset += messageLength; + Map hasJson = new HashMap(); try { List messages = getMessagesForSearchJobResponse.getMessages(); for (LogMessage message : messages) { @@ -725,7 +725,27 @@ private static int getMessages(CSVWriter csvWriter, // Write as JSON. if (outputFormat == OutputFormat.JSON) { - String json = objectMapper.writeValueAsString(fields); + Map jsonFields = new HashMap(); + for (int i = 0; i < fieldNames.size(); i++) { + String fieldName = fieldNames.get(i); + String fieldValue = fields.get(fieldName); + jsonFields.put(fieldName, fieldValue); + + // Replace with JSON if possible. + Boolean fieldHasJson = hasJson.get(fieldName); + if (fieldHasJson == null || fieldHasJson) { + TypeReference> typeRef = + new TypeReference>() {}; + try { + HashMap actualValue = objectMapper.readValue(fieldValue, typeRef); + jsonFields.put(fieldName, actualValue); + hasJson.put(fieldName, true); + } catch (JsonProcessingException jpe) { + // Ta... + } + } + } + String json = objectMapper.writeValueAsString(jsonFields); System.out.println(json); } } From 36c17dd4a4aa9e5e1d52c4da995ce6503b23f00d Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 24 Dec 2016 12:12:32 +0100 Subject: [PATCH 02/11] Adding option to lift JSON contents in a field to the top level JSON output --- .../searchjob/SearchJobResultDumper.java | 97 +++++++++++-------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index b984b9d..92a6341 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -1,33 +1,11 @@ package com.sumologic.client.searchjob; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.URL; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - import au.com.bytecode.opencsv.CSVWriter; -import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; - +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; import com.sumologic.client.Credentials; import com.sumologic.client.SumoLogicClient; import com.sumologic.client.model.LogMessage; @@ -35,6 +13,14 @@ import com.sumologic.client.searchjob.model.GetRecordsForSearchJobResponse; import com.sumologic.client.searchjob.model.GetSearchJobStatusResponse; import com.sumologic.client.searchjob.model.SearchJobRecord; +import org.apache.commons.cli.*; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import java.io.*; +import java.net.URL; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; /** * A small but useful tool that executes a search job and dumps the results to @@ -100,6 +86,10 @@ public static void main(String[] args) throws Exception { // How many times to retry if the query fails. int retry = 1; + // If outputting JSON, which fields' keys to lift to the top level; + // this assumes that the lifted field contains valid JSON. + String liftJsonField = null; + // Create the command line options. Options options = createOptions(); @@ -183,6 +173,13 @@ public static void main(String[] args) throws Exception { if (commandLine.hasOption("json")) { outputFormat = OutputFormat.JSON; } + if (commandLine.hasOption("lift")) { + if (commandLine.hasOption("json")) { + liftJsonField = commandLine.getOptionValue("lift"); + } else { + throw new ParseException("--json required if --lift is specified"); + } + } if (commandLine.hasOption("aggregates")) { dumpAggregates = true; } @@ -279,7 +276,8 @@ public static void main(String[] args) throws Exception { "" + chunkEndMillis, timezone, retry, - lastEndFile); + lastEndFile, + liftJsonField); if (failure) { break; } @@ -421,6 +419,12 @@ private static Options createOptions() { .withArgName("json") .withDescription("Format the output as JSON") .create()); + options.addOption( + OptionBuilder.withLongOpt("lift") + .withArgName("lift") + .withDescription("Name of JSON field in the result to lift to the top level JSON output") + .hasArg() + .create()); options.addOption( OptionBuilder.withLongOpt("last-end-file") .withArgName("last-end-file") @@ -475,7 +479,8 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, String endTimestamp, String timeZone, int retry, - String lastEndFile) { + String lastEndFile, + String liftedJsonField) { int triesLeft = retry; int attempt = 1; @@ -498,7 +503,8 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, endTimestamp, timeZone, attempt, - lastEndFile); + lastEndFile, + liftedJsonField); if (failure) { System.err.println(String.format( @@ -524,7 +530,8 @@ private static boolean executeSearch(CSVWriter csvWriter, String endTimestamp, String timeZone, int attempt, - String lastEndFile) { + String lastEndFile, + String liftedJsonField) { // Create the search job. String searchJobId = sumoClient.createSearchJob( @@ -534,7 +541,7 @@ private static boolean executeSearch(CSVWriter csvWriter, timeZone); System.err.printf("[%s] %s - Search job ID: '%s', attempt: '%d'\n", - new Date(), prefix, searchJobId, attempt); + new Date(), prefix, searchJobId, attempt); try { @@ -592,7 +599,8 @@ private static boolean executeSearch(CSVWriter csvWriter, sumoClient, searchJobId, offset, - messageCount); + messageCount, + liftedJsonField); } // Wait if necessary. @@ -668,7 +676,8 @@ private static int getMessages(CSVWriter csvWriter, SumoLogicClient sumoClient, String searchJobId, int messageOffset, - int messageCount) { + int messageCount, + String liftedJsonField) { int messageLength = 0; while ((messageLength = messageCount - messageOffset) > 0) { @@ -734,17 +743,27 @@ private static int getMessages(CSVWriter csvWriter, // Replace with JSON if possible. Boolean fieldHasJson = hasJson.get(fieldName); if (fieldHasJson == null || fieldHasJson) { - TypeReference> typeRef = - new TypeReference>() {}; - try { - HashMap actualValue = objectMapper.readValue(fieldValue, typeRef); - jsonFields.put(fieldName, actualValue); - hasJson.put(fieldName, true); - } catch (JsonProcessingException jpe) { - // Ta... + TypeReference> typeRef = + new TypeReference>() { + }; + try { + HashMap actualValue = objectMapper.readValue(fieldValue, typeRef); + if (liftedJsonField != null && fieldName.equals(liftedJsonField)) { + for (Map.Entry entry: actualValue.entrySet()) { + String liftedFieldName = entry.getKey(); + Object liftedFieldValue = entry.getValue(); + jsonFields.put(liftedFieldName, liftedFieldValue); + } + } else { + jsonFields.put(fieldName, actualValue); } + hasJson.put(fieldName, true); + } catch (JsonProcessingException jpe) { + hasJson.put(fieldName, false); + } } } + String json = objectMapper.writeValueAsString(jsonFields); System.out.println(json); } From 9b42c9db761c220f45816520af2f6e1f03c07544 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 24 Dec 2016 12:13:26 +0100 Subject: [PATCH 03/11] Forgot to remove imports in last commit --- .../com/sumologic/client/searchjob/SearchJobResultDumper.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index 92a6341..1aab83c 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; import com.sumologic.client.Credentials; import com.sumologic.client.SumoLogicClient; import com.sumologic.client.model.LogMessage; From 07b50c829f1161740d6655ff111f08e9da62560f Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sun, 25 Dec 2016 15:25:25 +0100 Subject: [PATCH 04/11] Don't add field if it is lifted... --- .../com/sumologic/client/searchjob/SearchJobResultDumper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index 1aab83c..4b243db 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -736,7 +736,6 @@ private static int getMessages(CSVWriter csvWriter, for (int i = 0; i < fieldNames.size(); i++) { String fieldName = fieldNames.get(i); String fieldValue = fields.get(fieldName); - jsonFields.put(fieldName, fieldValue); // Replace with JSON if possible. Boolean fieldHasJson = hasJson.get(fieldName); @@ -758,7 +757,10 @@ private static int getMessages(CSVWriter csvWriter, hasJson.put(fieldName, true); } catch (JsonProcessingException jpe) { hasJson.put(fieldName, false); + jsonFields.put(fieldName, fieldValue); } + } else { + jsonFields.put(fieldName, fieldValue); } } From eee9b3def8a548caa7dbee5ea8848cf02f1d6495 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 7 Jan 2017 15:28:00 -0800 Subject: [PATCH 05/11] Full flat JSON. Take that, Ikea. --- .../searchjob/SearchJobResultDumper.java | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index 4b243db..a833ece 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.util.ClassUtil; import com.sumologic.client.Credentials; import com.sumologic.client.SumoLogicClient; import com.sumologic.client.model.LogMessage; @@ -744,15 +745,11 @@ private static int getMessages(CSVWriter csvWriter, new TypeReference>() { }; try { - HashMap actualValue = objectMapper.readValue(fieldValue, typeRef); + HashMap jsonValue = objectMapper.readValue(fieldValue, typeRef); if (liftedJsonField != null && fieldName.equals(liftedJsonField)) { - for (Map.Entry entry: actualValue.entrySet()) { - String liftedFieldName = entry.getKey(); - Object liftedFieldValue = entry.getValue(); - jsonFields.put(liftedFieldName, liftedFieldValue); - } + addToJsonFields(objectMapper, jsonFields, jsonValue, null); } else { - jsonFields.put(fieldName, actualValue); + jsonFields.put(fieldName, jsonValue); } hasJson.put(fieldName, true); } catch (JsonProcessingException jpe) { @@ -764,7 +761,8 @@ private static int getMessages(CSVWriter csvWriter, } } - String json = objectMapper.writeValueAsString(jsonFields); + String json = objectMapper.writeValueAsString( + new TreeMap((Map) jsonFields)); System.out.println(json); } } @@ -777,6 +775,46 @@ private static int getMessages(CSVWriter csvWriter, return messageOffset; } + private static void addToJsonFields(ObjectMapper objectMapper, + Map jsonFields, + Map jsonValue, + String prefix) { + for (Map.Entry entry : jsonValue.entrySet()) { + String fieldName = entry.getKey(); + Object fieldValue = entry.getValue(); + if (ClassUtil.isCollectionMapOrArray(fieldValue.getClass())) { + if (fieldValue instanceof Map) { + String fieldNamePrefix = (prefix == null) + ? fieldName + "_" + : prefix + fieldName + "_"; + addToJsonFields(objectMapper, jsonFields, ((Map) fieldValue), fieldNamePrefix); + } else { + List valueList = (List) fieldValue; + for (int i = 0; i < valueList.size(); i++) { + String fieldNamePrefix = (prefix == null) + ? fieldName + "_" + i + "_" + : prefix + fieldName + "_" + i + "_"; + addToJsonFields(objectMapper, jsonFields, ((Map) valueList.get(i)), fieldNamePrefix); + } + } + } else { + // Primitive + addToJsonFieldsWithPrefix(jsonFields, prefix, fieldName, fieldValue); + } + } + } + + private static void addToJsonFieldsWithPrefix(Map jsonFields, + String prefix, + String fieldName, + Object fieldValue) { + if (prefix != null) { + jsonFields.put(prefix + fieldName, fieldValue); + } else { + jsonFields.put(fieldName, fieldValue); + } + } + private static int getRecords(CSVWriter csvWriter, AtomicBoolean headerWritten, ObjectMapper objectMapper, From 32a63cd3740ee987e809bd8fb3a4d974741e7261 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Tue, 10 Jan 2017 17:57:33 -0800 Subject: [PATCH 06/11] Array handling improvements --- .../searchjob/SearchJobResultDumper.java | 116 ++++++++++++++---- 1 file changed, 92 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index a833ece..ba6d521 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -85,9 +85,15 @@ public static void main(String[] args) throws Exception { // How many times to retry if the query fails. int retry = 1; - // If outputting JSON, which fields' keys to lift to the top level; + // If outputting JSON, which field to flatten, or: add to the output as columns; // this assumes that the lifted field contains valid JSON. - String liftJsonField = null; + String flattenJson = null; + + // Whether to skip arrays when flattening JSON. + boolean skipArrays = false; + + // Whether to turn arrays into a JSON string when flattening JSON. + boolean arraysAsJson = false; // Create the command line options. Options options = createOptions(); @@ -172,11 +178,28 @@ public static void main(String[] args) throws Exception { if (commandLine.hasOption("json")) { outputFormat = OutputFormat.JSON; } - if (commandLine.hasOption("lift")) { + if (commandLine.hasOption("flatten")) { if (commandLine.hasOption("json")) { - liftJsonField = commandLine.getOptionValue("lift"); + flattenJson = commandLine.getOptionValue("flatten"); + } else { + throw new ParseException("--json required if --flatten is specified"); + } + } + if (commandLine.hasOption("skip-arrays")) { + if (commandLine.hasOption("flatten")) { + skipArrays = true; + } else { + throw new ParseException("--flatten required if --skip-arrays is specified"); + } + } + if (commandLine.hasOption("arrays-as-json")) { + if (commandLine.hasOption("flatten")) { + if (commandLine.hasOption("skip-arrays")) { + throw new ParseException("--skip-arrays and --arrays-as-json cannot be specified together"); + } + arraysAsJson = true; } else { - throw new ParseException("--json required if --lift is specified"); + throw new ParseException("--flatten required if --skip-arrays is specified"); } } if (commandLine.hasOption("aggregates")) { @@ -276,7 +299,9 @@ public static void main(String[] args) throws Exception { timezone, retry, lastEndFile, - liftJsonField); + flattenJson, + skipArrays, + arraysAsJson); if (failure) { break; } @@ -419,11 +444,21 @@ private static Options createOptions() { .withDescription("Format the output as JSON") .create()); options.addOption( - OptionBuilder.withLongOpt("lift") - .withArgName("lift") - .withDescription("Name of JSON field in the result to lift to the top level JSON output") + OptionBuilder.withLongOpt("flatten") + .withArgName("flatten") + .withDescription("Name of the JSON field in the result to flatten into columns in the output") .hasArg() .create()); + options.addOption( + OptionBuilder.withLongOpt("skip-arrays") + .withArgName("skip-arrays") + .withDescription("When flattening JSON in the output, skip arrays") + .create()); + options.addOption( + OptionBuilder.withLongOpt("arrays-as-json") + .withArgName("arrays-as-json") + .withDescription("Turns arrays into a JSON string") + .create()); options.addOption( OptionBuilder.withLongOpt("last-end-file") .withArgName("last-end-file") @@ -479,7 +514,9 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, String timeZone, int retry, String lastEndFile, - String liftedJsonField) { + String jsonFieldToFlatten, + boolean skipArrays, + boolean arraysAsJson) { int triesLeft = retry; int attempt = 1; @@ -503,7 +540,9 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, timeZone, attempt, lastEndFile, - liftedJsonField); + jsonFieldToFlatten, + skipArrays, + arraysAsJson); if (failure) { System.err.println(String.format( @@ -530,7 +569,9 @@ private static boolean executeSearch(CSVWriter csvWriter, String timeZone, int attempt, String lastEndFile, - String liftedJsonField) { + String jsonFieldToFlatten, + boolean skipArrays, + boolean arraysAsJson) { // Create the search job. String searchJobId = sumoClient.createSearchJob( @@ -599,7 +640,9 @@ private static boolean executeSearch(CSVWriter csvWriter, searchJobId, offset, messageCount, - liftedJsonField); + jsonFieldToFlatten, + skipArrays, + arraysAsJson); } // Wait if necessary. @@ -676,7 +719,9 @@ private static int getMessages(CSVWriter csvWriter, String searchJobId, int messageOffset, int messageCount, - String liftedJsonField) { + String jsonFieldToFlatten, + boolean skipArrays, + boolean arraysAsJson) { int messageLength = 0; while ((messageLength = messageCount - messageOffset) > 0) { @@ -746,8 +791,8 @@ private static int getMessages(CSVWriter csvWriter, }; try { HashMap jsonValue = objectMapper.readValue(fieldValue, typeRef); - if (liftedJsonField != null && fieldName.equals(liftedJsonField)) { - addToJsonFields(objectMapper, jsonFields, jsonValue, null); + if (jsonFieldToFlatten != null && fieldName.equals(jsonFieldToFlatten)) { + addToJsonFields(objectMapper, jsonFields, jsonValue, null, skipArrays, arraysAsJson); } else { jsonFields.put(fieldName, jsonValue); } @@ -778,7 +823,9 @@ private static int getMessages(CSVWriter csvWriter, private static void addToJsonFields(ObjectMapper objectMapper, Map jsonFields, Map jsonValue, - String prefix) { + String prefix, + boolean skipArrays, + boolean arraysAsJson) { for (Map.Entry entry : jsonValue.entrySet()) { String fieldName = entry.getKey(); Object fieldValue = entry.getValue(); @@ -787,14 +834,35 @@ private static void addToJsonFields(ObjectMapper objectMapper, String fieldNamePrefix = (prefix == null) ? fieldName + "_" : prefix + fieldName + "_"; - addToJsonFields(objectMapper, jsonFields, ((Map) fieldValue), fieldNamePrefix); + addToJsonFields(objectMapper, + jsonFields, + ((Map) fieldValue), + fieldNamePrefix, + skipArrays, + arraysAsJson); } else { - List valueList = (List) fieldValue; - for (int i = 0; i < valueList.size(); i++) { - String fieldNamePrefix = (prefix == null) - ? fieldName + "_" + i + "_" - : prefix + fieldName + "_" + i + "_"; - addToJsonFields(objectMapper, jsonFields, ((Map) valueList.get(i)), fieldNamePrefix); + if (!skipArrays) { + if (!arraysAsJson) { + List valueList = (List) fieldValue; + for (int i = 0; i < valueList.size(); i++) { + String fieldNamePrefix = (prefix == null) + ? fieldName + "_" + i + "_" + : prefix + fieldName + "_" + i + "_"; + addToJsonFields(objectMapper, + jsonFields, + ((Map) valueList.get(i)), + fieldNamePrefix, + skipArrays, + arraysAsJson); + } + } else { + try { + String value = objectMapper.writeValueAsString(fieldValue); + addToJsonFieldsWithPrefix(jsonFields, prefix, fieldName, value); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } } } } else { From a85037f235fcdb44479781f4e16a37627689db4b Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Wed, 11 Jan 2017 11:27:49 -0800 Subject: [PATCH 07/11] Prefix field name on the first level for JSON --- .../sumologic/client/searchjob/SearchJobResultDumper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index ba6d521..bc0a510 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -792,7 +792,12 @@ private static int getMessages(CSVWriter csvWriter, try { HashMap jsonValue = objectMapper.readValue(fieldValue, typeRef); if (jsonFieldToFlatten != null && fieldName.equals(jsonFieldToFlatten)) { - addToJsonFields(objectMapper, jsonFields, jsonValue, null, skipArrays, arraysAsJson); + addToJsonFields(objectMapper, + jsonFields, + jsonValue, + fieldName + "_", + skipArrays, + arraysAsJson); } else { jsonFields.put(fieldName, jsonValue); } From 2fd0197c34429551921bc4dadb7ffef39e404b70 Mon Sep 17 00:00:00 2001 From: Stefan Zier Date: Mon, 21 Aug 2017 07:37:36 +0530 Subject: [PATCH 08/11] Adding an AuthCache to HttpUtil. --- .../com/sumologic/client/util/HttpUtils.java | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/sumologic/client/util/HttpUtils.java b/src/main/java/com/sumologic/client/util/HttpUtils.java index c870107..92536e9 100644 --- a/src/main/java/com/sumologic/client/util/HttpUtils.java +++ b/src/main/java/com/sumologic/client/util/HttpUtils.java @@ -8,10 +8,12 @@ import com.sumologic.client.model.HttpPostRequest; import com.sumologic.client.model.HttpPutRequest; import org.apache.http.HttpEntity; +import org.apache.http.client.AuthCache; import org.apache.http.client.CookieStore; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.*; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.*; @@ -31,6 +33,8 @@ public class HttpUtils { private final CookieStore cookieStore = new BasicCookieStore(); + private final AuthCache authCache = new BasicAuthCache(); + // Public HTTP request methods public Response @@ -168,13 +172,6 @@ public static Map toRequestHeaders(String... parts) { // Private methods - private CloseableHttpClient getHttpClient(ConnectionConfig config) { - CredentialsProvider provider = new BasicCredentialsProvider(); - provider.setCredentials(config.getAuthScope(), config.getUsernamePasswordCredentials()); - return HttpClients.custom().setDefaultCookieStore(cookieStore) - .setDefaultCredentialsProvider(provider) - .build(); - } private static String getEndpointURI(String endpoint) { return "/" + UrlParameters.API_SERVICE + @@ -189,20 +186,30 @@ private static String getEndpointURI(String endpoint) { } private Response - doRequest(ConnectionConfig config, int timeout, HttpUriRequest method, Map requestHeaders, + doRequest(ConnectionConfig config, int timeout, HttpUriRequest uriRequest, Map requestHeaders, Request request, ResponseHandler handler, int expectedStatusCode) { // Set headers for (Map.Entry header : requestHeaders.entrySet()) { - method.setHeader(header.getKey(), header.getValue()); + uriRequest.setHeader(header.getKey(), header.getValue()); } - CloseableHttpClient httpClient = getHttpClient(config); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(config.getAuthScope(), config.getUsernamePasswordCredentials()); + CloseableHttpClient httpClient = HttpClients.custom() + .setDefaultCookieStore(cookieStore) + .build(); + + // NOTE(stefan, 2017-08-21): Pass in a long-lived authCache so that on subsequent calls we don't have to make + // two requests. + HttpClientContext context = HttpClientContext.create(); + context.setCredentialsProvider(credsProvider); + context.setAuthCache(authCache); InputStream httpStream = null; CloseableHttpResponse httpResponse = null; try { - httpResponse = httpClient.execute(method); + httpResponse = httpClient.execute(uriRequest, context); HttpEntity entity = httpResponse.getEntity(); httpStream = entity.getContent(); @@ -223,10 +230,10 @@ private static String getEndpointURI(String endpoint) { String json = writer.toString(); if (JacksonUtils.isValidJson(json)) - throw new SumoServerException(method.getURI().toString(), writer.toString()); + throw new SumoServerException(uriRequest.getURI().toString(), writer.toString()); else throw new SumoServerException( - method.getURI().toString(), + uriRequest.getURI().toString(), httpResponse.getStatusLine().getStatusCode()); } } @@ -254,8 +261,8 @@ private static String getEndpointURI(String endpoint) { } } - if (method != null) { - try { method.abort();} catch (Exception ex) {} + if (uriRequest != null) { + try { uriRequest.abort();} catch (Exception ex) {} } try { From 6db8ce4e36dc43d4aef79e615fcb266d77d6b174 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 26 Aug 2017 17:27:33 +0530 Subject: [PATCH 09/11] Messages per request command line parameter --- .../searchjob/SearchJobResultDumper.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index bc0a510..05cb7a1 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -95,6 +95,9 @@ public static void main(String[] args) throws Exception { // Whether to turn arrays into a JSON string when flattening JSON. boolean arraysAsJson = false; + // How many messages to grab in each request. + int messagesPerRequest = 1000; + // Create the command line options. Options options = createOptions(); @@ -219,6 +222,11 @@ public static void main(String[] args) throws Exception { retry = Integer.parseInt(retryValue); } + if (commandLine.hasOption("messages-per-request")) { + String messagesPerRequestValue = commandLine.getOptionValue("messages-per-batch"); + messagesPerRequest = Integer.parseInt(messagesPerRequestValue); + } + } catch (ParseException exp) { System.err.println(exp.getMessage()); @@ -301,7 +309,8 @@ public static void main(String[] args) throws Exception { lastEndFile, flattenJson, skipArrays, - arraysAsJson); + arraysAsJson, + messagesPerRequest); if (failure) { break; } @@ -516,7 +525,8 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, String lastEndFile, String jsonFieldToFlatten, boolean skipArrays, - boolean arraysAsJson) { + boolean arraysAsJson, + int messagesPerRequest) { int triesLeft = retry; int attempt = 1; @@ -542,7 +552,8 @@ private static boolean executeSearchJobWithRetry(CSVWriter csvWriter, lastEndFile, jsonFieldToFlatten, skipArrays, - arraysAsJson); + arraysAsJson, + messagesPerRequest); if (failure) { System.err.println(String.format( @@ -571,7 +582,8 @@ private static boolean executeSearch(CSVWriter csvWriter, String lastEndFile, String jsonFieldToFlatten, boolean skipArrays, - boolean arraysAsJson) { + boolean arraysAsJson, + int messagesPerRequest) { // Create the search job. String searchJobId = sumoClient.createSearchJob( @@ -642,7 +654,8 @@ private static boolean executeSearch(CSVWriter csvWriter, messageCount, jsonFieldToFlatten, skipArrays, - arraysAsJson); + arraysAsJson, + messagesPerRequest); } // Wait if necessary. @@ -721,7 +734,8 @@ private static int getMessages(CSVWriter csvWriter, int messageCount, String jsonFieldToFlatten, boolean skipArrays, - boolean arraysAsJson) { + boolean arraysAsJson, + int messagesPerRequest) { int messageLength = 0; while ((messageLength = messageCount - messageOffset) > 0) { @@ -746,7 +760,7 @@ private static int getMessages(CSVWriter csvWriter, } } - messageLength = Math.min(messageLength, 1000); + messageLength = Math.min(messageLength, messagesPerRequest); if (messageLength > 0) { System.err.printf( "[%s] %s - Search job ID: '%s', messages: '%s', getting offset: '%d', length: '%d'\n", From 4b0cfdd5516f35bf92c8e3819feb7aa185311f23 Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 26 Aug 2017 17:30:53 +0530 Subject: [PATCH 10/11] Messages per request command line parameter --- .../sumologic/client/searchjob/SearchJobResultDumper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index 05cb7a1..8a0e925 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -480,6 +480,12 @@ private static Options createOptions() { .withDescription("Number of times to retry a query in case of an error") .hasArg() .create("r")); + options.addOption( + OptionBuilder.withLongOpt("messages-per-request") + .withArgName("messages-per-request") + .withDescription("Number of messages to fetch per request") + .hasArg() + .create()); return options; } From 5a64c98cba409d56e88ee8fb868ab5ebac48b19a Mon Sep 17 00:00:00 2001 From: Christian Beedgen Date: Sat, 26 Aug 2017 17:33:40 +0530 Subject: [PATCH 11/11] Messages per request command line parameter --- bin/run-search-job-result-dumper.sh | 2 +- .../com/sumologic/client/searchjob/SearchJobResultDumper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/run-search-job-result-dumper.sh b/bin/run-search-job-result-dumper.sh index b9a692e..e0f87b1 100755 --- a/bin/run-search-job-result-dumper.sh +++ b/bin/run-search-job-result-dumper.sh @@ -1 +1 @@ -java -cp "target/sumo-java-client-2.3-SNAPSHOT-jar-with-dependencies.jar" -server -Xmx128m com.sumologic.client.searchjob.SearchJobResultDumper $* +java -cp "target/sumo-java-client-2.4-SNAPSHOT-jar-with-dependencies.jar" -server -Xmx128m com.sumologic.client.searchjob.SearchJobResultDumper $* diff --git a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java index 8a0e925..512de08 100644 --- a/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java +++ b/src/main/java/com/sumologic/client/searchjob/SearchJobResultDumper.java @@ -223,7 +223,7 @@ public static void main(String[] args) throws Exception { } if (commandLine.hasOption("messages-per-request")) { - String messagesPerRequestValue = commandLine.getOptionValue("messages-per-batch"); + String messagesPerRequestValue = commandLine.getOptionValue("messages-per-request"); messagesPerRequest = Integer.parseInt(messagesPerRequestValue); }