From 501c69aa266d836664227ae0e55820af17726b02 Mon Sep 17 00:00:00 2001 From: Adam M Date: Fri, 28 Jun 2019 09:07:53 +0200 Subject: [PATCH 01/23] Performance changes --- .gitignore | 2 +- .../appium/mitmproxy/InterceptedMessage.java | 16 ++--- .../io/appium/mitmproxy/MitmproxyJava.java | 71 ++++++++----------- .../io/appium/mitmproxy/MitmproxyServer.java | 29 ++++---- .../appium/mitmproxy/MitmproxyJavaTest.java | 28 ++++---- 5 files changed, 67 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 3b9e72b..43936ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .gradle -.idea/* +.idea out build __pycache__ diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 47326b2..527c7be 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -26,9 +26,10 @@ public class InterceptedMessage { public int responseCode; public List responseHeaders; + private final static ObjectMapper objectMapper = new ObjectMapper(); public InterceptedMessage(ByteBuffer buffer) throws IOException { - buffer = buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.order(ByteOrder.LITTLE_ENDIAN); int metadataSize = buffer.getInt(); int request_content_size = buffer.getInt(); @@ -43,8 +44,7 @@ public InterceptedMessage(ByteBuffer buffer) throws IOException { responseBody = new byte[response_content_size]; buffer.get(responseBody); - ObjectMapper mapper = new ObjectMapper(); - JsonNode metadata = mapper.readTree(metadataBytes); + JsonNode metadata = objectMapper.readTree(metadataBytes); requestMethod = metadata.get("request").get("method").asText(); requestURL = new URL(metadata.get("request").get("url").asText()); JsonNode headers = metadata.get("request").get("headers"); @@ -72,13 +72,12 @@ public ByteBuffer serializedResponseToMitmproxy() throws JsonProcessingException // create JSON for metadata. Which is the responseCode and responseHeaders. // while we're at it, set the Content-Length header - ObjectMapper mapper = new ObjectMapper(); - ObjectNode metadataRoot = mapper.createObjectNode(); + ObjectNode metadataRoot = objectMapper.createObjectNode(); metadataRoot.put("status_code", responseCode); - ArrayNode headersNode = mapper.createArrayNode(); + ArrayNode headersNode = objectMapper.createArrayNode(); List headerNodes = responseHeaders.stream().map((h) -> { - ArrayNode headerPair = mapper.createArrayNode(); + ArrayNode headerPair = objectMapper.createArrayNode(); headerPair.add(h[0]); if (h[0].equals("content-length")) { headerPair.add(Integer.toString(contentLength)); @@ -90,11 +89,10 @@ public ByteBuffer serializedResponseToMitmproxy() throws JsonProcessingException headersNode.addAll(headerNodes); metadataRoot.set("headers", headersNode); - String metadataJson = mapper.writeValueAsString(metadataRoot); + String metadataJson = objectMapper.writeValueAsString(metadataRoot); byte[] metadata = metadataJson.getBytes(StandardCharsets.UTF_8); int metadataLength = metadata.length; - ByteBuffer buffer = ByteBuffer.allocate(8 + metadataLength + contentLength); buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putInt(metadataLength); diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java index 61a7897..b940a79 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java @@ -1,5 +1,6 @@ package io.appium.mitmproxy; +import org.apache.commons.io.IOUtils; import org.zeroturnaround.exec.ProcessExecutor; import org.zeroturnaround.exec.ProcessResult; import org.zeroturnaround.exec.stream.slf4j.Slf4jStream; @@ -9,8 +10,8 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; +import java.net.ServerSocket; import java.net.Socket; -import java.net.URISyntaxException; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.function.Function; @@ -18,18 +19,27 @@ public class MitmproxyJava { private String mitmproxyPath; + private Function messageInterceptor; + private int proxyPort; private MitmproxyServer server; private Future mitmproxyProcess; public static final int WEBSOCKET_PORT = 8765; - public MitmproxyJava(String mitmproxyPath, Function messageInterceptor) { + public MitmproxyJava(String mitmproxyPath, Function messageInterceptor, int proxyPort) { this.mitmproxyPath = mitmproxyPath; - server = new MitmproxyServer(new InetSocketAddress("localhost", WEBSOCKET_PORT), messageInterceptor); - server.start(); + this.messageInterceptor = messageInterceptor; + this.proxyPort = proxyPort; + } + + public MitmproxyJava(String mitmproxyPath, Function messageInterceptor) { + this(mitmproxyPath, messageInterceptor, 8080); } - public void start() throws IOException, TimeoutException, URISyntaxException { - System.out.println("starting mitmproxy on port 8080"); + public void start() throws IOException, TimeoutException { + System.out.println("starting mitmproxy on port " + proxyPort); + + server = new MitmproxyServer(new InetSocketAddress("localhost", WEBSOCKET_PORT), messageInterceptor); + server.start(); // python script file is zipped inside our jar. extract it into a temporary file. String pythonScriptPath = extractPythonScriptToFile(); @@ -41,35 +51,25 @@ public void start() throws IOException, TimeoutException, URISyntaxException { .start() .getFuture(); - waitForPortToBeInUse(8080); - System.out.println("mitmproxy started on port 8080"); + waitForPortToBeInUse(proxyPort); + System.out.println("mitmproxy started on port " + proxyPort); } - private String extractPythonScriptToFile() throws URISyntaxException, IOException { + private String extractPythonScriptToFile() throws IOException { File outfile = File.createTempFile("mitmproxy-python-plugin", ".py"); - InputStream instream = getClass().getClassLoader().getResourceAsStream("scripts/proxy.py"); - FileOutputStream outstream = new FileOutputStream(outfile); - - byte[] buffer = new byte[1024]; + try ( + InputStream inputStream = getClass().getClassLoader().getResourceAsStream("scripts/proxy.py"); + FileOutputStream outputStream = new FileOutputStream(outfile)) { - int length; - /*copying the contents from input stream to - * output stream using read and write methods - */ - while ((length = instream.read(buffer)) > 0){ - outstream.write(buffer, 0, length); + IOUtils.copy(inputStream, outputStream); } - //Closing the input/output file streams - instream.close(); - outstream.close(); - return outfile.getCanonicalPath(); } - public void stop() throws IOException, InterruptedException { + public void stop() throws InterruptedException { if (mitmproxyProcess != null) { mitmproxyProcess.cancel(true); } @@ -79,30 +79,17 @@ public void stop() throws IOException, InterruptedException { private void waitForPortToBeInUse(int port) throws TimeoutException { boolean inUse = false; - Socket s = null; int tries = 0; int maxTries = 60 * 1000 / 100; while (!inUse) { - try - { - s = new Socket("localhost", port); - } - catch (IOException e) - { + + try (Socket s = new Socket("localhost", port)) { + break; + } catch (IOException e) { inUse = false; } - finally - { - if(s != null) { - inUse = true; - try { - s.close(); - } catch (Exception e) { - } - break; - } - } + tries++; if (tries == maxTries) { throw new TimeoutException("Timed out waiting for mitmproxy to start"); diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyServer.java b/src/main/java/io/appium/mitmproxy/MitmproxyServer.java index 309af12..d0f97ce 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyServer.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyServer.java @@ -31,40 +31,39 @@ public void onClose(WebSocket conn, int code, String reason, boolean remote) { @Override public void onMessage(WebSocket conn, String message) { - System.out.println("received message from " + conn.getRemoteSocketAddress() + ": " + message); + System.out.println("received message from " + conn.getRemoteSocketAddress() + ": " + message); } @Override - public void onMessage( WebSocket conn, ByteBuffer message ) { + public void onMessage(WebSocket conn, ByteBuffer rawInputMessage) { InterceptedMessage intercepted = null; - InterceptedMessage modifiedMessage = null; try { - intercepted = new InterceptedMessage(message); + intercepted = new InterceptedMessage(rawInputMessage); } catch (IOException e) { - System.out.println("Could not parse message"); + System.out.println("Could not parse rawInputMessage"); e.printStackTrace(); } - modifiedMessage = interceptor.apply(intercepted); + InterceptedMessage modifiedMessage = interceptor.apply(intercepted); // if the supplied interceptor function does not return a message, assume no changes were intended and just // complete the request if (modifiedMessage == null) { - modifiedMessage = intercepted; - } - - try { - conn.send(modifiedMessage.serializedResponseToMitmproxy()); - } catch (JsonProcessingException e) { - System.out.println("Could not encode response to mitmproxy"); - e.printStackTrace(); + conn.send(rawInputMessage); + } else { + try { + conn.send(modifiedMessage.serializedResponseToMitmproxy()); + } catch (JsonProcessingException e) { + System.out.println("Could not encode response to mitmproxy"); + e.printStackTrace(); + } } } @Override public void onError(WebSocket conn, Exception ex) { - System.err.println("an error occured on connection " + conn.getRemoteSocketAddress() + ":" + ex); + System.err.println("an error occured on connection " + conn.getRemoteSocketAddress() + ":" + ex); } @Override diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index f893cfc..ac3fecb 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -11,28 +11,32 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import static junit.framework.TestCase.assertTrue; public class MitmproxyJavaTest { +// private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; + private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump"; + + @Test - public void ConstructorTest() throws URISyntaxException, IOException, InterruptedException { - MitmproxyJava proxy = new MitmproxyJava("/usr/local/bin/mitmdump", (InterceptedMessage m) -> { + public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { + MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { System.out.println(m.requestURL.toString()); return m; }); + proxy.start(); System.out.println("advanced in test"); proxy.stop(); } @Test - public void SimpleTest() throws InterruptedException, ExecutionException, TimeoutException, IOException, URISyntaxException, UnirestException { - List messages = new ArrayList(); + public void SimpleTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + List messages = new ArrayList<>(); - MitmproxyJava proxy = new MitmproxyJava("/usr/local/bin/mitmdump", (InterceptedMessage m) -> { + MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { messages.add(m); return m; }); @@ -51,10 +55,10 @@ public void SimpleTest() throws InterruptedException, ExecutionException, Timeou } @Test - public void NullInterceptorReturnTest() throws InterruptedException, ExecutionException, TimeoutException, IOException, URISyntaxException, UnirestException { - List messages = new ArrayList(); + public void NullInterceptorReturnTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + List messages = new ArrayList<>(); - MitmproxyJava proxy = new MitmproxyJava("/usr/local/bin/mitmdump", (InterceptedMessage m) -> { + MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { messages.add(m); return null; }); @@ -73,10 +77,10 @@ public void NullInterceptorReturnTest() throws InterruptedException, ExecutionEx } @Test - public void ResponseModificationTest() throws InterruptedException, ExecutionException, TimeoutException, IOException, URISyntaxException, UnirestException { - List messages = new ArrayList(); + public void ResponseModificationTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + List messages = new ArrayList<>(); - MitmproxyJava proxy = new MitmproxyJava("/usr/local/bin/mitmdump", (InterceptedMessage m) -> { + MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { messages.add(m); m.responseBody = "Hi from Test".getBytes(StandardCharsets.UTF_8); return m; From ad45166c25fb6b38bb780321ff3176426f93c19b Mon Sep 17 00:00:00 2001 From: Adam M Date: Thu, 4 Jul 2019 22:54:39 +0200 Subject: [PATCH 02/23] Fixed response header reading bug --- src/main/java/io/appium/mitmproxy/InterceptedMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 527c7be..7fcc857 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -57,7 +57,7 @@ public InterceptedMessage(ByteBuffer buffer) throws IOException { } responseCode = metadata.get("response").get("status_code").asInt(); - headers = metadata.get("request").get("headers"); + headers = metadata.get("response").get("headers"); responseHeaders = new ArrayList<>(); for (JsonNode headerNode : headers) { String[] headerArray = new String[2]; From 5770662c18ebee101787e211fc21e71779b778cf Mon Sep 17 00:00:00 2001 From: Adam M Date: Fri, 5 Jul 2019 00:08:33 +0200 Subject: [PATCH 03/23] Reverting mitmdump path --- src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index ac3fecb..b867041 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -17,9 +17,7 @@ public class MitmproxyJavaTest { -// private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; - private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump"; - + private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; @Test public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { From 6fa890b39598f04385cafb72f409aa477dc34491 Mon Sep 17 00:00:00 2001 From: Adam M Date: Fri, 5 Jul 2019 00:12:58 +0200 Subject: [PATCH 04/23] Reading directly byte[] --- src/main/java/io/appium/mitmproxy/InterceptedMessage.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 7fcc857..0933ebb 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -89,8 +89,7 @@ public ByteBuffer serializedResponseToMitmproxy() throws JsonProcessingException headersNode.addAll(headerNodes); metadataRoot.set("headers", headersNode); - String metadataJson = objectMapper.writeValueAsString(metadataRoot); - byte[] metadata = metadataJson.getBytes(StandardCharsets.UTF_8); + byte[] metadata = objectMapper.writeValueAsBytes(metadataRoot); int metadataLength = metadata.length; ByteBuffer buffer = ByteBuffer.allocate(8 + metadataLength + contentLength); From 9675af0a1bacf33e8e91a534f7a1263613c84f3c Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 11:56:05 +0200 Subject: [PATCH 05/23] Removing .idea from tracking --- .idea/encodings.xml | 4 ---- .idea/gradle.xml | 18 ------------------ .idea/misc.xml | 7 ------- .idea/vcs.xml | 6 ------ 4 files changed, 35 deletions(-) delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/gradle.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 15a15b2..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/gradle.xml b/.idea/gradle.xml deleted file mode 100644 index 3163861..0000000 --- a/.idea/gradle.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index bc8d0a3..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 6b5d01616b9052bb444b7e4333e0dc5ee518dcac Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 12:14:43 +0200 Subject: [PATCH 06/23] Added lombok and assertj, serializing the message is easier, forcing mitmproxy to run on specific port, adding extra params --- build.gradle | 7 + .../appium/mitmproxy/InterceptedMessage.java | 161 ++++++++---------- .../appium/mitmproxy/MessageSerializer.java | 67 ++++++++ .../io/appium/mitmproxy/MitmproxyJava.java | 58 +++++-- .../io/appium/mitmproxy/MitmproxyServer.java | 46 ++--- .../appium/mitmproxy/MitmproxyJavaTest.java | 78 ++++++--- 6 files changed, 271 insertions(+), 146 deletions(-) create mode 100644 src/main/java/io/appium/mitmproxy/MessageSerializer.java diff --git a/build.gradle b/build.gradle index 605a4a0..e059ce4 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,14 @@ dependencies { implementation group: 'org.zeroturnaround', name: 'zt-process-killer', version: '1.9' implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.26' testCompile group: 'junit', name: 'junit', version: '4.12' + testCompile group: 'org.mockito', name: 'mockito-all', version: '1.10.19' testCompile group: 'com.mashape.unirest', name: 'unirest-java', version: '1.4.9' + testCompile group: 'org.assertj', name: 'assertj-core', version: '3.12.2' + + compileOnly 'org.projectlombok:lombok:1.18.8' + annotationProcessor 'org.projectlombok:lombok:1.18.8' + + compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.0' } task javadocJar(type: Jar) { diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 0933ebb..7b16b1b 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -1,104 +1,93 @@ package io.appium.mitmproxy; -import java.io.IOException; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; + import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +@Data public class InterceptedMessage { - public byte[] requestBody; - public byte[] responseBody; + private final static ObjectMapper objectMapper = new ObjectMapper(); + /** + * Use getRequest().getMethod() instead + */ + @JsonIgnore + @Deprecated public String requestMethod; + + /** + * Use getRequest().getUrl() instead + */ + @JsonIgnore + @Deprecated public URL requestURL; + + /** + * Use getRequest().getHeaders() instead + */ + @JsonIgnore + @Deprecated public List requestHeaders; + + /** + * Use getResponse().getStatusCode() instead + */ + @JsonIgnore + @Deprecated public int responseCode; + + /** + * Use getResponse().getHeaders() instead + */ + @JsonIgnore + @Deprecated public List responseHeaders; - private final static ObjectMapper objectMapper = new ObjectMapper(); + /** + * Use getRequest().getBody() instead + */ + @JsonIgnore + @Deprecated + private byte[] requestBody; - public InterceptedMessage(ByteBuffer buffer) throws IOException { - buffer.order(ByteOrder.LITTLE_ENDIAN); - - int metadataSize = buffer.getInt(); - int request_content_size = buffer.getInt(); - int response_content_size = buffer.getInt(); - - byte[] metadataBytes = new byte[metadataSize]; - buffer.get(metadataBytes); - - requestBody = new byte[request_content_size]; - buffer.get(requestBody); - - responseBody = new byte[response_content_size]; - buffer.get(responseBody); - - JsonNode metadata = objectMapper.readTree(metadataBytes); - requestMethod = metadata.get("request").get("method").asText(); - requestURL = new URL(metadata.get("request").get("url").asText()); - JsonNode headers = metadata.get("request").get("headers"); - requestHeaders = new ArrayList<>(); - for (JsonNode headerNode : headers) { - String[] headerArray = new String[2]; - headerArray[0] = headerNode.get(0).asText(); - headerArray[1] = headerNode.get(1).asText(); - requestHeaders.add(headerArray); - } - - responseCode = metadata.get("response").get("status_code").asInt(); - headers = metadata.get("response").get("headers"); - responseHeaders = new ArrayList<>(); - for (JsonNode headerNode : headers) { - String[] headerArray = new String[2]; - headerArray[0] = headerNode.get(0).asText(); - headerArray[1] = headerNode.get(1).asText(); - responseHeaders.add(headerArray); - } - } + /** + * Use getResponse().getBody() instead + */ + @JsonIgnore + @Deprecated + private byte[] responseBody; + + private Request request; + + private Response response; + + @Data + static class Request { + + private String method; + + private String url; + + private List headers; + + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private byte[] body; + } + + @Data + public static class Response { + + @JsonProperty("status_code") + private int statusCode; + + private List headers; - public ByteBuffer serializedResponseToMitmproxy() throws JsonProcessingException { - int contentLength = responseBody.length; - - // create JSON for metadata. Which is the responseCode and responseHeaders. - // while we're at it, set the Content-Length header - ObjectNode metadataRoot = objectMapper.createObjectNode(); - metadataRoot.put("status_code", responseCode); - - ArrayNode headersNode = objectMapper.createArrayNode(); - List headerNodes = responseHeaders.stream().map((h) -> { - ArrayNode headerPair = objectMapper.createArrayNode(); - headerPair.add(h[0]); - if (h[0].equals("content-length")) { - headerPair.add(Integer.toString(contentLength)); - } else { - headerPair.add(h[1]); - } - return headerPair; - }).collect(Collectors.toList()); - headersNode.addAll(headerNodes); - metadataRoot.set("headers", headersNode); - - byte[] metadata = objectMapper.writeValueAsBytes(metadataRoot); - int metadataLength = metadata.length; - - ByteBuffer buffer = ByteBuffer.allocate(8 + metadataLength + contentLength); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putInt(metadataLength); - buffer.putInt(contentLength); - buffer.put(metadata); - buffer.put(responseBody); - - return (ByteBuffer) buffer.rewind(); + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private byte[] body; } } \ No newline at end of file diff --git a/src/main/java/io/appium/mitmproxy/MessageSerializer.java b/src/main/java/io/appium/mitmproxy/MessageSerializer.java new file mode 100644 index 0000000..ed21768 --- /dev/null +++ b/src/main/java/io/appium/mitmproxy/MessageSerializer.java @@ -0,0 +1,67 @@ +package io.appium.mitmproxy; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class MessageSerializer { + + private final static ObjectMapper objectMapper = new ObjectMapper(); + + @SneakyThrows(IOException.class) + public InterceptedMessage deserializeMessage(ByteBuffer buffer) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + + int metadataSize = buffer.getInt(); + int request_content_size = buffer.getInt(); + int response_content_size = buffer.getInt(); + + byte[] metadataBytes = new byte[metadataSize]; + buffer.get(metadataBytes); + + + byte[] requestBody = new byte[request_content_size]; + buffer.get(requestBody); + + byte[] responseBody = new byte[response_content_size]; + buffer.get(responseBody); + + InterceptedMessage interceptedMessage = objectMapper.readValue(metadataBytes, InterceptedMessage.class); + interceptedMessage.getRequest().setBody(requestBody); + interceptedMessage.getResponse().setBody(responseBody); + + //in order not to break the old API + interceptedMessage.setRequestBody(requestBody); + interceptedMessage.setResponseBody(responseBody); + interceptedMessage.setRequestHeaders(interceptedMessage.getRequest().getHeaders()); + interceptedMessage.setResponseHeaders(interceptedMessage.getResponse().getHeaders()); + interceptedMessage.setResponseCode(interceptedMessage.getResponse().getStatusCode()); + interceptedMessage.setRequestURL(new URL(interceptedMessage.getRequest().getUrl())); + interceptedMessage.setRequestMethod(interceptedMessage.getRequest().getMethod()); + + return interceptedMessage; + } + + public ByteBuffer serializeMessage(InterceptedMessage message) throws JsonProcessingException { + byte[] responseBody = message.getResponse().getBody(); + + int contentLength = responseBody.length; + + byte[] metadata = objectMapper.writeValueAsBytes(message.getResponse()); + int metadataLength = metadata.length; + + ByteBuffer buffer = ByteBuffer.allocate(8 + metadataLength + contentLength); + buffer.order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(metadataLength); + buffer.putInt(contentLength); + buffer.put(metadata); + buffer.put(message.getResponse().getBody()); + + return (ByteBuffer) buffer.rewind(); + } +} diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java index b940a79..2d60aec 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java @@ -1,5 +1,6 @@ package io.appium.mitmproxy; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.zeroturnaround.exec.ProcessExecutor; import org.zeroturnaround.exec.ProcessResult; @@ -10,50 +11,75 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; -import java.net.ServerSocket; import java.net.Socket; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.function.Function; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; + +@Slf4j public class MitmproxyJava { + private static final String LOCALHOST_IP = "127.0.0.1"; + private static final int WEBSOCKET_PORT = 8765; + private String mitmproxyPath; + private Function messageInterceptor; + private int proxyPort; + private MitmproxyServer server; + + private List extraMitmdumpParams; + private Future mitmproxyProcess; - public static final int WEBSOCKET_PORT = 8765; - public MitmproxyJava(String mitmproxyPath, Function messageInterceptor, int proxyPort) { + public MitmproxyJava(String mitmproxyPath, Function messageInterceptor, int proxyPort, List extraMitmdumpParams) { this.mitmproxyPath = mitmproxyPath; this.messageInterceptor = messageInterceptor; this.proxyPort = proxyPort; + this.extraMitmdumpParams = extraMitmdumpParams; } public MitmproxyJava(String mitmproxyPath, Function messageInterceptor) { - this(mitmproxyPath, messageInterceptor, 8080); + this(mitmproxyPath, messageInterceptor, 8080, null); } public void start() throws IOException, TimeoutException { - System.out.println("starting mitmproxy on port " + proxyPort); + log.info("Starting mitmproxy on port {}", proxyPort); - server = new MitmproxyServer(new InetSocketAddress("localhost", WEBSOCKET_PORT), messageInterceptor); + server = new MitmproxyServer(new InetSocketAddress(LOCALHOST_IP, WEBSOCKET_PORT), messageInterceptor); server.start(); // python script file is zipped inside our jar. extract it into a temporary file. String pythonScriptPath = extractPythonScriptToFile(); + final List mitmproxyStartParams = new ArrayList<>(); + mitmproxyStartParams.add(mitmproxyPath); + mitmproxyStartParams.add("--anticache"); + mitmproxyStartParams.add("-p"); + mitmproxyStartParams.add(String.valueOf(proxyPort)); + mitmproxyStartParams.add("-s"); + mitmproxyStartParams.add(pythonScriptPath); + + // adding params if needed for mitmproxy + if (isNotEmpty(this.extraMitmdumpParams)) { + mitmproxyStartParams.addAll(this.extraMitmdumpParams); + } + mitmproxyProcess = new ProcessExecutor() - .command(mitmproxyPath, "--anticache", "-s", pythonScriptPath) + .command(mitmproxyStartParams) .redirectOutput(Slf4jStream.ofCaller().asInfo()) .destroyOnExit() .start() .getFuture(); waitForPortToBeInUse(proxyPort); - System.out.println("mitmproxy started on port " + proxyPort); - + log.info("Mitmproxy started on port {}", proxyPort); } private String extractPythonScriptToFile() throws IOException { @@ -74,7 +100,17 @@ public void stop() throws InterruptedException { mitmproxyProcess.cancel(true); } server.stop(1000); - Thread.sleep(200); // this pains me. but it seems that it takes a moment for the server to actually relinquish the port it uses. + waitForPortToBeFree(proxyPort); + } + + private void waitForPortToBeFree(int port) { + + while (true) { + try (Socket s = new Socket(LOCALHOST_IP, port)) { + } catch (IOException e) { + return; + } + } } private void waitForPortToBeInUse(int port) throws TimeoutException { @@ -84,7 +120,7 @@ private void waitForPortToBeInUse(int port) throws TimeoutException { while (!inUse) { - try (Socket s = new Socket("localhost", port)) { + try (Socket s = new Socket(LOCALHOST_IP, port)) { break; } catch (IOException e) { inUse = false; diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyServer.java b/src/main/java/io/appium/mitmproxy/MitmproxyServer.java index d0f97ce..0d13004 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyServer.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyServer.java @@ -4,70 +4,70 @@ import org.java_websocket.WebSocket; import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.server.WebSocketServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.function.Function; public class MitmproxyServer extends WebSocketServer { + private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketServer.class); + private Function interceptor; + private MessageSerializer messageSerializer; + public MitmproxyServer(InetSocketAddress address, Function interceptor) { super(address); this.interceptor = interceptor; + this.messageSerializer = new MessageSerializer(); } @Override public void onOpen(WebSocket conn, ClientHandshake handshake) { - System.out.println("new connection to websocket server" + conn.getRemoteSocketAddress()); + LOGGER.debug("new connection to websocket server" + conn.getRemoteSocketAddress()); } @Override public void onClose(WebSocket conn, int code, String reason, boolean remote) { - System.out.println("closed " + conn.getRemoteSocketAddress() + " with exit code " + code + " additional info: " + reason); + LOGGER.debug("closed " + conn.getRemoteSocketAddress() + " with exit code " + code + " additional info: " + reason); } @Override public void onMessage(WebSocket conn, String message) { - System.out.println("received message from " + conn.getRemoteSocketAddress() + ": " + message); + LOGGER.debug("received message from " + conn.getRemoteSocketAddress() + " : " + message); } @Override public void onMessage(WebSocket conn, ByteBuffer rawInputMessage) { - InterceptedMessage intercepted = null; - - try { - intercepted = new InterceptedMessage(rawInputMessage); - } catch (IOException e) { - System.out.println("Could not parse rawInputMessage"); - e.printStackTrace(); - } + InterceptedMessage incomingMessage = this.messageSerializer.deserializeMessage(rawInputMessage); - InterceptedMessage modifiedMessage = interceptor.apply(intercepted); + InterceptedMessage modifiedMessage = interceptor.apply(incomingMessage); // if the supplied interceptor function does not return a message, assume no changes were intended and just // complete the request + InterceptedMessage messageToSendBack = modifiedMessage; + if (modifiedMessage == null) { - conn.send(rawInputMessage); - } else { - try { - conn.send(modifiedMessage.serializedResponseToMitmproxy()); - } catch (JsonProcessingException e) { - System.out.println("Could not encode response to mitmproxy"); - e.printStackTrace(); - } + messageToSendBack = incomingMessage; + } + + try { + conn.send(this.messageSerializer.serializeMessage(messageToSendBack)); + } catch (JsonProcessingException e) { + LOGGER.error(e.getMessage()); } } @Override public void onError(WebSocket conn, Exception ex) { - System.err.println("an error occured on connection " + conn.getRemoteSocketAddress() + ":" + ex); + LOGGER.error("an error occured on connection " + conn.getRemoteSocketAddress() + ":" + ex); } @Override public void onStart() { - System.out.println("websocket server started successfully"); + LOGGER.info("websocket server started successfully"); } } diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index b867041..0af848b 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -7,22 +7,23 @@ import org.junit.Test; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeoutException; -import static junit.framework.TestCase.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; public class MitmproxyJavaTest { - private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; + private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump.exe"; @Test public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { - System.out.println(m.requestURL.toString()); + System.out.println(m.getRequest().getUrl()); return m; }); proxy.start(); @@ -31,7 +32,7 @@ public void ConstructorTest() throws InterruptedException, IOException, TimeoutE } @Test - public void SimpleTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + public void SimpleTest() throws InterruptedException, TimeoutException, IOException, UnirestException { List messages = new ArrayList<>(); MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { @@ -41,61 +42,86 @@ public void SimpleTest() throws InterruptedException, TimeoutException, IOExcept proxy.start(); Unirest.setProxy(new HttpHost("localhost", 8080)); - Unirest.get("http://appium.io").asString(); + Unirest.get("http://appium.io").header("myTestHeader", "myTestValue").asString(); proxy.stop(); + final InterceptedMessage firstMessage = messages.get(0); - assertTrue(messages.size() > 0); - - InterceptedMessage appiumIORequest = messages.stream().filter((m) -> m.requestURL.getHost().equals("appium.io")).findFirst().get(); - - assertTrue(appiumIORequest.responseCode == 200); + assertThat(firstMessage.getRequest().getUrl()).startsWith("http://appium.io"); + assertThat(firstMessage.getRequest().getHeaders()).containsOnlyOnce(new String[]{"myTestHeader", "myTestValue"}); + assertThat(firstMessage.getResponse().getStatusCode()).isEqualTo(200); } @Test - public void NullInterceptorReturnTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + public void NullInterceptorReturnTest() throws InterruptedException, TimeoutException, IOException, UnirestException { List messages = new ArrayList<>(); MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { messages.add(m); return null; - }); + }, 8087, null); proxy.start(); - Unirest.setProxy(new HttpHost("localhost", 8080)); - Unirest.get("http://appium.io").asString(); + Unirest.setProxy(new HttpHost("localhost", 8087)); + Unirest.get("http://appium.io").header("myTestHeader", "myTestValue").asString(); proxy.stop(); - assertTrue(messages.size() > 0); + assertThat(messages).isNotEmpty(); - InterceptedMessage appiumIORequest = messages.stream().filter((m) -> m.requestURL.getHost().equals("appium.io")).findFirst().get(); + final InterceptedMessage firstMessage = messages.get(0); - assertTrue(appiumIORequest.responseCode == 200); + assertThat(firstMessage.getRequest().getUrl()).startsWith("http://appium.io"); + assertThat(firstMessage.getRequest().getHeaders()).containsOnlyOnce(new String[]{"myTestHeader", "myTestValue"}); + assertThat(firstMessage.getResponse().getStatusCode()).isEqualTo(200); } @Test - public void ResponseModificationTest() throws InterruptedException, TimeoutException, IOException, URISyntaxException, UnirestException { + public void ResponseModificationTest() throws InterruptedException, TimeoutException, IOException, UnirestException { List messages = new ArrayList<>(); MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { messages.add(m); - m.responseBody = "Hi from Test".getBytes(StandardCharsets.UTF_8); + m.getResponse().setBody("Hi from Test".getBytes(StandardCharsets.UTF_8)); + m.getResponse().getHeaders().add(new String[]{"myTestResponseHeader", "myTestResponseHeaderValue"}); + m.getResponse().setStatusCode(208); return m; }); proxy.start(); Unirest.setProxy(new HttpHost("localhost", 8080)); - HttpResponse response = Unirest.get("http://appium.io").asString(); + HttpResponse response = Unirest.get("http://appium.io").header("myTestHeader", "myTestValue").asString(); + proxy.stop(); - assertTrue(response.getBody().equals("Hi from Test")); + assertThat(response.getBody()).isEqualTo("Hi from Test"); - proxy.stop(); + final InterceptedMessage firstMessage = messages.get(0); + + assertThat(firstMessage.getRequest().getUrl()).startsWith("http://appium.io"); + assertThat(firstMessage.getRequest().getHeaders()).containsOnlyOnce(new String[]{"myTestHeader", "myTestValue"}); + assertThat(firstMessage.getResponse().getHeaders()).containsOnlyOnce(new String[]{"myTestResponseHeader", "myTestResponseHeaderValue"}); + assertThat(firstMessage.getResponse().getStatusCode()).isEqualTo(208); + } - assertTrue(messages.size() > 0); + @Test + public void shouldAddParametersToMitmdumpStart() throws IOException, TimeoutException, InterruptedException { + List mitmdumpParams = new ArrayList<>(); + mitmdumpParams.add("testParam"); + + List spiedParams = spy(mitmdumpParams); + + MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { + m.getResponse().setBody("Hi from Test".getBytes(StandardCharsets.UTF_8)); + m.getResponse().getHeaders().add(new String[]{"myTestResponseHeader", "myTestResponseHeaderValue"}); + m.getResponse().setStatusCode(208); + return m; + }, 8087, spiedParams); + + proxy.start(); + proxy.stop(); - InterceptedMessage appiumIORequest = messages.stream().filter((m) -> m.requestURL.getHost().equals("appium.io")).findFirst().get(); + //to verify that additional params were actually included to start path + verify(spiedParams).toArray(); - assertTrue(appiumIORequest.responseCode == 200); } } \ No newline at end of file From d8e6c2d0f1fb6bec23b6b9c022f72b649f06caf6 Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 12:22:09 +0200 Subject: [PATCH 07/23] Added finals, extended README.md --- README.md | 10 ++++++++-- src/main/java/io/appium/mitmproxy/MitmproxyJava.java | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9b94467..8735b0a 100644 --- a/README.md +++ b/README.md @@ -49,12 +49,18 @@ testCompile group: 'io.appium', name: 'mitmproxy-java', version: '1.6.1' ```java List messages = new ArrayList(); +//optional, default port is 8080 +int mitmproxyPort = 8090; + +//optional, you can pass null if no extra params +List extraMitmproxyParams = Arrays.asList("param1", "value1", "param2", "value2"); + // remember to set local OS proxy settings in the Network Preferences proxy = new MitmproxyJava("/usr/local/bin/mitmdump", (InterceptedMessage m) -> { - System.out.println("intercepted request for " + m.requestURL.toString()); + System.out.println("intercepted request for " + m.getRequest().getUrl()); messages.add(m); return m; -}); +}, mitmproxyPort, extraMitmproxyParams); proxy.start(); diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java index 2d60aec..bb1f963 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java @@ -26,15 +26,15 @@ public class MitmproxyJava { private static final String LOCALHOST_IP = "127.0.0.1"; private static final int WEBSOCKET_PORT = 8765; - private String mitmproxyPath; + private final String mitmproxyPath; - private Function messageInterceptor; + private final Function messageInterceptor; - private int proxyPort; + private final int proxyPort; private MitmproxyServer server; - private List extraMitmdumpParams; + private final List extraMitmdumpParams; private Future mitmproxyProcess; From 42b9873cc69f09d6dc46d2b29782fb0e8fff014e Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 16:33:55 +0200 Subject: [PATCH 08/23] Added proper deprecation description --- .../appium/mitmproxy/InterceptedMessage.java | 14 ++--- .../appium/mitmproxy/MessageSerializer.java | 4 +- .../io/appium/mitmproxy/MitmproxyJava.java | 51 +++++++++++-------- .../io/appium/mitmproxy/MitmproxyServer.java | 6 +-- 4 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 7b16b1b..8108aa6 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -14,49 +14,49 @@ public class InterceptedMessage { private final static ObjectMapper objectMapper = new ObjectMapper(); /** - * Use getRequest().getMethod() instead + * @deprecated use {@link #getResponse().getMethod()} instead */ @JsonIgnore @Deprecated public String requestMethod; /** - * Use getRequest().getUrl() instead + * @deprecated use {@link #getRequest().getUrl() } instead */ @JsonIgnore @Deprecated public URL requestURL; /** - * Use getRequest().getHeaders() instead + * @deprecated use {@link #getRequest().getHeaders() } instead */ @JsonIgnore @Deprecated public List requestHeaders; /** - * Use getResponse().getStatusCode() instead + * @deprecated use {@link #getResponse().getStatusCode() } instead */ @JsonIgnore @Deprecated public int responseCode; /** - * Use getResponse().getHeaders() instead + * @deprecated use {@link #getResponse().getHeaders() } instead */ @JsonIgnore @Deprecated public List responseHeaders; /** - * Use getRequest().getBody() instead + * @deprecated use {@link #getRequest().getBody() } instead */ @JsonIgnore @Deprecated private byte[] requestBody; /** - * Use getResponse().getBody() instead + * @deprecated use {@link #getResponse().getBody() } instead */ @JsonIgnore @Deprecated diff --git a/src/main/java/io/appium/mitmproxy/MessageSerializer.java b/src/main/java/io/appium/mitmproxy/MessageSerializer.java index ed21768..89025b7 100644 --- a/src/main/java/io/appium/mitmproxy/MessageSerializer.java +++ b/src/main/java/io/appium/mitmproxy/MessageSerializer.java @@ -12,6 +12,7 @@ public class MessageSerializer { private final static ObjectMapper objectMapper = new ObjectMapper(); + private static final int START_BYTES = 8; @SneakyThrows(IOException.class) public InterceptedMessage deserializeMessage(ByteBuffer buffer) { @@ -24,7 +25,6 @@ public InterceptedMessage deserializeMessage(ByteBuffer buffer) { byte[] metadataBytes = new byte[metadataSize]; buffer.get(metadataBytes); - byte[] requestBody = new byte[request_content_size]; buffer.get(requestBody); @@ -55,7 +55,7 @@ public ByteBuffer serializeMessage(InterceptedMessage message) throws JsonProces byte[] metadata = objectMapper.writeValueAsBytes(message.getResponse()); int metadataLength = metadata.length; - ByteBuffer buffer = ByteBuffer.allocate(8 + metadataLength + contentLength); + ByteBuffer buffer = ByteBuffer.allocate(START_BYTES + metadataLength + contentLength); buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putInt(metadataLength); buffer.putInt(contentLength); diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java index bb1f963..267e41e 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java @@ -1,5 +1,6 @@ package io.appium.mitmproxy; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.zeroturnaround.exec.ProcessExecutor; @@ -12,6 +13,7 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.net.Socket; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; @@ -25,6 +27,7 @@ public class MitmproxyJava { private static final String LOCALHOST_IP = "127.0.0.1"; private static final int WEBSOCKET_PORT = 8765; + private static final int TIMEOUT_FOR_SOCKET_CHECKING_MINS = 5; private final String mitmproxyPath; @@ -49,7 +52,7 @@ public MitmproxyJava(String mitmproxyPath, Function interceptor; + private final Function interceptor; - private MessageSerializer messageSerializer; + private final MessageSerializer messageSerializer; - public MitmproxyServer(InetSocketAddress address, Function interceptor) { + MitmproxyServer(InetSocketAddress address, Function interceptor) { super(address); this.interceptor = interceptor; this.messageSerializer = new MessageSerializer(); From 77d080deb3bdffc714cff9597d21cf0499f67d82 Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 17:57:33 +0200 Subject: [PATCH 09/23] Fixed proper javadoc generation --- .../io/appium/mitmproxy/InterceptedMessage.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 8108aa6..6853d3e 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -14,49 +14,49 @@ public class InterceptedMessage { private final static ObjectMapper objectMapper = new ObjectMapper(); /** - * @deprecated use {@link #getResponse().getMethod()} instead + * @deprecated use {@link #getRequest()}.{@link Request#getMethod()} instead */ @JsonIgnore @Deprecated public String requestMethod; /** - * @deprecated use {@link #getRequest().getUrl() } instead + * @deprecated use {@link #getRequest()}.{@link Request#getUrl()} instead */ @JsonIgnore @Deprecated public URL requestURL; /** - * @deprecated use {@link #getRequest().getHeaders() } instead + * @deprecated use {@link #getRequest()}.{@link Request#getHeaders()} instead */ @JsonIgnore @Deprecated public List requestHeaders; /** - * @deprecated use {@link #getResponse().getStatusCode() } instead + * @deprecated use {@link #getResponse()}.{@link Response#getStatusCode()} instead */ @JsonIgnore @Deprecated public int responseCode; /** - * @deprecated use {@link #getResponse().getHeaders() } instead + * @deprecated use {@link #getResponse()}.{@link Response#getHeaders()} instead */ @JsonIgnore @Deprecated public List responseHeaders; /** - * @deprecated use {@link #getRequest().getBody() } instead + * @deprecated use {@link #getRequest()}.{@link Request#getBody()} instead */ @JsonIgnore @Deprecated private byte[] requestBody; /** - * @deprecated use {@link #getResponse().getBody() } instead + * @deprecated use {@link #getResponse()}.{@link Response#getBody()} instead */ @JsonIgnore @Deprecated @@ -77,7 +77,7 @@ static class Request { @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) private byte[] body; - } + } @Data public static class Response { From eeeff1bb14a3edbf40d1ee8929009ba2bee7e080 Mon Sep 17 00:00:00 2001 From: Adam M Date: Sat, 6 Jul 2019 18:00:31 +0200 Subject: [PATCH 10/23] Fixing wrong timeout calculation --- src/main/java/io/appium/mitmproxy/MitmproxyJava.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java index 267e41e..2da256a 100644 --- a/src/main/java/io/appium/mitmproxy/MitmproxyJava.java +++ b/src/main/java/io/appium/mitmproxy/MitmproxyJava.java @@ -112,7 +112,7 @@ private void waitForPortToBeFree(int port) { final LocalDateTime timeoutTime = LocalDateTime.now().plusMinutes(TIMEOUT_FOR_SOCKET_CHECKING_MINS); while (true) { - if(timeoutTime.isAfter(LocalDateTime.now())){ + if(timeoutTime.isBefore(LocalDateTime.now())){ throw new TimeoutException("Timed out waiting for mitmproxy to stop"); } try (Socket s = new Socket(LOCALHOST_IP, port)) { @@ -134,7 +134,7 @@ private void waitForPortToBeInUse(int port) { while (true) { - if(timeoutTime.isAfter(LocalDateTime.now())){ + if(timeoutTime.isBefore(LocalDateTime.now())){ throw new TimeoutException("Timed out waiting for mitmproxy to start"); } From d5cb7d6c39b9042d2540bc5c91ace120875fa8a5 Mon Sep 17 00:00:00 2001 From: Jonah Stiennon Date: Mon, 8 Jul 2019 11:12:52 -0700 Subject: [PATCH 11/23] remove deprecated properties on InterceptedMessage in favor of the lombok methods. --- .../appium/mitmproxy/InterceptedMessage.java | 49 ------------------- .../appium/mitmproxy/MessageSerializer.java | 9 ---- .../appium/mitmproxy/MitmproxyJavaTest.java | 3 +- 3 files changed, 2 insertions(+), 59 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 6853d3e..5d071c7 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -13,55 +13,6 @@ public class InterceptedMessage { private final static ObjectMapper objectMapper = new ObjectMapper(); - /** - * @deprecated use {@link #getRequest()}.{@link Request#getMethod()} instead - */ - @JsonIgnore - @Deprecated - public String requestMethod; - - /** - * @deprecated use {@link #getRequest()}.{@link Request#getUrl()} instead - */ - @JsonIgnore - @Deprecated - public URL requestURL; - - /** - * @deprecated use {@link #getRequest()}.{@link Request#getHeaders()} instead - */ - @JsonIgnore - @Deprecated - public List requestHeaders; - - /** - * @deprecated use {@link #getResponse()}.{@link Response#getStatusCode()} instead - */ - @JsonIgnore - @Deprecated - public int responseCode; - - /** - * @deprecated use {@link #getResponse()}.{@link Response#getHeaders()} instead - */ - @JsonIgnore - @Deprecated - public List responseHeaders; - - /** - * @deprecated use {@link #getRequest()}.{@link Request#getBody()} instead - */ - @JsonIgnore - @Deprecated - private byte[] requestBody; - - /** - * @deprecated use {@link #getResponse()}.{@link Response#getBody()} instead - */ - @JsonIgnore - @Deprecated - private byte[] responseBody; - private Request request; private Response response; diff --git a/src/main/java/io/appium/mitmproxy/MessageSerializer.java b/src/main/java/io/appium/mitmproxy/MessageSerializer.java index 89025b7..2429c65 100644 --- a/src/main/java/io/appium/mitmproxy/MessageSerializer.java +++ b/src/main/java/io/appium/mitmproxy/MessageSerializer.java @@ -35,15 +35,6 @@ public InterceptedMessage deserializeMessage(ByteBuffer buffer) { interceptedMessage.getRequest().setBody(requestBody); interceptedMessage.getResponse().setBody(responseBody); - //in order not to break the old API - interceptedMessage.setRequestBody(requestBody); - interceptedMessage.setResponseBody(responseBody); - interceptedMessage.setRequestHeaders(interceptedMessage.getRequest().getHeaders()); - interceptedMessage.setResponseHeaders(interceptedMessage.getResponse().getHeaders()); - interceptedMessage.setResponseCode(interceptedMessage.getResponse().getStatusCode()); - interceptedMessage.setRequestURL(new URL(interceptedMessage.getRequest().getUrl())); - interceptedMessage.setRequestMethod(interceptedMessage.getRequest().getMethod()); - return interceptedMessage; } diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index 0af848b..c49907b 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -18,7 +18,8 @@ public class MitmproxyJavaTest { - private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump.exe"; + //private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump.exe"; + private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; @Test public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { From 9989e8ba548cbe28984e3480fd16cd52d5617477 Mon Sep 17 00:00:00 2001 From: Jonah Stiennon Date: Mon, 8 Jul 2019 11:13:19 -0700 Subject: [PATCH 12/23] version 2.0.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e059ce4..f67b436 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group = 'io.appium' -version = '1.6.2' +version = '2.0.0' archivesBaseName = 'mitmproxy-java' sourceCompatibility = 1.8 From a10ef7b8681adc11580505d301c4a3d6df7d3fc0 Mon Sep 17 00:00:00 2001 From: Jonah Stiennon Date: Mon, 8 Jul 2019 11:24:53 -0700 Subject: [PATCH 13/23] update readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 8735b0a..2b3f069 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,6 @@ I'm no Java expert! You may see some bad patterns, like my terrible disregard fo See [node-mitmproxy](https://github.com/jvilk/mitmproxy-node/blob/master/README.md#your-python-plugin-is-bad-and-you-should-feel-bad). Pull requests welcome! +## Develop + +Upload to Central Repository with command `./gradlew uploadArchives` \ No newline at end of file From 82396029342d0621fc20465554281db66ece7003 Mon Sep 17 00:00:00 2001 From: Adam M Date: Tue, 16 Jul 2019 20:21:33 +0200 Subject: [PATCH 14/23] Changing nested class scope to public --- src/main/java/io/appium/mitmproxy/InterceptedMessage.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java index 5d071c7..18f347f 100644 --- a/src/main/java/io/appium/mitmproxy/InterceptedMessage.java +++ b/src/main/java/io/appium/mitmproxy/InterceptedMessage.java @@ -1,11 +1,9 @@ package io.appium.mitmproxy; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; -import java.net.URL; import java.util.List; @Data @@ -18,7 +16,7 @@ public class InterceptedMessage { private Response response; @Data - static class Request { + public static class Request { private String method; From 8699e9c70af8458aefdf60159ffbfc89a9dcf2fb Mon Sep 17 00:00:00 2001 From: Jonah Stiennon Date: Wed, 17 Jul 2019 10:39:35 -0700 Subject: [PATCH 15/23] v2.0.1 --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index f67b436..b5d501c 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group = 'io.appium' -version = '2.0.0' +version = '2.0.1' archivesBaseName = 'mitmproxy-java' sourceCompatibility = 1.8 @@ -30,7 +30,7 @@ dependencies { compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' - + compile group: 'org.apache.commons', name: 'commons-collections4', version: '4.0' } @@ -98,4 +98,4 @@ uploadArchives { } } } -} \ No newline at end of file +} From e1c164aa660847f2fa803821e7aad878858f0fff Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Thu, 9 Mar 2023 16:57:17 +0530 Subject: [PATCH 16/23] Fixed issue while running MITM Proxy client on Mac --- build.gradle | 9 ++++++--- src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java | 9 ++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index b5d501c..9114bcd 100644 --- a/build.gradle +++ b/build.gradle @@ -46,7 +46,10 @@ artifacts { archives javadocJar, sourcesJar } signing { - sign configurations.archives + setRequired { + // signing is only required if the artifacts are to be published + sign configurations.archives + } } @@ -60,10 +63,10 @@ uploadArchives { // Destination repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) + authentication(userName: hasProperty('ossrhUsername'), password: hasProperty('ossrhPassword')) } snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: ossrhUsername, password: ossrhPassword) + authentication(userName: hasProperty('ossrhUsername'), password: hasProperty('ossrhPassword')) } // Add required metadata to POM diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index c49907b..6db086f 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -19,10 +19,17 @@ public class MitmproxyJavaTest { //private static final String MITMDUMP_PATH = "C:\\Python37\\Scripts\\mitmdump.exe"; - private static final String MITMDUMP_PATH = "/usr/local/bin/mitmdump"; + private static final String MITMDUMP_PATH = getMITMDumpPath(); + static String getMITMDumpPath(){ + if(System.getProperty("os.name").contains("Mac")){ + return "/opt/homebrew/bin/mitmdump"; + } + return "/usr/local/bin/mitmdump"; + } @Test public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { + System.out.println(); MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { System.out.println(m.getRequest().getUrl()); return m; From 9694813b06922a25183939fd13a463b216dd069e Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Thu, 9 Mar 2023 17:00:35 +0530 Subject: [PATCH 17/23] Removed JCenter due to its shutdown https://blog.gradle.org/jcenter-shutdown --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9114bcd..6ca3e8c 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,6 @@ sourceCompatibility = 1.8 repositories { mavenCentral() - jcenter() } dependencies { From 7e8e5af8b0560bdb6623b87c3bc82d75cf121117 Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Thu, 9 Mar 2023 17:06:46 +0530 Subject: [PATCH 18/23] Updated version from 2.0.1 -> 2.0.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6ca3e8c..623443b 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group = 'io.appium' -version = '2.0.1' +version = '2.0.2' archivesBaseName = 'mitmproxy-java' sourceCompatibility = 1.8 From 890b0cb1c6e4fac2bf425a5c59f869022b4c6739 Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Thu, 9 Mar 2023 17:10:02 +0530 Subject: [PATCH 19/23] Updated version from 2.0.1 -> 2.0.2 --- src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java index 6db086f..85ea9c3 100644 --- a/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java +++ b/src/test/java/io/appium/mitmproxy/MitmproxyJavaTest.java @@ -29,7 +29,6 @@ static String getMITMDumpPath(){ } @Test public void ConstructorTest() throws InterruptedException, IOException, TimeoutException { - System.out.println(); MitmproxyJava proxy = new MitmproxyJava(MITMDUMP_PATH, (InterceptedMessage m) -> { System.out.println(m.getRequest().getUrl()); return m; From 30b2f65935f5bc912725d23e4c05ffcb983f16e7 Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Fri, 10 Mar 2023 11:43:16 +0530 Subject: [PATCH 20/23] The mitmproxy API seems to have changed and the syntax is now http.Response.make --- src/main/resources/scripts/proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/scripts/proxy.py b/src/main/resources/scripts/proxy.py index f108b1f..01ab562 100644 --- a/src/main/resources/scripts/proxy.py +++ b/src/main/resources/scripts/proxy.py @@ -150,7 +150,7 @@ def request(self, flow): new_metadata = message_response[0] new_body = message_response[1] - flow.response = http.HTTPResponse.make( + flow.response = http.Response.make( new_metadata['status_code'], new_body, map(convert_headers_to_bytes, new_metadata['headers']) @@ -198,7 +198,7 @@ def response(self, flow): #print("Prepping response!") - flow.response = http.HTTPResponse.make( + flow.response = http.Response.make( new_metadata['status_code'], new_body, map(convert_headers_to_bytes, new_metadata['headers']) From 3b5ec7e521c706012b0b6c4ce81e4cc46d650449 Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Mon, 13 Mar 2023 10:04:06 +0530 Subject: [PATCH 21/23] Updated README.md as the code now support the latest stable mitmproxy v9 --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2b3f069..740e1fd 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ For transparently rewriting HTTP/HTTPS responses. The mitmproxy plugin lets ever ## Pre-requisites -* [`mitmproxy` V4](https://mitmproxy.org/) must be installed and runnable from the terminal. The install method cannot be a prebuilt binary or homebrew, since those packages are missing the Python websockets module. Install via `pip` or from source. -* Python 3.6, since we use the new async/await syntax in the mitmproxy plugin +* [`mitmproxy` V9](https://mitmproxy.org/) must be installed and runnable from the terminal. The install method cannot be a prebuilt binary or homebrew, since those packages are missing the Python websockets module. Install via `pip` or from source. +* Python 3.6 and above, since we use the new async/await syntax in the mitmproxy plugin * `pip3 install websockets` Maven: @@ -35,13 +35,13 @@ Maven: io.appium mitmproxy-java - 1.6.2 + 2.0.2 ``` Gradle: ``` -testCompile group: 'io.appium', name: 'mitmproxy-java', version: '1.6.1' +testCompile group: 'io.appium', name: 'mitmproxy-java', version: '2.0.0' ``` ## Usage @@ -67,10 +67,16 @@ proxy.start(); // do stuff proxy.stop(); +``` +If the above code doesn't work, and you are getting +`Error=2, No such file or directory` please re-check your mitmdump path in new MitmproxyJava initialization. +You can get mitmdump path using below command: +```shell +whereis mitmdump ``` -See AppriumPro article for more guidelines: https://appiumpro.com/editions/65 +See AppiumPro article for more guidelines: https://appiumpro.com/editions/65 Example can be found here: https://github.com/cloudgrey-io/appiumpro/blob/master/java/src/test/java/Edition065_Capture_Network_Requests.java ## Your Java code is bad and you should feel bad From 23d70343b888aee216bdcb661d1e9281c4757178 Mon Sep 17 00:00:00 2001 From: himanshunikhare25 Date: Mon, 13 Mar 2023 10:27:19 +0530 Subject: [PATCH 22/23] Fixed typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 740e1fd..99ca370 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Maven: Gradle: ``` -testCompile group: 'io.appium', name: 'mitmproxy-java', version: '2.0.0' +testCompile group: 'io.appium', name: 'mitmproxy-java', version: '2.0.2' ``` ## Usage From 01aae080fd64e2f9e2e90f304c43dabaf6d17a4d Mon Sep 17 00:00:00 2001 From: Jonah Stiennon Date: Fri, 22 Sep 2023 15:51:29 -0700 Subject: [PATCH 23/23] simplify publishing --- README.md | 3 ++- build.gradle | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 99ca370..b9b1b3d 100644 --- a/README.md +++ b/README.md @@ -89,4 +89,5 @@ See [node-mitmproxy](https://github.com/jvilk/mitmproxy-node/blob/master/README. ## Develop -Upload to Central Repository with command `./gradlew uploadArchives` \ No newline at end of file +Upload to Central Repository with command `./gradlew uploadArchives` +Set username and password in `build.gradle` file \ No newline at end of file diff --git a/build.gradle b/build.gradle index 623443b..6b779fb 100644 --- a/build.gradle +++ b/build.gradle @@ -46,9 +46,10 @@ artifacts { } signing { setRequired { - // signing is only required if the artifacts are to be published - sign configurations.archives + // condition for when signing is required + gradle.taskGraph.hasTask("uploadArchives") } + sign configurations.archives } @@ -62,10 +63,10 @@ uploadArchives { // Destination repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: hasProperty('ossrhUsername'), password: hasProperty('ossrhPassword')) + authentication(userName: 'username', password: 'password') } snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: hasProperty('ossrhUsername'), password: hasProperty('ossrhPassword')) + authentication(userName: 'username', password: 'password') } // Add required metadata to POM