From 0c8d50963e6790c3dd7715da554c42e233b78545 Mon Sep 17 00:00:00 2001 From: Bob Pielock Date: Sun, 8 Sep 2013 10:50:38 +0000 Subject: [PATCH 1/3] added some awesome statistics tools for ford dev team to check out. --- openxc/graphing/README | 9 ++++ openxc/graphing/open-xc-graphite.py | 72 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 openxc/graphing/README create mode 100755 openxc/graphing/open-xc-graphite.py diff --git a/openxc/graphing/README b/openxc/graphing/README new file mode 100644 index 00000000..d01d02ca --- /dev/null +++ b/openxc/graphing/README @@ -0,0 +1,9 @@ +With this script you can dump all vehicle json into graphite for longer term logging of data, monitoring driving details, or just having some data +to look back on. + +Ben Lau Re-Worked this for OpenXC Hackathon Mashery Team - he was working on this for a few hours this weekend. + +Usage: tail the device, loop file system, or file that your trace or Vehicle Interface serial is connected to, and this will parse and feed +into graphite via carbon-relay. + +tail -f file | open-xc-graphite.py & diff --git a/openxc/graphing/open-xc-graphite.py b/openxc/graphing/open-xc-graphite.py new file mode 100755 index 00000000..5bb547d0 --- /dev/null +++ b/openxc/graphing/open-xc-graphite.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +# 2013 Benjamin Lau, Mashery +# Copyright: CC0 + +import sys +import fileinput +import json +import datetime +import time +from threading import Timer + + +CARBON_SERVER="0.0.0.0" +CARBON_PORT=2004 + +counts = {} +counter = 0 +duration = 60 #adjust this to make the polling period shorter for testing + +def process_input(line): + global counter + global counts + json_dump = json.loads(line) + if counts.has_key(json_dump['name']): + counts[json_dump['name']]+=1 + else: + counts[json_dump['name']] = 1 +# print counts + counter+=1 + +def send_to_graphite(): + global counts + global mytimer + ts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + stats = [] + for key in counts.keys(): + stats.append("openxc.vehicle.data.%s %d %d" % (key,counts[key],time.mktime(time.strptime(ts,'%Y-%m-%d %H:%M')))) + msg='\n'.join(stats) + if msg == "": + print "No data for this period" + else: + print msg + #comment out this block if you need to test and don't have a local instance of carbon/graphite + try: + sock=socket.socket() + sock.connect((CARBON_SERVER,CARBON_PORT)) + sock.sendall(msg) + except: + print "Error connecting to %s:%d"%(CARBON_SERVER,CARBON_PORT) + counts = {} + mytimer = Timer(duration, send_to_graphite) + mytimer.start() + process() + +mytimer = Timer(duration, send_to_graphite) + +def process(): + global mytimer + while 1: + line = sys.stdin.readline() + if not line: break +# sys.stdout.write("processing: %s" % line) + process_input(line) + +def hook_nobuf(filename, mode): + return open(filename, mode, 0) + +fi = fileinput.FileInput(openhook=hook_nobuf) +mytimer.start() +process() + From 9c5adb38966843e86c2f3e031a93f92a6aee44a6 Mon Sep 17 00:00:00 2001 From: Bob Pielock Date: Sun, 8 Sep 2013 10:57:34 +0000 Subject: [PATCH 2/3] changed name fron graphing to metrics --- openxc/{graphing => metrics}/README | 0 openxc/{graphing => metrics}/open-xc-graphite.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename openxc/{graphing => metrics}/README (100%) rename openxc/{graphing => metrics}/open-xc-graphite.py (100%) diff --git a/openxc/graphing/README b/openxc/metrics/README similarity index 100% rename from openxc/graphing/README rename to openxc/metrics/README diff --git a/openxc/graphing/open-xc-graphite.py b/openxc/metrics/open-xc-graphite.py similarity index 100% rename from openxc/graphing/open-xc-graphite.py rename to openxc/metrics/open-xc-graphite.py From 09731861c846a360371936852ece80a7df9b0ee1 Mon Sep 17 00:00:00 2001 From: Bob Pielock Date: Sun, 8 Sep 2013 19:39:33 +0000 Subject: [PATCH 3/3] removed counters to dump raw data will need additional logic later on --- openxc/metrics/open-xc-graphite-nocounters.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 openxc/metrics/open-xc-graphite-nocounters.py diff --git a/openxc/metrics/open-xc-graphite-nocounters.py b/openxc/metrics/open-xc-graphite-nocounters.py new file mode 100755 index 00000000..c19e23c9 --- /dev/null +++ b/openxc/metrics/open-xc-graphite-nocounters.py @@ -0,0 +1,76 @@ +#!/usr/bin/python + +# 2013 Benjamin Lau, Mashery +# Copyright: CC0 +# Minor Changes by Bob + +import sys +import fileinput +import json +import datetime +import time +import socket +from threading import Timer + + +CARBON_SERVER="127.0.0.1" +CARBON_PORT=2003 +timeout = 10 +socket.setdefaulttimeout(timeout) + +counts = {} +counter = 0 +duration = 1 #adjust this to make the polling period shorter for testing + +def process_input(line): + global counter + global counts + json_dump = json.loads(line) + if counts.has_key(json_dump['name']): + counts[json_dump['name']]+=1 + else: + counts[json_dump['name']] = 1 +# print counts + counter+=1 + +def send_to_graphite(): + global counts + global mytimer + ts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + stats = [] + for key in counts.keys(): + stats.append("openxc.vehicle.data.%s %d %d" % (key,counts[key],time.mktime(time.strptime(counts[ts],'%Y-%m-%d %H:%M')))) + msg='\n'.join(stats) + if msg == "": + print "No data for this period" + else: + print msg + #comment out this block if you need to test and don't have a local instance of carbon/graphite + try: + sock=socket.socket() + sock.connect((CARBON_SERVER,CARBON_PORT)) + sock.sendall(msg) + except: + print "Error connecting to %s:%d"%(CARBON_SERVER,CARBON_PORT) + counts = {} + mytimer = Timer(duration, send_to_graphite) + mytimer.start() + process() + +mytimer = Timer(duration, send_to_graphite) + +def process(): + global mytimer + while 1: + line = sys.stdin.readline() + if not line: break + sys.stdout.write("processing: %s" % line) + process_input(line) + +def hook_nobuf(filename, mode): + return open(filename, mode, 0) + +fi = fileinput.FileInput(openhook=hook_nobuf) +mytimer.start() +process() +