diff --git a/.gitignore b/.gitignore index b26fb4c..beb3b7a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ pom.xml.asc *.class /.lein-* /.nrepl-port -issues +dump diff --git a/README.md b/README.md new file mode 100644 index 0000000..81de2ba --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# GitHub Issues Dump + +Dumps raw JSON of GitHub issues retrieved from the GitHub API. + +## Usage + +```bash +lein run -- -t TOKEN +lein run -- -t TOKEN -o ORGANIZATION +``` + +You may need to go to [personal access tokens](https://github.com/settings/tokens) settings on GitHub to set up a token. + +## License + +The MIT License (MIT) + +Copyright (c) 2015 ClipCard + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..1459b0c --- /dev/null +++ b/project.clj @@ -0,0 +1,12 @@ +(defproject github-issues-dump "1.0.0" + :license {:name "MIT License" + :url "http://opensource.org/licenses/MIT" + :key "mit" + :year 2015 + :distribution :repo} + :main github-issues-dump.core + :dependencies [[org.clojure/clojure "1.6.0"] + [org.clojure/data.json "0.2.5"] + [org.clojure/tools.cli "0.2.4"] + [clj-time "0.4.1"] + [http-kit "2.1.18"]]) diff --git a/src/github_issues_dump/api.clj b/src/github_issues_dump/api.clj new file mode 100644 index 0000000..5727ec3 --- /dev/null +++ b/src/github_issues_dump/api.clj @@ -0,0 +1,152 @@ +(ns github-issues-dump.api + (:require [clojure.data.json :as json] + [clojure.string :as string] + [clj-time.core :as time] + [clj-time.coerce :as coerce] + [org.httpkit.client :as http])) + +(defn- error-message [response] + (get-in response [:parsed-body :message])) + +(defn- auth-failure + [response] + (when (= 401 (:status response)) + (throw (ex-info "Authorization Failure" {:response response})))) + +(defn bad-req [response] + (when (#{400 404} (:status response)) + (throw (ex-info "Bad Request" response)))) + +(defn rate-limit-ms + "GitHub gives us 'the time at which the current rate limit window + resets in UTC epoch seconds', as a String. We produce a number of + milliseconds from current time, as a number." + [x-ratelimit-reset] + (let [end-ms (* 1000 (Long. x-ratelimit-reset)) + now-ms (coerce/to-long (time/now))] + (max 0 (- end-ms now-ms)))) + +(defn- service-rate-limit + [{:keys [headers status] :as response}] + (when (and (= 403 status) + (= "0" (:x-ratelimit-remaining headers))) + (throw (ex-info "Rate Limited" + {:rate-limit-ms (rate-limit-ms (:x-ratelimit-reset headers)) + :message (error-message response)})))) + +(defn- service-unexpected-cond + [response] + (when (>= (:status response) 500) + (throw (ex-info "Unexpected condition" + {:response response :message (error-message response)})))) + +(defn- response-error + [response] + (when (>= (:status response) 400) + (or (auth-failure response) + (bad-req response) + (service-rate-limit response) + (service-unexpected-cond response)))) + +(defn json->map + "Convert JSON body to a map." + [{:keys [body] :as response}] + (when-not (string/blank? body) + (json/read-str body :key-fn keyword))) + +(defn http-get [url options] + (let [response @(http/get url options) + error (:error response)] + (if error + (throw error) + response))) + +(def base-url "https://api.github.com") + +(defn get-resource [token url-path & [query-params]] + (let [url (format (str base-url "%s") (string/replace url-path base-url "")) + headers {"Accept" "application/vnd.github.raw+json"} + options (merge {:oauth-token token + :headers headers} + (when query-params {:query-params query-params})) + response (http-get url options) + body (json->map response) + parsed-response (assoc response :parsed-body body)] + (or (response-error parsed-response) + parsed-response))) + +;; Pagination + +(defn parse-link [link] + (let [[_ url] (re-find #"<(.*)>" link) + [_ rel] (re-find #"rel=\W(.*)\W" link)] + [(keyword rel) url])) + +(defn parse-links + "Takes the content of the link header from a github resp, returns a map of links" + [link-body] + (->> (string/split link-body #",") + (map parse-link) + (into {}))) + +(defn next-page + [{:keys [link]}] + (when link + (let [link-map (parse-links link)] + (:next link-map)))) + +(defn page-number [url] + (let [[_ page-num] (re-find #"(?:&|\?)page=(\d+)" (or url ""))] + (if page-num + (Integer. page-num) + 1))) + +(defn all-results [body-fn token url & [query-params]] + (let [response (get-resource token url query-params) + {:keys [headers]} response + body (body-fn response) + page-num (-> response :opts :url page-number) + results [[page-num body]] + next-page-link (next-page headers)] + (if next-page-link + (lazy-cat results (all-results body-fn token next-page-link)) + results))) + +;; Resources + +(def base-query + {:per_page 100 + :sort "created" + :direction "asc"}) + +(defn issues [token & [organization]] + (let [url (if organization + (format "/orgs/%s/issues" organization) + "/issues") + query-params (assoc base-query :filter "all" :state "all")] + (all-results :body token url query-params))) + +(defn repositories [token & [organization]] + (let [url (if organization + (format "/orgs/%s/repos" organization) + "/user/repos") + query-params (assoc base-query :visibility "all")] + (mapcat second (all-results :parsed-body token url query-params)))) + +(defn issue-comments* [token repository] + (let [owner (get-in repository [:owner :login]) + repo-name (:name repository) + url (format "/repos/%s/%s/issues/comments" owner repo-name)] + [[owner repo-name] (all-results :body token url base-query)])) + +(defn issue-comments [token repos] + (map #(issue-comments* token %) repos)) + +(defn review-comments* [token repository] + (let [owner (get-in repository [:owner :login]) + repo-name (:name repository) + url (format "/repos/%s/%s/pulls/comments" owner repo-name)] + [[owner repo-name] (all-results :body token url base-query)])) + +(defn review-comments [token repos] + (map #(review-comments* token %) repos)) diff --git a/src/github_issues_dump/core.clj b/src/github_issues_dump/core.clj new file mode 100644 index 0000000..6357ca9 --- /dev/null +++ b/src/github_issues_dump/core.clj @@ -0,0 +1,47 @@ +(ns github-issues-dump.core + (:require [clojure.java.io :as io] + [clojure.tools.cli :refer [cli]] + [github-issues-dump.api :as api])) + +(defn spit* [filename body] + (println "Writing file: " filename) + (io/make-parents filename) + (spit filename body)) + +(defn dump-issues [token organization] + (let [issues (api/issues token organization)] + (doseq [[page body] issues + :let [filename (if organization + (format "dump/%s/issues/%s.json" organization page) + (format "dump/issues/%s.json" page))]] + (spit* filename body)))) + +(defn dump-comments* [filename-pattern comments] + (doseq [[[owner repo] pages] comments] + (doseq [[page body] pages + :let [filename (format filename-pattern owner repo page)]] + (spit* filename body)))) + +(defn dump-comments [token organization] + (let [repos (api/repositories token organization) + issue-comments (api/issue-comments token repos) + review-comments (api/review-comments token repos)] + (dump-comments* "dump/%s/%s/issues/comments/%s.json" issue-comments) + (dump-comments* "dump/%s/%s/pulls/comments/%s.json" review-comments))) + +(def cli-args + [["-h" "--help" "Show help" :flag true :default false] + ["-t" "--token" + "GitHub authorization token - create one at https://github.com/settings/tokens"] + ["-o" "--organization" "Optional organization limits the scope of issues captured"]]) + +(defn -main [& args] + (let [[options _ banner] (apply cli args cli-args) + {:keys [help organization token]} options] + + (when help + (println banner) + (System/exit 0)) + + (dump-issues token organization) + (dump-comments token organization)))