From f83f18cc485a2723202e3fc32f21df7809497a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Mon, 28 Apr 2025 11:57:00 +0200 Subject: [PATCH 1/4] ci: add SourceHut builds.sr.ht configuration --- .build.yml | 23 +++++++++++++++++++++++ Makefile | 11 +++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 .build.yml diff --git a/.build.yml b/.build.yml new file mode 100644 index 000000000..69396e390 --- /dev/null +++ b/.build.yml @@ -0,0 +1,23 @@ +--- +image: debian/testing +packages: + - curl + - golang + - make + - gnupg +sources: + - "https://git.sr.ht/~mcepl/git-bug" +environment: + DESTDIR: ./out + GOFLAGS: "" + CC: gcc +tasks: + - validate: | + cd git-bug + test -z "$(gofmt -d .)" || exit 1 + - download-dependencies: | + cd git-bug + go mod download > /dev/null + - build: | + cd git-bug + make test diff --git a/Makefile b/Makefile index 4745d195d..ea9b950de 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,15 @@ releases: go generate go run github.com/mitchellh/gox@v1.0.1 -ldflags "$(LDFLAGS)" -osarch '!darwin/386' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -.PHONY: secure -secure: +secure: secure-practices secure-vulnerabilities + +.PHONY: secure-practices +secure-practices: + go run github.com/praetorian-inc/gokart scan + # eventually go run github.com/securego/gosec/v2/cmd/gosec@latest ./... + +.PHONY: secure-vulnerabilities +secure-vulnerabilities: go run golang.org/x/vuln/cmd/govulncheck ./... .PHONY: test From 942f524808f874cbebab0a11bced2ef64b786e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Tue, 29 Apr 2025 10:46:43 +0200 Subject: [PATCH 2/4] WIP feat: stub for the bridge for https://todo.sr.ht This is absolutely not working, it just builds. --- bridge/bridges.go | 2 + bridge/todosrht/client.go | 1461 +++++++++++++++++++++++ bridge/todosrht/config.go | 220 ++++ bridge/todosrht/export.go | 475 ++++++++ bridge/todosrht/import.go | 657 ++++++++++ bridge/todosrht/todosrht.go | 145 +++ doc/man/git-bug-bridge-auth-add-token.1 | 2 +- doc/man/git-bug-bridge-new.1 | 2 +- doc/md/git-bug_bridge_auth_add-token.md | 2 +- doc/md/git-bug_bridge_new.md | 2 +- 10 files changed, 2964 insertions(+), 4 deletions(-) create mode 100644 bridge/todosrht/client.go create mode 100644 bridge/todosrht/config.go create mode 100644 bridge/todosrht/export.go create mode 100644 bridge/todosrht/import.go create mode 100644 bridge/todosrht/todosrht.go diff --git a/bridge/bridges.go b/bridge/bridges.go index 5fe0c395d..5b826776b 100644 --- a/bridge/bridges.go +++ b/bridge/bridges.go @@ -7,6 +7,7 @@ import ( "github.com/git-bug/git-bug/bridge/gitlab" "github.com/git-bug/git-bug/bridge/jira" "github.com/git-bug/git-bug/bridge/launchpad" + "github.com/git-bug/git-bug/bridge/todosrht" "github.com/git-bug/git-bug/cache" "github.com/git-bug/git-bug/repository" ) @@ -15,6 +16,7 @@ func init() { core.Register(&github.Github{}) core.Register(&gitlab.Gitlab{}) core.Register(&launchpad.Launchpad{}) + core.Register(&todosrht.TodoSourceHut{}) core.Register(&jira.Jira{}) } diff --git a/bridge/todosrht/client.go b/bridge/todosrht/client.go new file mode 100644 index 000000000..7d70be553 --- /dev/null +++ b/bridge/todosrht/client.go @@ -0,0 +1,1461 @@ +package todosrht + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/git-bug/git-bug/entities/common" +) + +var errDone = errors.New("Iteration Done") +var errTransitionNotFound = errors.New("Transition not found") +var errTransitionNotAllowed = errors.New("Transition not allowed") + +// ============================================================================= +// Extended JSON +// ============================================================================= + +const TimeFormat = "2006-01-02T15:04:05.999999999Z0700" + +// ParseTime parse an RFC3339 string with nanoseconds +func ParseTime(timeStr string) (time.Time, error) { + out, err := time.Parse(time.RFC3339Nano, timeStr) + if err != nil { + out, err = time.Parse(TimeFormat, timeStr) + } + return out, err +} + +// Time is just a time.Time with a JSON serialization +type Time struct { + time.Time +} + +// UnmarshalJSON parses an RFC3339 date string into a time object +// borrowed from: https://stackoverflow.com/a/39180230/141023 +func (t *Time) UnmarshalJSON(data []byte) (err error) { + str := string(data) + + // Get rid of the quotes "" around the value. + // A second option would be to include them in the date format string + // instead, like so below: + // time.Parse(`"`+time.RFC3339Nano+`"`, s) + str = str[1 : len(str)-1] + + timeObj, err := ParseTime(str) + t.Time = timeObj + return +} + +// ============================================================================= +// JSON Objects +// ============================================================================= + +// Session credential cookie name/value pair received after logging in and +// required to be sent on all subsequent requests +type Session struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// SessionResponse the JSON object returned from a /session query (login) +type SessionResponse struct { + Session Session `json:"session"` +} + +// SessionQuery the JSON object that is POSTed to the /session endpoint +// in order to login and get a session cookie +type SessionQuery struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// User the JSON object representing a TODOSRHT user +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/user +type User struct { + DisplayName string `json:"displayName"` + EmailAddress string `json:"emailAddress"` + Key string `json:"key"` + Name string `json:"name"` +} + +// Comment the JSON object for a single comment item returned in a list of +// comments +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getComments +type Comment struct { + ID string `json:"id"` + Body string `json:"body"` + Author User `json:"author"` + UpdateAuthor User `json:"updateAuthor"` + Created Time `json:"created"` + Updated Time `json:"updated"` +} + +// CommentPage the JSON object holding a single page of comments returned +// either by direct query or within an issue query +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getComments +type CommentPage struct { + StartAt int `json:"startAt"` + MaxResults int `json:"maxResults"` + Total int `json:"total"` + Comments []Comment `json:"comments"` +} + +// NextStartAt return the index of the first item on the next page +func (cp *CommentPage) NextStartAt() int { + return cp.StartAt + len(cp.Comments) +} + +// IsLastPage return true if there are no more items beyond this page +func (cp *CommentPage) IsLastPage() bool { + return cp.NextStartAt() >= cp.Total +} + +// IssueFields the JSON object returned as the "fields" member of an issue. +// There are a very large number of fields and many of them are custom. We +// only grab a few that we need. +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getIssue +type IssueFields struct { + Creator User `json:"creator"` + Created Time `json:"created"` + Description string `json:"description"` + Summary string `json:"summary"` + Comments CommentPage `json:"comment"` + Labels []string `json:"labels"` +} + +// ChangeLogItem "field-change" data within a changelog entry. A single +// changelog entry might effect multiple fields. For example, closing an issue +// generally requires a change in "status" and "resolution" +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getIssue +type ChangeLogItem struct { + Field string `json:"field"` + FieldType string `json:"fieldtype"` + From string `json:"from"` + FromString string `json:"fromString"` + To string `json:"to"` + ToString string `json:"toString"` +} + +// ChangeLogEntry One entry in a changelog +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getIssue +type ChangeLogEntry struct { + ID string `json:"id"` + Author User `json:"author"` + Created Time `json:"created"` + Items []ChangeLogItem `json:"items"` +} + +// ChangeLogPage A collection of changes to issue metadata +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getIssue +type ChangeLogPage struct { + StartAt int `json:"startAt"` + MaxResults int `json:"maxResults"` + Total int `json:"total"` + IsLast bool `json:"isLast"` // Cloud-only + Entries []ChangeLogEntry `json:"histories"` + Values []ChangeLogEntry `json:"values"` +} + +// NextStartAt return the index of the first item on the next page +func (clp *ChangeLogPage) NextStartAt() int { + return clp.StartAt + len(clp.Entries) +} + +// IsLastPage return true if there are no more items beyond this page +func (clp *ChangeLogPage) IsLastPage() bool { + // NOTE(josh): The "isLast" field is returned on TODOSRHT cloud, but not on + // TODOSRHT server. If we can distinguish which one we are working with, we can + // possibly rely on that instead. + return clp.NextStartAt() >= clp.Total +} + +// Issue Top-level object for an issue +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getIssue +type Issue struct { + ID string `json:"id"` + Key string `json:"key"` + Fields IssueFields `json:"fields"` + ChangeLog ChangeLogPage `json:"changelog"` +} + +// SearchResult The result type from querying the search endpoint +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/search +type SearchResult struct { + StartAt int `json:"startAt"` + MaxResults int `json:"maxResults"` + Total int `json:"total"` + Issues []Issue `json:"issues"` +} + +// NextStartAt return the index of the first item on the next page +func (sr *SearchResult) NextStartAt() int { + return sr.StartAt + len(sr.Issues) +} + +// IsLastPage return true if there are no more items beyond this page +func (sr *SearchResult) IsLastPage() bool { + return sr.NextStartAt() >= sr.Total +} + +// SearchRequest the JSON object POSTed to the /search endpoint +type SearchRequest struct { + JQL string `json:"jql"` + StartAt int `json:"startAt"` + MaxResults int `json:"maxResults"` + Fields []string `json:"fields"` +} + +// Project the JSON object representing a project. Note that we don't use all +// the fields so we have only implemented a couple. +type Project struct { + ID string `json:"id,omitempty"` + Key string `json:"key,omitempty"` +} + +// IssueType the JSON object representing an issue type (i.e. "bug", "task") +// Note that we don't use all the fields so we have only implemented a couple. +type IssueType struct { + ID string `json:"id"` +} + +// IssueCreateFields fields that are included in an IssueCreate request +type IssueCreateFields struct { + Project Project `json:"project"` + Summary string `json:"summary"` + Description string `json:"description"` + IssueType IssueType `json:"issuetype"` +} + +// IssueCreate the JSON object that is POSTed to the /issue endpoint to create +// a new issue +type IssueCreate struct { + Fields IssueCreateFields `json:"fields"` +} + +// IssueCreateResult the JSON object returned after issue creation. +type IssueCreateResult struct { + ID string `json:"id"` + Key string `json:"key"` +} + +// CommentCreate the JSOn object that is POSTed to the /comment endpoint to +// create a new comment +type CommentCreate struct { + Body string `json:"body"` +} + +// StatusCategory the JSON object representing a status category +type StatusCategory struct { + ID int `json:"id"` + Key string `json:"key"` + Self string `json:"self"` + ColorName string `json:"colorName"` + Name string `json:"name"` +} + +// Status the JSON object representing a status (i.e. "Open", "Closed") +type Status struct { + ID string `json:"id"` + Name string `json:"name"` + Self string `json:"self"` + Description string `json:"description"` + StatusCategory StatusCategory `json:"statusCategory"` +} + +// Transition the JSON object represenging a transition from one Status to +// another Status in a TODOSRHT workflow +type Transition struct { + ID string `json:"id"` + Name string `json:"name"` + To Status `json:"to"` +} + +// TransitionList the JSON object returned from the /transitions endpoint +type TransitionList struct { + Transitions []Transition `json:"transitions"` +} + +// ServerInfo general server information returned by the /serverInfo endpoint. +// Notably `ServerTime` will tell you the time on the server. +type ServerInfo struct { + BaseURL string `json:"baseUrl"` + Version string `json:"version"` + VersionNumbers []int `json:"versionNumbers"` + BuildNumber int `json:"buildNumber"` + BuildDate Time `json:"buildDate"` + ServerTime Time `json:"serverTime"` + ScmInfo string `json:"scmInfo"` + BuildPartnerName string `json:"buildPartnerName"` + ServerTitle string `json:"serverTitle"` +} + +// ============================================================================= +// REST Client +// ============================================================================= + +// ClientTransport wraps http.RoundTripper by adding a +// "Content-Type=application/json" header +type ClientTransport struct { + underlyingTransport http.RoundTripper + basicAuthString string +} + +// RoundTrip overrides the default by adding the content-type header +func (ct *ClientTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Content-Type", "application/json") + if ct.basicAuthString != "" { + req.Header.Add("Authorization", + fmt.Sprintf("Basic %s", ct.basicAuthString)) + } + + return ct.underlyingTransport.RoundTrip(req) +} + +func (ct *ClientTransport) SetCredentials(username string, token string) { + credString := fmt.Sprintf("%s:%s", username, token) + ct.basicAuthString = base64.StdEncoding.EncodeToString([]byte(credString)) +} + +// Client Thin wrapper around the http.Client providing todosrht-specific methods +// for API endpoints +type Client struct { + *http.Client + serverURL string + ctx context.Context +} + +// NewClient Construct a new client connected to the provided server and +// utilizing the given context for asynchronous events +func NewClient(ctx context.Context, serverURL string) *Client { + cookiJar, _ := cookiejar.New(nil) + client := &http.Client{ + Transport: &ClientTransport{underlyingTransport: http.DefaultTransport}, + Jar: cookiJar, + } + + return &Client{client, serverURL, ctx} +} + +// Login POST credentials to the /session endpoint and get a session cookie +func (client *Client) Login(credType, login, password string) error { + switch credType { + case "SESSION": + return client.RefreshSessionToken(login, password) + case "TOKEN": + return client.SetTokenCredentials(login, password) + default: + panic("unknown todo.sr.ht cred type") + } +} + +// RefreshSessionToken formulate the JSON request object from the user +// credentials and POST it to the /session endpoint and get a session cookie +func (client *Client) RefreshSessionToken(username, password string) error { + params := SessionQuery{ + Username: username, + Password: password, + } + + data, err := json.Marshal(params) + if err != nil { + return err + } + + return client.RefreshSessionTokenRaw(data) +} + +// SetTokenCredentials POST credentials to the /session endpoint and get a +// session cookie +func (client *Client) SetTokenCredentials(username, password string) error { + switch transport := client.Transport.(type) { + case *ClientTransport: + transport.SetCredentials(username, password) + default: + return fmt.Errorf("Invalid transport type") + } + return nil +} + +// RefreshSessionTokenRaw POST credentials to the /session endpoint and get a +// session cookie +func (client *Client) RefreshSessionTokenRaw(credentialsJSON []byte) error { + postURL := fmt.Sprintf("%s/rest/auth/1/session", client.serverURL) + + req, err := http.NewRequest("POST", postURL, bytes.NewBuffer(credentialsJSON)) + if err != nil { + return err + } + + urlobj, err := url.Parse(client.serverURL) + if err != nil { + fmt.Printf("Failed to parse %s\n", client.serverURL) + } else { + // Clear out cookies + client.Jar.SetCookies(urlobj, []*http.Cookie{}) + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + req = req.WithContext(ctx) + } + + response, err := client.Do(req) + if err != nil { + return err + } + + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + content, _ := io.ReadAll(response.Body) + return fmt.Errorf( + "error creating token %v: %s", response.StatusCode, content) + } + + data, _ := io.ReadAll(response.Body) + var aux SessionResponse + err = json.Unmarshal(data, &aux) + if err != nil { + return err + } + + var cookies []*http.Cookie + cookie := &http.Cookie{ + Name: aux.Session.Name, + Value: aux.Session.Value, + } + cookies = append(cookies, cookie) + client.Jar.SetCookies(urlobj, cookies) + + return nil +} + +// ============================================================================= +// Endpoint Wrappers +// ============================================================================= + +// Search Perform an issue a JQL search on the /search endpoint +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/search +func (client *Client) Search(jql string, maxResults int, startAt int) (*SearchResult, error) { + url := fmt.Sprintf("%s/rest/api/2/search", client.serverURL) + + requestBody, err := json.Marshal(SearchRequest{ + JQL: jql, + StartAt: startAt, + MaxResults: maxResults, + Fields: []string{ + "comment", + "created", + "creator", + "description", + "labels", + "status", + "summary"}}) + if err != nil { + return nil, err + } + + request, err := http.NewRequest("POST", url, bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s, %s", response.StatusCode, + url, requestBody) + return nil, err + } + + var message SearchResult + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &message) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &message, nil +} + +// SearchIterator cursor within paginated results from the /search endpoint +type SearchIterator struct { + client *Client + jql string + searchResult *SearchResult + Err error + + pageSize int + itemIdx int +} + +// HasError returns true if the iterator is holding an error +func (si *SearchIterator) HasError() bool { + if si.Err == errDone { + return false + } + if si.Err == nil { + return false + } + return true +} + +// HasNext returns true if there is another item available in the result set +func (si *SearchIterator) HasNext() bool { + return si.Err == nil && si.itemIdx < len(si.searchResult.Issues) +} + +// Next Return the next item in the result set and advance the iterator. +// Advancing the iterator may require fetching a new page. +func (si *SearchIterator) Next() *Issue { + if si.Err != nil { + return nil + } + + issue := si.searchResult.Issues[si.itemIdx] + if si.itemIdx+1 < len(si.searchResult.Issues) { + // We still have an item left in the currently cached page + si.itemIdx++ + } else { + if si.searchResult.IsLastPage() { + si.Err = errDone + } else { + // There are still more pages to fetch, so fetch the next page and + // cache it + si.searchResult, si.Err = si.client.Search( + si.jql, si.pageSize, si.searchResult.NextStartAt()) + // NOTE(josh): we don't deal with the error now, we just cache it. + // HasNext() will return false and the caller can check the error + // afterward. + si.itemIdx = 0 + } + } + return &issue +} + +// IterSearch return an iterator over paginated results for a JQL search +func (client *Client) IterSearch(jql string, pageSize int) *SearchIterator { + result, err := client.Search(jql, pageSize, 0) + + iter := &SearchIterator{ + client: client, + jql: jql, + searchResult: result, + Err: err, + pageSize: pageSize, + itemIdx: 0, + } + + return iter +} + +// GetIssue fetches an issue object via the /issue/{IssueIdOrKey} endpoint +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue +func (client *Client) GetIssue(idOrKey string, fields []string, expand []string, + properties []string) (*Issue, error) { + + url := fmt.Sprintf("%s/rest/api/2/issue/%s", client.serverURL, idOrKey) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + err := fmt.Errorf("Creating request %v", err) + return nil, err + } + + query := request.URL.Query() + if len(fields) > 0 { + query.Add("fields", strings.Join(fields, ",")) + } + if len(expand) > 0 { + query.Add("expand", strings.Join(expand, ",")) + } + if len(properties) > 0 { + query.Add("properties", strings.Join(properties, ",")) + } + request.URL.RawQuery = query.Encode() + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var issue Issue + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &issue) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &issue, nil +} + +// GetComments returns a page of comments via the issue/{IssueIdOrKey}/comment +// endpoint +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue-getComment +func (client *Client) GetComments(idOrKey string, maxResults int, startAt int) (*CommentPage, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/comment", client.serverURL, idOrKey) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + err := fmt.Errorf("Creating request %v", err) + return nil, err + } + + query := request.URL.Query() + if maxResults > 0 { + query.Add("maxResults", fmt.Sprintf("%d", maxResults)) + } + if startAt > 0 { + query.Add("startAt", fmt.Sprintf("%d", startAt)) + } + request.URL.RawQuery = query.Encode() + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var comments CommentPage + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &comments) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &comments, nil +} + +// CommentIterator cursor within paginated results from the /comment endpoint +type CommentIterator struct { + client *Client + idOrKey string + message *CommentPage + Err error + + pageSize int + itemIdx int +} + +// HasError returns true if the iterator is holding an error +func (ci *CommentIterator) HasError() bool { + if ci.Err == errDone { + return false + } + if ci.Err == nil { + return false + } + return true +} + +// HasNext returns true if there is another item available in the result set +func (ci *CommentIterator) HasNext() bool { + return ci.Err == nil && ci.itemIdx < len(ci.message.Comments) +} + +// Next Return the next item in the result set and advance the iterator. +// Advancing the iterator may require fetching a new page. +func (ci *CommentIterator) Next() *Comment { + if ci.Err != nil { + return nil + } + + comment := ci.message.Comments[ci.itemIdx] + if ci.itemIdx+1 < len(ci.message.Comments) { + // We still have an item left in the currently cached page + ci.itemIdx++ + } else { + if ci.message.IsLastPage() { + ci.Err = errDone + } else { + // There are still more pages to fetch, so fetch the next page and + // cache it + ci.message, ci.Err = ci.client.GetComments( + ci.idOrKey, ci.pageSize, ci.message.NextStartAt()) + // NOTE(josh): we don't deal with the error now, we just cache it. + // HasNext() will return false and the caller can check the error + // afterward. + ci.itemIdx = 0 + } + } + return &comment +} + +// IterComments returns an iterator over paginated comments within an issue +func (client *Client) IterComments(idOrKey string, pageSize int) *CommentIterator { + message, err := client.GetComments(idOrKey, pageSize, 0) + + iter := &CommentIterator{ + client: client, + idOrKey: idOrKey, + message: message, + Err: err, + pageSize: pageSize, + itemIdx: 0, + } + + return iter +} + +// GetChangeLog fetch one page of the changelog for an issue via the +// /issue/{IssueIdOrKey}/changelog endpoint (for TODOSRHT cloud) or +// /issue/{IssueIdOrKey} with (fields=*none&expand=changelog) +// (for TODOSRHT server) +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue +func (client *Client) GetChangeLog(idOrKey string, maxResults int, startAt int) (*ChangeLogPage, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/changelog", client.serverURL, idOrKey) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + err := fmt.Errorf("Creating request %v", err) + return nil, err + } + + query := request.URL.Query() + if maxResults > 0 { + query.Add("maxResults", fmt.Sprintf("%d", maxResults)) + } + if startAt > 0 { + query.Add("startAt", fmt.Sprintf("%d", startAt)) + } + request.URL.RawQuery = query.Encode() + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode == http.StatusNotFound { + // The issue/{IssueIdOrKey}/changelog endpoint is only available on TODOSRHT cloud + // products, not on TODOSRHT server. In order to get the information we have to + // query the issue and ask for a changelog expansion. Unfortunately this means + // that the changelog is not paginated and we have to fetch the entire thing + // at once. Hopefully things don't break for very long changelogs. + issue, err := client.GetIssue( + idOrKey, []string{"*none"}, []string{"changelog"}, []string{}) + if err != nil { + return nil, err + } + + return &issue.ChangeLog, nil + } + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var changelog ChangeLogPage + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &changelog) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + // TODOSRHT cloud returns changelog entries in the "values" list, whereas TODOSRHT + // server returns them in the "histories" list when embedded in an issue + // object. + changelog.Entries = changelog.Values + changelog.Values = nil + + return &changelog, nil +} + +// ChangeLogIterator cursor within paginated results from the /search endpoint +type ChangeLogIterator struct { + client *Client + idOrKey string + message *ChangeLogPage + Err error + + pageSize int + itemIdx int +} + +// HasError returns true if the iterator is holding an error +func (cli *ChangeLogIterator) HasError() bool { + if cli.Err == errDone { + return false + } + if cli.Err == nil { + return false + } + return true +} + +// HasNext returns true if there is another item available in the result set +func (cli *ChangeLogIterator) HasNext() bool { + return cli.Err == nil && cli.itemIdx < len(cli.message.Entries) +} + +// Next Return the next item in the result set and advance the iterator. +// Advancing the iterator may require fetching a new page. +func (cli *ChangeLogIterator) Next() *ChangeLogEntry { + if cli.Err != nil { + return nil + } + + item := cli.message.Entries[cli.itemIdx] + if cli.itemIdx+1 < len(cli.message.Entries) { + // We still have an item left in the currently cached page + cli.itemIdx++ + } else { + if cli.message.IsLastPage() { + cli.Err = errDone + } else { + // There are still more pages to fetch, so fetch the next page and + // cache it + cli.message, cli.Err = cli.client.GetChangeLog( + cli.idOrKey, cli.pageSize, cli.message.NextStartAt()) + // NOTE(josh): we don't deal with the error now, we just cache it. + // HasNext() will return false and the caller can check the error + // afterward. + cli.itemIdx = 0 + } + } + return &item +} + +// IterChangeLog returns an iterator over entries in the changelog for an issue +func (client *Client) IterChangeLog(idOrKey string, pageSize int) *ChangeLogIterator { + message, err := client.GetChangeLog(idOrKey, pageSize, 0) + + iter := &ChangeLogIterator{ + client: client, + idOrKey: idOrKey, + message: message, + Err: err, + pageSize: pageSize, + itemIdx: 0, + } + + return iter +} + +// GetProject returns the project JSON object given its id or key +func (client *Client) GetProject(projectIDOrKey string) (*Project, error) { + url := fmt.Sprintf( + "%s/rest/api/2/project/%s", client.serverURL, projectIDOrKey) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, url) + return nil, err + } + + var project Project + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &project) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &project, nil +} + +// CreateIssue creates a new TODOSRHT issue and returns it +func (client *Client) CreateIssue(projectIDOrKey, title, body string, + extra map[string]interface{}) (*IssueCreateResult, error) { + + url := fmt.Sprintf("%s/rest/api/2/issue", client.serverURL) + + fields := make(map[string]interface{}) + fields["summary"] = title + fields["description"] = body + for key, value := range extra { + fields[key] = value + } + + // If the project string is an integer than assume it is an ID. Otherwise it + // is a key. + _, err := strconv.Atoi(projectIDOrKey) + if err == nil { + fields["project"] = map[string]string{"id": projectIDOrKey} + } else { + fields["project"] = map[string]string{"key": projectIDOrKey} + } + + message := make(map[string]interface{}) + message["fields"] = fields + + data, err := json.Marshal(message) + if err != nil { + return nil, err + } + + request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusCreated { + content, _ := io.ReadAll(response.Body) + err := fmt.Errorf( + "HTTP response %d, query was %s\n data: %s\n response: %s", + response.StatusCode, request.URL.String(), data, content) + return nil, err + } + + var result IssueCreateResult + + data, _ = io.ReadAll(response.Body) + err = json.Unmarshal(data, &result) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &result, nil +} + +// UpdateIssueTitle changes the "summary" field of a TODOSRHT issue +func (client *Client) UpdateIssueTitle(issueKeyOrID, title string) (time.Time, error) { + + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s", client.serverURL, issueKeyOrID) + var responseTime time.Time + + // NOTE(josh): Since updates are a list of heterogeneous objects let's just + // manually build the JSON text + data, err := json.Marshal(title) + if err != nil { + return responseTime, err + } + + var buffer bytes.Buffer + _, _ = fmt.Fprintf(&buffer, `{"update":{"summary":[`) + _, _ = fmt.Fprintf(&buffer, `{"set":%s}`, data) + _, _ = fmt.Fprintf(&buffer, `]}}`) + + data = buffer.Bytes() + request, err := http.NewRequest("PUT", url, bytes.NewBuffer(data)) + if err != nil { + return responseTime, err + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return responseTime, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusNoContent { + content, _ := io.ReadAll(response.Body) + err := fmt.Errorf( + "HTTP response %d, query was %s\n data: %s\n response: %s", + response.StatusCode, request.URL.String(), data, content) + return responseTime, err + } + + dateHeader, ok := response.Header["Date"] + if !ok || len(dateHeader) != 1 { + // No "Date" header, or empty, or multiple of them. Regardless, we don't + // have a date we can return + return responseTime, nil + } + + responseTime, err = http.ParseTime(dateHeader[0]) + if err != nil { + return time.Time{}, err + } + + return responseTime, nil +} + +// UpdateIssueBody changes the "description" field of a TODOSRHT issue +func (client *Client) UpdateIssueBody(issueKeyOrID, body string) (time.Time, error) { + + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s", client.serverURL, issueKeyOrID) + var responseTime time.Time + // NOTE(josh): Since updates are a list of heterogeneous objects let's just + // manually build the JSON text + data, err := json.Marshal(body) + if err != nil { + return responseTime, err + } + + var buffer bytes.Buffer + _, _ = fmt.Fprintf(&buffer, `{"update":{"description":[`) + _, _ = fmt.Fprintf(&buffer, `{"set":%s}`, data) + _, _ = fmt.Fprintf(&buffer, `]}}`) + + data = buffer.Bytes() + request, err := http.NewRequest("PUT", url, bytes.NewBuffer(data)) + if err != nil { + return responseTime, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return responseTime, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusNoContent { + content, _ := io.ReadAll(response.Body) + err := fmt.Errorf( + "HTTP response %d, query was %s\n data: %s\n response: %s", + response.StatusCode, request.URL.String(), data, content) + return responseTime, err + } + + dateHeader, ok := response.Header["Date"] + if !ok || len(dateHeader) != 1 { + // No "Date" header, or empty, or multiple of them. Regardless, we don't + // have a date we can return + return responseTime, nil + } + + responseTime, err = http.ParseTime(dateHeader[0]) + if err != nil { + return time.Time{}, err + } + + return responseTime, nil +} + +// AddComment adds a new comment to an issue (and returns it). +func (client *Client) AddComment(issueKeyOrID, body string) (*Comment, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/comment", client.serverURL, issueKeyOrID) + + params := CommentCreate{Body: body} + data, err := json.Marshal(params) + if err != nil { + return nil, err + } + + request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusCreated { + content, _ := io.ReadAll(response.Body) + err := fmt.Errorf( + "HTTP response %d, query was %s\n data: %s\n response: %s", + response.StatusCode, request.URL.String(), data, content) + return nil, err + } + + var result Comment + + data, _ = io.ReadAll(response.Body) + err = json.Unmarshal(data, &result) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &result, nil +} + +// UpdateComment changes the text of a comment +func (client *Client) UpdateComment(issueKeyOrID, commentID, body string) ( + *Comment, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/comment/%s", client.serverURL, issueKeyOrID, + commentID) + + params := CommentCreate{Body: body} + data, err := json.Marshal(params) + if err != nil { + return nil, err + } + + request, err := http.NewRequest("PUT", url, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var result Comment + + data, _ = io.ReadAll(response.Body) + err = json.Unmarshal(data, &result) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &result, nil +} + +// UpdateLabels changes labels for an issue +func (client *Client) UpdateLabels(issueKeyOrID string, added, removed []common.Label) (time.Time, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/", client.serverURL, issueKeyOrID) + var responseTime time.Time + + // NOTE(josh): Since updates are a list of heterogeneous objects let's just + // manually build the JSON text + var buffer bytes.Buffer + _, _ = fmt.Fprintf(&buffer, `{"update":{"labels":[`) + first := true + for _, label := range added { + if !first { + _, _ = fmt.Fprintf(&buffer, ",") + } + _, _ = fmt.Fprintf(&buffer, `{"add":"%s"}`, label) + first = false + } + for _, label := range removed { + if !first { + _, _ = fmt.Fprintf(&buffer, ",") + } + _, _ = fmt.Fprintf(&buffer, `{"remove":"%s"}`, label) + first = false + } + _, _ = fmt.Fprintf(&buffer, "]}}") + + data := buffer.Bytes() + request, err := http.NewRequest("PUT", url, bytes.NewBuffer(data)) + if err != nil { + return responseTime, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return responseTime, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusNoContent { + content, _ := io.ReadAll(response.Body) + err := fmt.Errorf( + "HTTP response %d, query was %s\n data: %s\n response: %s", + response.StatusCode, request.URL.String(), data, content) + return responseTime, err + } + + dateHeader, ok := response.Header["Date"] + if !ok || len(dateHeader) != 1 { + // No "Date" header, or empty, or multiple of them. Regardless, we don't + // have a date we can return + return responseTime, nil + } + + responseTime, err = http.ParseTime(dateHeader[0]) + if err != nil { + return time.Time{}, err + } + + return responseTime, nil +} + +// GetTransitions returns a list of available transitions for an issue +func (client *Client) GetTransitions(issueKeyOrID string) (*TransitionList, error) { + + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/transitions", client.serverURL, issueKeyOrID) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + err := fmt.Errorf("Creating request %v", err) + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var message TransitionList + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &message) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &message, nil +} + +func getTransitionTo(tlist *TransitionList, desiredStateNameOrID string) *Transition { + for _, transition := range tlist.Transitions { + if transition.To.ID == desiredStateNameOrID { + return &transition + } else if transition.To.Name == desiredStateNameOrID { + return &transition + } + } + return nil +} + +// DoTransition changes the "status" of an issue +func (client *Client) DoTransition(issueKeyOrID string, transitionID string) (time.Time, error) { + url := fmt.Sprintf( + "%s/rest/api/2/issue/%s/transitions", client.serverURL, issueKeyOrID) + var responseTime time.Time + + // TODO(josh)[767ee72]: Figure out a good way to "configure" the + // open/close state mapping. It would be *great* if we could actually + // *compute* the necessary transitions and prompt for missing metadata... + // but that is complex + var buffer bytes.Buffer + _, _ = fmt.Fprintf(&buffer, + `{"transition":{"id":"%s"}, "resolution": {"name": "Done"}}`, + transitionID) + request, err := http.NewRequest("POST", url, bytes.NewBuffer(buffer.Bytes())) + if err != nil { + return responseTime, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return responseTime, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusNoContent { + err := errors.Wrap(errTransitionNotAllowed, fmt.Sprintf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String())) + return responseTime, err + } + + dateHeader, ok := response.Header["Date"] + if !ok || len(dateHeader) != 1 { + // No "Date" header, or empty, or multiple of them. Regardless, we don't + // have a date we can return + return responseTime, nil + } + + responseTime, err = http.ParseTime(dateHeader[0]) + if err != nil { + return time.Time{}, err + } + + return responseTime, nil +} + +// GetServerInfo Fetch server information from the /serverinfo endpoint +// https://docs.atlassian.com/software/todosrht/docs/api/REST/8.2.6/#api/2/issue +func (client *Client) GetServerInfo() (*ServerInfo, error) { + url := fmt.Sprintf("%s/rest/api/2/serverinfo", client.serverURL) + + request, err := http.NewRequest("GET", url, nil) + if err != nil { + err := fmt.Errorf("Creating request %v", err) + return nil, err + } + + if client.ctx != nil { + ctx, cancel := context.WithTimeout(client.ctx, defaultTimeout) + defer cancel() + request = request.WithContext(ctx) + } + + response, err := client.Do(request) + if err != nil { + err := fmt.Errorf("Performing request %v", err) + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + err := fmt.Errorf( + "HTTP response %d, query was %s", response.StatusCode, + request.URL.String()) + return nil, err + } + + var message ServerInfo + + data, _ := io.ReadAll(response.Body) + err = json.Unmarshal(data, &message) + if err != nil { + err := fmt.Errorf("Decoding response %v", err) + return nil, err + } + + return &message, nil +} + +// GetServerTime returns the current time on the server +func (client *Client) GetServerTime() (Time, error) { + var result Time + info, err := client.GetServerInfo() + if err != nil { + return result, err + } + return info.ServerTime, nil +} diff --git a/bridge/todosrht/config.go b/bridge/todosrht/config.go new file mode 100644 index 000000000..aceace639 --- /dev/null +++ b/bridge/todosrht/config.go @@ -0,0 +1,220 @@ +package todosrht + +import ( + "context" + "fmt" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/commands/input" + "github.com/git-bug/git-bug/repository" +) + +const moreConfigText = ` +NOTE: There are a few optional configuration values that you can additionally +set in your git configuration to influence the behavior of the bridge. Please +see the notes at: +https://github.com/git-bug/git-bug/blob/master/doc/todosrht_bridge.md +` + +const credTypeText = ` +TODOSRHT has recently altered it's authentication strategies. Servers deployed +prior to October 1st 2019 must use "SESSION" authentication, whereby the REST +client logs in with an actual username and password, is assigned a session, and +passes the session cookie with each request. TODOSRHT Cloud and servers deployed +after October 1st 2019 must use "TOKEN" authentication. You must create a user +API token and the client will provide this along with your username with each +request.` + +func (*TodoSourceHut) ValidParams() map[string]interface{} { + return map[string]interface{}{ + "BaseURL": nil, + "Login": nil, + "CredPrefix": nil, + "Project": nil, + "TokenRaw": nil, + } +} + +// Configure sets up the bridge configuration +func (j *TodoSourceHut) Configure(repo *cache.RepoCache, params core.BridgeParams, interactive bool) (core.Configuration, error) { + var err error + + baseURL := params.BaseURL + if baseURL == "" { + if !interactive { + return nil, fmt.Errorf("Non-interactive-mode is active. Please specify the TODOSRHT server URL via the --base-url option.") + } + // terminal prompt + baseURL, err = input.Prompt("TODOSRHT server URL", "URL", input.Required, input.IsURL) + if err != nil { + return nil, err + } + } + + project := params.Project + if project == "" { + if !interactive { + return nil, fmt.Errorf("Non-interactive-mode is active. Please specify the TODOSRHT project key via the --project option.") + } + project, err = input.Prompt("TODOSRHT project key", "project", input.Required) + if err != nil { + return nil, err + } + } + + var login string + var credType string + var cred auth.Credential + + switch { + case params.CredPrefix != "": + cred, err = auth.LoadWithPrefix(repo, params.CredPrefix) + if err != nil { + return nil, err + } + l, ok := cred.GetMetadata(auth.MetaKeyLogin) + if !ok { + return nil, fmt.Errorf("credential doesn't have a login") + } + login = l + default: + if params.Login == "" { + if !interactive { + return nil, fmt.Errorf("Non-interactive-mode is active. Please specify the login name via the --login option.") + } + login, err = input.Prompt("TODOSRHT login", "login", input.Required) + if err != nil { + return nil, err + } + } else { + login = params.Login + } + // TODO: validate username + + if params.TokenRaw == "" { + if !interactive { + return nil, fmt.Errorf("Non-interactive-mode is active. Please specify the access token via the --token option.") + } + fmt.Println(credTypeText) + credTypeInput, err := input.PromptChoice("Authentication mechanism", []string{"SESSION", "TOKEN"}) + if err != nil { + return nil, err + } + credType = []string{"SESSION", "TOKEN"}[credTypeInput] + cred, err = promptCredOptions(repo, login, baseURL) + if err != nil { + return nil, err + } + } else { + credType = "TOKEN" + } + } + + conf := make(core.Configuration) + conf[core.ConfigKeyTarget] = target + conf[confKeyBaseUrl] = baseURL + conf[confKeyProject] = project + conf[confKeyCredentialType] = credType + conf[confKeyDefaultLogin] = login + + err = j.ValidateConfig(conf) + if err != nil { + return nil, err + } + + fmt.Printf("Attempting to login with credentials...\n") + client, err := buildClient(context.TODO(), baseURL, credType, cred) + if err != nil { + return nil, err + } + + // verify access to the project with credentials + fmt.Printf("Checking project ...\n") + _, err = client.GetProject(project) + if err != nil { + return nil, fmt.Errorf( + "Project %s doesn't exist on %s, or authentication credentials for (%s)"+ + " are invalid", + project, baseURL, login) + } + + // don't forget to store the now known valid token + if !auth.IdExist(repo, cred.ID()) { + err = auth.Store(repo, cred) + if err != nil { + return nil, err + } + } + + err = core.FinishConfig(repo, metaKeyTodoSourceHutLogin, login) + if err != nil { + return nil, err + } + + fmt.Print(moreConfigText) + return conf, nil +} + +// ValidateConfig returns true if all required keys are present +func (*TodoSourceHut) ValidateConfig(conf core.Configuration) error { + if v, ok := conf[core.ConfigKeyTarget]; !ok { + return fmt.Errorf("missing %s key", core.ConfigKeyTarget) + } else if v != target { + return fmt.Errorf("unexpected target name: %v", v) + } + if _, ok := conf[confKeyBaseUrl]; !ok { + return fmt.Errorf("missing %s key", confKeyBaseUrl) + } + if _, ok := conf[confKeyProject]; !ok { + return fmt.Errorf("missing %s key", confKeyProject) + } + if _, ok := conf[confKeyCredentialType]; !ok { + return fmt.Errorf("missing %s key", confKeyCredentialType) + } + if _, ok := conf[confKeyDefaultLogin]; !ok { + return fmt.Errorf("missing %s key", confKeyDefaultLogin) + } + + return nil +} + +func promptCredOptions(repo repository.RepoKeyring, login, baseUrl string) (auth.Credential, error) { + creds, err := auth.List(repo, + auth.WithTarget(target), + auth.WithKind(auth.KindToken), + auth.WithMeta(auth.MetaKeyLogin, login), + auth.WithMeta(auth.MetaKeyBaseURL, baseUrl), + ) + if err != nil { + return nil, err + } + + cred, index, err := input.PromptCredential(target, "password", creds, []string{ + "enter my password", + "ask my password each time", + }) + switch { + case err != nil: + return nil, err + case cred != nil: + return cred, nil + case index == 0: + password, err := input.PromptPassword("Password", "password", input.Required) + if err != nil { + return nil, err + } + lp := auth.NewLoginPassword(target, login, password) + lp.SetMetadata(auth.MetaKeyLogin, login) + lp.SetMetadata(auth.MetaKeyBaseURL, baseUrl) + return lp, nil + case index == 1: + l := auth.NewLogin(target, login) + l.SetMetadata(auth.MetaKeyLogin, login) + l.SetMetadata(auth.MetaKeyBaseURL, baseUrl) + return l, nil + default: + panic("missed case") + } +} diff --git a/bridge/todosrht/export.go b/bridge/todosrht/export.go new file mode 100644 index 000000000..ba4918516 --- /dev/null +++ b/bridge/todosrht/export.go @@ -0,0 +1,475 @@ +package todosrht + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "time" + + "github.com/pkg/errors" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/entities/bug" + "github.com/git-bug/git-bug/entity" + "github.com/git-bug/git-bug/entity/dag" +) + +var ( + ErrMissingCredentials = errors.New("missing credentials") +) + +// todosrhtExporter implement the Exporter interface +type todosrhtExporter struct { + conf core.Configuration + + // cache identities clients + identityClient map[entity.Id]*Client + + // the mapping from git-bug "status" to TODOSRHT "status" id + statusMap map[string]string + + // cache identifiers used to speed up exporting operations + // cleared for each bug + cachedOperationIDs map[entity.Id]string + + // cache labels used to speed up exporting labels events + cachedLabels map[string]string + + // store TODOSRHT project information + project *Project +} + +// Init . +func (je *todosrhtExporter) Init(ctx context.Context, repo *cache.RepoCache, conf core.Configuration) error { + je.conf = conf + je.identityClient = make(map[entity.Id]*Client) + je.cachedOperationIDs = make(map[entity.Id]string) + je.cachedLabels = make(map[string]string) + + statusMap, err := getStatusMap(je.conf) + if err != nil { + return err + } + je.statusMap = statusMap + + // preload all clients + err = je.cacheAllClient(ctx, repo) + if err != nil { + return err + } + + if len(je.identityClient) == 0 { + return fmt.Errorf("no credentials for this bridge") + } + + var client *Client + for _, c := range je.identityClient { + client = c + break + } + + if client == nil { + panic("nil client") + } + + je.project, err = client.GetProject(je.conf[confKeyProject]) + if err != nil { + return err + } + + return nil +} + +func (je *todosrhtExporter) cacheAllClient(ctx context.Context, repo *cache.RepoCache) error { + creds, err := auth.List(repo, + auth.WithTarget(target), + auth.WithKind(auth.KindLoginPassword), auth.WithKind(auth.KindLogin), + auth.WithMeta(auth.MetaKeyBaseURL, je.conf[confKeyBaseUrl]), + ) + if err != nil { + return err + } + + for _, cred := range creds { + login, ok := cred.GetMetadata(auth.MetaKeyLogin) + if !ok { + _, _ = fmt.Fprintf(os.Stderr, "credential %s is not tagged with a SourceHut login\n", cred.ID().Human()) + continue + } + + user, err := repo.Identities().ResolveIdentityImmutableMetadata(metaKeyTodoSourceHutLogin, login) + if entity.IsErrNotFound(err) { + continue + } + if err != nil { + return nil + } + + if _, ok := je.identityClient[user.Id()]; !ok { + client, err := buildClient(ctx, je.conf[confKeyBaseUrl], je.conf[confKeyCredentialType], cred) + if err != nil { + return err + } + je.identityClient[user.Id()] = client + } + } + + return nil +} + +// getClientForIdentity return an API client configured with the credentials +// of the given identity. If no client were found it will initialize it from +// the known credentials and cache it for next use. +func (je *todosrhtExporter) getClientForIdentity(userId entity.Id) (*Client, error) { + client, ok := je.identityClient[userId] + if ok { + return client, nil + } + + return nil, ErrMissingCredentials +} + +// ExportAll export all event made by the current user to TodoSourceHut +func (je *todosrhtExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) { + out := make(chan core.ExportResult) + + go func() { + defer close(out) + + var allIdentitiesIds []entity.Id + for id := range je.identityClient { + allIdentitiesIds = append(allIdentitiesIds, id) + } + + allBugsIds := repo.Bugs().AllIds() + + for _, id := range allBugsIds { + b, err := repo.Bugs().Resolve(id) + if err != nil { + out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id) + return + } + + select { + + case <-ctx.Done(): + // stop iterating if context cancel function is called + return + + default: + snapshot := b.Snapshot() + + // ignore issues whose last modification date is before the query date + // TODO: compare the Lamport time instead of using the unix time + if snapshot.CreateTime.Before(since) { + out <- core.NewExportNothing(b.Id(), "bug created before the since date") + continue + } + + if snapshot.HasAnyActor(allIdentitiesIds...) { + // try to export the bug and it associated events + err := je.exportBug(ctx, b, out) + if err != nil { + out <- core.NewExportError(errors.Wrap(err, "can't export bug"), id) + return + } + } else { + out <- core.NewExportNothing(id, "not an actor") + } + } + } + }() + + return out, nil +} + +// exportBug publish bugs and related events +func (je *todosrhtExporter) exportBug(ctx context.Context, b *cache.BugCache, out chan<- core.ExportResult) error { + snapshot := b.Snapshot() + + var bugTodoSourceHutID string + + // Special case: + // if a user try to export a bug that is not already exported to todosrht (or + // imported from todosrht) and we do not have the token of the bug author, + // there is nothing we can do. + + // first operation is always createOp + createOp := snapshot.Operations[0].(*bug.CreateOperation) + author := snapshot.Author + + // skip bug if it was imported from some other bug system + origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin) + if ok && origin != target { + out <- core.NewExportNothing( + b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin)) + return nil + } + + // skip bug if it is a todosrht bug but is associated with another project + // (one bridge per TODOSRHT project) + project, ok := snapshot.GetCreateMetadata(metaKeyTodoSourceHutProject) + if ok && !stringInSlice(project, []string{je.project.ID, je.project.Key}) { + out <- core.NewExportNothing( + b.Id(), fmt.Sprintf("issue tagged with project: %s", project)) + return nil + } + + // get todosrht bug ID + todosrhtID, ok := snapshot.GetCreateMetadata(metaKeyTodoSourceHutId) + if ok { + // will be used to mark operation related to a bug as exported + bugTodoSourceHutID = todosrhtID + } else { + // check that we have credentials for operation author + client, err := je.getClientForIdentity(author.Id()) + if err != nil { + // if bug is not yet exported and we do not have the author's credentials + // then there is nothing we can do, so just skip this bug + out <- core.NewExportNothing( + b.Id(), fmt.Sprintf("missing author credentials for user %.8s", + author.Id().String())) + return err + } + + // Load any custom fields required to create an issue from the git + // config file. + fields := make(map[string]interface{}) + defaultFields, hasConf := je.conf[confKeyCreateDefaults] + if hasConf { + err = json.Unmarshal([]byte(defaultFields), &fields) + if err != nil { + return err + } + } else { + // If there is no configuration provided, at the very least the + // "issueType" field is always required. 10001 is "story" which I'm + // pretty sure is standard/default on all TODOSRHT instances. + fields["issuetype"] = map[string]interface{}{ + "id": "10001", + } + } + bugIDField, hasConf := je.conf[confKeyCreateGitBug] + if hasConf { + // If the git configuration also indicates it, we can assign the git-bug + // id to a custom field to assist in integrations + fields[bugIDField] = b.Id().String() + } + + // create bug + result, err := client.CreateIssue( + je.project.ID, createOp.Title, createOp.Message, fields) + if err != nil { + err := errors.Wrap(err, "exporting todosrht issue") + out <- core.NewExportError(err, b.Id()) + return err + } + + id := result.ID + out <- core.NewExportBug(b.Id()) + // mark bug creation operation as exported + err = markOperationAsExported( + b, createOp.Id(), id, je.project.Key, time.Time{}) + if err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return err + } + + // commit operation to avoid creating multiple issues with multiple pushes + err = b.CommitAsNeeded() + if err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return err + } + + // cache bug todosrht ID + bugTodoSourceHutID = id + } + + // cache operation todosrht id + je.cachedOperationIDs[createOp.Id()] = bugTodoSourceHutID + + for _, op := range snapshot.Operations[1:] { + // ignore SetMetadata operations + if _, ok := op.(dag.OperationDoesntChangeSnapshot); ok { + continue + } + + // ignore operations already existing in todosrht (due to import or export) + // cache the ID of already exported or imported issues and events from + // TodoSourceHut + if id, ok := op.GetMetadata(metaKeyTodoSourceHutId); ok { + je.cachedOperationIDs[op.Id()] = id + continue + } + + opAuthor := op.Author() + client, err := je.getClientForIdentity(opAuthor.Id()) + if err != nil { + out <- core.NewExportError( + fmt.Errorf("missing operation author credentials for user %.8s", + author.Id().String()), b.Id()) + continue + } + + var id string + var exportTime time.Time + switch opr := op.(type) { + case *bug.AddCommentOperation: + comment, err := client.AddComment(bugTodoSourceHutID, opr.Message) + if err != nil { + err := errors.Wrap(err, "adding comment") + out <- core.NewExportError(err, b.Id()) + return err + } + id = comment.ID + out <- core.NewExportComment(b.Id()) + + // cache comment id + je.cachedOperationIDs[op.Id()] = id + + case *bug.EditCommentOperation: + if opr.Target == createOp.Id() { + // An EditCommentOpreation with the Target set to the create operation + // encodes a modification to the long-description/summary. + exportTime, err = client.UpdateIssueBody(bugTodoSourceHutID, opr.Message) + if err != nil { + err := errors.Wrap(err, "editing issue") + out <- core.NewExportError(err, b.Id()) + return err + } + out <- core.NewExportCommentEdition(b.Id()) + id = bugTodoSourceHutID + } else { + // Otherwise it's an edit to an actual comment. A comment cannot be + // edited before it was created, so it must be the case that we have + // already observed and cached the AddCommentOperation. + commentID, ok := je.cachedOperationIDs[opr.Target] + if !ok { + // Since an edit has to come after the creation, we expect we would + // have cached the creation id. + panic("unexpected error: comment id not found") + } + comment, err := client.UpdateComment(bugTodoSourceHutID, commentID, opr.Message) + if err != nil { + err := errors.Wrap(err, "editing comment") + out <- core.NewExportError(err, b.Id()) + return err + } + out <- core.NewExportCommentEdition(b.Id()) + // TODOSRHT doesn't track all comment edits, they will only tell us about + // the most recent one. We must invent a consistent id for the operation + // so we use the comment ID plus the timestamp of the update, as + // reported by TODOSRHT. Note that this must be consistent with the importer + // during ensureComment() + id = getTimeDerivedID(comment.ID, comment.Updated) + } + + case *bug.SetStatusOperation: + todosrhtStatus, hasStatus := je.statusMap[opr.Status.String()] + if hasStatus { + exportTime, err = UpdateIssueStatus(client, bugTodoSourceHutID, todosrhtStatus) + if err != nil { + err := errors.Wrap(err, "editing status") + out <- core.NewExportWarning(err, b.Id()) + // Failure to update status isn't necessarily a big error. It's + // possible that we just don't have enough information to make that + // update. In this case, just don't export the operation. + continue + } + out <- core.NewExportStatusChange(b.Id()) + id = bugTodoSourceHutID + } else { + out <- core.NewExportError(fmt.Errorf( + "No todosrht status mapped for %.8s", opr.Status.String()), b.Id()) + } + + case *bug.SetTitleOperation: + exportTime, err = client.UpdateIssueTitle(bugTodoSourceHutID, opr.Title) + if err != nil { + err := errors.Wrap(err, "editing title") + out <- core.NewExportError(err, b.Id()) + return err + } + out <- core.NewExportTitleEdition(b.Id()) + id = bugTodoSourceHutID + + case *bug.LabelChangeOperation: + exportTime, err = client.UpdateLabels( + bugTodoSourceHutID, opr.Added, opr.Removed) + if err != nil { + err := errors.Wrap(err, "updating labels") + out <- core.NewExportError(err, b.Id()) + return err + } + out <- core.NewExportLabelChange(b.Id()) + id = bugTodoSourceHutID + + default: + panic("unhandled operation type case") + } + + // mark operation as exported + err = markOperationAsExported( + b, op.Id(), id, je.project.Key, exportTime) + if err != nil { + err := errors.Wrap(err, "marking operation as exported") + out <- core.NewExportError(err, b.Id()) + return err + } + + // commit at each operation export to avoid exporting same events multiple + // times + err = b.CommitAsNeeded() + if err != nil { + err := errors.Wrap(err, "bug commit") + out <- core.NewExportError(err, b.Id()) + return err + } + } + + return nil +} + +func markOperationAsExported(b *cache.BugCache, target entity.Id, todosrhtID, todosrhtProject string, exportTime time.Time) error { + newMetadata := map[string]string{ + metaKeyTodoSourceHutId: todosrhtID, + metaKeyTodoSourceHutProject: todosrhtProject, + } + if !exportTime.IsZero() { + newMetadata[metaKeyTodoSourceHutExportTime] = exportTime.Format(http.TimeFormat) + } + + _, err := b.SetMetadata(target, newMetadata) + return err +} + +// UpdateIssueStatus attempts to change the "status" field by finding a +// transition which achieves the desired state and then performing that +// transition +func UpdateIssueStatus(client *Client, issueKeyOrID string, desiredStateNameOrID string) (time.Time, error) { + var responseTime time.Time + + tlist, err := client.GetTransitions(issueKeyOrID) + if err != nil { + return responseTime, err + } + + transition := getTransitionTo(tlist, desiredStateNameOrID) + if transition == nil { + return responseTime, errTransitionNotFound + } + + responseTime, err = client.DoTransition(issueKeyOrID, transition.ID) + if err != nil { + return responseTime, err + } + + return responseTime, nil +} diff --git a/bridge/todosrht/import.go b/bridge/todosrht/import.go new file mode 100644 index 000000000..77129de5e --- /dev/null +++ b/bridge/todosrht/import.go @@ -0,0 +1,657 @@ +package todosrht + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/entities/bug" + "github.com/git-bug/git-bug/entities/common" + "github.com/git-bug/git-bug/entity" + "github.com/git-bug/git-bug/entity/dag" + "github.com/git-bug/git-bug/util/text" +) + +const ( + defaultPageSize = 10 +) + +// todosrhtImporter implement the Importer interface +type todosrhtImporter struct { + conf core.Configuration + + client *Client + + // send only channel + out chan<- core.ImportResult +} + +// Init . +func (ji *todosrhtImporter) Init(ctx context.Context, repo *cache.RepoCache, conf core.Configuration) error { + ji.conf = conf + + var cred auth.Credential + + // Prioritize LoginPassword credentials to avoid a prompt + creds, err := auth.List(repo, + auth.WithTarget(target), + auth.WithKind(auth.KindLoginPassword), + auth.WithMeta(auth.MetaKeyBaseURL, conf[confKeyBaseUrl]), + auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]), + ) + if err != nil { + return err + } + if len(creds) > 0 { + cred = creds[0] + goto end + } + + creds, err = auth.List(repo, + auth.WithTarget(target), + auth.WithKind(auth.KindLogin), + auth.WithMeta(auth.MetaKeyBaseURL, conf[confKeyBaseUrl]), + auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]), + ) + if err != nil { + return err + } + if len(creds) > 0 { + cred = creds[0] + } + +end: + if cred == nil { + return fmt.Errorf("no credential for this bridge") + } + + // TODO(josh)[da52062]: Validate token and if it is expired then prompt for + // credentials and generate a new one + ji.client, err = buildClient(ctx, conf[confKeyBaseUrl], conf[confKeyCredentialType], cred) + return err +} + +// ImportAll iterate over all the configured repository issues and ensure the +// creation of the missing issues / timeline items / edits / label events ... +func (ji *todosrhtImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) { + sinceStr := since.Format("2006-01-02 15:04") + project := ji.conf[confKeyProject] + + out := make(chan core.ImportResult) + ji.out = out + + go func() { + defer close(ji.out) + + message, err := ji.client.Search( + fmt.Sprintf("project=%s AND updatedDate>\"%s\"", project, sinceStr), 0, 0) + if err != nil { + out <- core.NewImportError(err, "") + return + } + + fmt.Printf("So far so good. Have %d issues to import\n", message.Total) + + jql := fmt.Sprintf("project=%s AND updatedDate>\"%s\"", project, sinceStr) + var searchIter *SearchIterator + for searchIter = + ji.client.IterSearch(jql, defaultPageSize); searchIter.HasNext(); { + issue := searchIter.Next() + b, err := ji.ensureIssue(repo, *issue) + if err != nil { + err := fmt.Errorf("issue creation: %v", err) + out <- core.NewImportError(err, "") + return + } + + var commentIter *CommentIterator + for commentIter = + ji.client.IterComments(issue.ID, defaultPageSize); commentIter.HasNext(); { + comment := commentIter.Next() + err := ji.ensureComment(repo, b, *comment) + if err != nil { + out <- core.NewImportError(err, "") + } + } + if commentIter.HasError() { + out <- core.NewImportError(commentIter.Err, "") + } + + snapshot := b.Snapshot() + opIdx := 0 + + var changelogIter *ChangeLogIterator + for changelogIter = + ji.client.IterChangeLog(issue.ID, defaultPageSize); changelogIter.HasNext(); { + changelogEntry := changelogIter.Next() + + // Advance the operation iterator up to the first operation which has + // an export date not before the changelog entry date. If the changelog + // entry was created in response to an exported operation, then this + // will be that operation. + var exportTime time.Time + for ; opIdx < len(snapshot.Operations); opIdx++ { + exportTimeStr, hasTime := snapshot.Operations[opIdx].GetMetadata( + metaKeyTodoSourceHutExportTime) + if !hasTime { + continue + } + exportTime, err = http.ParseTime(exportTimeStr) + if err != nil { + continue + } + if !exportTime.Before(changelogEntry.Created.Time) { + break + } + } + if opIdx < len(snapshot.Operations) { + err = ji.ensureChange(repo, b, *changelogEntry, snapshot.Operations[opIdx]) + } else { + err = ji.ensureChange(repo, b, *changelogEntry, nil) + } + if err != nil { + out <- core.NewImportError(err, "") + } + + } + if changelogIter.HasError() { + out <- core.NewImportError(changelogIter.Err, "") + } + + if !b.NeedCommit() { + out <- core.NewImportNothing(b.Id(), "no imported operation") + } else if err := b.Commit(); err != nil { + err = fmt.Errorf("bug commit: %v", err) + out <- core.NewImportError(err, "") + return + } + } + if searchIter.HasError() { + out <- core.NewImportError(searchIter.Err, "") + } + }() + + return out, nil +} + +// Create a bug.Person from a TODOSRHT user +func (ji *todosrhtImporter) ensurePerson(repo *cache.RepoCache, user User) (*cache.IdentityCache, error) { + // Look first in the cache + i, err := repo.Identities().ResolveIdentityImmutableMetadata( + metaKeyTodoSourceHutUser, string(user.Key)) + if err == nil { + return i, nil + } + if _, ok := err.(entity.ErrMultipleMatch); ok { + return nil, err + } + + i, err = repo.Identities().NewRaw( + user.DisplayName, + user.EmailAddress, + user.Key, + "", + nil, + map[string]string{ + metaKeyTodoSourceHutUser: user.Key, + }, + ) + + if err != nil { + return nil, err + } + + ji.out <- core.NewImportIdentity(i.Id()) + return i, nil +} + +// Create a bug.Bug based from a TODOSRHT issue +func (ji *todosrhtImporter) ensureIssue(repo *cache.RepoCache, issue Issue) (*cache.BugCache, error) { + author, err := ji.ensurePerson(repo, issue.Fields.Creator) + if err != nil { + return nil, err + } + + b, err := repo.Bugs().ResolveMatcher(func(excerpt *cache.BugExcerpt) bool { + if _, ok := excerpt.CreateMetadata[metaKeyTodoSourceHutBaseUrl]; ok && + excerpt.CreateMetadata[metaKeyTodoSourceHutBaseUrl] != ji.conf[confKeyBaseUrl] { + return false + } + + return excerpt.CreateMetadata[core.MetaKeyOrigin] == target && + excerpt.CreateMetadata[metaKeyTodoSourceHutId] == issue.ID && + excerpt.CreateMetadata[metaKeyTodoSourceHutProject] == ji.conf[confKeyProject] + }) + if err != nil && !entity.IsErrNotFound(err) { + return nil, err + } + + if entity.IsErrNotFound(err) { + b, _, err = repo.Bugs().NewRaw( + author, + issue.Fields.Created.Unix(), + text.CleanupOneLine(issue.Fields.Summary), + text.Cleanup(issue.Fields.Description), + nil, + map[string]string{ + core.MetaKeyOrigin: target, + metaKeyTodoSourceHutId: issue.ID, + metaKeyTodoSourceHutKey: issue.Key, + metaKeyTodoSourceHutProject: ji.conf[confKeyProject], + metaKeyTodoSourceHutBaseUrl: ji.conf[confKeyBaseUrl], + }) + if err != nil { + return nil, err + } + + ji.out <- core.NewImportBug(b.Id()) + } + + return b, nil +} + +// Return a unique string derived from a unique todosrht id and a timestamp +func getTimeDerivedID(todosrhtID string, timestamp Time) string { + return fmt.Sprintf("%s-%d", todosrhtID, timestamp.Unix()) +} + +// Create a bug.Comment from a TODOSRHT comment +func (ji *todosrhtImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, item Comment) error { + // ensure person + author, err := ji.ensurePerson(repo, item.Author) + if err != nil { + return err + } + + targetOpID, err := b.ResolveOperationWithMetadata(metaKeyTodoSourceHutId, item.ID) + if err != nil && err != cache.ErrNoMatchingOp { + return err + } + + // If the comment is a new comment then create it + if targetOpID == "" && err == cache.ErrNoMatchingOp { + var cleanText string + if item.Updated != item.Created { + // We don't know the original text... we only have the updated text. + cleanText = "" + } else { + cleanText = text.Cleanup(item.Body) + } + + // add comment operation + commentId, op, err := b.AddCommentRaw( + author, + item.Created.Unix(), + cleanText, + nil, + map[string]string{ + metaKeyTodoSourceHutId: item.ID, + }, + ) + if err != nil { + return err + } + + ji.out <- core.NewImportComment(b.Id(), commentId) + targetOpID = op.Id() + } + + // If there are no updates to this comment, then we are done + if item.Updated == item.Created { + return nil + } + + // If there has been an update to this comment, we try to find it in the + // database. We need a unique id so we'll concat the issue id with the update + // timestamp. Note that this must be consistent with the exporter during + // export of an EditCommentOperation + derivedID := getTimeDerivedID(item.ID, item.Updated) + _, err = b.ResolveOperationWithMetadata(metaKeyTodoSourceHutId, derivedID) + if err == nil { + // Already imported this edition + return nil + } + + if err != cache.ErrNoMatchingOp { + return err + } + + // ensure editor identity + editor, err := ji.ensurePerson(repo, item.UpdateAuthor) + if err != nil { + return err + } + + commentId := entity.CombineIds(b.Id(), targetOpID) + + // comment edition + _, err = b.EditCommentRaw( + editor, + item.Updated.Unix(), + commentId, + text.Cleanup(item.Body), + map[string]string{ + metaKeyTodoSourceHutId: derivedID, + }, + ) + + if err != nil { + return err + } + + ji.out <- core.NewImportCommentEdition(b.Id(), commentId) + + return nil +} + +// Return a unique string derived from a unique todosrht id and an index into the +// data referred to by that todosrht id. +func getIndexDerivedID(todosrhtID string, idx int) string { + return fmt.Sprintf("%s-%d", todosrhtID, idx) +} + +func labelSetsMatch(todosrhtSet []string, gitbugSet []common.Label) bool { + if len(todosrhtSet) != len(gitbugSet) { + return false + } + + sort.Strings(todosrhtSet) + gitbugStrSet := make([]string, len(gitbugSet)) + for idx, label := range gitbugSet { + gitbugStrSet[idx] = label.String() + } + sort.Strings(gitbugStrSet) + + for idx, value := range todosrhtSet { + if value != gitbugStrSet[idx] { + return false + } + } + + return true +} + +// Create a bug.Operation (or a series of operations) from a TODOSRHT changelog +// entry +func (ji *todosrhtImporter) ensureChange(repo *cache.RepoCache, b *cache.BugCache, entry ChangeLogEntry, potentialOp dag.Operation) error { + + // If we have an operation which is already mapped to the entire changelog + // entry then that means this changelog entry was induced by an export + // operation and we've already done the match, so we skip this one + _, err := b.ResolveOperationWithMetadata(metaKeyTodoSourceHutDerivedId, entry.ID) + if err == nil { + return nil + } else if err != cache.ErrNoMatchingOp { + return err + } + + // In general, multiple fields may be changed in changelog entry on + // TODOSRHT. For example, when an issue is closed both its "status" and its + // "resolution" are updated within a single changelog entry. + // I don't thing git-bug has a single operation to modify an arbitrary + // number of fields in one go, so we break up the single TODOSRHT changelog + // entry into individual field updates. + author, err := ji.ensurePerson(repo, entry.Author) + if err != nil { + return err + } + + if len(entry.Items) < 1 { + return fmt.Errorf("Received changelog entry with no item! (%s)", entry.ID) + } + + statusMap, err := getStatusMapReverse(ji.conf) + if err != nil { + return err + } + + // NOTE(josh): first do an initial scan and see if any of the changed items + // matches the current potential operation. If it does, then we know that this + // entire changelog entry was created in response to that git-bug operation. + // So we associate the operation with the entire changelog, and not a specific + // entry. + for _, item := range entry.Items { + switch item.Field { + case "labels": + fromLabels := removeEmpty(strings.Split(item.FromString, " ")) + toLabels := removeEmpty(strings.Split(item.ToString, " ")) + removedLabels, addedLabels, _ := setSymmetricDifference(fromLabels, toLabels) + + opr, isRightType := potentialOp.(*bug.LabelChangeOperation) + if isRightType && labelSetsMatch(addedLabels, opr.Added) && labelSetsMatch(removedLabels, opr.Removed) { + _, err := b.SetMetadata(opr.Id(), map[string]string{ + metaKeyTodoSourceHutDerivedId: entry.ID, + }) + if err != nil { + return err + } + return nil + } + + case "status": + opr, isRightType := potentialOp.(*bug.SetStatusOperation) + if isRightType && statusMap[opr.Status.String()] == item.To { + _, err := b.SetMetadata(opr.Id(), map[string]string{ + metaKeyTodoSourceHutDerivedId: entry.ID, + }) + if err != nil { + return err + } + return nil + } + + case "summary": + // NOTE(josh): TODOSRHT calls it "summary", which sounds more like the body + // text, but it's the title + opr, isRightType := potentialOp.(*bug.SetTitleOperation) + if isRightType && opr.Title == item.To { + _, err := b.SetMetadata(opr.Id(), map[string]string{ + metaKeyTodoSourceHutDerivedId: entry.ID, + }) + if err != nil { + return err + } + return nil + } + + case "description": + // NOTE(josh): TODOSRHT calls it "description", which sounds more like the + // title but it's actually the body + opr, isRightType := potentialOp.(*bug.EditCommentOperation) + if isRightType && + opr.Target == b.Snapshot().Operations[0].Id() && + opr.Message == item.ToString { + _, err := b.SetMetadata(opr.Id(), map[string]string{ + metaKeyTodoSourceHutDerivedId: entry.ID, + }) + if err != nil { + return err + } + return nil + } + } + } + + // Since we didn't match the changelog entry to a known export operation, + // then this is a changelog entry that we should import. We import each + // changelog entry item as a separate git-bug operation. + for idx, item := range entry.Items { + derivedID := getIndexDerivedID(entry.ID, idx) + _, err := b.ResolveOperationWithMetadata(metaKeyTodoSourceHutDerivedId, derivedID) + if err == nil { + continue + } + if err != cache.ErrNoMatchingOp { + return err + } + + switch item.Field { + case "labels": + fromLabels := removeEmpty(strings.Split(item.FromString, " ")) + toLabels := removeEmpty(strings.Split(item.ToString, " ")) + removedLabels, addedLabels, _ := setSymmetricDifference(fromLabels, toLabels) + + op, err := b.ForceChangeLabelsRaw( + author, + entry.Created.Unix(), + text.CleanupOneLineArray(addedLabels), + text.CleanupOneLineArray(removedLabels), + map[string]string{ + metaKeyTodoSourceHutId: entry.ID, + metaKeyTodoSourceHutDerivedId: derivedID, + }, + ) + if err != nil { + return err + } + + ji.out <- core.NewImportLabelChange(b.Id(), op.Id()) + + case "status": + statusStr, hasMap := statusMap[item.To] + if hasMap { + switch statusStr { + case common.OpenStatus.String(): + op, err := b.OpenRaw( + author, + entry.Created.Unix(), + map[string]string{ + metaKeyTodoSourceHutId: entry.ID, + metaKeyTodoSourceHutDerivedId: derivedID, + }, + ) + if err != nil { + return err + } + ji.out <- core.NewImportStatusChange(b.Id(), op.Id()) + + case common.ClosedStatus.String(): + op, err := b.CloseRaw( + author, + entry.Created.Unix(), + map[string]string{ + metaKeyTodoSourceHutId: entry.ID, + metaKeyTodoSourceHutDerivedId: derivedID, + }, + ) + if err != nil { + return err + } + ji.out <- core.NewImportStatusChange(b.Id(), op.Id()) + } + } else { + ji.out <- core.NewImportError( + fmt.Errorf( + "No git-bug status mapped for todosrht status %s (%s)", + item.ToString, item.To), "") + } + + case "summary": + // NOTE(josh): TODOSRHT calls it "summary", which sounds more like the body + // text, but it's the title + op, err := b.SetTitleRaw( + author, + entry.Created.Unix(), + text.CleanupOneLine(item.ToString), + map[string]string{ + metaKeyTodoSourceHutId: entry.ID, + metaKeyTodoSourceHutDerivedId: derivedID, + }, + ) + if err != nil { + return err + } + + ji.out <- core.NewImportTitleEdition(b.Id(), op.Id()) + + case "description": + // NOTE(josh): TODOSRHT calls it "description", which sounds more like the + // title but it's actually the body + commentId, _, err := b.EditCreateCommentRaw( + author, + entry.Created.Unix(), + text.Cleanup(item.ToString), + map[string]string{ + metaKeyTodoSourceHutId: entry.ID, + metaKeyTodoSourceHutDerivedId: derivedID, + }, + ) + if err != nil { + return err + } + + ji.out <- core.NewImportCommentEdition(b.Id(), commentId) + + default: + ji.out <- core.NewImportWarning( + fmt.Errorf( + "Unhandled changelog event %s", item.Field), "") + } + + // Other Examples: + // "assignee" (todosrht) + // "Attachment" (todosrht) + // "Epic Link" (custom) + // "Rank" (custom) + // "resolution" (todosrht) + // "Sprint" (custom) + } + return nil +} + +func getStatusMap(conf core.Configuration) (map[string]string, error) { + mapStr, hasConf := conf[confKeyIDMap] + if !hasConf { + return map[string]string{ + common.OpenStatus.String(): "1", + common.ClosedStatus.String(): "6", + }, nil + } + + statusMap := make(map[string]string) + err := json.Unmarshal([]byte(mapStr), &statusMap) + return statusMap, err +} + +func getStatusMapReverse(conf core.Configuration) (map[string]string, error) { + fwdMap, err := getStatusMap(conf) + if err != nil { + return fwdMap, err + } + + outMap := map[string]string{} + for key, val := range fwdMap { + outMap[val] = key + } + + mapStr, hasConf := conf[confKeyIDRevMap] + if !hasConf { + return outMap, nil + } + + revMap := make(map[string]string) + err = json.Unmarshal([]byte(mapStr), &revMap) + for key, val := range revMap { + outMap[key] = val + } + + return outMap, err +} + +func removeEmpty(values []string) []string { + output := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + output = append(output, value) + } + } + return output +} diff --git a/bridge/todosrht/todosrht.go b/bridge/todosrht/todosrht.go new file mode 100644 index 000000000..06b3c842d --- /dev/null +++ b/bridge/todosrht/todosrht.go @@ -0,0 +1,145 @@ +// Package todosrht contains the TodoSourceHut bridge implementation +package todosrht + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/commands/input" +) + +const ( + target = "todosrht" + + metaKeyTodoSourceHutId = "todosrht-id" + metaKeyTodoSourceHutDerivedId = "todosrht-derived-id" + metaKeyTodoSourceHutKey = "todosrht-key" + metaKeyTodoSourceHutUser = "todosrht-user" + metaKeyTodoSourceHutProject = "todosrht-project" + metaKeyTodoSourceHutBaseUrl = "todosrht-base-url" + metaKeyTodoSourceHutExportTime = "todosrht-export-time" + metaKeyTodoSourceHutLogin = "todosrht-login" + + confKeyBaseUrl = "base-url" + confKeyProject = "project" + confKeyDefaultLogin = "default-login" + confKeyCredentialType = "credentials-type" // "SESSION" or "TOKEN" + confKeyIDMap = "bug-id-map" + confKeyIDRevMap = "bug-id-revmap" + // the issue type when exporting a new bug. Default is Story (10001) + confKeyCreateDefaults = "create-issue-defaults" + // if set, the bridge fill this TODOSRHT field with the `git-bug` id when exporting + confKeyCreateGitBug = "create-issue-gitbug-id" + + defaultTimeout = 60 * time.Second +) + +var _ core.BridgeImpl = &TodoSourceHut{} + +// TodoSourceHut Main object for the bridge +type TodoSourceHut struct{} + +// Target returns "todosrht" +func (*TodoSourceHut) Target() string { + return target +} + +func (*TodoSourceHut) LoginMetaKey() string { + return metaKeyTodoSourceHutLogin +} + +// NewImporter returns the todosrht importer +func (*TodoSourceHut) NewImporter() core.Importer { + return &todosrhtImporter{} +} + +// NewExporter returns the todosrht exporter +func (*TodoSourceHut) NewExporter() core.Exporter { + return &todosrhtExporter{} +} + +func buildClient(ctx context.Context, baseURL string, credType string, cred auth.Credential) (*Client, error) { + client := NewClient(ctx, baseURL) + + var login, password string + + switch cred := cred.(type) { + case *auth.LoginPassword: + login = cred.Login + password = cred.Password + case *auth.Login: + login = cred.Login + p, err := input.PromptPassword(fmt.Sprintf("Password for %s", login), "password", input.Required) + if err != nil { + return nil, err + } + password = p + } + + err := client.Login(credType, login, password) + if err != nil { + return nil, err + } + + return client, nil +} + +// stringInSlice returns true if needle is found in haystack +func stringInSlice(needle string, haystack []string) bool { + for _, match := range haystack { + if match == needle { + return true + } + } + return false +} + +// Given two string slices, return three lists containing: +// 1. elements found only in the first input list +// 2. elements found only in the second input list +// 3. elements found in both input lists +func setSymmetricDifference(setA, setB []string) ([]string, []string, []string) { + sort.Strings(setA) + sort.Strings(setB) + + maxLen := len(setA) + len(setB) + onlyA := make([]string, 0, maxLen) + onlyB := make([]string, 0, maxLen) + both := make([]string, 0, maxLen) + + idxA := 0 + idxB := 0 + + for idxA < len(setA) && idxB < len(setB) { + if setA[idxA] < setB[idxB] { + // In the first set, but not the second + onlyA = append(onlyA, setA[idxA]) + idxA++ + } else if setA[idxA] > setB[idxB] { + // In the second set, but not the first + onlyB = append(onlyB, setB[idxB]) + idxB++ + } else { + // In both + both = append(both, setA[idxA]) + idxA++ + idxB++ + } + } + + for ; idxA < len(setA); idxA++ { + // Leftovers in the first set, not the second + onlyA = append(onlyA, setA[idxA]) + } + + for ; idxB < len(setB); idxB++ { + // Leftovers in the second set, not the first + onlyB = append(onlyB, setB[idxB]) + } + + return onlyA, onlyB, both +} diff --git a/doc/man/git-bug-bridge-auth-add-token.1 b/doc/man/git-bug-bridge-auth-add-token.1 index 9314e5e00..6845446c2 100644 --- a/doc/man/git-bug-bridge-auth-add-token.1 +++ b/doc/man/git-bug-bridge-auth-add-token.1 @@ -15,7 +15,7 @@ Store a new token .SH OPTIONS \fB-t\fP, \fB--target\fP="" - The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview] + The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview,todosrht] .PP \fB-l\fP, \fB--login\fP="" diff --git a/doc/man/git-bug-bridge-new.1 b/doc/man/git-bug-bridge-new.1 index f9b11853d..b5cde630f 100644 --- a/doc/man/git-bug-bridge-new.1 +++ b/doc/man/git-bug-bridge-new.1 @@ -19,7 +19,7 @@ Configure a new bridge by passing flags or/and using interactive terminal prompt .PP \fB-t\fP, \fB--target\fP="" - The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview] + The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview,todosrht] .PP \fB-u\fP, \fB--url\fP="" diff --git a/doc/md/git-bug_bridge_auth_add-token.md b/doc/md/git-bug_bridge_auth_add-token.md index 3fb33036c..8add68f02 100644 --- a/doc/md/git-bug_bridge_auth_add-token.md +++ b/doc/md/git-bug_bridge_auth_add-token.md @@ -9,7 +9,7 @@ git-bug bridge auth add-token [TOKEN] [flags] ### Options ``` - -t, --target string The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview] + -t, --target string The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview,todosrht] -l, --login string The login in the remote bug-tracker -u, --user string The user to add the token to. Default is the current user -h, --help help for add-token diff --git a/doc/md/git-bug_bridge_new.md b/doc/md/git-bug_bridge_new.md index 443ee385d..641aac47d 100644 --- a/doc/md/git-bug_bridge_new.md +++ b/doc/md/git-bug_bridge_new.md @@ -71,7 +71,7 @@ git bug bridge new \ ``` -n, --name string A distinctive name to identify the bridge - -t, --target string The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview] + -t, --target string The target of the bridge. Valid values are [github,gitlab,jira,launchpad-preview,todosrht] -u, --url string The URL of the remote repository -b, --base-url string The base URL of your remote issue tracker -l, --login string The login on your remote issue tracker From 1d23ec5efafcc5baed7812a919846908f3e0a472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Tue, 29 Apr 2025 11:37:06 +0200 Subject: [PATCH 3/4] WIP add tests --- bridge/todosrht/config_test.go | 226 ++++++++++++ bridge/todosrht/export_test.go | 368 ++++++++++++++++++++ bridge/todosrht/import_integration_test.go | 386 +++++++++++++++++++++ bridge/todosrht/import_test.go | 242 +++++++++++++ bridge/todosrht/mocks/Client.go | 44 +++ 5 files changed, 1266 insertions(+) create mode 100644 bridge/todosrht/config_test.go create mode 100644 bridge/todosrht/export_test.go create mode 100644 bridge/todosrht/import_integration_test.go create mode 100644 bridge/todosrht/import_test.go create mode 100644 bridge/todosrht/mocks/Client.go diff --git a/bridge/todosrht/config_test.go b/bridge/todosrht/config_test.go new file mode 100644 index 000000000..1a10f7954 --- /dev/null +++ b/bridge/todosrht/config_test.go @@ -0,0 +1,226 @@ +package github + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/git-bug/git-bug/bridge/core/auth" +) + +func TestSplitURL(t *testing.T) { + type args struct { + url string + } + type want struct { + owner string + project string + err error + } + tests := []struct { + name string + args args + want want + }{ + { + name: "default url", + args: args{ + url: "https://todo.sr.ht/git-bug/git-bug", + }, + want: want{ + owner: "git-bug", + project: "git-bug", + err: nil, + }, + }, + { + name: "default issues url", + args: args{ + url: "https://todo.sr.ht/git-bug/git-bug/issues", + }, + want: want{ + owner: "git-bug", + project: "git-bug", + err: nil, + }, + }, + { + name: "default url with git extension", + args: args{ + url: "https://todo.sr.ht/git-bug/git-bug.git", + }, + want: want{ + owner: "git-bug", + project: "git-bug", + err: nil, + }, + }, + { + name: "url with git protocol", + args: args{ + url: "git://todo.sr.ht/git-bug/git-bug.git", + }, + want: want{ + owner: "git-bug", + project: "git-bug", + err: nil, + }, + }, + { + name: "ssh url", + args: args{ + url: "git@todo.sr.ht:git-bug/git-bug.git", + }, + want: want{ + owner: "git-bug", + project: "git-bug", + err: nil, + }, + }, + { + name: "bad url", + args: args{ + url: "https://githb.com/git-bug/git-bug.git", + }, + want: want{ + err: ErrBadProjectURL, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + owner, project, err := splitURL(tt.args.url) + assert.Equal(t, tt.want.err, err) + assert.Equal(t, tt.want.owner, owner) + assert.Equal(t, tt.want.project, project) + }) + } +} + +func TestValidateUsername(t *testing.T) { + if env := os.Getenv("TRAVIS"); env == "true" { + t.Skip("Travis environment: avoiding non authenticated requests") + } + if _, has := os.LookupEnv("CI"); has { + t.Skip("Github action environment: avoiding non authenticated requests") + } + + tests := []struct { + name string + input string + fixed string + ok bool + }{ + { + name: "existing username", + input: "git-bug", + fixed: "git-bug", + ok: true, + }, + { + name: "existing username with bad case", + input: "GiT-bUg", + fixed: "git-bug", + ok: true, + }, + { + name: "existing organisation", + input: "git-bug", + fixed: "git-bug", + ok: true, + }, + { + name: "existing organisation with bad case", + input: "gIt-BuG", + fixed: "git-bug", + ok: true, + }, + { + name: "non existing username", + input: "cant-find-this", + fixed: "", + ok: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, fixed, err := validateUsername(tt.input) + assert.NoError(t, err) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.fixed, fixed) + }) + } +} + +func TestValidateProject(t *testing.T) { + envPrivate := os.Getenv("GITHUB_TOKEN_PRIVATE") + if envPrivate == "" { + t.Skip("Env var GITHUB_TOKEN_PRIVATE missing") + } + + envPublic := os.Getenv("GITHUB_TOKEN_PUBLIC") + if envPublic == "" { + t.Skip("Env var GITHUB_TOKEN_PUBLIC missing") + } + + tokenPrivate := auth.NewToken(target, envPrivate) + tokenPublic := auth.NewToken(target, envPublic) + + type args struct { + owner string + project string + token *auth.Token + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "public repository and token with scope 'public_repo'", + args: args{ + project: "git-bug", + owner: "git-bug", + token: tokenPublic, + }, + want: true, + }, + { + name: "private repository and token with scope 'repo'", + args: args{ + project: "test-github-bridge", + owner: "git-bug", + token: tokenPrivate, + }, + want: true, + }, + { + name: "private repository and token with scope 'public_repo'", + args: args{ + project: "test-github-bridge", + owner: "git-bug", + token: tokenPublic, + }, + want: false, + }, + { + name: "project not existing", + args: args{ + project: "cant-find-this", + owner: "organisation-not-found", + token: tokenPublic, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ok, _ := validateProject(tt.args.owner, tt.args.project, tt.args.token) + assert.Equal(t, tt.want, ok) + }) + } +} diff --git a/bridge/todosrht/export_test.go b/bridge/todosrht/export_test.go new file mode 100644 index 000000000..9b10020dc --- /dev/null +++ b/bridge/todosrht/export_test.go @@ -0,0 +1,368 @@ +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/entity" + "github.com/git-bug/git-bug/entity/dag" + "github.com/git-bug/git-bug/repository" + "github.com/git-bug/git-bug/util/interrupt" +) + +const ( + testRepoBaseName = "git-bug-test-github-exporter" +) + +type testCase struct { + name string + bug *cache.BugCache + numOrOp int // number of original operations +} + +func testCases(t *testing.T, repo *cache.RepoCache) []*testCase { + // simple bug + simpleBug, _, err := repo.Bugs().New("simple bug", "new bug") + require.NoError(t, err) + + // bug with comments + bugWithComments, _, err := repo.Bugs().New("bug with comments", "new bug") + require.NoError(t, err) + + _, _, err = bugWithComments.AddComment("new comment") + require.NoError(t, err) + + // bug with label changes + bugLabelChange, _, err := repo.Bugs().New("bug label change", "new bug") + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"bug"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"core"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels(nil, []string{"bug"}) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"InVaLiD"}, nil) + require.NoError(t, err) + + _, _, err = bugLabelChange.ChangeLabels([]string{"bUG"}, nil) + require.NoError(t, err) + + // bug with comments editions + bugWithCommentEditions, createOp, err := repo.Bugs().New("bug with comments editions", "new bug") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment( + entity.CombineIds(bugWithCommentEditions.Id(), createOp.Id()), "first comment edited") + require.NoError(t, err) + + commentId, _, err := bugWithCommentEditions.AddComment("first comment") + require.NoError(t, err) + + _, err = bugWithCommentEditions.EditComment(commentId, "first comment edited") + require.NoError(t, err) + + // bug status changed + bugStatusChanged, _, err := repo.Bugs().New("bug status changed", "new bug") + require.NoError(t, err) + + _, err = bugStatusChanged.Close() + require.NoError(t, err) + + _, err = bugStatusChanged.Open() + require.NoError(t, err) + + // bug title changed + bugTitleEdited, _, err := repo.Bugs().New("bug title edited", "new bug") + require.NoError(t, err) + + _, err = bugTitleEdited.SetTitle("bug title edited again") + require.NoError(t, err) + + return []*testCase{ + { + name: "simple bug", + bug: simpleBug, + numOrOp: 1, + }, + { + name: "bug with comments", + bug: bugWithComments, + numOrOp: 2, + }, + { + name: "bug label change", + bug: bugLabelChange, + numOrOp: 6, + }, + { + name: "bug with comment editions", + bug: bugWithCommentEditions, + numOrOp: 4, + }, + { + name: "bug changed status", + bug: bugStatusChanged, + numOrOp: 3, + }, + { + name: "bug title edited", + bug: bugTitleEdited, + numOrOp: 2, + }, + } +} + +func TestGithubPushPull(t *testing.T) { + // repo owner + envUser := os.Getenv("GITHUB_TEST_USER") + + // token must have 'repo' and 'delete_repo' scopes + envToken := os.Getenv("GITHUB_TOKEN_ADMIN") + if envToken == "" { + t.Skip("Env var GITHUB_TOKEN_ADMIN missing") + } + + // create repo backend + repo := repository.CreateGoGitTestRepo(t, false) + + backend, err := cache.NewRepoCacheNoEvents(repo) + require.NoError(t, err) + + // set author identity + login := "identity-test" + author, err := backend.Identities().New("test identity", "test@test.org") + require.NoError(t, err) + author.SetMetadata(metaKeyGithubLogin, login) + err = author.Commit() + require.NoError(t, err) + + err = backend.SetUserIdentity(author) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + // Setup token + cleanup + token := auth.NewToken(target, envToken) + token.SetMetadata(auth.MetaKeyLogin, login) + err = auth.Store(repo, token) + require.NoError(t, err) + + cleanToken := func() error { + return auth.Remove(repo, token.ID()) + } + defer cleanToken() + interrupt.RegisterCleaner(cleanToken) + + tests := testCases(t, backend) + + // generate project name + projectName := generateRepoName() + + // create target Github repository + err = createRepository(projectName, envToken) + require.NoError(t, err) + + fmt.Println("created repository", projectName) + + // Let Github handle the repo creation and update all their internal caches. + // Avoid HTTP error 404 retrieving repository node id + time.Sleep(10 * time.Second) + + // Make sure to remove the Github repository when the test end + defer func(t *testing.T) { + if err := deleteRepository(projectName, envUser, envToken); err != nil { + t.Fatal(err) + } + fmt.Println("deleted repository:", projectName) + }(t) + + interrupt.RegisterCleaner(func() error { + return deleteRepository(projectName, envUser, envToken) + }) + + ctx := context.Background() + + // initialize exporter + exporter := &githubExporter{} + err = exporter.Init(ctx, backend, core.Configuration{ + confKeyOwner: envUser, + confKeyProject: projectName, + confKeyDefaultLogin: login, + }) + require.NoError(t, err) + + start := time.Now() + + // export all bugs + exportEvents, err := exporter.ExportAll(ctx, backend, time.Time{}) + require.NoError(t, err) + + for result := range exportEvents { + require.NoError(t, result.Err) + } + require.NoError(t, err) + + fmt.Printf("test repository exported in %f seconds\n", time.Since(start).Seconds()) + + repoTwo := repository.CreateGoGitTestRepo(t, false) + + // create a second backend + backendTwo, err := cache.NewRepoCacheNoEvents(repoTwo) + require.NoError(t, err) + + importer := &githubImporter{} + err = importer.Init(ctx, backend, core.Configuration{ + confKeyOwner: envUser, + confKeyProject: projectName, + confKeyDefaultLogin: login, + }) + require.NoError(t, err) + + // import all exported bugs to the second backend + importEvents, err := importer.ImportAll(ctx, backendTwo, time.Time{}) + require.NoError(t, err) + + for result := range importEvents { + require.NoError(t, result.Err) + } + + require.Len(t, backendTwo.Bugs().AllIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // for each operation a SetMetadataOperation will be added + // so number of operations should double + require.Len(t, tt.bug.Snapshot().Operations, tt.numOrOp*2) + + // verify operation have correct metadata + for _, op := range tt.bug.Snapshot().Operations { + // Check if the originals operations (*not* SetMetadata) are tagged properly + if _, ok := op.(dag.OperationDoesntChangeSnapshot); !ok { + _, haveIDMetadata := op.GetMetadata(metaKeyGithubId) + require.True(t, haveIDMetadata) + + _, haveURLMetada := op.GetMetadata(metaKeyGithubUrl) + require.True(t, haveURLMetada) + } + } + + // get bug github ID + bugGithubID, ok := tt.bug.Snapshot().GetCreateMetadata(metaKeyGithubId) + require.True(t, ok) + + // retrieve bug from backendTwo + importedBug, err := backendTwo.Bugs().ResolveBugCreateMetadata(metaKeyGithubId, bugGithubID) + require.NoError(t, err) + + // verify bug have same number of original operations + require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp) + + // verify bugs are tagged with origin=github + issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.MetaKeyOrigin) + require.True(t, ok) + require.Equal(t, issueOrigin, target) + + // TODO: maybe more tests to ensure bug final state + }) + } +} + +func generateRepoName() string { + rand.Seed(time.Now().UnixNano()) + var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + b := make([]rune, 8) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return fmt.Sprintf("%s-%s", testRepoBaseName, string(b)) +} + +// create repository need a token with scope 'repo' +func createRepository(project, token string) error { + // This function use the V3 Github API because repository creation is not supported yet on the V4 API. + url := fmt.Sprintf("%s/user/repos", githubV3Url) + + params := struct { + Name string `json:"name"` + Description string `json:"description"` + Private bool `json:"private"` + HasIssues bool `json:"has_issues"` + }{ + Name: project, + Description: "git-bug exporter temporary test repository", + Private: true, + HasIssues: true, + } + + data, err := json.Marshal(params) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + return err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + + return resp.Body.Close() +} + +// delete repository need a token with scope 'delete_repo' +func deleteRepository(project, owner, token string) error { + // This function use the V3 Github API because repository removal is not supported yet on the V4 API. + url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + // need the token for private repositories + req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + + client := &http.Client{ + Timeout: defaultTimeout, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("error deleting repository") + } + + return nil +} diff --git a/bridge/todosrht/import_integration_test.go b/bridge/todosrht/import_integration_test.go new file mode 100644 index 000000000..365427e17 --- /dev/null +++ b/bridge/todosrht/import_integration_test.go @@ -0,0 +1,386 @@ +package github + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/shurcooL/githubv4" + m "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/git-bug/git-bug/bridge/github/mocks" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/entities/bug" + "github.com/git-bug/git-bug/entities/common" + "github.com/git-bug/git-bug/repository" + "github.com/git-bug/git-bug/util/interrupt" +) + +// using testify/mock and mockery + +var userName = githubv4.String("marcus") +var userEmail = githubv4.String("marcus@rom.com") +var unedited = githubv4.String("unedited") +var edited = githubv4.String("edited") + +func TestGithubImporterIntegration(t *testing.T) { + // mock + clientMock := &mocks.Client{} + setupExpectations(t, clientMock) + importer := githubImporter{} + importer.client = &rateLimitHandlerClient{sc: clientMock} + + // arrange + repo := repository.CreateGoGitTestRepo(t, false) + backend, err := cache.NewRepoCacheNoEvents(repo) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + require.NoError(t, err) + + // act + events, err := importer.ImportAll(context.Background(), backend, time.Time{}) + + // assert + require.NoError(t, err) + for e := range events { + require.NoError(t, e.Err) + } + require.Len(t, backend.Bugs().AllIds(), 5) + require.Len(t, backend.Identities().AllIds(), 2) + + b1, err := backend.Bugs().ResolveBugCreateMetadata(metaKeyGithubUrl, "https://github.com/marcus/to-himself/issues/1") + require.NoError(t, err) + ops1 := b1.Snapshot().Operations + require.Equal(t, "marcus", ops1[0].Author().Name()) + require.Equal(t, "title 1", ops1[0].(*bug.CreateOperation).Title) + require.Equal(t, "body text 1", ops1[0].(*bug.CreateOperation).Message) + + b3, err := backend.Bugs().ResolveBugCreateMetadata(metaKeyGithubUrl, "https://github.com/marcus/to-himself/issues/3") + require.NoError(t, err) + ops3 := b3.Snapshot().Operations + require.Equal(t, "issue 3 comment 1", ops3[1].(*bug.AddCommentOperation).Message) + require.Equal(t, "issue 3 comment 2", ops3[2].(*bug.AddCommentOperation).Message) + require.Equal(t, []common.Label{"bug"}, ops3[3].(*bug.LabelChangeOperation).Added) + require.Equal(t, "title 3, edit 1", ops3[4].(*bug.SetTitleOperation).Title) + + b4, err := backend.Bugs().ResolveBugCreateMetadata(metaKeyGithubUrl, "https://github.com/marcus/to-himself/issues/4") + require.NoError(t, err) + ops4 := b4.Snapshot().Operations + require.Equal(t, "edited", ops4[1].(*bug.EditCommentOperation).Message) + +} + +func setupExpectations(t *testing.T, mock *mocks.Client) { + rateLimitingError(mock) + expectIssueQuery1(mock) + expectIssueQuery2(mock) + expectIssueQuery3(mock) + expectUserQuery(t, mock) +} + +func rateLimitingError(mock *mocks.Client) { + mock.On("Query", m.Anything, m.AnythingOfType("*github.issueQuery"), m.Anything).Return(errors.New("API rate limit exceeded")).Once() + mock.On("Query", m.Anything, m.AnythingOfType("*github.rateLimitQuery"), m.Anything).Return(nil).Run( + func(args m.Arguments) { + retVal := args.Get(1).(*rateLimitQuery) + retVal.RateLimit.ResetAt.Time = time.Now().Add(time.Millisecond * 200) + }, + ).Once() +} + +func expectIssueQuery1(mock *mocks.Client) { + mock.On("Query", m.Anything, m.AnythingOfType("*github.issueQuery"), m.Anything).Return(nil).Run( + func(args m.Arguments) { + retVal := args.Get(1).(*issueQuery) + retVal.Repository.Issues.Nodes = []issueNode{ + { + issue: issue{ + authorEvent: authorEvent{ + Id: 1, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Title: "title 1", + Number: 1, + Body: "body text 1", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/1", + }, + }, + }, + UserContentEdits: userContentEditConnection{}, + TimelineItems: timelineItemsConnection{}, + }, + { + issue: issue{ + authorEvent: authorEvent{ + Id: 2, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Title: "title 2", + Number: 2, + Body: "body text 2", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/2", + }, + }, + }, + UserContentEdits: userContentEditConnection{}, + TimelineItems: timelineItemsConnection{}, + }, + } + retVal.Repository.Issues.PageInfo = pageInfo{ + EndCursor: "end-cursor-1", + HasNextPage: true, + } + }, + ).Once() +} + +func expectIssueQuery2(mock *mocks.Client) { + mock.On("Query", m.Anything, m.AnythingOfType("*github.issueQuery"), m.Anything).Return(nil).Run( + func(args m.Arguments) { + retVal := args.Get(1).(*issueQuery) + retVal.Repository.Issues.Nodes = []issueNode{ + { + issue: issue{ + authorEvent: authorEvent{ + Id: 3, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Title: "title 3", + Number: 3, + Body: "body text 3", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/3", + }, + }, + }, + UserContentEdits: userContentEditConnection{}, + TimelineItems: timelineItemsConnection{ + Nodes: []timelineItem{ + { + Typename: "IssueComment", + IssueComment: issueComment{ + authorEvent: authorEvent{ + Id: 301, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Body: "issue 3 comment 1", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/3#issuecomment-1", + }, + }, + UserContentEdits: userContentEditConnection{}, + }, + }, + { + Typename: "IssueComment", + IssueComment: issueComment{ + authorEvent: authorEvent{ + Id: 302, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Body: "issue 3 comment 2", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/3#issuecomment-2", + }, + }, + UserContentEdits: userContentEditConnection{}, + }, + }, + { + Typename: "LabeledEvent", + LabeledEvent: labeledEvent{ + actorEvent: actorEvent{ + Id: 303, + Actor: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Label: label{ + Name: "bug", + }, + }, + }, + { + Typename: "RenamedTitleEvent", + RenamedTitleEvent: renamedTitleEvent{ + actorEvent: actorEvent{ + Id: 304, + Actor: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + CurrentTitle: "title 3, edit 1", + }, + }, + }, + PageInfo: pageInfo{}, + }, + }, + { + issue: issue{ + authorEvent: authorEvent{ + Id: 4, + Author: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + }, + Title: "title 4", + Number: 4, + Body: unedited, + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/4", + }, + }, + }, + UserContentEdits: userContentEditConnection{ + Nodes: []userContentEdit{ + // Github is weird: here the order is reversed chronological + { + Id: 402, + Editor: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + Diff: &edited, + }, + { + Id: 401, + Editor: &actor{ + Typename: "User", + User: userActor{ + Name: &userName, + Email: userEmail, + }, + }, + // Github is weird: whenever an issue has issue edits, then the first item + // (issue edit) holds the original (unedited) content and the second item + // (issue edit) holds the (first) edited content. + Diff: &unedited, + }, + }, + PageInfo: pageInfo{}, + }, + TimelineItems: timelineItemsConnection{}, + }, + } + retVal.Repository.Issues.PageInfo = pageInfo{ + EndCursor: "end-cursor-2", + HasNextPage: true, + } + }, + ).Once() +} + +func expectIssueQuery3(mock *mocks.Client) { + mock.On("Query", m.Anything, m.AnythingOfType("*github.issueQuery"), m.Anything).Return(nil).Run( + func(args m.Arguments) { + retVal := args.Get(1).(*issueQuery) + retVal.Repository.Issues.Nodes = []issueNode{ + { + issue: issue{ + authorEvent: authorEvent{ + Author: nil, + }, + Title: "title 5", + Number: 5, + Body: "body text 5", + Url: githubv4.URI{ + URL: &url.URL{ + Scheme: "https", + Host: "github.com", + Path: "marcus/to-himself/issues/5", + }, + }, + }, + UserContentEdits: userContentEditConnection{}, + TimelineItems: timelineItemsConnection{}, + }, + } + retVal.Repository.Issues.PageInfo = pageInfo{} + }, + ).Once() +} + +func expectUserQuery(t *testing.T, mock *mocks.Client) { + mock.On("Query", m.Anything, m.AnythingOfType("*github.userQuery"), m.AnythingOfType("map[string]interface {}")).Return(nil).Run( + func(args m.Arguments) { + vars := args.Get(2).(map[string]interface{}) + ghost := githubv4.String("ghost") + require.Equal(t, ghost, vars["login"]) + + retVal := args.Get(1).(*userQuery) + retVal.User.Name = &ghost + retVal.User.Login = "ghost-login" + }, + ).Once() +} diff --git a/bridge/todosrht/import_test.go b/bridge/todosrht/import_test.go new file mode 100644 index 000000000..380c09a7c --- /dev/null +++ b/bridge/todosrht/import_test.go @@ -0,0 +1,242 @@ +package github + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/git-bug/git-bug/bridge/core" + "github.com/git-bug/git-bug/bridge/core/auth" + "github.com/git-bug/git-bug/cache" + "github.com/git-bug/git-bug/entities/bug" + "github.com/git-bug/git-bug/entities/common" + "github.com/git-bug/git-bug/entities/identity" + "github.com/git-bug/git-bug/entity/dag" + "github.com/git-bug/git-bug/repository" + "github.com/git-bug/git-bug/util/interrupt" +) + +func TestGithubImporter(t *testing.T) { + envToken := os.Getenv("GITHUB_TOKEN_PRIVATE") + if envToken == "" { + t.Skip("Env var GITHUB_TOKEN_PRIVATE missing") + } + + repo := repository.CreateGoGitTestRepo(t, false) + + backend, err := cache.NewRepoCacheNoEvents(repo) + require.NoError(t, err) + + defer backend.Close() + interrupt.RegisterCleaner(backend.Close) + + author, err := identity.NewIdentity(repo, "Michael Muré", "no-reply@git-bug.test") + require.NoError(t, err) + + complexIssueEditAuthor, err := identity.NewIdentity(repo, "sudoforge", "no-reply@git-bug.test") + require.NoError(t, err) + + tests := []struct { + name string + url string + bug *bug.Snapshot + }{ + { + name: "simple issue", + url: "https://github.com/git-bug/test-github-bridge/issues/1", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewAddCommentOp(author, 0, "second comment", nil), + }, + }, + }, + { + name: "empty issue", + url: "https://github.com/git-bug/test-github-bridge/issues/2", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "empty issue", "", nil), + }, + }, + }, + { + name: "complex issue", + url: "https://github.com/git-bug/test-github-bridge/issues/3", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "complex issue", "initial comment", nil), + bug.NewLabelChangeOperation(author, 0, []common.Label{"bug"}, []common.Label{}), + bug.NewLabelChangeOperation(author, 0, []common.Label{"duplicate"}, []common.Label{}), + bug.NewLabelChangeOperation(author, 0, []common.Label{}, []common.Label{"duplicate"}), + bug.NewAddCommentOp(author, 0, strings.Join([]string{ + "### header", + "**bold**", + "_italic_", + "> with quote", + "`inline code`", + "```\nmultiline code\n```", + "- bulleted\n- list", + "1. numbered\n1. list", + "- [ ] task\n- [x] list", + "@MichaelMure mention", + "#2 reference issue\n#3 auto-reference issue", + "![image](https://user-images.githubusercontent.com/294669/56870222-811faf80-6a0c-11e9-8f2c-f0beb686303f.png)", + }, "\n\n"), nil), + bug.NewEditCommentOp(complexIssueEditAuthor, 0, "", strings.Join([]string{ + "### header", + "**bold**", + "_italic_", + "> with quote", + "`inline code`", + "```\nmultiline code\n```", + "- bulleted\n- list", + "1. numbered\n1. list", + "- [ ] task\n- [x] list", + "@git-bug/maintainers mention", + "#2 reference issue\n#3 auto-reference issue", + "![image](https://user-images.githubusercontent.com/294669/56870222-811faf80-6a0c-11e9-8f2c-f0beb686303f.png)", + }, "\n\n"), nil), + bug.NewSetTitleOp(author, 0, "complex issue edited", "complex issue"), + bug.NewSetTitleOp(author, 0, "complex issue", "complex issue edited"), + bug.NewSetStatusOp(author, 0, common.ClosedStatus), + bug.NewSetStatusOp(author, 0, common.OpenStatus), + }, + }, + }, + { + name: "editions", + url: "https://github.com/git-bug/test-github-bridge/issues/4", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "editions", "initial comment edited", nil), + bug.NewEditCommentOp(author, 0, "", "erased then edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited", nil), + }, + }, + }, + { + name: "comment deletion", + url: "https://github.com/git-bug/test-github-bridge/issues/5", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "comment deletion", "", nil), + }, + }, + }, + { + name: "edition deletion", + url: "https://github.com/git-bug/test-github-bridge/issues/6", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "edition deletion", "initial comment", nil), + bug.NewEditCommentOp(author, 0, "", "initial comment edited again", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + bug.NewEditCommentOp(author, 0, "", "first comment edited again", nil), + }, + }, + }, + { + name: "hidden comment", + url: "https://github.com/git-bug/test-github-bridge/issues/7", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "hidden comment", "initial comment", nil), + bug.NewAddCommentOp(author, 0, "first comment", nil), + }, + }, + }, + { + name: "transferred issue", + url: "https://github.com/git-bug/test-github-bridge/issues/8", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "transfered issue", "", nil), + }, + }, + }, + { + name: "unicode control characters", + url: "https://github.com/git-bug/test-github-bridge/issues/10", + bug: &bug.Snapshot{ + Operations: []dag.Operation{ + bug.NewCreateOp(author, 0, "unicode control characters", "u0000: \nu0001: \nu0002: \nu0003: \nu0004: \nu0005: \nu0006: \nu0007: \nu0008: \nu0009: \t\nu0010: \nu0011: \nu0012: \nu0013: \nu0014: \nu0015: \nu0016: \nu0017: \nu0018: \nu0019:", nil), + }, + }, + }, + } + + login := "test-identity" + author.SetMetadata(metaKeyGithubLogin, login) + + token := auth.NewToken(target, envToken) + token.SetMetadata(auth.MetaKeyLogin, login) + err = auth.Store(repo, token) + require.NoError(t, err) + + ctx := context.Background() + + importer := &githubImporter{} + err = importer.Init(ctx, backend, core.Configuration{ + confKeyOwner: "git-bug", + confKeyProject: "test-github-bridge", + confKeyDefaultLogin: login, + }) + require.NoError(t, err) + + start := time.Now() + + events, err := importer.ImportAll(ctx, backend, time.Time{}) + require.NoError(t, err) + + for result := range events { + require.NoError(t, result.Err) + } + + fmt.Printf("test repository imported in %f seconds\n", time.Since(start).Seconds()) + + require.Len(t, backend.Bugs().AllIds(), len(tests)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := backend.Bugs().ResolveBugCreateMetadata(metaKeyGithubUrl, tt.url) + require.NoError(t, err) + + ops := b.Snapshot().Operations + require.Len(t, tt.bug.Operations, len(b.Snapshot().Operations)) + + for i, op := range tt.bug.Operations { + require.IsType(t, ops[i], op) + require.Equal(t, op.Author().Name(), ops[i].Author().Name()) + + switch op := op.(type) { + case *bug.CreateOperation: + require.Equal(t, op.Title, ops[i].(*bug.CreateOperation).Title) + require.Equal(t, op.Message, ops[i].(*bug.CreateOperation).Message) + case *bug.SetStatusOperation: + require.Equal(t, op.Status, ops[i].(*bug.SetStatusOperation).Status) + case *bug.SetTitleOperation: + require.Equal(t, op.Was, ops[i].(*bug.SetTitleOperation).Was) + require.Equal(t, op.Title, ops[i].(*bug.SetTitleOperation).Title) + case *bug.LabelChangeOperation: + require.ElementsMatch(t, op.Added, ops[i].(*bug.LabelChangeOperation).Added) + require.ElementsMatch(t, op.Removed, ops[i].(*bug.LabelChangeOperation).Removed) + case *bug.AddCommentOperation: + require.Equal(t, op.Message, ops[i].(*bug.AddCommentOperation).Message) + case *bug.EditCommentOperation: + require.Equal(t, op.Message, ops[i].(*bug.EditCommentOperation).Message) + + default: + panic("unknown operation type") + } + } + }) + } +} diff --git a/bridge/todosrht/mocks/Client.go b/bridge/todosrht/mocks/Client.go new file mode 100644 index 000000000..1270abb81 --- /dev/null +++ b/bridge/todosrht/mocks/Client.go @@ -0,0 +1,44 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + githubv4 "github.com/shurcooL/githubv4" + + mock "github.com/stretchr/testify/mock" +) + +// Client is an autogenerated mock type for the Client type +type Client struct { + mock.Mock +} + +// Mutate provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *Client) Mutate(_a0 context.Context, _a1 interface{}, _a2 githubv4.Input, _a3 map[string]interface{}) error { + ret := _m.Called(_a0, _a1, _a2, _a3) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, interface{}, githubv4.Input, map[string]interface{}) error); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Query provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Client) Query(_a0 context.Context, _a1 interface{}, _a2 map[string]interface{}) error { + ret := _m.Called(_a0, _a1, _a2) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, interface{}, map[string]interface{}) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} From 77131a9212e2e926df7bb671db545f385aea2033 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 13 Apr 2024 15:01:34 +0200 Subject: [PATCH 4/4] fix ssh authenticate --- repository/gogit.go | 82 ++++++++++++++++++++++++++++++++++++++-- repository/gogit_test.go | 46 ++++++++++++++++++++++ repository/mock_repo.go | 5 +++ repository/repo.go | 4 ++ 4 files changed, 133 insertions(+), 4 deletions(-) diff --git a/repository/gogit.go b/repository/gogit.go index c91179bfa..ec8ce968b 100644 --- a/repository/gogit.go +++ b/repository/gogit.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -20,6 +21,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "golang.org/x/sync/errgroup" "golang.org/x/sys/execabs" @@ -387,15 +389,32 @@ func (repo *GoGitRepo) FetchRefs(remote string, prefixes ...string) (string, err return "", err } - err = repo.r.Fetch(&gogit.FetchOptions{ + fetchOptions := &gogit.FetchOptions{ RemoteName: remote, RemoteURL: remoteUrl, RefSpecs: refSpecs, Progress: buf, - }) + } + + publicKeys, err := repo.SSHAuth(remote) + if err != nil { + return "", err + } + + err = repo.r.Fetch(fetchOptions) if err == gogit.NoErrAlreadyUpToDate { return "already up-to-date", nil } + // ssh-agent is required if ssh or scp-like url is configured in repository config + // we can not fetch if the ssh-agent has invalid keys or ssh-agent is not running + // retry to fetch again if we can retreive public keys from the users home directory + if err != nil && publicKeys != nil { + fetchOptions.Auth = publicKeys + err = repo.r.Fetch(fetchOptions) + if err == gogit.NoErrAlreadyUpToDate { + return "already up-to-date", nil + } + } if err != nil { return "", err } @@ -483,15 +502,32 @@ func (repo *GoGitRepo) PushRefs(remote string, prefixes ...string) (string, erro return "", err } - err = remo.Push(&gogit.PushOptions{ + pushOptions := &gogit.PushOptions{ RemoteName: remote, RemoteURL: remoteUrl, RefSpecs: refSpecs, Progress: buf, - }) + } + + publicKeys, err := repo.SSHAuth(remote) + if err != nil { + return "", err + } + + err = repo.r.Push(pushOptions) if err == gogit.NoErrAlreadyUpToDate { return "already up-to-date", nil } + // ssh-agent is required if ssh or scp-like url is configured in repository config + // we can not push if the ssh-agent has invalid keys or ssh-agent is not running + // retry to push again if we can retreive public keys from the users home directory + if err != nil && publicKeys != nil { + pushOptions.Auth = publicKeys + err = repo.r.Push(pushOptions) + if err == gogit.NoErrAlreadyUpToDate { + return "already up-to-date", nil + } + } if err != nil { return "", err } @@ -499,6 +535,44 @@ func (repo *GoGitRepo) PushRefs(remote string, prefixes ...string) (string, erro return buf.String(), nil } +// SSHAuth will attempt to read public keys for SSH auth +// if the repository remote contains a ssh or scp-like url +func (repo *GoGitRepo) SSHAuth(remote string) (*ssh.PublicKeys, error) { + // get the repository config + config, err := repo.r.Config() + if err != nil { + return nil, err + } + + // check if the repository config has at least one remote url + remotes, found := config.Remotes[remote] + if !found || len(remotes.URLs) < 1 { + return nil, fmt.Errorf("remote %s url not found in repository config", remote) + } + + schemeRegexp := regexp.MustCompile(`^[^:]+://`) + scpLikeRegexp := regexp.MustCompile(`^(?:(?P[^@]+)@)?(?P)`) + + // try to load public keys from the users home directory + // if the repository remote contains a ssh or scp-like url + if strings.HasPrefix(remotes.URLs[0], "ssh://") || (scpLikeRegexp.MatchString(remotes.URLs[0]) && !schemeRegexp.MatchString(remotes.URLs[0])) { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + // try to find and load valid public keys from the users home directory + attemptKeys := []string{"id_rsa", "id_ecdsa", "id_ecdsa_sk", "id_ed25519", "id_ed25519_sk", "id_xmss", "id_dsa"} + for _, key := range attemptKeys { + authMethod, err := ssh.NewPublicKeysFromFile("git", filepath.Join(home, ".ssh", key), "") + if err == nil { + return authMethod, nil + } + } + } + return nil, nil +} + // StoreData will store arbitrary data and return the corresponding hash func (repo *GoGitRepo) StoreData(data []byte) (Hash, error) { obj := repo.r.Storer.NewEncodedObject() diff --git a/repository/gogit_test.go b/repository/gogit_test.go index 7e1f6687f..ba72e9c43 100644 --- a/repository/gogit_test.go +++ b/repository/gogit_test.go @@ -2,6 +2,7 @@ package repository import ( "fmt" + "log" "os" "path" "path/filepath" @@ -96,3 +97,48 @@ func TestGoGit_DetectsSubmodules(t *testing.T) { assert.Empty(t, err) assert.Equal(t, expected, result) } + +func TestGoGitRepoSSH(t *testing.T) { + repo := CreateGoGitTestRepo(t, false) + + err := repo.AddRemote("ssh", "ssh://git@github.com:MichaelMure/git-bug.git") + if err != nil { + log.Fatal(err) + } + keys, err := repo.SSHAuth("ssh") + require.NotNil(t, keys) + require.Empty(t, err) + + err = repo.AddRemote("http", "http://github.com/MichaelMure/git-bug.git") + if err != nil { + log.Fatal(err) + } + keys, err = repo.SSHAuth("http") + require.Nil(t, keys) + require.Empty(t, err) + + err = repo.AddRemote("https", "https://github.com/MichaelMure/git-bug.git") + if err != nil { + log.Fatal(err) + } + keys, err = repo.SSHAuth("https") + require.Nil(t, keys) + require.Empty(t, err) + + err = repo.AddRemote("git", "git://github.com/MichaelMure/git-bug.git") + if err != nil { + log.Fatal(err) + } + keys, err = repo.SSHAuth("git") + require.Nil(t, keys) + require.Empty(t, err) + + err = repo.AddRemote("scp-like", "git@github.com:MichaelMure/git-bug.git") + if err != nil { + log.Fatal(err) + } + keys, err = repo.SSHAuth("scp-like") + require.NotNil(t, keys) + require.Empty(t, err) + +} diff --git a/repository/mock_repo.go b/repository/mock_repo.go index b9cbe138d..3f8555ee3 100644 --- a/repository/mock_repo.go +++ b/repository/mock_repo.go @@ -10,6 +10,7 @@ import ( "github.com/99designs/keyring" "github.com/ProtonMail/go-crypto/openpgp" "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/git-bug/git-bug/util/lamport" ) @@ -252,6 +253,10 @@ func (r *mockRepoData) PushRefs(remote string, prefixes ...string) (string, erro panic("implement me") } +func (r *mockRepoData) SSHAuth(remote string) (*ssh.PublicKeys, error) { + panic("implement me") +} + func (r *mockRepoData) StoreData(data []byte) (Hash, error) { rawHash := sha1.Sum(data) hash := Hash(fmt.Sprintf("%x", rawHash)) diff --git a/repository/repo.go b/repository/repo.go index 234742772..3876a42f5 100644 --- a/repository/repo.go +++ b/repository/repo.go @@ -7,6 +7,7 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/go-git/go-billy/v5" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/git-bug/git-bug/util/lamport" ) @@ -142,6 +143,9 @@ type RepoData interface { // the remote state. PushRefs(remote string, prefixes ...string) (string, error) + // SSHAuth will attempt to read public keys for SSH auth + SSHAuth(remote string) (*ssh.PublicKeys, error) + // StoreData will store arbitrary data and return the corresponding hash StoreData(data []byte) (Hash, error)