-
Notifications
You must be signed in to change notification settings - Fork 26
Expand file tree
/
Copy pathrunTransfer.go
More file actions
145 lines (122 loc) · 3.7 KB
/
Copy pathrunTransfer.go
File metadata and controls
145 lines (122 loc) · 3.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
package main
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
)
func handleUpdates(
cancel context.CancelFunc,
statusChannel <-chan string,
errorChannel <-chan error,
transfer Transfer,
) {
// all transfer updates are sent via channel - this function handles those updates
exit := false
for {
select {
case ip := <-transfer.CancelChannel:
cancel()
transfer.Status = StatusCancelled
transfer.StoppedAt = fmt.Sprint(time.Now())
infoLog.Printf("transfer %v cancelled by ip %v", transfer.Id, ip)
exit = true
defer func() {
transfer.CancelledChannel <- true
}()
case err := <-errorChannel:
cancel()
transfer.Status = StatusError
transfer.StoppedAt = fmt.Sprint(time.Now())
transfer.Err = err.Error()
errorLog.Printf("error running transfer %v :: %v", transfer.Id, err)
exit = true
case status := <-statusChannel:
transfer.Status = status
infoLog.Printf("transfer %v is now %v", transfer.Id, status)
}
transferMap.Set(transfer.Id, transfer)
if exit {
return
}
}
}
func runTransfer(ctx context.Context, cancel context.CancelFunc, transfer Transfer) {
statusChannel := make(chan string)
errorChannel := make(chan error)
go handleUpdates(cancel, statusChannel, errorChannel, transfer)
statusChannel <- StatusRunning
// create temp directories
err := os.MkdirAll(transfer.TmpDir, 0600)
if err != nil {
errorChannel <- fmt.Errorf("error creating temp dir :: %v", err)
}
defer func() {
if !transfer.KeepFiles {
os.RemoveAll(transfer.TmpDir)
infoLog.Printf("temp dir %v removed", transfer.TmpDir)
}
}()
infoLog.Printf("temp dir %v created", transfer.TmpDir)
transfer.PipeFileDir = filepath.Join(transfer.TmpDir, "pipe-files")
err = os.MkdirAll(transfer.PipeFileDir, 0600)
if err != nil {
errorChannel <- fmt.Errorf("error creating pipe file dir :: %v", err)
}
infoLog.Printf("pipe file dir %v created", transfer.PipeFileDir)
transfer.FinalCsvDir = filepath.Join(transfer.TmpDir, "final-csv")
err = os.MkdirAll(transfer.FinalCsvDir, 0600)
if err != nil {
errorChannel <- fmt.Errorf("error creating final csv dir :: %v", err)
}
infoLog.Printf("final csv dir %v created", transfer.FinalCsvDir)
source, err := newSystem(
transfer.SourceName, transfer.SourceType, transfer.SourceConnectionString,
)
if err != nil {
errorChannel <- fmt.Errorf("error creating source system :: %v", err)
return
}
target, err := newSystem(
transfer.TargetName, transfer.TargetType, transfer.TargetConnectionString,
)
if err != nil {
errorChannel <- fmt.Errorf("error creating target system :: %v", err)
return
}
if transfer.DropTargetTableIfExists {
dropped, err := target.dropTableIfExists(transfer)
if err != nil {
errorChannel <- fmt.Errorf("error dropping target table :: %v", err)
return
}
infoLog.Printf("transfer %v dropped %v if exists", transfer.Id, dropped)
}
rows, err := source.query(transfer.Query)
if err != nil {
errorChannel <- fmt.Errorf("error querying source :: %v", err)
return
}
defer rows.Close()
columnInfo, err := source.getColumnInfo(rows)
if err != nil {
errorChannel <- fmt.Errorf("error getting column info :: %v", err)
return
}
if transfer.CreateTargetTable {
created, err := target.createTable(columnInfo, transfer)
if err != nil {
errorChannel <- fmt.Errorf("error creating target table :: %v", err)
return
}
infoLog.Printf("transfer %v created %v", transfer.Id, created)
}
pipeFileChannel := source.createPipeFiles(ctx, errorChannel, columnInfo, transfer, rows)
err = target.insertPipeFiles(ctx, pipeFileChannel, errorChannel, columnInfo, transfer)
if err != nil {
errorChannel <- fmt.Errorf("error inserting pipe files :: %v", err)
return
}
statusChannel <- StatusComplete
}