History

lots of fixes, but it's still broken because of filemarshal issues

fritz committed 14 years ago

Commit d85aa851 parent 7b395d07 files · +61 −71Browse files

Unified · Split

7 files
  1. common.go +22 −29
  2. main.go +17 −17
  3. master.go +3 −6
  4. mexec.go +10 −9
  5. misc.go +4 −5
  6. slave.go +2 −2
  7. web.go +3 −3
common.go+22−29
@@ -10,13 +10,13 @@ · 9 unmodified lines
1010 package main
1111
1212 import (
13− "os"
14− "net"
13+ "bitbucket.org/floren/filemarshal"
14+ "encoding/gob"
1515 "fmt"
16− "log"
1716 "io"
18− "gob"
19− "bitbucket.org/floren/filemarshal"
17+ "log"
18+ "net"
19+ "os"
2020 "strconv"
2121 "strings"
2222 "syscall"
@@ -30,7 +30,6 @@ func (s SlaveResp) String() string { · 7 unmodified lines
3030 return fmt.Sprint("id: ", s.Id)
3131 }
3232
33−
3433 type Resp struct {
3534 NumNodes int
3635 Msg string
@@ -52,7 +51,7 @@ type cmdToExec struct { · 15 unmodified lines
5251 DestName string
5352 SymlinkTarget string
5453 Local int
55− Fi *os.FileInfo
54+ Fi os.FileInfo
5655 }
5756
5857 func (a *cmdToExec) String() string {
@@ -72,7 +71,7 @@ type vitalData struct { · 13 unmodified lines
7271 HostAddr string
7372 ParentAddr string
7473 ServerAddr string
75− Id string
74+ Id string
7675 Nodes []string
7776 Exceptlist map[string]bool
7877 }
@@ -144,7 +143,6 @@ func (s *SlaveInfo) String() string { · 65 unmodified lines
144143 return fmt.Sprint(s.Id, "@", s.Addr)
145144 }
146145
147−
148146 func Dprint(level int, arg ...interface{}) {
149147 if *DebugLevel >= level {
150148 log.Print(arg...)
@@ -168,7 +166,6 @@ const ( · 17 unmodified lines
168166 Recv
169167 )
170168
171−
172169 func IoString(i interface{}, dir int) string {
173170 switch i.(type) {
174171 case net.Conn:
@@ -227,7 +224,7 @@ func (r *RpcClientServer) Send(funcname string, arg interface{}) { · 52 unmodified lines
227224
228225 var onRecvFunc func(funcname string, r io.Reader, arg interface{})
229226
230−func (r *RpcClientServer) Recv(funcname string, arg interface{}) (err os.Error) {
227+func (r *RpcClientServer) Recv(funcname string, arg interface{}) (err error) {
231228 err = r.D.Decode(arg)
232229 if err != nil {
233230 log.Print(funcname, ": Recv error: ", err)
@@ -242,10 +239,9 @@ func (r *RpcClientServer) Recv(funcname string, arg interface{}) (err os.Error) · 8 unmodified lines
242239 return
243240 }
244241
245−
246242 var onDialFunc func(fam, laddr, raddr string)
247243
248−func Dial(fam, laddr, raddr string) (c net.Conn, err os.Error) {
244+func Dial(fam, laddr, raddr string) (c net.Conn, err error) {
249245 if onDialFunc != nil {
250246 onDialFunc(fam, laddr, raddr)
251247 }
@@ -265,7 +261,6 @@ func Dial(fam, laddr, raddr string) (c net.Conn, err os.Error) { · 13 unmodified lines
265261 return
266262 }
267263
268−
269264 type Listener struct {
270265 l net.Listener
271266 }
@@ -276,7 +271,7 @@ func (l Listener) Addr() net.Addr { · 4 unmodified lines
276271
277272 var onListenFunc func(fam, laddr string)
278273
279−func Listen(fam, laddr string) (l Listener, err os.Error) {
274+func Listen(fam, laddr string) (l Listener, err error) {
280275 if onListenFunc != nil {
281276 onListenFunc(fam, laddr)
282277 }
@@ -287,7 +282,7 @@ func Listen(fam, laddr string) (l Listener, err os.Error) { · 4 unmodified lines
287282
288283 var onAcceptFunc func(c net.Conn)
289284
290−func (l Listener) Accept() (c net.Conn, err os.Error) {
285+func (l Listener) Accept() (c net.Conn, err error) {
291286 c, err = l.l.Accept()
292287 if err != nil {
293288 return
@@ -299,13 +294,12 @@ func (l Listener) Accept() (c net.Conn, err os.Error) { · 5 unmodified lines
299294 return
300295 }
301296
302−
303297 // depends on syscall
304298 func WaitAllChildren() {
305299 var status syscall.WaitStatus
306300 for {
307301 pid, err := syscall.Wait4(-1, &status, 0, nil)
308− if err <= 0 {
302+ if err != nil {
309303 break
310304 }
311305 log.Printf("wait4 returns pid %v status %v\n", pid, status)
@@ -318,7 +312,7 @@ func WaitAllChildren() { · 6 unmodified lines
318312 * This function builds up a list of files that need to go out to the current node's sub-nodes.
319313 * It is called by both the master and, if a more complex hierarchy is used, the upper-level slaves.
320314 */
321−func cacheRelayFilesAndDelegateExec(arg *StartReq, root, clientnode string) os.Error {
315+func cacheRelayFilesAndDelegateExec(arg *StartReq, root, clientnode string) error {
322316 Dprint(2, "cacheRelayFilesAndDelegateExec: files ", arg.Cmds, " nodes: ", clientnode, " fileServer: ", arg.Lfam, arg.Lserver)
323317
324318 larg := newStartReq(arg)
@@ -328,7 +322,7 @@ func cacheRelayFilesAndDelegateExec(arg *StartReq, root, clientnode string) os.E · 3 unmodified lines
328322 comesfrom := root + c.DestName
329323 Dprint(2, "current cmd comesfrom = ", comesfrom, ", DestName = ", c.DestName, ", CurrentName = ", c.CurrentName, ", SymlinkTarget = ", c.SymlinkTarget)
330324 f := new(filemarshal.File)
331− if c.Fi.IsRegular() || c.Fi.IsDirectory() || c.Fi.IsSymlink() {
325+ if !c.Fi.IsDir() || c.Fi.IsDir() || ((c.Fi.Mode() & os.ModeSymlink) != 0) {
332326 f = &filemarshal.File{CurrentName: comesfrom, Fi: *c.Fi, SymlinkTarget: c.SymlinkTarget, DestName: c.DestName}
333327 } else {
334328 continue
@@ -360,6 +354,7 @@ func cacheRelayFilesAndDelegateExec(arg *StartReq, root, clientnode string) os.E · 25 unmodified lines
360354
361355 return nil
362356 }
357+
363358 /*
364359 * The ioProxy listens for incoming connections. Sub-nodes will connect to it
365360 * and use the connection as stdin, stdout, and stderr for the programs they
@@ -371,7 +366,7 @@ func cacheRelayFilesAndDelegateExec(arg *StartReq, root, clientnode string) os.E · 5 unmodified lines
371366 * finished. workerChan will contain one int for every client which has
372367 * completed and disconnected.
373368 */
374−func ioProxy(fam, server string, dest io.Writer) (workerChan chan int, l Listener, err os.Error) {
369+func ioProxy(fam, server string, dest io.Writer) (workerChan chan int, l Listener, err error) {
375370 workerChan = make(chan int, 0)
376371 l, err = Listen(fam, server)
377372 if err != nil {
@@ -416,7 +411,7 @@ type nodeExecList struct { · 38 unmodified lines
416411 * the really hard way: few people understand regular expressions. So we
417412 * don't use them.
418413 */
419−func parseNodeList(l string) (rl []nodeExecList, err os.Error) {
414+func parseNodeList(l string) (rl []nodeExecList, err error) {
420415 /* bust it apart by , */
421416 ranges := strings.SplitN(l, ",", -1)
422417 for _, n := range ranges {
@@ -465,7 +460,7 @@ func doPrivateMount(pathbase string) { · 42 unmodified lines
465460 * Make a TCP connection to a server, return both the connection and
466461 * a File pointing to that connection.
467462 */
468−func fileTcpDial(server string) (*os.File, net.Conn, os.Error) {
463+func fileTcpDial(server string) (*os.File, net.Conn, error) {
469464 var laddr net.TCPAddr
470465 raddr, err := net.ResolveTCPAddr("tcp4", server)
471466 if err != nil {
@@ -503,13 +498,11 @@ func newStartReq(arg *StartReq) *StartReq { · 31 unmodified lines
503498 }
504499 }
505500
506−
507501 /*
508502 * Functions and data types for keeping track of slave nodes
509503 */
510504
511−
512−func registerSlaves() os.Error {
505+func registerSlaves() error {
513506 l, err := Listen(*defaultFam, myListenAddress)
514507 if err != nil {
515508 log.Fatal("listen error:", err)
@@ -584,8 +577,8 @@ func (sv *Slaves) Add(vd *vitalData, r *RpcClientServer) (resp SlaveResp) { · 68 unmodified lines
584577
585578 func (sv *Slaves) Remove(s *SlaveInfo) {
586579 Dprintln(4, "Remove %v ", s, " slave %v", sv.Slaves[s.Id])
587− sv.Slaves[s.Id] = nil, false
588− sv.Addr2id[s.Server] = "", false
580+ delete(sv.Slaves, s.Id)
581+ delete(sv.Addr2id, s.Server)
589582 Dprintln(2, "slave Remove: Id: ", s)
590583 return
591584 }
@@ -600,7 +593,7 @@ func (sv *Slaves) Remove(s *SlaveInfo) { · 8 unmodified lines
600593 func (sv *Slaves) Get(n string) (s *SlaveInfo, ok bool) {
601594 Dprint(6, "Get: ", n)
602595 s, ok = sv.Slaves[n]
603− if ! ok {
596+ if !ok {
604597 s, ok = sv.Slaves[sv.Addr2id[n]]
605598 }
606599 Dprint(6, " Returns: ", s)
main.go+17−17
@@ -10,14 +10,14 @@ · 9 unmodified lines
1010 package main
1111
1212 import (
13+ "bitbucket.org/rminnich/forth"
14+ "encoding/gob"
15+ "flag"
16+ "fmt"
1317 "log"
18+ "net/rpc"
1419 "os"
15− "rpc"
16− "fmt"
1720 "strconv"
18− "flag"
19− "gob"
20− "bitbucket.org/rminnich/forth"
2121 )
2222
2323 func usage() {
@@ -43,14 +43,14 @@ var ( · 19 unmodified lines
4343 defaultMasterUDS = flag.String("defaultMasterUDS", "/tmp/g", "Default Master Unix Domain Socket")
4444 ioProxyPort = flag.String("iopp", "0", "io proxy port")
4545 cmdPort = flag.String("cmdport", "6666", "command port")
46− defaultFam = flag.String("fam", "tcp4", "network type")
47− gprocBin = flag.String("gprocBin", "gproc", "name of gproc binary")
46+ defaultFam = flag.String("fam", "tcp4", "network type")
47+ gprocBin = flag.String("gprocBin", "gproc", "name of gproc binary")
4848 /* required in the command line */
49− parent = flag.String("parent", "hostname", "parent for some configurations")
49+ parent = flag.String("parent", "hostname", "parent for some configurations")
5050 myAddress = flag.String("myAddress", "hostname", "Required set to my address")
51− myId = flag.String("myId", "0", "Required -- tell slaves their id")
51+ myId = flag.String("myId", "0", "Required -- tell slaves their id")
5252 /* these are not switches */
53− role = "client"
53+ role = "client"
5454 myListenAddress string
5555 )
5656
@@ -63,9 +63,9 @@ var ( · 6 unmodified lines
6363 * You can just put a simple string in for the argument and it will push and pop it
6464 * ./gproc_linux_arm -parent='hostname base 7 roundup sb strcat 10.1.1.1 hostname base 7 % ifelse' -myId='hostname base 7 % 1 + hostname base 7 / hostname base 7 % ifelse' -myAddress=hostname s
6565 * is used for strongbox, where there is a single root, 49 level 1s defined by name %7 == 0, and
66− */
66+ */
6767 func main() {
68− var err os.Error
68+ var err error
6969 flag.Usage = usage
7070 flag.Parse()
7171 interp := forth.New()
@@ -115,11 +115,11 @@ func main() { · 43 unmodified lines
115115 info := getInfo(*defaultMasterUDS, flag.Arg(1))
116116 fmt.Print("Nodes:\n", info)
117117 /* not yet
118− case "EXCEPT", "except", "x":
119− loc.Init("init")
120− exceptOK := except(*defaultMasterUDS, flag.Args()[1:])
121− fmt.Print(exceptOK)
122− */
118+ case "EXCEPT", "except", "x":
119+ loc.Init("init")
120+ exceptOK := except(*defaultMasterUDS, flag.Args()[1:])
121+ fmt.Print(exceptOK)
122+ */
123123 case "R":
124124 /* This is for executing a program from the slave */
125125 slaveProc(NewRpcClientServer(os.Stdin, *binRoot), &RpcClientServer{E: gob.NewEncoder(os.Stdout), D: gob.NewDecoder(os.Stdout)}, &RpcClientServer{E: gob.NewEncoder(os.NewFile(3, "pipe")), D: gob.NewDecoder(os.NewFile(3, "pipe"))})
master.go+3−6
@@ -9,10 +9,7 @@ · 8 unmodified lines
99
1010 package main
1111
12−import (
13− "os"
14− "log"
15−)
12+import "log"
1613
1714 var (
1815 Workers []Worker
@@ -74,7 +71,7 @@ func sendCommandsToNodes(r *RpcClientServer, sendReq *StartReq, root string) (nu · 55 unmodified lines
7471 slaveNodes, err := parseNodeList(sendReq.Nodes)
7572 Dprint(2, "receiveCmds: sendReq.Nodes: ", sendReq.Nodes, " expands to ", slaveNodes)
7673 if err != nil {
77− r.Send("receiveCmds", Resp{NumNodes: 0, Msg: "startExecution: bad slaveNodeList: " + err.String()})
74+ r.Send("receiveCmds", Resp{NumNodes: 0, Msg: "startExecution: bad slaveNodeList: " + err.Error()})
7875 return
7976 }
8077 for _, aNode := range slaveNodes {
@@ -91,7 +88,7 @@ func sendCommandsToNodes(r *RpcClientServer, sendReq *StartReq, root string) (nu · 10 unmodified lines
9188 /*
9289 * The master sits in a loop listening for commands to come in over the Unix domain socket.
9390 */
94−func receiveCmds(domainSock string) os.Error {
91+func receiveCmds(domainSock string) error {
9592 vitalData := vitalData{HostAddr: "", HostReady: false, Error: "No hosts ready", Exceptlist: exceptFiles}
9693 l, err := Listen("unix", *defaultMasterUDS)
9794 if err != nil {
mexec.go+10−9
@@ -18,13 +18,14 @@ · 17 unmodified lines
1818 package main
1919
2020 import (
21+ "bitbucket.org/rminnich/ldd"
22+ "errors"
23+ "fmt"
2124 "log"
2225 "os"
23− "strings"
24− "bitbucket.org/rminnich/ldd"
2526 "path"
2627 "path/filepath"
27− "fmt"
28+ "strings"
2829 )
2930
3031 /*
@@ -136,7 +137,7 @@ func startExecution(masterAddr, fam, ioProxyPort, slaveNodes string, cmd []strin · 105 unmodified lines
136137 }
137138
138139 var (
139− BadRangeErr = os.NewError("bad range format")
140+ BadRangeErr = errors.New("bad range format")
140141 )
141142
142143 type packVisitor struct {
@@ -149,7 +150,7 @@ func newPackVisitor() (p *packVisitor) { · 6 unmodified lines
149150 return &packVisitor{alreadyVisited: make(map[string]bool)}
150151 }
151152
152−func (p *packVisitor) VisitDir(filePath string, f *os.FileInfo) bool {
153+func (p *packVisitor) VisitDir(filePath string, f os.FileInfo) bool {
153154 filePath = strings.TrimSpace(filePath)
154155 filePath = strings.TrimRightFunc(filePath, isNull)
155156
@@ -180,7 +181,7 @@ func isNull(r int) bool { · 24 unmodified lines
180181 return r == 0
181182 }
182183
183−func (p *packVisitor) VisitFile(filePath string, f *os.FileInfo) {
184+func (p *packVisitor) VisitFile(filePath string, f os.FileInfo) {
184185 // shouldn't need to do this, need to fix ldd
185186 filePath = strings.TrimSpace(filePath)
186187 filePath = strings.TrimRightFunc(filePath, isNull)
@@ -194,13 +195,13 @@ func (p *packVisitor) VisitFile(filePath string, f *os.FileInfo) { · 7 unmodified lines
194195 Local: 0,
195196 Fi: f,
196197 }
197− Dprint(4, "VisitFile: appending ", f.Name, " ", f.Size, " ", []byte(filePath), " ", p.alreadyVisited)
198+ Dprint(4, "VisitFile: appending ", f.Name(), " ", f.Size(), " ", []byte(filePath), " ", p.alreadyVisited)
198199
199200 p.cmds = append(p.cmds, c)
200201
201202 switch {
202− case f.IsRegular():
203− p.bytesToTransfer += f.Size
203+ case !f.IsDir():
204+ p.bytesToTransfer += f.Size()
204205 case f.IsSymlink():
205206 /* we have to read the link but also get a path the file for
206207 * further walking. We think.
misc.go+4−5
@@ -31,11 +31,10 @@ import ( · 30 unmodified lines
3131 // return cmds
3232 // }
3333
34−
35−func readitin(s, root string) ([]byte, os.FileInfo, os.Error) {
34+func readitin(s, root string) ([]byte, os.FileInfo, error) {
3635 fi, _ := os.Stat(root + s)
3736 f, _ := os.Open(s)
38− bytes := make([]byte, fi.Size)
37+ bytes := make([]byte, fi.Size())
3938 f.Read(bytes)
4039 return bytes, *fi, nil
4140 }
@@ -44,12 +43,12 @@ type Arg struct { · 2 unmodified lines
4443 Msg []byte
4544 }
4645
47−func Ping(arg *Arg, resp *Resp) os.Error {
46+func Ping(arg *Arg, resp *Resp) error {
4847 resp.Msg = string(arg.Msg)
4948 return nil
5049 }
5150
52−func Debug(arg *SetDebugLevel, resp *SetDebugLevel) os.Error {
51+func Debug(arg *SetDebugLevel, resp *SetDebugLevel) error {
5352 resp.level = *DebugLevel
5453 *DebugLevel = arg.level
5554 return nil
slave.go+2−2
@@ -15,9 +15,9 @@ import ( · 14 unmodified lines
1515 "os"
1616 "net"
1717 "fmt"
18− "gob"
18+ "encoding/gob"
1919 "time"
20− "rand"
20+ "math/rand"
2121 )
2222
2323 var id string
web.go+3−3
@@ -10,13 +10,13 @@ · 9 unmodified lines
1010 package main
1111
1212 import (
13− "http"
13+ "net/http"
1414 "io"
1515 "log"
1616 "old/template"
17− "exec"
17+ "os/exec"
1818 "os"
19− "url"
19+ "net/url"
2020 )
2121
2222 var fmap = template.FormatterMap{