Unified · Split
| @@ -24,6 +24,7 @@ · 23 unmodified lines | ||
| 24 | 24 | #include <libsec.h> |
| 25 | 25 | #include <stdio.h> |
| 26 | 26 | #include "debug.h" |
| 27 | +#include "timeit.h" | |
| 27 | 28 | |
| 28 | 29 | int remote_override = 0; |
| 29 | 30 | int remote = 0; |
| @@ -534,7 +535,7 @@ static int · 504 unmodified lines | ||
| 534 | 535 | checkmount(char *addr) |
| 535 | 536 | { |
| 536 | 537 | char *dest = estrdup9p(addr); |
| 537 | − int retries = 0; | |
| 538 | + int retries; | |
| 538 | 539 | int err = 0; |
| 539 | 540 | int fd; |
| 540 | 541 | char *srvpt = smprint("/srv/%s", addr); |
| @@ -568,10 +569,31 @@ checkmount(char *addr) · 27 unmodified lines | ||
| 568 | 569 | |
| 569 | 570 | /* no dir, spawn off an import */ |
| 570 | 571 | DPRINT(DNET, "\tno dir, spawn off an import +++\n"); |
| 572 | +/* | |
| 573 | + for (retries=0, err=-1; retries<=5000; retries++){ | |
| 574 | + if(0==retries%1000) | |
| 575 | + proccreate(startimport, dest, STACK); | |
| 576 | + | |
| 577 | + // we need to wait for this to be done | |
| 578 | + if((tmp = dirstat(mtpt)) == nil){ | |
| 579 | + err = 0; | |
| 580 | + break; | |
| 581 | + } | |
| 582 | + | |
| 583 | + // ugh, arbitrary delay | |
| 584 | + sleep(10); | |
| 585 | + } | |
| 586 | + if(err) | |
| 587 | + DPRINT(DERR, "*ERROR*: checkmount: import for (%s) not complete after 10 seconds, giving up\n", dest); | |
| 588 | +*/ | |
| 589 | + | |
| 590 | + | |
| 591 | + | |
| 592 | +for (retries=0, tmp=nil; retries<=5 && tmp==nil; retries++){ | |
| 571 | 593 | proccreate(startimport, dest, STACK); |
| 572 | 594 | |
| 573 | − /* we need to wait for this to be done */ | |
| 574 | − while((tmp = dirstat(mtpt)) == nil) { /* ugh, arbitrary 100ms delay */ | |
| 595 | + // we need to wait for this to be done | |
| 596 | + while((tmp = dirstat(mtpt)) == nil) { // ugh, arbitrary 100ms delay | |
| 575 | 597 | if(retries++ > 1009) { |
| 576 | 598 | DPRINT(DERR, "*ERROR*: checkmount: import for (%s) not complete after 10 seconds, giving up\n", dest); |
| 577 | 599 | err = -1; |
| @@ -579,6 +601,7 @@ checkmount(char *addr) · 1 unmodified line | ||
| 579 | 601 | } |
| 580 | 602 | sleep(10); |
| 581 | 603 | } |
| 604 | +} | |
| 582 | 605 | |
| 583 | 606 | out: |
| 584 | 607 | if(srvpt) |
| @@ -993,6 +1016,9 @@ releasegang(Gang *g) · 408 unmodified lines | ||
| 993 | 1016 | } |
| 994 | 1017 | |
| 995 | 1018 | wunlock(&glock); |
| 1019 | + | |
| 1020 | + // FIXME: dump the time stamps | |
| 1021 | + timeit_dump(); | |
| 996 | 1022 | } |
| 997 | 1023 | |
| 998 | 1024 | static void |
| @@ -1328,7 +1354,9 @@ setupstdio(Session *s) · 329 unmodified lines | ||
| 1328 | 1354 | |
| 1329 | 1355 | snprint(dest, 255, "%s/g%d/%d", gangpath, g->index, s->index); |
| 1330 | 1356 | |
| 1357 | +if(!s->remote)stamp("Before local bind"); | |
| 1331 | 1358 | n = bind(buf, dest, MREPL); |
| 1359 | +if(!s->remote)timeit_tag_dt_float("Before local bind", "Time for local bind"); | |
| 1332 | 1360 | DPRINT(DEXE, "\tbind buf=(%s) dest=(%s) ret=%d\n", buf, dest, n); |
| 1333 | 1361 | if(n < 0) |
| 1334 | 1362 | return smprint("couldn't bind %s %s: %r\n", buf, dest); |
| @@ -1341,7 +1369,9 @@ setupstdio(Session *s) · 6 unmodified lines | ||
| 1341 | 1369 | |
| 1342 | 1370 | snprint(dest, 255, "%s/g%d/stdin", xgangpath, g->index); |
| 1343 | 1371 | DPRINT(DEXE, "\tsplicefrom buf=(%s) dest=(%s)\n", buf, dest); |
| 1372 | +if(!s->remote)stamp("Before local splicefrom/stdin"); | |
| 1344 | 1373 | n = splicefrom(buf, dest); /* execfs initiates splicefrom gangfs */ |
| 1374 | +if(!s->remote)timeit_tag_dt_float("Before local splicefrom/stdin", "Time for local splicefrom/stdin"); | |
| 1345 | 1375 | DPRINT(DEXE, "\t\tret=%d\n", n); |
| 1346 | 1376 | if(n < 0) |
| 1347 | 1377 | return smprint("setupstdio: %r\n"); |
| @@ -1354,7 +1384,9 @@ setupstdio(Session *s) · 6 unmodified lines | ||
| 1354 | 1384 | |
| 1355 | 1385 | snprint(dest, 255, "%s/g%d/stdout", xgangpath, g->index); |
| 1356 | 1386 | DPRINT(DEXE, "\tspliceto buf=(%s) dest=(%s)\n", buf, dest); |
| 1387 | +if(!s->remote)stamp("Before local spliceto/stdout"); | |
| 1357 | 1388 | n = spliceto(buf, dest); /* execfs initiates spliceto gangfs stdout */ |
| 1389 | +if(!s->remote)timeit_tag_dt_float("Before local spliceto/stdout", "Time for local spliceto/stdout"); | |
| 1358 | 1390 | DPRINT(DEXE, "\t\tret=%d\n", n); |
| 1359 | 1391 | if(n < 0) |
| 1360 | 1392 | return smprint("setupstdio: %r\n"); |
| @@ -1367,11 +1399,14 @@ setupstdio(Session *s) · 6 unmodified lines | ||
| 1367 | 1399 | |
| 1368 | 1400 | snprint(dest, 255, "%s/g%d/stderr", xgangpath, g->index); |
| 1369 | 1401 | DPRINT(DEXE, "\tspliceto buf=(%s) dest=(%s)\n", buf, dest); |
| 1402 | +if(!s->remote)stamp("Before local spliceto/stderr"); | |
| 1370 | 1403 | n = spliceto(buf, dest); /* execfs initiates spliceto gangfs stderr */ |
| 1404 | +if(!s->remote)timeit_tag_dt_float("Before local spliceto/stderr", "Time for local spliceto/stderr"); | |
| 1371 | 1405 | DPRINT(DEXE, "\t\tret=%d\n", n); |
| 1372 | 1406 | if(n < 0) |
| 1373 | 1407 | return smprint("setupstdio: %r\n"); |
| 1374 | 1408 | |
| 1409 | +if(!s->remote)timeit_tag_dt_float("Before local bind", "Time for local setupstdio"); | |
| 1375 | 1410 | return nil; |
| 1376 | 1411 | } |
| 1377 | 1412 | |
| @@ -1386,22 +1421,27 @@ cloneproc(void *arg) · 8 unmodified lines | ||
| 1386 | 1421 | int n = -1; |
| 1387 | 1422 | char *err; |
| 1388 | 1423 | |
| 1389 | − if(sess->remote) | |
| 1424 | + if(sess->remote){ | |
| 1390 | 1425 | snprint(buf, 255, "%s/gclone", sess->path); |
| 1391 | − else | |
| 1426 | +//stamp("gclone"); | |
| 1427 | + }else{ | |
| 1392 | 1428 | snprint(buf, 255, "%s/clone", sess->path); |
| 1429 | +stamp("clone"); | |
| 1430 | + } | |
| 1393 | 1431 | |
| 1394 | 1432 | DPRINT(DEXE, "cloneproc: **** clone proc %s (remote=%d)\n", |
| 1395 | 1433 | buf, sess->remote); |
| 1396 | 1434 | |
| 1397 | 1435 | /* FUTURE: This shouldn't be necessary with exec2fs */ |
| 1398 | − for(retries=0; retries <= 50; retries++) { | |
| 1436 | +if(!sess->remote)stamp("Before sess"); | |
| 1437 | + for(retries=0; retries <= 10; retries++) { | |
| 1399 | 1438 | sess->fd = open(buf, ORDWR); |
| 1400 | 1439 | |
| 1401 | 1440 | if(sess->fd <= 0) { |
| 1402 | − sleep(100); | |
| 1403 | − DPRINT(DERR, "WARNING: retry: %d on %d:%d opening exec2fs returned %d: %r -- retrying\n", | |
| 1404 | − retries, sess->index, sess->subindex, sess->fd); | |
| 1441 | +stamp("sess sleep1"); | |
| 1442 | + sleep(0); | |
| 1443 | + DPRINT(DERR, "WARNING: retry: %d opening execfs returned %d: %r -- retrying\n", | |
| 1444 | + retries, sess->fd); | |
| 1405 | 1445 | continue; |
| 1406 | 1446 | } |
| 1407 | 1447 | |
| @@ -1410,13 +1450,16 @@ cloneproc(void *arg) · 2 unmodified lines | ||
| 1410 | 1450 | buf[n] = 0; |
| 1411 | 1451 | break; |
| 1412 | 1452 | } |
| 1453 | +stamp("sess sleep2"); | |
| 1413 | 1454 | |
| 1414 | − sleep(100); | |
| 1415 | − DPRINT(DERR, "WARNING: retry: %d exec2fs's pread=(%d) failed for fd=(%d): %r -- retrying\n", | |
| 1455 | + sleep(0); | |
| 1456 | + DPRINT(DERR, "WARNING: retry: %d execfs's pread=(%d) failed for fd=(%d): %r -- retrying\n", | |
| 1416 | 1457 | retries, n, sess->fd); |
| 1417 | 1458 | close(sess->fd); |
| 1418 | 1459 | } |
| 1419 | 1460 | |
| 1461 | +if(!sess->remote)timeit_tag_dt_float("Before sess", "Time for local sess"); | |
| 1462 | + | |
| 1420 | 1463 | DPRINT(DEXE, "\tcontrol channel on sess->fd=(%d) for subsession (%d) pread=(%d)\n", |
| 1421 | 1464 | sess->fd, sess->subindex, n); |
| 1422 | 1465 | |
| @@ -1441,6 +1484,7 @@ cloneproc(void *arg) · 18 unmodified lines | ||
| 1441 | 1484 | DPRINT(DEXE, "cloneproc: sess->pid=%ld buf=(%s)\n", sess->pid, buf); |
| 1442 | 1485 | |
| 1443 | 1486 | /* if we are a remote session, setup backmount */ |
| 1487 | +//stamp("Before remote"); | |
| 1444 | 1488 | if(sess->remote) { |
| 1445 | 1489 | DPRINT(DEXE, "Attempting to remote checkmount %s on %s\n", |
| 1446 | 1490 | mysysname, sess->path); |
| @@ -1472,6 +1516,7 @@ cloneproc(void *arg) · 25 unmodified lines | ||
| 1472 | 1516 | } |
| 1473 | 1517 | |
| 1474 | 1518 | /* finish by setting up stdio */ |
| 1519 | +if(!sess->remote)stamp("Before local setupstdio"); | |
| 1475 | 1520 | if(err = setupstdio(sess)) { |
| 1476 | 1521 | DPRINT(DEXE, "reporting failure from stdio setup session %d:%d: %s\n", |
| 1477 | 1522 | sess->index, sess->subindex, err); |
| @@ -1480,6 +1525,7 @@ cloneproc(void *arg) · 2 unmodified lines | ||
| 1480 | 1525 | DPRINT(DEXE, "reporting success from session %d:%d\n", sess->index, sess->subindex); |
| 1481 | 1526 | sendp(sess->chan, nil); |
| 1482 | 1527 | } |
| 1528 | +if(!sess->remote)timeit_tag_dt_float("Before local setupstdio", "Time for local setupstdio outer"); | |
| 1483 | 1529 | |
| 1484 | 1530 | error: |
| 1485 | 1531 | threadexits(nil); |
| @@ -1525,9 +1571,13 @@ reslocal(Gang *g) · 39 unmodified lines | ||
| 1525 | 1571 | g->sess = emalloc9p(sizeof(Session)*g->size); |
| 1526 | 1572 | |
| 1527 | 1573 | for(count = 0; count < g->size; count++) { |
| 1574 | +stamp("Before local setupsession"); | |
| 1528 | 1575 | setupsess(g, &g->sess[count], nil, 0, count); |
| 1576 | +timeit_tag_dt_float("Before local setupsession", "Time for local setupsession"); | |
| 1529 | 1577 | DPRINT(DEXE, "reslocal: reservation: count=%d out of %d\n", count+1, g->size); |
| 1578 | +stamp("Before local cloneproc"); | |
| 1530 | 1579 | proccreate(cloneproc, &g->sess[count], STACK); |
| 1580 | +timeit_tag_dt_float("Before local cloneproc", "Time for local cloneproc"); | |
| 1531 | 1581 | } |
| 1532 | 1582 | |
| 1533 | 1583 | /* |
| @@ -1538,15 +1588,19 @@ reslocal(Gang *g) · 4 unmodified lines | ||
| 1538 | 1588 | * |
| 1539 | 1589 | */ |
| 1540 | 1590 | |
| 1591 | +stamp("Before local recvp"); | |
| 1541 | 1592 | for(count = 0; count < g->size; count++) { |
| 1542 | 1593 | char *myerr; |
| 1543 | 1594 | DPRINT(DEXE, "\twaiting on reservation: count=%d out of %d\n", |
| 1544 | 1595 | count+1, g->size); |
| 1596 | +stamp("local recvp"); | |
| 1545 | 1597 | myerr = recvp(g->sess[count].chan); |
| 1598 | +timeit_tag_dt_float("local recvp", "Time for single local recvp"); | |
| 1546 | 1599 | if(myerr != nil) |
| 1547 | 1600 | err = myerr; |
| 1548 | 1601 | /* MAYBE: retry here? for reliability ? */ |
| 1549 | 1602 | } |
| 1603 | +timeit_tag_dt_float("Before local recvp", "Time for local recvp total"); | |
| 1550 | 1604 | |
| 1551 | 1605 | return err; |
| 1552 | 1606 | } |
| @@ -1582,6 +1636,7 @@ resgang(Gang *g) · 29 unmodified lines | ||
| 1582 | 1636 | remaining = size; |
| 1583 | 1637 | /* walk children tree, allocating cores and incrementing stats.njobs |
| 1584 | 1638 | counting sessions and keeping children per subsession in an array? */ |
| 1639 | +//stamp("Before walk"); | |
| 1585 | 1640 | for(current=mystats.next; current !=nil; current = current->next) { |
| 1586 | 1641 | int avail = current->ntask - current->njobs; |
| 1587 | 1642 | if(avail >= remaining) { |
| @@ -1596,7 +1651,11 @@ resgang(Gang *g) · 8 unmodified lines | ||
| 1596 | 1651 | if(njobs[scount] > 0) /* only increment when we've allocated a subnode */ |
| 1597 | 1652 | scount++; |
| 1598 | 1653 | } |
| 1599 | − | |
| 1654 | +//stamp("After walk"); | |
| 1655 | +//timeit_settype(TIMIT_FLOAT); | |
| 1656 | +//timeit_tag_dt("Before walk", "After walk", "Time for walk"); | |
| 1657 | +//timeit_settype(TIMIT_DEFAULT); | |
| 1658 | + | |
| 1600 | 1659 | scount++; /* count instead of index now */ |
| 1601 | 1660 | |
| 1602 | 1661 | /* allocate subsessions */ |
| @@ -1606,23 +1665,29 @@ resgang(Gang *g) · 3 unmodified lines | ||
| 1606 | 1665 | DPRINT(DGAN, "Gang opening up %d sessions\n", scount); |
| 1607 | 1666 | |
| 1608 | 1667 | current = mystats.next; |
| 1668 | +stamp("Before loop"); | |
| 1609 | 1669 | for(count = 0; count < scount; count++) { |
| 1610 | − // FIXME: try moving checkmount to initstatus | |
| 1611 | − // checkmount(current->name); | |
| 1670 | +//stamp("Before checkmount"); | |
| 1671 | + checkmount(current->name); | |
| 1672 | +//timeit_dt_last_float("Time for checkmount"); | |
| 1612 | 1673 | |
| 1613 | 1674 | if(njobs[count] == 0) |
| 1614 | 1675 | continue; |
| 1615 | 1676 | DPRINT(DGAN, "resgang: Session %d Target %s Njobs: %d From %s\n", |
| 1616 | 1677 | count, current->name, njobs[count], mysysname); |
| 1617 | 1678 | |
| 1679 | +//stamp("Before setupsess"); | |
| 1618 | 1680 | setupsess(g, &g->sess[count], current->name, njobs[count], count); |
| 1619 | − | |
| 1681 | +//timeit_dt_last_float("Time for setupsess"); | |
| 1682 | + | |
| 1620 | 1683 | proccreate(cloneproc, &g->sess[count], STACK); |
| 1621 | 1684 | current = current->next; |
| 1622 | 1685 | } |
| 1686 | +timeit_tag_dt_float("Before loop", "Time for loop"); | |
| 1623 | 1687 | |
| 1624 | 1688 | DPRINT(DGAN, "waiting on %d sessions\n", scount); |
| 1625 | 1689 | |
| 1690 | +stamp("Before loop2"); | |
| 1626 | 1691 | for(count = 0; count < scount; count++) { |
| 1627 | 1692 | char *myerr; |
| 1628 | 1693 | DPRINT(DGAN, "\twaiting on reservation: count=%d out of %d\n", count+1, scount); |
| @@ -1636,6 +1701,7 @@ resgang(Gang *g) · 7 unmodified lines | ||
| 1636 | 1701 | g->status = GANG_BROKE; |
| 1637 | 1702 | } |
| 1638 | 1703 | } |
| 1704 | +timeit_dt_last_float("Time for loop2"); | |
| 1639 | 1705 | |
| 1640 | 1706 | return err; |
| 1641 | 1707 | } |
| @@ -1687,6 +1753,7 @@ cmdres(Req *r, int num) · 45 unmodified lines | ||
| 1687 | 1753 | respond(r, Ebadimode); |
| 1688 | 1754 | goto cleanuperror; |
| 1689 | 1755 | }; |
| 1756 | +stamp("Before mpipes"); | |
| 1690 | 1757 | if (mpipe(dest, "stdout") < 0) { |
| 1691 | 1758 | respond(r, Estdout); |
| 1692 | 1759 | goto cleanuperror; |
| @@ -1695,14 +1762,20 @@ cmdres(Req *r, int num) · 2 unmodified lines | ||
| 1695 | 1762 | respond(r, Estderr); |
| 1696 | 1763 | goto cleanuperror; |
| 1697 | 1764 | } |
| 1765 | +timeit_dt_last_float("Time for mpipes"); | |
| 1698 | 1766 | |
| 1699 | 1767 | g->size = num; |
| 1700 | 1768 | |
| 1701 | 1769 | /* this assumes top down reservation model */ |
| 1702 | − if(mystats.nchild) | |
| 1770 | + if(mystats.nchild){ | |
| 1771 | + stamp("Before resgang"); | |
| 1703 | 1772 | err = resgang(g); |
| 1704 | − else | |
| 1773 | +timeit_tag_dt_float("Before resgang", "Time for RES gang"); | |
| 1774 | + }else{ | |
| 1775 | +stamp("Before reslocal"); | |
| 1705 | 1776 | err = reslocal(g); |
| 1777 | +timeit_tag_dt_float("Before reslocal", "Time for RES local"); | |
| 1778 | + } | |
| 1706 | 1779 | |
| 1707 | 1780 | if(err) { |
| 1708 | 1781 | DPRINT(DERR, "*ERROR*: problem establishing gang reservation: %s\n", err); |
| @@ -2159,6 +2232,9 @@ threadmain(int argc, char **argv) · 450 unmodified lines | ||
| 2159 | 2232 | proccreate(updatestatus, (void *)parent, STACK); |
| 2160 | 2233 | } |
| 2161 | 2234 | |
| 2235 | + timeit_init(); | |
| 2236 | + //timeit_setsilent(1); | |
| 2237 | + | |
| 2162 | 2238 | /* spawn off a io thread */ |
| 2163 | 2239 | iochan = chancreate(sizeof(void *), 0); |
| 2164 | 2240 | clunkchan = chancreate(sizeof(void *), 0); |