Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Experimental option --fast on the clone command only clones manifest artifacts and related tags. This is an experiment to see how much bandwidth and time are saved by omitting the download of file content until it is actually needed. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | fast-clone |
Files: | files | file ages | folders |
SHA3-256: |
9e8a7d4ca85aaf6b1e19badbf96d802b |
User & Date: | drh 2018-06-04 16:27:52 |
Context
2018-06-04
| ||
16:27 | Experimental option --fast on the clone command only clones manifest artifacts and related tags. This is an experiment to see how much bandwidth and time are saved by omitting the download of file content until it is actually needed. Leaf check-in: 9e8a7d4ca8 user: drh tags: fast-clone | |
14:47 | Update the sync protocol document. check-in: 55cd6153de user: drh tags: trunk | |
Changes
Changes to src/clone.c.
︙ | ︙ | |||
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | ** ** By default, your current login name is used to create the default ** admin user. This can be overridden using the -A|--admin-user ** parameter. ** ** Options: ** --admin-user|-A USERNAME Make USERNAME the administrator ** --once Don't remember the URI. ** --private Also clone private branches ** --ssl-identity FILENAME Use the SSL identity if requested by the server ** --ssh-command|-c SSH Use SSH as the "ssh" command ** --httpauth|-B USER:PASS Add HTTP Basic Authorization to requests ** -u|--unversioned Also sync unversioned content ** -v|--verbose Show more statistics in output ** ** See also: init */ void clone_cmd(void){ char *zPassword; const char *zDefaultUser; /* Optional name of the default user */ const char *zHttpAuth; /* HTTP Authorization user:pass information */ int nErr = 0; int urlFlags = URL_PROMPT_PW | URL_REMEMBER; int syncFlags = SYNC_CLONE; /* Also clone private branches */ if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE; if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER; if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE; if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED; zHttpAuth = find_option("httpauth","B",1); zDefaultUser = find_option("admin-user","A",1); clone_ssh_find_options(); url_proxy_options(); | > > | 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | ** ** By default, your current login name is used to create the default ** admin user. This can be overridden using the -A|--admin-user ** parameter. ** ** Options: ** --admin-user|-A USERNAME Make USERNAME the administrator ** --fast Minimal clone - most content left on server ** --once Don't remember the URI. ** --private Also clone private branches ** --ssl-identity FILENAME Use the SSL identity if requested by the server ** --ssh-command|-c SSH Use SSH as the "ssh" command ** --httpauth|-B USER:PASS Add HTTP Basic Authorization to requests ** -u|--unversioned Also sync unversioned content ** -v|--verbose Show more statistics in output ** ** See also: init */ void clone_cmd(void){ char *zPassword; const char *zDefaultUser; /* Optional name of the default user */ const char *zHttpAuth; /* HTTP Authorization user:pass information */ int nErr = 0; int urlFlags = URL_PROMPT_PW | URL_REMEMBER; int syncFlags = SYNC_CLONE; /* Also clone private branches */ if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE; if( find_option("fast",0,0)!=0 ) syncFlags |= SYNC_ONLY_CKIN; if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER; if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE; if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED; zHttpAuth = find_option("httpauth","B",1); zDefaultUser = find_option("admin-user","A",1); clone_ssh_find_options(); url_proxy_options(); |
︙ | ︙ |
Changes to src/xfer.c.
︙ | ︙ | |||
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | int nFileRcvd; /* Number of files received */ int nDeltaRcvd; /* Number of deltas received */ int nDanglingFile; /* Number of dangling deltas received */ int mxSend; /* Stop sending "file" when pOut reaches this size */ int resync; /* Send igot cards for all holdings */ u8 syncPrivate; /* True to enable syncing private content */ u8 nextIsPrivate; /* If true, next "file" received is a private */ u32 clientVersion; /* Version of the client software */ time_t maxTime; /* Time when this transfer should be finished */ }; /* ** The input blob contains an artifact. Convert it into a record ID. ** Create a phantom record if no prior record exists and ** phantomize is true. | > > > | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | int nFileRcvd; /* Number of files received */ int nDeltaRcvd; /* Number of deltas received */ int nDanglingFile; /* Number of dangling deltas received */ int mxSend; /* Stop sending "file" when pOut reaches this size */ int resync; /* Send igot cards for all holdings */ u8 syncPrivate; /* True to enable syncing private content */ u8 nextIsPrivate; /* If true, next "file" received is a private */ u8 ckinOnly; /* Clone only check-ins and related artifacts */ u8 useOkToSend; /* Omit artifacts found in okToSend */ u32 clientVersion; /* Version of the client software */ time_t maxTime; /* Time when this transfer should be finished */ Bag okToSend; /* IDs of BLOBs acceptable to send */ }; /* ** The input blob contains an artifact. Convert it into a record ID. ** Create a phantom record if no prior record exists and ** phantomize is true. |
︙ | ︙ | |||
1350 1351 1352 1353 1354 1355 1356 | if( !g.perm.Clone ){ cgi_reset_content(); @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x")) @ error not\sauthorized\sto\sclone nErr++; break; } | | > > > > > | | 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 | if( !g.perm.Clone ){ cgi_reset_content(); @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x")) @ error not\sauthorized\sto\sclone nErr++; break; } if( db_get_boolean("uv-sync",0) && !uvCatalogSent && !xfer.ckinOnly ){ @ pragma uv-pull-only send_unversioned_catalog(&xfer); uvCatalogSent = 1; } if( xfer.nToken==3 && blob_is_int(&xfer.aToken[1], &iVers) && iVers>=2 ){ int seqno, max; if( iVers>=3 ){ cgi_set_content_type("application/x-fossil-uncompressed"); } blob_is_int(&xfer.aToken[2], &seqno); max = db_int(0, "SELECT max(rid) FROM blob"); while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){ if( time(NULL) >= xfer.maxTime ) break; if( xfer.useOkToSend && bag_find(&xfer.okToSend, seqno)==0 ){ /* no-op */ }else if( iVers>=3 ){ send_compressed_file(&xfer, seqno); }else{ send_file(&xfer, seqno, 0, 1); } seqno++; } if( seqno>max ) seqno = 0; |
︙ | ︙ | |||
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 | /* pragma send-catalog ** ** Send igot cards for all known artifacts. */ if( blob_eq(&xfer.aToken[1], "send-catalog") ){ xfer.resync = 0x7fffffff; } /* pragma client-version VERSION ** ** Let the server know what version of Fossil is running on the client. */ if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){ xfer.clientVersion = atoi(blob_str(&xfer.aToken[2])); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 | /* pragma send-catalog ** ** Send igot cards for all known artifacts. */ if( blob_eq(&xfer.aToken[1], "send-catalog") ){ xfer.resync = 0x7fffffff; } /* pragma ckin-only ** ** Only send manifests and related tag artifacts. Omit ** file content, wiki, and ticket artifacts. */ if( blob_eq(&xfer.aToken[1], "ckin-only") ){ Stmt q; if( !xfer.useOkToSend ){ xfer.useOkToSend = 1; bag_init(&xfer.okToSend); } db_prepare(&q, "SELECT objid FROM event WHERE type='ci'"); while( db_step(&q)==SQLITE_ROW ){ bag_insert(&xfer.okToSend, db_column_int(&q, 0)); } db_finalize(&q); db_prepare(&q, "SELECT srcid FROM tagxref" " WHERE srcid!=0" " AND srcid!=rid" " AND rid IN (SELECT objid FROM event WHERE type='ci')" ); while( db_step(&q)==SQLITE_ROW ){ bag_insert(&xfer.okToSend, db_column_int(&q, 0)); } db_finalize(&q); xfer.ckinOnly = 1; } /* pragma client-version VERSION ** ** Let the server know what version of Fossil is running on the client. */ if( xfer.nToken>=3 && blob_eq(&xfer.aToken[1], "client-version") ){ xfer.clientVersion = atoi(blob_str(&xfer.aToken[2])); |
︙ | ︙ | |||
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 | #define SYNC_VERBOSE 0x0010 /* Extra diagnostics */ #define SYNC_RESYNC 0x0020 /* --verily */ #define SYNC_UNVERSIONED 0x0040 /* Sync unversioned content */ #define SYNC_UV_REVERT 0x0080 /* Copy server unversioned to client */ #define SYNC_FROMPARENT 0x0100 /* Pull from the parent project */ #define SYNC_UV_TRACE 0x0200 /* Describe UV activities */ #define SYNC_UV_DRYRUN 0x0400 /* Do not actually exchange files */ #endif /* ** Floating-point absolute value */ static double fossil_fabs(double x){ return x>0.0 ? x : -x; | > | 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 | #define SYNC_VERBOSE 0x0010 /* Extra diagnostics */ #define SYNC_RESYNC 0x0020 /* --verily */ #define SYNC_UNVERSIONED 0x0040 /* Sync unversioned content */ #define SYNC_UV_REVERT 0x0080 /* Copy server unversioned to client */ #define SYNC_FROMPARENT 0x0100 /* Pull from the parent project */ #define SYNC_UV_TRACE 0x0200 /* Describe UV activities */ #define SYNC_UV_DRYRUN 0x0400 /* Do not actually exchange files */ #define SYNC_ONLY_CKIN 0x0800 /* Only manifasts & related tags */ #endif /* ** Floating-point absolute value */ static double fossil_fabs(double x){ return x>0.0 ? x : -x; |
︙ | ︙ | |||
1743 1744 1745 1746 1747 1748 1749 | blob_zero(&xfer.err); blob_zero(&xfer.line); origConfigRcvMask = 0; /* Send the send-private pragma if we are trying to sync private data */ if( syncFlags & SYNC_PRIVATE ){ | | > > > > > | 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 | blob_zero(&xfer.err); blob_zero(&xfer.line); origConfigRcvMask = 0; /* Send the send-private pragma if we are trying to sync private data */ if( syncFlags & SYNC_PRIVATE ){ blob_append(&send, "pragma send-private\n", -1); nCardSent++; } /* Let the server know if we are only interested in check-ins */ if( syncFlags & SYNC_ONLY_CKIN ){ blob_appendf(&send, "pragma ckin-only\n"); nCardSent++; } /* When syncing unversioned files, create a TEMP table in which to store ** the names of files that need to be sent from client to server. ** ** The initial assumption is that all unversioned files need to be sent ** to the other side. But "uvigot" cards received back from the remote |
︙ | ︙ | |||
1820 1821 1822 1823 1824 1825 1826 | blob_appendf(&send, "cookie %s\n", zCookie); } /* Generate gimme cards for phantoms and leaf cards ** for all leaves. */ if( (syncFlags & SYNC_PULL)!=0 | > | | 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 | blob_appendf(&send, "cookie %s\n", zCookie); } /* Generate gimme cards for phantoms and leaf cards ** for all leaves. */ if( (syncFlags & SYNC_PULL)!=0 || ((syncFlags & (SYNC_CLONE|SYNC_ONLY_CKIN))==SYNC_CLONE && cloneSeqno==1) ){ request_phantoms(&xfer, mxPhantomReq); } if( syncFlags & SYNC_PUSH ){ send_unsent(&xfer); nCardSent += send_unclustered(&xfer); if( syncFlags & SYNC_PRIVATE ) send_private(&xfer); |
︙ | ︙ | |||
2167 2168 2169 2170 2171 2172 2173 | && (syncFlags & SYNC_CLONE)!=0 && blob_is_hname(&xfer.aToken[2]) ){ if( zPCode==0 ){ zPCode = mprintf("%b", &xfer.aToken[2]); db_set("project-code", zPCode, 0); } | | > | > > > | 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 | && (syncFlags & SYNC_CLONE)!=0 && blob_is_hname(&xfer.aToken[2]) ){ if( zPCode==0 ){ zPCode = mprintf("%b", &xfer.aToken[2]); db_set("project-code", zPCode, 0); } if( cloneSeqno>0 ){ if( syncFlags & SYNC_ONLY_CKIN ){ blob_appendf(&send, "pragma ckin-only\n"); nCardSent++; } blob_appendf(&send, "clone 3 %d\n", cloneSeqno); nCardSent++; } }else /* config NAME SIZE \n CONTENT ** ** Receive a configuration value from the server. ** ** The received configuration setting is silently ignored if it was |
︙ | ︙ |