From bb4a9fbf362d0791e148195d86815d65ebc26fba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Jun 2018 07:12:21 -0700 Subject: [PATCH] Docs for CSV export, refs #266 --- docs/advanced_export.png | Bin 0 -> 7837 bytes docs/config.rst | 14 ++++++--- docs/csv_export.rst | 63 +++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/json_api.rst | 4 +-- docs/sql_queries.rst | 2 ++ 6 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 docs/advanced_export.png create mode 100644 docs/csv_export.rst diff --git a/docs/advanced_export.png b/docs/advanced_export.png new file mode 100644 index 0000000000000000000000000000000000000000..cec0c4ebee78a32f698b63f9de9180dd32cc4de2 GIT binary patch literal 7837 zcmb_>cQ71)v%eNC(R&R-bdf00dv8$>cS!VddWj$rgy_BZUJgebNA%u;Xy??Bh<-Y! zb1&cD@837`X5PI0WB0Q=yZgt^?C$Jrw7#A?2{9uv78VwXriO|k78VZlABGX){^M`+ z@gM#fZuE7GRWTUM{r$t$)ioOz@8#v?>FFs}00xal-{0Nc-rn8Z++18-oS&c5zr14S zRJr-*jk(>r!{laR=tVH2s9QokOkOnjUk+~PSj^4t3N$0Fb|KD1>3jCqQv|GR`cQ>~ zmyJvO;sJAVhoP79!rWghFE2kF%-mrxmQlGupF;(o^B$mYCXOC9wzlWyX4^9^+`W9W za`S;-?)Fi863=*kt*%W@PI7VbM#iW3`UiPQagd7NG&i>d=QJ|DGDO{Bj%Kd6&+qOo zkB*Oy&i9sCgcR7GzgRkZ;1LqldC7Hmy1BQr+0gs*Xuc;qz`{sV9&>kmcl@g~&Rb1d zOkMoyVtv>Jfsg^5)>Y9 zYo=qU@Z$dbHaGa<=f=q#a(=w4-VyAoWNdHog8N(Bef#kA;m-Xtn;U-3>tAEtQ-f{L zj|q};f< zT#t^91m}0`3|2Km@^wFDN_!=8YIuh`m{d$XbdNp6H9X`u-QVopbS@v)BahdnM}D=J z4nmWozJ#`91_fvfV#Q&wLNVQc9~P(Xe)nEyx6bD`^-ou(y>U-TPfarS2-UKBpW>^} zE2ZI|dH?C-m5t@qa>=EI+GWqw%E885dPVc^VOU94T0(48by{eYx4nqGhKrVjjOewN z&Q(Ntt5a~|Qe2rUi1BlW`57 z-K!h^o8Pgz_p^n|@=I`#o07V&kBJ7=M~va;+xd;7-rlaf&o;2(2U(BX+OHR($tAU0 z7^Nf(eLUuD^(NcnBC&DVC#;|=H##T6=ko{#%Nv7bfid&BW#{L$G-BGr!eX-0R8chk zu(bcn+4YGv^YBUcHB~!kDLu;M^3Wv>M&WFa0tjYE@oNQ4bJWe?NApu&=EgirE}D70 zE=SqIp-APWvVPH*Oqfi#`XMs&VRCR7NF>rpW|F3yYnukrpr z^{ma~^cW=ZIrTYqm6%M8df584Vl($`-uTr;Ljs?KV1EA9B+N(|0E%HT57?q_YRpKu zaC<0#Ih_?-OG+BGc)xkvMdlGeOLts&SpNs0R^8tE%~?4OJnSz$lxu!xP^=YT-ar{r z`cE@`qXiX05dXAoGJrpS@u)UJ0gU5GU24MQ!R(Papi zB7i3|g7`)OT15805#K^eC5p~uBYgchrr{qQ9F@(5>)ZQ8_<`l7oe zt3;-poh-YfoZa6L`bukv+o+%KNerV2g}0GTFd@8xAs1fo&Z=J~b|B5edMif_0$1kr zPT~mymhnpLS+R=T)+ZL-XlyLGF3JHoX@2HD2i3eKpZ*}$gOTLP)wbK-IU|DSlqPtT z?E}TEN1HV4jw&?lzV4$lYsRjb#R@UnZ%bfN-GOq63Bo>{Gl^`v3pH?4Ru8$%;r?aK z(A0&Q)`9@(Y4NSrRZGg>Ip6kLsX$wlVx9=x9|e;Y&&+*8OKX9DOTWIT2n>LxL4H8^ zK?UTx0`H6_2>o>9N3s_X1^-Ejt{K<{jeXQ#vi_O(PGB?@*cen5JeJjdN$g`p;g^bI z0)0>(tjP*;K>tWBE(maGR8%?yag54c)|upPZw-U<3x;c6H}AV-$VXojpEVX$&ta7E z2iI2V`;(6R<7n}m`v6c$)43c!`7=_Hm9`y`u%CQ&+mi4V>TyWjg9gJb z6xdRe%cE)hHG}G!@%EhIWtIqkY%Hq0-D+GHJ6g^PrLz?|ICu@ulMEYr@osBew#+48 zTZQtz6cVs?@XL40ztwQ}gzXSElK=&_YbtHOwX(aoEQLAoM4CXC%etZ+Ou5t@J?xuZ zSkyLH1mHmBhym~}j}ICUL|h9j@5s6+bJ&o21(wy=TDC@FOTu53#(j!%#MvEIgGMXP zan12f!gCns>|5-Z&(amuAE$s1l8jM?M%VQ&P;(bX?emy9rJh=}TNA*qCp8?{UWz=o ztpr5Ba&T0*4{QP80ETq2=MHNH_Gt9#{3@z*?S5sO8;|9Ffwi=?X+iPAqWQ5$?@1}r z0c%_+(ps4=N~a_SJAe-t`4&@am6+@z1kHO!vKui(Cw!D&cL-b2CLRa2+(*^u!P6~J z1Hxq8N7>x!j_DBTa)X+zr`7GX+}`~(^(JkWoztD(%-v&#W5~%zGFx3bRqj)OALim! zxIf}k7Wb%$;6I_HdSE$vw{EF z7Lzm;mxKMW$s=t8H$w-12G%g&KgZo3CUD9STKG>DXpt_|M~>uP0}F^C4go4^SJ9k@ z0u`VBUkmGYq(N~=M*-7~w-Ze^kmS?#mAg>%3^}i4ao4E&NKkG@ed$vR0(bl!$H()V zhK-{F<_#jW6cPmoKu7N&MtJOC7FcD&jj*xm+UenO+-pv$6CW)^ymYE=iz(+oV1?WA z#d+g{+wyeiyrN6NY(sV@N?yMjRPRH?Jn$ZBS&lT7so}HTw2L+pPb{3^aoZ3~uSN=f zngF(ACV=YAvT&K>G2mv+BK)G!#j+{aP>?Z2^*7tvqV(AUZNG4cd`Gftia*reprjXJ z;F=&fz=M0`A~;{IFA+9T zqIxA(O!r#lNQzl0_RCzY7NVMXrVR2ti!;JHLq{pJ)p-QK|6EnCns}Tg#EDT0RE1)T z0Sip?fD$ol7*z zW}_l9ud4ChJ@4S@an#WME@QQ=9!T?@k4=eY6TmZtbW7Xl)ngOsP@-7vchqVOX!XSn zTPQZj@3GF1IywpBliurHWb?{)Qvh+J1FDnVehq91Si#1NodX) zLDIt0J4SFRHpmgT+R3aezD!6|%s9MGs%k?`H0>y3aBF6XM*pi+4me*5cd^?4k&NO~ z3>NwWS+942;osT0ReZ($DIQR|4@BIGrObe(xF96!$5JtLfkNjwsNbj_8{ojO1moU7 z>kfvN&v@80WBcvl`?|ZOc+rpyS++xx@_nQ{??&y_?Gtc###o4Dfm`TBEXurW7*zfO z>OG>=M1H=KOoDV%ZTbu=%||Ji=<-uZY^VW2^CeGgNB{cSp&av*%P>x8BRkFCUv@yY z@oI}PqGR%s$@%)w_@%Z)*MoB*KaTqe z(8<}eSFHd_N0sy6_+|0GH*e5XcC}0?PIhG}Y@kSDpEr!RUHF>Bx0mM&qxnWBq1Q~= zFC=vE6EKDQ4}S&xI|1IXGC_Sdp?=4w1;1o;`-}(vTieQEa3P4*xNaPBp%J| zloaAKz-QRJWmM0GwOnTrDTU<>)5+Q27*BI0d229vNeHq{{kl+H!6*@0J{yUxqMhL# zB8$DQ+Z_ZP1?)S|e9%-0LDr}agH}5GsHr0kX0&((2~s`y&)C9DmtJ#a?K=?< zQZI+N9;lx3-dls9KcBkAe@(JUQ#o*kAc&?&YnpDJg%|NghFNUv+GnZsmLrIawHg&C zxgoMw8;wN&s*b-9^S&#C*rgH-85aI%e<-M1>F)}|k9_QHr8t80I-&>D5x)t1xGc%lewJo|XMfK#ME3}*-y7uY=olVn0yrZps@_I`k-R;zOIpqA%5#r(h5Fkn z@AhLBT<}K7Xl|S|5CCF;iI+4Dm@aTp>hH&quD~w6$BLPjpdpq=#$o=cP!rJ_e5Z@(fI9VnV-bowaxwD}e#RPkEEUJYs$3{_DZd~AexjOAgO76>R zFUq|^ZnFkeW!H$s%s_wXilPj^;o^t03VJL9p;koqa+*2fq0@|XAcaok*(X$a?Xa`( z=^#grY`w|P^1l#skvXVC;rNdaZ|>b6Mr(QLsrj*X|>yy%L2Mn80g z4=KhH0H%t5;0}(mD43_;VV}~iU@MUlJqPnfGtRt`NH=tC`16u?`Q|}aP=gZz6nHT! z%(kd+TYP56kF<^*%%^Q5$iV}!csUmyfoiE&T{$23RTPR8@-OriiRTfE;?=n@ zT|KK!L-14{bEA^^s1<+VeFBIUB>3mdAdmgbSWWW68_&rzql01#i!Kze6zti^+fS-n zmaWh(J<5%D4mn&7H;Qfu`X}8bOg02niR72EFB3Uzd>xJPy}w^lGb+CrIFxr}Rr;&s zCXp@`p7-wk%iivUYMbLi?m37$F0$WeeB{Q}L^J|=kUN8fe&?oU`z@Cc8Sp)p?vsKX zoK=%*`r^a^{9Ti*TXQWFpNi_`nf=-p%ujTPQ%A7#yg78cbVe?&f<>M` z7Qw_aBBh%e^kn-Xh6Vz~lXMdC;sk}UPV`KjrfjTUZ(-8$1`cGPO}1vS#4y+gV4AOS z#u(smO~WKP``vM=fCP#Bm%H-f-7`e3&2vL@kpbp*VXH$m){@LDL!eqf+KdEO&q#Td zJ-==R->+*$ngVwEQf37^>G(dlev;^&Qk3$kSyryKZ8D8!Y8veD-6x>wf&!;sEyoO1G%+{SW< z2%QOpr+6QCapc1iA3+2H&n;|&6XK-_gKyoB{dTA*93I89aJqOV(oLN$WO)K1{j%x4 z!R`cn3-Njy6{Q4h;}BV^4t6sKsvrjr)s%Ux7IjMNmE z@=>+i=&Fm>?+sM7ewV9%-`;}KL{5E~um(GRe9Ao8@nArzbUwpKW1R zwVsaoAyf&T$dra{uwIkJru9He}yXlXu>eO*u~f(g4@uab6rlv z{xaBDMrf>oJ78&#!8exGUwxNr9Tpz8i@>f-iTT0%1^Fj?6$U-%(XAo$RLs#mlh-QE z)myhs?c!f2U*>IP3Lwlu*5;{{p@x%RIcyl-z2kRr^Vlke@o%1r78 zRhG&be$qu)mq~}JqZb1U{Kxi|V`BVUTj!YEQ_GmDX<}kxNVZq^Y8K5Uszyu%??kt+ zFG!jAmt;(h$Mr>9phdhA85OB~)7t*|q&Q#&8yc(4^!P~vvFov%Uw+Fj>7N+f9 zTMv`5m%nyFT$kIp$o-aFlE`$LsGDMKJ$n>ZaLMJEp6xE#fDqla@u&=oQ~Wy)ePoXpOyCrnxa;*#^sY zjABt!$-<{A)8ufu)qm+}|623Gq@X$2_DDxLm#{?Qq4`Zqqzx4BFVtDb4H{i!z|YDn z{v6IHi&qakQR;0EQ->+FoICoc7JyJA4MbNn@8sDu4HI8yl zi=tzqF35CK|JYv~d=Ae3)JPQfIf?W)*V4YMu=5mKSgFR8R@qvXQMrFC;uKERM=L$ zq_{&s6j~x*l1lI$cfx$-G1;8E==ks{`%k(k`wfSb9`)g?`2O_|2Xi~JTq83qH1MIt zqNZkbT^6MoDIzg+oa2-4u1B0|X@NU(rzbp|(^E4~XGK0{Rz(iq&h9-fk;^(2=bkD( z%+pG!@=Ewcq{+Wei`KX%d;v{d0I_FBZa01J6!EE;=7SQJr9(?hbEL+~b0?Lg%6-|E zTQk$MIRBLZz$N}Zdg-00yXhI}vB_$O8PwP+w*|8Ii;F|y%WS@5xXF`o)B3DX_96B#dWq!__(h;(mu(sy5Ucw;$E_nNl={L??w=re5zdW zjh5uKgo}W}bJNzl;7YymLTNyWt<8hPXJESlm<@W z&F&o0WbaEx5&0Mhk7Qnqmr)@HmeZsFobe{IH%q=aXP1y?`#oE8WOp3v+`xIel46I` z>uj%L?oq7g$zbXvWK5(HXb0@{0dI=5fS{spxsv1!z!FY#{(s7;XC)gCtkkoc7pk{z zv0Df>0;BaH?5uj$LPvF6Sp26mF2E*tyho0%F_$wrseYQvo4QY_SYiMPFIOr*kv8CN zesgC;aR0?qqixtBmmLpR0D7DPxej=(#197z^(g(o*FR(e^~iyeE7w>$+r&ZiOj zZtWfKDyl%&s@!rgXOZ;W^=E0jZ@@pXswdDd?wuws9`~X0#-7hUs*R2`dh}>FPiw2H zAM5FqGpQ=-%HZd#kqc_221Y21DO;T!h|Zzub+YI7K!-9sdZcPON_=tMHmM}X#i|ZW zL{z>@(J3+jUKfP$yL?X1X({`ezJPH{59Hd*P6W_VSJz$jbwDOpZk^-p0nHJSfUM+D znrjr;%Tj8;LvGzDuDq4$@Ky(sG}6SwYFfZN|^0&Yls#WA>zz@N4@5A zQpT#-cTGhja&1*UU#U$;tty4grW-XjLnM^pgX=N%!_LXlU1yQW`)6bn;Q(hTqA?H#i zYV}SM_Q~NDewK*>P^^%=zL{n${%{(VRMo@l?LJ6N3K}Aidu2NQ*X)OB}70WIb|1&XemP0!R5Ah$?eqV@w zhjWt%P49b0pqH@$OB>;GMWb?_-gtNhah+nAQWHSjeN9O7`*s}cn5NG@^x&}0h$PwG zy-NVfcQELkHm7Jgt*u1fnnOH3d*>V_NjxOoKF}tWSK*02AD0}npg>;&59++iqZZbL z1F8Q41aZhz4!*Hd=*i|i6G-NW)fkf*oP6a$YcQSHxc}UV)o>=}_sp^+NU(ADfmSPf zh%nX;M|U>Re);LW5Xy9*sc#a3-oPs!O+FqYb|lu}4>yVtQ%p4p?Y8*5_PSnP3)S6) zjjrQX})2(b`!1GCd`4WH>LMsxutIyBk zQmzd3tx6|_&lCfstS(S&k=FF#VO5GM4Zl_}9b$@mVNyXH6Vm-VkE>{oPK;4yfHY<3 zhr~)?BqO{e%cLed!2*2Q563RAdQ)@Z2Hks7rHGDGW}D|zo|D_griR(XH<|!hO()S( zAsltm$SkA` where an entire table +(potentially hundreds of thousands of rows) can be exported as a single CSV +file. This is turned on by default - you can turn it off like this: :: datasette mydatabase.db --config allow_csv_stream:off +.. _config_max_csv_mb: max_csv_mb ---------- The maximum size of CSV that can be exported, in megabytes. Defaults to 100MB. -You can disable the limit entirely by settings this to 0:: +You can disable the limit entirely by settings this to 0: + +:: datasette mydatabase.db --config max_csv_mb:0 diff --git a/docs/csv_export.rst b/docs/csv_export.rst new file mode 100644 index 00000000..10f65b76 --- /dev/null +++ b/docs/csv_export.rst @@ -0,0 +1,63 @@ +.. _csv_export: + +CSV Export +========== + +Any Datasette table, view or custom SQL query can be exported as CSV. + +To obtain the CSV representation of the table you are looking, click the "this +data as CSV" link. + +You can also use the advanced export form for more control over the resulting +file, which looks like this and has the following options: + +.. image:: advanced_export.png + +* **download file** - instead of displaying CSV in your browser, this forces + your browser to download the CSV to your downloads directory. + +* **expand labels** - if your table has any foreign key references this option + will cause the CSV to gain additional ``COLUMN_NAME_label`` columns with a + label for each foreign key derived from the linked table. `In this example + `_ + the ``city_id`` column is accompanied by a ``city_id_label`` column. + +* **stream all records** - by default CSV files only contain the first + :ref:`config_max_returned_rows` records. This option will cause Datasette to + loop through every matching record and return them as a single CSV file. + +You can try that out on https://latest.datasette.io/fixtures/facetable?_size=4 + +Streaming all records +--------------------- + +The *stream all records* option is designed to be as efficient as possible - +under the hood it takes advantage of Python 3 asyncio capabilities and +Datasette's efficient :ref:`pagination ` to stream back the full +CSV file. + +Since databases can get pretty large, by default this option is capped at 100MB - +if a table returns more than 100MB of data the last line of the CSV will be a +truncation error message. + +You can increase or remove this limit using the :ref:`config_max_csv_mb` config +setting. You can also disable the CSV export feature entirely using +:ref:`config_allow_csv_stream`. + +A note on URLs +-------------- + +The default URL for the CSV representation of a table is that table with +``.csv`` appended to it: + +* https://latest.datasette.io/fixtures/facetable - HTML interface +* https://latest.datasette.io/fixtures/facetable.csv - CSV export +* https://latest.datasette.io/fixtures/facetable.json - JSON API + +This pattern doesn't work for tables with names that already end in ``.csv`` or +``.json``. For those tables, you can instead use the ``_format=`` querystring +parameter: + +* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv - HTML interface +* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=csv - CSV export +* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=json - JSON API diff --git a/docs/index.rst b/docs/index.rst index 031ff0fa..4fa403fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Contents getting_started json_api sql_queries + csv_export facets full_text_search spatialite diff --git a/docs/json_api.rst b/docs/json_api.rst index 97029bb3..f010d08c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1,5 +1,5 @@ -The Datasette JSON API -====================== +JSON API +======== Datasette provides a JSON API for your SQLite databases. Anything you can do through the Datasette user interface can also be accessed as JSON via the API. diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 0355d9d5..cfff8136 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -95,6 +95,8 @@ will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL. +.. _pagination: + Pagination ----------