From f1968ebe7c15ffe8238bac6199e2743ee88fe17e Mon Sep 17 00:00:00 2001 From: Carl Kadie Date: Sat, 14 Nov 2026 07:40:31 -0800 Subject: [PATCH] Publish documentation only through the Pages artifact workflow Stop tracking the generated Sphinx HTML under docs/ and drop the CI and release checks that compared it with a fresh build. docs.yml already builds the site strictly and deploys it to GitHub Pages. --- .github/workflows/ci.yml | 3 - .github/workflows/release.yml | 3 - RELEASING.md | 6 +- docs/.buildinfo | 4 - docs/.nojekyll | 0 docs/_images/example1.png | Bin 2775 -> 0 bytes docs/_images/example2.png | Bin 5776 -> 0 bytes docs/_sources/index.rst.txt | 693 -- docs/_static/base-stemmer.js | 476 -- docs/_static/basic.css | 906 --- docs/_static/classic.css | 262 - docs/_static/default.css | 1 - docs/_static/doctools.js | 150 - docs/_static/documentation_options.js | 13 - docs/_static/english-stemmer.js | 1066 --- docs/_static/file.png | Bin 286 -> 0 bytes docs/_static/language_data.js | 13 - docs/_static/minus.png | Bin 90 -> 0 bytes docs/_static/placeholder.txt | 0 docs/_static/plus.png | Bin 90 -> 0 bytes docs/_static/pygments.css | 75 - docs/_static/searchtools.js | 693 -- docs/_static/sidebar.js | 64 - docs/_static/sphinx_highlight.js | 159 - docs/genindex.html | 953 --- docs/index.html | 10142 ------------------------ docs/objects.inv | Bin 2372 -> 0 bytes docs/py-modindex.html | 143 - docs/search.html | 100 - docs/searchindex.js | 1 - useful.md | 1 - 31 files changed, 4 insertions(+), 15923 deletions(-) delete mode 100644 docs/.buildinfo delete mode 100644 docs/.nojekyll delete mode 100644 docs/_images/example1.png delete mode 100644 docs/_images/example2.png delete mode 100644 docs/_sources/index.rst.txt delete mode 100644 docs/_static/base-stemmer.js delete mode 100644 docs/_static/basic.css delete mode 100644 docs/_static/classic.css delete mode 100644 docs/_static/default.css delete mode 100644 docs/_static/doctools.js delete mode 100644 docs/_static/documentation_options.js delete mode 100644 docs/_static/english-stemmer.js delete mode 100644 docs/_static/file.png delete mode 100644 docs/_static/language_data.js delete mode 100644 docs/_static/minus.png delete mode 100644 docs/_static/placeholder.txt delete mode 100644 docs/_static/plus.png delete mode 100644 docs/_static/pygments.css delete mode 100644 docs/_static/searchtools.js delete mode 100644 docs/_static/sidebar.js delete mode 100644 docs/_static/sphinx_highlight.js delete mode 100644 docs/genindex.html delete mode 100644 docs/index.html delete mode 100644 docs/objects.inv delete mode 100644 docs/py-modindex.html delete mode 100644 docs/search.html delete mode 100644 docs/searchindex.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59e347d6..d126684a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,6 @@ jobs: - name: Build documentation from clean sources run: uv run --frozen --no-sync sphinx-build -W --keep-going -E -a -d "${{ runner.temp }}/doctrees" -b html doc/source "${{ runner.temp }}/generated-docs" - - name: Verify generated documentation is current - run: diff -ru --exclude=.nojekyll "${{ runner.temp }}/generated-docs" docs - test: name: Test (${{ matrix.os }}, Python ${{ matrix.python-version }}) strategy: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 957914d0..968c81cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,9 +61,6 @@ jobs: - name: Build documentation from clean sources run: uv run --frozen --no-sync sphinx-build -W --keep-going -E -a -d "${{ runner.temp }}/doctrees" -b html doc/source "${{ runner.temp }}/generated-docs" - - name: Verify generated documentation is current - run: diff -ru --exclude=.nojekyll "${{ runner.temp }}/generated-docs" docs - - name: Build wheel and source distribution run: uv build --no-sources diff --git a/RELEASING.md b/RELEASING.md index 5e48454e..d237ef8a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -69,15 +69,17 @@ repository or GitHub Actions. notebook environment. Inspect numerical results and committed output changes rather than accepting changed output mechanically. 8. Build both the source and published documentation with warnings treated as - errors, verify that generated pages are committed, and check external links: + errors and check external links: ```console pysnptools_docs_output="$(mktemp -d)" uv run --frozen --no-sync sphinx-build -W --keep-going -E -a -d /tmp/pysnptools-doctrees -b html doc/source "$pysnptools_docs_output" - diff -ru --exclude=.nojekyll "$pysnptools_docs_output" docs uv run --frozen --no-sync sphinx-build -W --keep-going -E -a -b linkcheck doc/source doc/build/linkcheck ``` + Documentation is deployed from the strict Sphinx build by the Pages artifact + workflow (`docs.yml`). Generated HTML is not checked into Git. + 9. Validate every entry in the package's hashdown manifests, including the packaged tutorial sample files. 10. Build the source distribution and wheel without local source overrides: diff --git a/docs/.buildinfo b/docs/.buildinfo deleted file mode 100644 index aeeda401..00000000 --- a/docs/.buildinfo +++ /dev/null @@ -1,4 +0,0 @@ -# Sphinx build info version 1 -# This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 4c4721524ee508aab9d40431dfc5f47e -tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/.nojekyll b/docs/.nojekyll deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/_images/example1.png b/docs/_images/example1.png deleted file mode 100644 index f6e73bccd075bfc1a43d5f1038a74c4889896128..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2775 zcmb_e4>;5N8ed6Qq{q}5XOptBPFguK(@N89Xr25^F^p=fwqqh2B8_`a#aP2RO8$qs zmBzGK6zP`P2pgdlN(?L0pHW1~{q5YIoZG4U+~?f$d!FB)_kDlA_kBO_@BO^rZ<43G ztD!zr9|Qs!61Ng|fIxE_fajfhbAhYTsL%JnZ4PUPs}o2dGkXRI^Jq9X90(*#HF&>Q z7mybm+Umywffm(i&pFbtOZz||{SYDnw=+8Ec^;0bL>oD?&zG>1bDh*?&aMq{Xup^| z{7E0|NgZRi=E~$oD3f zUX(?gQihABnj@{{E9uCLx3xi|T^IJt_zvc;n>MU_!j+7n#&Y8>Dby;`cZ106Tt(bt z#Hzp+x%v%7G47yXqstmw-;MPe(AJOTq9(&7n>Q&`Nx4VAe{^iKG)RTwO^Txg^077?rEw<%gp5~AWTHvT3XwynkWqd z6Sex*3;QQ5@&q$T+0Z@~knI(Nu$5%&>RQoLzsHq*x~Kc zJJIOJ)^}|u(Kdea5pyWh#XR~Gnu=e~9mWmauFWLNXo6S( zb2|x{j#jWI7f+P64?l{nIKxvBBMOntMUP^h_uphnE~cdiPW)q4)836bW43FQ-;TRP zuurDhmGZFwxyj~-xm@kl%xvM;gE+Cmh$SV zAXNUvELDW>kv)+RLm9*IUOV-PuVTYOtFiG4~5 zv@h`hF;V5Ynr)mw&-`ausWjDekwbccP!IPTM}8wkLtDq))rEgnHCgp2v{<A|b!#9NDH+P-`QY+vcuLiF@qN)XGzwxtM;hg{Mr~bqCh5TqpNFE= zQ0_ZvK?}BIWO*xzPHtOHxj$>nvWN64X+o8UXC7NT3@#A1Ru$pIC9|28Px)+}xm({C zRQ^Ys62@)ZWU&Cs{o7}7Y#GQ4FXLY&*V}$(@J2(fXEl_OTS*Ro{|xVhPs4OxT9f6{ zdw;OybUHqEK%gK{Ulb=(^ncg!XisQm2kAVlJ3yn6pA}VzsZWyU_tqC7!IjUf^n3Bu ztd1eK-NIKnRt1)lOLo>w6s;xPE~#o4|GeplcZ(lu8y#8XPg$t%i%*esuI%?I{graX z!;aPhG!-W}$>uwa9cdowJ0E+o_c5|hJO*NSJ;UGS-+>X|9^QWriWxUY{~UE=tJ)!{BRQ27u?WR zl}+fmylcn8@#3KpjAac4+-XeptO9r`K!7WQPcJ#fzKmhGC%uJnj~*>?>@5))g?c+z zh)tIW63lpfqPW!GvVPCmU|F{J)mGBcpj%UCtWp#{9RXiWG@Mu8R{V7@PSh`Fuz=^h z^NT5?AYN;2=AYnOxGIBB>6?a9JCQrwfPIl7OZJ*skp& z;TD3}gmE|pm_Qp-T z6B3>w7#!~QKlmyVzBPAT&sO&0Uq8nkxK51jAdwS6n%EzasL&7&hr`h3ClF@VM!$;8 zLhGg1-p<5kcY)unw@2`&>xOm4JwIp$Wh-&i!f+~{Kp?ljFcSf@roEP8&Hwg>&@_&6 z)8?6Wq6jVEaKVoNnr5g0+gtIVX^~5Xki-b3|2~ibuYS~x{dhyf_S&$nUz)p)uMU73 z1PLD`&erdJX__C+W(Hf(+jBd&nOb3;S!8ch-gyesGvSTfDHTWL?b1^+>ykV^wI!fzbecc2DP4`lnT9 ca=APnWtdl(R%kf^Y(_yuXLo|YY0uC90=Y&z0{{R3 diff --git a/docs/_images/example2.png b/docs/_images/example2.png deleted file mode 100644 index e6fe7421874967801b1e7e560912e1505eb86f3f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5776 zcmc&&c|4SD+ZLhcv4w;zlWHOvitIJn!r1p^ER)b!2U%lKX(JjY#%@Zsm@L^U*_VmI z)RYlInHbARc4Pf!p7;GdPk(%`_xJtv{V~gZUFUt?=Xo5*d0y9b$6J^g@cen~PZkyy z9*Ciy6$=ZSKI6aFAy&pW^iX&zjYt=qp%5f0c&RYh;iX&MD4HN>7GAGHuRV z!e`wM1mOXj2?X9UOAb7d^FE3;Z1XD|;*6Zm6y2E`9B28wF;~$%Z4H2ep}oWMAl#%< z8S3bJIQM(U>trNu5_!l-!van|v*=zNd6K-8Ibgu`Ly6;uoRTk?d;@$CC{dy`m%349 z-N^4t@11?lz9hA#nUw*RyN}U!Ol)U3cIyNE z*afwLoeSYSa~bpfr|!O*3X$-oo6N-RYMeG%?MJ7dPLIu9YchuSWuBO25MhHP-<0GC z9nSz~2{Uq(Ft%<>n(!D}ofmv$nfW1Az%Uenqx^%HN*P zS8u5#BGK4utJhJ-?r)1dzZYS5}i|+Hc-wuvDc4Y#Lu!KwkOj`1B=6R1A^j znmbb?Zs0l|v+SeAV4cLdGhjv)OPvtf zhK%z3DA5{?xn-VN(2EIvi`b57CYzK$?&w}}oLO9qJGb3?^qMK>*qe3xi6`IJh5rSm zhtWpsY*E~vxyTnchfj}Hghk8*=L!_p;A zV-gz|bPF!a*SADFTYv$GC=NmTsa^_2Q+lCzP;4h{;#xeU!@Ft9s;S;X7A&=huMqpg z1w61S&D#?~6fNqJ{$5Jd;L!a-Ac_QO<%vEB#TtgXG9~V33Q~=zC|BmTo}Vb2wXH#l z*KI~C0`|x@8)H^Ie)PKw4mgFXN6(=IX}8BYL(Q@Ap_F6-IHtTJ1PNQi@i8isYHi(C zp_q2QtV)&>JY3$xdd%K-Jywj`fFVj^d*GOka(IWglR|&;k)UAP zsrTxDN0QBgb6sF~w9b17X5@pW>WIP?GcK?q@T=O|b7gs@8q_o_eQO#pF+u8`c&;%=)tZt_Sy9RyCkx_tV zN@H`vu=7y~(aVjWn1L$YFuIE^thdZg@*cfM7o0abR=4Zgr%d~7?yms`CH5BIlis|$ zJIwL(0_Dv;F8XVS^5|v@0%LTF_KSZ}in=y47{b>_O%_yFfUNhf#{ig$sytIAe{OO% zxLqvJxq%#>u%*>34>PXX4k%KDf|Ixj#VL6DYd4x^p?Du3fS9!MgW`oUWR6bKh8zJh z8n%*|=4>EvD*XIZKd+9^Up`CDe82CDbC{q`Hn4uD@LV=|>}RjYv-1eAe7bJdrlAH2 z6-_fgYrZ@*mhVXHl+=x#pQNY>r(hUkgdqz0;rR{E_UgO&D~&%=rKdi%Ju_F_UP4K8 zKEp7D1^<%z6kysm9jZI%pC{T?H+hs z64hILI~dGy{{r*gmefFs%J#<&(8XN?!%`c>hWoXiZ(%@x_0e-6tTp$0>Jh#r$IVA7 zJ6y^*0%9-v(i+>ccK_q9oaI_dS0P24o|Au`%3i+^0xFsmWUdrnF9E-1tUmw3NA13oKxL7^Cn(}r1NG(!@+4d+!7$=061L~?p z!?g0J9rF}#`UR!(@sWrOzD-{f@cWx4{e2hZu-7>zaf2|XT7#?!NmCZ2HzY*MtWuE+ zbnnQ}%e?}}w4j}_u4?25nER4;zxxdY)sbGKbQ<2dtHZz{F&m0FJicd}k=AN_pPZoK z<0+X-l;T^}j_tnc8ltF1)s@U5O2q~LTxX(|m@R-fg4&$Ba#$q28qH)JKm3_=jU49f_J3P(o4*RO1p=CPS7Q$!)IPlO?-^mbQ!}R%mo|yYuy5T7?|@KX&kz9{E!ttV}uc8mKKCln_(iGubQvI zfJIPd<~U>C@XCt*XAYo*7h~CExIOsd;n>4rMsG5GS`f#a7y2k>ncax~c{z3)MZw1I zt}pEE(#u-Y7&wq?(9j*_L9W2dA!$wBss7(}46@8^DnyizrP_(Io`+F6wM8BLVmBm5VH zFcFqFEBjl_MOLILf#V2yA=UxGkkmipBhHO}p_>4fhxKGs)`-MB*+!?j)`esj+hjmr z#b{$^KYY{S=90!1bucfXu4k(-9am3~kY{Y27#Q8L(F@S???FEnW~oKzyJnyAgluQn zOCt8HB5KIOTU0Jjqa~4T?|Wt9d|oq%xL0_!PpNhmBxcgd3DZd>eXbDu7su*e*4{XT z8#q)Yq~W0>{lRX43Gm(hxv5w;2t(mkvL${21NuhxHb|mInY3G+7ZIG3J^I{5v z)PGhBGO!Bt5Yo4Nc$hHId0T?CTwM(1J%UQP3<}_)9snhYW%os%i{g%Py&lA$8p4!9 z!HLL|KGv9!Dzz_Tdp9o~uEX$@Ui4DR6XA--wLSgx^=K~1*@uS3y1v~L`5ruhb+ttU z`pT6_R&}+b2)YPQhNyoz4-OpL5D>7 zS94VS^vnW{$D7VtXS8AvfZ7y^%zf_*{wCZn$4AcGYB+V^?GpW)R6w|hQ!A} zs!&t+&kglPr&}&M*t*8WJD>3uVthY!3?|^s%|01*(uUD&`Th zGy;ykX#1}5{WLpDt|rVfz&)7wBUBxe=ayfvnXx+H+4q3igJc%L=Xj+~=I4F7pHyNE zl-SU|x?fTL+04rNlp`qx{H5}~(SPnjjqeV~1i2RiRB%#7u{#v=X~%1q4vk7-)NK-g z9HPRF`~e)X5$vH~zdbdNZZWtI`ji}PNnE{sQ>tC@0n_bO@g9feGj9FNgRcol6a=U< z(OtW8X^?a39ydW;_6lV6c94vFU-4b_=Zw?Kxmu4JmbLK!&iQTyV(|-~eaW zvqWXAz*HjRxZC;}{c*{SaKQuiIXL>N>-);yXI6}|5P6_TMjB-(F}td@u|DC!e+*>M z`0Y+?tIDqXn}I!hfkjw#Xb0EOl)2U+>c#WTQElz}`d=PEo5`Lgl>#fSPGF>+yH#Qj zmI6m}l_iQWi5NboBB`zXyg9aIo%^1=X$*2&)KaUatlM%FFd^5@R5_74lU8Nx=t^F) zGs9NGG2H?N1DHp05Iz z_|4YvxTFiQm-r$zy3mat*bYgda4vgJ}D zWW=AeLAuwJG{Y>LI*9|U`%B>p#9+XXLqV&Qy*JG2ZLWm7bW!$?TEA z_aQ3$5w(t7JDlaHsmJ#!*7oM!@s5KZ2X=NNJ%RcENIcPd-JQ(@b#zW*nU0w=1~RtEn0vv_-NIx{nP41BgV2G?Be6#*nF{uYi|J8NawOC_} zIcHzDYGXHVp}3JA-%hdtL^s~I82p8wKI4qWK%m8d1FZNfD z9z_{1C*ocJ|Lx(gnaqKIn%>{O88g+JsUH2YdpmpX)@6mAVmu{`x5zAzD`tA-I#=)f E4|oDQF#rGn diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt deleted file mode 100644 index 598dd40e..00000000 --- a/docs/_sources/index.rst.txt +++ /dev/null @@ -1,693 +0,0 @@ -################################ -:mod:`pysnptools` Documentation -################################ - -PySnpTools: A library for reading and manipulating genetic data. - -See `PySnpTools's README.md `_ for installation instructions, documentation, and code. - - -HDF5 and BGEN support are optional. Install ``pysnptools[hdf5]``, -``pysnptools[bgen]``, or ``pysnptools[hdf5,bgen]`` as needed. -Existing HDF5 users upgrading from an earlier release must install the HDF5 extra. -Core Python 3.15 qualification excludes both backends; HDF5 and BGEN remain -covered by the Python 3.11 through 3.14 test matrix. - - -:Synopsis: - -* :mod:`.snpreader`: Efficiently read genetic PLINK formats including \*.bed/bim/fam and phenotype files. Also, efficiently read *parts* of files and standardize data. - -* :new: :mod:`.distreader`: Efficiently work with unphased BGEN format and other diploid, biallelic distribution data. Also, efficiently read *parts* of files. - See `Distribution IPython Notebook `_ - -* :mod:`.kernelreader`: Efficiently create, read, and manipulate kernel data. - -* :mod:`.standardizer`: Specify standardizers for :mod:`.snpreader`. - -* :mod:`.kernelstandardizer`: Specify standardizers for :mod:`.kernelreader`. - -* :mod:`.pstreader`: Generalizes :mod:`.snpreader` and :mod:`.kernelreader` (provides the efficiency of numpy arrays with some of the flexibility of pandas) - -* :mod:`.util`: In one line, intersect and re-order IIDs from :mod:`.snpreader`, :mod:`.kernelreader` and other sources. Also, efficiently extract a submatrix from an ndarray. - -* :class:`.util.IntRangeSet`: Efficiently manipulate ranges of integers -- for example, genetic position -- with set operators including - union, intersection, and set difference. - - -* :mod:`.util.mapreduce1`: Run loops in parallel on multiple processes, threads, or clusters. - -* :mod:`.util.filecache`: Automatically copy files to and from any remote storage. - - -:Tutorial: - -* `White Paper `_ - -*From PyData Conference, Seattle* - -* `Slides `_ -* `Video `_ -* IPython Notebooks - * `General Notebook `_ - * `Distribution Notebook `_ - -:Code: - -* `PyPi `_ -* `GitHub `_ - -:Contacts: - -* Email the developers at `fastlmm-dev@python.org `_. -* `Join `_ the user discussion - and announcements list (or use `web sign up `_). -* `Open an issue `_ on GitHub. -* `Project Home `_. - - - -.. automodule:: pysnptools - :undoc-members: - :show-inheritance: - :special-members: - -*********************** -:mod:`snpreader` Module -*********************** - -.. automodule:: pysnptools.snpreader - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:class:`snpreader.SnpReader` -+++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpReader - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs - - -:class:`snpreader.Bed` -++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.Bed - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.SnpData` -++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpData - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.Pheno` -++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.Pheno - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.Ped` -+++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.Ped - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.Dat` -+++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.Dat - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.Dense` -+++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.Dense - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -:class:`snpreader.SnpHdf5` -+++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpHdf5 - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`snpreader.SnpNpz` -+++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpNpz - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`snpreader.SnpMemMap` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpMemMap - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`snpreader.SnpGen` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.SnpGen - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`snpreader.DistributedBed` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.snpreader.DistributedBed - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -**************************** -:mod:`kernelreader` Module -**************************** -.. automodule:: pysnptools.kernelreader - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:class:`kernelreader.KernelReader` -+++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.KernelReader - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs, col_property, row_property - - -:class:`kernelreader.KernelData` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.KernelData - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`kernelreader.KernelNpz` -++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.KernelNpz - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`kernelreader.KernelHdf5` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.KernelHdf5 - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`kernelreader.Identity` -+++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.Identity - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`kernelreader.SnpKernel` -+++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelreader.SnpKernel - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - - - -*************************** -:mod:`standardizer` Module -*************************** - -.. automodule:: pysnptools.standardizer - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:class:`standardizer.Standardizer` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.Standardizer - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs - -:class:`standardizer.Unit` -+++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.Unit - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize - -:class:`standardizer.Beta` -+++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.Beta - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize - -:class:`standardizer.Identity` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.Identity - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize,is_constant - -:class:`standardizer.DiagKtoN` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.DiagKtoN - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize - -:class:`standardizer.UnitTrained` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.UnitTrained - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize,is_constant - -:class:`standardizer.DiagKtoNTrained` -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.standardizer.DiagKtoNTrained - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize,is_constant - - -****************************************************** -:mod:`kernelstandardizer` Module -****************************************************** - -:class:`kernelstandardizer.KernelStandardizer` -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelstandardizer.KernelStandardizer - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs - -:class:`kernelstandardizer.DiagKtoN` -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelstandardizer.DiagKtoN - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize - -:class:`kernelstandardizer.DiagKtoNTrained` -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelstandardizer.DiagKtoNTrained - :no-index: - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize,is_constant - -:class:`kernelstandardizer.Identity` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.kernelstandardizer.Identity - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs,standardize - -************************** -:mod:`distreader` Module -************************** - -.. automodule:: pysnptools.distreader - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:class:`distreader.DistReader` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistReader - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs - - -:class:`distreader.Bgen` -+++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.Bgen - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - -.. autofunction:: pysnptools.distreader.bgen.default_iid_function - -.. autofunction:: pysnptools.distreader.bgen.default_sid_function - -.. autofunction:: pysnptools.distreader.bgen.default_sample_function - -.. autofunction:: pysnptools.distreader.bgen.default_id_rsid_function - -:class:`distreader.DistData` -+++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistData - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row - - -:class:`distreader.DistHdf5` -++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistHdf5 - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`distreader.DistNpz` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistNpz - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`distreader.DistMemMap` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistMemMap - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`distreader.DistGen` -++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.distreader.DistGen - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -*********************** -:mod:`pstreader` Module -*********************** - -.. automodule:: pysnptools.pstreader - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:class:`pstreader.PstReader` -+++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.pstreader.PstReader - :members: - :undoc-members: - :show-inheritance: - :special-members: __getitem__ - :exclude-members: copyinputs - - -:class:`pstreader.PstData` -++++++++++++++++++++++++++ -.. autoclass:: pysnptools.pstreader.PstData - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`pstreader.PstHdf5` -+++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.pstreader.PstHdf5 - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -:class:`pstreader.PstNpz` -+++++++++++++++++++++++++ -.. autoclass:: pysnptools.pstreader.PstNpz - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - -:class:`pstreader.PstMemMap` -++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.pstreader.PstMemMap - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: copyinputs, col, col_property, row, row_property - - -*********************** -:mod:`util` Module -*********************** - -:mod:`util` -++++++++++++++++++++++++++ -.. automodule:: pysnptools.util - :members: - :undoc-members: - :show-inheritance: - -.. autofunction:: pysnptools.util.example_file - -.. autofunction:: pysnptools.util.snp_gen - -:class:`util.IntRangeSet` -++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.IntRangeSet - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: __and__, __weakref__,__module__,__dict__, __add__ - - -:mod:`util.pheno` -+++++++++++++++++++++++++++++++++++++++++++++ -.. automodule:: pysnptools.util.pheno - :members: - :undoc-members: - :show-inheritance: - :special-members: - -******************************************* -:mod:`util.mapreduce1` Module -******************************************* -.. automodule:: pysnptools.util.mapreduce1 - :members: - :undoc-members: - :show-inheritance: - :special-members: - - -:mod:`util.mapreduce1` -++++++++++++++++++++++++++ - -.. autofunction:: pysnptools.util.mapreduce1.map_reduce - :no-index: - -:class:`util.mapreduce1.runner.Runner` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.mapreduce1.runner.Runner - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: run - -:class:`util.mapreduce1.runner.Local` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.mapreduce1.runner.Local - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: run - -:class:`util.mapreduce1.runner.LocalMultiProc` -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.mapreduce1.runner.LocalMultiProc - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: run - -:class:`util.mapreduce1.runner.LocalMultiThread` -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.mapreduce1.runner.LocalMultiThread - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: run - -:class:`util.mapreduce1.runner.LocalInParts` -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.mapreduce1.runner.LocalInParts - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: run - - -******************************************* -:mod:`util.filecache` Module -******************************************* -.. automodule:: pysnptools.util.filecache - :members: - :undoc-members: - :show-inheritance: - :special-members: - -:class:`util.filecache.FileCache` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.filecache.FileCache - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: - -:class:`util.filecache.LocalCache` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.filecache.LocalCache - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: - -:class:`util.filecache.PeerToPeer` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.filecache.PeerToPeer - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: - -:class:`util.filecache.Hashdown` -++++++++++++++++++++++++++++++++++++++++++++++++++ -.. autoclass:: pysnptools.util.filecache.Hashdown - :members: - :undoc-members: - :show-inheritance: - :special-members: - :exclude-members: - - -.. only:: html - -*********************** -Indices and Tables -*********************** - - * :ref:`genindex` - * :ref:`modindex` - * :ref:`search` diff --git a/docs/_static/base-stemmer.js b/docs/_static/base-stemmer.js deleted file mode 100644 index e6fa0c49..00000000 --- a/docs/_static/base-stemmer.js +++ /dev/null @@ -1,476 +0,0 @@ -// @ts-check - -/**@constructor*/ -BaseStemmer = function() { - /** @protected */ - this.current = ''; - this.cursor = 0; - this.limit = 0; - this.limit_backward = 0; - this.bra = 0; - this.ket = 0; - - /** - * @param {string} value - */ - this.setCurrent = function(value) { - this.current = value; - this.cursor = 0; - this.limit = this.current.length; - this.limit_backward = 0; - this.bra = this.cursor; - this.ket = this.limit; - }; - - /** - * @return {string} - */ - this.getCurrent = function() { - return this.current; - }; - - /** - * @param {BaseStemmer} other - */ - this.copy_from = function(other) { - /** @protected */ - this.current = other.current; - this.cursor = other.cursor; - this.limit = other.limit; - this.limit_backward = other.limit_backward; - this.bra = other.bra; - this.ket = other.ket; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.in_grouping = function(s, min, max) { - /** @protected */ - if (this.cursor >= this.limit) return false; - var ch = this.current.charCodeAt(this.cursor); - if (ch > max || ch < min) return false; - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false; - this.cursor++; - return true; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.go_in_grouping = function(s, min, max) { - /** @protected */ - while (this.cursor < this.limit) { - var ch = this.current.charCodeAt(this.cursor); - if (ch > max || ch < min) - return true; - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) - return true; - this.cursor++; - } - return false; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.in_grouping_b = function(s, min, max) { - /** @protected */ - if (this.cursor <= this.limit_backward) return false; - var ch = this.current.charCodeAt(this.cursor - 1); - if (ch > max || ch < min) return false; - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false; - this.cursor--; - return true; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.go_in_grouping_b = function(s, min, max) { - /** @protected */ - while (this.cursor > this.limit_backward) { - var ch = this.current.charCodeAt(this.cursor - 1); - if (ch > max || ch < min) return true; - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return true; - this.cursor--; - } - return false; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.out_grouping = function(s, min, max) { - /** @protected */ - if (this.cursor >= this.limit) return false; - var ch = this.current.charCodeAt(this.cursor); - if (ch > max || ch < min) { - this.cursor++; - return true; - } - ch -= min; - if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) == 0) { - this.cursor++; - return true; - } - return false; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.go_out_grouping = function(s, min, max) { - /** @protected */ - while (this.cursor < this.limit) { - var ch = this.current.charCodeAt(this.cursor); - if (ch <= max && ch >= min) { - ch -= min; - if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) != 0) { - return true; - } - } - this.cursor++; - } - return false; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.out_grouping_b = function(s, min, max) { - /** @protected */ - if (this.cursor <= this.limit_backward) return false; - var ch = this.current.charCodeAt(this.cursor - 1); - if (ch > max || ch < min) { - this.cursor--; - return true; - } - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) { - this.cursor--; - return true; - } - return false; - }; - - /** - * @param {number[]} s - * @param {number} min - * @param {number} max - * @return {boolean} - */ - this.go_out_grouping_b = function(s, min, max) { - /** @protected */ - while (this.cursor > this.limit_backward) { - var ch = this.current.charCodeAt(this.cursor - 1); - if (ch <= max && ch >= min) { - ch -= min; - if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) != 0) { - return true; - } - } - this.cursor--; - } - return false; - }; - - /** - * @param {string} s - * @return {boolean} - */ - this.eq_s = function(s) - { - /** @protected */ - if (this.limit - this.cursor < s.length) return false; - if (this.current.slice(this.cursor, this.cursor + s.length) != s) - { - return false; - } - this.cursor += s.length; - return true; - }; - - /** - * @param {string} s - * @return {boolean} - */ - this.eq_s_b = function(s) - { - /** @protected */ - if (this.cursor - this.limit_backward < s.length) return false; - if (this.current.slice(this.cursor - s.length, this.cursor) != s) - { - return false; - } - this.cursor -= s.length; - return true; - }; - - /** - * @param {Among[]} v - * @return {number} - */ - this.find_among = function(v) - { - /** @protected */ - var i = 0; - var j = v.length; - - var c = this.cursor; - var l = this.limit; - - var common_i = 0; - var common_j = 0; - - var first_key_inspected = false; - - while (true) - { - var k = i + ((j - i) >>> 1); - var diff = 0; - var common = common_i < common_j ? common_i : common_j; // smaller - // w[0]: string, w[1]: substring_i, w[2]: result, w[3]: function (optional) - var w = v[k]; - var i2; - for (i2 = common; i2 < w[0].length; i2++) - { - if (c + common == l) - { - diff = -1; - break; - } - diff = this.current.charCodeAt(c + common) - w[0].charCodeAt(i2); - if (diff != 0) break; - common++; - } - if (diff < 0) - { - j = k; - common_j = common; - } - else - { - i = k; - common_i = common; - } - if (j - i <= 1) - { - if (i > 0) break; // v->s has been inspected - if (j == i) break; // only one item in v - - // - but now we need to go round once more to get - // v->s inspected. This looks messy, but is actually - // the optimal approach. - - if (first_key_inspected) break; - first_key_inspected = true; - } - } - do { - var w = v[i]; - if (common_i >= w[0].length) - { - this.cursor = c + w[0].length; - if (w.length < 4) return w[2]; - var res = w[3](this); - this.cursor = c + w[0].length; - if (res) return w[2]; - } - i = w[1]; - } while (i >= 0); - return 0; - }; - - // find_among_b is for backwards processing. Same comments apply - /** - * @param {Among[]} v - * @return {number} - */ - this.find_among_b = function(v) - { - /** @protected */ - var i = 0; - var j = v.length - - var c = this.cursor; - var lb = this.limit_backward; - - var common_i = 0; - var common_j = 0; - - var first_key_inspected = false; - - while (true) - { - var k = i + ((j - i) >> 1); - var diff = 0; - var common = common_i < common_j ? common_i : common_j; - var w = v[k]; - var i2; - for (i2 = w[0].length - 1 - common; i2 >= 0; i2--) - { - if (c - common == lb) - { - diff = -1; - break; - } - diff = this.current.charCodeAt(c - 1 - common) - w[0].charCodeAt(i2); - if (diff != 0) break; - common++; - } - if (diff < 0) - { - j = k; - common_j = common; - } - else - { - i = k; - common_i = common; - } - if (j - i <= 1) - { - if (i > 0) break; - if (j == i) break; - if (first_key_inspected) break; - first_key_inspected = true; - } - } - do { - var w = v[i]; - if (common_i >= w[0].length) - { - this.cursor = c - w[0].length; - if (w.length < 4) return w[2]; - var res = w[3](this); - this.cursor = c - w[0].length; - if (res) return w[2]; - } - i = w[1]; - } while (i >= 0); - return 0; - }; - - /* to replace chars between c_bra and c_ket in this.current by the - * chars in s. - */ - /** - * @param {number} c_bra - * @param {number} c_ket - * @param {string} s - * @return {number} - */ - this.replace_s = function(c_bra, c_ket, s) - { - /** @protected */ - var adjustment = s.length - (c_ket - c_bra); - this.current = this.current.slice(0, c_bra) + s + this.current.slice(c_ket); - this.limit += adjustment; - if (this.cursor >= c_ket) this.cursor += adjustment; - else if (this.cursor > c_bra) this.cursor = c_bra; - return adjustment; - }; - - /** - * @return {boolean} - */ - this.slice_check = function() - { - /** @protected */ - if (this.bra < 0 || - this.bra > this.ket || - this.ket > this.limit || - this.limit > this.current.length) - { - return false; - } - return true; - }; - - /** - * @param {number} c_bra - * @return {boolean} - */ - this.slice_from = function(s) - { - /** @protected */ - var result = false; - if (this.slice_check()) - { - this.replace_s(this.bra, this.ket, s); - result = true; - } - return result; - }; - - /** - * @return {boolean} - */ - this.slice_del = function() - { - /** @protected */ - return this.slice_from(""); - }; - - /** - * @param {number} c_bra - * @param {number} c_ket - * @param {string} s - */ - this.insert = function(c_bra, c_ket, s) - { - /** @protected */ - var adjustment = this.replace_s(c_bra, c_ket, s); - if (c_bra <= this.bra) this.bra += adjustment; - if (c_bra <= this.ket) this.ket += adjustment; - }; - - /** - * @return {string} - */ - this.slice_to = function() - { - /** @protected */ - var result = ''; - if (this.slice_check()) - { - result = this.current.slice(this.bra, this.ket); - } - return result; - }; - - /** - * @return {string} - */ - this.assign_to = function() - { - /** @protected */ - return this.current.slice(0, this.limit); - }; -}; diff --git a/docs/_static/basic.css b/docs/_static/basic.css deleted file mode 100644 index 4738b2ed..00000000 --- a/docs/_static/basic.css +++ /dev/null @@ -1,906 +0,0 @@ -/* - * Sphinx stylesheet -- basic theme. - */ - -/* -- main layout ----------------------------------------------------------- */ - -div.clearer { - clear: both; -} - -div.section::after { - display: block; - content: ''; - clear: left; -} - -/* -- relbar ---------------------------------------------------------------- */ - -div.related { - width: 100%; - font-size: 90%; -} - -div.related h3 { - display: none; -} - -div.related ul { - margin: 0; - padding: 0 0 0 10px; - list-style: none; -} - -div.related li { - display: inline; -} - -div.related li.right { - float: right; - margin-right: 5px; -} - -/* -- sidebar --------------------------------------------------------------- */ - -div.sphinxsidebarwrapper { - padding: 10px 5px 0 10px; -} - -div.sphinxsidebar { - float: left; - width: 230px; - margin-left: -100%; - font-size: 90%; - word-wrap: break-word; - overflow-wrap : break-word; -} - -div.sphinxsidebar ul { - list-style: none; -} - -div.sphinxsidebar ul ul, -div.sphinxsidebar ul.want-points { - margin-left: 20px; - list-style: square; -} - -div.sphinxsidebar ul ul { - margin-top: 0; - margin-bottom: 0; -} - -div.sphinxsidebar form { - margin-top: 10px; -} - -div.sphinxsidebar input { - border: 1px solid #98dbcc; - font-family: sans-serif; - font-size: 1em; -} - -div.sphinxsidebar #searchbox form.search { - overflow: hidden; -} - -div.sphinxsidebar #searchbox input[type="text"] { - float: left; - width: 80%; - padding: 0.25em; - box-sizing: border-box; -} - -div.sphinxsidebar #searchbox input[type="submit"] { - float: left; - width: 20%; - border-left: none; - padding: 0.25em; - box-sizing: border-box; -} - - -img { - border: 0; - max-width: 100%; -} - -/* -- search page ----------------------------------------------------------- */ - -ul.search { - margin-top: 10px; -} - -ul.search li { - padding: 5px 0; -} - -ul.search li a { - font-weight: bold; -} - -ul.search li p.context { - color: #888; - margin: 2px 0 0 30px; - text-align: left; -} - -ul.keywordmatches li.goodmatch a { - font-weight: bold; -} - -/* -- index page ------------------------------------------------------------ */ - -table.contentstable { - width: 90%; - margin-left: auto; - margin-right: auto; -} - -table.contentstable p.biglink { - line-height: 150%; -} - -a.biglink { - font-size: 1.3em; -} - -span.linkdescr { - font-style: italic; - padding-top: 5px; - font-size: 90%; -} - -/* -- general index --------------------------------------------------------- */ - -table.indextable { - width: 100%; -} - -table.indextable td { - text-align: left; - vertical-align: top; -} - -table.indextable ul { - margin-top: 0; - margin-bottom: 0; - list-style-type: none; -} - -table.indextable > tbody > tr > td > ul { - padding-left: 0em; -} - -table.indextable tr.pcap { - height: 10px; -} - -table.indextable tr.cap { - margin-top: 10px; - background-color: #f2f2f2; -} - -img.toggler { - margin-right: 3px; - margin-top: 3px; - cursor: pointer; -} - -div.modindex-jumpbox { - border-top: 1px solid #ddd; - border-bottom: 1px solid #ddd; - margin: 1em 0 1em 0; - padding: 0.4em; -} - -div.genindex-jumpbox { - border-top: 1px solid #ddd; - border-bottom: 1px solid #ddd; - margin: 1em 0 1em 0; - padding: 0.4em; -} - -/* -- domain module index --------------------------------------------------- */ - -table.modindextable td { - padding: 2px; - border-collapse: collapse; -} - -/* -- general body styles --------------------------------------------------- */ - -div.body { - min-width: 360px; - max-width: 800px; -} - -div.body p, div.body dd, div.body li, div.body blockquote { - -moz-hyphens: auto; - -ms-hyphens: auto; - -webkit-hyphens: auto; - hyphens: auto; -} - -a.headerlink { - visibility: hidden; -} - -a:visited { - color: #551A8B; -} - -h1:hover > a.headerlink, -h2:hover > a.headerlink, -h3:hover > a.headerlink, -h4:hover > a.headerlink, -h5:hover > a.headerlink, -h6:hover > a.headerlink, -dt:hover > a.headerlink, -caption:hover > a.headerlink, -p.caption:hover > a.headerlink, -div.code-block-caption:hover > a.headerlink { - visibility: visible; -} - -div.body p.caption { - text-align: inherit; -} - -div.body td { - text-align: left; -} - -.first { - margin-top: 0 !important; -} - -p.rubric { - margin-top: 30px; - font-weight: bold; -} - -img.align-left, figure.align-left, .figure.align-left, object.align-left { - clear: left; - float: left; - margin-right: 1em; -} - -img.align-right, figure.align-right, .figure.align-right, object.align-right { - clear: right; - float: right; - margin-left: 1em; -} - -img.align-center, figure.align-center, .figure.align-center, object.align-center { - display: block; - margin-left: auto; - margin-right: auto; -} - -img.align-default, figure.align-default, .figure.align-default { - display: block; - margin-left: auto; - margin-right: auto; -} - -.align-left { - text-align: left; -} - -.align-center { - text-align: center; -} - -.align-default { - text-align: center; -} - -.align-right { - text-align: right; -} - -/* -- sidebars -------------------------------------------------------------- */ - -div.sidebar, -aside.sidebar { - margin: 0 0 0.5em 1em; - border: 1px solid #ddb; - padding: 7px; - background-color: #ffe; - width: 40%; - float: right; - clear: right; - overflow-x: auto; -} - -p.sidebar-title { - font-weight: bold; -} - -nav.contents, -aside.topic, -div.admonition, div.topic, blockquote { - clear: left; -} - -/* -- topics ---------------------------------------------------------------- */ - -nav.contents, -aside.topic, -div.topic { - border: 1px solid #ccc; - padding: 7px; - margin: 10px 0 10px 0; -} - -p.topic-title { - font-size: 1.1em; - font-weight: bold; - margin-top: 10px; -} - -/* -- admonitions ----------------------------------------------------------- */ - -div.admonition { - margin-top: 10px; - margin-bottom: 10px; - padding: 7px; -} - -div.admonition dt { - font-weight: bold; -} - -p.admonition-title { - margin: 0px 10px 5px 0px; - font-weight: bold; -} - -div.body p.centered { - text-align: center; - margin-top: 25px; -} - -/* -- content of sidebars/topics/admonitions -------------------------------- */ - -div.sidebar > :last-child, -aside.sidebar > :last-child, -nav.contents > :last-child, -aside.topic > :last-child, -div.topic > :last-child, -div.admonition > :last-child { - margin-bottom: 0; -} - -div.sidebar::after, -aside.sidebar::after, -nav.contents::after, -aside.topic::after, -div.topic::after, -div.admonition::after, -blockquote::after { - display: block; - content: ''; - clear: both; -} - -/* -- tables ---------------------------------------------------------------- */ - -table.docutils { - margin-top: 10px; - margin-bottom: 10px; - border: 0; - border-collapse: collapse; -} - -table.align-center { - margin-left: auto; - margin-right: auto; -} - -table.align-default { - margin-left: auto; - margin-right: auto; -} - -table caption span.caption-number { - font-style: italic; -} - -table caption span.caption-text { -} - -table.docutils td, table.docutils th { - padding: 1px 8px 1px 5px; - border-top: 0; - border-left: 0; - border-right: 0; - border-bottom: 1px solid #aaa; -} - -th { - text-align: left; - padding-right: 5px; -} - -table.citation { - border-left: solid 1px gray; - margin-left: 1px; -} - -table.citation td { - border-bottom: none; -} - -th > :first-child, -td > :first-child { - margin-top: 0px; -} - -th > :last-child, -td > :last-child { - margin-bottom: 0px; -} - -/* -- figures --------------------------------------------------------------- */ - -div.figure, figure { - margin: 0.5em; - padding: 0.5em; -} - -div.figure p.caption, figcaption { - padding: 0.3em; -} - -div.figure p.caption span.caption-number, -figcaption span.caption-number { - font-style: italic; -} - -div.figure p.caption span.caption-text, -figcaption span.caption-text { -} - -/* -- field list styles ----------------------------------------------------- */ - -table.field-list td, table.field-list th { - border: 0 !important; -} - -.field-list ul { - margin: 0; - padding-left: 1em; -} - -.field-list p { - margin: 0; -} - -.field-name { - -moz-hyphens: manual; - -ms-hyphens: manual; - -webkit-hyphens: manual; - hyphens: manual; -} - -/* -- hlist styles ---------------------------------------------------------- */ - -table.hlist { - margin: 1em 0; -} - -table.hlist td { - vertical-align: top; -} - -/* -- object description styles --------------------------------------------- */ - -.sig { - font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; -} - -.sig-name, code.descname { - background-color: transparent; - font-weight: bold; -} - -.sig-name { - font-size: 1.1em; -} - -code.descname { - font-size: 1.2em; -} - -.sig-prename, code.descclassname { - background-color: transparent; -} - -.optional { - font-size: 1.3em; -} - -.sig-paren { - font-size: larger; -} - -.sig-param.n { - font-style: italic; -} - -/* C++ specific styling */ - -.sig-inline.c-texpr, -.sig-inline.cpp-texpr { - font-family: unset; -} - -.sig.c .k, .sig.c .kt, -.sig.cpp .k, .sig.cpp .kt { - color: #0033B3; -} - -.sig.c .m, -.sig.cpp .m { - color: #1750EB; -} - -.sig.c .s, .sig.c .sc, -.sig.cpp .s, .sig.cpp .sc { - color: #067D17; -} - - -/* -- other body styles ----------------------------------------------------- */ - -ol.arabic { - list-style: decimal; -} - -ol.loweralpha { - list-style: lower-alpha; -} - -ol.upperalpha { - list-style: upper-alpha; -} - -ol.lowerroman { - list-style: lower-roman; -} - -ol.upperroman { - list-style: upper-roman; -} - -:not(li) > ol > li:first-child > :first-child, -:not(li) > ul > li:first-child > :first-child { - margin-top: 0px; -} - -:not(li) > ol > li:last-child > :last-child, -:not(li) > ul > li:last-child > :last-child { - margin-bottom: 0px; -} - -ol.simple ol p, -ol.simple ul p, -ul.simple ol p, -ul.simple ul p { - margin-top: 0; -} - -ol.simple > li:not(:first-child) > p, -ul.simple > li:not(:first-child) > p { - margin-top: 0; -} - -ol.simple p, -ul.simple p { - margin-bottom: 0; -} - -aside.footnote > span, -div.citation > span { - float: left; -} -aside.footnote > span:last-of-type, -div.citation > span:last-of-type { - padding-right: 0.5em; -} -aside.footnote > p { - margin-left: 2em; -} -div.citation > p { - margin-left: 4em; -} -aside.footnote > p:last-of-type, -div.citation > p:last-of-type { - margin-bottom: 0em; -} -aside.footnote > p:last-of-type:after, -div.citation > p:last-of-type:after { - content: ""; - clear: both; -} - -dl.field-list { - display: grid; - grid-template-columns: fit-content(30%) auto; -} - -dl.field-list > dt { - font-weight: bold; - word-break: break-word; - padding-left: 0.5em; - padding-right: 5px; -} - -dl.field-list > dd { - padding-left: 0.5em; - margin-top: 0em; - margin-left: 0em; - margin-bottom: 0em; -} - -dl { - margin-bottom: 15px; -} - -dd > :first-child { - margin-top: 0px; -} - -dd ul, dd table { - margin-bottom: 10px; -} - -dd { - margin-top: 3px; - margin-bottom: 10px; - margin-left: 30px; -} - -.sig dd { - margin-top: 0px; - margin-bottom: 0px; -} - -.sig dl { - margin-top: 0px; - margin-bottom: 0px; -} - -dl > dd:last-child, -dl > dd:last-child > :last-child { - margin-bottom: 0; -} - -dt:target, span.highlighted { - background-color: #fbe54e; -} - -rect.highlighted { - fill: #fbe54e; -} - -dl.glossary dt { - font-weight: bold; - font-size: 1.1em; -} - -.versionmodified { - font-style: italic; -} - -.system-message { - background-color: #fda; - padding: 5px; - border: 3px solid red; -} - -.footnote:target { - background-color: #ffa; -} - -.line-block { - display: block; - margin-top: 1em; - margin-bottom: 1em; -} - -.line-block .line-block { - margin-top: 0; - margin-bottom: 0; - margin-left: 1.5em; -} - -.guilabel, .menuselection { - font-family: sans-serif; -} - -.accelerator { - text-decoration: underline; -} - -.classifier { - font-style: oblique; -} - -.classifier:before { - font-style: normal; - margin: 0 0.5em; - content: ":"; - display: inline-block; -} - -abbr, acronym { - border-bottom: dotted 1px; - cursor: help; -} - -/* -- code displays --------------------------------------------------------- */ - -pre { - overflow: auto; - overflow-y: hidden; /* fixes display issues on Chrome browsers */ -} - -pre, div[class*="highlight-"] { - clear: both; -} - -span.pre { - -moz-hyphens: none; - -ms-hyphens: none; - -webkit-hyphens: none; - hyphens: none; - white-space: nowrap; -} - -div[class*="highlight-"] { - margin: 1em 0; -} - -td.linenos pre { - border: 0; - background-color: transparent; - color: #aaa; -} - -table.highlighttable { - display: block; -} - -table.highlighttable tbody { - display: block; -} - -table.highlighttable tr { - display: flex; -} - -table.highlighttable td { - margin: 0; - padding: 0; -} - -table.highlighttable td.linenos { - padding-right: 0.5em; -} - -table.highlighttable td.code { - flex: 1; - overflow: hidden; -} - -.highlight .hll { - display: block; -} - -div.highlight pre, -table.highlighttable pre { - margin: 0; -} - -div.code-block-caption + div { - margin-top: 0; -} - -div.code-block-caption { - margin-top: 1em; - padding: 2px 5px; - font-size: small; -} - -div.code-block-caption code { - background-color: transparent; -} - -table.highlighttable td.linenos, -span.linenos, -div.highlight span.gp { /* gp: Generic.Prompt */ - user-select: none; - -webkit-user-select: text; /* Safari fallback only */ - -webkit-user-select: none; /* Chrome/Safari */ - -moz-user-select: none; /* Firefox */ - -ms-user-select: none; /* IE10+ */ -} - -div.code-block-caption span.caption-number { - padding: 0.1em 0.3em; - font-style: italic; -} - -div.code-block-caption span.caption-text { -} - -div.literal-block-wrapper { - margin: 1em 0; -} - -code.xref, a code { - background-color: transparent; - font-weight: bold; -} - -h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { - background-color: transparent; -} - -.viewcode-link { - float: right; -} - -.viewcode-back { - float: right; - font-family: sans-serif; -} - -div.viewcode-block:target { - margin: -1px -10px; - padding: 0 10px; -} - -/* -- math display ---------------------------------------------------------- */ - -img.math { - vertical-align: middle; -} - -div.body div.math p { - text-align: center; -} - -span.eqno { - float: right; -} - -span.eqno a.headerlink { - position: absolute; - z-index: 1; -} - -div.math:hover a.headerlink { - visibility: visible; -} - -/* -- printout stylesheet --------------------------------------------------- */ - -@media print { - div.document, - div.documentwrapper, - div.bodywrapper { - margin: 0 !important; - width: 100%; - } - - div.sphinxsidebar, - div.related, - div.footer, - #top-link { - display: none; - } -} \ No newline at end of file diff --git a/docs/_static/classic.css b/docs/_static/classic.css deleted file mode 100644 index 8851cb72..00000000 --- a/docs/_static/classic.css +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Sphinx stylesheet -- classic theme. - */ - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -html { - /* CSS hack for macOS's scrollbar (see #1125) */ - background-color: #FFFFFF; -} - -body { - font-family: sans-serif; - font-size: 100%; - background-color: #11303d; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - display: flex; - background-color: #1c4e63; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 230px; -} - -div.body { - background-color: #ffffff; - color: #000000; - padding: 0 20px 30px 20px; -} - -div.footer { - color: #ffffff; - width: 100%; - padding: 9px 0 9px 0; - text-align: center; - font-size: 75%; -} - -div.footer a { - color: #ffffff; - text-decoration: underline; -} - -div.related { - background-color: #133f52; - line-height: 30px; - color: #ffffff; -} - -div.related a { - color: #ffffff; -} - -div.sphinxsidebar { -} - -div.sphinxsidebar h3 { - font-family: 'Trebuchet MS', sans-serif; - color: #ffffff; - font-size: 1.4em; - font-weight: normal; - margin: 0; - padding: 0; -} - -div.sphinxsidebar h3 a { - color: #ffffff; -} - -div.sphinxsidebar h4 { - font-family: 'Trebuchet MS', sans-serif; - color: #ffffff; - font-size: 1.3em; - font-weight: normal; - margin: 5px 0 0 0; - padding: 0; -} - -div.sphinxsidebar p { - color: #ffffff; -} - -div.sphinxsidebar p.topless { - margin: 5px 10px 10px 10px; -} - -div.sphinxsidebar ul { - margin: 10px; - padding: 0; - color: #ffffff; -} - -div.sphinxsidebar a { - color: #98dbcc; -} - -div.sphinxsidebar input { - border: 1px solid #98dbcc; - font-family: sans-serif; - font-size: 1em; -} - - - -/* -- hyperlink styles ------------------------------------------------------ */ - -a { - color: #355f7c; - text-decoration: none; -} - -a:visited { - color: #551a8b; - text-decoration: none; -} - -a:hover { - text-decoration: underline; -} - - - -/* -- body styles ----------------------------------------------------------- */ - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: 'Trebuchet MS', sans-serif; - background-color: #f2f2f2; - font-weight: normal; - color: #20435c; - border-bottom: 1px solid #ccc; - margin: 20px -20px 10px -20px; - padding: 3px 0 3px 10px; -} - -div.body h1 { margin-top: 0; font-size: 200%; } -div.body h2 { font-size: 160%; } -div.body h3 { font-size: 140%; } -div.body h4 { font-size: 120%; } -div.body h5 { font-size: 110%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: #c60f0f; - font-size: 0.8em; - padding: 0 4px 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - background-color: #c60f0f; - color: white; -} - -div.body p, div.body dd, div.body li, div.body blockquote { - text-align: justify; - line-height: 130%; -} - -div.admonition p.admonition-title + p { - display: inline; -} - -div.admonition p { - margin-bottom: 5px; -} - -div.admonition pre { - margin-bottom: 5px; -} - -div.admonition ul, div.admonition ol { - margin-bottom: 5px; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -nav.contents, -aside.topic, -div.topic { - background-color: #eee; -} - -div.warning { - background-color: #ffe4e4; - border: 1px solid #f66; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre { - padding: 5px; - background-color: unset; - color: unset; - line-height: 120%; - border: 1px solid #ac9; - border-left: none; - border-right: none; -} - -code { - background-color: #ecf0f3; - padding: 0 1px 0 1px; - font-size: 0.95em; -} - -th, dl.field-list > dt { - background-color: #ede; -} - -.warning code { - background: #efc2c2; -} - -.note code { - background: #d6d6d6; -} - -.viewcode-back { - font-family: sans-serif; -} - -div.viewcode-block:target { - background-color: #f4debf; - border-top: 1px solid #ac9; - border-bottom: 1px solid #ac9; -} - -div.code-block-caption { - color: #efefef; - background-color: #1c4e63; -} \ No newline at end of file diff --git a/docs/_static/default.css b/docs/_static/default.css deleted file mode 100644 index 81b93636..00000000 --- a/docs/_static/default.css +++ /dev/null @@ -1 +0,0 @@ -@import url("classic.css"); diff --git a/docs/_static/doctools.js b/docs/_static/doctools.js deleted file mode 100644 index 807cdb17..00000000 --- a/docs/_static/doctools.js +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Base JavaScript utilities for all Sphinx HTML documentation. - */ -"use strict"; - -const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([ - "TEXTAREA", - "INPUT", - "SELECT", - "BUTTON", -]); - -const _ready = (callback) => { - if (document.readyState !== "loading") { - callback(); - } else { - document.addEventListener("DOMContentLoaded", callback); - } -}; - -/** - * Small JavaScript module for the documentation. - */ -const Documentation = { - init: () => { - Documentation.initDomainIndexTable(); - Documentation.initOnKeyListeners(); - }, - - /** - * i18n support - */ - TRANSLATIONS: {}, - PLURAL_EXPR: (n) => (n === 1 ? 0 : 1), - LOCALE: "unknown", - - // gettext and ngettext don't access this so that the functions - // can safely bound to a different name (_ = Documentation.gettext) - gettext: (string) => { - const translated = Documentation.TRANSLATIONS[string]; - switch (typeof translated) { - case "undefined": - return string; // no translation - case "string": - return translated; // translation exists - default: - return translated[0]; // (singular, plural) translation tuple exists - } - }, - - ngettext: (singular, plural, n) => { - const translated = Documentation.TRANSLATIONS[singular]; - if (typeof translated !== "undefined") - return translated[Documentation.PLURAL_EXPR(n)]; - return n === 1 ? singular : plural; - }, - - addTranslations: (catalog) => { - Object.assign(Documentation.TRANSLATIONS, catalog.messages); - Documentation.PLURAL_EXPR = new Function( - "n", - `return (${catalog.plural_expr})`, - ); - Documentation.LOCALE = catalog.locale; - }, - - /** - * helper function to focus on search bar - */ - focusSearchBar: () => { - document.querySelectorAll("input[name=q]")[0]?.focus(); - }, - - /** - * Initialise the domain index toggle buttons - */ - initDomainIndexTable: () => { - const toggler = (el) => { - const idNumber = el.id.substr(7); - const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`); - if (el.src.substr(-9) === "minus.png") { - el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`; - toggledRows.forEach((el) => (el.style.display = "none")); - } else { - el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`; - toggledRows.forEach((el) => (el.style.display = "")); - } - }; - - const togglerElements = document.querySelectorAll("img.toggler"); - togglerElements.forEach((el) => - el.addEventListener("click", (event) => toggler(event.currentTarget)), - ); - togglerElements.forEach((el) => (el.style.display = "")); - if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler); - }, - - initOnKeyListeners: () => { - // only install a listener if it is really needed - if ( - !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS - && !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS - ) - return; - - document.addEventListener("keydown", (event) => { - // bail for input elements - if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) - return; - // bail with special keys - if (event.altKey || event.ctrlKey || event.metaKey) return; - - if (!event.shiftKey) { - switch (event.key) { - case "ArrowLeft": - if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; - - const prevLink = document.querySelector('link[rel="prev"]'); - if (prevLink && prevLink.href) { - window.location.href = prevLink.href; - event.preventDefault(); - } - break; - case "ArrowRight": - if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; - - const nextLink = document.querySelector('link[rel="next"]'); - if (nextLink && nextLink.href) { - window.location.href = nextLink.href; - event.preventDefault(); - } - break; - } - } - - // some keyboard layouts may need Shift to get / - switch (event.key) { - case "/": - if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; - Documentation.focusSearchBar(); - event.preventDefault(); - } - }); - }, -}; - -// quick alias for translations -const _ = Documentation.gettext; - -_ready(Documentation.init); diff --git a/docs/_static/documentation_options.js b/docs/_static/documentation_options.js deleted file mode 100644 index 89435bb4..00000000 --- a/docs/_static/documentation_options.js +++ /dev/null @@ -1,13 +0,0 @@ -const DOCUMENTATION_OPTIONS = { - VERSION: '1.0.0', - LANGUAGE: 'en', - COLLAPSE_INDEX: false, - BUILDER: 'html', - FILE_SUFFIX: '.html', - LINK_SUFFIX: '.html', - HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt', - NAVIGATION_WITH_KEYS: false, - SHOW_SEARCH_SUMMARY: true, - ENABLE_SEARCH_SHORTCUTS: true, -}; \ No newline at end of file diff --git a/docs/_static/english-stemmer.js b/docs/_static/english-stemmer.js deleted file mode 100644 index 056760ee..00000000 --- a/docs/_static/english-stemmer.js +++ /dev/null @@ -1,1066 +0,0 @@ -// Generated from english.sbl by Snowball 3.0.1 - https://snowballstem.org/ - -/**@constructor*/ -var EnglishStemmer = function() { - var base = new BaseStemmer(); - - /** @const */ var a_0 = [ - ["arsen", -1, -1], - ["commun", -1, -1], - ["emerg", -1, -1], - ["gener", -1, -1], - ["later", -1, -1], - ["organ", -1, -1], - ["past", -1, -1], - ["univers", -1, -1] - ]; - - /** @const */ var a_1 = [ - ["'", -1, 1], - ["'s'", 0, 1], - ["'s", -1, 1] - ]; - - /** @const */ var a_2 = [ - ["ied", -1, 2], - ["s", -1, 3], - ["ies", 1, 2], - ["sses", 1, 1], - ["ss", 1, -1], - ["us", 1, -1] - ]; - - /** @const */ var a_3 = [ - ["succ", -1, 1], - ["proc", -1, 1], - ["exc", -1, 1] - ]; - - /** @const */ var a_4 = [ - ["even", -1, 2], - ["cann", -1, 2], - ["inn", -1, 2], - ["earr", -1, 2], - ["herr", -1, 2], - ["out", -1, 2], - ["y", -1, 1] - ]; - - /** @const */ var a_5 = [ - ["", -1, -1], - ["ed", 0, 2], - ["eed", 1, 1], - ["ing", 0, 3], - ["edly", 0, 2], - ["eedly", 4, 1], - ["ingly", 0, 2] - ]; - - /** @const */ var a_6 = [ - ["", -1, 3], - ["bb", 0, 2], - ["dd", 0, 2], - ["ff", 0, 2], - ["gg", 0, 2], - ["bl", 0, 1], - ["mm", 0, 2], - ["nn", 0, 2], - ["pp", 0, 2], - ["rr", 0, 2], - ["at", 0, 1], - ["tt", 0, 2], - ["iz", 0, 1] - ]; - - /** @const */ var a_7 = [ - ["anci", -1, 3], - ["enci", -1, 2], - ["ogi", -1, 14], - ["li", -1, 16], - ["bli", 3, 12], - ["abli", 4, 4], - ["alli", 3, 8], - ["fulli", 3, 9], - ["lessli", 3, 15], - ["ousli", 3, 10], - ["entli", 3, 5], - ["aliti", -1, 8], - ["biliti", -1, 12], - ["iviti", -1, 11], - ["tional", -1, 1], - ["ational", 14, 7], - ["alism", -1, 8], - ["ation", -1, 7], - ["ization", 17, 6], - ["izer", -1, 6], - ["ator", -1, 7], - ["iveness", -1, 11], - ["fulness", -1, 9], - ["ousness", -1, 10], - ["ogist", -1, 13] - ]; - - /** @const */ var a_8 = [ - ["icate", -1, 4], - ["ative", -1, 6], - ["alize", -1, 3], - ["iciti", -1, 4], - ["ical", -1, 4], - ["tional", -1, 1], - ["ational", 5, 2], - ["ful", -1, 5], - ["ness", -1, 5] - ]; - - /** @const */ var a_9 = [ - ["ic", -1, 1], - ["ance", -1, 1], - ["ence", -1, 1], - ["able", -1, 1], - ["ible", -1, 1], - ["ate", -1, 1], - ["ive", -1, 1], - ["ize", -1, 1], - ["iti", -1, 1], - ["al", -1, 1], - ["ism", -1, 1], - ["ion", -1, 2], - ["er", -1, 1], - ["ous", -1, 1], - ["ant", -1, 1], - ["ent", -1, 1], - ["ment", 15, 1], - ["ement", 16, 1] - ]; - - /** @const */ var a_10 = [ - ["e", -1, 1], - ["l", -1, 2] - ]; - - /** @const */ var a_11 = [ - ["andes", -1, -1], - ["atlas", -1, -1], - ["bias", -1, -1], - ["cosmos", -1, -1], - ["early", -1, 5], - ["gently", -1, 3], - ["howe", -1, -1], - ["idly", -1, 2], - ["news", -1, -1], - ["only", -1, 6], - ["singly", -1, 7], - ["skies", -1, 1], - ["sky", -1, -1], - ["ugly", -1, 4] - ]; - - /** @const */ var /** Array */ g_aeo = [17, 64]; - - /** @const */ var /** Array */ g_v = [17, 65, 16, 1]; - - /** @const */ var /** Array */ g_v_WXY = [1, 17, 65, 208, 1]; - - /** @const */ var /** Array */ g_valid_LI = [55, 141, 2]; - - var /** boolean */ B_Y_found = false; - var /** number */ I_p2 = 0; - var /** number */ I_p1 = 0; - - - /** @return {boolean} */ - function r_prelude() { - B_Y_found = false; - /** @const */ var /** number */ v_1 = base.cursor; - lab0: { - base.bra = base.cursor; - if (!(base.eq_s("'"))) - { - break lab0; - } - base.ket = base.cursor; - if (!base.slice_del()) - { - return false; - } - } - base.cursor = v_1; - /** @const */ var /** number */ v_2 = base.cursor; - lab1: { - base.bra = base.cursor; - if (!(base.eq_s("y"))) - { - break lab1; - } - base.ket = base.cursor; - if (!base.slice_from("Y")) - { - return false; - } - B_Y_found = true; - } - base.cursor = v_2; - /** @const */ var /** number */ v_3 = base.cursor; - lab2: { - while(true) - { - /** @const */ var /** number */ v_4 = base.cursor; - lab3: { - golab4: while(true) - { - /** @const */ var /** number */ v_5 = base.cursor; - lab5: { - if (!(base.in_grouping(g_v, 97, 121))) - { - break lab5; - } - base.bra = base.cursor; - if (!(base.eq_s("y"))) - { - break lab5; - } - base.ket = base.cursor; - base.cursor = v_5; - break golab4; - } - base.cursor = v_5; - if (base.cursor >= base.limit) - { - break lab3; - } - base.cursor++; - } - if (!base.slice_from("Y")) - { - return false; - } - B_Y_found = true; - continue; - } - base.cursor = v_4; - break; - } - } - base.cursor = v_3; - return true; - }; - - /** @return {boolean} */ - function r_mark_regions() { - I_p1 = base.limit; - I_p2 = base.limit; - /** @const */ var /** number */ v_1 = base.cursor; - lab0: { - lab1: { - /** @const */ var /** number */ v_2 = base.cursor; - lab2: { - if (base.find_among(a_0) == 0) - { - break lab2; - } - break lab1; - } - base.cursor = v_2; - if (!base.go_out_grouping(g_v, 97, 121)) - { - break lab0; - } - base.cursor++; - if (!base.go_in_grouping(g_v, 97, 121)) - { - break lab0; - } - base.cursor++; - } - I_p1 = base.cursor; - if (!base.go_out_grouping(g_v, 97, 121)) - { - break lab0; - } - base.cursor++; - if (!base.go_in_grouping(g_v, 97, 121)) - { - break lab0; - } - base.cursor++; - I_p2 = base.cursor; - } - base.cursor = v_1; - return true; - }; - - /** @return {boolean} */ - function r_shortv() { - lab0: { - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab1: { - if (!(base.out_grouping_b(g_v_WXY, 89, 121))) - { - break lab1; - } - if (!(base.in_grouping_b(g_v, 97, 121))) - { - break lab1; - } - if (!(base.out_grouping_b(g_v, 97, 121))) - { - break lab1; - } - break lab0; - } - base.cursor = base.limit - v_1; - lab2: { - if (!(base.out_grouping_b(g_v, 97, 121))) - { - break lab2; - } - if (!(base.in_grouping_b(g_v, 97, 121))) - { - break lab2; - } - if (base.cursor > base.limit_backward) - { - break lab2; - } - break lab0; - } - base.cursor = base.limit - v_1; - if (!(base.eq_s_b("past"))) - { - return false; - } - } - return true; - }; - - /** @return {boolean} */ - function r_R1() { - return I_p1 <= base.cursor; - }; - - /** @return {boolean} */ - function r_R2() { - return I_p2 <= base.cursor; - }; - - /** @return {boolean} */ - function r_Step_1a() { - var /** number */ among_var; - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab0: { - base.ket = base.cursor; - if (base.find_among_b(a_1) == 0) - { - base.cursor = base.limit - v_1; - break lab0; - } - base.bra = base.cursor; - if (!base.slice_del()) - { - return false; - } - } - base.ket = base.cursor; - among_var = base.find_among_b(a_2); - if (among_var == 0) - { - return false; - } - base.bra = base.cursor; - switch (among_var) { - case 1: - if (!base.slice_from("ss")) - { - return false; - } - break; - case 2: - lab1: { - /** @const */ var /** number */ v_2 = base.limit - base.cursor; - lab2: { - { - /** @const */ var /** number */ c1 = base.cursor - 2; - if (c1 < base.limit_backward) - { - break lab2; - } - base.cursor = c1; - } - if (!base.slice_from("i")) - { - return false; - } - break lab1; - } - base.cursor = base.limit - v_2; - if (!base.slice_from("ie")) - { - return false; - } - } - break; - case 3: - if (base.cursor <= base.limit_backward) - { - return false; - } - base.cursor--; - if (!base.go_out_grouping_b(g_v, 97, 121)) - { - return false; - } - base.cursor--; - if (!base.slice_del()) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_Step_1b() { - var /** number */ among_var; - base.ket = base.cursor; - among_var = base.find_among_b(a_5); - base.bra = base.cursor; - lab0: { - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab1: { - switch (among_var) { - case 1: - /** @const */ var /** number */ v_2 = base.limit - base.cursor; - lab2: { - lab3: { - /** @const */ var /** number */ v_3 = base.limit - base.cursor; - lab4: { - if (base.find_among_b(a_3) == 0) - { - break lab4; - } - if (base.cursor > base.limit_backward) - { - break lab4; - } - break lab3; - } - base.cursor = base.limit - v_3; - if (!r_R1()) - { - break lab2; - } - if (!base.slice_from("ee")) - { - return false; - } - } - } - base.cursor = base.limit - v_2; - break; - case 2: - break lab1; - case 3: - among_var = base.find_among_b(a_4); - if (among_var == 0) - { - break lab1; - } - switch (among_var) { - case 1: - /** @const */ var /** number */ v_4 = base.limit - base.cursor; - if (!(base.out_grouping_b(g_v, 97, 121))) - { - break lab1; - } - if (base.cursor > base.limit_backward) - { - break lab1; - } - base.cursor = base.limit - v_4; - base.bra = base.cursor; - if (!base.slice_from("ie")) - { - return false; - } - break; - case 2: - if (base.cursor > base.limit_backward) - { - break lab1; - } - break; - } - break; - } - break lab0; - } - base.cursor = base.limit - v_1; - /** @const */ var /** number */ v_5 = base.limit - base.cursor; - if (!base.go_out_grouping_b(g_v, 97, 121)) - { - return false; - } - base.cursor--; - base.cursor = base.limit - v_5; - if (!base.slice_del()) - { - return false; - } - base.ket = base.cursor; - base.bra = base.cursor; - /** @const */ var /** number */ v_6 = base.limit - base.cursor; - among_var = base.find_among_b(a_6); - switch (among_var) { - case 1: - if (!base.slice_from("e")) - { - return false; - } - return false; - case 2: - { - /** @const */ var /** number */ v_7 = base.limit - base.cursor; - lab5: { - if (!(base.in_grouping_b(g_aeo, 97, 111))) - { - break lab5; - } - if (base.cursor > base.limit_backward) - { - break lab5; - } - return false; - } - base.cursor = base.limit - v_7; - } - break; - case 3: - if (base.cursor != I_p1) - { - return false; - } - /** @const */ var /** number */ v_8 = base.limit - base.cursor; - if (!r_shortv()) - { - return false; - } - base.cursor = base.limit - v_8; - if (!base.slice_from("e")) - { - return false; - } - return false; - } - base.cursor = base.limit - v_6; - base.ket = base.cursor; - if (base.cursor <= base.limit_backward) - { - return false; - } - base.cursor--; - base.bra = base.cursor; - if (!base.slice_del()) - { - return false; - } - } - return true; - }; - - /** @return {boolean} */ - function r_Step_1c() { - base.ket = base.cursor; - lab0: { - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab1: { - if (!(base.eq_s_b("y"))) - { - break lab1; - } - break lab0; - } - base.cursor = base.limit - v_1; - if (!(base.eq_s_b("Y"))) - { - return false; - } - } - base.bra = base.cursor; - if (!(base.out_grouping_b(g_v, 97, 121))) - { - return false; - } - lab2: { - if (base.cursor > base.limit_backward) - { - break lab2; - } - return false; - } - if (!base.slice_from("i")) - { - return false; - } - return true; - }; - - /** @return {boolean} */ - function r_Step_2() { - var /** number */ among_var; - base.ket = base.cursor; - among_var = base.find_among_b(a_7); - if (among_var == 0) - { - return false; - } - base.bra = base.cursor; - if (!r_R1()) - { - return false; - } - switch (among_var) { - case 1: - if (!base.slice_from("tion")) - { - return false; - } - break; - case 2: - if (!base.slice_from("ence")) - { - return false; - } - break; - case 3: - if (!base.slice_from("ance")) - { - return false; - } - break; - case 4: - if (!base.slice_from("able")) - { - return false; - } - break; - case 5: - if (!base.slice_from("ent")) - { - return false; - } - break; - case 6: - if (!base.slice_from("ize")) - { - return false; - } - break; - case 7: - if (!base.slice_from("ate")) - { - return false; - } - break; - case 8: - if (!base.slice_from("al")) - { - return false; - } - break; - case 9: - if (!base.slice_from("ful")) - { - return false; - } - break; - case 10: - if (!base.slice_from("ous")) - { - return false; - } - break; - case 11: - if (!base.slice_from("ive")) - { - return false; - } - break; - case 12: - if (!base.slice_from("ble")) - { - return false; - } - break; - case 13: - if (!base.slice_from("og")) - { - return false; - } - break; - case 14: - if (!(base.eq_s_b("l"))) - { - return false; - } - if (!base.slice_from("og")) - { - return false; - } - break; - case 15: - if (!base.slice_from("less")) - { - return false; - } - break; - case 16: - if (!(base.in_grouping_b(g_valid_LI, 99, 116))) - { - return false; - } - if (!base.slice_del()) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_Step_3() { - var /** number */ among_var; - base.ket = base.cursor; - among_var = base.find_among_b(a_8); - if (among_var == 0) - { - return false; - } - base.bra = base.cursor; - if (!r_R1()) - { - return false; - } - switch (among_var) { - case 1: - if (!base.slice_from("tion")) - { - return false; - } - break; - case 2: - if (!base.slice_from("ate")) - { - return false; - } - break; - case 3: - if (!base.slice_from("al")) - { - return false; - } - break; - case 4: - if (!base.slice_from("ic")) - { - return false; - } - break; - case 5: - if (!base.slice_del()) - { - return false; - } - break; - case 6: - if (!r_R2()) - { - return false; - } - if (!base.slice_del()) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_Step_4() { - var /** number */ among_var; - base.ket = base.cursor; - among_var = base.find_among_b(a_9); - if (among_var == 0) - { - return false; - } - base.bra = base.cursor; - if (!r_R2()) - { - return false; - } - switch (among_var) { - case 1: - if (!base.slice_del()) - { - return false; - } - break; - case 2: - lab0: { - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab1: { - if (!(base.eq_s_b("s"))) - { - break lab1; - } - break lab0; - } - base.cursor = base.limit - v_1; - if (!(base.eq_s_b("t"))) - { - return false; - } - } - if (!base.slice_del()) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_Step_5() { - var /** number */ among_var; - base.ket = base.cursor; - among_var = base.find_among_b(a_10); - if (among_var == 0) - { - return false; - } - base.bra = base.cursor; - switch (among_var) { - case 1: - lab0: { - lab1: { - if (!r_R2()) - { - break lab1; - } - break lab0; - } - if (!r_R1()) - { - return false; - } - { - /** @const */ var /** number */ v_1 = base.limit - base.cursor; - lab2: { - if (!r_shortv()) - { - break lab2; - } - return false; - } - base.cursor = base.limit - v_1; - } - } - if (!base.slice_del()) - { - return false; - } - break; - case 2: - if (!r_R2()) - { - return false; - } - if (!(base.eq_s_b("l"))) - { - return false; - } - if (!base.slice_del()) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_exception1() { - var /** number */ among_var; - base.bra = base.cursor; - among_var = base.find_among(a_11); - if (among_var == 0) - { - return false; - } - base.ket = base.cursor; - if (base.cursor < base.limit) - { - return false; - } - switch (among_var) { - case 1: - if (!base.slice_from("sky")) - { - return false; - } - break; - case 2: - if (!base.slice_from("idl")) - { - return false; - } - break; - case 3: - if (!base.slice_from("gentl")) - { - return false; - } - break; - case 4: - if (!base.slice_from("ugli")) - { - return false; - } - break; - case 5: - if (!base.slice_from("earli")) - { - return false; - } - break; - case 6: - if (!base.slice_from("onli")) - { - return false; - } - break; - case 7: - if (!base.slice_from("singl")) - { - return false; - } - break; - } - return true; - }; - - /** @return {boolean} */ - function r_postlude() { - if (!B_Y_found) - { - return false; - } - while(true) - { - /** @const */ var /** number */ v_1 = base.cursor; - lab0: { - golab1: while(true) - { - /** @const */ var /** number */ v_2 = base.cursor; - lab2: { - base.bra = base.cursor; - if (!(base.eq_s("Y"))) - { - break lab2; - } - base.ket = base.cursor; - base.cursor = v_2; - break golab1; - } - base.cursor = v_2; - if (base.cursor >= base.limit) - { - break lab0; - } - base.cursor++; - } - if (!base.slice_from("y")) - { - return false; - } - continue; - } - base.cursor = v_1; - break; - } - return true; - }; - - this.stem = /** @return {boolean} */ function() { - lab0: { - /** @const */ var /** number */ v_1 = base.cursor; - lab1: { - if (!r_exception1()) - { - break lab1; - } - break lab0; - } - base.cursor = v_1; - lab2: { - { - /** @const */ var /** number */ v_2 = base.cursor; - lab3: { - { - /** @const */ var /** number */ c1 = base.cursor + 3; - if (c1 > base.limit) - { - break lab3; - } - base.cursor = c1; - } - break lab2; - } - base.cursor = v_2; - } - break lab0; - } - base.cursor = v_1; - r_prelude(); - r_mark_regions(); - base.limit_backward = base.cursor; base.cursor = base.limit; - /** @const */ var /** number */ v_3 = base.limit - base.cursor; - r_Step_1a(); - base.cursor = base.limit - v_3; - /** @const */ var /** number */ v_4 = base.limit - base.cursor; - r_Step_1b(); - base.cursor = base.limit - v_4; - /** @const */ var /** number */ v_5 = base.limit - base.cursor; - r_Step_1c(); - base.cursor = base.limit - v_5; - /** @const */ var /** number */ v_6 = base.limit - base.cursor; - r_Step_2(); - base.cursor = base.limit - v_6; - /** @const */ var /** number */ v_7 = base.limit - base.cursor; - r_Step_3(); - base.cursor = base.limit - v_7; - /** @const */ var /** number */ v_8 = base.limit - base.cursor; - r_Step_4(); - base.cursor = base.limit - v_8; - /** @const */ var /** number */ v_9 = base.limit - base.cursor; - r_Step_5(); - base.cursor = base.limit - v_9; - base.cursor = base.limit_backward; - /** @const */ var /** number */ v_10 = base.cursor; - r_postlude(); - base.cursor = v_10; - } - return true; - }; - - /**@return{string}*/ - this['stemWord'] = function(/**string*/word) { - base.setCurrent(word); - this.stem(); - return base.getCurrent(); - }; -}; diff --git a/docs/_static/file.png b/docs/_static/file.png deleted file mode 100644 index a858a410e4faa62ce324d814e4b816fff83a6fb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 286 zcmV+(0pb3MP)s`hMrGg#P~ix$^RISR_I47Y|r1 z_CyJOe}D1){SET-^Amu_i71Lt6eYfZjRyw@I6OQAIXXHDfiX^GbOlHe=Ae4>0m)d(f|Me07*qoM6N<$f}vM^LjV8( diff --git a/docs/_static/language_data.js b/docs/_static/language_data.js deleted file mode 100644 index 57767864..00000000 --- a/docs/_static/language_data.js +++ /dev/null @@ -1,13 +0,0 @@ -/* - * This script contains the language-specific data used by searchtools.js, - * namely the set of stopwords, stemmer, scorer and splitter. - */ - -const stopwords = new Set(["a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "aren't", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can't", "cannot", "could", "couldn't", "did", "didn't", "do", "does", "doesn't", "doing", "don't", "down", "during", "each", "few", "for", "from", "further", "had", "hadn't", "has", "hasn't", "have", "haven't", "having", "he", "he'd", "he'll", "he's", "her", "here", "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", "isn't", "it", "it's", "its", "itself", "let's", "me", "more", "most", "mustn't", "my", "myself", "no", "nor", "not", "of", "off", "on", "once", "only", "or", "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", "shan't", "she", "she'd", "she'll", "she's", "should", "shouldn't", "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "wasn't", "we", "we'd", "we'll", "we're", "we've", "were", "weren't", "what", "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", "why's", "with", "won't", "would", "wouldn't", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves"]); -window.stopwords = stopwords; // Export to global scope - - -/* Non-minified versions are copied as separate JavaScript files, if available */ -BaseStemmer=function(){this.current="",this.cursor=0,this.limit=0,this.limit_backward=0,this.bra=0,this.ket=0,this.setCurrent=function(t){this.current=t,this.cursor=0,this.limit=this.current.length,this.limit_backward=0,this.bra=this.cursor,this.ket=this.limit},this.getCurrent=function(){return this.current},this.copy_from=function(t){this.current=t.current,this.cursor=t.cursor,this.limit=t.limit,this.limit_backward=t.limit_backward,this.bra=t.bra,this.ket=t.ket},this.in_grouping=function(t,r,i){return!(this.cursor>=this.limit||i<(i=this.current.charCodeAt(this.cursor))||i>>3]&1<<(7&i))||(this.cursor++,0))},this.go_in_grouping=function(t,r,i){for(;this.cursor>>3]&1<<(7&s)))return!0;this.cursor++}return!1},this.in_grouping_b=function(t,r,i){return!(this.cursor<=this.limit_backward||i<(i=this.current.charCodeAt(this.cursor-1))||i>>3]&1<<(7&i))||(this.cursor--,0))},this.go_in_grouping_b=function(t,r,i){for(;this.cursor>this.limit_backward;){var s=this.current.charCodeAt(this.cursor-1);if(i>>3]&1<<(7&s)))return!0;this.cursor--}return!1},this.out_grouping=function(t,r,i){return!(this.cursor>=this.limit)&&(i<(i=this.current.charCodeAt(this.cursor))||i>>3]&1<<(7&i)))&&(this.cursor++,!0)},this.go_out_grouping=function(t,r,i){for(;this.cursor>>3]&1<<(7&s)))return!0;this.cursor++}return!1},this.out_grouping_b=function(t,r,i){return!(this.cursor<=this.limit_backward)&&(i<(i=this.current.charCodeAt(this.cursor-1))||i>>3]&1<<(7&i)))&&(this.cursor--,!0)},this.go_out_grouping_b=function(t,r,i){for(;this.cursor>this.limit_backward;){var s=this.current.charCodeAt(this.cursor-1);if(s<=i&&r<=s&&0!=(t[(s-=r)>>>3]&1<<(7&s)))return!0;this.cursor--}return!1},this.eq_s=function(t){return!(this.limit-this.cursor>>1),o=0,a=e=(l=t[r])[0].length){if(this.cursor=s+l[0].length,l.length<4)return l[2];var g=l[3](this);if(this.cursor=s+l[0].length,g)return l[2]}}while(0<=(r=l[1]));return 0},this.find_among_b=function(t){for(var r=0,i=t.length,s=this.cursor,h=this.limit_backward,e=0,n=0,c=!1;;){for(var u,o=r+(i-r>>1),a=0,l=e=(u=t[r])[0].length){if(this.cursor=s-u[0].length,u.length<4)return u[2];var g=u[3](this);if(this.cursor=s-u[0].length,g)return u[2]}}while(0<=(r=u[1]));return 0},this.replace_s=function(t,r,i){var s=i.length-(r-t);return this.current=this.current.slice(0,t)+i+this.current.slice(r),this.limit+=s,this.cursor>=r?this.cursor+=s:this.cursor>t&&(this.cursor=t),s},this.slice_check=function(){return!(this.bra<0||this.bra>this.ket||this.ket>this.limit||this.limit>this.current.length)},this.slice_from=function(t){var r=!1;return this.slice_check()&&(this.replace_s(this.bra,this.ket,t),r=!0),r},this.slice_del=function(){return this.slice_from("")},this.insert=function(t,r,i){r=this.replace_s(t,r,i);t<=this.bra&&(this.bra+=r),t<=this.ket&&(this.ket+=r)},this.slice_to=function(){var t="";return t=this.slice_check()?this.current.slice(this.bra,this.ket):t},this.assign_to=function(){return this.current.slice(0,this.limit)}}; -var EnglishStemmer=function(){var a=new BaseStemmer,c=[["arsen",-1,-1],["commun",-1,-1],["emerg",-1,-1],["gener",-1,-1],["later",-1,-1],["organ",-1,-1],["past",-1,-1],["univers",-1,-1]],o=[["'",-1,1],["'s'",0,1],["'s",-1,1]],u=[["ied",-1,2],["s",-1,3],["ies",1,2],["sses",1,1],["ss",1,-1],["us",1,-1]],t=[["succ",-1,1],["proc",-1,1],["exc",-1,1]],l=[["even",-1,2],["cann",-1,2],["inn",-1,2],["earr",-1,2],["herr",-1,2],["out",-1,2],["y",-1,1]],n=[["",-1,-1],["ed",0,2],["eed",1,1],["ing",0,3],["edly",0,2],["eedly",4,1],["ingly",0,2]],f=[["",-1,3],["bb",0,2],["dd",0,2],["ff",0,2],["gg",0,2],["bl",0,1],["mm",0,2],["nn",0,2],["pp",0,2],["rr",0,2],["at",0,1],["tt",0,2],["iz",0,1]],_=[["anci",-1,3],["enci",-1,2],["ogi",-1,14],["li",-1,16],["bli",3,12],["abli",4,4],["alli",3,8],["fulli",3,9],["lessli",3,15],["ousli",3,10],["entli",3,5],["aliti",-1,8],["biliti",-1,12],["iviti",-1,11],["tional",-1,1],["ational",14,7],["alism",-1,8],["ation",-1,7],["ization",17,6],["izer",-1,6],["ator",-1,7],["iveness",-1,11],["fulness",-1,9],["ousness",-1,10],["ogist",-1,13]],m=[["icate",-1,4],["ative",-1,6],["alize",-1,3],["iciti",-1,4],["ical",-1,4],["tional",-1,1],["ational",5,2],["ful",-1,5],["ness",-1,5]],b=[["ic",-1,1],["ance",-1,1],["ence",-1,1],["able",-1,1],["ible",-1,1],["ate",-1,1],["ive",-1,1],["ize",-1,1],["iti",-1,1],["al",-1,1],["ism",-1,1],["ion",-1,2],["er",-1,1],["ous",-1,1],["ant",-1,1],["ent",-1,1],["ment",15,1],["ement",16,1]],k=[["e",-1,1],["l",-1,2]],g=[["andes",-1,-1],["atlas",-1,-1],["bias",-1,-1],["cosmos",-1,-1],["early",-1,5],["gently",-1,3],["howe",-1,-1],["idly",-1,2],["news",-1,-1],["only",-1,6],["singly",-1,7],["skies",-1,1],["sky",-1,-1],["ugly",-1,4]],d=[17,64],v=[17,65,16,1],i=[1,17,65,208,1],w=[55,141,2],p=!1,y=0,h=0;function q(){var r=a.limit-a.cursor;return!!(a.out_grouping_b(i,89,121)&&a.in_grouping_b(v,97,121)&&a.out_grouping_b(v,97,121)||(a.cursor=a.limit-r,a.out_grouping_b(v,97,121)&&a.in_grouping_b(v,97,121)&&!(a.cursor>a.limit_backward))||(a.cursor=a.limit-r,a.eq_s_b("past")))}function z(){return h<=a.cursor}function Y(){return y<=a.cursor}this.stem=function(){var r=a.cursor;if(!(()=>{var r;if(a.bra=a.cursor,0!=(r=a.find_among(g))&&(a.ket=a.cursor,!(a.cursora.limit)a.cursor=i;else{a.cursor=e,a.cursor=r,(()=>{p=!1;var r=a.cursor;if(a.bra=a.cursor,!a.eq_s("'")||(a.ket=a.cursor,a.slice_del())){a.cursor=r;r=a.cursor;if(a.bra=a.cursor,a.eq_s("y")){if(a.ket=a.cursor,!a.slice_from("Y"))return;p=!0}a.cursor=r;for(r=a.cursor;;){var i=a.cursor;r:{for(;;){var e=a.cursor;if(a.in_grouping(v,97,121)&&(a.bra=a.cursor,a.eq_s("y"))){a.ket=a.cursor,a.cursor=e;break}if(a.cursor=e,a.cursor>=a.limit)break r;a.cursor++}if(!a.slice_from("Y"))return;p=!0;continue}a.cursor=i;break}a.cursor=r}})(),h=a.limit,y=a.limit;i=a.cursor;r:{var s=a.cursor;if(0==a.find_among(c)){if(a.cursor=s,!a.go_out_grouping(v,97,121))break r;if(a.cursor++,!a.go_in_grouping(v,97,121))break r;a.cursor++}h=a.cursor,a.go_out_grouping(v,97,121)&&(a.cursor++,a.go_in_grouping(v,97,121))&&(a.cursor++,y=a.cursor)}a.cursor=i,a.limit_backward=a.cursor,a.cursor=a.limit;var e=a.limit-a.cursor,r=((()=>{var r=a.limit-a.cursor;if(a.ket=a.cursor,0==a.find_among_b(o))a.cursor=a.limit-r;else if(a.bra=a.cursor,!a.slice_del())return;if(a.ket=a.cursor,0!=(r=a.find_among_b(u)))switch(a.bra=a.cursor,r){case 1:if(a.slice_from("ss"))break;return;case 2:r:{var i=a.limit-a.cursor,e=a.cursor-2;if(!(e{a.ket=a.cursor,o=a.find_among_b(n),a.bra=a.cursor;r:{var r=a.limit-a.cursor;i:{switch(o){case 1:var i=a.limit-a.cursor;e:{var e=a.limit-a.cursor;if(0==a.find_among_b(t)||a.cursor>a.limit_backward){if(a.cursor=a.limit-e,!z())break e;if(!a.slice_from("ee"))return}}a.cursor=a.limit-i;break;case 2:break i;case 3:if(0==(o=a.find_among_b(l)))break i;switch(o){case 1:var s=a.limit-a.cursor;if(!a.out_grouping_b(v,97,121))break i;if(a.cursor>a.limit_backward)break i;if(a.cursor=a.limit-s,a.bra=a.cursor,a.slice_from("ie"))break;return;case 2:if(a.cursor>a.limit_backward)break i}}break r}a.cursor=a.limit-r;var c=a.limit-a.cursor;if(!a.go_out_grouping_b(v,97,121))return;if(a.cursor--,a.cursor=a.limit-c,!a.slice_del())return;a.ket=a.cursor,a.bra=a.cursor;var o,c=a.limit-a.cursor;switch(o=a.find_among_b(f)){case 1:return a.slice_from("e");case 2:var u=a.limit-a.cursor;if(a.in_grouping_b(d,97,111)&&!(a.cursor>a.limit_backward))return;a.cursor=a.limit-u;break;case 3:return a.cursor!=h||(u=a.limit-a.cursor,q()&&(a.cursor=a.limit-u,a.slice_from("e")))}if(a.cursor=a.limit-c,a.ket=a.cursor,a.cursor<=a.limit_backward)return;if(a.cursor--,a.bra=a.cursor,!a.slice_del())return}})(),a.cursor=a.limit-r,a.limit-a.cursor),r=(a.ket=a.cursor,e=a.limit-a.cursor,(a.eq_s_b("y")||(a.cursor=a.limit-e,a.eq_s_b("Y")))&&(a.bra=a.cursor,a.out_grouping_b(v,97,121))&&a.cursor>a.limit_backward&&a.slice_from("i"),a.cursor=a.limit-i,a.limit-a.cursor),e=((()=>{var r;if(a.ket=a.cursor,0!=(r=a.find_among_b(_))&&(a.bra=a.cursor,z()))switch(r){case 1:if(a.slice_from("tion"))break;return;case 2:if(a.slice_from("ence"))break;return;case 3:if(a.slice_from("ance"))break;return;case 4:if(a.slice_from("able"))break;return;case 5:if(a.slice_from("ent"))break;return;case 6:if(a.slice_from("ize"))break;return;case 7:if(a.slice_from("ate"))break;return;case 8:if(a.slice_from("al"))break;return;case 9:if(a.slice_from("ful"))break;return;case 10:if(a.slice_from("ous"))break;return;case 11:if(a.slice_from("ive"))break;return;case 12:if(a.slice_from("ble"))break;return;case 13:if(a.slice_from("og"))break;return;case 14:if(!a.eq_s_b("l"))return;if(a.slice_from("og"))break;return;case 15:if(a.slice_from("less"))break;return;case 16:if(!a.in_grouping_b(w,99,116))return;if(a.slice_del())break}})(),a.cursor=a.limit-r,a.limit-a.cursor),i=((()=>{var r;if(a.ket=a.cursor,0!=(r=a.find_among_b(m))&&(a.bra=a.cursor,z()))switch(r){case 1:if(a.slice_from("tion"))break;return;case 2:if(a.slice_from("ate"))break;return;case 3:if(a.slice_from("al"))break;return;case 4:if(a.slice_from("ic"))break;return;case 5:if(a.slice_del())break;return;case 6:if(!Y())return;if(a.slice_del())break}})(),a.cursor=a.limit-e,a.limit-a.cursor),r=((()=>{var r;if(a.ket=a.cursor,0!=(r=a.find_among_b(b))&&(a.bra=a.cursor,Y()))switch(r){case 1:if(a.slice_del())break;return;case 2:var i=a.limit-a.cursor;if(!a.eq_s_b("s")&&(a.cursor=a.limit-i,!a.eq_s_b("t")))return;if(a.slice_del())break}})(),a.cursor=a.limit-i,a.limit-a.cursor),e=((()=>{var r;if(a.ket=a.cursor,0!=(r=a.find_among_b(k)))switch(a.bra=a.cursor,r){case 1:if(!Y()){if(!z())return;var i=a.limit-a.cursor;if(q())return;a.cursor=a.limit-i}if(a.slice_del())break;return;case 2:if(!Y())return;if(!a.eq_s_b("l"))return;if(a.slice_del())break}})(),a.cursor=a.limit-r,a.cursor=a.limit_backward,a.cursor);(()=>{if(p)for(;;){var r=a.cursor;r:{for(;;){var i=a.cursor;if(a.bra=a.cursor,a.eq_s("Y")){a.ket=a.cursor,a.cursor=i;break}if(a.cursor=i,a.cursor>=a.limit)break r;a.cursor++}if(a.slice_from("y"))continue;return}a.cursor=r;break}})(),a.cursor=e}}return!0},this.stemWord=function(r){return a.setCurrent(r),this.stem(),a.getCurrent()}}; -window.Stemmer = EnglishStemmer; diff --git a/docs/_static/minus.png b/docs/_static/minus.png deleted file mode 100644 index d96755fdaf8bb2214971e0db9c1fd3077d7c419d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90 zcmeAS@N?(olHy`uVBq!ia0vp^+#t*WBp7;*Yy1LIik>cxAr*|t7R?Mi>2?kWtu=nj kDsEF_5m^0CR;1wuP-*O&G^0G}KYk!hp00i_>zopr08q^qX#fBK diff --git a/docs/_static/placeholder.txt b/docs/_static/placeholder.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/_static/plus.png b/docs/_static/plus.png deleted file mode 100644 index 7107cec93a979b9a5f64843235a16651d563ce2d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90 zcmeAS@N?(olHy`uVBq!ia0vp^+#t*WBp7;*Yy1LIik>cxAr*|t7R?Mi>2?kWtu>-2 m3q%Vub%g%s<8sJhVPMczOq}xhg9DJoz~JfX=d#Wzp$Pyb1r*Kz diff --git a/docs/_static/pygments.css b/docs/_static/pygments.css deleted file mode 100644 index 5f2b0a25..00000000 --- a/docs/_static/pygments.css +++ /dev/null @@ -1,75 +0,0 @@ -pre { line-height: 125%; } -td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -.highlight .hll { background-color: #ffffcc } -.highlight { background: #eeffcc; } -.highlight .c { color: #408090; font-style: italic } /* Comment */ -.highlight .err { border: 1px solid #F00 } /* Error */ -.highlight .k { color: #007020; font-weight: bold } /* Keyword */ -.highlight .o { color: #666 } /* Operator */ -.highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ -.highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ -.highlight .cp { color: #007020 } /* Comment.Preproc */ -.highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ -.highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ -.highlight .cs { color: #408090; background-color: #FFF0F0 } /* Comment.Special */ -.highlight .gd { color: #A00000 } /* Generic.Deleted */ -.highlight .ge { font-style: italic } /* Generic.Emph */ -.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */ -.highlight .gr { color: #F00 } /* Generic.Error */ -.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.highlight .gi { color: #00A000 } /* Generic.Inserted */ -.highlight .go { color: #333 } /* Generic.Output */ -.highlight .gp { color: #C65D09; font-weight: bold } /* Generic.Prompt */ -.highlight .gs { font-weight: bold } /* Generic.Strong */ -.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.highlight .gt { color: #04D } /* Generic.Traceback */ -.highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ -.highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ -.highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ -.highlight .kp { color: #007020 } /* Keyword.Pseudo */ -.highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ -.highlight .kt { color: #902000 } /* Keyword.Type */ -.highlight .m { color: #208050 } /* Literal.Number */ -.highlight .s { color: #4070A0 } /* Literal.String */ -.highlight .na { color: #4070A0 } /* Name.Attribute */ -.highlight .nb { color: #007020 } /* Name.Builtin */ -.highlight .nc { color: #0E84B5; font-weight: bold } /* Name.Class */ -.highlight .no { color: #60ADD5 } /* Name.Constant */ -.highlight .nd { color: #555; font-weight: bold } /* Name.Decorator */ -.highlight .ni { color: #D55537; font-weight: bold } /* Name.Entity */ -.highlight .ne { color: #007020 } /* Name.Exception */ -.highlight .nf { color: #06287E } /* Name.Function */ -.highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ -.highlight .nn { color: #0E84B5; font-weight: bold } /* Name.Namespace */ -.highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ -.highlight .nv { color: #BB60D5 } /* Name.Variable */ -.highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ -.highlight .w { color: #BBB } /* Text.Whitespace */ -.highlight .mb { color: #208050 } /* Literal.Number.Bin */ -.highlight .mf { color: #208050 } /* Literal.Number.Float */ -.highlight .mh { color: #208050 } /* Literal.Number.Hex */ -.highlight .mi { color: #208050 } /* Literal.Number.Integer */ -.highlight .mo { color: #208050 } /* Literal.Number.Oct */ -.highlight .sa { color: #4070A0 } /* Literal.String.Affix */ -.highlight .sb { color: #4070A0 } /* Literal.String.Backtick */ -.highlight .sc { color: #4070A0 } /* Literal.String.Char */ -.highlight .dl { color: #4070A0 } /* Literal.String.Delimiter */ -.highlight .sd { color: #4070A0; font-style: italic } /* Literal.String.Doc */ -.highlight .s2 { color: #4070A0 } /* Literal.String.Double */ -.highlight .se { color: #4070A0; font-weight: bold } /* Literal.String.Escape */ -.highlight .sh { color: #4070A0 } /* Literal.String.Heredoc */ -.highlight .si { color: #70A0D0; font-style: italic } /* Literal.String.Interpol */ -.highlight .sx { color: #C65D09 } /* Literal.String.Other */ -.highlight .sr { color: #235388 } /* Literal.String.Regex */ -.highlight .s1 { color: #4070A0 } /* Literal.String.Single */ -.highlight .ss { color: #517918 } /* Literal.String.Symbol */ -.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ -.highlight .fm { color: #06287E } /* Name.Function.Magic */ -.highlight .vc { color: #BB60D5 } /* Name.Variable.Class */ -.highlight .vg { color: #BB60D5 } /* Name.Variable.Global */ -.highlight .vi { color: #BB60D5 } /* Name.Variable.Instance */ -.highlight .vm { color: #BB60D5 } /* Name.Variable.Magic */ -.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/docs/_static/searchtools.js b/docs/_static/searchtools.js deleted file mode 100644 index e29b1c75..00000000 --- a/docs/_static/searchtools.js +++ /dev/null @@ -1,693 +0,0 @@ -/* - * Sphinx JavaScript utilities for the full-text search. - */ -"use strict"; - -/** - * Simple result scoring code. - */ -if (typeof Scorer === "undefined") { - var Scorer = { - // Implement the following function to further tweak the score for each result - // The function takes a result array [docname, title, anchor, descr, score, filename] - // and returns the new score. - /* - score: result => { - const [docname, title, anchor, descr, score, filename, kind] = result - return score - }, - */ - - // query matches the full name of an object - objNameMatch: 11, - // or matches in the last dotted part of the object name - objPartialMatch: 6, - // Additive scores depending on the priority of the object - objPrio: { - 0: 15, // used to be importantResults - 1: 5, // used to be objectResults - 2: -5, // used to be unimportantResults - }, - // Used when the priority is not in the mapping. - objPrioDefault: 0, - - // query found in title - title: 15, - partialTitle: 7, - // query found in terms - term: 5, - partialTerm: 2, - }; -} - -// Global search result kind enum, used by themes to style search results. -// prettier-ignore -class SearchResultKind { - static get index() { return "index"; } - static get object() { return "object"; } - static get text() { return "text"; } - static get title() { return "title"; } -} - -const _removeChildren = (element) => { - while (element && element.lastChild) element.removeChild(element.lastChild); -}; - -/** - * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping - */ -const _escapeRegExp = (string) => - string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string - -const _escapeHTML = (text) => { - return text - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -}; - -const _displayItem = (item, searchTerms, highlightTerms) => { - const docBuilder = DOCUMENTATION_OPTIONS.BUILDER; - const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX; - const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX; - const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY; - const contentRoot = document.documentElement.dataset.content_root; - - const [docName, title, anchor, descr, score, _filename, kind] = item; - - let listItem = document.createElement("li"); - // Add a class representing the item's type: - // can be used by a theme's CSS selector for styling - // See SearchResultKind for the class names. - listItem.classList.add(`kind-${kind}`); - let requestUrl; - let linkUrl; - if (docBuilder === "dirhtml") { - // dirhtml builder - let dirname = docName + "/"; - if (dirname.match(/\/index\/$/)) - dirname = dirname.substring(0, dirname.length - 6); - else if (dirname === "index/") dirname = ""; - requestUrl = contentRoot + dirname; - linkUrl = requestUrl; - } else { - // normal html builders - requestUrl = contentRoot + docName + docFileSuffix; - linkUrl = docName + docLinkSuffix; - } - let linkEl = listItem.appendChild(document.createElement("a")); - linkEl.href = linkUrl + anchor; - linkEl.dataset.score = score; - linkEl.innerHTML = _escapeHTML(title); - if (descr) { - listItem.appendChild(document.createElement("span")).innerHTML = - ` (${_escapeHTML(descr)})`; - // highlight search terms in the description - if (SPHINX_HIGHLIGHT_ENABLED) - // SPHINX_HIGHLIGHT_ENABLED is set in sphinx_highlight.js - highlightTerms.forEach((term) => - _highlightText(listItem, term, "highlighted"), - ); - } else if (showSearchSummary) - fetch(requestUrl) - .then((responseData) => responseData.text()) - .then((data) => { - if (data) - listItem.appendChild( - Search.makeSearchSummary(data, searchTerms, anchor), - ); - // highlight search terms in the summary - if (SPHINX_HIGHLIGHT_ENABLED) - // SPHINX_HIGHLIGHT_ENABLED is set in sphinx_highlight.js - highlightTerms.forEach((term) => - _highlightText(listItem, term, "highlighted"), - ); - }); - Search.output.appendChild(listItem); -}; -const _finishSearch = (resultCount) => { - Search.stopPulse(); - Search.title.innerText = _("Search Results"); - if (!resultCount) - Search.status.innerText = Documentation.gettext( - "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories.", - ); - else - Search.status.innerText = Documentation.ngettext( - "Search finished, found one page matching the search query.", - "Search finished, found ${resultCount} pages matching the search query.", - resultCount, - ).replace("${resultCount}", resultCount); -}; -const _displayNextItem = ( - results, - resultCount, - searchTerms, - highlightTerms, -) => { - // results left, load the summary and display it - // this is intended to be dynamic (don't sub resultsCount) - if (results.length) { - _displayItem(results.pop(), searchTerms, highlightTerms); - setTimeout( - () => _displayNextItem(results, resultCount, searchTerms, highlightTerms), - 5, - ); - } - // search finished, update title and status message - else _finishSearch(resultCount); -}; -// Helper function used by query() to order search results. -// Each input is an array of [docname, title, anchor, descr, score, filename, kind]. -// Order the results by score (in opposite order of appearance, since the -// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically. -const _orderResultsByScoreThenName = (a, b) => { - const leftScore = a[4]; - const rightScore = b[4]; - if (leftScore === rightScore) { - // same score: sort alphabetically - const leftTitle = a[1].toLowerCase(); - const rightTitle = b[1].toLowerCase(); - if (leftTitle === rightTitle) return 0; - return leftTitle > rightTitle ? -1 : 1; // inverted is intentional - } - return leftScore > rightScore ? 1 : -1; -}; - -/** - * Default splitQuery function. Can be overridden in ``sphinx.search`` with a - * custom function per language. - * - * The regular expression works by splitting the string on consecutive characters - * that are not Unicode letters, numbers, underscores, or emoji characters. - * This is the same as ``\W+`` in Python, preserving the surrogate pair area. - */ -if (typeof splitQuery === "undefined") { - var splitQuery = (query) => - query - .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu) - .filter((term) => term); // remove remaining empty strings -} - -/** - * Search Module - */ -const Search = { - _index: null, - _queued_query: null, - _pulse_status: -1, - - htmlToText: (htmlString, anchor) => { - const htmlElement = new DOMParser().parseFromString( - htmlString, - "text/html", - ); - for (const removalQuery of [".headerlink", "script", "style"]) { - htmlElement.querySelectorAll(removalQuery).forEach((el) => { - el.remove(); - }); - } - if (anchor) { - const anchorContent = htmlElement.querySelector( - `[role="main"] ${anchor}`, - ); - if (anchorContent) return anchorContent.textContent; - - console.warn( - `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.`, - ); - } - - // if anchor not specified or not found, fall back to main content - const docContent = htmlElement.querySelector('[role="main"]'); - if (docContent) return docContent.textContent; - - console.warn( - "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template.", - ); - return ""; - }, - - init: () => { - const query = new URLSearchParams(window.location.search).get("q"); - document - .querySelectorAll('input[name="q"]') - .forEach((el) => (el.value = query)); - if (query) Search.performSearch(query); - }, - - loadIndex: (url) => - (document.body.appendChild(document.createElement("script")).src = url), - - setIndex: (index) => { - Search._index = index; - if (Search._queued_query !== null) { - const query = Search._queued_query; - Search._queued_query = null; - Search.query(query); - } - }, - - hasIndex: () => Search._index !== null, - - deferQuery: (query) => (Search._queued_query = query), - - stopPulse: () => (Search._pulse_status = -1), - - startPulse: () => { - if (Search._pulse_status >= 0) return; - - const pulse = () => { - Search._pulse_status = (Search._pulse_status + 1) % 4; - Search.dots.innerText = ".".repeat(Search._pulse_status); - if (Search._pulse_status >= 0) window.setTimeout(pulse, 500); - }; - pulse(); - }, - - /** - * perform a search for something (or wait until index is loaded) - */ - performSearch: (query) => { - // create the required interface elements - const searchText = document.createElement("h2"); - searchText.textContent = _("Searching"); - const searchSummary = document.createElement("p"); - searchSummary.classList.add("search-summary"); - searchSummary.innerText = ""; - const searchList = document.createElement("ul"); - searchList.setAttribute("role", "list"); - searchList.classList.add("search"); - - const out = document.getElementById("search-results"); - Search.title = out.appendChild(searchText); - Search.dots = Search.title.appendChild(document.createElement("span")); - Search.status = out.appendChild(searchSummary); - Search.output = out.appendChild(searchList); - - const searchProgress = document.getElementById("search-progress"); - // Some themes don't use the search progress node - if (searchProgress) { - searchProgress.innerText = _("Preparing search..."); - } - Search.startPulse(); - - // index already loaded, the browser was quick! - if (Search.hasIndex()) Search.query(query); - else Search.deferQuery(query); - }, - - _parseQuery: (query) => { - // stem the search terms and add them to the correct list - const stemmer = new Stemmer(); - const searchTerms = new Set(); - const excludedTerms = new Set(); - const highlightTerms = new Set(); - const objectTerms = new Set(splitQuery(query.toLowerCase().trim())); - splitQuery(query.trim()).forEach((queryTerm) => { - const queryTermLower = queryTerm.toLowerCase(); - - // maybe skip this "word" - // stopwords set is from language_data.js - if (stopwords.has(queryTermLower) || queryTerm.match(/^\d+$/)) return; - - // stem the word - let word = stemmer.stemWord(queryTermLower); - // select the correct list - if (word[0] === "-") excludedTerms.add(word.substr(1)); - else { - searchTerms.add(word); - highlightTerms.add(queryTermLower); - } - }); - - if (SPHINX_HIGHLIGHT_ENABLED) { - // SPHINX_HIGHLIGHT_ENABLED is set in sphinx_highlight.js - localStorage.setItem( - "sphinx_highlight_terms", - [...highlightTerms].join(" "), - ); - } - - // console.debug("SEARCH: searching for:"); - // console.info("required: ", [...searchTerms]); - // console.info("excluded: ", [...excludedTerms]); - - return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; - }, - - /** - * execute search (requires search index to be loaded) - */ - _performSearch: ( - query, - searchTerms, - excludedTerms, - highlightTerms, - objectTerms, - ) => { - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const titles = Search._index.titles; - const allTitles = Search._index.alltitles; - const indexEntries = Search._index.indexentries; - - // Collect multiple result groups to be sorted separately and then ordered. - // Each is an array of [docname, title, anchor, descr, score, filename, kind]. - const normalResults = []; - const nonMainIndexResults = []; - - _removeChildren(document.getElementById("search-progress")); - - const queryLower = query.toLowerCase().trim(); - for (const [title, foundTitles] of Object.entries(allTitles)) { - if ( - title.toLowerCase().trim().includes(queryLower) - && queryLower.length >= title.length / 2 - ) { - for (const [file, id] of foundTitles) { - const score = Math.round( - (Scorer.title * queryLower.length) / title.length, - ); - const boost = titles[file] === title ? 1 : 0; // add a boost for document titles - normalResults.push([ - docNames[file], - titles[file] !== title ? `${titles[file]} > ${title}` : title, - id !== null ? "#" + id : "", - null, - score + boost, - filenames[file], - SearchResultKind.title, - ]); - } - } - } - - // search for explicit entries in index directives - for (const [entry, foundEntries] of Object.entries(indexEntries)) { - if (entry.includes(queryLower) && queryLower.length >= entry.length / 2) { - for (const [file, id, isMain] of foundEntries) { - const score = Math.round((100 * queryLower.length) / entry.length); - const result = [ - docNames[file], - titles[file], - id ? "#" + id : "", - null, - score, - filenames[file], - SearchResultKind.index, - ]; - if (isMain) { - normalResults.push(result); - } else { - nonMainIndexResults.push(result); - } - } - } - } - - // lookup as object - objectTerms.forEach((term) => - normalResults.push(...Search.performObjectSearch(term, objectTerms)), - ); - - // lookup as search terms in fulltext - normalResults.push( - ...Search.performTermsSearch(searchTerms, excludedTerms), - ); - - // let the scorer override scores with a custom scoring function - if (Scorer.score) { - normalResults.forEach((item) => (item[4] = Scorer.score(item))); - nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item))); - } - - // Sort each group of results by score and then alphabetically by name. - normalResults.sort(_orderResultsByScoreThenName); - nonMainIndexResults.sort(_orderResultsByScoreThenName); - - // Combine the result groups in (reverse) order. - // Non-main index entries are typically arbitrary cross-references, - // so display them after other results. - let results = [...nonMainIndexResults, ...normalResults]; - - // remove duplicate search results - // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept - let seen = new Set(); - results = results.reverse().reduce((acc, result) => { - let resultStr = result - .slice(0, 4) - .concat([result[5]]) - .map((v) => String(v)) - .join(","); - if (!seen.has(resultStr)) { - acc.push(result); - seen.add(resultStr); - } - return acc; - }, []); - - return results.reverse(); - }, - - query: (query) => { - const [ - searchQuery, - searchTerms, - excludedTerms, - highlightTerms, - objectTerms, - ] = Search._parseQuery(query); - const results = Search._performSearch( - searchQuery, - searchTerms, - excludedTerms, - highlightTerms, - objectTerms, - ); - - // for debugging - //Search.lastresults = results.slice(); // a copy - // console.info("search results:", Search.lastresults); - - // print the results - _displayNextItem(results, results.length, searchTerms, highlightTerms); - }, - - /** - * search for object names - */ - performObjectSearch: (object, objectTerms) => { - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const objects = Search._index.objects; - const objNames = Search._index.objnames; - const titles = Search._index.titles; - - const results = []; - - const objectSearchCallback = (prefix, match) => { - const name = match[4]; - const fullname = (prefix ? prefix + "." : "") + name; - const fullnameLower = fullname.toLowerCase(); - if (fullnameLower.indexOf(object) < 0) return; - - let score = 0; - const parts = fullnameLower.split("."); - - // check for different match types: exact matches of full name or - // "last name" (i.e. last dotted part) - if (fullnameLower === object || parts.slice(-1)[0] === object) - score += Scorer.objNameMatch; - else if (parts.slice(-1)[0].indexOf(object) > -1) - score += Scorer.objPartialMatch; // matches in last name - - const objName = objNames[match[1]][2]; - const title = titles[match[0]]; - - // If more than one term searched for, we require other words to be - // found in the name/title/description - const otherTerms = new Set(objectTerms); - otherTerms.delete(object); - if (otherTerms.size > 0) { - const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase(); - if ( - [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0) - ) - return; - } - - let anchor = match[3]; - if (anchor === "") anchor = fullname; - else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname; - - const descr = objName + _(", in ") + title; - - // add custom score for some objects according to scorer - if (Scorer.objPrio.hasOwnProperty(match[2])) - score += Scorer.objPrio[match[2]]; - else score += Scorer.objPrioDefault; - - results.push([ - docNames[match[0]], - fullname, - "#" + anchor, - descr, - score, - filenames[match[0]], - SearchResultKind.object, - ]); - }; - Object.keys(objects).forEach((prefix) => - objects[prefix].forEach((array) => objectSearchCallback(prefix, array)), - ); - return results; - }, - - /** - * search for full-text terms in the index - */ - performTermsSearch: (searchTerms, excludedTerms) => { - // prepare search - const terms = Search._index.terms; - const titleTerms = Search._index.titleterms; - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const titles = Search._index.titles; - - const scoreMap = new Map(); - const fileMap = new Map(); - - // perform the search on the required terms - searchTerms.forEach((word) => { - const files = []; - // find documents, if any, containing the query word in their text/title term indices - // use Object.hasOwnProperty to avoid mismatching against prototype properties - const arr = [ - { - files: terms.hasOwnProperty(word) ? terms[word] : undefined, - score: Scorer.term, - }, - { - files: titleTerms.hasOwnProperty(word) ? titleTerms[word] : undefined, - score: Scorer.title, - }, - ]; - // add support for partial matches - if (word.length > 2) { - const escapedWord = _escapeRegExp(word); - if (!terms.hasOwnProperty(word)) { - Object.keys(terms).forEach((term) => { - if (term.match(escapedWord)) - arr.push({ files: terms[term], score: Scorer.partialTerm }); - }); - } - if (!titleTerms.hasOwnProperty(word)) { - Object.keys(titleTerms).forEach((term) => { - if (term.match(escapedWord)) - arr.push({ files: titleTerms[term], score: Scorer.partialTitle }); - }); - } - } - - // no match but word was a required one - if (arr.every((record) => record.files === undefined)) return; - - // found search word in contents - arr.forEach((record) => { - if (record.files === undefined) return; - - let recordFiles = record.files; - if (recordFiles.length === undefined) recordFiles = [recordFiles]; - files.push(...recordFiles); - - // set score for the word in each file - recordFiles.forEach((file) => { - if (!scoreMap.has(file)) scoreMap.set(file, new Map()); - const fileScores = scoreMap.get(file); - fileScores.set(word, record.score); - }); - }); - - // create the mapping - files.forEach((file) => { - if (!fileMap.has(file)) fileMap.set(file, [word]); - else if (fileMap.get(file).indexOf(word) === -1) - fileMap.get(file).push(word); - }); - }); - - // now check if the files don't contain excluded terms - const results = []; - for (const [file, wordList] of fileMap) { - // check if all requirements are matched - - // as search terms with length < 3 are discarded - const filteredTermCount = [...searchTerms].filter( - (term) => term.length > 2, - ).length; - if ( - wordList.length !== searchTerms.size - && wordList.length !== filteredTermCount - ) - continue; - - // ensure that none of the excluded terms is in the search result - if ( - [...excludedTerms].some( - (term) => - terms[term] === file - || titleTerms[term] === file - || (terms[term] || []).includes(file) - || (titleTerms[term] || []).includes(file), - ) - ) - break; - - // select one (max) score for the file. - const score = Math.max(...wordList.map((w) => scoreMap.get(file).get(w))); - // add result to the result list - results.push([ - docNames[file], - titles[file], - "", - null, - score, - filenames[file], - SearchResultKind.text, - ]); - } - return results; - }, - - /** - * helper function to return a node containing the - * search summary for a given text. keywords is a list - * of stemmed words. - */ - makeSearchSummary: (htmlText, keywords, anchor) => { - const text = Search.htmlToText(htmlText, anchor); - if (text === "") return null; - - const textLower = text.toLowerCase(); - const actualStartPosition = [...keywords] - .map((k) => textLower.indexOf(k.toLowerCase())) - .filter((i) => i > -1) - .slice(-1)[0]; - const startWithContext = Math.max(actualStartPosition - 120, 0); - - const top = startWithContext === 0 ? "" : "..."; - const tail = startWithContext + 240 < text.length ? "..." : ""; - - let summary = document.createElement("p"); - summary.classList.add("context"); - summary.textContent = - top + text.substr(startWithContext, 240).trim() + tail; - - return summary; - }, -}; - -_ready(Search.init); diff --git a/docs/_static/sidebar.js b/docs/_static/sidebar.js deleted file mode 100644 index ac7b39ee..00000000 --- a/docs/_static/sidebar.js +++ /dev/null @@ -1,64 +0,0 @@ -/* - * This script makes the Sphinx sidebar collapsible. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds - * in .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton - * used to collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden - * and the width of the sidebar and the margin-left of the document - * are decreased. When the sidebar is expanded the opposite happens. - * This script saves a per-browser/per-session cookie used to - * remember the position of the sidebar among the pages. - * Once the browser is closed the cookie is deleted and the position - * reset to the default (expanded). - * - */ - -const initialiseSidebar = () => { - - - - - // global elements used by the functions. - const bodyWrapper = document.getElementsByClassName("bodywrapper")[0] - const sidebar = document.getElementsByClassName("sphinxsidebar")[0] - const sidebarWrapper = document.getElementsByClassName('sphinxsidebarwrapper')[0] - const sidebarButton = document.getElementById("sidebarbutton") - const sidebarArrow = sidebarButton.querySelector('span') - - // for some reason, the document has no sidebar; do not run into errors - if (typeof sidebar === "undefined") return; - - const flipArrow = element => element.innerText = (element.innerText === "»") ? "«" : "»" - - const collapse_sidebar = () => { - bodyWrapper.style.marginLeft = ".8em"; - sidebar.style.width = ".8em" - sidebarWrapper.style.display = "none" - flipArrow(sidebarArrow) - sidebarButton.title = _('Expand sidebar') - window.localStorage.setItem("sidebar", "collapsed") - } - - const expand_sidebar = () => { - bodyWrapper.style.marginLeft = "" - sidebar.style.removeProperty("width") - sidebarWrapper.style.display = "" - flipArrow(sidebarArrow) - sidebarButton.title = _('Collapse sidebar') - window.localStorage.setItem("sidebar", "expanded") - } - - sidebarButton.addEventListener("click", () => { - (sidebarWrapper.style.display === "none") ? expand_sidebar() : collapse_sidebar() - }) - - if (!window.localStorage.getItem("sidebar")) return - const value = window.localStorage.getItem("sidebar") - if (value === "collapsed") collapse_sidebar(); - else if (value === "expanded") expand_sidebar(); -} - -if (document.readyState !== "loading") initialiseSidebar() -else document.addEventListener("DOMContentLoaded", initialiseSidebar) \ No newline at end of file diff --git a/docs/_static/sphinx_highlight.js b/docs/_static/sphinx_highlight.js deleted file mode 100644 index a74e103a..00000000 --- a/docs/_static/sphinx_highlight.js +++ /dev/null @@ -1,159 +0,0 @@ -/* Highlighting utilities for Sphinx HTML documentation. */ -"use strict"; - -const SPHINX_HIGHLIGHT_ENABLED = true; - -/** - * highlight a given string on a node by wrapping it in - * span elements with the given class name. - */ -const _highlight = (node, addItems, text, className) => { - if (node.nodeType === Node.TEXT_NODE) { - const val = node.nodeValue; - const parent = node.parentNode; - const pos = val.toLowerCase().indexOf(text); - if ( - pos >= 0 - && !parent.classList.contains(className) - && !parent.classList.contains("nohighlight") - ) { - let span; - - const closestNode = parent.closest("body, svg, foreignObject"); - const isInSVG = closestNode && closestNode.matches("svg"); - if (isInSVG) { - span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); - } else { - span = document.createElement("span"); - span.classList.add(className); - } - - span.appendChild(document.createTextNode(val.substr(pos, text.length))); - const rest = document.createTextNode(val.substr(pos + text.length)); - parent.insertBefore(span, parent.insertBefore(rest, node.nextSibling)); - node.nodeValue = val.substr(0, pos); - /* There may be more occurrences of search term in this node. So call this - * function recursively on the remaining fragment. - */ - _highlight(rest, addItems, text, className); - - if (isInSVG) { - const rect = document.createElementNS( - "http://www.w3.org/2000/svg", - "rect", - ); - const bbox = parent.getBBox(); - rect.x.baseVal.value = bbox.x; - rect.y.baseVal.value = bbox.y; - rect.width.baseVal.value = bbox.width; - rect.height.baseVal.value = bbox.height; - rect.setAttribute("class", className); - addItems.push({ parent: parent, target: rect }); - } - } - } else if (node.matches && !node.matches("button, select, textarea")) { - node.childNodes.forEach((el) => _highlight(el, addItems, text, className)); - } -}; -const _highlightText = (thisNode, text, className) => { - let addItems = []; - _highlight(thisNode, addItems, text, className); - addItems.forEach((obj) => - obj.parent.insertAdjacentElement("beforebegin", obj.target), - ); -}; - -/** - * Small JavaScript module for the documentation. - */ -const SphinxHighlight = { - /** - * highlight the search words provided in localstorage in the text - */ - highlightSearchWords: () => { - if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight - - // get and clear terms from localstorage - const url = new URL(window.location); - const highlight = - localStorage.getItem("sphinx_highlight_terms") - || url.searchParams.get("highlight") - || ""; - localStorage.removeItem("sphinx_highlight_terms"); - // Update history only if '?highlight' is present; otherwise it - // clears text fragments (not set in window.location by the browser) - if (url.searchParams.has("highlight")) { - url.searchParams.delete("highlight"); - window.history.replaceState({}, "", url); - } - - // get individual terms from highlight string - const terms = highlight - .toLowerCase() - .split(/\s+/) - .filter((x) => x); - if (terms.length === 0) return; // nothing to do - - // There should never be more than one element matching "div.body" - const divBody = document.querySelectorAll("div.body"); - const body = divBody.length ? divBody[0] : document.querySelector("body"); - window.setTimeout(() => { - terms.forEach((term) => _highlightText(body, term, "highlighted")); - }, 10); - - const searchBox = document.getElementById("searchbox"); - if (searchBox === null) return; - searchBox.appendChild( - document - .createRange() - .createContextualFragment( - '

' - + '' - + _("Hide Search Matches") - + "

", - ), - ); - }, - - /** - * helper function to hide the search marks again - */ - hideSearchWords: () => { - document - .querySelectorAll("#searchbox .highlight-link") - .forEach((el) => el.remove()); - document - .querySelectorAll("span.highlighted") - .forEach((el) => el.classList.remove("highlighted")); - localStorage.removeItem("sphinx_highlight_terms"); - }, - - initEscapeListener: () => { - // only install a listener if it is really needed - if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; - - document.addEventListener("keydown", (event) => { - // bail for input elements - if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) - return; - // bail with special keys - if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) - return; - if ( - DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS - && event.key === "Escape" - ) { - SphinxHighlight.hideSearchWords(); - event.preventDefault(); - } - }); - }, -}; - -_ready(() => { - /* Do not call highlightSearchWords() when we are on the search page. - * It will highlight words from the *previous* search query. - */ - if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); - SphinxHighlight.initEscapeListener(); -}); diff --git a/docs/genindex.html b/docs/genindex.html deleted file mode 100644 index a5eb566e..00000000 --- a/docs/genindex.html +++ /dev/null @@ -1,953 +0,0 @@ - - - - - - - Index — pysnptools 1.0.0 documentation - - - - - - - - - - - - -
-
-
-
- - -

Index

- -
- _ - | A - | B - | C - | D - | E - | F - | G - | H - | I - | J - | K - | L - | M - | N - | O - | P - | R - | S - | T - | U - | V - | W - -
-

_

- - - -
- -

A

- - - -
- -

B

- - - -
- -

C

- - - -
- -

D

- - - -
- -

E

- - - -
- -

F

- - - -
- -

G

- - - -
- -

H

- - - -
- -

I

- - - -
- -

J

- - -
- -

K

- - - -
- -

L

- - - -
- -

M

- - -
- -

N

- - -
- -

O

- - - -
- -

P

- - - -
    -
  • - pysnptools.kernelreader - -
    • -
  • - pysnptools.pstreader - -
    • -
  • - pysnptools.snpreader - -
    • -
  • - pysnptools.standardizer - -
    • -
  • - pysnptools.util - -
    • -
  • - pysnptools.util.filecache - -
    • -
  • - pysnptools.util.mapreduce1 - -
    • -
  • - pysnptools.util.pheno - -
    • -
- -

R

- - - -
- -

S

- - - -
- -

T

- - -
- -

U

- - - -
- -

V

- - - -
- -

W

- - -
- - - -
-
-
-
- -
-
- - - - \ No newline at end of file diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index 24559ab5..00000000 --- a/docs/index.html +++ /dev/null @@ -1,10142 +0,0 @@ - - - - - - - - pysnptools Documentation — pysnptools 1.0.0 documentation - - - - - - - - - - - - -
-
-
-
- -
-

pysnptools Documentation

-

PySnpTools: A library for reading and manipulating genetic data.

-

See PySnpTools’s README.md for installation instructions, documentation, and code.

-

HDF5 and BGEN support are optional. Install pysnptools[hdf5], -pysnptools[bgen], or pysnptools[hdf5,bgen] as needed. -Existing HDF5 users upgrading from an earlier release must install the HDF5 extra. -Core Python 3.15 qualification excludes both backends; HDF5 and BGEN remain -covered by the Python 3.11 through 3.14 test matrix.

-
-
Synopsis:
-

-
-
    -

  • snpreader: Efficiently read genetic PLINK formats including *.bed/bim/fam and phenotype files. Also, efficiently read parts of files and standardize data.

    • -
  • -
    new:
    -

    distreader: Efficiently work with unphased BGEN format and other diploid, biallelic distribution data. Also, efficiently read parts of files. -See Distribution IPython Notebook

    -
    -
    -
    • -

  • kernelreader: Efficiently create, read, and manipulate kernel data.

    • -

  • standardizer: Specify standardizers for snpreader.

    • -

  • kernelstandardizer: Specify standardizers for kernelreader.

    • -

  • pstreader: Generalizes snpreader and kernelreader (provides the efficiency of numpy arrays with some of the flexibility of pandas)

    • -

  • util: In one line, intersect and re-order IIDs from snpreader, kernelreader and other sources. Also, efficiently extract a submatrix from an ndarray.

    • -

  • util.IntRangeSet: Efficiently manipulate ranges of integers – for example, genetic position – with set operators including -union, intersection, and set difference.

    • -

  • util.mapreduce1: Run loops in parallel on multiple processes, threads, or clusters.

    • -

  • util.filecache: Automatically copy files to and from any remote storage.

    • -
-
-
Tutorial:
-

-
- -

From PyData Conference, Seattle

- -
-
Code:
-

-
- -
-
Contacts:
-

-
- -
-

snpreader Module

-

Tools for reading and manipulating SNP data.

-
-
-class pysnptools.snpreader.Bed(filename, count_A1=None, iid=None, sid=None, pos=None, num_threads=None, skip_format_check=False, fam_filename=None, bim_filename=None, chrom_map={'MT': 26, 'X': 23, 'XY': 25, 'Y': 24})
-

A SnpReader for random-access reads of Bed/Bim/Fam files from disk.

-

See SnpReader for details and examples.

-

The format is described here.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The *.bed file to read. The ‘.bed’ suffix is optional. The related *.bim and *.fam files will also be read.

    • -

  • count_A1 (bool) – Tells if it should count the number of A1 alleles (the PLINK standard) or the number of A2 alleles. False is the current default, but in the future the default will change to True.

    • -
-

The following three options are never needed, but can be used to avoid reading large ‘.fam’ and ‘.bim’ files when their information is already known.

-
    -

  • iid (an array of strings) – The SnpReader.iid information. If not given, reads info from ‘.fam’ file.

    • -

  • sid (an array of strings) – The SnpReader.sid information. If not given, reads info from ‘.bim’ file.

    • -

  • pos (optional, an array of strings) – The SnpReader.pos information. If not given, reads info from ‘.bim’ file.

    • -
  • -
    num_threads (optinal, int) – The number of threads with which to read data. Defaults to all available processors.

    Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    -
    -
    -
    • -
  • -
    skip_format_check (bool) – By default (False), checks that the ‘.bed’ file starts with the expected bytes

    the first time any file (‘.bed’, ‘.fam’, or ‘.bim’) is opened.

    -
    -
    -
    • -

  • fam_filename (optional, string) – The *.fam file to read. Defaults to the bed filename with the suffix replaced.

    • -

  • bim_filename (optional, string) – The *.bim file to read. Defaults to the bed filename with the suffix replaced.

    • -
  • -
    chrom_map (optional, dictionary) – A dictionary from non-numeric chromosome names to numbers. Defaults to the PLINK

    mapping, namely, plink_chrom_map.

    -
    -
    -
    • -
-
-
-
-
-

Constants

-
    -
  • -
    plink_chrom_map

    A dictionary that ‘X’ to 23, ‘Y’ to 24, ‘XY’ to 25, and ‘MT’ to 26.

    -
    -
    -
    • -
  • -
    reverse_plink_chrom_map

    A dictionary that maps 23 to ‘X’, 24 to ‘Y’, 25 to ‘XY’, and 26 to ‘MT’

    -
    -
    -
    • -
-

Methods beyond SnpReader

-
-

The SnpReader.read() method returns a SnpData with a SnpData.val ndarray. By default, this ndarray is -numpy.float32. Optionally, it can be numpy.float16. For Bed, however, it can also be numpy.int8 with missing values -represented by -127.

-

When reading, any chromosome and position values of 0 (the PLINK standard for missing) will be represented in pos as NaN.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/distributed_bed_test1_X.*","*.bed")
->>> snp_on_disk = Bed(bedfile, count_A1=False)
->>> snpdata1 = snp_on_disk.read() # Read into the default, an float64 ndarray
->>> snpdata1.val.dtype
-dtype('float64')
->>> snpdata1 = snp_on_disk.read(dtype='int8',_require_float32_64=False) #Read into an 'int8' ndarray.
->>> snpdata1.val.dtype
-dtype('int8')
-
-
-
-
-
-property col
-

same as sid

-
- -
-
-property col_property
-

same as pos

-
- -
-
-copyinputs(copier)
-
- -
-
-property row
-

same as iid

-
- -
-
-static write(filename, snpdata, count_A1=False, force_python_only=False, _require_float32_64=True, num_threads=None, reverse_chrom_map={})
-

Writes a SnpData to Bed format and returns the Bed.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • count_A1 (bool) – Tells if it should count the number of A1 alleles (the PLINK standard) or the number of A2 alleles. False is the current default, but in the future the default will change to True.

    • -

  • force_python_only (bool) – Defaults to False. Tells to use Python code rather than faster Rust code. (Might be useful for debugging).

    • -

  • _require_float32_64 (bool) – Defaults to True. Requires that snpdata’s dtype is float32 or float64. (False is useful for writing int8 data.)

    • -

  • num_threads (int) – Maximum number of threads to use. Currently ignored and number of threads is 1.

    • -

  • reverse_chrom_map (dictionary) – Dictionary from chromosome number to chromsome string to write in the *.bim file. Defaults to empty dictionary.

    • -
-
-
Return type:
-

Bed

-
-
-

Any pos values of NaN will be written as 0, the PLINK standard for missing chromosome and position values.

-
>>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_fn = example_file("pysnptools/examples/toydata.5chrom.*", "*.bed")
->>> snpdata = Bed(bed_fn)[:,::2].read() # Read every-other SNP
->>> pstutil.create_directory_if_necessary("tempdir/everyother.bed")
->>> Bed.write("tempdir/everyother.bed",snpdata,count_A1=False)   # Write data in Bed format
-Bed('tempdir/everyother.bed',count_A1=False)
->>> # Can write from an int8 array, too.
->>> snpdata_int = SnpData(val=np.int_(snpdata.val).astype('int8'),iid=snpdata.iid,sid=snpdata.sid,pos=snpdata.pos,_require_float32_64=False)
->>> snpdata_int.val.dtype
-dtype('int8')
->>> Bed.write("tempdir/everyother.bed",snpdata_int,count_A1=False,_require_float32_64=False)
-Bed('tempdir/everyother.bed',count_A1=False)
-
-
-
- -
- -
-
-class pysnptools.snpreader.Dat(filename, skiprows=0)
-

A SnpReader for reading Dat/Fam/Map-formatted files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This is a text format that can store any numeric values. (In contrast, Bed and Ped can only store 0,1,2, and missing). Its Dat files look like:

-
null_0          j       n       0.333   1       2
-null_100        j       n       2       1       1
-null_200        j       n       0       nan     1
-...
-
-
-

Its Map and Fam files are described in the PLINK format reference.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Dat file to read.

    • -

  • skiprows (int) – Number of lines to skip before reading. Defaults to 0.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Dat
->>> from pysnptools.util import example_file # Download and return local file name
->>> dat_file = example_file("pysnptools/examples/toydata.*","*.dat")
->>> data_on_disk = Dat(dat_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-copyinputs(copier)
-
- -
-
-static write(filename, snpdata)
-

Writes a SnpData to dat/fam/map format and returns the Dat.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

Dat

-
-
-
>>> from pysnptools.snpreader import Dat, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dat")
->>> Dat.write("tempdir/toydata10.dat",snpdata)              # Write data in dat/fam/map format
-Dat('tempdir/toydata10.dat')
-
-
-
- -
- -
-
-class pysnptools.snpreader.Dense(filename, extract_iid_function=<function zero_family>, extract_sid_pos_function=<function zero_pos>)
-

A SnpReader for reading *.dense.txt (0,1,2 text files) from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This format looks like:

-
var     4006    9570    22899   37676   41236   41978   55159   66828...
-1-10004-rs12354060      22222222...
-1-707348-rs12184279     222222?2...
-1-724325-rs12564807     00000000...
-...
-
-
-

where rows are sids and columns are iids.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Dense file to read.

    • -

  • extract_iid_function (string) – A function that breaks the row id from the file into a family id and an individual id. Defaults to -setting the family id to 0 and using the whole row id as the iid.

    • -

  • extract_sid_pos_function (string) – A function that breaks the column id from the file into (chromosome, sid, genetic distance, basepair distance). -Defaults to setting the SnpReader.pos information to 0 and using the whole column id as the sid.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Dense
->>> from pysnptools.util import example_file # Download and return local file name
->>> dense_file = example_file("pysnptools/examples/toydata100.dense.txt")
->>> data_on_disk = Dense(dense_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 100)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata, join_iid_function=<function just_case>, join_sid_pos_function=<function just_sid>)
-

Writes a SnpData to Dense format. The values must be 0,1,2 (or missing). Returns a Dense with the default extractors.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • join_iid_function (a function) – function to turn a family id and individual id into a file id for columns. -Defaults ignoring the family id and using the individual id as the column id.

    • -

  • join_sid_pos_function (a function) – function to turn a sid and pos data into a file id for rows. -Defaults ignoring the SnpReader.pos information and using the sid id as the row id.

    • -
-
-
Return type:
-

Dense

-
-
-
>>> from pysnptools.snpreader import Dense, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dense.txt")
->>> Dense.write("tempdir/toydata10.dense.txt",snpdata)        # Write data in Dense format
-Dense('tempdir/toydata10.dense.txt')
-
-
-
- -
- -
-
-class pysnptools.snpreader.DistributedBed(storage)
-

A class that implements the SnpReader interface. It stores Bed-like data in pieces on storage. When you request data, it retrieves only the needed pieces.

-
-
Constructor:
-
Parameters:
-

storage (string or FileCache) – Tells where the DistributedBed data is stored. -A string can be given and will be interpreted as the path to a directory. -A FileCache instance can be given, which provides a method to specify cluster-distributed storage.

-
-
type storage:
-

string or FileCache

-
-
Example:
-

-
-
>>> import os
->>> from pysnptools.snpreader import DistributedBed
->>> from pysnptools.util import example_file # Download and return local file name
->>> folder = os.path.dirname(example_file('pysnptools/examples/toydataSkip10.distributedbed/*'))
->>> data_on_disk = DistributedBed(folder)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 1000)
-
-
-
-
-
-
-property col
-

same as sid

-
- -
-
-property col_property
-

same as pos

-
- -
-
-copyinputs(copier)
-
- -
-
-property row
-

same as iid

-
- -
-
-static write(storage, snpreader, piece_per_chrom_count=1, updater=None, runner=None)
-

Uploads from any Bed-like data to cluster storage for efficient retrieval later. -If some of the contents already exists in storage, it skips uploading that part of the contents. (To avoid this behavior, -clear the storage.)

-
-
Parameters:
-
    -

  • storage (string or FileCache or None.) – Tells where to store SNP data. -A string can be given and will be interpreted as the path of a local directory to use for storage. (The local -directory will not be automatically erased and so must be user managed.) -A FileCache instance can be given, which provides a -method to specify cluster-distributed storage. (FileCache’s will not be automatically erased and must be user managed.) -If None, the storage will be in an automatically-erasing temporary directory. (If the TEMP environment variable is set, Python places the temp directory under it.)

    • -

  • snpreader (SnpReader) – A Bed or other SnpReader with values of 0,1,2, or missing. -(Note that this differs from most other write methods that take a SnpData)

    • -

  • piece_per_chrom_count (A number) – The number of pieces in which to store the data from each chromosome. Data is split across -SNPs. For example, if piece_per_chrom_count is set to 100 and 22 chromosomes are uploaded, then data will be stored in 2200 pieces. Later, when data is requested -only the pieces necessary for the request will be downloaded to local storage.

    • -

  • updater (A function or lambda) – A single argument function to write logging message to, for example, the function created by log_in_place().

    • -

  • runner (Runner) – a Runner, optional: Tells how to run. -(Note that Local and LocalMultProc are good options.) -If not given, the function is run locally.

    • -
-
-
Return type:
-

DistributedBed

-
-
-
>>> from pysnptools.snpreader import DistributedBed, Bed
->>> import shutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> directory = 'tempdir/toydataSkip10.distributedbed'
->>> if os.path.exists(directory):
-...     shutil.rmtree(directory)
->>> bedfile = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpreader = Bed(bedfile,count_A1=False)[:,::10]  # Read every 10 snps from Bed format
->>> DistributedBed.write(directory,snpreader,piece_per_chrom_count=5)  # Write data in DistributedBed format
-DistributedBed(LocalCache('tempdir/toydataSkip10.distributedbed'))
-
-
-
- -
- -
-
-class pysnptools.snpreader.Hdf5(*args, **kwargs)
-
- -
-
-class pysnptools.snpreader.Ped(filename, missing='0')
-

A SnpReader for reading Ped-formatted (and the related Map-formatted) files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This format is described here and looks like:

-
FAM001  1  0 0  1  2  A A  G G  A C
-FAM001  2  0 0  1  2  A A  A G  0 0
-...
-
-
-

the direction of the encoding from allele pair to 0,1,2 is arbitrary. That is, if the alleles are “A” and “G”, then “G G” could be 0 and “A A” could be 2 or visa versa. The pair “A G” will always be 1.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Ped file to read.

    • -

  • missing (string) – The value in the file that represents missing data.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Ped
->>> from pysnptools.util import example_file # Download and return local file name
->>> ped_file = example_file('pysnptools/examples/toydata.ped')
->>> data_on_disk = Ped(ped_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-copyinputs(copier)
-
- -
-
-static write(filename, snpdata)
-

Writes a SnpData to Ped format. The values must be 0,1,2. The direction of the encoding to allele pairs is arbitrary. This means -that if a SnpData is written in Ped format and then read back, then 0’s may become 2’s and 2’s may become 0’s. (1’s will stay 1’s). -Returns the Ped

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

Ped

-
-
-
>>> from pysnptools.snpreader import Ped, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.ped")
->>> Ped.write("tempdir/toydata10.ped",snpdata)            # Write data in Ped format
-Ped('tempdir/toydata10.ped')
-
-
-
- -
- -
-
-class pysnptools.snpreader.Pheno(input, iid_if_none=None, missing=None)
-

A SnpReader for reading “alternative” phenotype files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This text format is described here and looks like:

-
FID    IID      qt1   bmi    site
-F1     1110     2.3   22.22  2
-F2     2202     34.12 18.23  1
-...
-
-
-

where the heading row is optional.

-
-
Constructor:
-
Parameters:
-
    -

  • input (string) – The phenotype file to read.

    • -

  • iid_if_none (None or array of strings) – An SnpReader.iid to use if the file is empty.

    • -

  • missing (None or string) – The value in the file that represents missing data.

    • -
-
-
-

For backwards compatibility with older code, input can be a dictionary (instead of a filename) with these fields:

-
    -

  • ‘header’ : [1] array phenotype name,

    • -

  • ‘vals’ : [N*1] array of phenotype data,

    • -

  • ‘iid’ : [N*2] array of family IDs and individual IDs

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Pheno, Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> pheno_file = example_file('pysnptools/examples/toydata.phe')
->>> data_on_disk = Pheno(pheno_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 1)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata, missing='NaN', sep='\t')
-

Writes a SnpData to Pheno format and returns the Pheno.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • missing (string) – value to threat as missing (default ‘NaN’)

    • -

  • sep (string) – the separator (default ‘ ‘)

    • -
-
-
Return type:
-

Pheno

-
-
-
>>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.phe")
->>> Pheno.write("tempdir/toydata10.txt",snpdata)       # Write data in Pheno format
-Pheno('tempdir/toydata10.txt')
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpData(iid, sid, val, pos=None, name=None, parent_string=None, copyinputs_function=None, xp=None, _require_float32_64=True)
-

A SnpReader for holding SNP values (or similar values) in-memory, along with related iid and sid information. -It is usually created by calling the SnpReader.read() method on another SnpReader, for example, Bed. It can also be constructed directly.

-

See SnpReader for details and examples.

-
-
Constructor:
-
Parameters:
-
    -

  • iid (an array of string pair) – The SnpReader.iid information.

    • -

  • sid (an array of strings) – The SnpReader.sid information.

    • -

  • val (a 2-D array of floats) – The SNP values

    • -

  • pos (optional, an array of strings) – The SnpReader.pos information

    • -

  • name (optional, string) – Information to be display about the origin of this data

    • -

  • copyinputs_function (optional, function) – Used internally by optional clustering code

    • -
  • -
    xp (optional, a Python module or string) – The array module that controls val,

    for example, ‘numpy’ or ‘cupy’. Defaults to numpy. Can also be set with the -‘ARRAY_MODULE’ environment variable.

    -
    -
    -
    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpData
->>> snpdata = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> print((float(snpdata.val[0,1]), snpdata.iid_count, snpdata.sid_count))
-(2.0, 2, 3)
-
-
-
-
-

Equality:

-
-

Two SnpData objects are equal if their four arrays (SnpData.val, SnpReader.iid, SnpReader.sid, and SnpReader.pos) -are ‘array_equal’. (Their ‘name’ does not need to be the same). -If either SnpData.val contains NaN, the objects will not be equal. However, SnpData.allclose() can be used to treat NaN as -regular values.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.snpreader import SnpData
->>> snpdata1 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata2 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata1 == snpdata2) #True, because all the arrays have the same values.
-True
->>> print(snpdata1.val is snpdata2.val) #False, because the two arrays have different memory.
-False
->>> snpdata3 = SnpData(iid=[['a','0'],['b','0']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata4 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata3 == snpdata4) #False, because the iids are different.
-False
->>> snpdata5 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5 == snpdata6) #False, because the val's contain NaN
-False
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
-
-
-
-

Methods beyond SnpReader

-
-
-allclose(value, equal_nan=True)
-
-
Parameters:
-
    -

  • value (SnpData) – Other object with which to compare.

    • -

  • equal_nan (bool) – (Default: True) Tells if NaN in SnpData.val should be treated as regular values when testing equality.

    • -
-
-
-
>>> import numpy as np
->>> snpdata5 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
->>> print(snpdata5.allclose(snpdata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
-False
-
-
-
- -
-
-standardize(standardizer=Unit(), block_size=None, return_trained=False, force_python_only=False, num_threads=None)
-

Does in-place standardization of the in-memory -SNP data. By default, it applies ‘Unit’ standardization, that is: the values for each SNP will have mean zero and standard deviation 1.0. -NaN values are then filled with zero, the mean (consequently, if there are NaN values, the final standard deviation will not be zero. -Note that, for efficiency, this method works in-place, actually changing values in the ndarray. Although it works in place, for convenience -it also returns the SnpData.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – optional – Specify standardization to be applied. -Any Standardizer may be used. Some choices include Unit (default, makes values for each SNP have mean zero and -standard deviation 1.0) and Beta.

    • -

  • block_size (None) – optional – Deprecated.

    • -

  • return_trained (bool) – If true, returns a second value containing a constant Standardizer trained on this data.

    • -

  • force_python_only (bool) – optional – If true, will use pure Python instead of faster C++ libraries.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData (standardizes in place, but for convenience, returns ‘self’)

-
-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bed_file,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1) # Prints the specification for this SnpData
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> print(snpdata1.val[0,0])
-2.0
->>> snpdata1.standardize() # standardize changes the values in snpdata1.val and changes the specification.
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False),Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
->>> snpdata2 = snp_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print('{0:.6f}'.format(snpdata2.val[0,0]))
-0.229416
-
-
-
- -
-
-train_standardizer(apply_in_place, standardizer=Unit(), force_python_only=False, num_thread=None)
-
-

Deprecated since version 0.2.23: Use standardize() with return_trained=True instead.

-
-
- -
-
-property val
-

The 2D NumPy array of floats that represents the values of the SNPs. You can get or set this property.

-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:5,:].read() #read data for first 5 iids
->>> print(snpdata.val[4,100]) #print one of the SNP values
-2.0
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpGen(seed, iid_count, sid_count, chrom_count=22, cache_file=None, block_size=None, sid_batch_size=None)
-

A SnpReader that generates deterministic “random” SNP data on the fly.

-

See SnpReader for general examples of using SnpReader.

-
-
Constructor:
-
Parameters:
-
    -

  • seed (number) – The random seed that (along with block_size) determines the SNP values.

    • -
-
-
Parameters:
-
    -

  • iid_count (number) – the number of iids (number of individuals)

    • -
-
-
Parameters:
-
    -

  • sid_count (number) – the number of sids (number of SNPs)

    • -
-
-
Parameters:
-
    -

  • chrom_count (number) – the number of chromosomes to generate (must be 22 or fewer)

    • -
-
-
Parameters:
-
    -

  • cache_file (string) – (default None) If provided, tells where to cache the common iid, sid, and pos information. Using it can save time.

    • -
-
-
Parameters:
-
    -

  • block_size (number or None) – (default None) Tells how many SNP to generate at once. The default picks a block_size such that iid_count * *block_size is about 100,000.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpGen
->>> #Prepare to generate data for 1000 individuals and 1,000,000 SNPs
->>> snp_gen = SnpGen(seed=332,iid_count=1000,sid_count=1000*1000)
->>> print(snp_gen.iid_count,snp_gen.sid_count)
-1000 1000000
->>> snp_data = snp_gen[:,200*1000:201*1000].read() #Generate for all users and for SNPs 200K to 201K
->>> print(snp_data.val[1,1], snp_data.iid_count, snp_data.sid_count)
-0.0 1000 1000
-
-
-
-
-
-
Also See:
-

snp_gen()

-
-
-
-
-property col
-

A ndarray of the cols id. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.col[-1]) # print the last column id.
-z
->>> data2 = PstData(row=['a','b','c'],col=[[1,0],[2,0]],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.col[-1]) # print the last column id.
-[2 0]
-
-
-
- -
-
-property col_property
-

A ndarray of the additional information for each col. Each element is a ndarray.

-
-
Return type:
-

1- or 2-dimensional ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file)
->>> print(on_disk.col_property[:3]) # print column information for the first three cols: #The '...' is an optional space
-[[1.         0.00800801 0.        ]
- [1.         0.023023   1.        ]
- [1.         0.0700701  4.        ]]
-
-
-
- -
-
-copyinputs(copier)
-
- -
-
-property row
-

A ndarray of the row ids. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length row_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.row[:2]) # print the first two row ids
-['a' 'b']
->>> data2 = PstData(row=[[1,0],[2,0],[2,1]],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.row[:2]) # print the first two row ids
-[[1 0]
- [2 0]]
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpHdf5(*args, **kwargs)
-

A SnpReader for reading *.snp.hdf5 files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

The general HDF5 format is described here. The SnpHdf5 format stores -val, iid, sid, and pos information in Hdf5 format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The SnpHdf5 file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hd5f_file = example_file("pysnptools/examples/toydata.snpmajor.snp.hdf5")
->>> data_on_disk = SnpHdf5(hd5f_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-property col
-

A ndarray of the cols id. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.col[-1]) # print the last column id.
-z
->>> data2 = PstData(row=['a','b','c'],col=[[1,0],[2,0]],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.col[-1]) # print the last column id.
-[2 0]
-
-
-
- -
-
-property row
-

A ndarray of the row ids. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length row_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.row[:2]) # print the first two row ids
-['a' 'b']
->>> data2 = PstData(row=[[1,0],[2,0],[2,1]],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.row[:2]) # print the first two row ids
-[[1 0]
- [2 0]]
-
-
-
- -
-
-static write(filename, snpdata, hdf5_dtype=None, sid_major=True)
-

Writes a SnpData to SnpHdf5 format and return a the SnpHdf5.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • hdf5_dtype (string) – None (use the .val’s dtype) or a Hdf5 dtype, e.g. ‘f8’,’f4’,etc.

    • -

  • sid_major (bool) – Tells if vals should be stored on disk in sid_major (default) or iid_major format.

    • -
-
-
Return type:
-

SnpHdf5

-
-
-
>>> from pysnptools.snpreader import SnpHdf5, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bedfile,count_A1=False)[:,:10].read()     # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.snp.hdf5")
->>> SnpHdf5.write("tempdir/toydata10.snp.hdf5",snpdata)        # Write data in SnpHdf5 format
-SnpHdf5('tempdir/toydata10.snp.hdf5')
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpMemMap(*args, **kwargs)
-

A SnpData that keeps its data in a memory-mapped file. This allows data large than fits in main memory.

-

See SnpData for general examples of using SnpData.

-
-
Constructor:
-
Parameters:
-

filename (string) – The *.snp.memmap file to read.

-
-
-

Also see SnpMemMap.empty() and SnpMemMap.write().

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file('pysnptools/examples/tiny.snp.memmap')
->>> snp_mem_map = SnpMemMap(mem_map_file)
->>> print(snp_mem_map.val[0,1], snp_mem_map.iid_count, snp_mem_map.sid_count)
-2.0 2 3
-
-
-
-
-

Methods inherited from SnpData

-
-
-

Methods beyond SnpReader

-
-
-static empty(iid, sid, filename, pos=None, order='F', dtype=<class 'numpy.float64'>)
-

Create an empty SnpMemMap on disk.

-
-
Parameters:
-
    -

  • iid (an array of string pairs) – The SnpReader.iid information

    • -

  • sid (an array of strings) – The SnpReader.sid information

    • -

  • filename (string) – name of memory-mapped file to create

    • -

  • pos (an array of numeric triples) – optional – The additional SnpReader.pos information associated with each sid. Default: None

    • -

  • order (string or None) – {‘F’ (default), ‘C’}, optional – Specify the order of the ndarray.

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the SnpMemMap.val ndarray.

    • -
-
-
Return type:
-

SnpMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.snpreader import SnpMemMap
->>> filename = "tempdir/tiny.snp.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> snp_mem_map = SnpMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> snp_mem_map.val[:,:] = [[0.,2.,0.],[0.,1.,2.]]
->>> snp_mem_map.flush()
-
-
-
- -
-
-property filename
-

The name of the memory-mapped file

-
- -
-
-flush()
-

Flush SnpMemMap.val to disk and close the file. (If values or properties are accessed again, the file will be reopened.)

-
>>> import pysnptools.util as pstutil
->>> from pysnptools.snpreader import SnpMemMap
->>> filename = "tempdir/tiny.snp.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> snp_mem_map = SnpMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> snp_mem_map.val[:,:] = [[0.,2.,0.],[0.,1.,2.]]
->>> snp_mem_map.flush()
-
-
-
- -
-
-property offset
-

The byte position in the file where the memory-mapped values start.

-

(The disk space before this is used to store SnpReader.iid, etc. information. -This property is useful when interfacing with, for example, external Fortran and C matrix libraries.)

-
- -
-
-property val
-

The 2D NumPy memmap array of floats that represents the values. You can get this property, but cannot set it (except with itself)

-
>>> from pysnptools.snpreader import SnpMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file('pysnptools/examples/tiny.snp.memmap')
->>> snp_mem_map = SnpMemMap(mem_map_file)
->>> print(snp_mem_map.val[0,1])
-2.0
-
-
-
- -
-
-static write(filename, snpreader, standardizer=Identity(), order='A', dtype=None, block_size=None, num_threads=None)
-

Writes a SnpReader to SnpMemMap format.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpreader (SnpReader) – The data that should be written to disk.

    • -
-
-
Return type:
-

SnpMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> from pysnptools.snpreader import Bed, SnpMemMap
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> bed = Bed(bed_file)
->>> pstutil.create_directory_if_necessary("tempdir/toydata.5chrom.snp.memmap") #LATER should we just promise to create directories?
->>> SnpMemMap.write("tempdir/toydata.5chrom.snp.memmap",bed)      # Write bed in SnpMemMap format
-SnpMemMap('tempdir/toydata.5chrom.snp.memmap')
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpNpz(*args, **kwargs)
-

A SnpReader for reading *.snp.npz files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

The general NPZ format is described here. The SnpNpz format stores -val, iid, sid, and pos information in NPZ format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The SnpNpz file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file('pysnptools/examples/toydata10.snp.npz')
->>> data_on_disk = SnpNpz(npz_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10)
-
-
-
-
-

Methods beyond SnpReader

-
-
-property col
-

A ndarray of the cols id. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.col[-1]) # print the last column id.
-z
->>> data2 = PstData(row=['a','b','c'],col=[[1,0],[2,0]],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.col[-1]) # print the last column id.
-[2 0]
-
-
-
- -
-
-property row
-

A ndarray of the row ids. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length row_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.row[:2]) # print the first two row ids
-['a' 'b']
->>> data2 = PstData(row=[[1,0],[2,0],[2,1]],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.row[:2]) # print the first two row ids
-[[1 0]
- [2 0]]
-
-
-
- -
-
-static write(filename, snpdata)
-

Writes a SnpData to SnpNpz format and returns the SnpNpz

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

SnpNpz

-
-
-
>>> from pysnptools.snpreader import SnpNpz, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()     # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.snp.npz")
->>> SnpNpz.write("tempdir/toydata10.snp.npz",snpdata)          # Write data in SnpNpz format
-SnpNpz('tempdir/toydata10.snp.npz')
-
-
-
- -
- -
-
-class pysnptools.snpreader.SnpReader(*args, **kwargs)
-

A SnpReader is one of three things:

-
    -

  • A class such as Bed for you to specify a file with data. For example,

    • -
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk) # prints the name of the file reader
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)
->>> snp_on_disk.sid_count # prints the number of SNPS (but doesn't read any SNP values)
-1015
-
-
-
    -

  • A SnpData class that holds SNP data in memory, typically after reading it from disk:

    • -
-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> snpdata1 = snp_on_disk.read() #reads the SNP values
->>> type(snpdata1.val).__name__ # The val property is an ndarray of SNP values
-'ndarray'
->>> print(snpdata1) # prints the name of the file reader
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> snpdata1.iid_count #prints the number of iids (number of individuals) in this in-memory data
-300
-
-
-
    -
  • -
    A subset of any SnpReader, specified with “[ iid_index , sid_index ]”, to read only some SNP values.

    It can also be used to re-order the values.

    -
    -
    -
    • -
-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> subset_on_disk = snp_on_disk[[3,4],::2] # specification for a subset of the data on disk. No SNP values are read yet.
->>> print(subset_on_disk.sid_count) # prints the number of sids in this subset (but still doesn't read any SNP values)
-508
->>> print(subset_on_disk) #prints a specification of 'subset_on_disk'
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2]
->>> snpdata_subset = subset_on_disk.read() # efficiently reads the specified subset of values from the disk
->>> print(snpdata_subset) # prints the name of the file reader
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2])
->>> print((int(snpdata_subset.val.shape[0]), int(snpdata_subset.val.shape[1]))) # The dimensions of the ndarray of SNP values
-(2, 508)
-
-
-
-

The SnpReaders Classes

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

Format

Random Access

Suffixes

Write method?

SnpData

in-memory floats

Yes

n/a

n/a

Bed

binary, 0,1,2

Yes (by sid)

.bed/.bim/.fam

Yes

Pheno

text, floats

No

.txt,.phe

Yes

Dat

text, floats

No

.dat/.map/.fam

Yes

Ped

text, 0,1,2

No

.ped/.map

Yes

Dense

text, 0,1,2

No

.dense.txt

Yes

SnpNpz

binary, floats

No

.snp.npz

Yes

SnpHdf5

binary, floats

Yes (by sid or iid)

.snp.hdf5

Yes

SnpMemMap

mem-mapped floats

Yes

.snp.memmap

Yes

SnpGen

generated values

Yes (by sid)

n/a

n/a

-
-

Methods & Properties:

-
-

Every SnpReader, such as Bed and SnpData, has these properties: iid, iid_count, sid, sid_count, -pos and these methods: read(), iid_to_index(), sid_to_index(), read_kernel(). See below for details.

-

SnpData is a SnpReader so it supports the above properties and methods. In addition, it supports property SnpData.val, -method SnpData.standardize(), and equality testing. -See below for details.

-

Many of the classes, such as Bed, also provide a static Bed.write() method for writing SnpData to disk.

-
>>> # read from Pheno, write to Bed
->>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
-
-
-
>>> bed_fn = example_file("pysnptools/examples/toydata.5chrom.*", "*.bed")
->>> snpdata = Bed(bed_fn)[:,::2].read() # Read every-other SNP
->>> pstutil.create_directory_if_necessary("tempdir/everyother.bed")
->>> Bed.write("tempdir/everyother.bed",snpdata,count_A1=False)   # Write data in Bed format
-Bed('tempdir/everyother.bed',count_A1=False)
-
-
-
-

iids and sids:

-
-

Individuals are identified with an iid, which is a ndarray of two strings: a family ID and a individual ID. -SNP locations are identified with an sid string in a ndarray. For example:

-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.iid[:3]) # print the first three iids
-[['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']]
->>> print(snp_on_disk.sid[:9]) # print the first nine sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> print(snp_on_disk.iid_to_index([['POP1','44'],['POP1','12']])) #Find the indexes for two iids.
-[2 1]
-
-
-
-

Selecting and Reordering Individuals and SNPs

-
-

You often don’t want to read the SNP values for all iids and sids. You can use indexing to create a subsetting SnpReader that -will read only the SNP values of interest.

-

SnpReaders support the indexing formats supported by ndarray plus two generalizations. Here are examples of indexing with an array -of indexes, with slicing, and with an array of Booleans.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_1 = snp_on_disk[[3,-1],:] #index with an array of indexes. Negatives count from end.
->>> print((subset_snpreader_1.iid_count, subset_snpreader_1.sid_count))
-(2, 1015)
->>> snpdata1 = subset_snpreader_1.read() # read just the two rows of interest from the disk
->>> subset_snpreader_2 = snp_on_disk[:,:0:-2] #index with a slice
->>> print((subset_snpreader_2.iid_count, subset_snpreader_2.sid_count))
-(300, 507)
->>> boolindexes = [s.startswith('23_') for s in snp_on_disk.sid] # create a Boolean index of sids that start '23_'
->>> subset_snpreader_3 = snp_on_disk[:,boolindexes] #index with array of Booleans
->>> print((subset_snpreader_3.iid_count, subset_snpreader_3.sid_count))
-(300, 24)
-
-
-

The first generalization over what ndarray offers is full indexing on both the iid dimension and the sid dimension, in other words, -full multidimensional indexing. For example,

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_4 = snp_on_disk[[3,4],:0:-2] # index on two dimensions at once
->>> print((subset_snpreader_4.iid_count, subset_snpreader_4.sid_count))
-(2, 507)
-
-
-

The second generalization is indexing on a single integer index.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_5 = snp_on_disk[5,:] #index with single integer
->>> print((subset_snpreader_5.iid_count, subset_snpreader_5.sid_count))
-(1, 1015)
-
-
-

Indexing is also useful when you have SNP values in memory via a SnpData index and want to copy a subset of those values. -While you could instead index directly on the SnpData.val ndarray, by indexing on the SnpData instance you -also get iid and cid information.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1.sid[:9]) # print the first 9 sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> snpdata_subset = snpdata1[:,::2].read(view_ok=True,order='A') # create a copy or view with every other sid
->>> print(snpdata_subset.sid[:9])# print the first 9 sids in the subset
-['1_12' '1_10' '1_28' '1_36' '1_4' '1_11' '1_32' '1_9' '1_17']
-
-
-

You can apply indexing on top of indexing to specify subsets of subsets of data to read. In this example, -only the SNP values for every 16th sid is actually read from the disk.

-
>>> # These are just SnpReaders, nothing is read from disk yet
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> half_snpreader = snp_on_disk[:,::2] # a reader for half the sids
->>> quarter_snpreader = half_snpreader[:,::2] # a reader for half of half the sids
->>> sixteenth_snpreader = quarter_snpreader[:,::2][:,::2] # a reader for half of half of half of half the sids
->>> print(sixteenth_snpreader) #Print the specification of this reader
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[:,::2][:,::2][:,::2][:,::2]
->>> # Now we read from disk. Only values for one sid in every 16 will be read.
->>> snpdata_sixteenth = sixteenth_snpreader.read()
->>> print(snpdata_sixteenth.val[0,3])
-2.0
-
-
-
-

When Data is Read:

-
-

SNP data can be enormous so we generally avoid reading it to the degree practical. Specifically,

-
    -

  • Constructing and printing a SnpReader causes no file reading. For example, these commands read no data:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk) # Print the Bed SnpReader specification. No data is read.
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)
->>> subset_on_disk = snp_on_disk[[3,4],::2] # Construct a subsetting SnpReader. No data is read.
->>> print(subset_on_disk) # print the subset SnpReader. No data is read.
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2]
-
    -
    -
    • -

  • Properties and methods related to the iids and sids (to the degree practical) read just some iid and sid data from the disk, -not SNP value data. Moreover, the iid and sid data is read from file only once. Consider these commands:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk.sid[:9]) # without reading any SNP values data from disk, read the sid and iid data from disk, cache it, and then print the first nine sids.
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> print(snp_on_disk.sid_to_index(['1_10','1_13'])) #use the cached sid information to find the indexes of '1_10' and '1_13'. (No data is read from disk.)
-[2 9]
-
    -
    -
    • -

  • The only methods that read SNP values from file are read() and read_kernel() (to the degree practical). For example:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() #read all the SNP values from disk, creating a new SnpData instance that keeps these values in memory
->>> print(snpdata1.val[0,2])# print the SNP value for the iid with index 0 and the sid with index 2. (No data is read from disk.)
-1.0
-
    -
    -
    • -

  • If you request the values for only a subset of the iids or sids, (to the degree practical) only that subset will be read from disk. -for example:

    -
    >>> subset_on_disk = Bed(bedfile,count_A1=False)[[3,4],::2] # Construct a subsetting SnpReader. No data is read.
->>> snpdata_subset = subset_on_disk.read() # from disk, read the SNP values for the iids with index 3 and 4 AND sids with even numbered indexes.
->>> print(snpdata_subset.val[0,2]) # print the SNP value with subset iid index 0 and sid index 2 (corresponding to iid index 3 and sid index 4 in the full data). No data is read from disk.
-2.0
-
    -
    -
    • -
-
-

When Data is Re-Read and Copied:

-
-

Every time you call a SnpReader’s read() method, the SnpReader re-reads the SNP value data and returns a new in-memory SnpData -(with SnpData.val property containing a new ndarray of the SNP values). Likewise, when you call the kernel() method, the SnpReader re-reads -the data and returns a new kernel ndarray.

-

Here is an example of what not to do, because it causes all the SNP value data to be read twice.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> # The following is not recommended because it inefficiently reads all the SNP values twice.
->>> print(snp_on_disk.read().val[0,2]) # read all values into a new SnpData, print a SNP value
-1.0
->>> print(snp_on_disk.read().val[0,3]) # read all values (again) into a second new SnpData, print a SNP value
-2.0
-
-
-

Here are two efficient alternatives. First, if all SNP values can all fit in memory, read them once into a SnpData and then -access that SnpData multiple times.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() # read all values into a new SnpData
->>> print(snpdata1.val[0,2]) # print a SNP value from snpdata1's in-memory ndarray
-1.0
->>> print(snpdata1.val[0,3]) # print another SNP value from snpdata1's in-memory ndarray.
-2.0
-
-
-

Second, if the SNP value data is too large to fit in memory, use subsetting to read only the SNP values of interest from disk.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk[0,2].read().val[0,0]) #Define the subset of data and read only that subset from disk.
-1.0
->>> print(snp_on_disk[0,3].read().val[0,0]) #Define a second subset of data and read only that subset from disk.
-2.0
-
-
-

Because the in-memory SnpData class is a kind of SnpReader, you may read from it, too. -Doing so create a new SnpData instance containing a copy of the SNP values in a new ndarray.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() # read all SNP values from disk into a new SnpData
->>> print(snpdata1.val is snpdata1.val) # Do the in-memory SNP values use the same memory as themselves? Yes
-True
->>> snpdata2 = snpdata1.read() # copy all the SNP values into a new ndarray in a new SnpData
->>> print(snpdata2.val is snpdata1.val) # Do the two ndarrays of in-memory SNP values use the same memory?
-False
-
-
-
-

Avoiding Unwanted ndarray Allocations

-
-

You may want a subset of SNPs values from an in-memory SnpData and you may know that this subset and the original SnpData -can safely share the memory of the ndarray of SNP values. For this case, the read() has optional parameters called view_ok and order. If you override -the defaults of “view_ok=False,order=’F’” with “view_ok=True,order=’A’, the read() will, if practical, return a new -SnpData with a ndarray shares memory with the original ndarray. -Use these parameters with care because any change to either ndarray (for example, via SnpData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

-
-
-
-
snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
>>> snpdata1 = snp_on_disk.read() # read all data from disk into a SnpData with a new ndarray
->>> column01 = snpdata1[:,0:1].read(view_ok=True,order='A') #create SnpData with the data from just the first two SNPs. Sharing memory is OK. The memory may be laid out in any order (that is sid-major and iid-major are both OK).
->>> import numpy as np
->>> #print np.may_share_memory(snpdata1.val, column01.val) # Do the two ndarray's share memory? They could (but currently they won't)
->>> column201 = snpdata1[:,[2,0,1]].read(view_ok=True,order='A') #create SnpData with the data from three SNPs, permuted. Sharing memory is OK.
->>> print(np.may_share_memory(snpdata1.val, column201.val)) # Do the two ndarray's share memory? No, ndarray decided that this indexing was too complex for sharing.
-False
-
-
-

The read() Method

-
-

By default the read() returns a ndarray of numpy.float64 laid out in memory in F-contiguous order (iid-index varies the fastest). You may, instead, -ask for numpy.float32 or for C-contiguous order or any order. See read() for details.

-
-
-
The SnpData.standardize() Method

The SnpData.standardize() method, available only on SnpData, does in-place standardization of the in-memory -SNP data. By default, it applies ‘Unit’ standardization, that is: the values for each SNP will have mean zero and standard deviation 1.0. -NaN values are then filled with zeros (consequently, if there are NaN values, the final standard deviation will not be zero). -Note that, for efficiently, this method works in-place, actually changing values in the ndarray. Although it works in place, for convenience -it also returns itself. See SnpData.standardize() for options and details.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1) # Prints the specification for this SnpData
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> print(snpdata1.val[0,0])
-2.0
->>> snpdata1.standardize() # standardize changes the values in snpdata1.val and changes the specification.
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False),Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
->>> snpdata2 = snp_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print('{0:.6f}'.format(snpdata2.val[0,0]))
-0.229416
-
-
-
-
-

The read_kernel() Method

-
-

The read_kernel() method, available on any SnpReader, returns a KernelData. The val property of the KernelData is -an ndarray of the (possibility standardized) SNP values multiplied with their transposed selves. When applied to an read-from-disk SnpReader, such as Bed, -the method can save memory by reading (and standardizing) the data in blocks. See read_kernel() for details.

-
>>> from pysnptools.standardizer import Unit
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> kerneldata1 = snp_on_disk.read_kernel(Unit()) #Create an in-memory kernel from the snp data on disk.
->>> print('{0:.6f}'.format(kerneldata1.val[0,0]))
-901.421836
->>> kerneldata2 = snp_on_disk.read_kernel(Unit(),block_size=10) #Create an in-memory kernel from the snp data on disk, but only read 10 SNPS at a time from the disk.
->>> print('{0:.6f}'.format(kerneldata2.val[0,0]))
-901.421836
-
-
-
-

Details of Methods & Properties:

-
-
-
-
-as_dist(max_weight=2.0, block_size=None)
-

Returns a pysnptools.distreader.DistReader such that turns the an allele count of 0,1, or 2 into a probability distribution of -[1,0,0],[0,1,0], or [0,0,1], respectively. Any other values produce a distribution of [NaN,NaN,NaN].

-
-
Parameters:
-
    -

  • max_weight (number) – optional – Tells the maximum allele count. Default of 2.

    • -

  • block_size (int or None) – optional – Default of None (meaning to load all). Suggested number of sids to read into memory at a time.

    • -
-
-
Return type:
-

class:DistReader

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpreader = Bed(bedfile, count_A1=False)
->>> print(snpreader[0,0].read().val)
-[[2.]]
->>> distreader = snpreader.as_dist(max_weight=2)
->>> print(distreader[0,0].read().val)
-[[[0. 0. 1.]]]
-
-
-
- -
-
-copyinputs(copier)
-
- -
-
-property iid
-

A ndarray of the iids. Each iid is a ndarray of two strings (a family ID and a individual ID) that identifies an individual.

-
-
Return type:
-

ndarray of strings with shape [iid_count,2]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.iid[:3]) # print the first three iids
-[['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']]
-
-
-
- -
-
-property iid_count
-

number of iids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-iid_to_index(list)
-

Takes a list of iids and returns a list of index numbers

-
-
Parameters:
-

list – list of iids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> print(snp_on_disk.iid_to_index([['POP1','44'],['POP1','12']])) #Find the indexes for two iids.
-[2 1]
-
-
-
- -
-
-kernel(standardizer, allowlowrank=False, block_size=10000, blocksize=None, num_threads=None)
-
-

Warning

-

Deprecated. Use read_kernel() instead.

-
-

Returns a ndarray of size iid_count x iid_count. The returned array has the value of the standardized SNP values multiplied with their transposed selves.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – – Specify standardization to be applied before the matrix multiply. Any Standardizer may be used. Some choices include Standardizer.Identity -(do nothing), Unit (make values for each SNP have mean zero and standard deviation 1.0), Beta.

    • -

  • block_size (int or None) – optional – Default of 10000. None means to load all. Suggested number of sids to read into memory at a time.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

ndarray of size iid_count x iid_count

-
-
-

Calling the method again causes the SNP values to be re-read and allocates a new ndarray.

-

When applied to an read-from-disk SnpReader, such as Bed, the method can save memory by reading (and standardizing) the data in blocks.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> kernel = snp_on_disk.kernel(Unit())
->>> print(((int(kernel.shape[0]),int(kernel.shape[1])), '{0:.6f}'.format(kernel[0,0])))
-((300, 300), '901.421836')
-
-
-
- -
-
-property pos
-

A ndarray of the position information for each sid. Each element is a ndarray of three numpy.numbers (chromosome, genetic distance, basepair distance).

-
-
Return type:
-

ndarray of float64 with shape [sid_count, 3]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.pos[:3,]) # print position information for the first three sids: #The '...' is for possible space char
-[[1.         0.00800801        nan]
- [1.         0.023023   1.        ]
- [1.         0.0700701  4.        ]]
-
-
-
- -
-
-read(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None, _require_float32_64=True)
-

Reads the SNP values and returns a SnpData (with SnpData.val property containing a new ndarray of the SNP values).

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (iid-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (sid-index varies the fastest). -If order is ‘A’, then the SnpData.val -ndarray may be in any order (either C-, Fortran-contiguous).

    • -

  • dtype (data-type.) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the SnpData.val ndarray. -(For Bed, only, it can also be numpy.int8. Hidden option _require_float32_64 must be set -to False. See Bed for an example.)

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the SnpData.val’s ndarray. If True, -if practical and reading from a SnpData, will return a new -SnpData with a ndarray shares memory with the original SnpData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray (for example, via SnpData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to read data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData

-
-
-

Calling the method again causes the SNP values to be re-read and creates a new in-memory SnpData with a new ndarray of SNP values.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile, count_A1=False) # Specify SNP data on disk
->>> snpdata1 = snp_on_disk.read() # Read all the SNP data returning a SnpData instance
->>> print(type(snpdata1.val).__name__) # The SnpData instance contains a ndarray of the data.
-ndarray
->>> subset_snpdata = snp_on_disk[:,::2].read() # From the disk, read SNP values for every other sid
->>> print(subset_snpdata.val[0,0]) # Print the first SNP value in the subset
-2.0
->>> subsub_snpdata = subset_snpdata[:10,:].read(order='A',view_ok=True) # Create an in-memory subset of the subset with SNP values for the first ten iids. Share memory if practical.
->>> import numpy as np
->>> # print np.may_share_memory(subset_snpdata.val, subsub_snpdata.val) # Do the two ndarray's share memory? They could. Currently they won't.
-
-
-
- -
-
-read_kernel(standardizer=None, block_size=None, order='A', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Returns a KernelData such that the KernelData.val() property will be a ndarray of the standardized SNP values multiplied with their transposed selves.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – – (required) Specify standardization to be applied before the matrix multiply. Any Standardizer may be used. Some choices include Standardizer.Identity -(do nothing), Unit (make values for each SNP have mean zero and standard deviation 1.0) and Beta.

    • -

  • block_size (int or None) – optional – Default of None (meaning to load all). Suggested number of sids to read into memory at a time.

    • -
-
-
Return type:
-

class:KernelData

-
-
-

Calling the method again causes the SNP values to be re-read and allocates a new class:KernelData.

-

When applied to an read-from-disk SnpReader, such as Bed, the method can save memory by reading (and standardizing) the data in blocks.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> kerneldata1 = snp_on_disk.read_kernel(Unit())
->>> print((int(kerneldata1.iid_count), '{0:.6f}'.format(kerneldata1.val[0,0])))
-(300, '901.421836')
-
-
-
- -
-
-property row_property
-

Defined as a zero-width array for compatibility with PstReader, but not used.

-
- -
-
-property sid
-

A ndarray of the sids. Each sid is a string that identifies a SNP.

-
-
Return type:
-

ndarray (length sid_count) of strings

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.sid[:9]) # print the first nine sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
-
-
-
- -
-
-property sid_count
-

number of sids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-sid_to_index(list)
-

Takes a list of sids and returns a list of index numbers

-
-
Parameters:
-

list (list of strings) – list of sids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> print(snp_on_disk.sid_to_index(['1_10','1_13'])) #Find the indexes for two sids.
-[2 9]
-
-
-
- -
-
-property val_shape
-

Tells the shape of value for a given individual and SNP. For SnpReaders always returns None, meaning a single scalar value.

-
- -
- -
-

snpreader.SnpReader

-
-
-class pysnptools.snpreader.SnpReader(*args, **kwargs)
-

A SnpReader is one of three things:

-
    -

  • A class such as Bed for you to specify a file with data. For example,

    • -
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk) # prints the name of the file reader
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)
->>> snp_on_disk.sid_count # prints the number of SNPS (but doesn't read any SNP values)
-1015
-
-
-
    -

  • A SnpData class that holds SNP data in memory, typically after reading it from disk:

    • -
-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> snpdata1 = snp_on_disk.read() #reads the SNP values
->>> type(snpdata1.val).__name__ # The val property is an ndarray of SNP values
-'ndarray'
->>> print(snpdata1) # prints the name of the file reader
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> snpdata1.iid_count #prints the number of iids (number of individuals) in this in-memory data
-300
-
-
-
    -
  • -
    A subset of any SnpReader, specified with “[ iid_index , sid_index ]”, to read only some SNP values.

    It can also be used to re-order the values.

    -
    -
    -
    • -
-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> subset_on_disk = snp_on_disk[[3,4],::2] # specification for a subset of the data on disk. No SNP values are read yet.
->>> print(subset_on_disk.sid_count) # prints the number of sids in this subset (but still doesn't read any SNP values)
-508
->>> print(subset_on_disk) #prints a specification of 'subset_on_disk'
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2]
->>> snpdata_subset = subset_on_disk.read() # efficiently reads the specified subset of values from the disk
->>> print(snpdata_subset) # prints the name of the file reader
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2])
->>> print((int(snpdata_subset.val.shape[0]), int(snpdata_subset.val.shape[1]))) # The dimensions of the ndarray of SNP values
-(2, 508)
-
-
-
-

The SnpReaders Classes

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

Format

Random Access

Suffixes

Write method?

SnpData

in-memory floats

Yes

n/a

n/a

Bed

binary, 0,1,2

Yes (by sid)

.bed/.bim/.fam

Yes

Pheno

text, floats

No

.txt,.phe

Yes

Dat

text, floats

No

.dat/.map/.fam

Yes

Ped

text, 0,1,2

No

.ped/.map

Yes

Dense

text, 0,1,2

No

.dense.txt

Yes

SnpNpz

binary, floats

No

.snp.npz

Yes

SnpHdf5

binary, floats

Yes (by sid or iid)

.snp.hdf5

Yes

SnpMemMap

mem-mapped floats

Yes

.snp.memmap

Yes

SnpGen

generated values

Yes (by sid)

n/a

n/a

-
-

Methods & Properties:

-
-

Every SnpReader, such as Bed and SnpData, has these properties: iid, iid_count, sid, sid_count, -pos and these methods: read(), iid_to_index(), sid_to_index(), read_kernel(). See below for details.

-

SnpData is a SnpReader so it supports the above properties and methods. In addition, it supports property SnpData.val, -method SnpData.standardize(), and equality testing. -See below for details.

-

Many of the classes, such as Bed, also provide a static Bed.write() method for writing SnpData to disk.

-
>>> # read from Pheno, write to Bed
->>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
-
-
-
>>> bed_fn = example_file("pysnptools/examples/toydata.5chrom.*", "*.bed")
->>> snpdata = Bed(bed_fn)[:,::2].read() # Read every-other SNP
->>> pstutil.create_directory_if_necessary("tempdir/everyother.bed")
->>> Bed.write("tempdir/everyother.bed",snpdata,count_A1=False)   # Write data in Bed format
-Bed('tempdir/everyother.bed',count_A1=False)
-
-
-
-

iids and sids:

-
-

Individuals are identified with an iid, which is a ndarray of two strings: a family ID and a individual ID. -SNP locations are identified with an sid string in a ndarray. For example:

-
>>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.iid[:3]) # print the first three iids
-[['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']]
->>> print(snp_on_disk.sid[:9]) # print the first nine sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> print(snp_on_disk.iid_to_index([['POP1','44'],['POP1','12']])) #Find the indexes for two iids.
-[2 1]
-
-
-
-

Selecting and Reordering Individuals and SNPs

-
-

You often don’t want to read the SNP values for all iids and sids. You can use indexing to create a subsetting SnpReader that -will read only the SNP values of interest.

-

SnpReaders support the indexing formats supported by ndarray plus two generalizations. Here are examples of indexing with an array -of indexes, with slicing, and with an array of Booleans.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_1 = snp_on_disk[[3,-1],:] #index with an array of indexes. Negatives count from end.
->>> print((subset_snpreader_1.iid_count, subset_snpreader_1.sid_count))
-(2, 1015)
->>> snpdata1 = subset_snpreader_1.read() # read just the two rows of interest from the disk
->>> subset_snpreader_2 = snp_on_disk[:,:0:-2] #index with a slice
->>> print((subset_snpreader_2.iid_count, subset_snpreader_2.sid_count))
-(300, 507)
->>> boolindexes = [s.startswith('23_') for s in snp_on_disk.sid] # create a Boolean index of sids that start '23_'
->>> subset_snpreader_3 = snp_on_disk[:,boolindexes] #index with array of Booleans
->>> print((subset_snpreader_3.iid_count, subset_snpreader_3.sid_count))
-(300, 24)
-
-
-

The first generalization over what ndarray offers is full indexing on both the iid dimension and the sid dimension, in other words, -full multidimensional indexing. For example,

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_4 = snp_on_disk[[3,4],:0:-2] # index on two dimensions at once
->>> print((subset_snpreader_4.iid_count, subset_snpreader_4.sid_count))
-(2, 507)
-
-
-

The second generalization is indexing on a single integer index.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> subset_snpreader_5 = snp_on_disk[5,:] #index with single integer
->>> print((subset_snpreader_5.iid_count, subset_snpreader_5.sid_count))
-(1, 1015)
-
-
-

Indexing is also useful when you have SNP values in memory via a SnpData index and want to copy a subset of those values. -While you could instead index directly on the SnpData.val ndarray, by indexing on the SnpData instance you -also get iid and cid information.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1.sid[:9]) # print the first 9 sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> snpdata_subset = snpdata1[:,::2].read(view_ok=True,order='A') # create a copy or view with every other sid
->>> print(snpdata_subset.sid[:9])# print the first 9 sids in the subset
-['1_12' '1_10' '1_28' '1_36' '1_4' '1_11' '1_32' '1_9' '1_17']
-
-
-

You can apply indexing on top of indexing to specify subsets of subsets of data to read. In this example, -only the SNP values for every 16th sid is actually read from the disk.

-
>>> # These are just SnpReaders, nothing is read from disk yet
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> half_snpreader = snp_on_disk[:,::2] # a reader for half the sids
->>> quarter_snpreader = half_snpreader[:,::2] # a reader for half of half the sids
->>> sixteenth_snpreader = quarter_snpreader[:,::2][:,::2] # a reader for half of half of half of half the sids
->>> print(sixteenth_snpreader) #Print the specification of this reader
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[:,::2][:,::2][:,::2][:,::2]
->>> # Now we read from disk. Only values for one sid in every 16 will be read.
->>> snpdata_sixteenth = sixteenth_snpreader.read()
->>> print(snpdata_sixteenth.val[0,3])
-2.0
-
-
-
-

When Data is Read:

-
-

SNP data can be enormous so we generally avoid reading it to the degree practical. Specifically,

-
    -

  • Constructing and printing a SnpReader causes no file reading. For example, these commands read no data:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk) # Print the Bed SnpReader specification. No data is read.
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)
->>> subset_on_disk = snp_on_disk[[3,4],::2] # Construct a subsetting SnpReader. No data is read.
->>> print(subset_on_disk) # print the subset SnpReader. No data is read.
-Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False)[[3,4],::2]
-
    -
    -
    • -

  • Properties and methods related to the iids and sids (to the degree practical) read just some iid and sid data from the disk, -not SNP value data. Moreover, the iid and sid data is read from file only once. Consider these commands:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk.sid[:9]) # without reading any SNP values data from disk, read the sid and iid data from disk, cache it, and then print the first nine sids.
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
->>> print(snp_on_disk.sid_to_index(['1_10','1_13'])) #use the cached sid information to find the indexes of '1_10' and '1_13'. (No data is read from disk.)
-[2 9]
-
    -
    -
    • -

  • The only methods that read SNP values from file are read() and read_kernel() (to the degree practical). For example:

    -
    >>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() #read all the SNP values from disk, creating a new SnpData instance that keeps these values in memory
->>> print(snpdata1.val[0,2])# print the SNP value for the iid with index 0 and the sid with index 2. (No data is read from disk.)
-1.0
-
    -
    -
    • -

  • If you request the values for only a subset of the iids or sids, (to the degree practical) only that subset will be read from disk. -for example:

    -
    >>> subset_on_disk = Bed(bedfile,count_A1=False)[[3,4],::2] # Construct a subsetting SnpReader. No data is read.
->>> snpdata_subset = subset_on_disk.read() # from disk, read the SNP values for the iids with index 3 and 4 AND sids with even numbered indexes.
->>> print(snpdata_subset.val[0,2]) # print the SNP value with subset iid index 0 and sid index 2 (corresponding to iid index 3 and sid index 4 in the full data). No data is read from disk.
-2.0
-
    -
    -
    • -
-
-

When Data is Re-Read and Copied:

-
-

Every time you call a SnpReader’s read() method, the SnpReader re-reads the SNP value data and returns a new in-memory SnpData -(with SnpData.val property containing a new ndarray of the SNP values). Likewise, when you call the kernel() method, the SnpReader re-reads -the data and returns a new kernel ndarray.

-

Here is an example of what not to do, because it causes all the SNP value data to be read twice.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> # The following is not recommended because it inefficiently reads all the SNP values twice.
->>> print(snp_on_disk.read().val[0,2]) # read all values into a new SnpData, print a SNP value
-1.0
->>> print(snp_on_disk.read().val[0,3]) # read all values (again) into a second new SnpData, print a SNP value
-2.0
-
-
-

Here are two efficient alternatives. First, if all SNP values can all fit in memory, read them once into a SnpData and then -access that SnpData multiple times.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() # read all values into a new SnpData
->>> print(snpdata1.val[0,2]) # print a SNP value from snpdata1's in-memory ndarray
-1.0
->>> print(snpdata1.val[0,3]) # print another SNP value from snpdata1's in-memory ndarray.
-2.0
-
-
-

Second, if the SNP value data is too large to fit in memory, use subsetting to read only the SNP values of interest from disk.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> print(snp_on_disk[0,2].read().val[0,0]) #Define the subset of data and read only that subset from disk.
-1.0
->>> print(snp_on_disk[0,3].read().val[0,0]) #Define a second subset of data and read only that subset from disk.
-2.0
-
-
-

Because the in-memory SnpData class is a kind of SnpReader, you may read from it, too. -Doing so create a new SnpData instance containing a copy of the SNP values in a new ndarray.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
->>> snpdata1 = snp_on_disk.read() # read all SNP values from disk into a new SnpData
->>> print(snpdata1.val is snpdata1.val) # Do the in-memory SNP values use the same memory as themselves? Yes
-True
->>> snpdata2 = snpdata1.read() # copy all the SNP values into a new ndarray in a new SnpData
->>> print(snpdata2.val is snpdata1.val) # Do the two ndarrays of in-memory SNP values use the same memory?
-False
-
-
-
-

Avoiding Unwanted ndarray Allocations

-
-

You may want a subset of SNPs values from an in-memory SnpData and you may know that this subset and the original SnpData -can safely share the memory of the ndarray of SNP values. For this case, the read() has optional parameters called view_ok and order. If you override -the defaults of “view_ok=False,order=’F’” with “view_ok=True,order=’A’, the read() will, if practical, return a new -SnpData with a ndarray shares memory with the original ndarray. -Use these parameters with care because any change to either ndarray (for example, via SnpData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

-
-
-
-
snp_on_disk = Bed(bedfile,count_A1=False) # Construct a Bed SnpReader. No data is read.
>>> snpdata1 = snp_on_disk.read() # read all data from disk into a SnpData with a new ndarray
->>> column01 = snpdata1[:,0:1].read(view_ok=True,order='A') #create SnpData with the data from just the first two SNPs. Sharing memory is OK. The memory may be laid out in any order (that is sid-major and iid-major are both OK).
->>> import numpy as np
->>> #print np.may_share_memory(snpdata1.val, column01.val) # Do the two ndarray's share memory? They could (but currently they won't)
->>> column201 = snpdata1[:,[2,0,1]].read(view_ok=True,order='A') #create SnpData with the data from three SNPs, permuted. Sharing memory is OK.
->>> print(np.may_share_memory(snpdata1.val, column201.val)) # Do the two ndarray's share memory? No, ndarray decided that this indexing was too complex for sharing.
-False
-
-
-

The read() Method

-
-

By default the read() returns a ndarray of numpy.float64 laid out in memory in F-contiguous order (iid-index varies the fastest). You may, instead, -ask for numpy.float32 or for C-contiguous order or any order. See read() for details.

-
-
-
The SnpData.standardize() Method

The SnpData.standardize() method, available only on SnpData, does in-place standardization of the in-memory -SNP data. By default, it applies ‘Unit’ standardization, that is: the values for each SNP will have mean zero and standard deviation 1.0. -NaN values are then filled with zeros (consequently, if there are NaN values, the final standard deviation will not be zero). -Note that, for efficiently, this method works in-place, actually changing values in the ndarray. Although it works in place, for convenience -it also returns itself. See SnpData.standardize() for options and details.

-
>>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1) # Prints the specification for this SnpData
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> print(snpdata1.val[0,0])
-2.0
->>> snpdata1.standardize() # standardize changes the values in snpdata1.val and changes the specification.
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False),Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
->>> snpdata2 = snp_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print('{0:.6f}'.format(snpdata2.val[0,0]))
-0.229416
-
-
-
-
-

The read_kernel() Method

-
-

The read_kernel() method, available on any SnpReader, returns a KernelData. The val property of the KernelData is -an ndarray of the (possibility standardized) SNP values multiplied with their transposed selves. When applied to an read-from-disk SnpReader, such as Bed, -the method can save memory by reading (and standardizing) the data in blocks. See read_kernel() for details.

-
>>> from pysnptools.standardizer import Unit
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify some data on disk in Bed format
->>> kerneldata1 = snp_on_disk.read_kernel(Unit()) #Create an in-memory kernel from the snp data on disk.
->>> print('{0:.6f}'.format(kerneldata1.val[0,0]))
-901.421836
->>> kerneldata2 = snp_on_disk.read_kernel(Unit(),block_size=10) #Create an in-memory kernel from the snp data on disk, but only read 10 SNPS at a time from the disk.
->>> print('{0:.6f}'.format(kerneldata2.val[0,0]))
-901.421836
-
-
-
-

Details of Methods & Properties:

-
-
-
-
-as_dist(max_weight=2.0, block_size=None)
-

Returns a pysnptools.distreader.DistReader such that turns the an allele count of 0,1, or 2 into a probability distribution of -[1,0,0],[0,1,0], or [0,0,1], respectively. Any other values produce a distribution of [NaN,NaN,NaN].

-
-
Parameters:
-
    -

  • max_weight (number) – optional – Tells the maximum allele count. Default of 2.

    • -

  • block_size (int or None) – optional – Default of None (meaning to load all). Suggested number of sids to read into memory at a time.

    • -
-
-
Return type:
-

class:DistReader

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpreader = Bed(bedfile, count_A1=False)
->>> print(snpreader[0,0].read().val)
-[[2.]]
->>> distreader = snpreader.as_dist(max_weight=2)
->>> print(distreader[0,0].read().val)
-[[[0. 0. 1.]]]
-
-
-
- -
-
-property iid
-

A ndarray of the iids. Each iid is a ndarray of two strings (a family ID and a individual ID) that identifies an individual.

-
-
Return type:
-

ndarray of strings with shape [iid_count,2]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.iid[:3]) # print the first three iids
-[['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']]
-
-
-
- -
-
-property iid_count
-

number of iids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-iid_to_index(list)
-

Takes a list of iids and returns a list of index numbers

-
-
Parameters:
-

list – list of iids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> print(snp_on_disk.iid_to_index([['POP1','44'],['POP1','12']])) #Find the indexes for two iids.
-[2 1]
-
-
-
- -
-
-kernel(standardizer, allowlowrank=False, block_size=10000, blocksize=None, num_threads=None)
-
-

Warning

-

Deprecated. Use read_kernel() instead.

-
-

Returns a ndarray of size iid_count x iid_count. The returned array has the value of the standardized SNP values multiplied with their transposed selves.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – – Specify standardization to be applied before the matrix multiply. Any Standardizer may be used. Some choices include Standardizer.Identity -(do nothing), Unit (make values for each SNP have mean zero and standard deviation 1.0), Beta.

    • -

  • block_size (int or None) – optional – Default of 10000. None means to load all. Suggested number of sids to read into memory at a time.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

ndarray of size iid_count x iid_count

-
-
-

Calling the method again causes the SNP values to be re-read and allocates a new ndarray.

-

When applied to an read-from-disk SnpReader, such as Bed, the method can save memory by reading (and standardizing) the data in blocks.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> kernel = snp_on_disk.kernel(Unit())
->>> print(((int(kernel.shape[0]),int(kernel.shape[1])), '{0:.6f}'.format(kernel[0,0])))
-((300, 300), '901.421836')
-
-
-
- -
-
-property pos
-

A ndarray of the position information for each sid. Each element is a ndarray of three numpy.numbers (chromosome, genetic distance, basepair distance).

-
-
Return type:
-

ndarray of float64 with shape [sid_count, 3]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.pos[:3,]) # print position information for the first three sids: #The '...' is for possible space char
-[[1.         0.00800801        nan]
- [1.         0.023023   1.        ]
- [1.         0.0700701  4.        ]]
-
-
-
- -
-
-read(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None, _require_float32_64=True)
-

Reads the SNP values and returns a SnpData (with SnpData.val property containing a new ndarray of the SNP values).

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (iid-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (sid-index varies the fastest). -If order is ‘A’, then the SnpData.val -ndarray may be in any order (either C-, Fortran-contiguous).

    • -

  • dtype (data-type.) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the SnpData.val ndarray. -(For Bed, only, it can also be numpy.int8. Hidden option _require_float32_64 must be set -to False. See Bed for an example.)

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the SnpData.val’s ndarray. If True, -if practical and reading from a SnpData, will return a new -SnpData with a ndarray shares memory with the original SnpData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray (for example, via SnpData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to read data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData

-
-
-

Calling the method again causes the SNP values to be re-read and creates a new in-memory SnpData with a new ndarray of SNP values.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile, count_A1=False) # Specify SNP data on disk
->>> snpdata1 = snp_on_disk.read() # Read all the SNP data returning a SnpData instance
->>> print(type(snpdata1.val).__name__) # The SnpData instance contains a ndarray of the data.
-ndarray
->>> subset_snpdata = snp_on_disk[:,::2].read() # From the disk, read SNP values for every other sid
->>> print(subset_snpdata.val[0,0]) # Print the first SNP value in the subset
-2.0
->>> subsub_snpdata = subset_snpdata[:10,:].read(order='A',view_ok=True) # Create an in-memory subset of the subset with SNP values for the first ten iids. Share memory if practical.
->>> import numpy as np
->>> # print np.may_share_memory(subset_snpdata.val, subsub_snpdata.val) # Do the two ndarray's share memory? They could. Currently they won't.
-
-
-
- -
-
-read_kernel(standardizer=None, block_size=None, order='A', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Returns a KernelData such that the KernelData.val() property will be a ndarray of the standardized SNP values multiplied with their transposed selves.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – – (required) Specify standardization to be applied before the matrix multiply. Any Standardizer may be used. Some choices include Standardizer.Identity -(do nothing), Unit (make values for each SNP have mean zero and standard deviation 1.0) and Beta.

    • -

  • block_size (int or None) – optional – Default of None (meaning to load all). Suggested number of sids to read into memory at a time.

    • -
-
-
Return type:
-

class:KernelData

-
-
-

Calling the method again causes the SNP values to be re-read and allocates a new class:KernelData.

-

When applied to an read-from-disk SnpReader, such as Bed, the method can save memory by reading (and standardizing) the data in blocks.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> kerneldata1 = snp_on_disk.read_kernel(Unit())
->>> print((int(kerneldata1.iid_count), '{0:.6f}'.format(kerneldata1.val[0,0])))
-(300, '901.421836')
-
-
-
- -
-
-property row_property
-

Defined as a zero-width array for compatibility with PstReader, but not used.

-
- -
-
-property sid
-

A ndarray of the sids. Each sid is a string that identifies a SNP.

-
-
Return type:
-

ndarray (length sid_count) of strings

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False)
->>> print(snp_on_disk.sid[:9]) # print the first nine sids
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
-
-
-
- -
-
-property sid_count
-

number of sids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-sid_to_index(list)
-

Takes a list of sids and returns a list of index numbers

-
-
Parameters:
-

list (list of strings) – list of sids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bedfile,count_A1=False) # Specify SNP data on disk
->>> print(snp_on_disk.sid_to_index(['1_10','1_13'])) #Find the indexes for two sids.
-[2 9]
-
-
-
- -
-
-property val_shape
-

Tells the shape of value for a given individual and SNP. For SnpReaders always returns None, meaning a single scalar value.

-
- -
- -
-
-

snpreader.Bed

-
-
-class pysnptools.snpreader.Bed(filename, count_A1=None, iid=None, sid=None, pos=None, num_threads=None, skip_format_check=False, fam_filename=None, bim_filename=None, chrom_map={'MT': 26, 'X': 23, 'XY': 25, 'Y': 24})
-

A SnpReader for random-access reads of Bed/Bim/Fam files from disk.

-

See SnpReader for details and examples.

-

The format is described here.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The *.bed file to read. The ‘.bed’ suffix is optional. The related *.bim and *.fam files will also be read.

    • -

  • count_A1 (bool) – Tells if it should count the number of A1 alleles (the PLINK standard) or the number of A2 alleles. False is the current default, but in the future the default will change to True.

    • -
-

The following three options are never needed, but can be used to avoid reading large ‘.fam’ and ‘.bim’ files when their information is already known.

-
    -

  • iid (an array of strings) – The SnpReader.iid information. If not given, reads info from ‘.fam’ file.

    • -

  • sid (an array of strings) – The SnpReader.sid information. If not given, reads info from ‘.bim’ file.

    • -

  • pos (optional, an array of strings) – The SnpReader.pos information. If not given, reads info from ‘.bim’ file.

    • -
  • -
    num_threads (optinal, int) – The number of threads with which to read data. Defaults to all available processors.

    Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    -
    -
    -
    • -
  • -
    skip_format_check (bool) – By default (False), checks that the ‘.bed’ file starts with the expected bytes

    the first time any file (‘.bed’, ‘.fam’, or ‘.bim’) is opened.

    -
    -
    -
    • -

  • fam_filename (optional, string) – The *.fam file to read. Defaults to the bed filename with the suffix replaced.

    • -

  • bim_filename (optional, string) – The *.bim file to read. Defaults to the bed filename with the suffix replaced.

    • -
  • -
    chrom_map (optional, dictionary) – A dictionary from non-numeric chromosome names to numbers. Defaults to the PLINK

    mapping, namely, plink_chrom_map.

    -
    -
    -
    • -
-
-
-
-
-

Constants

-
    -
  • -
    plink_chrom_map

    A dictionary that ‘X’ to 23, ‘Y’ to 24, ‘XY’ to 25, and ‘MT’ to 26.

    -
    -
    -
    • -
  • -
    reverse_plink_chrom_map

    A dictionary that maps 23 to ‘X’, 24 to ‘Y’, 25 to ‘XY’, and 26 to ‘MT’

    -
    -
    -
    • -
-

Methods beyond SnpReader

-
-

The SnpReader.read() method returns a SnpData with a SnpData.val ndarray. By default, this ndarray is -numpy.float32. Optionally, it can be numpy.float16. For Bed, however, it can also be numpy.int8 with missing values -represented by -127.

-

When reading, any chromosome and position values of 0 (the PLINK standard for missing) will be represented in pos as NaN.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/distributed_bed_test1_X.*","*.bed")
->>> snp_on_disk = Bed(bedfile, count_A1=False)
->>> snpdata1 = snp_on_disk.read() # Read into the default, an float64 ndarray
->>> snpdata1.val.dtype
-dtype('float64')
->>> snpdata1 = snp_on_disk.read(dtype='int8',_require_float32_64=False) #Read into an 'int8' ndarray.
->>> snpdata1.val.dtype
-dtype('int8')
-
-
-
-
-
-static write(filename, snpdata, count_A1=False, force_python_only=False, _require_float32_64=True, num_threads=None, reverse_chrom_map={})
-

Writes a SnpData to Bed format and returns the Bed.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • count_A1 (bool) – Tells if it should count the number of A1 alleles (the PLINK standard) or the number of A2 alleles. False is the current default, but in the future the default will change to True.

    • -

  • force_python_only (bool) – Defaults to False. Tells to use Python code rather than faster Rust code. (Might be useful for debugging).

    • -

  • _require_float32_64 (bool) – Defaults to True. Requires that snpdata’s dtype is float32 or float64. (False is useful for writing int8 data.)

    • -

  • num_threads (int) – Maximum number of threads to use. Currently ignored and number of threads is 1.

    • -

  • reverse_chrom_map (dictionary) – Dictionary from chromosome number to chromsome string to write in the *.bim file. Defaults to empty dictionary.

    • -
-
-
Return type:
-

Bed

-
-
-

Any pos values of NaN will be written as 0, the PLINK standard for missing chromosome and position values.

-
>>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_fn = example_file("pysnptools/examples/toydata.5chrom.*", "*.bed")
->>> snpdata = Bed(bed_fn)[:,::2].read() # Read every-other SNP
->>> pstutil.create_directory_if_necessary("tempdir/everyother.bed")
->>> Bed.write("tempdir/everyother.bed",snpdata,count_A1=False)   # Write data in Bed format
-Bed('tempdir/everyother.bed',count_A1=False)
->>> # Can write from an int8 array, too.
->>> snpdata_int = SnpData(val=np.int_(snpdata.val).astype('int8'),iid=snpdata.iid,sid=snpdata.sid,pos=snpdata.pos,_require_float32_64=False)
->>> snpdata_int.val.dtype
-dtype('int8')
->>> Bed.write("tempdir/everyother.bed",snpdata_int,count_A1=False,_require_float32_64=False)
-Bed('tempdir/everyother.bed',count_A1=False)
-
-
-
- -
- -
-
-

snpreader.SnpData

-
-
-class pysnptools.snpreader.SnpData(iid, sid, val, pos=None, name=None, parent_string=None, copyinputs_function=None, xp=None, _require_float32_64=True)
-

A SnpReader for holding SNP values (or similar values) in-memory, along with related iid and sid information. -It is usually created by calling the SnpReader.read() method on another SnpReader, for example, Bed. It can also be constructed directly.

-

See SnpReader for details and examples.

-
-
Constructor:
-
Parameters:
-
    -

  • iid (an array of string pair) – The SnpReader.iid information.

    • -

  • sid (an array of strings) – The SnpReader.sid information.

    • -

  • val (a 2-D array of floats) – The SNP values

    • -

  • pos (optional, an array of strings) – The SnpReader.pos information

    • -

  • name (optional, string) – Information to be display about the origin of this data

    • -

  • copyinputs_function (optional, function) – Used internally by optional clustering code

    • -
  • -
    xp (optional, a Python module or string) – The array module that controls val,

    for example, ‘numpy’ or ‘cupy’. Defaults to numpy. Can also be set with the -‘ARRAY_MODULE’ environment variable.

    -
    -
    -
    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpData
->>> snpdata = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> print((float(snpdata.val[0,1]), snpdata.iid_count, snpdata.sid_count))
-(2.0, 2, 3)
-
-
-
-
-

Equality:

-
-

Two SnpData objects are equal if their four arrays (SnpData.val, SnpReader.iid, SnpReader.sid, and SnpReader.pos) -are ‘array_equal’. (Their ‘name’ does not need to be the same). -If either SnpData.val contains NaN, the objects will not be equal. However, SnpData.allclose() can be used to treat NaN as -regular values.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.snpreader import SnpData
->>> snpdata1 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata2 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata1 == snpdata2) #True, because all the arrays have the same values.
-True
->>> print(snpdata1.val is snpdata2.val) #False, because the two arrays have different memory.
-False
->>> snpdata3 = SnpData(iid=[['a','0'],['b','0']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata4 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata3 == snpdata4) #False, because the iids are different.
-False
->>> snpdata5 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5 == snpdata6) #False, because the val's contain NaN
-False
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
-
-
-
-

Methods beyond SnpReader

-
-
-allclose(value, equal_nan=True)
-
-
Parameters:
-
    -

  • value (SnpData) – Other object with which to compare.

    • -

  • equal_nan (bool) – (Default: True) Tells if NaN in SnpData.val should be treated as regular values when testing equality.

    • -
-
-
-
>>> import numpy as np
->>> snpdata5 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
->>> print(snpdata5.allclose(snpdata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
-False
-
-
-
- -
-
-standardize(standardizer=Unit(), block_size=None, return_trained=False, force_python_only=False, num_threads=None)
-

Does in-place standardization of the in-memory -SNP data. By default, it applies ‘Unit’ standardization, that is: the values for each SNP will have mean zero and standard deviation 1.0. -NaN values are then filled with zero, the mean (consequently, if there are NaN values, the final standard deviation will not be zero. -Note that, for efficiency, this method works in-place, actually changing values in the ndarray. Although it works in place, for convenience -it also returns the SnpData.

-
-
Parameters:
-
    -

  • standardizer (Standardizer) – optional – Specify standardization to be applied. -Any Standardizer may be used. Some choices include Unit (default, makes values for each SNP have mean zero and -standard deviation 1.0) and Beta.

    • -

  • block_size (None) – optional – Deprecated.

    • -

  • return_trained (bool) – If true, returns a second value containing a constant Standardizer trained on this data.

    • -

  • force_python_only (bool) – optional – If true, will use pure Python instead of faster C++ libraries.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData (standardizes in place, but for convenience, returns ‘self’)

-
-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snp_on_disk = Bed(bed_file,count_A1=False) # Specify some data on disk in Bed format
->>> snpdata1 = snp_on_disk.read() # read all SNP values into memory
->>> print(snpdata1) # Prints the specification for this SnpData
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False))
->>> print(snpdata1.val[0,0])
-2.0
->>> snpdata1.standardize() # standardize changes the values in snpdata1.val and changes the specification.
-SnpData(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False),Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
->>> snpdata2 = snp_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print('{0:.6f}'.format(snpdata2.val[0,0]))
-0.229416
-
-
-
- -
-
-train_standardizer(apply_in_place, standardizer=Unit(), force_python_only=False, num_thread=None)
-
-

Deprecated since version 0.2.23: Use standardize() with return_trained=True instead.

-
-
- -
-
-property val
-

The 2D NumPy array of floats that represents the values of the SNPs. You can get or set this property.

-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:5,:].read() #read data for first 5 iids
->>> print(snpdata.val[4,100]) #print one of the SNP values
-2.0
-
-
-
- -
- -
-
-

snpreader.Pheno

-
-
-class pysnptools.snpreader.Pheno(input, iid_if_none=None, missing=None)
-

A SnpReader for reading “alternative” phenotype files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This text format is described here and looks like:

-
FID    IID      qt1   bmi    site
-F1     1110     2.3   22.22  2
-F2     2202     34.12 18.23  1
-...
-
-
-

where the heading row is optional.

-
-
Constructor:
-
Parameters:
-
    -

  • input (string) – The phenotype file to read.

    • -

  • iid_if_none (None or array of strings) – An SnpReader.iid to use if the file is empty.

    • -

  • missing (None or string) – The value in the file that represents missing data.

    • -
-
-
-

For backwards compatibility with older code, input can be a dictionary (instead of a filename) with these fields:

-
    -

  • ‘header’ : [1] array phenotype name,

    • -

  • ‘vals’ : [N*1] array of phenotype data,

    • -

  • ‘iid’ : [N*2] array of family IDs and individual IDs

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Pheno, Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> pheno_file = example_file('pysnptools/examples/toydata.phe')
->>> data_on_disk = Pheno(pheno_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 1)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata, missing='NaN', sep='\t')
-

Writes a SnpData to Pheno format and returns the Pheno.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • missing (string) – value to threat as missing (default ‘NaN’)

    • -

  • sep (string) – the separator (default ‘ ‘)

    • -
-
-
Return type:
-

Pheno

-
-
-
>>> from pysnptools.snpreader import Pheno, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.phe")
->>> Pheno.write("tempdir/toydata10.txt",snpdata)       # Write data in Pheno format
-Pheno('tempdir/toydata10.txt')
-
-
-
- -
- -
-
-

snpreader.Ped

-
-
-class pysnptools.snpreader.Ped(filename, missing='0')
-

A SnpReader for reading Ped-formatted (and the related Map-formatted) files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This format is described here and looks like:

-
FAM001  1  0 0  1  2  A A  G G  A C
-FAM001  2  0 0  1  2  A A  A G  0 0
-...
-
-
-

the direction of the encoding from allele pair to 0,1,2 is arbitrary. That is, if the alleles are “A” and “G”, then “G G” could be 0 and “A A” could be 2 or visa versa. The pair “A G” will always be 1.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Ped file to read.

    • -

  • missing (string) – The value in the file that represents missing data.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Ped
->>> from pysnptools.util import example_file # Download and return local file name
->>> ped_file = example_file('pysnptools/examples/toydata.ped')
->>> data_on_disk = Ped(ped_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata)
-

Writes a SnpData to Ped format. The values must be 0,1,2. The direction of the encoding to allele pairs is arbitrary. This means -that if a SnpData is written in Ped format and then read back, then 0’s may become 2’s and 2’s may become 0’s. (1’s will stay 1’s). -Returns the Ped

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

Ped

-
-
-
>>> from pysnptools.snpreader import Ped, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.ped")
->>> Ped.write("tempdir/toydata10.ped",snpdata)            # Write data in Ped format
-Ped('tempdir/toydata10.ped')
-
-
-
- -
- -
-
-

snpreader.Dat

-
-
-class pysnptools.snpreader.Dat(filename, skiprows=0)
-

A SnpReader for reading Dat/Fam/Map-formatted files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This is a text format that can store any numeric values. (In contrast, Bed and Ped can only store 0,1,2, and missing). Its Dat files look like:

-
null_0          j       n       0.333   1       2
-null_100        j       n       2       1       1
-null_200        j       n       0       nan     1
-...
-
-
-

Its Map and Fam files are described in the PLINK format reference.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Dat file to read.

    • -

  • skiprows (int) – Number of lines to skip before reading. Defaults to 0.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Dat
->>> from pysnptools.util import example_file # Download and return local file name
->>> dat_file = example_file("pysnptools/examples/toydata.*","*.dat")
->>> data_on_disk = Dat(dat_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata)
-

Writes a SnpData to dat/fam/map format and returns the Dat.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

Dat

-
-
-
>>> from pysnptools.snpreader import Dat, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dat")
->>> Dat.write("tempdir/toydata10.dat",snpdata)              # Write data in dat/fam/map format
-Dat('tempdir/toydata10.dat')
-
-
-
- -
- -
-
-

snpreader.Dense

-
-
-class pysnptools.snpreader.Dense(filename, extract_iid_function=<function zero_family>, extract_sid_pos_function=<function zero_pos>)
-

A SnpReader for reading *.dense.txt (0,1,2 text files) from disk.

-

See SnpReader for general examples of using SnpReaders.

-

This format looks like:

-
var     4006    9570    22899   37676   41236   41978   55159   66828...
-1-10004-rs12354060      22222222...
-1-707348-rs12184279     222222?2...
-1-724325-rs12564807     00000000...
-...
-
-
-

where rows are sids and columns are iids.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The Dense file to read.

    • -

  • extract_iid_function (string) – A function that breaks the row id from the file into a family id and an individual id. Defaults to -setting the family id to 0 and using the whole row id as the iid.

    • -

  • extract_sid_pos_function (string) – A function that breaks the column id from the file into (chromosome, sid, genetic distance, basepair distance). -Defaults to setting the SnpReader.pos information to 0 and using the whole column id as the sid.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Dense
->>> from pysnptools.util import example_file # Download and return local file name
->>> dense_file = example_file("pysnptools/examples/toydata100.dense.txt")
->>> data_on_disk = Dense(dense_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 100)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata, join_iid_function=<function just_case>, join_sid_pos_function=<function just_sid>)
-

Writes a SnpData to Dense format. The values must be 0,1,2 (or missing). Returns a Dense with the default extractors.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • join_iid_function (a function) – function to turn a family id and individual id into a file id for columns. -Defaults ignoring the family id and using the individual id as the column id.

    • -

  • join_sid_pos_function (a function) – function to turn a sid and pos data into a file id for rows. -Defaults ignoring the SnpReader.pos information and using the sid id as the row id.

    • -
-
-
Return type:
-

Dense

-
-
-
>>> from pysnptools.snpreader import Dense, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()  # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dense.txt")
->>> Dense.write("tempdir/toydata10.dense.txt",snpdata)        # Write data in Dense format
-Dense('tempdir/toydata10.dense.txt')
-
-
-
- -
- -
-
-

snpreader.SnpHdf5

-
-
-class pysnptools.snpreader.SnpHdf5(*args, **kwargs)
-

A SnpReader for reading *.snp.hdf5 files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

The general HDF5 format is described here. The SnpHdf5 format stores -val, iid, sid, and pos information in Hdf5 format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The SnpHdf5 file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hd5f_file = example_file("pysnptools/examples/toydata.snpmajor.snp.hdf5")
->>> data_on_disk = SnpHdf5(hd5f_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10000)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata, hdf5_dtype=None, sid_major=True)
-

Writes a SnpData to SnpHdf5 format and return a the SnpHdf5.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -

  • hdf5_dtype (string) – None (use the .val’s dtype) or a Hdf5 dtype, e.g. ‘f8’,’f4’,etc.

    • -

  • sid_major (bool) – Tells if vals should be stored on disk in sid_major (default) or iid_major format.

    • -
-
-
Return type:
-

SnpHdf5

-
-
-
>>> from pysnptools.snpreader import SnpHdf5, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bedfile,count_A1=False)[:,:10].read()     # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.snp.hdf5")
->>> SnpHdf5.write("tempdir/toydata10.snp.hdf5",snpdata)        # Write data in SnpHdf5 format
-SnpHdf5('tempdir/toydata10.snp.hdf5')
-
-
-
- -
- -
-
-

snpreader.SnpNpz

-
-
-class pysnptools.snpreader.SnpNpz(*args, **kwargs)
-

A SnpReader for reading *.snp.npz files from disk.

-

See SnpReader for general examples of using SnpReaders.

-

The general NPZ format is described here. The SnpNpz format stores -val, iid, sid, and pos information in NPZ format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The SnpNpz file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file('pysnptools/examples/toydata10.snp.npz')
->>> data_on_disk = SnpNpz(npz_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 10)
-
-
-
-
-

Methods beyond SnpReader

-
-
-static write(filename, snpdata)
-

Writes a SnpData to SnpNpz format and returns the SnpNpz

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpdata (SnpData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

SnpNpz

-
-
-
>>> from pysnptools.snpreader import SnpNpz, Bed
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpdata = Bed(bed_file,count_A1=False)[:,:10].read()     # Read first 10 snps from Bed format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.snp.npz")
->>> SnpNpz.write("tempdir/toydata10.snp.npz",snpdata)          # Write data in SnpNpz format
-SnpNpz('tempdir/toydata10.snp.npz')
-
-
-
- -
- -
-
-

snpreader.SnpMemMap

-
-
-class pysnptools.snpreader.SnpMemMap(*args, **kwargs)
-

A SnpData that keeps its data in a memory-mapped file. This allows data large than fits in main memory.

-

See SnpData for general examples of using SnpData.

-
-
Constructor:
-
Parameters:
-

filename (string) – The *.snp.memmap file to read.

-
-
-

Also see SnpMemMap.empty() and SnpMemMap.write().

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file('pysnptools/examples/tiny.snp.memmap')
->>> snp_mem_map = SnpMemMap(mem_map_file)
->>> print(snp_mem_map.val[0,1], snp_mem_map.iid_count, snp_mem_map.sid_count)
-2.0 2 3
-
-
-
-
-

Methods inherited from SnpData

-
-
-

Methods beyond SnpReader

-
-
-static empty(iid, sid, filename, pos=None, order='F', dtype=<class 'numpy.float64'>)
-

Create an empty SnpMemMap on disk.

-
-
Parameters:
-
    -

  • iid (an array of string pairs) – The SnpReader.iid information

    • -

  • sid (an array of strings) – The SnpReader.sid information

    • -

  • filename (string) – name of memory-mapped file to create

    • -

  • pos (an array of numeric triples) – optional – The additional SnpReader.pos information associated with each sid. Default: None

    • -

  • order (string or None) – {‘F’ (default), ‘C’}, optional – Specify the order of the ndarray.

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the SnpMemMap.val ndarray.

    • -
-
-
Return type:
-

SnpMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.snpreader import SnpMemMap
->>> filename = "tempdir/tiny.snp.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> snp_mem_map = SnpMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> snp_mem_map.val[:,:] = [[0.,2.,0.],[0.,1.,2.]]
->>> snp_mem_map.flush()
-
-
-
- -
-
-property filename
-

The name of the memory-mapped file

-
- -
-
-flush()
-

Flush SnpMemMap.val to disk and close the file. (If values or properties are accessed again, the file will be reopened.)

-
>>> import pysnptools.util as pstutil
->>> from pysnptools.snpreader import SnpMemMap
->>> filename = "tempdir/tiny.snp.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> snp_mem_map = SnpMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> snp_mem_map.val[:,:] = [[0.,2.,0.],[0.,1.,2.]]
->>> snp_mem_map.flush()
-
-
-
- -
-
-property offset
-

The byte position in the file where the memory-mapped values start.

-

(The disk space before this is used to store SnpReader.iid, etc. information. -This property is useful when interfacing with, for example, external Fortran and C matrix libraries.)

-
- -
-
-property val
-

The 2D NumPy memmap array of floats that represents the values. You can get this property, but cannot set it (except with itself)

-
>>> from pysnptools.snpreader import SnpMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file('pysnptools/examples/tiny.snp.memmap')
->>> snp_mem_map = SnpMemMap(mem_map_file)
->>> print(snp_mem_map.val[0,1])
-2.0
-
-
-
- -
-
-static write(filename, snpreader, standardizer=Identity(), order='A', dtype=None, block_size=None, num_threads=None)
-

Writes a SnpReader to SnpMemMap format.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • snpreader (SnpReader) – The data that should be written to disk.

    • -
-
-
Return type:
-

SnpMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> from pysnptools.snpreader import Bed, SnpMemMap
->>> bed_file = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> bed = Bed(bed_file)
->>> pstutil.create_directory_if_necessary("tempdir/toydata.5chrom.snp.memmap") #LATER should we just promise to create directories?
->>> SnpMemMap.write("tempdir/toydata.5chrom.snp.memmap",bed)      # Write bed in SnpMemMap format
-SnpMemMap('tempdir/toydata.5chrom.snp.memmap')
-
-
-
- -
- -
-
-

snpreader.SnpGen

-
-
-class pysnptools.snpreader.SnpGen(seed, iid_count, sid_count, chrom_count=22, cache_file=None, block_size=None, sid_batch_size=None)
-

A SnpReader that generates deterministic “random” SNP data on the fly.

-

See SnpReader for general examples of using SnpReader.

-
-
Constructor:
-
Parameters:
-
    -

  • seed (number) – The random seed that (along with block_size) determines the SNP values.

    • -
-
-
Parameters:
-
    -

  • iid_count (number) – the number of iids (number of individuals)

    • -
-
-
Parameters:
-
    -

  • sid_count (number) – the number of sids (number of SNPs)

    • -
-
-
Parameters:
-
    -

  • chrom_count (number) – the number of chromosomes to generate (must be 22 or fewer)

    • -
-
-
Parameters:
-
    -

  • cache_file (string) – (default None) If provided, tells where to cache the common iid, sid, and pos information. Using it can save time.

    • -
-
-
Parameters:
-
    -

  • block_size (number or None) – (default None) Tells how many SNP to generate at once. The default picks a block_size such that iid_count * *block_size is about 100,000.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.snpreader import SnpGen
->>> #Prepare to generate data for 1000 individuals and 1,000,000 SNPs
->>> snp_gen = SnpGen(seed=332,iid_count=1000,sid_count=1000*1000)
->>> print(snp_gen.iid_count,snp_gen.sid_count)
-1000 1000000
->>> snp_data = snp_gen[:,200*1000:201*1000].read() #Generate for all users and for SNPs 200K to 201K
->>> print(snp_data.val[1,1], snp_data.iid_count, snp_data.sid_count)
-0.0 1000 1000
-
-
-
-
-
-
Also See:
-

snp_gen()

-
-
-
- -
-
-

snpreader.DistributedBed

-
-
-class pysnptools.snpreader.DistributedBed(storage)
-

A class that implements the SnpReader interface. It stores Bed-like data in pieces on storage. When you request data, it retrieves only the needed pieces.

-
-
Constructor:
-
Parameters:
-

storage (string or FileCache) – Tells where the DistributedBed data is stored. -A string can be given and will be interpreted as the path to a directory. -A FileCache instance can be given, which provides a method to specify cluster-distributed storage.

-
-
type storage:
-

string or FileCache

-
-
Example:
-

-
-
>>> import os
->>> from pysnptools.snpreader import DistributedBed
->>> from pysnptools.util import example_file # Download and return local file name
->>> folder = os.path.dirname(example_file('pysnptools/examples/toydataSkip10.distributedbed/*'))
->>> data_on_disk = DistributedBed(folder)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 1000)
-
-
-
-
-
-
-static write(storage, snpreader, piece_per_chrom_count=1, updater=None, runner=None)
-

Uploads from any Bed-like data to cluster storage for efficient retrieval later. -If some of the contents already exists in storage, it skips uploading that part of the contents. (To avoid this behavior, -clear the storage.)

-
-
Parameters:
-
    -

  • storage (string or FileCache or None.) – Tells where to store SNP data. -A string can be given and will be interpreted as the path of a local directory to use for storage. (The local -directory will not be automatically erased and so must be user managed.) -A FileCache instance can be given, which provides a -method to specify cluster-distributed storage. (FileCache’s will not be automatically erased and must be user managed.) -If None, the storage will be in an automatically-erasing temporary directory. (If the TEMP environment variable is set, Python places the temp directory under it.)

    • -

  • snpreader (SnpReader) – A Bed or other SnpReader with values of 0,1,2, or missing. -(Note that this differs from most other write methods that take a SnpData)

    • -

  • piece_per_chrom_count (A number) – The number of pieces in which to store the data from each chromosome. Data is split across -SNPs. For example, if piece_per_chrom_count is set to 100 and 22 chromosomes are uploaded, then data will be stored in 2200 pieces. Later, when data is requested -only the pieces necessary for the request will be downloaded to local storage.

    • -

  • updater (A function or lambda) – A single argument function to write logging message to, for example, the function created by log_in_place().

    • -

  • runner (Runner) – a Runner, optional: Tells how to run. -(Note that Local and LocalMultProc are good options.) -If not given, the function is run locally.

    • -
-
-
Return type:
-

DistributedBed

-
-
-
>>> from pysnptools.snpreader import DistributedBed, Bed
->>> import shutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> directory = 'tempdir/toydataSkip10.distributedbed'
->>> if os.path.exists(directory):
-...     shutil.rmtree(directory)
->>> bedfile = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> snpreader = Bed(bedfile,count_A1=False)[:,::10]  # Read every 10 snps from Bed format
->>> DistributedBed.write(directory,snpreader,piece_per_chrom_count=5)  # Write data in DistributedBed format
-DistributedBed(LocalCache('tempdir/toydataSkip10.distributedbed'))
-
-
-
- -
- -
-
-
-

kernelreader Module

-

Tools for reading and manipulating kernels. A kernels is a matrix from iid to iid that typically tells the relatedness of pairs of people.

-
-

kernelreader.KernelReader

-
-
-class pysnptools.kernelreader.KernelReader(*args, **kwargs)
-

A KernelReader is one of three things:

-
    -

  • A class such as KernelNpz for you to a file with data. For example,

    -
    >>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(kernel_file)
->>> print(kernel_on_disk) # prints specification for reading from file
-KernelNpz('...pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk.iid_count # prints the number of iids (but doesn't read any kernel values)
-500
-
    -
    -
    • -

  • A KernelData class that holds kernel data in memory, typically after computing from a SnpReader or reading it from a KernelReader:

    -
    >>> # Compute kernel from a SnpReader
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> bed_file = example_file('tests/datasets/all_chr.maf0.001.N300.*','*.bed')
->>> snp_on_disk = Bed(bed_file,count_A1=False)
->>> kerneldata1 = snp_on_disk.read_kernel(Unit()) #reads the SNP values and computes the kernel
->>> type(kerneldata1.val).__name__ # The val property is an ndarray of kernel values
-'ndarray'
->>> print(kerneldata1) # prints the specification of the in-memory kernel information
-KernelData(SnpKernel(Bed(...tests/datasets/all_chr.maf0.001.N300.bed',count_A1=False),standardizer=Unit()))
->>> kerneldata1.iid_count #prints the number of iids (number of individuals) in this in-memory data
-300
->>> # Read kernel from a KernelReader
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(kernel_file)
->>> kerneldata2 = kernel_on_disk.read() #reads the kernel values
->>> print(kerneldata2) # prints the specification of the in-memory kernel information
-KernelData(KernelNpz(...pysnptools/examples/toydata.kernel.npz'))
->>> kerneldata2.iid_count #prints the number of iids (number of individuals) in this in-memory data
-500
-
    -
    -
    • -

  • A subset of any KernelReader, specified with “[ iid_index ]” (or specified with “[ iid0_index , iid1_index ]”), to read only some kernel values. It can -also be used to re-order the values.

    -
    >>> kernel_on_disk = KernelNpz(kernel_file)
->>> subset_on_disk1 = kernel_on_disk[[3,4]] # specification for a subset of the data on disk. No kernel values are read yet.
->>> print(subset_on_disk1.iid_count) # prints the number of iids in this subset (but still doesn't read any kernel values)
-2
->>> print(subset_on_disk1) #prints a specification of 'subset_on_disk1'
-KernelNpz(...pysnptools/examples/toydata.kernel.npz')[[3,4],[3,4]]
->>> kerneldata_subset = subset_on_disk1.read() # efficiently (if possible) reads the specified subset of values from the disk
->>> print(kerneldata_subset) # prints the specification of the in-memory kernel information
-KernelData(KernelNpz(...pysnptools/examples/toydata.kernel.npz')[[3,4],[3,4]])
->>> print((int(kerneldata_subset.val.shape[0]), int(kerneldata_subset.val.shape[1]))) # The dimensions of the ndarray of kernel values
-(2, 2)
->>> subset_on_disk2 = kernel_on_disk[[3,4],::2] # specification for a subset of the data on disk. No kernel values are read yet.
->>> print((subset_on_disk2.iid0_count, subset_on_disk2.iid1_count))
-(2, 250)
-
    -
    -
    • -
-

The KernelReaders Classes

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

Format

Random Access

Suffixes

Write method?

KernelData

in-memory

Yes

n/a

n/a

KernelNpz

binary

No

.kernel.npz

Yes

KernelHdf5

binary

Yes

.kernel.hdf5

Yes

Identity

n/a

Yes

n/a

No

SnpKernel

depends

depends

n/a

No

-
-

Methods & Properties:

-
-

Every KernelReader, such as KernelNpz and KernelData, when square has these properties: iid, iid_count, -and these methods: read(), and iid_to_index(). A square kernel is one that has the same iid list for both its rows and columns.

-

More generally, KernelReaders can have one iid list for its rows and a different iid list for its columns, so these properties and methods are also defined: iid0, iid1, iid0_count, -iid1_count, iid0_to_index(), and iid1_to_index().

-

See below for details.

-

KernelData is a KernelReader so it supports the above properties and methods. In addition, it supports property KernelData.val, method KernelData.standardize(), and equality testing.

-

See below for details.

-

Some of the classes, such as KernelNpz, also provide a static KernelNpz.write() method for writing KernelData.

-
>>> # create a kernel from a Bed file and write to KernelNpz format
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> from pysnptools.standardizer import Unit
->>> import pysnptools.util as pstutil
-
-
-
>>> bed_file = example_file('pysnptools/examples/toydata.5chrom.bed')
->>> kerneldata = Bed(bed_file,count_A1=False).read_kernel(Unit())     # Create a kernel from the data in the Bed file
->>> pstutil.create_directory_if_necessary("tempdir/toydata.kernel.npz")
->>> KernelNpz.write("tempdir/toydata.kernel.npz",kerneldata)      # Write data in KernelNpz format
-KernelNpz('tempdir/toydata.kernel.npz')
-
-
-
-

iids:

-
-

Individual are identified with an iid, which is a ndarray of two strings: a family ID and an individual ID. For example:

-
>>> kernel_on_disk = KernelNpz(kernel_file)
->>> print(kernel_on_disk.iid[:3]) # print the first three iids
-[['per0' 'per0']
- ['per1' 'per1']
- ['per2' 'per2']]
->>> print(kernel_on_disk.iid_to_index([['per2','per2'],['per1','per1']])) #Find the indexes for two iids.
-[2 1]
-
-
-
-

KernelReader is a kind of PstReader. See the documentation for PstReader to learn about:

-
-
    -

  • When Data is Read

    • -

  • When Data is Re-Read and Copied

    • -

  • Avoiding Unwanted ndarray Allocations

    • -

  • Creating Subsetting PstReaders with Indexing

    • -
-
-

The read() Method

-
-

By default the read() returns a ndarray of numpy.float64 laid out in memory in F-contiguous order (iid0-index varies the fastest). You may, instead, -ask for numpy.float32 or for C-contiguous order or any order. See read() for details.

-
-
-
The KernelData.standardize() Method

The KernelData.standardize() method, available only on KernelData, does in-place standardization of the in-memory -kernel data. The method multiples the values with a scalar factor such that the diagonal sums to iid_count. Although it works in place, for convenience -it also returns itself. See KernelData.standardize() for details.

-
>>> kernel_on_disk = KernelNpz(kernel_file)
->>> kerneldata1 = kernel_on_disk.read() # read all kernel values into memory
->>> print(np.diag(kerneldata1.val).sum())
-5000000.0
->>> kerneldata1.standardize() # standardize changes the values in kerneldata1.val
-KernelData(KernelNpz(...pysnptools/examples/toydata.kernel.npz'))
->>> print(np.diag(kerneldata1.val).sum())
-500.0
->>> kerneldata2 = kernel_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print(np.diag(kerneldata2.val).sum())
-500.0
-
-
-
-
-

Details of Methods & Properties:

-
-
-property iid
-

A ndarray of the iids. Each iid is a ndarray of two strings (a family ID and a individual ID) that identifies an individual. -Assumes the kernel is square, so will throw an exception if the row iids are different from the column iids.

-
-
Return type:
-

ndarray (length iid_count) of ndarray (length 2) of strings

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not kernel value data. Moreover, the iid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(kernel_file)
->>> print(kernel_on_disk.iid[:3]) # print the first three iids
-[['per0' 'per0']
- ['per1' 'per1']
- ['per2' 'per2']]
-
-
-
- -
-
-property iid0
-

A ndarray of the row iids. See iid

-
- -
-
-property iid0_count
-

number of row iids. See iid_count

-
-
Return type:
-

integer

-
-
-
- -
-
-iid0_to_index(list)
-

Takes a list of row iids and returns a list of index numbers. See iid_to_index

-
- -
-
-property iid1
-

A ndarray of the column iids. See iid

-
- -
-
-property iid1_count
-

number of column iids. See iid_count

-
-
Return type:
-

integer

-
-
-
- -
-
-iid1_to_index(list)
-

Takes a list of column iids and returns a list of index numbers. See iid_to_index

-
- -
-
-property iid_count
-

number of iids -Assumes the kernel is square, so will throw an exception if the row iids are different from the column iids.

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid data from the disk, not kernel value data. Moreover, the iid data is read from file only once.

-
- -
-
-iid_to_index(list)
-

Takes a list of iids and returns a list of index numbers. -Assumes the kernel is square, so will throw an exception if the row iids are different from the column iids.

-
-
Parameters:
-

list – list of iids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid from the disk, not kernel value data. Moreover, the iid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(npz_file)
->>> print(kernel_on_disk.iid_to_index([['per2','per2'],['per1','per1']])) #Find the indexes for two iids.
-[2 1]
-
-
-
- -
-
-read(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Reads the kernel values and returns a KernelData (with KernelData.val property containing a new ndarray of the kernel values).

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (iid0-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (iid1-index varies the fastest). -If order is ‘A’, then the KernelData.val -ndarray may be in any order (either C-, Fortran-contiguous, or even discontiguous).

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the KernelData.val ndarray.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the KernelData.val’s ndarray. If True, -if practical and reading from a KernelData, will return a new -KernelData with a ndarray shares memory with the original KernelData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray (for example, via KernelData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

KernelData

-
-
-

Calling the method again causes the kernel values to be re-read and creates a new in-memory KernelData with a new ndarray of kernel values.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(npz_file)
->>> kerneldata1 = kernel_on_disk.read() # Read all the kernel data returning a KernelData instance
->>> print(type(kerneldata1.val).__name__) # The KernelData instance contains a ndarray of the data.
-ndarray
->>> subset_kerneldata = kernel_on_disk[::2].read() # From the disk, read kernel values for every other iid
->>> print('{0:.6f}'.format(subset_kerneldata.val[0,0])) # Print the first kernel value in the subset
-9923.069928
->>> subsub_kerneldata = subset_kerneldata[:10].read(order='A',view_ok=True) # Create an in-memory subset of the subset with kernel values for the first ten iids. Share memory if practical.
->>> import numpy as np
->>> #print(np.may_share_memory(subset_kerneldata.val, subsub_kerneldata.val)) # Do the two ndarray's share memory? They could. Currently they won't.
-
-
-
- -
-
-property val_shape
-

Tells the shape of value for a given individual and SNP. For KernelReaders always returns None, meaning a single scalar value.

-
- -
- -
-
-

kernelreader.KernelData

-
-
-class pysnptools.kernelreader.KernelData(iid=None, iid0=None, iid1=None, val=None, name=None, parent_string=None, xp=None)
-

A KernelReader for holding kernel values in-memory, along with related iid information. -It is usually created by calling the SnpReader.read_kernel() method on a SnpReader, for example, Bed. -It can also be created by calling the KernelReader.read() method on SnpKernel, for example, KernelNpz. -It can also be constructed directly.

-

See KernelReader for details and examples.

-
-
Constructor:
-
Parameters:
-
    -

  • iid (an array of string pairs) – The KernelReader.iid information.

    • -

  • iid0 (an array of string pairs) – The KernelReader.iid0 information.

    • -

  • iid1 (an array of strings pairs) – The KernelReader.iid1 information.

    • -

  • val (a 2-D array of floats) – The SNP values

    • -

  • name (optional, string) – Information to be display about the origin of this data

    • -
  • -
    xp (optional, a Python module or string) – The array module that controls val,

    for example, ‘numpy’ or ‘cupy’. Defaults to numpy. Can also be set with the -‘ARRAY_MODULE’ environment variable.

    -
    -
    -
    • -
-
-
-

If iid is provided, don’t provide iid0 and iid1. Likewise, if iid0 and iid1 are provided, don’t provide iid.

-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelData
->>> kerneldata = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,1.]])
->>> print((float(kerneldata.val[0,1]), kerneldata.iid_count))
-(0.5, 2)
-
-
-
-
-

Equality:

-
-

Two KernelData objects are equal if their three arrays (iid0, iid1, and KernelData.val) are ‘array_equal’. -(Their ‘string’ does not need to be the same). -If either KernelData.val contains NaN, the objects will not be equal. However, KernelData.allclose() can be used to treat NaN as -regular values.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.kernelreader import KernelData
->>> kerneldata1 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,1.]])
->>> kerneldata2 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,1.]])
->>> print(kerneldata1 == kerneldata2) #True, because all the arrays have the same values.
-True
->>> print(kerneldata1.val is kerneldata2.val) #False, because the two arrays have different memory.
-False
->>> kerneldata3 = KernelData(iid=[['a','0'],['b','0']], val=[[1.,.5],[.5,1.]])
->>> kerneldata4 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,1.]])
->>> print(kerneldata3 == kerneldata4) #False, because the iids are different.
-False
->>> kerneldata5 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,np.nan]])
->>> kerneldata6 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,np.nan]])
->>> print(kerneldata5 == kerneldata6) #False, because the val's contain NaN
-False
->>> print(kerneldata5.allclose(kerneldata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
-
-
-
-

Methods beyond KernelReader

-
-
-allclose(value, equal_nan=True)
-
-
Parameters:
-
    -

  • value (KernelData) – Other object with which to compare.

    • -

  • equal_nan (bool) – (Default: True) Tells if NaN in KernelData.val should be treated as regular values when testing equality.

    • -
-
-
-
>>> import numpy as np
->>> kerneldata5 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,np.nan]])
->>> kerneldata6 = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,np.nan]])
->>> print(kerneldata5.allclose(kerneldata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
->>> print(kerneldata5.allclose(kerneldata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
-False
-
-
-
- -
-
-standardize(standardizer=DiagKtoN(), return_trained=False, force_python_only=False, num_threads=None)
-

Does in-place standardization of the in-memory -kernel data. The method multiples the values with a scalar factor such that the diagonal sums to iid_count. Although it works in place, for convenience -it also returns the KernelData.

-
-
Return type:
-

KernelData (standardizes in place, but for convenience, returns ‘self’)

-
-
-
>>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> import numpy as np
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kernel_on_disk = KernelNpz(kernel_file)
->>> kerneldata1 = kernel_on_disk.read() # read all kernel values into memory
->>> print(np.diag(kerneldata1.val).sum())
-5000000.0
->>> kerneldata1.standardize() # standardize changes the values in kerneldata1.val
-KernelData(KernelNpz(...pysnptools/examples/toydata.kernel.npz'))
->>> print(np.diag(kerneldata1.val).sum())
-500.0
->>> kerneldata2 = kernel_on_disk.read().standardize() # Read and standardize in one expression with only one ndarray allocated.
->>> print(np.diag(kerneldata2.val).sum())
-500.0
-
-
-
- -
-
-property val
-

The 2D NumPy array of floats that represents the values of the kernel. You can get or set this property.

-
>>> from pysnptools.kernelreader import KernelData
->>> kerneldata = KernelData(iid=[['fam0','iid0'],['fam0','iid1']], val=[[1.,.5],[.5,1.]])
->>> print((float(kerneldata.val[0,1]), kerneldata.iid_count))
-(0.5, 2)
-
-
-
- -
- -
-
-

kernelreader.KernelNpz

-
-
-class pysnptools.kernelreader.KernelNpz(*args, **kwargs)
-

A KernelReader for reading *.kernel.npz files from disk.

-

See KernelReader for general examples of using KernelReaders.

-

The general NPZ format is described here. The KernelNpz format stores -val, iid0, and iid1 information in NPZ format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The KernelNpz file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> data_on_disk = KernelNpz(npz_file)
->>> print(data_on_disk.iid_count)
-500
-
-
-
-
-

Methods beyond KernelReader

-
-
-static write(filename, kerneldata)
-

Writes a KernelData to KernelNpz format and returns the KernelNpz.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • kerneldata (KernelData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

KernelNpz

-
-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file('pysnptools/examples/toydata.5chrom.*','*.bed')
->>> kerneldata = Bed(bed_file,count_A1=False).read_kernel(Unit())     # Create a kernel from the data in the Bed file
->>> pstutil.create_directory_if_necessary("tempdir/toydata.kernel.npz")
->>> KernelNpz.write("tempdir/toydata.kernel.npz",kerneldata)      # Write data in KernelNpz format
-KernelNpz('tempdir/toydata.kernel.npz')
-
-
-
- -
- -
-
-

kernelreader.KernelHdf5

-
-
-class pysnptools.kernelreader.KernelHdf5(*args, **kwargs)
-

A KernelReader for reading *.kernel.hdf5 files from disk.

-

See KernelReader for general examples of using KernelReaders.

-

The general HDF5 format is described in here. The KernelHdf5 format stores -val, iid, sid, and pos information in Hdf5 format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The KernelHdf5 file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import KernelHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file('pysnptools/examples/toydata.kernel.hdf5')
->>> data_on_disk = KernelHdf5(hdf5_file)
->>> print(data_on_disk.iid_count)
-500
-
-
-
-
-

Methods beyond KernelReader

-
-
-static write(filename, kerneldata, hdf5_dtype=None, sid_major=True)
-

Writes a KernelData to KernelHdf5 format and returns the KernelHdf5.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • kerneldata (KernelData) – The in-memory data that should be written to disk.

    • -

  • hdf5_dtype (string) – None (use the .val’s dtype) or a Hdf5 dtype, e.g. ‘f8’,’f4’,etc.

    • -

  • col_major (bool) – Tells if vals should be stored on disk in sid_major (default) or iid_major format.

    • -
-
-
Return type:
-

KernelHdf5

-
-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file('pysnptools/examples/toydata.5chrom.*','*.bed')
->>> kerneldata = Bed(bed_file,count_A1=False).read_kernel(Unit())     # Create a kernel from the data in the Bed file
->>> pstutil.create_directory_if_necessary("tempdir/toydata.kernel.hdf5")
->>> KernelHdf5.write("tempdir/toydata.kernel.hdf5",kerneldata)          # Write data in KernelHdf5 format
-KernelHdf5('tempdir/toydata.kernel.hdf5')
-
-
-
- -
- -
-
-

kernelreader.Identity

-
-
-class pysnptools.kernelreader.Identity(iid, test=None)
-

A KernelReader for that represents an identity matrix. No memory for the values is allocated until Identity.read() is called.

-

See KernelReader for general examples of using KernelReaders.

-
-
Constructor:
-
Parameters:
-
-
-
Example:
-

-
-
>>> from pysnptools.kernelreader import Identity
->>> identity = Identity(iid=[['fam0','iid0'],['fam0','iid1']])
->>> print(identity.iid_count)
-2
->>> print(identity.read().val) # '...' is possible space character
-[[...1.  0.]
- [...0.  1.]]
-
-
-
>>> identity = Identity(iid=[['fam0','iid0'],['fam0','iid1'],['fam0','iid2']],test=[['fam0','iid1'],['fam0','iid3']])
->>> print((identity.iid0_count, identity.iid1_count))
-(3, 2)
->>> print(identity.read().val) # '...' is possible space character
-[[...0.  0.]
- [...1.  0.]
- [...0.  0.]]
-
-
-
-
-
- -
-
-

kernelreader.SnpKernel

-
-
-class pysnptools.kernelreader.SnpKernel(snpreader, standardizer=None, block_size=None)
-

A KernelReader that creates a kernel from a SnpReader. No SNP data will be read until -the SnpKernel.read() method is called. Use block_size to avoid ever reading all the SNP data into memory. -at once.

-

See KernelReader for general examples of using KernelReaders.

-
-
Constructor:
-
Parameters:
-
    -

  • snpreader (SnpReader) – The SNP data

    • -

  • standardizer (Standardizer) – How the SNP data should be standardized

    • -

  • block_size (optional, int) – The number of SNPs to read at a time.

    • -
-
-
-

If block_size is not given, then all SNP data will be read at once.

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed
->>> from pysnptools.standardizer import Unit
->>> from pysnptools.util import example_file # Download and return local file name
->>> bed_file = example_file('pysnptools/examples/toydata.5chrom.*','*.bed')
->>> snp_on_disk = Bed(bed_file,count_A1=False)     # A Bed file is specified, but nothing is read from disk
->>> kernel_on_disk = SnpKernel(snp_on_disk, Unit(),block_size=500)  # A kernel is specified, but nothing is read from disk
->>> print(kernel_on_disk) #Print the specification
-SnpKernel(Bed(...pysnptools/examples/toydata.5chrom.bed',count_A1=False),standardizer=Unit(),block_size=500)
->>> print(kernel_on_disk.iid_count)                                  # iid information is read from disk, but not SNP data
-500
->>> kerneldata = kernel_on_disk.read().standardize()                # SNPs are read and Unit standardized, 500 at a time, to create a kernel, which is then standardized
->>> print('{0:.6f}'.format(kerneldata.val[0,0]))
-0.992307
-
-
-
-
-
-
-property pos
-

The SnpReader.pos property of the SNP data.

-
- -
-
-read_snps(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Reads the SNP values, applies the standardizer, and returns a SnpData.

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (iid-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (sid-index varies the fastest). -If order is ‘A’, then the SnpData.val -ndarray may be in any order (either C-, Fortran-contiguous).

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the SnpData.val ndarray.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the SnpData.val’s ndarray. If True, -if practical and reading from a SnpData, will return a new -SnpData with a ndarray shares memory with the original SnpData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray (for example, via SnpData.standardize()) will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to read data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData

-
-
-
- -
-
-property sid
-

The SnpReader.sid property of the SNP data.

-
- -
-
-property sid_count
-

The SnpReader.sid_count property of the SNP data.

-
- -
- -
-
-
-

standardizer Module

-
-

standardizer.Standardizer

-
-
-class pysnptools.standardizer.Standardizer
-

A Standardizer is a class such as Unit and Beta to be used by the SnpData.standardize() and SnpReader.read_kernel() method to standardize SNP data.

-
-
Example:
-

-
-

Read and standardize SNP data.

-
>>> from pysnptools.standardizer import Unit
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
-
-
-

Create a kernel from SNP data on disk.

-
>>> bedfile2 = example_file("pysnptools/examples/toydata.5chrom.*","*.bed")
->>> kerneldata = Bed(bedfile2,count_A1=False).read_kernel(Unit())
->>> print('{0:.6f}'.format(kerneldata.val[0,0]))
-9923.069928
-
-
-

Can also return a constant SNP standardizer that can be applied to other SnpData.

-
>>> snp_whole = Bed(bedfile,count_A1=False)
->>> train_idx, test_idx = range(10,snp_whole.iid_count), range(0,10) #test on the first 10, train on the rest
->>> snp_train, trained_standardizer = Unit().standardize(snp_whole[train_idx,:].read(),return_trained=True)
->>> print('{0:.6f}'.format(snp_train.val[0,0]))
-0.233550
->>> print(trained_standardizer.stats[0,:]) #The mean and stddev of the 1st SNP on the training data # '...' for a possible space character
-[...1.94827586  0.22146953]
->>> snp_test = snp_whole[test_idx,:].read().standardize(trained_standardizer)
->>> float(snp_test.val[0,0])
-0.23354968324845735
-
-
-

Standardize any Numpy array.

-
>>> val = Bed(bedfile,count_A1=False).read().val
->>> print(val[0,0])
-2.0
->>> val = Unit().standardize(val)
->>> print('{0:.6f}'.format(val[0,0]))
-0.229416
-
-
-

Details of Methods & Properties:

-
-
-property is_constant
-

Tell if a standardizer’s statistics are constant. For example, Unit_Trained contains a pre-specified mean and stddev for each SNP, so Unit_Trained.is_constant is True. -Unit, on the other hand, measures the mean and stddev for each SNP, so Unit.is_constant is False.

-
-
Return type:
-

bool

-
-
-
- -
-
-standardize(snps, block_size=None, return_trained=False, force_python_only=False, num_threads=None)
-

Applies standardization, in place, to SnpData (or a NumPy array). For convenience also returns the SnpData (or a NumPy array).

-
-
Parameters:
-
    -

  • snps (SnpData (or a NumPy array)) – SNP values to standardize

    • -

  • block_size (None) – Not used

    • -

  • return_trained (bool) – If true, returns a second value containing a constant standardizer trained on this data.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

SnpData (or a NumPy array), (optional) constant Standardizer

-
-
-
- -
- -
-
-

standardizer.Unit

-
-
-class pysnptools.standardizer.Unit
-

A Standardizer to unit standardize SNP data. For each sid, the mean of the values is zero with standard deviation 1. -NaN values are then filled with zero, the mean (consequently, if there are NaN values, the final standard deviation will not be zero.

-

See Standardizer for more information about standardization.

-
>>> from pysnptools.standardizer import Unit
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit())
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.229416
-
-
-
- -
-
-

standardizer.Beta

-
-
-class pysnptools.standardizer.Beta(a, b)
-

A Standardizer to beta standardize SNP data.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • a (float) – The a parameter of the beta distribution

    • -

  • b (float) – The b parameter of the beta distribution

    • -
-
-
-
-
-
>>> from pysnptools.standardizer import Beta
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Beta(1,25))
->>> print('{0:.6f}'.format(snpdata1.val[0,0]))
-0.680802
-
-
-
- -
-
-

standardizer.Identity

-
-
-class pysnptools.standardizer.Identity
-

A Standardizer that does nothing to SNP data.

-

See Standardizer for more information about standardization.

-
>>> from pysnptools.standardizer import Identity
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read()
->>> print(snpdata1.val[0,0])
-2.0
->>> snpdata1 = snpdata1.standardize(Identity())
->>> print(snpdata1.val[0,0])
-2.0
-
-
-
- -
-
-

standardizer.DiagKtoN

-
-
-class pysnptools.standardizer.DiagKtoN(deprecated_iid_count=None)
-

Both a Standardizer and a A KernelStandardizer.

-

When applied to a SnpData, it multiplies the SNP values by the square root of a factor such that a kernel -constructed from the SNP values will have a diagonal that sums to iid_count. This class thus -standardizes the kernel before it is even constructed.

-

When applied to a KernelData, it multiplies the kernel values by the a factor such that a kernel -will have a diagonal that sums to iid_count.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • deprecated_iid_count (int) (optional) – Deprecated.

    • -
-
-
-
-
-

Example of DiagKtoN to SnpData:

-
>>> from pysnptools.standardizer import DiagKtoN, Unit, Identity
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit()).standardize(DiagKtoN())
->>> kernel1 = snpdata1.read_kernel(Identity())
->>> print('{0:.6f}'.format(np.diag(kernel1.val).sum()))
-300.000000
-
-
-

Example of DiagKtoN to KernelData:

-
>>> import numpy as np
->>> from pysnptools.standardizer import DiagKtoN, Unit, Identity
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit())
->>> kernel1 = snpdata1.read_kernel(DiagKtoN(),block_size=None)
->>> print('{0:.6f}'.format(np.diag(kernel1.val).sum()))
-300.000000
-
-
-
- -
-
-

standardizer.UnitTrained

-
-
-class pysnptools.standardizer.UnitTrained(sid, stats)
-

A Standardizer to unit standardize one set of SNP data based on the mean and stddev of another set of SNP data. -NaN values are then filled with zero.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • stats (ndarray of float) – The mean and stddev of each sid

    • -
-
-
-
-
-
>>> from pysnptools.standardizer import Unit
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> train = Bed(bedfile,count_A1=False)[1:,:].read() # read SNP values for all but the first iid
->>> _, unittrained = train.standardize(Unit(),return_trained=True) #Unit standardize and remember the mean and stddev of each sid
->>> print(unittrained.stats[:5,:]) #Print the means and stddev of the first five sids
-[[...1.94983278  0.21828988]
- [...1.96989967  0.17086341]
- [...1.84280936  0.39057474]
- [...1.99665552  0.0577347 ]
- [...1.97658863  0.15120608]]
->>> test = Bed(bedfile,count_A1=False)[0,:].read() # read SNP values for the first iid
->>> test = test.standardize(unittrained) # Use the mean and stddev of the train data to unit standardize the test data.
->>> print('{0:.6f}'.format(test.val[0,0]))
-0.229819
-
-
-
- -
-
-

standardizer.DiagKtoNTrained

-
-
-class pysnptools.standardizer.DiagKtoNTrained(factor)
-

Both a Standardizer and a A KernelStandardizer.

-

When applied to a SnpData, DiagKtoN standardizes one set of SNP data based on another set of SNP data.

-

When applied to a KernelData, DiagKtoN standardizes one kernel data based on another kernel data.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • factor (float) – The number what when multiplied into the kernel values will make the diagonal of the kernel values to sum to iid_count.

    • -
-
-
-
-
-
- -
-
-
-

kernelstandardizer Module

-
-

kernelstandardizer.KernelStandardizer

-
-
-class pysnptools.kernelstandardizer.KernelStandardizer
-

A KernelStandardizer is a class such as DiagKtoN and Identity to be used by the KernelData.standardize() to standardize Kernel data. -It always works in-place and returns the KernelData on which it works.

-
-
Example:
-

-
-

Read and standardize Kernel data.

-
>>> from pysnptools.kernelstandardizer import DiagKtoN
->>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kerneldata1 = KernelNpz(kernel_file).read()
->>> print(np.diag(kerneldata1.val).sum())
-5000000.0
->>> kerneldata1 = kerneldata1.standardize(DiagKtoN())
->>> print(np.diag(kerneldata1.val).sum())
-500.0
-
-
-

Can also return a constant kernel standardizer that be applied to other KernelData.

-
>>> kernel_whole = KernelNpz(kernel_file)
->>> train_idx, test_idx = range(10,kernel_whole.iid_count), range(0,10)  #test on the first 10, train on the rest
->>> kernel_train, trained_standardizer = DiagKtoN().standardize(kernel_whole[train_idx].read(),return_trained=True)
->>> print('{0:.6f}'.format(np.diag(kernel_train.val).sum()))
-490.000000
->>> print('{0:.6f}'.format(trained_standardizer.factor))
-0.000100
->>> kernel_whole_test = kernel_whole[:,test_idx].read().standardize(trained_standardizer)
->>> print('{0:.6f}'.format(kernel_whole_test.val[0,0]))
-0.992217
-
-
-

Details of Methods & Properties:

-
-
-standardize(kerneldata, return_trained=False, force_python_only=False, num_threads=None)
-

Applies standardization, in place, to KernelData. For convenience also returns the KernelData.

-
-
Parameters:
-
    -

  • snps (KernelData) – kernel values to standardize

    • -

  • return_trained (bool) – If true, returns a second value containing a constant KernelStandardizer trained on this data.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • num_threads (None or int) – optional – The number of threads with which to standardize data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

KernelData, (optional) constant KernelStandardizer

-
-
-
- -
- -
-
-

kernelstandardizer.DiagKtoN

-
-
-class pysnptools.kernelstandardizer.DiagKtoN(deprecated_iid_count=None)
-

Both a Standardizer and a A KernelStandardizer.

-

When applied to a SnpData, it multiplies the SNP values by the square root of a factor such that a kernel -constructed from the SNP values will have a diagonal that sums to iid_count. This class thus -standardizes the kernel before it is even constructed.

-

When applied to a KernelData, it multiplies the kernel values by the a factor such that a kernel -will have a diagonal that sums to iid_count.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • deprecated_iid_count (int) (optional) – Deprecated.

    • -
-
-
-
-
-

Example of DiagKtoN to SnpData:

-
>>> from pysnptools.standardizer import DiagKtoN, Unit, Identity
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit()).standardize(DiagKtoN())
->>> kernel1 = snpdata1.read_kernel(Identity())
->>> print('{0:.6f}'.format(np.diag(kernel1.val).sum()))
-300.000000
-
-
-

Example of DiagKtoN to KernelData:

-
>>> import numpy as np
->>> from pysnptools.standardizer import DiagKtoN, Unit, Identity
->>> from pysnptools.snpreader import Bed
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> snpdata1 = Bed(bedfile,count_A1=False).read().standardize(Unit())
->>> kernel1 = snpdata1.read_kernel(DiagKtoN(),block_size=None)
->>> print('{0:.6f}'.format(np.diag(kernel1.val).sum()))
-300.000000
-
-
-
- -
-
-

kernelstandardizer.DiagKtoNTrained

-
-
-class pysnptools.kernelstandardizer.DiagKtoNTrained(factor)
-

Both a Standardizer and a A KernelStandardizer.

-

When applied to a SnpData, DiagKtoN standardizes one set of SNP data based on another set of SNP data.

-

When applied to a KernelData, DiagKtoN standardizes one kernel data based on another kernel data.

-

See Standardizer for more information about standardization.

-
-
Constructor:
-
Parameters:
-
    -

  • factor (float) – The number what when multiplied into the kernel values will make the diagonal of the kernel values to sum to iid_count.

    • -
-
-
-
-
-
- -
-
-

kernelstandardizer.Identity

-
-
-class pysnptools.kernelstandardizer.Identity
-

A KernelStandardizer that does nothing to kernel data.

-

See KernelStandardizer for more information about standardization.

-
>>> from pysnptools.kernelstandardizer import Identity as KS_Identity
->>> from pysnptools.kernelreader import KernelNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> kernel_file = example_file('pysnptools/examples/toydata.kernel.npz')
->>> kerneldata1 = KernelNpz(kernel_file).read()
->>> print(np.diag(kerneldata1.val).sum())
-5000000.0
->>> kerneldata1 = kerneldata1.standardize(KS_Identity())
->>> print(np.diag(kerneldata1.val).sum())
-5000000.0
-
-
-
- -
-
-
-

distreader Module

-

Tools for reading and manipulating SNP distribution data. For each individual and SNP location, it gives three probabilities: P(AA),P(AB),P(BB), -where A and B are the alleles. The probabilities should sum to 1.0. Missing data is represented by three numpy.NaN’s.

-
-

distreader.DistReader

-
-
-class pysnptools.distreader.DistReader(*args, **kwargs)
-

A DistReader is one of three things:

-
    -

  • A class such as Bgen for you to specify a file with data. For example,

    -
    >>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> print(dist_on_disk) # prints the name of the file reader
-Bgen('...pysnptools/examples/2500x100.bgen')
->>> dist_on_disk.sid_count # prints the number of SNPS (but doesn't read any SNP distribution values)
-100
-
    -
    -
    • -

  • A DistData class that holds SNP distribution data in memory, typically after reading it from disk:

    -
    >>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> distdata1 = dist_on_disk.read() #reads the SNP distribution values
->>> print(type(distdata1.val)) # The val property is an 3-D ndarray of SNP distribution values
-<class 'numpy.ndarray'>
->>> print(distdata1) # prints the name in-memory SNP distribution reader.
-DistData(Bgen('...pysnptools/examples/2500x100.bgen'))
->>> distdata1.iid_count #prints the number of iids (number of individuals) in this in-memory data
-2500
-
    -
    -
    • -

  • A subset of any DistReader, specified with “[ iid_index , sid_index ]”, to read only some SNP distribution values. It can -also be used to re-order the values.

    -
    >>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> subset_on_disk = dist_on_disk[[3,4],::2] # specification for a subset of the data on disk. No SNP distribution values are read yet.
->>> print(subset_on_disk.sid_count) # prints the number of sids in this subset (but still doesn't read any SNP distribution values)
-50
->>> print(subset_on_disk) #prints a specification of 'subset_on_disk'
-Bgen('...pysnptools/examples/2500x100.bgen')[[3,4],::2]
->>> distdata_subset = subset_on_disk.read() # efficiently reads the specified subset of values from the disk
->>> print(distdata_subset) # prints the specification of the in-memory SNP distribution information
-DistData(Bgen('...pysnptools/examples/2500x100.bgen')[[3,4],::2])
->>> print((int(distdata_subset.val.shape[0]), int(distdata_subset.val.shape[1]))) # The dimensions of the ndarray of SNP distribution values
-(2, 50)
-
    -
    -
    • -
-

The DistReaders Classes

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

Format

Random Access

Suffixes

Write method?

DistData

in-memory floats

Yes

n/a

n/a

Bgen

binary, floats

Yes (by sid)

*.bgen

Yes

DistNpz

binary, floats

No

.dist.npz

Yes

DistHdf5

binary, floats

Yes (by sid or iid)

.dist.hdf5

Yes

DistMemMap

mem-mapped floats

Yes

.dist.memmap

Yes

-
-

Methods & Properties:

-
-

Every DistReader, such as Bgen and DistData, has these properties: iid, iid_count, sid, sid_count, -pos and these methods: read(), iid_to_index(), sid_to_index(), as_snp(). See below for details.

-

DistData is a DistReader so it supports the above properties and methods. In addition, it supports property DistData.val.

-

See below for details.

-

Many of the classes, such as Bgen, also provide a static Bgen.write() method for writing DistReader or DistData to disk.

-
>>> from pysnptools.distreader import DistHdf5, Bgen
->>> import pysnptools.util as pstutil
->>> hdf5_file = example_file("pysnptools/examples/toydata.snpmajor.dist.hdf5")
->>> distreader = DistHdf5(hdf5_file)[:,:10] # A reader for the first 10 SNPs in Hdf5 format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.bgen")
->>> Bgen.write("tempdir/toydata10.bgen",distreader)        # Write data in BGEN format
-Bgen('tempdir/toydata10.bgen')
-
-
-
-

iids and sids:

-
-

same as with SnpReader

-
-

Selecting and Reordering Individuals and SNPs

-
-

same as with SnpReader

-
-

When Data is Read:

-
-

same as with SnpReader

-
-

When Data is Re-Read and Copied:

-
-

same as with SnpReader

-
-

Avoiding Unwanted ndarray Allocations

-
-

same as with SnpReader

-
-

The read() Method

-
-

By default the read() returns a 3-D ndarray of numpy.float64 laid out in memory in F-contiguous order -(iid-index varies the fastest). You may, instead, -ask for numpy.float32 or for C-contiguous order or any order. See read() for details.

-
-

Details of Methods & Properties:

-
-
-as_snp(max_weight=2.0, block_size=None)
-

Returns a pysnptools.snpreader.SnpReader such that turns the probability distribution into an expected value.

-

For example, if the probability distribution is [0.466804 0.38812848 0.14506752] and the max_weight is 2, then the expected -value will be (0.466804*0+0.38812848*.5+0.14506752*1)*2, 0.67826352 .

-
-
Parameters:
-
    -

  • max_weight (number) – optional – The expected values will range from 0 to max_weight. Default of 2.

    • -

  • block_size (int or None) – optional – Default of None (meaning to load all). Suggested number of sids to read into memory at a time.

    • -
-
-
Return type:
-

class:SnpReader

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> distreader = Bgen(bgen_file) # Specify distribution data on disk
->>> print(distreader[0,0].read().val)
-[[[0.466804   0.38812848 0.14506752]]]
->>> snpreader = distreader.as_snp(max_weight=2)
->>> print(snpreader[0,0].read().val)
-[[0.67826352]]
-
-
-
- -
-
-property iid
-

A ndarray of the iids. Each iid is a ndarray of two strings (a family ID and a individual ID) that identifies an individual.

-
-
Return type:
-

ndarray of strings with shape [iid_count,2]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP distribution data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> print(dist_on_disk.iid[:3]) # print the first three iids
-[['0' 'iid_0']
- ['0' 'iid_1']
- ['0' 'iid_2']]
-
-
-
- -
-
-property iid_count
-

number of iids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP distribution data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-iid_to_index(list)
-

Takes a list of iids and returns a list of index numbers

-
-
Parameters:
-

list – list of iids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file) # Specify SNP data on disk
->>> print(dist_on_disk.iid_to_index([['0','iid_2'],['0','iid_1']])) #Find the indexes for two iids.
-[2 1]
-
-
-
- -
-
-property pos
-

A ndarray of the position information for each sid. Each element is a ndarray of three numpy.numbers (chromosome, genetic distance, basepair distance).

-
-
Return type:
-

ndarray of float64 with shape [sid_count, 3]

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP distribution data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> print(dist_on_disk.pos[:4,].astype('int')) # print position information for the first four sids: #The '...' is for possible space char
-[[        1         0   9270273]
- [        1         0  39900273]
- [        1         0  70530273]
- [        1         0 101160273]]
-
-
-
- -
-
-read(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Reads the SNP values and returns a DistData (with DistData.val property containing a new 3D ndarray of the SNP distribution values).

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (iid-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (sid-index varies the fastest). -If order is ‘A’, then the DistData.val -ndarray may be in any order (either C-, Fortran-contiguous).

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the DistData.val ndarray.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the DistData.val’s ndarray. If True, -if practical and reading from a DistData, will return a new -DistData with a ndarray shares memory with the original DistData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to read data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

DistData

-
-
-

Calling the method again causes the SNP distribution values to be re-read and creates a new in-memory DistData with a new ndarray of SNP values.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file) # Specify SNP data on disk
->>> distdata1 = dist_on_disk.read() # Read all the SNP data returning a DistData instance
->>> print(type(distdata1.val).__name__) # The DistData instance contains a ndarray of the data.
-ndarray
->>> subset_distdata = dist_on_disk[:,::2].read() # From the disk, read SNP values for every other sid
->>> print(subset_distdata.val[0,0]) # Print the first SNP value in the subset
-[0.466804   0.38812848 0.14506752]
->>> subsub_distdata = subset_distdata[:10,:].read(order='A',view_ok=True) # Create an in-memory subset of the subset with SNP values for the first ten iids. Share memory if practical.
->>> import numpy as np
->>> # print np.may_share_memory(subset_distdata.val, subsub_distdata.val) # Do the two ndarray's share memory? They could. Currently they won't.
-
-
-
- -
-
-property row_property
-

Defined as a zero-width array for compatibility with PstReader, but not used.

-
- -
-
-property sid
-

A ndarray of the sids. Each sid is a string that identifies a SNP.

-
-
Return type:
-

ndarray (length sid_count) of strings

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP distribution data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file)
->>> print(dist_on_disk.sid[:9]) # print the first nine sids
-['sid_0' 'sid_1' 'sid_2' 'sid_3' 'sid_4' 'sid_5' 'sid_6' 'sid_7' 'sid_8']
-
-
-
- -
-
-property sid_count
-

number of sids

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only iid and sid data from the disk, not SNP distribution data. Moreover, the iid and sid data is read from file only once.

-
- -
-
-sid_to_index(list)
-

Takes a list of sids and returns a list of index numbers

-
-
Parameters:
-

list (list of strings) – list of sids

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only iid and sid data from the disk, not SNP value data. Moreover, the iid and sid data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> dist_on_disk = Bgen(bgen_file) # Specify SNP data on disk
->>> print(dist_on_disk.sid_to_index(['sid_2','sid_9'])) #Find the indexes for two sids.
-[2 9]
-
-
-
- -
-
-property val_shape
-

Tells the shape of value for a given individual and SNP. For DistReaders always returns 3.

-
- -
- -
-
-

distreader.Bgen

-
-
-class pysnptools.distreader.Bgen(filename, iid_function=<function default_iid_function>, sid_function=<function default_sid_function>, sample=None, num_threads=None, fresh_properties=True)
-

A DistReader for reading *.bgen files from disk.

-

See DistReader for general examples of using DistReaders.

-

The BGEN format is described here.

-
-

Tip: The ‘gen’ in BGEN stands for ‘genetic’. The ‘gen’ in DistGen stands for generate, because it generates random (genetic) data.*

-
-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The BGEN file to read.

    • -

  • iid_function (optional, function) – Function to turn a BGEN sample into a DistReader.iid. -(Default: bgen.default_iid_function().)

    • -

  • sid_function (optional, function or string) – Function to turn a BGEN (SNP) id and rsid into a DistReader.sid. -(Default: bgen.default_sid_function().) Can also be the string ‘id’ or ‘rsid’, which is faster than using a function.

    • -

  • sample (optional, string) – A GEN sample file. If given, overrides information in *.bgen file.

    • -
  • -
    num_threads (optional, int) – The number of threads with which to read data. Defaults to all available processors.

    Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    -
    -
    -
    • -

  • fresh_properties (optional, bool) – When true (default), memory will be allocated for the iid, sid, and -pos properties. This is safe. When false, the properties will use the Bgen’s on-disk ‘memmap’. That saves -memory, but is only safe if the Bgen object is still around when the properties are used.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/example.bgen")
->>> data_on_disk = Bgen(bgen_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 199)
-
-
-
-
-

Methods beyond DistReader

-
-
-flush()
-

Close the *.bgen file for reading. (If values or properties are accessed again, the file will be reopened.)

-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/example.bgen")
->>> data_on_disk = Bgen(bgen_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(500, 199)
->>> data_on_disk.flush()
-
-
-
- -
-
-static genwrite(filename, distreader, decimal_places=None, id_rsid_function=<function default_id_rsid_function>, sample_function=<function default_sample_function>, block_size=None)
-

Writes a DistReader to Gen format

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create (will also create filename_without_gen.sample file.)

    • -

  • distreader (DistReader) – The data that should be written to disk. It can also be any distreader, for example, DistNpz, DistData, or -another Bgen.

    • -

  • decimal_places – (Default: None) Number of decimal places with which to write the text numbers. None writes to full precision.

    • -

  • id_rsid_function (function) – Function to turn a a DistReader.sid into a GEN (SNP) id and rsid. -(Default: bgen.default_id_rsid_function().)

    • -

  • sid_function (function) – Function to turn a GEN (SNP) id and rsid into a DistReader.sid. -(Default: bgen.default_sid_function().)

    • -

  • block_size (number) – The number of SNPs to read in a batch from distreader. Defaults to a block_size such that block_size * iid_count is about 100,000.

    • -
-
-
Return type:
-

None

-
-
-
>>> from pysnptools.distreader import DistHdf5, Bgen
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file("pysnptools/examples/toydata.snpmajor.dist.hdf5")
->>> distreader = DistHdf5(hdf5_file)[:,:10] # A reader for the first 10 SNPs in Hdf5 format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.bgen")
->>> Bgen.genwrite("tempdir/toydata10.gen",distreader)        # Write data in GEN format
-
-
-
- -
-
-static write(filename, distreader, bits=16, compression=None, sample_function=<function default_sample_function>, id_rsid_function=<function default_id_rsid_function>, iid_function=<function default_iid_function>, sid_function=<function default_sid_function>, block_size=None, qctool_path=None, cleanup_temp_files=True)
-

Writes a DistReader to BGEN format and return a the Bgen. Requires access to the 3rd party QCTool.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • distreader (DistReader) – The data that should be written to disk. It can also be any distreader, for example, DistNpz, DistData, or -another Bgen.

    • -

  • bits (int) – Number of bits, between 1 and 32 used to represent each 0-to-1 probability value. Default is 16. -An np.float32 needs 23 bits. A np.float64 would need 52 bits, which the BGEN format doesn’t offer, so use 32.

    • -

  • compression (string) – How to compress the file. Can be None (default), ‘zlib’, or ‘zstd’.

    • -

  • sample_function (function) – Function to turn a DistReader.iid into a BGEN sample. -(Default: bgen.default_sample_function().)

    • -

  • id_rsid_function (function) – Function to turn a a DistReader.sid into a BGEN (SNP) id and rsid. -(Default: bgen.default_id_rsid_function().)

    • -

  • iid_function (function) – Function to turn a BGEN sample into a DistReader.iid. -(Default: bgen.default_iid_function().)

    • -

  • sid_function (function) – Function to turn a BGEN (SNP) id and rsid into a DistReader.sid. -(Default: bgen.default_sid_function().)

    • -

  • block_size (number) – The number of SNPs to read in a batch from distreader. Defaults to a block_size such that block_size * iid_count is about 100,000.

    • -

  • qctool_path (string) – Tells the path to the 3rd party QCTool. Defaults to reading -path from environment variable QCTOOLPATH. (To use on Windows, install Ubuntu for Windows, install QCTool in Ubuntu, -and then give the path as “ubuntu run <UBUNTU PATH TO QCTOOL”.)

    • -

  • cleanup_temp_files (bool) – Tells if delete temporary *.gen and *.sample files.

    • -
-
-
Return type:
-

Bgen

-
-
-
>>> from pysnptools.distreader import DistHdf5, Bgen
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file("pysnptools/examples/toydata.snpmajor.dist.hdf5")
->>> distreader = DistHdf5(hdf5_file)[:,:10] # A reader for the first 10 SNPs in Hdf5 format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.bgen")
->>> Bgen.write("tempdir/toydata10.bgen",distreader)        # Write data in BGEN format
-Bgen('tempdir/toydata10.bgen')
-
-
-
- -
- -
-
-pysnptools.distreader.bgen.default_iid_function(sample)
-

The default function for turning a Bgen sample into a -two-part pysnptools.distreader.DistReader.iid. -If the Bgen sample contains a single comma, we split on the comma to -create the iid. -Otherwise, the iid will be (‘0’,sample)

-
>>> default_iid_function('fam0,ind0')
-('fam0', 'ind0')
->>> default_iid_function('ind0')
-('0', 'ind0')
-
-
-
- -
-
-pysnptools.distreader.bgen.default_sid_function(id, rsid)
-

The default function for turning a Bgen (SNP) id and rsid into a -pysnptools.distreader.DistReader.sid. -If the Bgen rsid is ‘’ or ‘0’, the sid will be the (SNP) id. -Otherwise, the sid will be ‘ID,RSID’

-
>>> default_sid_function('SNP1','rs102343')
-'SNP1,rs102343'
->>> default_sid_function('SNP1','0')
-'SNP1'
-
-
-
- -
-
-pysnptools.distreader.bgen.default_sample_function(famid, indid)
-

The default function for turning a two-part pysnptools.distreader.DistReader.iid into a a Bgen sample. -If the iid’s first part (the family id) is ‘0’ or ‘’, the sample will be iid’s 2nd part. -Otherwise, the sample will be ‘FAMID,INDID’

-
>>> default_sample_function('fam0','ind0')
-'fam0,ind0'
->>> default_sample_function('0','ind0')
-'ind0'
-
-
-
- -
-
-pysnptools.distreader.bgen.default_id_rsid_function(sid)
-

The default function for turning a pysnptools.distreader.DistReader.sid into a Bgen (SNP) id and rsid. -If the sid contains a single comma, we split on the comma to create the id and rsid. -Otherwise, the (SNP) id will be the sid and the rsid will be ‘0’

-
>>> default_id_rsid_function('SNP1,rs102343')
-('SNP1', 'rs102343')
->>> default_id_rsid_function('SNP1')
-('SNP1', '0')
-
-
-
- -
-
-

distreader.DistData

-
-
-class pysnptools.distreader.DistData(iid, sid, val, pos=None, name=None, copyinputs_function=None)
-

A DistReader for holding SNP distributions (or similar values) in-memory, along with related iid, sid, and pos information. -It is usually created by calling the DistReader.read() method on another DistReader, for example, Bgen. -It can also be constructed directly.

-

See DistReader for details and examples.

-
-
Constructor:
-
Parameters:
-
    -

  • iid (an array of string pair) – The DistReader.iid information.

    • -

  • sid (an array of strings) – The DistReader.sid information.

    • -

  • val (a 3-D array of floats) – The SNP value distribution

    • -

  • pos (optional, an array of strings) – The DistReader.pos information

    • -

  • name (optional, string) – Information to be display about the origin of this data

    • -

  • copyinputs_function (optional, function) – Used internally by optional clustering code

    • -
-
-
Example:
-

-
-
>>> from pysnptools.distreader import DistData
->>> distdata = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[.5,.5,0]]])
->>> print((distdata.val[1,1], distdata.iid_count, distdata.sid_count))
-(array([0.  , 0.75, 0.25]), 2, 3)
-
-
-
-
-

Equality:

-
-

Two DistData objects are equal if their four arrays (DistData.val, DistReader.iid, DistReader.sid, and DistReader.pos) -are ‘array_equal’. (Their ‘name’ does not need to be the same). -If either DistData.val contains NaN, the objects will not be equal. However, DistData.allclose() can be used to treat NaN as -regular values.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.distreader import DistData
->>> snpdata1 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[.5,.5,0]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata2 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[.5,.5,0]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata1 == snpdata2) #True, because all the arrays have the same values.
-True
->>> print(snpdata1.val is snpdata2.val) #False, because the two arrays have different memory.
-False
->>> snpdata3 = DistData(iid=[['a','0'],['b','0']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[.5,.5,0]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata4 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[.5,.5,0]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata3 == snpdata4) #False, because the iids are different.
-False
->>> snpdata5 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[np.nan,np.nan,np.nan]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[np.nan,np.nan,np.nan]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5 == snpdata6) #False, because the val's contain NaN
-False
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
-
-
-
-

Methods beyond DistReader

-
-
-allclose(value, equal_nan=True)
-
-
Parameters:
-
    -

  • value (DistData) – Other object with which to compare.

    • -

  • equal_nan (bool) – (Default: True) Tells if NaN in DistData.val should be treated as regular values when testing equality.

    • -
-
-
-
>>> import numpy as np
->>> snpdata5 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[np.nan,np.nan,np.nan]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> snpdata6 = DistData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],
-...                     val=[[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                          [[0,1.,0],[0,.75,.25],[np.nan,np.nan,np.nan]]],
-...                     pos=[[0,0,0],[0,0,0],[0,0,0]])
->>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
->>> print(snpdata5.allclose(snpdata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
-False
-
-
-
- -
-
-property val
-

The 3D NumPy array of floats that represents the distribution of SNP values. You can get or set this property.

-
>>> from pysnptools.distreader import Bgen
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> distdata = Bgen(bgen_file)[:5,:].read() #read data for first 5 iids
->>> print(distdata.val[4,55]) #print one of the SNP values
-[0.23137255 0.65342184 0.11520562]
-
-
-
- -
- -
-
-

distreader.DistHdf5

-
-
-class pysnptools.distreader.DistHdf5(*args, **kwargs)
-

A DistReader for reading *.dist.hdf5 files from disk.

-

See DistReader for general examples of using DistReaders.

-

The general HDF5 format is described here. The DistHdf5 format stores -val, iid, sid, and pos information in Hdf5 format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The DistHdf5 file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.distreader import DistHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file("pysnptools/examples/toydata.snpmajor.dist.hdf5")
->>> data_on_disk = DistHdf5(hdf5_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(25, 10000)
-
-
-
-
-

Methods beyond DistReader

-
-
-static write(filename, distdata, hdf5_dtype=None, sid_major=True)
-

Writes a DistData to DistHdf5 format and return a the DistHdf5.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • distdata (DistData) – The in-memory data that should be written to disk.

    • -

  • hdf5_dtype (string) – None (use the .val’s dtype) or a Hdf5 dtype, e.g. ‘f8’,’f4’,etc.

    • -

  • sid_major – Tells if vals should be stored on disk in sid_major (default) or iid_major format.

    • -
-
-
Return type:
-

DistHdf5

-
-
-
>>> from pysnptools.distreader import DistHdf5, Bgen
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> distdata = Bgen(bgen_file)[:,:10].read()     # Read first 10 snps from BGEN format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dist.hdf5")
->>> DistHdf5.write("tempdir/toydata10.dist.hdf5",distdata)        # Write data in DistHdf5 format
-DistHdf5('tempdir/toydata10.dist.hdf5')
-
-
-
- -
- -
-
-

distreader.DistNpz

-
-
-class pysnptools.distreader.DistNpz(*args, **kwargs)
-

A DistReader for reading *.dist.npz files from disk.

-

See DistReader for general examples of using DistReaders.

-

The general NPZ format is described here. The DistNpz format stores -val, iid, sid, and pos information in NPZ format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The DistNpz file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.distreader import DistNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> npz_file = example_file("pysnptools/examples/toydata10.dist.npz")
->>> data_on_disk = DistNpz(npz_file)
->>> print((data_on_disk.iid_count, data_on_disk.sid_count))
-(25, 10)
-
-
-
-
-

Methods beyond DistReader

-
-
-static write(filename, distdata)
-

Writes a DistData to DistNpz format and returns the DistNpz

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • distdata (DistData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

DistNpz

-
-
-
>>> from pysnptools.distreader import DistNpz, DistHdf5
->>> import pysnptools.util as pstutil
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file("pysnptools/examples/toydata.iidmajor.dist.hdf5")
->>> distdata = DistHdf5(hdf5_file)[:,:10].read()     # Read first 10 snps from DistHdf5 format
->>> pstutil.create_directory_if_necessary("tempdir/toydata10.dist.npz")
->>> DistNpz.write("tempdir/toydata10.dist.npz",distdata)          # Write data in DistNpz format
-DistNpz('tempdir/toydata10.dist.npz')
-
-
-
- -
- -
-
-

distreader.DistMemMap

-
-
-class pysnptools.distreader.DistMemMap(*args, **kwargs)
-

A DistData that keeps its data in a memory-mapped file. This allows data large than fits in main memory.

-

See DistData for general examples of using DistData.

-
-
Constructor:
-
Parameters:
-

filename (string) – The *.dist.memmap file to read.

-
-
-

Also see DistMemMap.empty() and DistMemMap.write().

-
-
Example:
-

-
-
>>> from pysnptools.distreader import DistMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file("pysnptools/examples/tiny.dist.memmap")
->>> dist_mem_map = DistMemMap(mem_map_file)
->>> print(dist_mem_map.val[0,1], dist_mem_map.iid_count, dist_mem_map.sid_count)
-[0.43403135 0.28289911 0.28306954] 25 10
-
-
-
-
-

Methods inherited from DistData

-
-
-

Methods beyond DistReader

-
-
-static empty(iid, sid, filename, pos=None, order='F', dtype=<class 'numpy.float64'>)
-

Create an empty DistMemMap on disk.

-
-
Parameters:
-
    -

  • iid (an array of string pairs) – The DistReader.iid information

    • -

  • sid (an array of strings) – The DistReader.sid information

    • -

  • filename (string) – name of memory-mapped file to create

    • -

  • pos (an array of numeric triples) – optional – The additional DistReader.pos information associated with each sid. Default: None

    • -

  • order (string or None) – {‘F’ (default), ‘C’}, optional – Specify the order of the ndarray.

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the DistMemMap.val ndarray.

    • -
-
-
Return type:
-

DistMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.distreader import DistMemMap
->>> filename = "tempdir/tiny.dist.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> dist_mem_map = DistMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> dist_mem_map.val[:,:,:] = [[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                            [[0,1.,0],[0,.75,.25],[.5,.5,0]]]
->>> dist_mem_map.flush()
-
-
-
- -
-
-property filename
-

The name of the memory-mapped file

-
- -
-
-flush()
-

Flush DistMemMap.val to disk and close the file. (If values or properties are accessed again, the file will be reopened.)

-
>>> import pysnptools.util as pstutil
->>> from pysnptools.distreader import DistMemMap
->>> filename = "tempdir/tiny.dist.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> dist_mem_map = DistMemMap.empty(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'],filename=filename,order="F",dtype=np.float64)
->>> dist_mem_map.val[:,:,:] = [[[.5,.5,0],[0,0,1],[.5,.5,0]],
-...                            [[0,1.,0],[0,.75,.25],[.5,.5,0]]]
->>> dist_mem_map.flush()
-
-
-
- -
-
-property offset
-

The byte position in the file where the memory-mapped values start.

-

(The disk space before this is used to store DistReader.iid, etc. information. -This property is useful when interfacing with, for example, external Fortran and C matrix libraries.)

-
- -
-
-property val
-

The 3D NumPy memmap array of floats that represents the distribution of SNP values. You can get this property, but cannot set it (except with itself)

-
>>> from pysnptools.distreader import DistMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> mem_map_file = example_file("pysnptools/examples/tiny.dist.memmap")
->>> dist_mem_map = DistMemMap(mem_map_file)
->>> print(dist_mem_map.val[0,1])
-[0.43403135 0.28289911 0.28306954]
-
-
-
- -
-
-static write(filename, distreader, order='A', dtype=None, block_size=None, num_threads=None)
-

Writes a DistReader to DistMemMap format.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • distreader (DistReader) – The data that should be written to disk. It can also be any distreader, for example, DistNpz, DistData, or -another Bgen.

    • -

  • order (string or None) – {‘A’ (default), ‘F’, ‘C’}, optional – Specify the order of the ndarray. By default, will match the order of the input if knowable; otherwise, ‘F’

    • -

  • dtype (data-type) – {None (default), numpy.float64, numpy.float32}, optional – The data-type for the DistMemMap.val ndarray. -By default, will match the order of the input if knowable; otherwise np.float64.

    • -

  • block_size (number) – The number of SNPs to read in a batch from distreader. Defaults to a block_size such that block_size * iid_count is about 100,000.

    • -

  • num_threads (None or int) – optional – The number of threads with which to write data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

DistMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.distreader import Bgen, DistMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>> bgen_file = example_file("pysnptools/examples/2500x100.bgen")
->>> distreader = Bgen(bgen_file)[:,:10] #Create a reader for the first 10 SNPs
->>> pstutil.create_directory_if_necessary("tempdir/tiny.dist.memmap")
->>> DistMemMap.write("tempdir/tiny.dist.memmap",distreader)      # Write distreader in DistMemMap format
-DistMemMap('tempdir/tiny.dist.memmap')
-
-
-
- -
- -
-
-

distreader.DistGen

-
-
-class pysnptools.distreader.DistGen(seed, iid_count, sid_count, chrom_count=22, cache_file=None, block_size=None)
-

A DistReader that generates – on the fly – deterministic “random” distributions over SNP data..

-

See DistReader for general examples of using DistReader.

-
-

Tip: The ‘gen’ in DistGen stands for generate, because it generates random (genetic) data. The ‘gen’ in Bgen stands for ‘genetic’. *

-
-
-
Constructor:
-
Parameters:
-
    -

  • seed (number) – The random seed that (along with block_size) determines the SNP values.

    • -
-
-
Parameters:
-
    -

  • iid_count (number) – the number of iids (number of individuals)

    • -
-
-
Parameters:
-
    -

  • sid_count (number) – the number of sids (number of SNPs)

    • -
-
-
Parameters:
-
    -

  • chrom_count (number) – the number of chromosomes to generate. Default is 22. Must be 22 or less.

    • -
-
-
Parameters:
-
    -

  • cache_file (string) – (default None) If provided, tells where to cache the common iid, sid, and pos information. Using it can save time.

    • -
-
-
Parameters:
-
    -

  • block_size (number or None) – (default None) Tells how many SNP to generate at once. The default picks a block_size such that iid_count * *block_size is about 100,000.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.distreader import DistGen
->>> #Prepare to generate data for 1000 individuals and 1,000,000 SNPs
->>> dist_gen = DistGen(seed=332,iid_count=1000,sid_count=1000*1000)
->>> print(dist_gen.iid_count,dist_gen.sid_count)
-1000 1000000
->>> dist_data = dist_gen[:,200*1000:201*1000].read() #Generate data for all users and for SNPs 200K to 201K
->>> print(dist_data.val[2,2], dist_data.iid_count, dist_data.sid_count)
-[0.2760024  0.19624834 0.52774925] 1000 1000
-
-
-
-
-
- -
-
-
-

pstreader Module

-

Tools for reading and manipulating matrix data along with row and column ids and properties.

-
-

pstreader.PstReader

-
-
-class pysnptools.pstreader.PstReader(*args, **kwargs)
-

A PstReader is one of three things:

-
    -

  • A PstData class that holds matrix data in memory:

    -
    >>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> type(data1.val).__name__ # The val property is an ndarray of values
-'ndarray'
->>> data1.row_count #prints the number of rows in this in-memory data
-3
-
    -
    -
    • -

  • A class such as PstNpz for you to specify data in file. For example,

    -
    >>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file)
->>> print(on_disk) # prints the name of the file reader
-PstNpz(...tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk.col_count # prints the number of columns (but doesn't read any matrix values)
-1015
-
    -
    -
    • -

  • A subset of any PstReader, specified with “[ row_index , col_index ]”, to read just some values.

    -
    >>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file)
->>> subset_on_disk = on_disk[[3,4],::2] # specification for a subset of the data on disk. No values are read yet.
->>> print(subset_on_disk.col_count) # prints the number of columns in this subset (but still doesn't read any values)
-508
->>> print(subset_on_disk) #prints a specification of 'subset_on_disk'
-PstNpz(...tests/datasets/all_chr.maf0.001.N300.pst.npz')[[3,4],::2]
->>> data_subset = subset_on_disk.read() # efficiently (if practical) reads the specified subset of values from the disk
->>> print(data_subset) # prints the specification of the in-memory information
-PstData(PstNpz(...tests/datasets/all_chr.maf0.001.N300.pst.npz')[[3,4],::2])
->>> int(data_subset.val.shape[0]),int(data_subset.val.shape[1]) # The dimensions of the ndarray of values
-(2, 508)
-
    -
    -
    • -
-

The PstReaders Classes

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

Format

Random Access

Suffixes

PstData

in-memory floats

Yes

n/a

PstNpz

binary, floats

No

.pst.npz,.snp.npz, -.kernel.npz

PstHdf5

binary, floats

Yes

.pst.hdf5,.snp.hdf5, -.kernel.hdf5

PstMemMap

mem-mapped floats

Yes

.pst.memmap,.snp.memmap

various SnpReader

varies

varies

varies

various KernelReader

varies

varies

varies

-
-

A SnpReader and KernelReader are each a kind of PstReader. They have some restrictions summarized here: -================================== =============== ============ ============ ==================== ==================== -Class val type row type col type row_property type col_property type -PstReader float any any any any -SnpReader float str,str str none float,float,float -KernelReader float str,str str,str none none -================================== =============== ============ ============ ==================== ====================

-

For convenience, they allow additional ways to access rows and columns.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Class

val name

row name

col name

row_property name

col_property name

PstReader

val

row

col

row_property

col_property

SnpReader

val

iid,row

sid,col

none

col_property,pos

KernelReader

val

iid0,row,iid

iid1,col,iid

none

none

-
-
Note:
-

A KernelReader.iid may be used when KernelReader.iid0 is equal to KernelReader.iid1

-
-
-
-
-

Methods & Properties:

-
-

Every PstReader, such as PstNpz and PstData, has these properties: row, row_count, col, col_count, -row_property, col_property and these methods: read(), row_to_index(), col_to_index(). See below for details.

-

PstData is a PstReader so it supports the above properties and methods. In addition, it supports property PstData.val and equality testing. -See below for details.

-
-

Rows and Cols:

-
-

Example:

-
>>> from pysnptools.pstreader import PstHdf5
->>> from pysnptools.util import print2 #print bytes strings and Unicode strings the same
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file('pysnptools/examples/toydata.iidmajor.snp.hdf5')
->>> on_disk = PstHdf5(hdf5_file) # PstHdf5 can load .pst.hdf5, .snp.hdf5, and kernel.hdf5
->>> print2(on_disk.row[:3]) # print the first three rows
-[['per0' 'per0']
- ['per1' 'per1']
- ['per2' 'per2']]
->>> print2(on_disk.col[:7]) # print the first seven columns
-['null_0' 'null_1' 'null_2' 'null_3' 'null_4' 'null_5' 'null_6']
->>> print2(on_disk.row_to_index([[b'per2', b'per2'],[b'per1', b'per1']])) #Find the indexes for two rows.
-[2 1]
-
-
-
-

When Matrix Data is Read:

-
-

Matrix data can be enormous so we generally avoid reading it to the degree practical. Specifically,

-
    -

  • Constructing and printing a PstReader causes no file reading. For example, these commands read no data:

    -
    >>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> print(on_disk) # Print the PstHdf5 PstReader specification. No data is read.
-PstHdf5(...pysnptools/examples/toydata.iidmajor.snp.hdf5')
->>> subset_on_disk = on_disk[[3,4],::2] # Construct a subsetting PstReader. No data is read.
->>> print(subset_on_disk) # print the subset PstReader. No data is read.
-PstHdf5(...pysnptools/examples/toydata.iidmajor.snp.hdf5')[[3,4],::2]
-
    -
    -
    • -

  • Properties and methods related to the rows and columns (to the degree practical) read only row and col data from the disk, -not value data. Moreover, the row and col data is read from file only once. Consider these commands:

    -
    >>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> print2(on_disk.col[:7]) # without reading any values data from disk, read the row and col data from disk, cache it, and then print the first seven cols.
-['null_0' 'null_1' 'null_2' 'null_3' 'null_4' 'null_5' 'null_6']
->>> print(on_disk.col_to_index([b'null_7',b'null_2'])) #use the cached col information to find the indexes of 'null_7' and 'null_2'. (No data is read from disk.)
-[7 2]
-
    -
    -
    • -

  • The only method that reads values from file (to the degree practical) is read(). For example:

    -
    >>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> data1 = on_disk.read() #read all the values from disk, creating a new PstData instance that keeps these values in memory
->>> print(data1.val[0,2]) # print the value for the row with index 0 and the col with index 2. (No data is read from disk.)
-2.0
-
    -
    -
    • -

  • If you request the values for only a subset of the rows or columns, (to the degree practical) only that subset will be read from disk. -for example:

    -
    >>> hdf5_file2 = example_file('tests/datasets/all_chr.maf0.001.N300.hdf5')
->>> on_disk = PstHdf5(hdf5_file2)[[3,4],::2] # Construct a subsetting PstReader. No data is read.
->>> data_subset = subset_on_disk.read() # from disk, read the values for the rows with index 3 and 4 AND cols with even numbered indexes.
->>> print(data_subset.val[0,2]) # print the value with subset row index 0 and col index 2 (corresponding to row index 3 and col index 4 in the full data). No data is read from disk.
-1.0
-
    -
    -
    • -
-
-

When Matrix Data is Re-Read and Copied:

-
-

Every time you call a PstReader’s read() method, the PstReader re-reads the value data and returns a new in-memory PstData -(with PstData.val property containing a new ndarray of the values).

-

Here is an example of what not to do, because it causes all the matrix value data to be read twice.

-
>>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> # Not recommended because it inefficiently reads all the values twice.
->>> print(on_disk.read().val[0,2]) # read all values into a new PstData, print a value
-2.0
->>> print(on_disk.read().val[0,3]) # read all values (again) into a second new PstData, print a value
-2.0
-
-
-

Here are two efficient alternatives. First, if all values can all fit in memory, read them once into a PstData and then -access that PstData multiple times.

-
>>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> data1 = on_disk.read() # read all values into a new PstData
->>> print(data1.val[0,2]) # print a value from data1's in-memory ndarray
-2.0
->>> print(data1.val[0,3]) # print another value from data1's in-memory ndarray.
-2.0
-
-
-

Second, if the value data is too large to fit in memory, use subsetting to read only the values of interest from disk.

-
>>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> print(on_disk[0,2].read().val[0,0]) #Define the subset of data and read only that subset from disk.
-2.0
->>> print(on_disk[0,3].read().val[0,0]) #Define a second subset of data and read only that subset from disk.
-2.0
-
-
-

Because the in-memory PstData class is a kind of PstReader, you may read from it, too. -Doing so create a new PstData instance containing a copy of the matrix values in a new ndarray.

-
>>> on_disk = PstHdf5(hdf5_file) # Construct a PstHdf5 PstReader. No data is read.
->>> data1 = on_disk.read() # read all matrix values from disk into a new PstData
->>> print(data1.val is data1.val) # Do the in-memory SNP values use the same memory as themselves? Yes
-True
->>> data2 = data1.read() # copy all the matrix values into a new ndarray in a new PstData
->>> print(data2.val is data1.val) # Do the two ndarrays of in-memory matrix values use the same memory?
-False
-
-
-
-

Avoiding Unwanted ndarray Allocations

-
-

You may want a subset of matrix values from an in-memory PstData and you may know that this subset and the original PstData -can safely share the memory of the ndarray of matrix values. For this case, the read() has optional parameters called view_ok and order. If you override -the defaults of “view_ok=False,order=’F’” with “view_ok=True,order=’A’, the read() will, if practical, return a new -PstData with a ndarray shares memory with the original ndarray. -Use these parameters with care because any change to either ndarray will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

-
>>> on_disk = PstNpz(pstnpz_file) # Construct a PstNpz PstReader. No data is read.
->>> data1 = on_disk.read() # read all data from disk into a PstData with a new ndarray
->>> column01 = data1[:,0:1].read(view_ok=True,order='A') #create PstData with the data from just the first two columns. Sharing memory is OK. The memory may be laid out in any order (that is col-major and row-major are both OK).
->>> import numpy as np
->>> #print(np.may_share_memory(data1.val, column01.val)) # Do the two ndarray's share memory? They could (but currently they won't)
->>> column201 = data1[:,[2,0,1]].read(view_ok=True,order='A') #create PstData with the data from three columns, permuted. Sharing memory is OK.
->>> print(np.may_share_memory(data1.val, column201.val)) # Do the two ndarray's share memory? No, ndarray decided that this indexing was too complex for sharing.
-False
-
-
-
-

Creating Subsetting PstReaders with Indexing

-
-

You often don’t want to read the matrix values for all rows and cols. You can use indexing to create a subsetting PstReader that -will read only the matrix values of interest.

-

PstReaders support the indexing formats supported by ndarray plus two generalizations. Here are examples of indexing with an array -of indexes, with slicing, and with an array of Booleans.

-
>>> on_disk = PstNpz(pstnpz_file) # Specify some data on disk in PstNpz format
->>> subset_reader_1 = on_disk[[3,-1],:] #index with an array of indexes (negatives count from end)
->>> print((subset_reader_1.row_count, subset_reader_1.col_count))
-(2, 1015)
->>> data1 = subset_reader_1.read() # read just the two rows of interest from the disk
->>> subset_reader_2 = on_disk[:,:0:-2] #index with a slice
->>> print((subset_reader_2.row_count, subset_reader_2.col_count))
-(300, 507)
->>> boolindexes = [s.startswith(b'23_') for s in on_disk.col] # create a Boolean index of cols that start '23_'
->>> subset_reader_3 = on_disk[:,boolindexes] #index with array of Booleans
->>> print((subset_reader_3.row_count, subset_reader_3.col_count))
-(300, 24)
-
-
-

The first generalization over what ndarray offers is full indexing on both the row dimension and the col dimension, in other words, -full multidimensional indexing. For example,

-
>>> on_disk = PstNpz(pstnpz_file) # Specify some data on disk in PstNpz format
->>> subset_reader_4 = on_disk[[3,4],:0:-2] # index on two dimensions at once
->>> print((subset_reader_4.row_count, subset_reader_4.col_count))
-(2, 507)
-
-
-

The second generalization is indexing on a single integer index.

-
>>> on_disk = PstNpz(pstnpz_file) # Specify some data on disk in PstNpz format
->>> subset_reader_5 = on_disk[5,:] #index with single integer
->>> print((subset_reader_5.row_count, subset_reader_5.col_count))
-(1, 1015)
-
-
-

Indexing is also useful when you have matrix values in memory via a PstData index and want to copy a subset of those values. -While you could instead index directly on the PstData.val ndarray, by indexing on the PstData instance you -also get row and col information.

-
>>> on_disk = PstNpz(pstnpz_file) # Specify some data on disk in PstNpz format
->>> data1 = on_disk.read() # read all matrix values into memory
->>> print2(data1.col[:9]) # print the first 9 cols
-['1_12' '1_34' '1_10' '1_35' '1_28' '1_25' '1_36' '1_39' '1_4']
-
-
-
>>> data_subset = data1[:,::2].read(view_ok=True,order='A') # create a copy or view with every other col
->>> print2(data_subset.col[:9]) # print the first 9 cols in the subset
-['1_12' '1_10' '1_28' '1_36' '1_4' '1_11' '1_32' '1_9' '1_17']
-
-
-

You can apply indexing on top of indexing to specify subsets of subsets of data to read. In this example, -only the column values for every 16th col is actually read from the disk.

-
>>> # These are just PstReaders, nothing is read from disk yet
->>> on_disk = PstNpz(pstnpz_file) # Specify some data on disk in PstNpz format
->>> half_reader = on_disk[:,::2] # a reader for half the cols
->>> quarter_reader = half_reader[:,::2] # a reader for half of half the cols
->>> sixteenth_reader = quarter_reader[:,::2][:,::2] # a reader for half of half of half of half the cols
->>> print(sixteenth_reader) #Print the specification of this reader
-PstNpz(...tests/datasets/all_chr.maf0.001.N300.pst.npz')[:,::2][:,::2][:,::2][:,::2]
->>> # Now we read from disk. Only values for one col in every 16 will be read.
->>> data_sixteenth = sixteenth_reader.read()
->>> print(data_sixteenth.val[0,3])
-2.0
-
-
-
-

The read() Method

-
-

By default the read() returns a ndarray of numpy.float64 laid out in memory in F-contiguous order (row-index varies the fastest). You may, instead, -ask for numpy.float32 or for C-contiguous order or any order. See read() for details.

-
-

Details of Methods & Properties:

-
-
-property col
-

A ndarray of the cols id. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.col[-1]) # print the last column id.
-z
->>> data2 = PstData(row=['a','b','c'],col=[[1,0],[2,0]],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.col[-1]) # print the last column id.
-[2 0]
-
-
-
- -
-
-property col_count
-

number of cols

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
- -
-
-property col_property
-

A ndarray of the additional information for each col. Each element is a ndarray.

-
-
Return type:
-

1- or 2-dimensional ndarray (length col_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file)
->>> print(on_disk.col_property[:3]) # print column information for the first three cols: #The '...' is an optional space
-[[1.         0.00800801 0.        ]
- [1.         0.023023   1.        ]
- [1.         0.0700701  4.        ]]
-
-
-
- -
-
-col_to_index(list)
-

Takes a list of column ds and returns a list of index numbers

-
-
Parameters:
-

list (list) – list of cols

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file) # Specify matrix data on disk
->>> print(on_disk.col_to_index([b'1_10',b'1_13'])) #Find the indexes for two cols.
-[2 9]
-
-
-
- -
-
-read(order='F', dtype=<class 'numpy.float64'>, force_python_only=False, view_ok=False, num_threads=None)
-

Reads the matrix values and returns a PstData (with PstData.val property containing a new ndarray of the matrix values).

-
-
Parameters:
-
    -

  • order (string or None) – {‘F’ (default), ‘C’, ‘A’}, optional – Specify the order of the ndarray. If order is ‘F’ (default), -then the array will be in F-contiguous order (row-index varies the fastest). -If order is ‘C’, then the returned array will be in C-contiguous order (col-index varies the fastest). -If order is ‘A’, then the PstData.val -ndarray may be in any order (either C-, Fortran-contiguous).

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the PstData.val ndarray.

    • -

  • force_python_only (bool) – optional – If False (default), may use outside library code. If True, requests that the read -be done without outside library code.

    • -

  • view_ok (bool) – optional – If False (default), allocates new memory for the PstData.val’s ndarray. If True, -if practical and reading from a PstData, will return a new -PstData with a ndarray shares memory with the original PstData. -Typically, you’ll also wish to use “order=’A’” to increase the chance that sharing will be possible. -Use these parameters with care because any change to either ndarray will effect -the others. Also keep in mind that read() relies on ndarray’s mechanisms to decide whether to actually -share memory and so it may ignore your suggestion and allocate a new ndarray anyway.

    • -

  • num_threads (None or int) – optional – The number of threads with which to read data. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

PstData

-
-
-

Calling the method again causes the matrix values to be re-read and creates a new in-memory PstData with a new ndarray of matrix values.

-

If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file('pysnptools/examples/toydata.iidmajor.snp.hdf5')
->>> on_disk = PstHdf5(hdf5_file) # Specify matrix data on disk
->>> pstdata1 = on_disk.read() # Read all the matrix data returning a PstData instance
->>> print(type(pstdata1.val).__name__) # The PstData instance contains a ndarray of the data.
-ndarray
->>> subset_pstdata = on_disk[:,::2].read() # From the disk, read matrix values for every other sid
->>> print(subset_pstdata.val[0,0]) # Print the first matrix value in the subset
-1.0
->>> subsub_pstdata = subset_pstdata[:10,:].read(order='A',view_ok=True) # Create an in-memory subset of the subset with matrix values for the first ten iids. Share memory if practical.
->>> import numpy as np
->>> # print(np.may_share_memory(subset_snpdata.val, subsub_snpdata.val)) # Do the two ndarray's share memory? They could. Currently they won't.
-
-
-
- -
-
-property row
-

A ndarray of the row ids. Each id can be anything, for example, a string, an array of two strings, a number, etc.

-
-
Return type:
-

ndarray (length row_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. -Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.row[:2]) # print the first two row ids
-['a' 'b']
->>> data2 = PstData(row=[[1,0],[2,0],[2,1]],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data2.row[:2]) # print the first two row ids
-[[1 0]
- [2 0]]
-
-
-
- -
-
-property row_count
-

number of rows

-
-
Return type:
-

integer

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
- -
-
-property row_property
-

A ndarray of the additional information for each row. Each element is a ndarray.

-
-
Return type:
-

1- or 2-dimensional ndarray (length row_count)

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> print(data1.row_property[:2]) # print the first two row property values
-['A' 'B']
-
-
-
- -
-
-row_to_index(list)
-

Takes a list of row ids and returns a list of index numbers

-
-
Parameters:
-

list – list of rows

-
-
Return type:
-

ndarray of int

-
-
-

This method (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>> pstnpz_file = example_file('tests/datasets/all_chr.maf0.001.N300.pst.npz')
->>> on_disk = PstNpz(pstnpz_file) # Specify matrix data on disk
->>> print(on_disk.row_to_index([[b'POP1',b'44'],[b'POP1',b'12']])) #Find the indexes for two rows.
-[2 1]
-
-
-
- -
-
-property shape
-

number of rows and number of cols

-
-
Return type:
-

tuple of two integers

-
-
-

This property (to the degree practical) reads only row and col data from the disk, not matrix value data. Moreover, the row and col data is read from file only once.

-
- -
- -
-
-

pstreader.PstData

-
-
-class pysnptools.pstreader.PstData(row, col, val, row_property=None, col_property=None, name=None, parent_string=None, copyinputs_function=None)
-

A PstReader for holding values in-memory, along with related row and col information. -It is usually created by calling the PstReader.read() method on another PstReader, for example, PstNpz. It can also be constructed directly.

-

See PstReader for details and examples.

-
-
Constructor:
-
Parameters:
-
    -

  • row (an array of anything) – The PstReader.row information

    • -

  • col (an array of anything) – The PstReader.col information

    • -

  • val (a 2-D array of floats or an array of floats) – The values

    • -

  • row_property (optional, an array of anything) – Additional information associated with each row.

    • -

  • col_property (optional, an array of anything) – Additional information associated with each col.

    • -

  • name (optional, string) – Information to be display about the origin of this data

    • -

  • copyinputs_function (optional, function) – Used internally by optional clustering code

    • -
-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstData
->>> pstdata = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> print(pstdata.val[0,1], pstdata.row_count, pstdata.col_count)
-2.0 2 3
-
-
-
-
-

Equality:

-
-

Two PstData objects are equal if their five arrays (PstData.val, PstReader.row, PstReader.col, PstReader.row_property, and PstReader.col_property) arrays are equal. -(Their ‘name’ does not need to be the same). -If either PstData.val contains NaN, the objects will not be equal. However, PstData.allclose() can be used to treat NaN as regular values. -Any NaN’s in the other four arrays are treated as regular values.

-
-
Example:
-

-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData
->>> pstdata1 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> pstdata2 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> print(pstdata1 == pstdata2) #True, because all the arrays have the same values.
-True
->>> print(pstdata1.val is pstdata2.val) #False, because the two arrays have different memory.
-False
->>> pstdata3 = PstData(row=['a','b'], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> pstdata4 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,2.]])
->>> print(pstdata3 == pstdata4) #False, because the rows have different ids.
-False
->>> pstdata5 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]])
->>> pstdata6 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]])
->>> print(pstdata5 == pstdata6) #False, because the val's contain NaN
-False
->>> print(pstdata5.allclose(pstdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
-
-
-
-

Methods beyond PstReader

-
-
-allclose(value, equal_nan=True)
-
-
Parameters:
-
    -

  • value (PstData) – Other object with which to compare.

    • -

  • equal_nan (bool) – (Default: True) Tells if NaN in PstData.val should be treated as regular values when testing equality.

    • -
-
-
-
>>> import numpy as np
->>> pstdata5 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]])
->>> pstdata6 = PstData(row=[['fam0','iid0'],['fam0','iid1']], col=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]])
->>> print(pstdata5.allclose(pstdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
-True
->>> print(pstdata5.allclose(pstdata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
-False
-
-
-
- -
-
-property val
-

The 2D NumPy array of floats (or array of floats) that represents the values. You can get or set this property.

-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file
->>>
->>> pstnpz_file = example_file('tests/datasets/little.pst.npz')
->>> pstdata = PstNpz(pstnpz_file)[:5,:].read() #read data for first 5 rows
->>> print(pstdata.val[4,5]) #print one of the values
-2.0
-
-
-
- -
-
-property val_shape
-

Tells the shape of value for a given individual and SNP. If None, means a single scalar value.

-
- -
- -
-
-

pstreader.PstHdf5

-
-
-class pysnptools.pstreader.PstHdf5(filename, block_size=5000)
-

A PstReader for reading *.pst.hdf5 files from disk.

-

See PstReader for general examples of using PstReaders.

-

The general HDF5 format is described in here. The PstHdf5 format stores -val, row, col, row_property, and col_property information in Hdf5 format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The PstHdf5 file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> psthdf5_file = example_file('pysnptools/examples/toydata.iidmajor.snp.hdf5')
->>> on_disk = PstHdf5(psthdf5_file) # PstHdf5 can load .pst.hdf5, .snp.hdf5, and kernel.hdf5
->>> print(on_disk.row_count)
-500
-
-
-
-
-

Methods beyond PstReader

-
-
-flush()
-

Flush the connection to the HDF5 file. (If values or properties are accessed again, the file will be reopened.)

-
>>> import pysnptools.util as pstutil
->>> from pysnptools.pstreader import PstHdf5
->>> from pysnptools.util import example_file # Download and return local file name
->>> hdf5_file = example_file("pysnptools/examples/toydata.kernel.hdf5")
->>> reader = PstHdf5(hdf5_file)
->>> val1 = reader[0,0].read()
->>> reader.flush()
-
-
-
- -
-
-static write(filename, pstdata, hdf5_dtype=None, col_major=True)
-

Writes a PstData to PstHdf5 format and returns the PstHdf5.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • pstdata (PstData) – The in-memory data that should be written to disk.

    • -

  • hdf5_dtype (string) – None (use the .val’s dtype) or a Hdf5 dtype, e.g. ‘f8’,’f4’,etc.

    • -

  • col_major (bool) – Tells if vals should be stored on disk in col_major (default) or row_major format.

    • -
-
-
Return type:
-

PstHdf5

-
-
-
>>> import numpy as np
->>> from pysnptools.pstreader import PstData, PstHdf5
->>> import pysnptools.util as pstutil
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> pstutil.create_directory_if_necessary("tempdir/tiny.pst.hdf5")
->>> PstHdf5.write("tempdir/tiny.pst.hdf5",data1)          # Write data in PstHdf5 format
-PstHdf5('tempdir/tiny.pst.hdf5')
-
-
-
- -
- -
-
-

pstreader.PstNpz

-
-
-class pysnptools.pstreader.PstNpz(filename)
-

A PstReader for reading *.pst.npz files from disk.

-

See PstReader for general examples of using PstReaders.

-

The general NPZ format is described here. The PstNpz format stores -val, row, col, row_property, and col_property information in NPZ format.

-
-
Constructor:
-
Parameters:
-
    -

  • filename (string) – The PstNpz file to read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstNpz
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> pstnpz_file = example_file('tests/datasets/little.pst.npz')
->>> data_on_disk = PstNpz(pstnpz_file)
->>> print(data_on_disk.row_count)
-300
-
-
-
-
-

Methods beyond NpzReader

-
-
-static write(filename, pstdata)
-

Writes a PstData to PstNpz format and returns the PstNpz.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • pstdata (PstData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

PstNpz

-
-
-
>>> from pysnptools.pstreader import PstData, PstNpz
->>> import pysnptools.util as pstutil
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> pstutil.create_directory_if_necessary("tempdir/tiny.pst.npz")
->>> PstNpz.write("tempdir/tiny.pst.npz",data1)          # Write data in PstNz format
-PstNpz('tempdir/tiny.pst.npz')
-
-
-
- -
- -
-
-

pstreader.PstMemMap

-
-
-class pysnptools.pstreader.PstMemMap(filename)
-

A PstData that keeps its data in a memory-mapped file. This allows data large than fits in main memory.

-

See PstData for general examples of using PstData.

-
-
Constructor:
-
Parameters:
-

filename (string) – The *.pst.memmap file to read.

-
-
-

Also see PstMemMap.empty() and PstMemMap.write().

-
-
Example:
-

-
-
>>> from pysnptools.pstreader import PstMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> pst_mem_map_file = example_file('pysnptools/examples/tiny.pst.memmap')
->>> pst_mem_map = PstMemMap(pst_mem_map_file)
->>> print(pst_mem_map.val[0,1], pst_mem_map.row_count, pst_mem_map.col_count)
-2.0 3 2
-
-
-
-
-

Methods beyond PstReader

-
-
-static empty(row, col, filename, row_property=None, col_property=None, order='F', dtype=<class 'numpy.float64'>, val_shape=None)
-

Create an empty PstMemMap on disk.

-
-
Parameters:
-
    -

  • row (an array of anything) – The PstReader.row information

    • -

  • col (an array of anything) – The PstReader.col information

    • -

  • filename (string) – name of memory-mapped file to create

    • -

  • row_property (an array of anything) – optional – The additional PstReader.row_property information associated with each row. Default: None

    • -

  • col_property (an array of anything) – optional – The additional PstReader.col_property information associated with each col. Default: None

    • -

  • order (string or None) – {‘F’ (default), ‘C’}, optional – Specify the order of the ndarray.

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for the PstMemMap.val ndarray.

    • -

  • val_shape (None or a number) – (Default: None), optional – The shape of the last dimension of PstMemMap.val. None means each value is a scalar.

    • -
-
-
Return type:
-

PstMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.pstreader import PstMemMap
->>> filename = "tempdir/tiny.pst.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> pst_mem_map = PstMemMap.empty(row=['a','b','c'],col=['y','z'],filename=filename,row_property=['A','B','C'],order="F",dtype=np.float64)
->>> pst_mem_map.val[:,:] = [[1,2],[3,4],[np.nan,6]]
->>> pst_mem_map.flush()
-
-
-
- -
-
-property filename
-

The name of the memory-mapped file

-
- -
-
-flush()
-

Flush PstMemMap.val to disk and close the file. (If values or properties are accessed again, the file will be reopened.)

-
>>> import pysnptools.util as pstutil
->>> from pysnptools.pstreader import PstMemMap
->>> filename = "tempdir/tiny.pst.memmap"
->>> pstutil.create_directory_if_necessary(filename)
->>> pst_mem_map = PstMemMap.empty(row=['a','b','c'],col=['y','z'],filename=filename,row_property=['A','B','C'],order="F",dtype=np.float64)
->>> pst_mem_map.val[:,:] = [[1,2],[3,4],[np.nan,6]]
->>> pst_mem_map.flush()
-
-
-
- -
-
-property offset
-

The byte position in the file where the memory-mapped values start.

-
- -
-
-property val
-

The NumPy memmap array of floats that represents the values. You can get this property, but cannot set it (except with itself)

-
>>> from pysnptools.pstreader import PstMemMap
->>> from pysnptools.util import example_file # Download and return local file name
->>>
->>> pst_mem_map_file = example_file('pysnptools/examples/tiny.pst.memmap')
->>> pst_mem_map = PstMemMap(pst_mem_map_file)
->>> print(pst_mem_map.val[0,1])
-2.0
-
-
-
- -
-
-static write(filename, pstdata)
-

Writes a PstData to PstMemMap format and returns the PstMemMap.

-
-
Parameters:
-
    -

  • filename (string) – the name of the file to create

    • -

  • pstdata (PstData) – The in-memory data that should be written to disk.

    • -
-
-
Return type:
-

PstMemMap

-
-
-
>>> import pysnptools.util as pstutil
->>> from pysnptools.pstreader import PstData, PstMemMap
->>> data1 = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
->>> pstutil.create_directory_if_necessary("tempdir/tiny.pst.memmap")
->>> PstMemMap.write("tempdir/tiny.pst.memmap",data1)      # Write data1 in PstMemMap format
-PstMemMap('tempdir/tiny.pst.memmap')
-
-
-
- -
- -
-
-
-

util Module

-
-

util

-
-
-pysnptools.util.array_module(xp=None)
-

Find the array module to use, for example numpy or cupy.

-
-
Parameters:
-

xp (optional, string or Python module) – The array module to use, for example, ‘numpy’ -(normal CPU-based module) or ‘cupy’ (GPU-based module). -If not given, will try to read -from the ARRAY_MODULE environment variable. If not given and -ARRAY_MODULE is not set, -will use numpy. If ‘cupy’ is requested, will -try to ‘import cupy’. If that import fails, will -revert to numpy.

-
-
Return type:
-

Python module

-
-
-
>>> from pysnptools.util import array_module
->>> xp = array_module() # will look at environment variable
->>> print(xp.zeros((3)))
-[0. 0. 0.]
->>> xp = array_module('cupy') # will try to import 'cupy'
->>> print(xp.zeros((3)))
-[0. 0. 0.]
-
-
-
- -
-
-pysnptools.util.asnumpy(a)
-

Given an array created with any array module, return the equivalent -numpy array. (Returns a numpy array unchanged.)

-
>>> from pysnptools.util import asnumpy, array_module
->>> xp = array_module('cupy')
->>> zeros_xp = xp.zeros((3)) # will be cupy if available
->>> zeros_np = asnumpy(zeros_xp) # will be numpy
->>> zeros_np
-array([0., 0., 0.])
-
-
-
- -
-
-pysnptools.util.create_directory_if_necessary(name, isfile=True, robust=False)
-

Create a directory for a file if the directory doesn’t already exist.

-
-
Parameters:
-
    -

  • name (string) – file or directory name

    • -

  • isfile (bool) – If True (default), the name is a file, otherwise it is a directory.

    • -

  • robust (bool) – If False (default), will try once to create the directory. If True, will try 25 times.

    • -
-
-
-
- -
-
-pysnptools.util.format_delta(delta_seconds)
-

Format a time delta nicely.

-
-
Parameters:
-

delta_seconds (number) – The number of seconds

-
-
Return type:
-

string

-
-
-
>>> from pysnptools.util import format_delta
->>> print(format_delta(86403.5))
-1 day, 0:00:03.500000
-
-
-
- -
-
-pysnptools.util.get_array_module(a)
-

Given an array, returns the array’s -module, for example, numpy or cupy. -Works for numpy even when cupy is not available.

-
>>> import numpy as np
->>> zeros_np = np.zeros((3))
->>> xp = get_array_module(zeros_np)
->>> xp.ones((3))
-array([1., 1., 1.])
-
-
-
- -
-
-pysnptools.util.intersect_apply(data_list, sort_by_dataset=True, intersect_before_standardize=True, is_test=False)
-

Intersects and sorts the iids from a list of datasets, returning new -version of the datasets with all the same iids in the same order.

-
-
Parameters:
-
    -

  • data_list (list) – list of datasets

    • -

  • sort_by_dataset (bool) – optional, If True (default), the iids are ordered -according to the first non-None dataset. -If False, the order is arbitrary, but consistent.

    • -

  • intersect_before_standardize (bool) – optional. Special code for -SnpKernel, the class that postpones computing a kernel from SNP data. -If True (default), SnpKernel will remove any iids before SNP -standardization before computing the kernel. -If False, SNPs will be standardized with all iids, then the kernel -will be computed, then any iids will be removed.

    • -
-
-
Return type:
-

list of datasets

-
-
-

Here are the dataset formats understood and what is returned for each.

- - - - - - - - - - - - - - - - - - - - - - - -

Dataset Format

What is Returned

None

None

A SnpReader

A new subsetting SnpReader with adjusted iid

A KernelReader

A new subsetting KernelReader with adjusted iid

A dictionary with [‘iid’] and [‘vals’] keys

The same dictionary but with the iid and vals values adjusted

Tuple of the form (val ndarray, iid list)

A new tuple with the val ndarray and iid list adjusted

-

If the iids in all the datasets are already the same and in the same order, then the datasets are returned without change.

-

Notice that only dictionaries are processed in-place. Inputting a SnpReader and KernelReader returns a new class of the same type (unless its iids -are already ok). Inputting a tuple returns a new tuple (unless its iids are already ok).

-
-
Example:
-

-
-
>>> from pysnptools.snpreader import Bed, Pheno
->>> from pysnptools.kernelreader import SnpKernel
->>> from pysnptools.standardizer import Unit
->>> #Create five datasets in different formats
->>> ignore_in = None
->>> from pysnptools.util import example_file # Download and return local file name
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> phenofile = example_file("tests/datasets/phenSynthFrom22.23.N300.randcidorder.txt")
->>> covfile = example_file("tests/datasets/all_chr.maf0.001.covariates.N300.txt")
->>> kernel_in = SnpKernel(Bed(bedfile,count_A1=False),Unit()) # Create a kernel from a Bed file
->>> pheno_in = Pheno(phenofile,missing="")
->>> cov = Pheno(covfile,missing="").read()
->>> cov_as_tuple_in = (cov.val,cov.iid) #We could do cov directly, but as an example we make it a tuple.
->>>
->>> # Create five new datasets with consistent iids
->>> ignore_out, kernel_out, pheno_out, cov_as_tuple_out = intersect_apply([ignore_in, kernel_in, pheno_in, cov_as_tuple_in])
->>> # Print the first five iids from each dataset
->>> print(ignore_out, kernel_out.iid[:5], pheno_out.iid[:5], cov_as_tuple_out[1][:5])
-None [['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']
- ['POP1' '58']
- ['POP1' '65']] [['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']
- ['POP1' '58']
- ['POP1' '65']] [['POP1' '0']
- ['POP1' '12']
- ['POP1' '44']
- ['POP1' '58']
- ['POP1' '65']]
-
-
-
- -
-
-pysnptools.util.intersect_ids(idslist)
-
-

Deprecated since version Use: intersect_apply() instead.

-
-

Takes a list of 2d string arrays of family and individual ids. -These are intersected.

-
-
Return type:
-

indarr, an array of size N x L, where N is the number of -individuals in the intersection, and L is the number of lists in idslist, and which -contains the index to use (in order) such that all people will be identical and in order -across all data sets.

-
-
-

If one of the lists=None, it is ignored (but still has values reported in indarr, all equal to -1),

-
- -
-
-pysnptools.util.log_in_place(name, level, time_lambda=<built-in function time>, show_log_diffs=False)
-

Create an one-argument function to write messages to. If the logging level -is met, messages will appear. All messages will be on the same line and -will include time.

-
-
Example:
-

-
-
from pysnptools.util import log_in_place
-import logging
-import time
-logging.basicConfig(level=logging.INFO)
-
-with log_in_place("counting", logging.INFO) as updater:
-for i in range(100):
-        updater(i)
-        time.sleep(.1) #typically, some work -- not a delay -- goes here
-
-
-

Outputs 100 messages on the same line, ending with -something like “counting – time=0:00:09.99, 99”

-
- -
-
-pysnptools.util.print2(arg)
-

Make printing under Python3 look the same as under Python2.

-
- -
-
-pysnptools.util.sub_matrix(val, row_index_list, col_index_list, order='A', dtype=<class 'numpy.float64'>, num_threads=None)
-

Efficiently creates a sub-matrix from a 2-D or 3-D ndarray.

-
-
Parameters:
-
    -

  • val (ndarray) – The ndarray from which to copy.

    • -

  • row_index_list (list of integers) – Tells which rows (and in which order) to copy into the sub-matrix

    • -

  • col_index_list (list of integers) – Tells which columns (and in which order) to copy into the sub-matrix

    • -

  • order (string or None) – {‘A’ (default), ‘C’, ‘F’}, optional – Specify the order of the sub-matrix to create. -If order is ‘C’, then the returned sub-matrix will be in C-contiguous order (first index varies the fastest). -If order is ‘F’, then the array will be in F-contiguous order (second index varies the fastest). -If order is ‘A’, then sub-matrix may be in any order F or C.

    • -

  • dtype (data-type) – {numpy.float64 (default), numpy.float32}, optional – The data-type for sub-matrix created.

    • -

  • num_threads (None or int) – optional – The number of threads with which to work. Defaults to all available -processors. Can also be set with these environment variables (listed in priority order): -‘PST_NUM_THREADS’, ‘NUM_THREADS’, ‘MKL_NUM_THREADS’.

    • -
-
-
Return type:
-

ndarray

-
-
-
>>> import numpy as np
->>> import pysnptools.util as pstutil
->>> np.random.seed(0) # set seed so that results are deterministic
->>> matrix = np.random.rand(12,7) # create a 12 x 7 ndarray
->>> submatrix = pstutil.sub_matrix(matrix,[0,2,11],[6,5,4,3,2,1,0])
->>> print(int(submatrix.shape[0]),int(submatrix.shape[1]))
-3 7
->>> print(matrix[2,0] == submatrix[1,6]) #The row # 2 is now #1, the column #0 is now #6.
-True
-
-
-

Note: Behind the scenes, for performance, this function calls a Rust helper function.

-
- -
-
-pysnptools.util.weighted_mean(ys, weights)
-
-
Parameters:
-
    -

  • ys (ndarray) – The ndarray of values

    • -

  • weights (ndarray) – the weight of each value (unnormalized is fine)

    • -
-
-
Return type:
-

the weight mean

-
-
-
>>> ys = np.array([103.664086,89.80645161,83.86888046,90.54141176])
->>> weights = np.array([2.340862423,4.982888433,0.17522245,0.098562628])
->>> float(round(weighted_mean(ys, weights),5))
-93.9487
-
-
-
- -
-
-pysnptools.util.weighted_simple_linear_regression(xs, ys, weights)
-
-
Parameters:
-
    -

  • xs (ndarray) – The ndarray of independent values

    • -

  • ys (ndarray) – The ndarray of dependent values

    • -

  • weights (ndarray) – the weight of each case (unnormalized is fine)

    • -
-
-
Return type:
-

slope, intercept, xmean, ymean

-
-
-
>>> xs = np.array([53.8329911,57.49486653,60.07392197,60.21081451])
->>> ys = np.array([103.664086,89.80645161,83.86888046,90.54141176])
->>> weights = np.array([2.340862423,4.982888433,0.17522245,0.098562628])
->>> slope, intercept, xmean, ymean = weighted_simple_linear_regression(xs, ys, weights)
->>> print(round(slope,5), round(intercept,5), round(xmean,5), round(ymean,5))
--3.52643 293.05586 56.46133 93.9487
-
-
-
- -
-
-pysnptools.util.example_file(pattern, endswith=None)
-

Returns the local location of a PySnpTools example file, downloading it -if needed.

-
-
Parameters:
-
    -

  • pattern – The name of the example file of interest. A -file name pattern -may be given. All matching files will be downloaded (if needed) and -the name of one will be returned.

    • -

  • endswith – The pattern of the file name to return. By default, when -no endswith is given, the name of the first matching file will be -returned.

    • -
-
-
Return type:
-

string

-
-
-

By default, the local location will be under the system temp directory -(typically controlled with the TEMP environment variable). -Alternatively, the directory can be set with the PYSNPTOOLS_CACHE_HOME -environment variable.

-

This function knows the MD5 hash of all PySnpTools example files and uses -that content-based hash to decide if a file needs to be downloaded.

-
>>> from pysnptools.util import example_file # Download and return local file name
->>> # Download the phenotype file if necessary. Return its local location.
->>> pheno_fn = example_file("pysnptools/examples/toydata.phe")
->>> print('The local file name is ', pheno_fn)
-The local file name is ...pysnptools/examples/toydata.phe
->>>
->>> # Download the bed,bim,&fam files if necessary. Return the location of bed file.
->>> bedfile = example_file("tests/datasets/all_chr.maf0.001.N300.*","*.bed")
->>> print('The local file name is ', bedfile)
-The local file name is ...tests/datasets/all_chr.maf0.001.N300.bed
-
-
-
- -
-
-pysnptools.util.snp_gen(fst, dfr, iid_count, sid_count, maf_low=0.05, maf_high=0.5, seed=0, sibs_per_family=10, freq_pop_0=0.5, chr_count=None, label_with_pop=False)
-

Generates a random SnpData

-
-
Parameters:
-
    -

  • fst (float) – Degree of Population Structure, e.g. 0 (a special case), 0.005, 0.01, 0.05, 0.1

    • -

  • dfr (float) – Degree of Family Relatedness, the fraction of individuals belonging to a family, ie. fracSibs, e.g. 0.0, 0.5, 0.6, 0.7, 0.8, 0.9

    • -

  • iid_count (int) – The number of individuals to generate. Because of rounding the actual number may be less.

    • -

  • sid_count (int) – The number of snps to generate.

    • -

  • maf_low (float) – (default .05) lower bound of uniformly-generated Minor allele frequency

    • -

  • maf_high (float) – (default .5) upper bound of uniformly-generated Minor allele frequency

    • -

  • seed (int) – (default 0) Random seed

    • -

  • sibs_per_family (int) – (default 10) number of siblings in each family

    • -

  • freq_pop_0 (float) – (default .5) Fraction of individuals in population 0 (the rest will be in population 1)

    • -

  • chr_count (int) – (default one chromosome per SNP) Number of chromosomes to which SNPs should be assigned. The SNPs will -be assigned as evenly as possible. Chromosome names are integers starting with 1. SNP positions within a chromosome are sequential -integers starting with 1.

    • -
-
-
Return type:
-

SnpData

-
-
Example:
-

-
-
>>> snpdata = snp_gen(fst=.1,dfr=.5,iid_count=200,sid_count=20,maf_low=.05,seed=6)
->>> print(int(snpdata.iid_count), int(snpdata.sid_count)) #because of rounding got 190 individuals
-190 20
-
-
-
-
Also See:
-

snpreader.SnpGen

-
-
-
- -
-
-

util.IntRangeSet

-
-
-class pysnptools.util.IntRangeSet(*ranges_inputs)
-

Bases: object

-

A class for efficiently manipulating ranges of integers (including negatives and longs) using set operations such as union(), intersection(), and difference.

-

The class differs from the built-in set class (and from Boolean numpy arrays) because it does not need to store every element in the set, only for every contiguous range of elements. -It differs from other Python interval libraries (that we know of) by being specialized and optimized for integer elements.

-
-
Example:
-

-
-

Here we take the union (operator “|”) of two IntRangeSets:

-_images/example1.png -
>>> from pysnptools.util import IntRangeSet
->>> a = IntRangeSet("100:500,501:1000") # a is the set of integers from 100 to 500 (exclusive) and 501 to 1000 (exclusive)
->>> b = IntRangeSet("-20,400:600")      # b is the set of integers -20 and the range 400 to 600 (exclusive)
->>> c = a | b                           # c is the union of a and b, namely -20 and 100 to 1000 (exclusive)
->>> print(c)
-IntRangeSet('-20,100:1000')
-
-
-
-
Example:
-

-
-

Suppose we want to find the intron regions of a gene but we are given only the transcription region and the exon regions.

-_images/example2.png -
>>> from pysnptools.util import IntRangeSet
->>> line = "chr15   29370   37380   29370,32358,36715   30817,32561,37380"
->>> chr,trans_start,trans_last,exon_starts,exon_lasts = line.split() # split the line on white space
->>> trans_start = int(trans_start)
->>> trans_stop = int(trans_last) + 1 # add one to convert the inclusive "last" value into a Pythonesque exclusive "stop" value
->>> int_range_set = IntRangeSet((trans_start,trans_stop)) # creates a IntRangeSet from 29370 (inclusive) to 37381 (exclusive)
->>> print(int_range_set) # print at any time to see the current value
-IntRangeSet('29370:37381')
-
-
-

Parse the exon start and last lists from strings to lists of integers (converting ‘last’ to ‘stop’)

-
>>> exon_starts = [int(start) for start in exon_starts.strip(",").split(',')]
->>> exon_stops = [int(last)+1 for last in exon_lasts.strip(",").split(',')]
->>> assert len(exon_starts) == len(exon_stops)
-
-
-

Zip together the two lists to create an iterable of exon_start,exon_stop tuples. Then -‘set subtract’ all these ranges from int_range_set

-
>>> int_range_set -= zip(exon_starts,exon_stops)
->>> print(int_range_set) # See what it looks like
-IntRangeSet('30818:32358,32562:36715')
-
-
-

Create the desired output by iterating through each contiguous range of integers

-
>>> for start, stop in int_range_set.ranges():
-...    print("{0}   {1}     {2}".format(chr, start, stop-1))
-chr15   30818     32357
-chr15   32562     36714
-
-
-

Ranges Input

-

The input to the IntRangeSet constructor and many of its methods is a ranges input.

-

A ranges input is one of the following:

-
-
    -

  • A comma-separated string of integers or integer ranges, e.g., '100:500,500:1000,2000'

    • -

  • An integer or integer expression, e.g., 3

    • -

  • A tuple of two integers, start and stop, e.g., (2,8). stop is exclusive.

    • -

  • A slice with non-negative values, e.g. slice(2,8)

    • -

  • A IntRangeSet (or any class with a ranges() method), e.g., IntRangeSet(3)

    • -

  • A list or iterable (but not tuple) of ranges inputs, e.g., [1,6,7,(100,200)]

    • -
-
-
Example:
-

-
-

Strings:

-
>>> a = IntRangeSet("100:500,500:1000,2000")
->>> a = IntRangeSet('-10:-8,1,2,3:11')
->>> a = IntRangeSet('')
-
-
-

The ranges in an ranges input can overlap and can be in any order.

-
>>> assert IntRangeSet("2000,100:1500,500:1000,2000") == IntRangeSet('100:1500,2000')
-
-
-

Integers and Integer Expressions:

-
>>> a = IntRangeSet(7)
->>> a = IntRangeSet(151000000000) # longs are OK
->>> a = IntRangeSet(2*3+1) # integer expressions are OK
-
-
-

A tuple must have exactly two integers. They represent the start (inclusive) and stop (exclusive) integers in a range.

-
>>> assert IntRangeSet((2,8)) == IntRangeSet('2:8') # check 'set equality'
->>> assert 7 in IntRangeSet((2,8)) # The integer 7 is an element of the set
->>> assert IntRangeSet((2,3)) == IntRangeSet(2)
-
-
-

Lists and iterables of ranges inputs:

-
>>> assert IntRangeSet([1,6,7,(100,200)]) == IntRangeSet('1,6:8,100:200')
->>> assert IntRangeSet(range(0,100)) == IntRangeSet('0:100')
->>> assert IntRangeSet([range(100,200),3,'1000:2000',[4,6],(20,30)]) == IntRangeSet('3:5,6,20:30,100:200,1000:2000')
-
-
-

Some methods accept zero or more ranges input as their input. This is called a ranges_inputs. For example:

-
>>> a = IntRangeSet() # zero ranges inputs
->>> a = IntRangeSet(3) # one ranges input
->>> a = IntRangeSet('3:11,5',14,100) # Three ranges inputs, a string and two integers.
-
-
-
-

Most Important Methods and Operators

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Description

Method or Operator

For Details

is a ‘set equal’ to b?

a == b

__eq__()

is a is superset of b?

b in a, -a.issuperset(b), -a >= b

__contains__()

is a is subset of b?

a <= b, -a.issubset(b)

__le__()

remove i th smallest element(s)

del a[i]

__delitem__()

get i th smallest element(s)

a[i]

__getitem__()

is a not ‘set equal’ to b?

a != b

__ne__()

is a is proper superset of b?

a > b

__gt__()

is a is proper subset of b?

a < b

__lt__()

union b into a

a |= b, -a += b, -a.add(b,c), -a.update(b,c)

add()

union a and b

a | b, -a b, -a.union(b,c)

__or__()

intersect b into a

a &= b, -a.intersection_update(b,c),

__iand__()

intersect a and b

a & b, -a.intersection(b,c)

intersection()

subtract b from in a

a -= b, -a.difference_update(b), -a.discard(b),

__isub__()

remove b from a, error if missing

a.remove(b)

remove()

a ‘set difference’ b

a - b, -a.difference(b)

__sub__()

iterate integers in a, from low

for element in a:

__iter__()

iterate integers in a, from high

for element in reverse(a):

__reversed__()

symmetric difference into a

a ^= b

__ixor__()

a ‘symmetric difference’ b

a ^ b

__xor__()

count of integer elements in a

len(a)

__len__()

iterate ranges in a, from low

for start,stop in a.ranges():

ranges()

count of ranges in a

a.ranges_len

ranges_len()

index of range containing b

a.ranges_index(b)

ranges_index()

get i th smallest range

a.ranges_getitem(i)

ranges_getitem()

a as a string

str(a)

__str__()

remove all elements from a

a.clear()

clear()

copy a

a.copy()

copy()

find index of b in a

a.index(b)

index()

are a and b disjoint?

a.isdisjoint(b)

isdisjoint()

is a empty?

a.isempty(b)

isempty()

smallest element of a

a.min()

min()

largest element of a

a.max()

max()

sum of elements in a

a.sum()

sum()

remove and return an element

a.pop()

pop()

-

Examples, Tips, and Warnings

-

The argument to a method and the right-hand side of an operator can be a ranges input rather than IntRangeSet. For example,

-
-
>>> big = IntRangeSet('-1000:2000')
->>> print(big - '10:20,500') # We can subtract this string because it is a legal ranges input.
-IntRangeSet('-1000:10,20:500,501:2000')
-
-
-

This even works for equality testing:

-
>>> assert big - '10:21,500' == '-1000:10,21:500,501:2000'
-
-
-

The Python in operator is backwards from other operators, so the left-hand side can be any ranges input.

-
>>> assert 501 in big
-
-
-
-

Like most other Python libraries you specify a range with an inclusive start integer and an exclusive stop.

-
>>> print(7 in IntRangeSet('4:8')) # includes 7
-True
->>> print(8 in IntRangeSet('4:8')) # excludes 8
-False
->>> print(8 in IntRangeSet(range(4,7))) # also excludes 8
-False
-
-
-
-
Be careful with the ranges inputs specified with tuples. Suppose we want a IntRangeSet containing 4,5,6,7.
>>> assert IntRangeSet((4,8)) == '4:8' #OK
->>> assert IntRangeSet([(4,8)]) == '4:8' # List of a tuple is OK
->>> assert IntRangeSet(4,8) == '4,8' #No. This is not a tuple. It is two integer inputs, so we get only 4 and 8
->>> assert IntRangeSet([4,8]) == '4,8' #No. List of two integers, so get only 4 and 8
->>> #Illegal: IntRangeSet((4,8,10)) # tuples must be pairs
-
-
-
-
-

Methods and Operators

-
-
-__annotate_func__ = None
-
- -
-
-__annotations_cache__ = {}
-
- -
-
-__contains__(*ranges_inputs)
-

True exactly when all the ranges input is a subset of the IntRangeSet.

-

These are the same:

-
    -

  • b in a

    • -

  • a.issuperset(b)

    • -

  • a >= b

    • -
-
-
Example:
-

-
-
>>> print(3 in IntRangeSet('0:5,6:11'))
-True
->>> print(IntRangeSet('4:7') in IntRangeSet('0:5,6:11'))
-False
->>> '6:9' in IntRangeSet('0:5,6:11') # The left-hand of 'in' can be any ranges input
-True
->>> print(IntRangeSet('0:5,6:11') >= '6:9') # The right-hand of can be any ranges input
-True
-
-
-

The ‘issuperset’ method also supports unioning multiple ranges inputs.

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11').issuperset(4,7,8))
-True
->>> print(IntRangeSet('0:5,6:11').issuperset(4,7,8,100))
-False
-
-
-

Note: By definition, any set is a superset of itself.

-
- -
-
-__delitem__(key)
-

Remove elements from the IntRangeSet by position index. Position index can be specified by an integer with -negative integers counting from the end. Position indexes can also be specified with slices and a ranges input.

-
-
Example:
-

-
-

Removing with an integer position index:

-
>>> a = IntRangeSet('100:200,1000')
->>> del a[2]
->>> print(a)
-IntRangeSet('100:102,103:200,1000')
->>> del a[-1]
->>> print(a)
-IntRangeSet('100:102,103:200')
-
-
-
-
Example:
-

-
-

Removing with a slice:

-
>>> a = IntRangeSet('100:200,1000')
->>> del a[2:11]
->>> print(a)
-IntRangeSet('100:102,111:200,1000')
-
-
-
-
Example:
-

-
-

Removing with a ranges input:

-
>>> a = IntRangeSet('100:200,1000')
->>> del a['2:11']
->>> print(a)
-IntRangeSet('100:102,111:200,1000')
-
-
-
- -
-
-__eq__(other)
-

True exactly when the IntRangeSet on the left is set equivalent to the ranges input on the right.

-
    -

  • a == b

    • -
-
>>> print(IntRangeSet('0:10,12') == IntRangeSet('0:10,12'))
-True
->>> print(IntRangeSet('0:10,12') == IntRangeSet('0:10'))
-False
->>> print(IntRangeSet('0:10,12') == IntRangeSet('12,0:5,5:10'))
-True
->>> print(IntRangeSet('0:10,12') == '0:10,12') # The right-hand can be any ranges input
-True
-
-
-
- -
-
-__firstlineno__ = 13
-
- -
-
-__ge__(other)
-

See IntRangeSet.__contains__()

-
- -
-
-__getitem__(key)
-

a[i] returns the ith integer in sorted order (origin 0) from a, an IntRangeSet

-
>>> print(IntRangeSet('100:200,1000')[0])
-100
->>> print(IntRangeSet('100:200,1000')[10])
-110
-
-
-

If i is negative, the indexing goes from the end

-
>>> print(IntRangeSet('100:200,1000')[-1])
-1000
-
-
-

Python’s standard slice notation may be used and returns IntRangeSets. -(Remember that the Stop number in slice notation is exclusive.)

-
>>> print(IntRangeSet('100:200,1000')[0:10]) # Integers 0 (inclusive) to 10 (exclusive)
-IntRangeSet('100:110')
-
-
-
>>> print(IntRangeSet('100:200,1000')[0:10:2]) # Integers 0 (inclusive) to 10 (exclusive) with step 2
-IntRangeSet('100,102,104,106,108')
-
-
-
>>> print(IntRangeSet('100:200,1000')[-3:]) # The last three integers in the IntRangeSet.
-IntRangeSet('198:200,1000')
-
-
-

An IntRangeSet can also be accessed with any ranges input.

-
>>> IntRangeSet('100:200,1000')['0:10,20']
-IntRangeSet('100:110,120')
-
-
-
- -
-
-__gt__(other)
-

True exactly when the IntRangeSet is a proper superset of the ranges.

-
    -

  • a > b

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') > '7:10') # The right-hand can be any ranges input
-True
-
-
-

Note: By definition, no set is a proper superset of itself.

-
- -
-
-__hash__ = None
-
- -
-
-__iadd__(*ranges_inputs)
-

See IntRangeSet.add()

-
- -
-
-__iand__()
-

Set the IntRangeSet to itself intersected with a input range

-

These are the same:

-
    -

  • a &= b

    • -

  • a.intersection_update(b)

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> a &= '3:8'
->>> print(a)
-IntRangeSet('3:5,6:8')
-
-
-
- -
-
-__imul__(n)
-

Put produces n copies of a unioned together in a, where a is an IntRangeSet. -Because a is a set, the result will either be an empty IntRangeSet (n is 0 or less) or the original a.

-
    -

  • a *= n

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> a *= 5
->>> print(a) # should be unchanged
-IntRangeSet('0:5,6:11')
->>> a *= 0
->>> print(a) # should be empty
-IntRangeSet('')
-
-
-
- -
-
-__init__(*ranges_inputs)
-

Create a IntRangeSet.

-
- -
-
-__ior__(*ranges_inputs)
-

See IntRangeSet.add()

-
- -
-
-__isub__(*ranges_inputs)
-

Remove the elements of the range inputs from the IntRangeSet

-

These are the same:

-
    -

  • a -= b

    • -

  • a.difference_update(b)

    • -

  • a.discard(b)

    • -
-

remove is almost the same except that it raises a KeyError if any element of b is not in a.

-
    -

  • a.remove(b)

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> a -= '3:7'
->>> print(a)
-IntRangeSet('0:3,7:11')
-
-
-

The ‘difference_update’, ‘discard’ and ‘remove’ methods also support subtracting multiple ranges inputs.

-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> a.difference_update('3:7','8:100')
->>> print(a)
-IntRangeSet('0:3,7')
-
-
-
- -
-
-__iter__()
-

Iterate, in order from smallest to largest, the integer elements of the IntRangeSet

-
-
Example:
-

-
-
>>> for i in IntRangeSet('1:4,10'):
-...    print(i)
-1
-2
-3
-10
-
-
-
- -
-
-__ixor__(ranges)
-

Set the IntRangeSet to contains exactly those elements that appear in either itself or the input ranges but not both

-

These are the same:

-
    -

  • a ^= b

    • -

  • a.symmetric_difference_update(b)

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> a ^= '3:8'
->>> print(a)
-IntRangeSet('0:3,5,8:11')
-
-
-
- -
-
-__le__(ranges)
-

True exactly when the IntRangeSet is a subset of the ranges.

-

These are the same:

-
    -

  • a <= b

    • -

  • a.issubset(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') <= '-1:101') # The right-hand can be any ranges input
-True
-
-
-

Note: By definition, any set is a subset of itself.

-
- -
-
-__len__()
-

The number of integer elements in the IntRangeSet

-
>>> print(len(IntRangeSet('0:10,12')))
-11
-
-
-

Note: This is computed in time linear in the number of ranges, rather than integer elements.

-
- -
-
-__lt__(ranges)
-

True exactly when the IntRangeSet is a proper subset of the ranges.

-
    -

  • a < b

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') < '-1:101') # The right-hand can be any ranges input
-True
-
-
-

Note: By definition, no set is a proper subset of itself.

-
- -
-
-__mul__(n)
-

a * n, produces n shallow copies of a unioned, where a is an IntRangeSet. -Because a is a set, the result will either be an empty IntRangeSet (n is 0 or less) or a copy of -the original IntRangeSet.

-
    -

  • a * n

    • -
-
- -
-
-__ne__(other)
-

False exactly when the IntRangeSet on the left is set equivalent to the ranges input on the right.

-

These are the same.

-
    -

  • a != b

    • -

  • not a == b

    • -
-
- -
-
-__or__()
-

Return the union of a IntRangeSet with zero or more ranges inputs. The original IntRangeSet is not changed.

-

These are the same:

-
    -

  • a | b

    • -

  • a + b

    • -

  • a.union(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:10') | 5)
-IntRangeSet('0:10')
-
-
-

The ‘union’ method also support unioning multiple ranges inputs,

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:10').union(5,'100:200'))
-IntRangeSet('0:10,100:200')
-
-
-
- -
-
-__repr__()
-

Use the standard repr(a) function to create a string representation of a, an IntRangeSet.

-
>>> print("Hello " + repr(IntRangeSet(2,3,4,10)))
-Hello IntRangeSet('2:5,10')
-
-
-
- -
-
-__reversed__()
-

reversed(a) is a generator that produces the integer elements of a in order from largest to smallest.

-
-
Example:
-

-
-
>>> for i in reversed(IntRangeSet('1:4,10')):
-...     print(i)
-10
-3
-2
-1
-
-
-
- -
-
-__static_attributes__ = ('_start_items', '_start_to_length')
-
- -
-
-__str__()
-

Use the standard str(a) function to create a string representation of a, an IntRangeSet.

-
>>> print("Hello " + str(IntRangeSet(2,3,4,10)))
-Hello IntRangeSet('2:5,10')
-
-
-
- -
-
-__sub__(*ranges_inputs)
-

Return the set difference of a IntRangeSet with zero or more ranges inputs. The original IntRangeSet is not changed.

-

These are the same:

-
    -

  • a - b

    • -

  • a.difference(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') - 1)
-IntRangeSet('0,2:5,6:11')
->>> print(IntRangeSet('0:5,6:11') - '3:100')
-IntRangeSet('0:3')
-
-
-

The ‘difference’ method also supports subtracting multiple input ranges

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11').difference('3:100',1))
-IntRangeSet('0,2')
-
-
-
- -
-
-__xor__(ranges)
-

Returns a new IntRangeSet set with elements in either the input IntRangeSet or the input range but not both.

-

These are the same:

-
    -

  • a ^ b

    • -

  • a.symmetric_difference(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') ^ '3:9')
-IntRangeSet('0:3,5,9:11')
-
-
-
- -
-
-add(*ranges_inputs)
-

Union zero or more ranges inputs into the current IntRangeSet.

-

These are the same:

-
    -

  • a |= b

    • -

  • a += b

    • -

  • a.add(b)

    • -

  • a.update(b)

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:10')
->>> a |= 5
->>> print(a)
-IntRangeSet('0:10')
-
-
-

The ‘add’ and ‘update’ methods also support unioning multiple ranges inputs,

-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:10')
->>> a.add('5','100:200')
->>> print(a)
-IntRangeSet('0:10,100:200')
-
-
-
- -
-
-clear()
-

Remove all ranges from this IntRangeSet.

-
>>> a = IntRangeSet('0:10,12')
->>> a.clear()
->>> print(a)
-IntRangeSet('')
-
-
-
- -
-
-copy()
-

Create a deep copy of a IntRangeSet.

-
- -
-
-count(ranges)
-

The number of times that the elements of ranges appears in the IntRangeSet. Because IntRangeSet is -a set, the number will be either 0 or 1.

-
>>> print(IntRangeSet('100:110,1000').count('105:107,1000'))
-1
-
-
-
- -
-
-difference(*ranges_inputs)
-

Return the set difference of a IntRangeSet with zero or more ranges inputs. The original IntRangeSet is not changed.

-

These are the same:

-
    -

  • a - b

    • -

  • a.difference(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') - 1)
-IntRangeSet('0,2:5,6:11')
->>> print(IntRangeSet('0:5,6:11') - '3:100')
-IntRangeSet('0:3')
-
-
-

The ‘difference’ method also supports subtracting multiple input ranges

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11').difference('3:100',1))
-IntRangeSet('0,2')
-
-
-
- -
-
-difference_update(*ranges_inputs)
-

See IntRangeSet.__isub__()

-
- -
-
-discard(*ranges_inputs)
-

See IntRangeSet.__isub__()

-
- -
-
-index(other)
-

If x is an integer, returns the index of x in a, where a is an IntRangeSet. -If x is an ranges input, returns an IntRangeSet of index of every integer in x. -Raises an IndexError is x not in a.

-

* a.index(x)

-
>>> print(IntRangeSet('100:110,1000').index(109))
-9
->>> print(IntRangeSet('100:110,1000').index('109,100:104'))
-IntRangeSet('0:4,9')
-
-
-
- -
-
-intersection()
-

Return the intersection of a IntRangeSet and zero or more ranges inputs. The original IntRangeSet is not changed.

-

These are the same:

-
    -

  • a & b

    • -

  • a.intersection(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') & '3:8')
-IntRangeSet('3:5,6:8')
-
-
-

The ‘intersection’ method also support intersecting multiple ranges inputs,

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11').intersection('3:8','4:7'))
-IntRangeSet('4,6')
-
-
-
- -
-
-intersection_update()
-

See IntRangeSet.__iand__()

-
- -
-
-isdisjoint(ranges)
-

True exactly when the two sets have no integer elements in common.

-
-
Example:
-

-
-
>>> print(IntRangeSet('100:110,1000').isdisjoint('900:2000'))
-False
->>> print(IntRangeSet('100:110,1000').isdisjoint('1900:2000'))
-True
-
-
-
- -
-
-property isempty
-

True exactly when the IntRangeSet is empty.

-
>>> print(IntRangeSet().isempty)
-True
->>> print(IntRangeSet(4).isempty)
-False
-
-
-
- -
-
-issubset(ranges)
-

True exactly when the IntRangeSet is a subset of the ranges.

-

These are the same:

-
    -

  • a <= b

    • -

  • a.issubset(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') <= '-1:101') # The right-hand can be any ranges input
-True
-
-
-

Note: By definition, any set is a subset of itself.

-
- -
-
-issuperset(*ranges_inputs)
-

True exactly when all the ranges input is a subset of the IntRangeSet.

-

These are the same:

-
    -

  • b in a

    • -

  • a.issuperset(b)

    • -

  • a >= b

    • -
-
-
Example:
-

-
-
>>> print(3 in IntRangeSet('0:5,6:11'))
-True
->>> print(IntRangeSet('4:7') in IntRangeSet('0:5,6:11'))
-False
->>> '6:9' in IntRangeSet('0:5,6:11') # The left-hand of 'in' can be any ranges input
-True
->>> print(IntRangeSet('0:5,6:11') >= '6:9') # The right-hand of can be any ranges input
-True
-
-
-

The ‘issuperset’ method also supports unioning multiple ranges inputs.

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11').issuperset(4,7,8))
-True
->>> print(IntRangeSet('0:5,6:11').issuperset(4,7,8,100))
-False
-
-
-

Note: By definition, any set is a superset of itself.

-
- -
-
-max()
-

The largest integer element in the IntRangeSet

-
>>> print(IntRangeSet('0:10,12').max())
-12
-
-
-

Note: This is more efficient than max(IntRangeSet(‘0:10,12’)) because is computed -in constant time rather than in time linear to the number of integer elements.

-
- -
-
-min()
-

The smallest integer element in the IntRangeSet

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:10,12').min())
-0
-
-
-

Note: This is more efficient than min(IntRangeSet('0:10,12')) because is computed -in constant time rather than in time linear to the number of integer elements.

-
- -
-
-pop()
-

Remove and return the largest integer element from the IntRangeSet. Raises KeyError if the IntRangeSet is empty.

-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:11')
->>> print(a.pop())
-10
->>> print(a)
-IntRangeSet('0:5,6:10')
-
-
-
- -
-
-ranges()
-

Iterate, in order, the ranges of a IntRangeSet as (start,stop) tuples.

-
-
Example:
-

-
-
>>> for start,stop in IntRangeSet('0:10,100:200').ranges():
-...       print("start is {0}, stop is {1}".format(start,stop))
-start is 0, stop is 10
-start is 100, stop is 200
-
-
-
- -
-
-ranges_getitem(index)
-

Returns the index-th range in the collection of ranges

-
>>> print(IntRangeSet('-30:-20,0:10,12').ranges_getitem(1))
-(0, 10)
-
-
-
- -
-
-ranges_index(element)
-

Returns the ranges index of range containing the element

-
>>> int_range_set = IntRangeSet('-30:-20,0:10,12')
->>> index = int_range_set.ranges_index(5)
->>> print(index)
-1
->>> int_range_set.ranges_getitem(index)
-(0, 10)
-
-
-
- -
-
-property ranges_len
-

The number of contiguous ranges in the IntRangeSet

-
>>> print(IntRangeSet('0:10,12').ranges_len)
-2
-
-
-
- -
-
-remove(*ranges_inputs)
-

See IntRangeSet__isub__()

-
- -
-
-sum()
-

The sum of the integer elements in the IntRangeSet

-
>>> print(IntRangeSet('0:10,12').sum())
-57
-
-
-

Note: This is more efficient than sum(IntRangeSet('0:10,12')) because is computed -in time linear in the number of ranges, rather than integer elements.

-
- -
-
-symmetric_difference(ranges)
-

Returns a new IntRangeSet set with elements in either the input IntRangeSet or the input range but not both.

-

These are the same:

-
    -

  • a ^ b

    • -

  • a.symmetric_difference(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:11') ^ '3:9')
-IntRangeSet('0:3,5,9:11')
-
-
-
- -
-
-symmetric_difference_update(ranges)
-

See IntRangeSet.__ixor__()

-
- -
-
-union()
-

Return the union of a IntRangeSet with zero or more ranges inputs. The original IntRangeSet is not changed.

-

These are the same:

-
    -

  • a | b

    • -

  • a + b

    • -

  • a.union(b)

    • -
-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:10') | 5)
-IntRangeSet('0:10')
-
-
-

The ‘union’ method also support unioning multiple ranges inputs,

-
-
Example:
-

-
-
>>> print(IntRangeSet('0:5,6:10').union(5,'100:200'))
-IntRangeSet('0:10,100:200')
-
-
-
- -
-
-update(*ranges_inputs)
-

Union zero or more ranges inputs into the current IntRangeSet.

-

These are the same:

-
    -

  • a |= b

    • -

  • a += b

    • -

  • a.add(b)

    • -

  • a.update(b)

    • -
-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:10')
->>> a |= 5
->>> print(a)
-IntRangeSet('0:10')
-
-
-

The ‘add’ and ‘update’ methods also support unioning multiple ranges inputs,

-
-
Example:
-

-
-
>>> a = IntRangeSet('0:5,6:10')
->>> a.add('5','100:200')
->>> print(a)
-IntRangeSet('0:10,100:200')
-
-
-
- -
- -
-
-

util.pheno

-
-

Deprecated since version Use: Pheno instead.

-
-
-
-pysnptools.util.pheno.loadOnePhen(filename, i_pheno=0, missing='-9', vectorize=False)
-
-

Deprecated since version Use: Pheno instead.

-
-

Load one column of a phenotype file. Remove any rows with missing data

-
-
Parameters:
-
    -

  • filename (string) – name of the file

    • -

  • i_pheno (int) – column to return (default ‘0’, the first column)

    • -

  • missing (string) – value to threat as missing

    • -

  • vectorize (bool) – if true, return a 1-D vector rather than a 2-D array

    • -
-
-
Return type:
-

An output dictionary

-
-
-

The output dictionary looks like:

-
    -

  • ‘header’ : [1] array phenotype namesv (only if header line is specified in file),

    • -

  • ‘vals’ : [N*1] array of phenotype-data,

    • -

  • ‘iid’ : [N*2] array of family IDs and individual IDs

    • -
-
- -
-
-pysnptools.util.pheno.loadPhen(filename, missing='-9', famid='FID', sampid='ID')
-
-

Deprecated since version Use: Pheno instead.

-
-

Load a phenotype or covariate file. Covariates have the same file format.

-
-
Parameters:
-
    -

  • filename (string) – name of the file

    • -

  • missing (string) – value to threat as missing

    • -

  • vectorize (bool) – if true, return a 1-D vector rather than a 2-D array

    • -
-
-
Return type:
-

An output dictionary

-
-
-

The output dictionary looks like:

-
    -

  • ‘header’ : [1] array phenotype names (only if header line is specified in file),

    • -

  • ‘vals’ : [N*1] array of phenotype-data,

    • -

  • ‘iid’ : [N*2] array of family IDs and individual IDs

    • -
-
- -
-
-
-

util.mapreduce1 Module

-
-
-pysnptools.util.mapreduce1.map_reduce(input_seq, mapper=<function _identity>, reducer=<class 'list'>, input_files=None, output_files=None, name=None, runner=None, nested=None)
-

Runs a function on sequence of inputs and runs a second function on the results. Can be nested and clusterized.

-
-
Parameters:
-
    -

  • input_seq (a sequence) – a sequence of inputs. The sequence must support the len function and be indexable. e.g. a list, range(100)

    • -

  • mapper (a function) – A function to apply to each set of inputs (optional). Defaults to the identity function.

    • -

  • reducer (a function that takes a sequence) – A function to turn the results from the mapper to a single value (optional). Defaults to creating a list of the results.

    • -

  • input_files (a list) – An optional list that tells what input files are needed. The list can contain the names of files (strings), None (ignored), or -objects such as SnpReader’s that can self-report their input files.

    • -

  • output_files (a list) – An optional list that tells what output files will be produced. The list can contain the names of files (strings), None (ignored), or -objects such as SnpReader’s that can self-report their output files.

    • -

  • name (a string) – A name to be displayed if this work is done on a cluster.

    • -

  • runner (Runner) – a Runner, optional: Tells how to run locally, multi-processor, or on a cluster. -If not given, the function is run locally.

    • -

  • nested (a function) – a mapper function that is itself a map_reduce. Some runners can efficiently clusterize such nested mappers.

    • -
-
-
Return type:
-

The results from the reducer.

-
-
Example:
-

-
-

Square the numbers 0 to 99 and report their sum, locally:

-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> map_reduce(range(100),
-...        mapper=lambda x: x*x,
-...        reducer=sum)
-328350
-
-
-

Compute it again, this time run on four processors:

-
>>> from pysnptools.util.mapreduce1.runner import LocalMultiProc
->>> map_reduce(range(100),
-...        mapper=lambda x: x*x,
-...        reducer=sum,
-...        runner=LocalMultiProc(4))
-328350
-
-
-

Compute it using named functions, again using four processors:

-
>>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,LocalMultiProc(4))
-328350
-
-
-
- -
-

util.mapreduce1

-
-
-pysnptools.util.mapreduce1.map_reduce(input_seq, mapper=<function _identity>, reducer=<class 'list'>, input_files=None, output_files=None, name=None, runner=None, nested=None)
-

Runs a function on sequence of inputs and runs a second function on the results. Can be nested and clusterized.

-
-
Parameters:
-
    -

  • input_seq (a sequence) – a sequence of inputs. The sequence must support the len function and be indexable. e.g. a list, range(100)

    • -

  • mapper (a function) – A function to apply to each set of inputs (optional). Defaults to the identity function.

    • -

  • reducer (a function that takes a sequence) – A function to turn the results from the mapper to a single value (optional). Defaults to creating a list of the results.

    • -

  • input_files (a list) – An optional list that tells what input files are needed. The list can contain the names of files (strings), None (ignored), or -objects such as SnpReader’s that can self-report their input files.

    • -

  • output_files (a list) – An optional list that tells what output files will be produced. The list can contain the names of files (strings), None (ignored), or -objects such as SnpReader’s that can self-report their output files.

    • -

  • name (a string) – A name to be displayed if this work is done on a cluster.

    • -

  • runner (Runner) – a Runner, optional: Tells how to run locally, multi-processor, or on a cluster. -If not given, the function is run locally.

    • -

  • nested (a function) – a mapper function that is itself a map_reduce. Some runners can efficiently clusterize such nested mappers.

    • -
-
-
Return type:
-

The results from the reducer.

-
-
Example:
-

-
-

Square the numbers 0 to 99 and report their sum, locally:

-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> map_reduce(range(100),
-...        mapper=lambda x: x*x,
-...        reducer=sum)
-328350
-
-
-

Compute it again, this time run on four processors:

-
>>> from pysnptools.util.mapreduce1.runner import LocalMultiProc
->>> map_reduce(range(100),
-...        mapper=lambda x: x*x,
-...        reducer=sum,
-...        runner=LocalMultiProc(4))
-328350
-
-
-

Compute it using named functions, again using four processors:

-
>>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,LocalMultiProc(4))
-328350
-
-
-
- -
-
-

util.mapreduce1.runner.Runner

-
-
-class pysnptools.util.mapreduce1.runner.Runner
-

A Runner is an object that tells a map_reduce() how to run on a local machine or cluster. Examples include -Local and LocalMultiProc.

-
- -
-
-

util.mapreduce1.runner.Local

-
-
-class pysnptools.util.mapreduce1.runner.Local(mkl_num_threads=None, logging_handler=<StreamHandler <stdout> (NOTSET)>)
-

A Runner that runs a map_reduce() locally. To save memory, it will feed the results of the mapper to the reducer as those results are computed.

-
-
Constructor:
-
Parameters:
-
    -

  • mkl_num_threads (number) – (default None) Limit on the number threads used by the NumPy MKL library.

    • -
-
-
Parameters:
-
    -

  • logging_handler (stream) – (default stdout) Where to output logging messages.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> from pysnptools.util.mapreduce1.runner import Local
->>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,Local())
-328350
-
-
-
-
-
- -
-
-

util.mapreduce1.runner.LocalMultiProc

-
-
-class pysnptools.util.mapreduce1.runner.LocalMultiProc(taskcount, mkl_num_threads=None, weights=None, taskindex_to_environ=None, just_one_process=False, logging_handler=<StreamHandler <stdout> (NOTSET)>)
-

A Runner that runs a map_reduce() as multiple processes on a single machine.

-
-
Constructor:
-
Parameters:
-
    -

  • taskcount (number) – The number of processes to run on.

    • -

  • mkl_num_threads (number) – (default None) Limit on the number threads used by the NumPy MKL library.

    • -
  • -
    weights (array of integers) – (default None) If given, tells the relative amount of work assigned to

    each task. The length of the array must be taskcount. If not given, all tasks are assigned the -same amount of work.

    -
    -
    -
    • -
  • -
    taskindex_to_environ (function from integers to dictionaries) – (default None). If given, this

    should be function from taskindex to a dictionary. The dictionary tells how to temporarily set -environment variables while the task is running. The dictionary is a mapping of -variables and values (both strings).

    -
    -
    -
    • -

  • just_one_process (bool) – (default False) Divide the work for multiple processes, but runs sequentially on one process. Can be useful for debugging.

    • -

  • logging_handler (stream) – (default stdout) Where to output logging messages.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> from pysnptools.util.mapreduce1.runner import LocalMultiProc
->>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,LocalMultiProc(4))
-328350
-
-
-
-
-
- -
-
-

util.mapreduce1.runner.LocalMultiThread

-
-
-class pysnptools.util.mapreduce1.runner.LocalMultiThread(taskcount, mkl_num_threads=None, just_one_process=False)
-

A Runner that runs a map_reduce() as multiple threads on a single machine.

-

Note that Python has problems running some programs efficiently on multiple threads. (Search ‘python global interpreter lock’ for details.) -If this Runner doesn’t work, consider LocalMultiProc instead.

-
-
Constructor:
-
Parameters:
-
    -

  • taskcount (number) – The number of threads to run on.

    • -
-
-
Parameters:
-
    -

  • mkl_num_threads (number) – (default None) Limit on the number threads used by the NumPy MKL library.

    • -
-
-
Parameters:
-
    -

  • just_one_process (bool) – (default False) Divide the work for multiple threads, but sequentially on one thread. Can be useful for debugging.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> from pysnptools.util.mapreduce1.runner import LocalMultiThread
->>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,LocalMultiThread(4))
-328350
-
-
-
-
-
- -
-
-

util.mapreduce1.runner.LocalInParts

-
-
-class pysnptools.util.mapreduce1.runner.LocalInParts(taskindex, taskcount, mkl_num_threads=None, weights=None, environ=None, result_file=None, run_dir='.', temp_dir=None, logging_handler=<StreamHandler <stdout> (NOTSET)>)
-

A Runner that runs one piece of a map_reduce() job locally. Partial results are saved to disk. -Clustering runners and LocalMultiProc use this runner internally.

-
-
Constructor:
-
Parameters:
-
    -

  • taskindex (number) – Which piece of work to run. When 0 to taskcount-1, does map work. When taskcount, does the reduce work.

    • -

  • taskcount (number) – The number of pieces into which to divide the work.

    • -

  • mkl_num_threads (number) – (default None) Limit on the number threads used by the NumPy MKL library.

    • -
  • -
    weights (array of integers) – (default None) If given, tells the relative amount of work assigned to

    each task. The length of the array must be taskcount. If not given, all tasks are assigned the -same amount of work.

    -
    -
    -
    • -
  • -
    environ (dictionary of variables and values) – (default None). Temporarily assigns environment variables.

    both variables and values should be strings.

    -
    -
    -
    • -

  • result_file (string) – (default None) Where to pickle the final results. If no file is given, the final results are returned, but not saved to a file.

    • -

  • result_dir (string) – (default None) The directory for any result_file. Defaults to the current working directory.

    • -

  • temp_dir (string) – (default None) The directory for partial results. Defaults to the result_dir/.working_directory.{map_reduce’s Name}.

    • -

  • logging_handler (stream) – (default stdout) Where to output logging messages.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.mapreduce1 import map_reduce
->>> from pysnptools.util.mapreduce1.runner import LocalInParts
->>> def holder1(n,runner):
-...     def mapper1(x):
-...         return x*x
-...     def reducer1(sequence):
-...        return sum(sequence)
-...     return map_reduce(range(n),mapper=mapper1,reducer=reducer1,runner=runner)
->>> holder1(100,LocalInParts(0,4)) #Run part 0 of 4 and save partial results to disk as '0.4.p'.
->>> holder1(100,LocalInParts(1,4)) #Run part 1 of 4 and save partial results to disk as '1.4.p'.
->>> holder1(100,LocalInParts(2,4)) #Run part 2 of 4 and save partial results to disk as '2.4.p'.
->>> holder1(100,LocalInParts(3,4)) #Run part 3 of 4 and save partial results to disk as '3.4.p'.
->>> holder1(100,LocalInParts(4,4)) #Read the all partial results and then apply the reduce function and return the result.
-328350
-
-
-
-
-
- -
-
-
-

util.filecache Module

-

Tools for reading and writing files, locally or across clusters.

-
-
-pysnptools.util.filecache.ip_address()
-
- -
-
-pysnptools.util.filecache.ip_address_pid()
-

Return the ip address of this computer and process id of the executing process.

-
- -
-

util.filecache.FileCache

-
-
-class pysnptools.util.filecache.FileCache
-
-
A FileCache is class such as LocalCache or PeerToPeer such that
    -

  • for reading, copies a (possibly remote) file to a local disk. (If the local file already exists -and is up-to-date, retrieval is typically skipped.)

    • -

  • for writing, copies a newly-written local file to (possibly remote) storage.

    • -

  • provides an assortment of file- and directory-related commands.

    • -
-
-
-
-
Example:
-

Suppose all machines in a cluster can read & write the storage at ‘peertopeer1/common’. -Also, suppose that ‘peertopeer1/192.168.1.105’ is stored locally -on machine 192.168.1.105 but readable other machines in the cluster -(and so on for all the machines and their IP addresses).

-
>>> from pysnptools.util.filecache import PeerToPeer, ip_address
->>> def id_and_path_function():
-...     ip = ip_address()
-...     return ip, 'peertopeer1/{0}'.format(ip)
->>> file_cache = PeerToPeer(common_directory='peertopeer1/common',id_and_path_function=id_and_path_function)
->>> file_cache
-PeerToPeer('peertopeer1/common',id_and_path_function=...')
-
-
-

Remove anything already in remote storage

-
>>> file_cache.rmtree()
-
-
-

Create a random SNP file and store it remotely.

-
>>> from pysnptools.snpreader import SnpGen, Dense
->>> snp_gen = SnpGen(seed=123,iid_count=100,sid_count=500)
->>> with file_cache.open_write('r123.100x500.dense.txt') as local_filename:
-...    dense1 = Dense.write(local_filename,snp_gen.read())
->>> list(file_cache.walk())
-['r123.100x500.dense.txt']
-
-
-

Copy back from remote storage (if needed) and then read SNP data from local file.

-
>>> with file_cache.open_read('r123.100x500.dense.txt') as local_filename:
-...     dense2 = Dense(local_filename)
-...     print(dense2[:3,:3].read().val) #Read 1st 3 individuals and SNPs
-[[ 0. nan nan]
- [ 0.  0. nan]
- [ 0.  0.  0.]]
-
-
-
-
-

Given an appropriate module, such as LocalCache or PeerToPeer, the FileCache library provides a unified way -to work with any remote storage scheme. It differs from virtual file systems, such as CIFS VFS, because:

-
-
    -

  • FileCache works with all operating systems and requires no operating system changes.

    • -

  • FileCache can take advantage of the highest performance file retrieval methods (e.g. kernel-space file systems, peer-to-peer transfers, tree copies, etc).

    • -

  • FileCache can take advantage of the highest performance local read and write storage (e.g. SSDs)

    • -

  • FileCache can work on top of CIFS VFS and any other remote storage system with an appropriate module.

    • -
-

The downside of FileCache is that:

- -
-

Methods & Properties:

-
-

Every FileCache, such as LocalCache and PeerToPeer, has this property: -name, -and these methods: -file_exists(), getmtime(), join(), load(), open_read(), open_write(), remove(), rmtree(), -save(), and walk(). See below for details.

-
-

Details of Methods & Properties:

-
-
-file_exists(file_name)
-

Tells if there a file with this name. (A directory with the name doesn’t count.)

-
-
Parameters:
-

file_name (string) – The name of the file to look for

-
-
Return type:
-

bool

-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> file_cache.file_exists('localcache1/file1.txt')
-False
-
-
-
>>> file_cache.save('localcache1/file1.txt','Hello')
->>> file_cache.file_exists('localcache1/file1.txt')
-True
-
-
-
- -
-
-getmtime(file_name)
-

Return the time that the file was last modified.

-
-
Parameters:
-

file_name (string) – The name of the (possibly remote) file of interest

-
-
Return type:
-

number, the number of seconds since the epoch

-
-
-
- -
-
-join(path)
-

The FileCache created by appending a path to the current FileCache.

-
-
Parameters:
-

path (string) – path to join to current FileCache

-
-
Return type:
-

FileCache

-
-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache
-LocalCache('localcache1')
->>> sub = file_cache.join('sub1')
->>> sub
-LocalCache('localcache1/sub1')
-
-
-
- -
-
-load(file_name, updater=None)
-

Returns the contents of a file in storage as a string.

-
-
Parameters:
-
    -

  • file_name (string) – The name of the (possibly remote) file to read

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
Return type:
-

string - what was written in the file.

-
-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> file_cache.save('file1.txt','Hello')
->>> file_cache.load('file1.txt')
-'Hello'
-
-
-
- -
-
-property name
-

A path-like name for this FileCache.

-
-
Return type:
-

string

-
-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.name
-'localcache1'
-
-
-
- -
-
-open_read(file_name, updater=None)
-

Used with a ‘with’ statement to produce a local copy of a (possibly remote) file.

-
-
Parameters:
-
    -

  • file_name (string) – The name of the (possibly remote) file to read

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
Return type:
-

a local filename to read.

-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> file_cache.save('file1.txt','Hello')
->>> with file_cache.open_read('file1.txt') as local_filename:
-...     with open(local_filename,'r') as fp:
-...         line = fp.readline()
->>> line
-'Hello'
-
-
-
- -
-
-open_write(file_name, size=0, updater=None)
-

Used with a ‘with’ statement to produce a local file name that will be copied to remote storage -when the ‘with’ statement is exited.

-
-
Parameters:
-
    -

  • file_name (string) – The name of the (possibly remote) file to which to write.

    • -

  • size (number) – (default 0) if given, an error will be thrown immediately if local storage doesn’t -have room for that many bytes.

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
Return type:
-

a local filename to read.

-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> with file_cache.open_write('file1.txt') as local_filename:
-...     with open(local_filename,'w') as fp:
-...         _= fp.write('Hello')
->>> file_cache.load('file1.txt')
-'Hello'
-
-
-
- -
-
-remove(file_name, updater=None)
-

Remove a file from storage. It is an error to remove a directory this way.

-
-
Parameters:
-
    -

  • file_name (string) – The name of the (possibly remote) file to remove.

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
-
- -
-
-rmtree(path=None, updater=None)
-

Delete all files in this FileCache. It is OK if there are no files.

-
-
Parameters:
-
    -

  • path (string) – (Default, None, the current FileCache). Optional a path (subdirectory, not file) to start in.

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
-
- -
-
-save(file_name, contents, size=0, updater=None)
-

Write a string to a file in storage.

-
-
Parameters:
-
    -

  • file_name (string) – The name of the (possibly remote) file to which to write.

    • -

  • contents (string) – What to write to the file.

    • -

  • size (number) – (default 0) if given, an error will be thrown immediately if local storage doesn’t -have room for that many bytes.

    • -

  • updater (function or lambda) – (Default, None). Optional function to which status messages may be written. For example, -the function created by log_in_place().

    • -
-
-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> file_cache.save('file1.txt','Hello')
->>> file_cache.load('file1.txt')
-'Hello'
-
-
-
- -
-
-walk(path=None)
-

Generates the relative paths of the files in the FileCache. It is OK if there are no files.

-
-
Parameters:
-

path (string) – (Default, None, the current FileCache). Optional path (subdirectory, not file) to start in.

-
-
Return type:
-

a generator of strings

-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> list(file_cache.walk())
-[]
-
-
-
>>> file_cache.save('file1.txt','Hello')
->>> list(file_cache.walk())
-['file1.txt']
-
-
-
>>> list(file_cache.walk('no_such_sub_path'))
-[]
-
-
-
- -
- -
-
-

util.filecache.LocalCache

-
-
-class pysnptools.util.filecache.LocalCache(directory)
-

A FileCache for working with files locally.

-

See FileCache for general examples of using FileCache.

-

This is the simplest FileCache in that it stores everything on a local disk rather than storing things remotely.

-
-
Constructor:
-
Parameters:
-
    -

  • directory (string) – The directory under which files should be written and read.

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import LocalCache
->>> file_cache = LocalCache('localcache1')
->>> file_cache.rmtree()
->>> file_cache.save('sub1/file1.txt','Hello')
->>> file_cache.file_exists('sub1/file1.txt')
-True
-
-
-
-
-
-
-property name
-

A path-like name for this LocalCache.

-
-
Return type:
-

string

-
-
-
- -
- -
-
-

util.filecache.PeerToPeer

-
-
-class pysnptools.util.filecache.PeerToPeer(common_directory, id_and_path_function, leave_space=0)
-

A FileCache that allows multiple machines (or processes on one machine) to share files in a peer-to-peer fashion.

-

See FileCache for general examples of using FileCache.

-

This is useful for a cluster with multiple machines with persistent storage and a fast network. Every time a machine writes a file, -the file stays on that machine, but information about the file’s name and location is saved to a common directory. When a new -machine needs the file, it looks to the common directory for a location and then copies from that location. That second location -is also saved to the common directory. Later machines that need the file will randomly pick among the available locations. When a file is removed, it -is only removed from the common directory; clean up of local copies happens later when space is needed for other reads.

-
-
Constructor:
-
Parameters:
-
    -

  • common_directory (FileCache or string) – The path or FileCache where -each file’s name and locations (but not contents) are stored. This path must be read/writable by every machine in the cluster.

    • -

  • id_and_path_function (function) – A function (or lambda) that deterministically returns two values: 1. an id unique to each machine -(or process) and 2. a file path to this machine’s local storage that any machine -in the cluster can read.

    • -

  • leave_space (number) – (Default “0”) How much free space must be left on local drive after downloads and writes.

    • -
-
-
Example:
-

-
-
>>> #Suppose that every machine can access a shared 'peertopeer1' and that 'peertopeer1/IPADDRESS' is local to each machine.
->>> from pysnptools.util.filecache import PeerToPeer, ip_address
->>> def id_and_path_function():
-...     ip = ip_address()
-...     return ip, 'peertopeer1/{0}'.format(ip)
->>> file_cache = PeerToPeer(common_directory='peertopeer1/common',id_and_path_function=id_and_path_function)
->>> file_cache.rmtree()
->>> file_cache.save('sub1/file1.txt','Hello')
->>> file_cache.file_exists('sub1/file1.txt')
-True
-
-
-
-
-
-
-property name
-

A path-like name for this PeerToPeer.

-
-
Return type:
-

string

-
-
-
- -
- -
-
-

util.filecache.Hashdown

-
-
-class pysnptools.util.filecache.Hashdown(url, file_to_hash={}, directory=None, allow_unknown_files=False, trust_local_files=False, _relative_directory=None)
-

A FileCache for downloading files with known MD5 hashes from a URL. Unloading is not supported.

-

See FileCache for general examples of using FileCache.

-
-
Constructor:
-
Parameters:
-
    -

  • url (string) – The URL from which to download any needed files.

    • -

  • file_to_hash (dictionary) – A dictionary from file names to MD5 hashes.

    • -
  • -
    directory (optional, string) – Local location for files. If not given will be under the system temp directory

    (typically controlled with the TEMP environment variable).

    -
    -
    -
    • -
  • -
    allow_unknown_files (optional, bool) – By default, all requested files must be in the dictionary. If True,

    other files can be requested. If found under the URL, they will be downloaded and an entry will be added -to the dictionary.

    -
    -
    -
    • -
  • -
    trust_local_files (optional, bool) – By default, when allow_unknown_files is True, unknown files

    will be download. If trust_local_files is also True, then any local files in directory will -be assumed to have the correct hash.

    -
    -
    -
    • -

  • _relative_directory (string) – Used internally

    • -
-
-
Example:
-

-
-
>>> from pysnptools.util.filecache import Hashdown
->>> file_to_hash= {'pysnptools/examples/toydata.5chrom.bed': '766f55aa716bc7bc97cad4de41a50ec3',
-...                'pysnptools/examples/toydata.5chrom.bim': '6a07f96e521f9a86df7bfd7814eebcd6',
-...                'pysnptools/examples/toydata.5chrom.fam': 'f4eb01f67e0738d4865fad2014af8537'}
->>> hashdown = Hashdown('https://github.com/fastlmm/PySnpTools/raw/cf248cbf762516540470d693532590a77c76fba2',
-...                      file_to_hash=file_to_hash)
->>> hashdown.file_exists('pysnptools/examples/toydata.5chrom.bed')
-True
->>> hashdown.load('pysnptools/examples/toydata.5chrom.fam').split('\n')[0]
-'per0 per0 0 0 2 0.408848'
-
-
-
-
-
-
-static load_hashdown(filename, directory=None, allow_unknown_files=False, trust_local_files=False)
-

Load a Hashdown object from a json file.

-
-
Parameters:
-
    -

  • filename – name of file to load from.

    • -

  • directory – Local location for files. If not given will be under the system temp directory -(typically controlled with the TEMP environment variable).

    • -

  • allow_unknown_files – By default, all requested files must be in the dictionary. If True, -other files can be requested. If found under the URL, they will be downloaded and an entry will be added -to the dictionary.

    • -

  • trust_local_files – By default, when allow_unknown_files is True, unknown files -will be download. If trust_local_files is also True, then any local files in directory will -be assumed to have the correct hash.

    • -
-
-
Return type:
-

Hashdown

-
-
-
>>> from pysnptools.util.filecache import Hashdown
->>> file_to_hash= {'pysnptools/examples/toydata.5chrom.bed': '766f55aa716bc7bc97cad4de41a50ec3',
-...                'pysnptools/examples/toydata.5chrom.bim': '6a07f96e521f9a86df7bfd7814eebcd6',
-...                'pysnptools/examples/toydata.5chrom.fam': 'f4eb01f67e0738d4865fad2014af8537'}
->>> hashdown1 = Hashdown('https://github.com/fastlmm/PySnpTools/raw/cf248cbf762516540470d693532590a77c76fba2',
-...                      file_to_hash=file_to_hash)
->>> hashdown1.save_hashdown('tempdir/demo.hashdown.json')
->>> hashdown2 = Hashdown.load_hashdown('tempdir/demo.hashdown.json')
->>> hashdown2.file_exists('pysnptools/examples/toydata.5chrom.bed')
-True
->>> hashdown2.load('pysnptools/examples/toydata.5chrom.fam').split('\n')[0]
-'per0 per0 0 0 2 0.408848'
-
-
-
- -
-
-property name
-

A path-like name for this Hashdown.

-
-
Return type:
-

string

-
-
-
>>> from pysnptools.util.filecache import Hashdown
->>> file_to_hash= {'pysnptools/examples/toydata.5chrom.bed': '766f55aa716bc7bc97cad4de41a50ec3',
-...                'pysnptools/examples/toydata.5chrom.bim': '6a07f96e521f9a86df7bfd7814eebcd6',
-...                'pysnptools/examples/toydata.5chrom.fam': 'f4eb01f67e0738d4865fad2014af8537'}
->>> hashdown = Hashdown('https://github.com/fastlmm/PySnpTools/raw/cf248cbf762516540470d693532590a77c76fba2',
-...                      file_to_hash=file_to_hash)
->>> hashdown.name
-'hashdown/9ac30da2bf589db947e91744dff0ec24'
->>> hashdown.join('pysnptools').name
-'hashdown/9ac30da2bf589db947e91744dff0ec24/pysnptools'
-
-
-
- -
-
-save_hashdown(filename)
-

Save a Hashdown object to a json file.

-
-
Parameters:
-

filename – name of file to save to.

-
-
-
>>> from pysnptools.util.filecache import Hashdown
->>> file_to_hash= {'pysnptools/examples/toydata.5chrom.bed': '766f55aa716bc7bc97cad4de41a50ec3',
-...                'pysnptools/examples/toydata.5chrom.bim': '6a07f96e521f9a86df7bfd7814eebcd6',
-...                'pysnptools/examples/toydata.5chrom.fam': 'f4eb01f67e0738d4865fad2014af8537'}
->>> hashdown1 = Hashdown('https://github.com/fastlmm/PySnpTools/raw/cf248cbf762516540470d693532590a77c76fba2',
-...                      file_to_hash=file_to_hash)
->>> hashdown1.save_hashdown('tempdir/demo.hashdown.json')
->>> hashdown2 = Hashdown.load_hashdown('tempdir/demo.hashdown.json')
->>> hashdown2.file_exists('pysnptools/examples/toydata.5chrom.bed')
-True
->>> hashdown2.load('pysnptools/examples/toydata.5chrom.fam').split('\n')[0]
-'per0 per0 0 0 2 0.408848'
-
-
-
- -
-
-static scan_local(local_directory, url=None, logging_level=30)
-

Bootstrap a Hashdown by recursively walking a local directory and finding the local MD5 hashes. -(A local hash might be wrong if the files are out of date or have OS-dependent line endings.) -Typically, you’ll then want to save the result to a JSON file and then edit that JSON file -manually to remove uninteresting files.

-
-
Parameters:
-
    -

  • local_directory – Local directory to recursively walk

    • -

  • url – URL to give to the Hashdown. (It will not be checked.)

    • -

  • logging_level – Logging level for printing progress of the walk. Default -is logging.WARNING)

    • -
-
-
Return type:
-

Hashdown

-
-
-
- -
- -
-
-
-

Indices and Tables

-
-
-
-
- - -
-
-
-
- -
-
- - - - \ No newline at end of file diff --git a/docs/objects.inv b/docs/objects.inv deleted file mode 100644 index 4db4e347c09dad8eeb7aa4f454893f9db111fee3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2372 zcmV-K3A^?qAX9K?X>NERX>N99Zgg*Qc_4OWa&u{KZXhxWBOp+6Z)#;@bUGk#d2?=X zbZ>8La|$CMR%LQ?X>V>iATcg5E-(rsAXI2&AaZ4GVQFq;WpW^IW*~HEX>%ZEX>4U6 zX>%ZBZ*6dLWpi_7WFU2OX>MmAdTeQ8E(&pV6oU;(oshvm2H~JY}D~x-IQ&` zH@|FzEcsh@uyyrQ|JnV|oBKO8{%}?bV>M?b*H|ulzvt4!Ma7KSJY2HNP`0aDbLi}H zduU1!>;AlIVL1{#2WA=geotV=s;a0OL$IBFXO*XwZWPz{2;n~>mdbOV%H3~h zkTTg%ynbQ|Wx7W*UMn|81moFGR9v!}V<-J2gY)(ag=V)i+!BXD@SZ_|u$+IXAE>o- z9L4OHF~Wf@_h`mU?tX$|JrknDPW)LRFPc``6?ANqcNpoGRAY!k@t=@}pqtlxv`-t0 zAuo_wX@SgI3*ee2c=evD!;5G z1Iz+$hyhc9MYV@@2KBJ!9X_mjhii^(y@3G|-M9!mottPqOV!IE_S?vUlKzN<|G?K% z1~VgP8H3&S6A6mHh~Y#G9F5WQI(($YZS-GsnP2D>O|x@B-m=uCX`#~zVribN1uXeg zDsT!x?939XfSJVu2V(42YL&T9`>#V=EzQyy44An{TX5OfM4-3xeDTqWQiv2w;abrn zmTe);*%=1SE$0G|TrfxjgZo4yJ(HYapw{G^hNOTK8q(YifS}fHhzB=zgCf|u!!>n7 z5@^3eG;}9?a`A9Y;Y~sch5(oq4RWr|JTPI1serqdF(F`%8j=BIr$M}VCEbN@;?gG- z)e4RYtuU$BV6KKf-!{3;H$MX7(p`GDO>V-?&zQLM(16rbBO+Aztx(iG;Gh*3$UZ#b)2?7dmGZMakt|in_NY&cZ%5I87;>eVQoF`|}Z58Mg zJA`tV9!HLq%RV&)jn+e0gTAjUURujbgey*Iq*u;GC=NPP!>{603Fo#~BDh6L&-%~I z7QUKW+I(%%_qRQ8*?>puA*4Z9f2Y@wGYnco&IRCQ^IXGfCS-1|3VW0kq_mjO$!{86 zQ^-(phAAOpR6)D?pnwYr4#sTg0mIK+N0Ww(-))d4KmY_a0YW^u2@n(kGyy_9un7>7 zK<`vU6JWw8Hvz6GyuW%F?XJ-F!vu!39|E9%kIX->$D1a`Cd32N2oEgSV=BNpJ0$aM zY_uZI+Aw%>Hvx;w+b$kQfR6nb{xU6+j`9zxm)1o~hSGv+~ zQg9~7`Ja!;b*0@^RhS~#5i!woP2%r9RsyD&nW7PU${E=LmA4CWUE+ULGo(m%L`2kG zb?vxWWso3zV&c;Njl1h~IZb%>y4^&#Yx5=c_^!&y8=|u$HEp;zq=(*NAR$bX5A_Zw z27K@EI$KGTp7jnVy7Al~O+DB-Rg)tC4`QdqB?I8o2-eCuY*^fC#%HK0-Q%2jzaMS}lFp?5+XTe&rt4RJd5oHC9* zx73N)TMB((=0KbQ!OD^{1mQL#iLcv=#Jd1M;u}rEP5VOPEvMOkqm94fG+L5y(#jDX zN0RQGHtU)zBLAIh!$}*@_+zEW$G~BTfy9-iq~Xx^5osH^5Qs&KiZhLd8+F99LnjCW zmtwc$n#+RI74p`X@dP1LxMU8CL;6?&*!pcW{^lT6gE$q!xXJjp5%_f#W_)UsRMrIq z+&&zhnBWFrTIJ^htxz-ca@GR9imFjqRQn4w<{BpQ}j@%Pnfs82nM>hMzUHyJiNi6;YboW`#--NNC$Pkipp`o^- z=2RVjuuwIshLuze&^rN9701dP@O*Gt2=drEsss;yL6zESPF69DK9W~feZy&dV&+gb zuM)M1PH^XJU5Dc&g&zpX=|&MhscQ|$CA~7Qnu1k0q0b)0N2)VUnwl-=`fKC=MvdZ( zlSZ)^zN2mAXyjiqbz1#^i>DeI$t62N<*q2z?g&Mmf)@(8>R&v-sB|70_qf!1zF?QE zDd!2c@qY(BE7jTs8?e^snpxk&?jQf8&x$*Ub9`ns)`>Y&sybp}Q5rDXXZ9|W3_jdg zp4CfaE?zZzS9Ij6Vg+zIYNj+V+k!tJ^0hPUFZ2EyYd_Pil+NQmpMr5>7>g)8$}Q7< z)<+Y1YAY+Yx+zwY`g%B}OK7n#-9IFWE}&e`^|A}bmMYntkjgJfWPMP^Pb$e{sQnVS z!>x!eE@~WS9oR^0=L`$pfgjDFlro-(TS09P{M@2Cy|4s~Fa7IIKL7k%a{mq!EVi6+ z1gl-Z{uRc!ukt{$P2T*PKZtYD8D$`zN?eyYRjDY0Az2z0KWX z-{3N~d{?m-|6e&gY)+qcKeu`Gmv`qMEB(&6RN9;ewfWE#ZOvu>c>T`#LqJ{~?@!<8 q`KR8MdP5iAy@73yF2k2o+PJ$f4pXrItY@D4?{C?j-~A5)`CpY4B#iI? diff --git a/docs/py-modindex.html b/docs/py-modindex.html deleted file mode 100644 index a3197490..00000000 --- a/docs/py-modindex.html +++ /dev/null @@ -1,143 +0,0 @@ - - - - - - - Python Module Index — pysnptools 1.0.0 documentation - - - - - - - - - - - - - - - -
-
-
-
- - -

Python Module Index

- -
- p -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
- p
- pysnptools -
    - pysnptools.distreader -
    - pysnptools.kernelreader -
    - pysnptools.pstreader -
    - pysnptools.snpreader -
    - pysnptools.standardizer -
    - pysnptools.util -
    - pysnptools.util.filecache -
    - pysnptools.util.mapreduce1 -
    - pysnptools.util.pheno -
- - -
-
-
-
- -
-
- - - - \ No newline at end of file diff --git a/docs/search.html b/docs/search.html deleted file mode 100644 index 2ee44eb2..00000000 --- a/docs/search.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - Search — pysnptools 1.0.0 documentation - - - - - - - - - - - - - - - - - - - -
-
-
-
- -

Search

- - - - -

- Searching for multiple words only shows matches that contain - all words. -

- - -
- - - -
- - -
- - -
-
-
-
- -
-
- - - - \ No newline at end of file diff --git a/docs/searchindex.js b/docs/searchindex.js deleted file mode 100644 index 41102792..00000000 --- a/docs/searchindex.js +++ /dev/null @@ -1 +0,0 @@ -Search.setIndex({"alltitles":{"Indices and Tables":[[0,"indices-and-tables"]],"distreader Module":[[0,"module-pysnptools.distreader"]],"distreader.Bgen":[[0,"distreader-bgen"]],"distreader.DistData":[[0,"distreader-distdata"]],"distreader.DistGen":[[0,"distreader-distgen"]],"distreader.DistHdf5":[[0,"distreader-disthdf5"]],"distreader.DistMemMap":[[0,"distreader-distmemmap"]],"distreader.DistNpz":[[0,"distreader-distnpz"]],"distreader.DistReader":[[0,"distreader-distreader"]],"kernelreader Module":[[0,"module-pysnptools.kernelreader"]],"kernelreader.Identity":[[0,"kernelreader-identity"]],"kernelreader.KernelData":[[0,"kernelreader-kerneldata"]],"kernelreader.KernelHdf5":[[0,"kernelreader-kernelhdf5"]],"kernelreader.KernelNpz":[[0,"kernelreader-kernelnpz"]],"kernelreader.KernelReader":[[0,"kernelreader-kernelreader"]],"kernelreader.SnpKernel":[[0,"kernelreader-snpkernel"]],"kernelstandardizer Module":[[0,"kernelstandardizer-module"]],"kernelstandardizer.DiagKtoN":[[0,"kernelstandardizer-diagkton"]],"kernelstandardizer.DiagKtoNTrained":[[0,"kernelstandardizer-diagktontrained"]],"kernelstandardizer.Identity":[[0,"kernelstandardizer-identity"]],"kernelstandardizer.KernelStandardizer":[[0,"kernelstandardizer-kernelstandardizer"]],"pstreader Module":[[0,"module-pysnptools.pstreader"]],"pstreader.PstData":[[0,"pstreader-pstdata"]],"pstreader.PstHdf5":[[0,"pstreader-psthdf5"]],"pstreader.PstMemMap":[[0,"pstreader-pstmemmap"]],"pstreader.PstNpz":[[0,"pstreader-pstnpz"]],"pstreader.PstReader":[[0,"pstreader-pstreader"]],"pysnptools Documentation":[[0,null]],"snpreader Module":[[0,"module-pysnptools.snpreader"]],"snpreader.Bed":[[0,"snpreader-bed"]],"snpreader.Dat":[[0,"snpreader-dat"]],"snpreader.Dense":[[0,"snpreader-dense"]],"snpreader.DistributedBed":[[0,"snpreader-distributedbed"]],"snpreader.Ped":[[0,"snpreader-ped"]],"snpreader.Pheno":[[0,"snpreader-pheno"]],"snpreader.SnpData":[[0,"snpreader-snpdata"]],"snpreader.SnpGen":[[0,"snpreader-snpgen"]],"snpreader.SnpHdf5":[[0,"snpreader-snphdf5"]],"snpreader.SnpMemMap":[[0,"snpreader-snpmemmap"]],"snpreader.SnpNpz":[[0,"snpreader-snpnpz"]],"snpreader.SnpReader":[[0,"snpreader-snpreader"]],"standardizer Module":[[0,"module-pysnptools.standardizer"]],"standardizer.Beta":[[0,"standardizer-beta"]],"standardizer.DiagKtoN":[[0,"standardizer-diagkton"]],"standardizer.DiagKtoNTrained":[[0,"standardizer-diagktontrained"]],"standardizer.Identity":[[0,"standardizer-identity"]],"standardizer.Standardizer":[[0,"standardizer-standardizer"]],"standardizer.Unit":[[0,"standardizer-unit"]],"standardizer.UnitTrained":[[0,"standardizer-unittrained"]],"util":[[0,"module-pysnptools.util"]],"util Module":[[0,"util-module"]],"util.IntRangeSet":[[0,"util-intrangeset"]],"util.filecache Module":[[0,"module-pysnptools.util.filecache"]],"util.filecache.FileCache":[[0,"util-filecache-filecache"]],"util.filecache.Hashdown":[[0,"util-filecache-hashdown"]],"util.filecache.LocalCache":[[0,"util-filecache-localcache"]],"util.filecache.PeerToPeer":[[0,"util-filecache-peertopeer"]],"util.mapreduce1":[[0,"util-mapreduce1"]],"util.mapreduce1 Module":[[0,"module-pysnptools.util.mapreduce1"]],"util.mapreduce1.runner.Local":[[0,"util-mapreduce1-runner-local"]],"util.mapreduce1.runner.LocalInParts":[[0,"util-mapreduce1-runner-localinparts"]],"util.mapreduce1.runner.LocalMultiProc":[[0,"util-mapreduce1-runner-localmultiproc"]],"util.mapreduce1.runner.LocalMultiThread":[[0,"util-mapreduce1-runner-localmultithread"]],"util.mapreduce1.runner.Runner":[[0,"util-mapreduce1-runner-runner"]],"util.pheno":[[0,"module-pysnptools.util.pheno"]]},"docnames":["index"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2},"filenames":["index.rst"],"indexentries":{"__annotate_func__ (pysnptools.util.intrangeset attribute)":[[0,"pysnptools.util.IntRangeSet.__annotate_func__",false]],"__annotations_cache__ (pysnptools.util.intrangeset attribute)":[[0,"pysnptools.util.IntRangeSet.__annotations_cache__",false]],"__contains__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__contains__",false]],"__delitem__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__delitem__",false]],"__eq__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__eq__",false]],"__firstlineno__ (pysnptools.util.intrangeset attribute)":[[0,"pysnptools.util.IntRangeSet.__firstlineno__",false]],"__ge__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__ge__",false]],"__getitem__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__getitem__",false]],"__gt__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__gt__",false]],"__hash__ (pysnptools.util.intrangeset attribute)":[[0,"pysnptools.util.IntRangeSet.__hash__",false]],"__iadd__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__iadd__",false]],"__iand__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__iand__",false]],"__imul__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__imul__",false]],"__init__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__init__",false]],"__ior__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__ior__",false]],"__isub__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__isub__",false]],"__iter__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__iter__",false]],"__ixor__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__ixor__",false]],"__le__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__le__",false]],"__len__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__len__",false]],"__lt__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__lt__",false]],"__mul__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__mul__",false]],"__ne__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__ne__",false]],"__or__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__or__",false]],"__repr__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__repr__",false]],"__reversed__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__reversed__",false]],"__static_attributes__ (pysnptools.util.intrangeset attribute)":[[0,"pysnptools.util.IntRangeSet.__static_attributes__",false]],"__str__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__str__",false]],"__sub__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__sub__",false]],"__xor__() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.__xor__",false]],"add() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.add",false]],"allclose() (pysnptools.distreader.distdata method)":[[0,"pysnptools.distreader.DistData.allclose",false]],"allclose() (pysnptools.kernelreader.kerneldata method)":[[0,"pysnptools.kernelreader.KernelData.allclose",false]],"allclose() (pysnptools.pstreader.pstdata method)":[[0,"pysnptools.pstreader.PstData.allclose",false]],"allclose() (pysnptools.snpreader.snpdata method)":[[0,"pysnptools.snpreader.SnpData.allclose",false]],"array_module() (in module pysnptools.util)":[[0,"pysnptools.util.array_module",false]],"as_dist() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.as_dist",false]],"as_snp() (pysnptools.distreader.distreader method)":[[0,"pysnptools.distreader.DistReader.as_snp",false]],"asnumpy() (in module pysnptools.util)":[[0,"pysnptools.util.asnumpy",false]],"bed (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Bed",false]],"beta (class in pysnptools.standardizer)":[[0,"pysnptools.standardizer.Beta",false]],"bgen (class in pysnptools.distreader)":[[0,"pysnptools.distreader.Bgen",false]],"clear() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.clear",false]],"col (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.col",false]],"col (pysnptools.snpreader.bed property)":[[0,"pysnptools.snpreader.Bed.col",false]],"col (pysnptools.snpreader.distributedbed property)":[[0,"pysnptools.snpreader.DistributedBed.col",false]],"col (pysnptools.snpreader.snpgen property)":[[0,"pysnptools.snpreader.SnpGen.col",false]],"col (pysnptools.snpreader.snphdf5 property)":[[0,"pysnptools.snpreader.SnpHdf5.col",false]],"col (pysnptools.snpreader.snpnpz property)":[[0,"pysnptools.snpreader.SnpNpz.col",false]],"col_count (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.col_count",false]],"col_property (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.col_property",false]],"col_property (pysnptools.snpreader.bed property)":[[0,"pysnptools.snpreader.Bed.col_property",false]],"col_property (pysnptools.snpreader.distributedbed property)":[[0,"pysnptools.snpreader.DistributedBed.col_property",false]],"col_property (pysnptools.snpreader.snpgen property)":[[0,"pysnptools.snpreader.SnpGen.col_property",false]],"col_to_index() (pysnptools.pstreader.pstreader method)":[[0,"pysnptools.pstreader.PstReader.col_to_index",false]],"copy() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.copy",false]],"copyinputs() (pysnptools.snpreader.bed method)":[[0,"pysnptools.snpreader.Bed.copyinputs",false]],"copyinputs() (pysnptools.snpreader.dat method)":[[0,"pysnptools.snpreader.Dat.copyinputs",false]],"copyinputs() (pysnptools.snpreader.distributedbed method)":[[0,"pysnptools.snpreader.DistributedBed.copyinputs",false]],"copyinputs() (pysnptools.snpreader.ped method)":[[0,"pysnptools.snpreader.Ped.copyinputs",false]],"copyinputs() (pysnptools.snpreader.snpgen method)":[[0,"pysnptools.snpreader.SnpGen.copyinputs",false]],"copyinputs() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.copyinputs",false]],"count() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.count",false]],"create_directory_if_necessary() (in module pysnptools.util)":[[0,"pysnptools.util.create_directory_if_necessary",false]],"dat (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Dat",false]],"default_id_rsid_function() (in module pysnptools.distreader.bgen)":[[0,"pysnptools.distreader.bgen.default_id_rsid_function",false]],"default_iid_function() (in module pysnptools.distreader.bgen)":[[0,"pysnptools.distreader.bgen.default_iid_function",false]],"default_sample_function() (in module pysnptools.distreader.bgen)":[[0,"pysnptools.distreader.bgen.default_sample_function",false]],"default_sid_function() (in module pysnptools.distreader.bgen)":[[0,"pysnptools.distreader.bgen.default_sid_function",false]],"dense (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Dense",false]],"difference() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.difference",false]],"difference_update() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.difference_update",false]],"discard() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.discard",false]],"distdata (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistData",false]],"distgen (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistGen",false]],"disthdf5 (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistHdf5",false]],"distmemmap (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistMemMap",false]],"distnpz (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistNpz",false]],"distreader (class in pysnptools.distreader)":[[0,"pysnptools.distreader.DistReader",false]],"distributedbed (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.DistributedBed",false]],"empty() (pysnptools.distreader.distmemmap static method)":[[0,"pysnptools.distreader.DistMemMap.empty",false]],"empty() (pysnptools.pstreader.pstmemmap static method)":[[0,"pysnptools.pstreader.PstMemMap.empty",false]],"empty() (pysnptools.snpreader.snpmemmap static method)":[[0,"pysnptools.snpreader.SnpMemMap.empty",false]],"example_file() (in module pysnptools.util)":[[0,"pysnptools.util.example_file",false]],"file_exists() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.file_exists",false]],"filecache (class in pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.FileCache",false]],"filename (pysnptools.distreader.distmemmap property)":[[0,"pysnptools.distreader.DistMemMap.filename",false]],"filename (pysnptools.pstreader.pstmemmap property)":[[0,"pysnptools.pstreader.PstMemMap.filename",false]],"filename (pysnptools.snpreader.snpmemmap property)":[[0,"pysnptools.snpreader.SnpMemMap.filename",false]],"flush() (pysnptools.distreader.bgen method)":[[0,"pysnptools.distreader.Bgen.flush",false]],"flush() (pysnptools.distreader.distmemmap method)":[[0,"pysnptools.distreader.DistMemMap.flush",false]],"flush() (pysnptools.pstreader.psthdf5 method)":[[0,"pysnptools.pstreader.PstHdf5.flush",false]],"flush() (pysnptools.pstreader.pstmemmap method)":[[0,"pysnptools.pstreader.PstMemMap.flush",false]],"flush() (pysnptools.snpreader.snpmemmap method)":[[0,"pysnptools.snpreader.SnpMemMap.flush",false]],"format_delta() (in module pysnptools.util)":[[0,"pysnptools.util.format_delta",false]],"genwrite() (pysnptools.distreader.bgen static method)":[[0,"pysnptools.distreader.Bgen.genwrite",false]],"get_array_module() (in module pysnptools.util)":[[0,"pysnptools.util.get_array_module",false]],"getmtime() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.getmtime",false]],"hashdown (class in pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.Hashdown",false]],"hdf5 (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Hdf5",false]],"identity (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.Identity",false]],"identity (class in pysnptools.kernelstandardizer)":[[0,"pysnptools.kernelstandardizer.Identity",false]],"identity (class in pysnptools.standardizer)":[[0,"pysnptools.standardizer.Identity",false]],"iid (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.iid",false]],"iid (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid",false]],"iid (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.iid",false]],"iid0 (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid0",false]],"iid0_count (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid0_count",false]],"iid0_to_index() (pysnptools.kernelreader.kernelreader method)":[[0,"pysnptools.kernelreader.KernelReader.iid0_to_index",false]],"iid1 (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid1",false]],"iid1_count (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid1_count",false]],"iid1_to_index() (pysnptools.kernelreader.kernelreader method)":[[0,"pysnptools.kernelreader.KernelReader.iid1_to_index",false]],"iid_count (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.iid_count",false]],"iid_count (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.iid_count",false]],"iid_count (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.iid_count",false]],"iid_to_index() (pysnptools.distreader.distreader method)":[[0,"pysnptools.distreader.DistReader.iid_to_index",false]],"iid_to_index() (pysnptools.kernelreader.kernelreader method)":[[0,"pysnptools.kernelreader.KernelReader.iid_to_index",false]],"iid_to_index() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.iid_to_index",false]],"index() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.index",false]],"intersect_apply() (in module pysnptools.util)":[[0,"pysnptools.util.intersect_apply",false]],"intersect_ids() (in module pysnptools.util)":[[0,"pysnptools.util.intersect_ids",false]],"intersection() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.intersection",false]],"intersection_update() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.intersection_update",false]],"intrangeset (class in pysnptools.util)":[[0,"pysnptools.util.IntRangeSet",false]],"ip_address() (in module pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.ip_address",false]],"ip_address_pid() (in module pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.ip_address_pid",false]],"is_constant (pysnptools.standardizer.standardizer property)":[[0,"pysnptools.standardizer.Standardizer.is_constant",false]],"isdisjoint() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.isdisjoint",false]],"isempty (pysnptools.util.intrangeset property)":[[0,"pysnptools.util.IntRangeSet.isempty",false]],"issubset() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.issubset",false]],"issuperset() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.issuperset",false]],"join() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.join",false]],"kernel() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.kernel",false]],"kerneldata (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.KernelData",false]],"kernelhdf5 (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.KernelHdf5",false]],"kernelnpz (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.KernelNpz",false]],"kernelreader (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.KernelReader",false]],"kernelstandardizer (class in pysnptools.kernelstandardizer)":[[0,"pysnptools.kernelstandardizer.KernelStandardizer",false]],"load() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.load",false]],"load_hashdown() (pysnptools.util.filecache.hashdown static method)":[[0,"pysnptools.util.filecache.Hashdown.load_hashdown",false]],"loadonephen() (in module pysnptools.util.pheno)":[[0,"pysnptools.util.pheno.loadOnePhen",false]],"loadphen() (in module pysnptools.util.pheno)":[[0,"pysnptools.util.pheno.loadPhen",false]],"local (class in pysnptools.util.mapreduce1.runner)":[[0,"pysnptools.util.mapreduce1.runner.Local",false]],"localcache (class in pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.LocalCache",false]],"localinparts (class in pysnptools.util.mapreduce1.runner)":[[0,"pysnptools.util.mapreduce1.runner.LocalInParts",false]],"localmultiproc (class in pysnptools.util.mapreduce1.runner)":[[0,"pysnptools.util.mapreduce1.runner.LocalMultiProc",false]],"localmultithread (class in pysnptools.util.mapreduce1.runner)":[[0,"pysnptools.util.mapreduce1.runner.LocalMultiThread",false]],"log_in_place() (in module pysnptools.util)":[[0,"pysnptools.util.log_in_place",false]],"map_reduce() (in module pysnptools.util.mapreduce1)":[[0,"pysnptools.util.mapreduce1.map_reduce",false]],"max() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.max",false]],"min() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.min",false]],"module":[[0,"module-pysnptools",false],[0,"module-pysnptools.distreader",false],[0,"module-pysnptools.kernelreader",false],[0,"module-pysnptools.pstreader",false],[0,"module-pysnptools.snpreader",false],[0,"module-pysnptools.standardizer",false],[0,"module-pysnptools.util",false],[0,"module-pysnptools.util.filecache",false],[0,"module-pysnptools.util.mapreduce1",false],[0,"module-pysnptools.util.pheno",false]],"name (pysnptools.util.filecache.filecache property)":[[0,"pysnptools.util.filecache.FileCache.name",false]],"name (pysnptools.util.filecache.hashdown property)":[[0,"pysnptools.util.filecache.Hashdown.name",false]],"name (pysnptools.util.filecache.localcache property)":[[0,"pysnptools.util.filecache.LocalCache.name",false]],"name (pysnptools.util.filecache.peertopeer property)":[[0,"pysnptools.util.filecache.PeerToPeer.name",false]],"offset (pysnptools.distreader.distmemmap property)":[[0,"pysnptools.distreader.DistMemMap.offset",false]],"offset (pysnptools.pstreader.pstmemmap property)":[[0,"pysnptools.pstreader.PstMemMap.offset",false]],"offset (pysnptools.snpreader.snpmemmap property)":[[0,"pysnptools.snpreader.SnpMemMap.offset",false]],"open_read() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.open_read",false]],"open_write() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.open_write",false]],"ped (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Ped",false]],"peertopeer (class in pysnptools.util.filecache)":[[0,"pysnptools.util.filecache.PeerToPeer",false]],"pheno (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.Pheno",false]],"pop() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.pop",false]],"pos (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.pos",false]],"pos (pysnptools.kernelreader.snpkernel property)":[[0,"pysnptools.kernelreader.SnpKernel.pos",false]],"pos (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.pos",false]],"print2() (in module pysnptools.util)":[[0,"pysnptools.util.print2",false]],"pstdata (class in pysnptools.pstreader)":[[0,"pysnptools.pstreader.PstData",false]],"psthdf5 (class in pysnptools.pstreader)":[[0,"pysnptools.pstreader.PstHdf5",false]],"pstmemmap (class in pysnptools.pstreader)":[[0,"pysnptools.pstreader.PstMemMap",false]],"pstnpz (class in pysnptools.pstreader)":[[0,"pysnptools.pstreader.PstNpz",false]],"pstreader (class in pysnptools.pstreader)":[[0,"pysnptools.pstreader.PstReader",false]],"pysnptools":[[0,"module-pysnptools",false]],"pysnptools.distreader":[[0,"module-pysnptools.distreader",false]],"pysnptools.kernelreader":[[0,"module-pysnptools.kernelreader",false]],"pysnptools.pstreader":[[0,"module-pysnptools.pstreader",false]],"pysnptools.snpreader":[[0,"module-pysnptools.snpreader",false]],"pysnptools.standardizer":[[0,"module-pysnptools.standardizer",false]],"pysnptools.util":[[0,"module-pysnptools.util",false]],"pysnptools.util.filecache":[[0,"module-pysnptools.util.filecache",false]],"pysnptools.util.mapreduce1":[[0,"module-pysnptools.util.mapreduce1",false]],"pysnptools.util.pheno":[[0,"module-pysnptools.util.pheno",false]],"ranges() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.ranges",false]],"ranges_getitem() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.ranges_getitem",false]],"ranges_index() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.ranges_index",false]],"ranges_len (pysnptools.util.intrangeset property)":[[0,"pysnptools.util.IntRangeSet.ranges_len",false]],"read() (pysnptools.distreader.distreader method)":[[0,"pysnptools.distreader.DistReader.read",false]],"read() (pysnptools.kernelreader.kernelreader method)":[[0,"pysnptools.kernelreader.KernelReader.read",false]],"read() (pysnptools.pstreader.pstreader method)":[[0,"pysnptools.pstreader.PstReader.read",false]],"read() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.read",false]],"read_kernel() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.read_kernel",false]],"read_snps() (pysnptools.kernelreader.snpkernel method)":[[0,"pysnptools.kernelreader.SnpKernel.read_snps",false]],"remove() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.remove",false]],"remove() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.remove",false]],"rmtree() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.rmtree",false]],"row (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.row",false]],"row (pysnptools.snpreader.bed property)":[[0,"pysnptools.snpreader.Bed.row",false]],"row (pysnptools.snpreader.distributedbed property)":[[0,"pysnptools.snpreader.DistributedBed.row",false]],"row (pysnptools.snpreader.snpgen property)":[[0,"pysnptools.snpreader.SnpGen.row",false]],"row (pysnptools.snpreader.snphdf5 property)":[[0,"pysnptools.snpreader.SnpHdf5.row",false]],"row (pysnptools.snpreader.snpnpz property)":[[0,"pysnptools.snpreader.SnpNpz.row",false]],"row_count (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.row_count",false]],"row_property (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.row_property",false]],"row_property (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.row_property",false]],"row_property (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.row_property",false]],"row_to_index() (pysnptools.pstreader.pstreader method)":[[0,"pysnptools.pstreader.PstReader.row_to_index",false]],"runner (class in pysnptools.util.mapreduce1.runner)":[[0,"pysnptools.util.mapreduce1.runner.Runner",false]],"save() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.save",false]],"save_hashdown() (pysnptools.util.filecache.hashdown method)":[[0,"pysnptools.util.filecache.Hashdown.save_hashdown",false]],"scan_local() (pysnptools.util.filecache.hashdown static method)":[[0,"pysnptools.util.filecache.Hashdown.scan_local",false]],"shape (pysnptools.pstreader.pstreader property)":[[0,"pysnptools.pstreader.PstReader.shape",false]],"sid (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.sid",false]],"sid (pysnptools.kernelreader.snpkernel property)":[[0,"pysnptools.kernelreader.SnpKernel.sid",false]],"sid (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.sid",false]],"sid_count (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.sid_count",false]],"sid_count (pysnptools.kernelreader.snpkernel property)":[[0,"pysnptools.kernelreader.SnpKernel.sid_count",false]],"sid_count (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.sid_count",false]],"sid_to_index() (pysnptools.distreader.distreader method)":[[0,"pysnptools.distreader.DistReader.sid_to_index",false]],"sid_to_index() (pysnptools.snpreader.snpreader method)":[[0,"pysnptools.snpreader.SnpReader.sid_to_index",false]],"snp_gen() (in module pysnptools.util)":[[0,"pysnptools.util.snp_gen",false]],"snpdata (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpData",false]],"snpgen (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpGen",false]],"snphdf5 (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpHdf5",false]],"snpkernel (class in pysnptools.kernelreader)":[[0,"pysnptools.kernelreader.SnpKernel",false]],"snpmemmap (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpMemMap",false]],"snpnpz (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpNpz",false]],"snpreader (class in pysnptools.snpreader)":[[0,"pysnptools.snpreader.SnpReader",false]],"standardize() (pysnptools.kernelreader.kerneldata method)":[[0,"pysnptools.kernelreader.KernelData.standardize",false]],"standardize() (pysnptools.kernelstandardizer.kernelstandardizer method)":[[0,"pysnptools.kernelstandardizer.KernelStandardizer.standardize",false]],"standardize() (pysnptools.snpreader.snpdata method)":[[0,"pysnptools.snpreader.SnpData.standardize",false]],"standardize() (pysnptools.standardizer.standardizer method)":[[0,"pysnptools.standardizer.Standardizer.standardize",false]],"standardizer (class in pysnptools.standardizer)":[[0,"pysnptools.standardizer.Standardizer",false]],"sub_matrix() (in module pysnptools.util)":[[0,"pysnptools.util.sub_matrix",false]],"sum() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.sum",false]],"symmetric_difference() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.symmetric_difference",false]],"symmetric_difference_update() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.symmetric_difference_update",false]],"train_standardizer() (pysnptools.snpreader.snpdata method)":[[0,"pysnptools.snpreader.SnpData.train_standardizer",false]],"union() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.union",false]],"unit (class in pysnptools.standardizer)":[[0,"pysnptools.standardizer.Unit",false]],"unittrained (class in pysnptools.standardizer)":[[0,"pysnptools.standardizer.UnitTrained",false]],"update() (pysnptools.util.intrangeset method)":[[0,"pysnptools.util.IntRangeSet.update",false]],"val (pysnptools.distreader.distdata property)":[[0,"pysnptools.distreader.DistData.val",false]],"val (pysnptools.distreader.distmemmap property)":[[0,"pysnptools.distreader.DistMemMap.val",false]],"val (pysnptools.kernelreader.kerneldata property)":[[0,"pysnptools.kernelreader.KernelData.val",false]],"val (pysnptools.pstreader.pstdata property)":[[0,"pysnptools.pstreader.PstData.val",false]],"val (pysnptools.pstreader.pstmemmap property)":[[0,"pysnptools.pstreader.PstMemMap.val",false]],"val (pysnptools.snpreader.snpdata property)":[[0,"pysnptools.snpreader.SnpData.val",false]],"val (pysnptools.snpreader.snpmemmap property)":[[0,"pysnptools.snpreader.SnpMemMap.val",false]],"val_shape (pysnptools.distreader.distreader property)":[[0,"pysnptools.distreader.DistReader.val_shape",false]],"val_shape (pysnptools.kernelreader.kernelreader property)":[[0,"pysnptools.kernelreader.KernelReader.val_shape",false]],"val_shape (pysnptools.pstreader.pstdata property)":[[0,"pysnptools.pstreader.PstData.val_shape",false]],"val_shape (pysnptools.snpreader.snpreader property)":[[0,"pysnptools.snpreader.SnpReader.val_shape",false]],"walk() (pysnptools.util.filecache.filecache method)":[[0,"pysnptools.util.filecache.FileCache.walk",false]],"weighted_mean() (in module pysnptools.util)":[[0,"pysnptools.util.weighted_mean",false]],"weighted_simple_linear_regression() (in module pysnptools.util)":[[0,"pysnptools.util.weighted_simple_linear_regression",false]],"write() (pysnptools.distreader.bgen static method)":[[0,"pysnptools.distreader.Bgen.write",false]],"write() (pysnptools.distreader.disthdf5 static method)":[[0,"pysnptools.distreader.DistHdf5.write",false]],"write() (pysnptools.distreader.distmemmap static method)":[[0,"pysnptools.distreader.DistMemMap.write",false]],"write() (pysnptools.distreader.distnpz static method)":[[0,"pysnptools.distreader.DistNpz.write",false]],"write() (pysnptools.kernelreader.kernelhdf5 static method)":[[0,"pysnptools.kernelreader.KernelHdf5.write",false]],"write() (pysnptools.kernelreader.kernelnpz static method)":[[0,"pysnptools.kernelreader.KernelNpz.write",false]],"write() (pysnptools.pstreader.psthdf5 static method)":[[0,"pysnptools.pstreader.PstHdf5.write",false]],"write() (pysnptools.pstreader.pstmemmap static method)":[[0,"pysnptools.pstreader.PstMemMap.write",false]],"write() (pysnptools.pstreader.pstnpz static method)":[[0,"pysnptools.pstreader.PstNpz.write",false]],"write() (pysnptools.snpreader.bed static method)":[[0,"pysnptools.snpreader.Bed.write",false]],"write() (pysnptools.snpreader.dat static method)":[[0,"pysnptools.snpreader.Dat.write",false]],"write() (pysnptools.snpreader.dense static method)":[[0,"pysnptools.snpreader.Dense.write",false]],"write() (pysnptools.snpreader.distributedbed static method)":[[0,"pysnptools.snpreader.DistributedBed.write",false]],"write() (pysnptools.snpreader.ped static method)":[[0,"pysnptools.snpreader.Ped.write",false]],"write() (pysnptools.snpreader.pheno static method)":[[0,"pysnptools.snpreader.Pheno.write",false]],"write() (pysnptools.snpreader.snphdf5 static method)":[[0,"pysnptools.snpreader.SnpHdf5.write",false]],"write() (pysnptools.snpreader.snpmemmap static method)":[[0,"pysnptools.snpreader.SnpMemMap.write",false]],"write() (pysnptools.snpreader.snpnpz static method)":[[0,"pysnptools.snpreader.SnpNpz.write",false]]},"objects":{"":[[0,0,0,"-","pysnptools"]],"pysnptools":[[0,0,0,"-","distreader"],[0,0,0,"-","kernelreader"],[0,0,0,"-","pstreader"],[0,0,0,"-","snpreader"],[0,0,0,"-","standardizer"],[0,0,0,"-","util"]],"pysnptools.distreader":[[0,1,1,"","Bgen"],[0,1,1,"","DistData"],[0,1,1,"","DistGen"],[0,1,1,"","DistHdf5"],[0,1,1,"","DistMemMap"],[0,1,1,"","DistNpz"],[0,1,1,"","DistReader"]],"pysnptools.distreader.Bgen":[[0,2,1,"","flush"],[0,2,1,"","genwrite"],[0,2,1,"","write"]],"pysnptools.distreader.DistData":[[0,2,1,"","allclose"],[0,3,1,"","val"]],"pysnptools.distreader.DistHdf5":[[0,2,1,"","write"]],"pysnptools.distreader.DistMemMap":[[0,2,1,"","empty"],[0,3,1,"","filename"],[0,2,1,"","flush"],[0,3,1,"","offset"],[0,3,1,"","val"],[0,2,1,"","write"]],"pysnptools.distreader.DistNpz":[[0,2,1,"","write"]],"pysnptools.distreader.DistReader":[[0,2,1,"","as_snp"],[0,3,1,"","iid"],[0,3,1,"","iid_count"],[0,2,1,"","iid_to_index"],[0,3,1,"","pos"],[0,2,1,"","read"],[0,3,1,"","row_property"],[0,3,1,"","sid"],[0,3,1,"","sid_count"],[0,2,1,"","sid_to_index"],[0,3,1,"","val_shape"]],"pysnptools.distreader.bgen":[[0,4,1,"","default_id_rsid_function"],[0,4,1,"","default_iid_function"],[0,4,1,"","default_sample_function"],[0,4,1,"","default_sid_function"]],"pysnptools.kernelreader":[[0,1,1,"","Identity"],[0,1,1,"","KernelData"],[0,1,1,"","KernelHdf5"],[0,1,1,"","KernelNpz"],[0,1,1,"","KernelReader"],[0,1,1,"","SnpKernel"]],"pysnptools.kernelreader.KernelData":[[0,2,1,"","allclose"],[0,2,1,"","standardize"],[0,3,1,"","val"]],"pysnptools.kernelreader.KernelHdf5":[[0,2,1,"","write"]],"pysnptools.kernelreader.KernelNpz":[[0,2,1,"","write"]],"pysnptools.kernelreader.KernelReader":[[0,3,1,"","iid"],[0,3,1,"","iid0"],[0,3,1,"","iid0_count"],[0,2,1,"","iid0_to_index"],[0,3,1,"","iid1"],[0,3,1,"","iid1_count"],[0,2,1,"","iid1_to_index"],[0,3,1,"","iid_count"],[0,2,1,"","iid_to_index"],[0,2,1,"","read"],[0,3,1,"","val_shape"]],"pysnptools.kernelreader.SnpKernel":[[0,3,1,"","pos"],[0,2,1,"","read_snps"],[0,3,1,"","sid"],[0,3,1,"","sid_count"]],"pysnptools.kernelstandardizer":[[0,1,1,"","Identity"],[0,1,1,"","KernelStandardizer"]],"pysnptools.kernelstandardizer.KernelStandardizer":[[0,2,1,"","standardize"]],"pysnptools.pstreader":[[0,1,1,"","PstData"],[0,1,1,"","PstHdf5"],[0,1,1,"","PstMemMap"],[0,1,1,"","PstNpz"],[0,1,1,"","PstReader"]],"pysnptools.pstreader.PstData":[[0,2,1,"","allclose"],[0,3,1,"","val"],[0,3,1,"","val_shape"]],"pysnptools.pstreader.PstHdf5":[[0,2,1,"","flush"],[0,2,1,"","write"]],"pysnptools.pstreader.PstMemMap":[[0,2,1,"","empty"],[0,3,1,"","filename"],[0,2,1,"","flush"],[0,3,1,"","offset"],[0,3,1,"","val"],[0,2,1,"","write"]],"pysnptools.pstreader.PstNpz":[[0,2,1,"","write"]],"pysnptools.pstreader.PstReader":[[0,3,1,"","col"],[0,3,1,"","col_count"],[0,3,1,"","col_property"],[0,2,1,"","col_to_index"],[0,2,1,"","read"],[0,3,1,"","row"],[0,3,1,"","row_count"],[0,3,1,"","row_property"],[0,2,1,"","row_to_index"],[0,3,1,"","shape"]],"pysnptools.snpreader":[[0,1,1,"","Bed"],[0,1,1,"","Dat"],[0,1,1,"","Dense"],[0,1,1,"","DistributedBed"],[0,1,1,"","Hdf5"],[0,1,1,"","Ped"],[0,1,1,"","Pheno"],[0,1,1,"","SnpData"],[0,1,1,"","SnpGen"],[0,1,1,"","SnpHdf5"],[0,1,1,"","SnpMemMap"],[0,1,1,"","SnpNpz"],[0,1,1,"","SnpReader"]],"pysnptools.snpreader.Bed":[[0,3,1,"","col"],[0,3,1,"","col_property"],[0,2,1,"","copyinputs"],[0,3,1,"","row"],[0,2,1,"","write"]],"pysnptools.snpreader.Dat":[[0,2,1,"","copyinputs"],[0,2,1,"","write"]],"pysnptools.snpreader.Dense":[[0,2,1,"","write"]],"pysnptools.snpreader.DistributedBed":[[0,3,1,"","col"],[0,3,1,"","col_property"],[0,2,1,"","copyinputs"],[0,3,1,"","row"],[0,2,1,"","write"]],"pysnptools.snpreader.Ped":[[0,2,1,"","copyinputs"],[0,2,1,"","write"]],"pysnptools.snpreader.Pheno":[[0,2,1,"","write"]],"pysnptools.snpreader.SnpData":[[0,2,1,"","allclose"],[0,2,1,"","standardize"],[0,2,1,"","train_standardizer"],[0,3,1,"","val"]],"pysnptools.snpreader.SnpGen":[[0,3,1,"","col"],[0,3,1,"","col_property"],[0,2,1,"","copyinputs"],[0,3,1,"","row"]],"pysnptools.snpreader.SnpHdf5":[[0,3,1,"","col"],[0,3,1,"","row"],[0,2,1,"","write"]],"pysnptools.snpreader.SnpMemMap":[[0,2,1,"","empty"],[0,3,1,"","filename"],[0,2,1,"","flush"],[0,3,1,"","offset"],[0,3,1,"","val"],[0,2,1,"","write"]],"pysnptools.snpreader.SnpNpz":[[0,3,1,"","col"],[0,3,1,"","row"],[0,2,1,"","write"]],"pysnptools.snpreader.SnpReader":[[0,2,1,"","as_dist"],[0,2,1,"","copyinputs"],[0,3,1,"","iid"],[0,3,1,"","iid_count"],[0,2,1,"","iid_to_index"],[0,2,1,"","kernel"],[0,3,1,"","pos"],[0,2,1,"","read"],[0,2,1,"","read_kernel"],[0,3,1,"","row_property"],[0,3,1,"","sid"],[0,3,1,"","sid_count"],[0,2,1,"","sid_to_index"],[0,3,1,"","val_shape"]],"pysnptools.standardizer":[[0,1,1,"","Beta"],[0,1,1,"","Identity"],[0,1,1,"","Standardizer"],[0,1,1,"","Unit"],[0,1,1,"","UnitTrained"]],"pysnptools.standardizer.Standardizer":[[0,3,1,"","is_constant"],[0,2,1,"","standardize"]],"pysnptools.util":[[0,1,1,"","IntRangeSet"],[0,4,1,"","array_module"],[0,4,1,"","asnumpy"],[0,4,1,"","create_directory_if_necessary"],[0,4,1,"","example_file"],[0,0,0,"-","filecache"],[0,4,1,"","format_delta"],[0,4,1,"","get_array_module"],[0,4,1,"","intersect_apply"],[0,4,1,"","intersect_ids"],[0,4,1,"","log_in_place"],[0,0,0,"-","mapreduce1"],[0,0,0,"-","pheno"],[0,4,1,"","print2"],[0,4,1,"","snp_gen"],[0,4,1,"","sub_matrix"],[0,4,1,"","weighted_mean"],[0,4,1,"","weighted_simple_linear_regression"]],"pysnptools.util.IntRangeSet":[[0,5,1,"","__annotate_func__"],[0,5,1,"","__annotations_cache__"],[0,2,1,"","__contains__"],[0,2,1,"","__delitem__"],[0,2,1,"","__eq__"],[0,5,1,"","__firstlineno__"],[0,2,1,"","__ge__"],[0,2,1,"","__getitem__"],[0,2,1,"","__gt__"],[0,5,1,"","__hash__"],[0,2,1,"","__iadd__"],[0,2,1,"","__iand__"],[0,2,1,"","__imul__"],[0,2,1,"","__init__"],[0,2,1,"","__ior__"],[0,2,1,"","__isub__"],[0,2,1,"","__iter__"],[0,2,1,"","__ixor__"],[0,2,1,"","__le__"],[0,2,1,"","__len__"],[0,2,1,"","__lt__"],[0,2,1,"","__mul__"],[0,2,1,"","__ne__"],[0,2,1,"","__or__"],[0,2,1,"","__repr__"],[0,2,1,"","__reversed__"],[0,5,1,"","__static_attributes__"],[0,2,1,"","__str__"],[0,2,1,"","__sub__"],[0,2,1,"","__xor__"],[0,2,1,"","add"],[0,2,1,"","clear"],[0,2,1,"","copy"],[0,2,1,"","count"],[0,2,1,"","difference"],[0,2,1,"","difference_update"],[0,2,1,"","discard"],[0,2,1,"","index"],[0,2,1,"","intersection"],[0,2,1,"","intersection_update"],[0,2,1,"","isdisjoint"],[0,3,1,"","isempty"],[0,2,1,"","issubset"],[0,2,1,"","issuperset"],[0,2,1,"","max"],[0,2,1,"","min"],[0,2,1,"","pop"],[0,2,1,"","ranges"],[0,2,1,"","ranges_getitem"],[0,2,1,"","ranges_index"],[0,3,1,"","ranges_len"],[0,2,1,"","remove"],[0,2,1,"","sum"],[0,2,1,"","symmetric_difference"],[0,2,1,"","symmetric_difference_update"],[0,2,1,"","union"],[0,2,1,"","update"]],"pysnptools.util.filecache":[[0,1,1,"","FileCache"],[0,1,1,"","Hashdown"],[0,1,1,"","LocalCache"],[0,1,1,"","PeerToPeer"],[0,4,1,"","ip_address"],[0,4,1,"","ip_address_pid"]],"pysnptools.util.filecache.FileCache":[[0,2,1,"","file_exists"],[0,2,1,"","getmtime"],[0,2,1,"","join"],[0,2,1,"","load"],[0,3,1,"","name"],[0,2,1,"","open_read"],[0,2,1,"","open_write"],[0,2,1,"","remove"],[0,2,1,"","rmtree"],[0,2,1,"","save"],[0,2,1,"","walk"]],"pysnptools.util.filecache.Hashdown":[[0,2,1,"","load_hashdown"],[0,3,1,"","name"],[0,2,1,"","save_hashdown"],[0,2,1,"","scan_local"]],"pysnptools.util.filecache.LocalCache":[[0,3,1,"","name"]],"pysnptools.util.filecache.PeerToPeer":[[0,3,1,"","name"]],"pysnptools.util.mapreduce1":[[0,4,1,"","map_reduce"]],"pysnptools.util.mapreduce1.runner":[[0,1,1,"","Local"],[0,1,1,"","LocalInParts"],[0,1,1,"","LocalMultiProc"],[0,1,1,"","LocalMultiThread"],[0,1,1,"","Runner"]],"pysnptools.util.pheno":[[0,4,1,"","loadOnePhen"],[0,4,1,"","loadPhen"]]},"objnames":{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","property","Python property"],"4":["py","function","Python function"],"5":["py","attribute","Python attribute"]},"objtypes":{"0":"py:module","1":"py:class","2":"py:method","3":"py:property","4":"py:function","5":"py:attribute"},"terms":{"100x500":0,"16th":0,"1_10":0,"1_11":0,"1_12":0,"1_13":0,"1_17":0,"1_25":0,"1_28":0,"1_32":0,"1_34":0,"1_35":0,"1_36":0,"1_39":0,"1_4":0,"1_9":0,"1st":0,"200k":0,"201k":0,"23_":0,"2500x100":0,"2d":0,"2nd":0,"3d":0,"3rd":0,"5chrom":0,"6a07f96e521f9a86df7bfd7814eebcd6":0,"6f":0,"766f55aa716bc7bc97cad4de41a50ec3":0,"9ac30da2bf589db947e91744dff0ec24":0,"A":0,"AND":0,"All":0,"An":0,"Be":0,"Both":0,"By":0,"Do":0,"Doing":0,"Each":0,"For":0,"From":0,"Here":0,"How":0,"If":0,"In":0,"It":0,"Its":0,"More":0,"Most":0,"No":0,"Not":0,"Other":0,"Some":0,"TO":0,"That":0,"The":0,"Their":0,"Then":0,"These":0,"They":0,"This":0,"To":0,"We":0,"What":0,"When":0,"Where":0,"Which":0,"While":0,"You":0,"_":0,"__annotate_func__":0,"__annotations_cache__":0,"__contains__":0,"__delitem__":0,"__eq__":0,"__firstlineno__":0,"__ge__":0,"__getitem__":0,"__gt__":0,"__hash__":0,"__iadd__":0,"__iand__":0,"__imul__":0,"__init__":0,"__ior__":0,"__isub__":0,"__iter__":0,"__ixor__":0,"__le__":0,"__len__":0,"__lt__":0,"__mul__":0,"__name__":0,"__ne__":0,"__or__":0,"__repr__":0,"__reversed__":0,"__static_attributes__":0,"__str__":0,"__sub__":0,"__xor__":0,"_ident":0,"_relative_directori":0,"_require_float32_64":0,"_start_item":0,"_start_to_length":0,"a1":0,"a2":0,"aa":0,"ab":0,"abov":0,"accept":0,"access":0,"accord":0,"across":0,"actual":0,"add":0,"addit":0,"address":0,"adjust":0,"advantag":0,"all_chr":0,"allclos":0,"allel":0,"alloc":0,"allow":0,"allow_unknown_fil":0,"allowlowrank":0,"almost":0,"along":0,"alreadi":0,"also":0,"altern":0,"although":0,"alway":0,"among":0,"amount":0,"ani":0,"announc":0,"anoth":0,"anyth":0,"anyway":0,"appear":0,"append":0,"appli":0,"apply_in_plac":0,"appropri":0,"arbitrari":0,"arg":0,"argument":0,"around":0,"array":0,"array_equ":0,"array_modul":0,"as_dist":0,"as_snp":0,"ask":0,"asnumpi":0,"assert":0,"assign":0,"associ":0,"assort":0,"assum":0,"astyp":0,"automat":0,"avail":0,"avoid":0,"b":0,"back":0,"backend":0,"backward":0,"base":0,"basepair":0,"basicconfig":0,"batch":0,"bb":0,"becaus":0,"becom":0,"bed_fil":0,"bed_fn":0,"bedfil":0,"bedfile2":0,"befor":0,"behavior":0,"behind":0,"belong":0,"beyond":0,"bgen_fil":0,"biallel":0,"big":0,"bim":0,"bim_filenam":0,"binari":0,"bit":0,"block":0,"block_siz":0,"blocksiz":0,"bmi":0,"bool":0,"boolean":0,"boolindex":0,"bootstrap":0,"bound":0,"break":0,"built":0,"byte":0,"c":0,"cach":0,"cache_fil":0,"call":0,"can":0,"care":0,"case":0,"caus":0,"cf248cbf762516540470d693532590a77c76fba2":0,"chanc":0,"chang":0,"char":0,"charact":0,"check":0,"choic":0,"chr":0,"chr15":0,"chr_count":0,"chrom_count":0,"chrom_map":0,"chromosom":0,"chromsom":0,"cid":0,"cif":0,"class":0,"clean":0,"cleanup_temp_fil":0,"clear":0,"close":0,"cluster":0,"code":0,"col":0,"col_count":0,"col_index":0,"col_index_list":0,"col_major":0,"col_properti":0,"col_to_index":0,"collect":0,"column":0,"column01":0,"column201":0,"com":0,"comma":0,"command":0,"common":0,"common_directori":0,"compar":0,"compat":0,"complex":0,"compress":0,"comput":0,"confer":0,"connect":0,"consequ":0,"consid":0,"consist":0,"constant":0,"construct":0,"constructor":0,"contact":0,"contain":0,"content":0,"contigu":0,"contrast":0,"control":0,"conveni":0,"convert":0,"copi":0,"copier":0,"copyinput":0,"copyinputs_funct":0,"core":0,"correct":0,"correspond":0,"count":0,"count_a1":0,"cov":0,"cov_as_tuple_in":0,"cov_as_tuple_out":0,"covari":0,"cover":0,"covfil":0,"cpu":0,"creat":0,"create_directory_if_necessari":0,"cupi":0,"current":0,"d":0,"dat_fil":0,"data":0,"data1":0,"data2":0,"data_list":0,"data_on_disk":0,"data_sixteenth":0,"data_subset":0,"dataset":0,"date":0,"day":0,"debug":0,"decid":0,"decim":0,"decimal_plac":0,"deep":0,"def":0,"default":0,"default_id_rsid_funct":0,"default_iid_funct":0,"default_sample_funct":0,"default_sid_funct":0,"defin":0,"definit":0,"degre":0,"del":0,"delay":0,"delet":0,"delta":0,"delta_second":0,"demo":0,"dense1":0,"dense2":0,"dense_fil":0,"depend":0,"deprec":0,"deprecated_iid_count":0,"describ":0,"descript":0,"desir":0,"detail":0,"determin":0,"determinist":0,"dev":0,"develop":0,"deviat":0,"dfr":0,"diag":0,"diagon":0,"dictionari":0,"differ":0,"difference_upd":0,"dimens":0,"dimension":0,"diploid":0,"direct":0,"directori":0,"dirnam":0,"discard":0,"discontigu":0,"discuss":0,"disjoint":0,"disk":0,"display":0,"dist":0,"dist_data":0,"dist_gen":0,"dist_mem_map":0,"dist_on_disk":0,"distanc":0,"distdata1":0,"distdata_subset":0,"distribut":0,"distributed_bed_test1_x":0,"divid":0,"doe":0,"doesn":0,"don":0,"done":0,"download":0,"downsid":0,"drive":0,"ds":0,"dtype":0,"e":0,"earlier":0,"edit":0,"effect":0,"effici":0,"either":0,"element":0,"email":0,"empti":0,"encod":0,"end":0,"endswith":0,"enorm":0,"entri":0,"environ":0,"epoch":0,"equal":0,"equal_nan":0,"equival":0,"eras":0,"error":0,"etc":0,"even":0,"ever":0,"everi":0,"everyoth":0,"everyth":0,"exact":0,"exampl":0,"example_fil":0,"except":0,"exclud":0,"exclus":0,"execut":0,"exist":0,"exit":0,"exon":0,"exon_last":0,"exon_start":0,"exon_stop":0,"expect":0,"express":0,"extern":0,"extra":0,"extract":0,"extract_iid_funct":0,"extract_sid_pos_funct":0,"extractor":0,"f":0,"f1":0,"f2":0,"f4":0,"f4eb01f67e0738d4865fad2014af8537":0,"f8":0,"factor":0,"fail":0,"fals":0,"fam":0,"fam0":0,"fam001":0,"fam_filenam":0,"famid":0,"famili":0,"fashion":0,"fast":0,"faster":0,"fastest":0,"fastlmm":0,"feed":0,"fewer":0,"fid":0,"field":0,"file":0,"file1":0,"file_cach":0,"file_exist":0,"file_nam":0,"file_to_hash":0,"filenam":0,"filename_without_gen":0,"fill":0,"final":0,"find":0,"fine":0,"first":0,"fit":0,"five":0,"flexibl":0,"fli":0,"float":0,"float16":0,"float32":0,"float64":0,"flush":0,"folder":0,"follow":0,"force_python_on":0,"form":0,"format":0,"format_delta":0,"fortran":0,"found":0,"four":0,"fp":0,"fracsib":0,"fraction":0,"free":0,"freq_pop_0":0,"frequenc":0,"fresh_properti":0,"fst":0,"full":0,"function":0,"futur":0,"g":0,"gen":0,"gene":0,"general":0,"generat":0,"genet":0,"genwrit":0,"get":0,"get_array_modul":0,"getmtim":0,"github":0,"give":0,"given":0,"global":0,"goe":0,"good":0,"got":0,"gpu":0,"half":0,"half_read":0,"half_snpread":0,"hand":0,"happen":0,"hash":0,"hashdown1":0,"hashdown2":0,"hd5f_file":0,"hdf5":0,"hdf5_dtype":0,"hdf5_file":0,"hdf5_file2":0,"head":0,"header":0,"hello":0,"helper":0,"hidden":0,"high":0,"highest":0,"hold":0,"holder1":0,"home":0,"howev":0,"https":0,"i_pheno":0,"id":0,"id_and_path_funct":0,"id_rsid_funct":0,"identifi":0,"idslist":0,"ie":0,"ignor":0,"ignore_in":0,"ignore_out":0,"iid":0,"iid0":0,"iid0_count":0,"iid0_index":0,"iid0_to_index":0,"iid1":0,"iid1_count":0,"iid1_index":0,"iid1_to_index":0,"iid2":0,"iid3":0,"iid_0":0,"iid_1":0,"iid_2":0,"iid_count":0,"iid_funct":0,"iid_if_non":0,"iid_index":0,"iid_major":0,"iid_to_index":0,"iidmajor":0,"illeg":0,"immedi":0,"implement":0,"import":0,"includ":0,"inclus":0,"increas":0,"ind0":0,"indarr":0,"independ":0,"index":0,"indexerror":0,"indid":0,"individu":0,"ineffici":0,"info":0,"inform":0,"inherit":0,"input":0,"input_fil":0,"input_seq":0,"instal":0,"instanc":0,"instead":0,"instruct":0,"int":0,"int8":0,"int_":0,"int_range_set":0,"integ":0,"intercept":0,"interest":0,"interfac":0,"internal":0,"interpret":0,"intersect":0,"intersect_appli":0,"intersect_before_standard":0,"intersect_id":0,"intersection_upd":0,"interval":0,"intrangeset__isub__":0,"intron":0,"ip":0,"ip_address":0,"ip_address_pid":0,"ipaddress":0,"ipython":0,"is_const":0,"is_test":0,"isdisjoint":0,"isempti":0,"isfil":0,"issu":0,"issubset":0,"issuperset":0,"iter":0,"ith":0,"j":0,"job":0,"join":0,"join_iid_funct":0,"join_sid_pos_funct":0,"json":0,"just":0,"just_cas":0,"just_one_process":0,"just_sid":0,"keep":0,"kernel":0,"kernel1":0,"kernel_fil":0,"kernel_in":0,"kernel_on_disk":0,"kernel_out":0,"kernel_train":0,"kernel_whol":0,"kernel_whole_test":0,"kerneldata1":0,"kerneldata2":0,"kerneldata3":0,"kerneldata4":0,"kerneldata5":0,"kerneldata6":0,"kerneldata_subset":0,"key":0,"keyerror":0,"kind":0,"know":0,"knowabl":0,"known":0,"ks_ident":0,"kwarg":0,"l":0,"label_with_pop":0,"laid":0,"lambda":0,"larg":0,"largest":0,"last":0,"later":0,"learn":0,"leave_spac":0,"left":0,"legal":0,"len":0,"length":0,"less":0,"level":0,"librari":0,"like":0,"likewis":0,"limit":0,"line":0,"linear":0,"list":0,"littl":0,"ll":0,"load":0,"load_hashdown":0,"loadonephen":0,"loadphen":0,"local_directori":0,"local_filenam":0,"localcache1":0,"localmultproc":0,"locat":0,"lock":0,"log":0,"log_in_plac":0,"logging_handl":0,"logging_level":0,"long":0,"look":0,"loop":0,"low":0,"lower":0,"machin":0,"maf0":0,"maf_high":0,"maf_low":0,"main":0,"major":0,"make":0,"manag":0,"mani":0,"manipul":0,"manual":0,"map":0,"map_reduc":0,"mapper":0,"mapper1":0,"match":0,"matrix":0,"max":0,"max_weight":0,"maximum":0,"may":0,"may_share_memori":0,"md":0,"md5":0,"mean":0,"measur":0,"mechan":0,"mem":0,"mem_map_fil":0,"memmap":0,"memori":0,"messag":0,"met":0,"method":0,"might":0,"min":0,"mind":0,"minor":0,"miss":0,"mkl":0,"mkl_num_thread":0,"modifi":0,"moreov":0,"mt":0,"much":0,"multi":0,"multidimension":0,"multipl":0,"multipli":0,"must":0,"n":0,"n300":0,"name":0,"namesv":0,"nan":0,"ndarray":0,"necessari":0,"need":0,"negat":0,"nest":0,"network":0,"never":0,"new":0,"newli":0,"nice":0,"nine":0,"no_such_sub_path":0,"non":0,"none":0,"normal":0,"notat":0,"note":0,"notebook":0,"noth":0,"notic":0,"notset":0,"now":0,"np":0,"npz":0,"npz_file":0,"npzreader":0,"null_0":0,"null_1":0,"null_100":0,"null_2":0,"null_200":0,"null_3":0,"null_4":0,"null_5":0,"null_6":0,"null_7":0,"num_thread":0,"number":0,"numer":0,"numpi":0,"object":0,"offer":0,"offset":0,"often":0,"ok":0,"older":0,"on_disk":0,"onc":0,"one":0,"onli":0,"open":0,"open_read":0,"open_writ":0,"oper":0,"optim":0,"optin":0,"option":0,"order":0,"org":0,"origin":0,"os":0,"others":0,"otherwis":0,"output":0,"output_fil":0,"outsid":0,"overlap":0,"overrid":0,"p":0,"page":0,"pair":0,"panda":0,"paper":0,"parallel":0,"paramet":0,"parent_str":0,"pars":0,"part":0,"parti":0,"partial":0,"path":0,"pattern":0,"ped_fil":0,"peer":0,"peertopeer1":0,"peopl":0,"per":0,"per0":0,"per1":0,"per2":0,"perform":0,"permut":0,"persist":0,"phe":0,"pheno_fil":0,"pheno_fn":0,"pheno_in":0,"pheno_out":0,"phenofil":0,"phenotyp":0,"phensynthfrom22":0,"pick":0,"pickl":0,"piec":0,"piece_per_chrom_count":0,"place":0,"plink":0,"plink_chrom_map":0,"plus":0,"pop":0,"pop1":0,"popul":0,"pos":0,"posit":0,"possibl":0,"postpon":0,"practic":0,"pre":0,"precis":0,"prepar":0,"print":0,"print2":0,"prioriti":0,"probabl":0,"problem":0,"process":0,"processor":0,"produc":0,"program":0,"progress":0,"project":0,"promis":0,"proper":0,"properti":0,"provid":0,"pst":0,"pst_mem_map":0,"pst_mem_map_fil":0,"pst_num_thread":0,"pstdata1":0,"pstdata2":0,"pstdata3":0,"pstdata4":0,"pstdata5":0,"pstdata6":0,"psthdf5_file":0,"pstnpz_file":0,"pstnz":0,"pstutil":0,"pure":0,"put":0,"pydata":0,"pypi":0,"pysnptools_cache_hom":0,"python":0,"python2":0,"python3":0,"pythonesqu":0,"qctool":0,"qctool_path":0,"qctoolpath":0,"qt1":0,"qualif":0,"quarter_read":0,"quarter_snpread":0,"r":0,"r123":0,"rais":0,"rand":0,"randcidord":0,"random":0,"rang":0,"ranges_getitem":0,"ranges_index":0,"ranges_input":0,"ranges_len":0,"rather":0,"raw":0,"re":0,"read":0,"read_kernel":0,"read_snp":0,"readabl":0,"reader":0,"readlin":0,"readm":0,"recommend":0,"recurs":0,"reduc":0,"reducer1":0,"refer":0,"region":0,"regular":0,"relat":0,"related":0,"releas":0,"reli":0,"remain":0,"rememb":0,"remot":0,"remov":0,"reopen":0,"reorder":0,"replac":0,"report":0,"repr":0,"repres":0,"represent":0,"request":0,"requir":0,"respect":0,"rest":0,"restrict":0,"result":0,"result_dir":0,"result_fil":0,"retriev":0,"return":0,"return_train":0,"revers":0,"reverse_chrom_map":0,"reverse_plink_chrom_map":0,"revert":0,"right":0,"rmtree":0,"robust":0,"room":0,"root":0,"round":0,"row":0,"row_count":0,"row_index":0,"row_index_list":0,"row_major":0,"row_properti":0,"row_to_index":0,"rs102343":0,"rs12184279":0,"rs12354060":0,"rs12564807":0,"rsid":0,"run":0,"run_dir":0,"rust":0,"s":0,"safe":0,"sampid":0,"sampl":0,"sample_funct":0,"save":0,"save_hashdown":0,"scalar":0,"scan_loc":0,"scene":0,"scheme":0,"search":0,"seattl":0,"second":0,"see":0,"seed":0,"select":0,"self":0,"selv":0,"sep":0,"separ":0,"sequenc":0,"sequenti":0,"set":0,"seven":0,"shallow":0,"shape":0,"share":0,"show_log_diff":0,"shutil":0,"sibl":0,"sibs_per_famili":0,"sid":0,"sid_0":0,"sid_1":0,"sid_2":0,"sid_3":0,"sid_4":0,"sid_5":0,"sid_6":0,"sid_7":0,"sid_8":0,"sid_9":0,"sid_batch_s":0,"sid_count":0,"sid_funct":0,"sid_index":0,"sid_major":0,"sid_to_index":0,"side":0,"sign":0,"similar":0,"simplest":0,"sinc":0,"singl":0,"site":0,"sixteenth_read":0,"sixteenth_snpread":0,"size":0,"skip":0,"skip_format_check":0,"skiprow":0,"sleep":0,"slice":0,"slide":0,"slope":0,"smallest":0,"snp":0,"snp1":0,"snp334":0,"snp349":0,"snp921":0,"snp_data":0,"snp_gen":0,"snp_mem_map":0,"snp_on_disk":0,"snp_test":0,"snp_train":0,"snp_whole":0,"snpdata1":0,"snpdata2":0,"snpdata3":0,"snpdata4":0,"snpdata5":0,"snpdata6":0,"snpdata_int":0,"snpdata_sixteenth":0,"snpdata_subset":0,"snpmajor":0,"snps":0,"someth":0,"sort":0,"sort_by_dataset":0,"sourc":0,"space":0,"special":0,"specif":0,"specifi":0,"split":0,"squar":0,"ssds":0,"stand":0,"start":0,"startswith":0,"stat":0,"statement":0,"static":0,"statist":0,"status":0,"stay":0,"stddev":0,"stdout":0,"step":0,"still":0,"stop":0,"storag":0,"store":0,"str":0,"stream":0,"streamhandl":0,"string":0,"strip":0,"structur":0,"sub":0,"sub1":0,"sub_matrix":0,"subdirectori":0,"submatrix":0,"subset":0,"subset_distdata":0,"subset_kerneldata":0,"subset_on_disk":0,"subset_on_disk1":0,"subset_on_disk2":0,"subset_pstdata":0,"subset_reader_1":0,"subset_reader_2":0,"subset_reader_3":0,"subset_reader_4":0,"subset_reader_5":0,"subset_snpdata":0,"subset_snpreader_1":0,"subset_snpreader_2":0,"subset_snpreader_3":0,"subset_snpreader_4":0,"subset_snpreader_5":0,"subsub_distdata":0,"subsub_kerneldata":0,"subsub_pstdata":0,"subsub_snpdata":0,"subtract":0,"suffix":0,"suggest":0,"sum":0,"summar":0,"superset":0,"support":0,"suppos":0,"symmetr":0,"symmetric_differ":0,"symmetric_difference_upd":0,"synopsi":0,"system":0,"t":0,"take":0,"task":0,"taskcount":0,"taskindex":0,"taskindex_to_environ":0,"tell":0,"temp":0,"temp_dir":0,"tempdir":0,"temporari":0,"temporarili":0,"ten":0,"test":0,"test_idx":0,"text":0,"th":0,"themselv":0,"thing":0,"thread":0,"threat":0,"three":0,"throw":0,"thrown":0,"thus":0,"time":0,"time_lambda":0,"tini":0,"tip":0,"togeth":0,"tool":0,"top":0,"toydata":0,"toydata10":0,"toydata100":0,"toydataskip10":0,"train":0,"train_idx":0,"train_standard":0,"trained_standard":0,"trans_last":0,"trans_start":0,"trans_stop":0,"transcript":0,"transfer":0,"transpos":0,"treat":0,"tree":0,"tri":0,"tripl":0,"true":0,"trust_local_fil":0,"tupl":0,"turn":0,"tutori":0,"twice":0,"two":0,"txt":0,"type":0,"typic":0,"ubuntu":0,"unchang":0,"understood":0,"unicod":0,"unifi":0,"uniform":0,"uninterest":0,"union":0,"uniqu":0,"unit_train":0,"unknown":0,"unless":0,"unload":0,"unnorm":0,"unphas":0,"unwant":0,"updat":0,"upgrad":0,"upload":0,"upper":0,"url":0,"use":0,"user":0,"usual":0,"val":0,"val1":0,"val_shap":0,"valu":0,"var":0,"vari":0,"variabl":0,"various":0,"vector":0,"versa":0,"version":0,"vfs":0,"via":0,"video":0,"view":0,"view_ok":0,"virtual":0,"visa":0,"w":0,"walk":0,"want":0,"warn":0,"way":0,"web":0,"weight":0,"weighted_mean":0,"weighted_simple_linear_regress":0,"whether":0,"white":0,"whole":0,"width":0,"will":0,"window":0,"wish":0,"within":0,"without":0,"won":0,"word":0,"work":0,"working_directori":0,"writabl":0,"write":0,"written":0,"wrong":0,"x":0,"xmean":0,"xp":0,"xs":0,"xy":0,"y":0,"yes":0,"yet":0,"ymean":0,"ys":0,"z":0,"zero":0,"zero_famili":0,"zero_po":0,"zeros_np":0,"zeros_xp":0,"zip":0,"zlib":0,"zstd":0},"titles":["pysnptools Documentation"],"titleterms":{"bed":0,"beta":0,"bgen":0,"dat":0,"dens":0,"diagkton":0,"diagktontrain":0,"distdata":0,"distgen":0,"disthdf5":0,"distmemmap":0,"distnpz":0,"distread":0,"distributedb":0,"document":0,"filecach":0,"hashdown":0,"ident":0,"indic":0,"intrangeset":0,"kerneldata":0,"kernelhdf5":0,"kernelnpz":0,"kernelread":0,"kernelstandard":0,"local":0,"localcach":0,"localinpart":0,"localmultiproc":0,"localmultithread":0,"mapreduce1":0,"modul":0,"ped":0,"peertop":0,"pheno":0,"pstdata":0,"psthdf5":0,"pstmemmap":0,"pstnpz":0,"pstreader":0,"pysnptool":0,"runner":0,"snpdata":0,"snpgen":0,"snphdf5":0,"snpkernel":0,"snpmemmap":0,"snpnpz":0,"snpreader":0,"standard":0,"tabl":0,"unit":0,"unittrain":0,"util":0}}) \ No newline at end of file diff --git a/useful.md b/useful.md index f9255119..94b6f37a 100644 --- a/useful.md +++ b/useful.md @@ -13,7 +13,6 @@ pytest cd doc make.bat html build\html\index.html -xcopy /c /e /s /h build\html ..\docs cd .. ```