diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 412eeda..0000000 --- a/.gitattributes +++ /dev/null @@ -1,22 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=auto - -# Custom for Visual Studio -*.cs diff=csharp -*.sln merge=union -*.csproj merge=union -*.vbproj merge=union -*.fsproj merge=union -*.dbproj merge=union - -# Standard to msysgit -*.doc diff=astextplain -*.DOC diff=astextplain -*.docx diff=astextplain -*.DOCX diff=astextplain -*.dot diff=astextplain -*.DOT diff=astextplain -*.pdf diff=astextplain -*.PDF diff=astextplain -*.rtf diff=astextplain -*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore deleted file mode 100644 index cf8be89..0000000 --- a/.gitignore +++ /dev/null @@ -1,224 +0,0 @@ -################### -## project specific -################### -*.tosdb -prov/tmp* -python/tosdb/_tosdb.py -core -*.test - -README_API2.md - -################# -## intelliJ -################# -java/.idea/* -test/java/.idea/* -*.iml -java/out/* -java/examples/TimeAndSales/.idea/* -java/examples/TimeAndSales/out/* -.idea/* - -################# -## Java -################# -*.class - - -################# -## Eclipse -################# -.project -.pydevproject -.metadata/ - -################# -## Visual Studio -################# - -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. - -# User-specific files -*.suo -*.user -*.sln.docstates -*/.vs/ - -# Build results - -VisualStudioBuild/[Dd]ebug/ -VisualStudioBuild/[Rr]elease/ -build/ -bin/Debug -[Oo]bj/ -log/ -symbols/Debug/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -*_i.c -*_p.c -*.ilk -*.meta -*.obj -*.pch -#*.pdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.log -*.scc - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opensdf -*.sdf -*.cachefile - -# Visual Studio profiler -*.psess -*.vsp -*.vspx - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# NCrunch -*.ncrunch* -.*crunch*.local.xml - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.Publish.xml -*.pubxml - -# NuGet Packages Directory -## TODO: If you have NuGet Package Restore enabled, uncomment the next line -#packages/ - -# Windows Azure Build Output -csx -*.build.csdef - -# Windows Store app package directory -AppPackages/ - -# Others -sql/ -*.Cache -ClientBin/ -[Ss]tyle[Cc]op.* -~$* -*~ -*.dbmdl -*.[Pp]ublish.xml -*.pfx -*.publishsettings - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file to a newer -# Visual Studio version. Backup files are not needed, because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm - -# SQL Server files -App_Data/*.mdf -App_Data/*.ldf - -############# -## Windows detritus -############# - -# Windows image file caches -Thumbs.db -ehthumbs.db - -# Folder config file -Desktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Mac crap -.DS_Store - - -############# -## Python -############# - -*.py[co] - -# C Ext -*.pyd - -# Packages -*.egg -*.egg-info -dist/ -build/ -eggs/ -parts/ -var/ -sdist/ -develop-eggs/ -.installed.cfg - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.tox - -#Translations -*.mo - -#Mr Developer -.mr.developer.cfg diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt deleted file mode 100644 index e54d5ab..0000000 --- a/COPYRIGHT.txt +++ /dev/null @@ -1,15 +0,0 @@ -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. - diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 513ff4d..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,623 +0,0 @@ -GNU GENERAL PUBLIC LICENSE -Version 3, 29 June 2007 - -Copyright (C) 2007 Free Software Foundation, Inc. - -Everyone is permitted to copy and distribute verbatim copies -of this license document, but changing it is not allowed. - -Preamble - -The GNU General Public License is a free, copyleft license for -software and other kinds of works. - -The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - -When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - -To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - -For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - -Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - -For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - -Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - -Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - -The precise terms and conditions for copying, distribution and -modification follow. - -TERMS AND CONDITIONS - -0. Definitions. - -“This License” refers to version 3 of the GNU General Public License. - -“Copyright” also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - -“The Program” refers to any copyrightable work licensed under this -License. Each licensee is addressed as “you”. “Licensees” and -“recipients” may be individuals or organizations. - -To “modify” a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a “modified version” of the -earlier work or a work “based on” the earlier work. - -A “covered work” means either the unmodified Program or a work based -on the Program. - -To “propagate” a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - -To “convey” a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - -An interactive user interface displays “Appropriate Legal Notices” -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - -1. Source Code. - -The “source code” for a work means the preferred form of the work -for making modifications to it. “Object code” means any non-source -form of a work. - -A “Standard Interface” means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - -The “System Libraries” of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -“Major Component”, in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - -The “Corresponding Source” for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - -The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - -The Corresponding Source for a work in source code form is that -same work. - -2. Basic Permissions. - -All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - -You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - -Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - -3. Protecting Users' Legal Rights From Anti-Circumvention Law. - -No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - -When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - -4. Conveying Verbatim Copies. - -You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - -You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - -5. Conveying Modified Source Versions. - -You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - -a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - -b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - “keep intact all notices”. - -c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - -d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - -A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -“aggregate” if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - -6. Conveying Non-Source Forms. - -You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - -a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - -b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - -c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - -d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - -e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - -A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - -A “User Product” is either (1) a “consumer product”, which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, “normally used” refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - -“Installation Information” for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - -If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - -The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - -Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - -7. Additional Terms. - -“Additional permissions” are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - -When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - -Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - -a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - -b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - -c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - -d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - -e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - -f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - -All other non-permissive additional terms are considered “further -restrictions” within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - -If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - -Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - -8. Termination. - -You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - -However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - -Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - -Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - -9. Acceptance Not Required for Having Copies. - -You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - -10. Automatic Licensing of Downstream Recipients. - -Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - -An “entity transaction” is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - -You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - -11. Patents. - -A “contributor” is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's “contributor version”. - -A contributor's “essential patent claims” are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, “control” includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - -Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - -In the following three paragraphs, a “patent license” is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To “grant” such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - -If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. “Knowingly relying” means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - -If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - -A patent license is “discriminatory” if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - -Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - -12. No Surrender of Others' Freedom. - -If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - -13. Use with the GNU Affero General Public License. - -Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - -14. Revised Versions of this License. - -The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License “or any later version” applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - -If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - -Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - -15. Disclaimer of Warranty. - -THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -16. Limitation of Liability. - -IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - -17. Interpretation of Sections 15 and 16. - -If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - -END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md deleted file mode 100644 index 0e4797d..0000000 --- a/README.md +++ /dev/null @@ -1,278 +0,0 @@ -## TOSDataBridge -- - - -TOSDataBridge (TOSDB) is an open-source collection of resources for pulling real-time streaming data off of TDAmeritrade's ThinkOrSwim(TOS) platform, providing C, C++, Java and Python interfaces. - -TOSDB uses TOS's antiquated, yet still useful, DDE feature, directly through the Windows API. The C / C++ interfaces are implemented as a shared library that communicates with a backend Windows Service. - -![](./res/tosdb_diagram2.png) - -The Python and Java interfaces wrap this library in a more object-oriented, user-friendly format. - -The core implementation is not portable, but the python interface does provides a thin virtualization layer over TCP. A user running Windows in a Virtual Machine, for instance, can expose the same python interface to a different host system running python3. - -![](./res/main_diagram.png) - -
- -### Updates -- - - - -#### Accessing the TDAmeritrade API Directly - -Ameritrade recently expanded and opened up their API for streaming/historical data, account access, -and trade execution. [Take a look at a new front-end library for it that's currently in development.](https://github.com/jeog/TDAmeritradeAPI) If you're simply looking for light-weight access to real-time market data and/or custom TOS studies/strategies data TOSDataBridge is probably a better approach. - -#### New Symbols for Futures Contracts - -Ameritrade recently changed the symbols needed for futures contracts. For example, "/ES" is now "/ES:XCME". - -/ES:XCME -/NQ:XCME -/YM:XCBT -/RTY:XCME -/NKD:XCME -/EMD:XCME -/VX:XCBF -/BTC:XCME -/XBT:XCBF -/CL:XNYM -/QM:XNYM -/NG:XNYM -/QG:XNYM -/RB:XNYM -/HO:XNYM -/BZ:XNYM -/GC:XCEC -/MGC:XCEC -/SI:XCEC -/SIL:XCEC -/HG:XCEC -/PL:XNYM -/PA:XNYM -/ZB:XCBT -/ZN:XCBT -/ZF:XCBT -/ZT:XCBT -/UB:XCBT -/TN:XCBT -/GE:XCME -/ZQ:XCBT -/GLB:XCME -/6A:XCME -/M6A:XCME -/6B:XCME -/M6B:XCME -/6C:XCME -/6E:XCME -/E7:XCME -/M6E:XCME -/6J:XCME -/J7:XCME -/6M:XCME -/6N:XCME -/6S:XCME -/ZC:XCBT -/XC:XCBT -/ZS:XCBT -/XK:XCBT -/ZW:XCBT -/XW:XCBT -/KE:XCBT -/ZM:XCBT -/ZL:XCBT -/ZO:XCBT -/HE:XCME -/GF:XCME -/LE:XCME -/LBS:XCME -/FNG:IFUS -/MME:IFUS -/DX:IFUS -/YG:IFUS -/YI:IFUS -/CT:IFUS -/CC:IFUS -/KC:IFUS -/SB:IFUS -/OJ:IFUS - - -### Requirements -- - - -- Windows for the core implementation. (The python interface is available to any system running python3.) -- TDAmeritrade's ThinkOrSwim(TOS) platform that exposes DDE functionality (the Window's verion) -- VC++ 2012 Redistributable (included) - -### Versions -- - - - -- **v0.8** \[branch 'v0.8'\] - 'stable' version that guarantees up-to-date binaries, signatures, and symbols (use the README from branch 'v0.8') - -- **v0.9** \[branch 'master'\] - development version that may lack up-to-date binaries, signatures, and/or symbols - -### New Features -- - - -- ***'CUSTOM' Topics*** - [access the 19 CUSTOM fields exported by TOS](README_DETAILS.md#custom-topics). - - export calculations, studies, strategy triggers etc. - - use ThinkScript or the Condition Wizard from within TOS to build custom quotes. -- ***'GetNFromMarker'*** - calls added to C API allowing for stateless blocking calls to be built on top. -- ***'StreamingSession'*** - python Context Manager that provides blocking streaming data for any combination of items/topics - -### Quick Setup -- - - -- tosdb-setup.bat will attempt to install the necessary modules/dependencies for you but you should refer to [Installation Details](README_INSTALL.md) for a more detailed explanation -- Be sure to know what build you need(x86 vs x64); it should match your system, all the modules you'll need, and your version of Python(if you plan on using the python wrapper), or your version of the Java Runtime(if you plan on using the java wrapper) - - ##### Core C/C++ Libraries - ``` - (Admin) C:\[...TOSDataBridge]\tosdb-setup.bat [x86|x64] [admin] [session] - ``` - - - [x86|x64] : the version to build (required) - - [admin] : does your TOS platform require elevation? (optional) - - [session] : override the service's attempt to determine the session id when exiting from session-0 isolation. The tos-databridge-engine.exe[] binary needs to run in the same session as the ThinkOrSwim platform. **Most users shouldn't worry about this** unless they plan to run in a non-standard environment (e.g an EC2 instance). [An explanation of Sessions, Desktops, and Stations.](https://blogs.technet.microsoft.com/askperf/2007/07/24/sessions-desktops-and-windows-stations/) (optional) - - ``` - Example 1: C:\TOSDataBridge\> tosdb-setup.bat x86 - Example 2: C:\TOSDataBridge\> tosdb-setup.bat x64 admin - Example 3: C:\TOSDataBridge\> tosdb-setup.bat x64 admin 2 - ``` - - ##### Python Wrapper (optional) - ``` - C:\[...TOSDataBridge]\python\python setup.py install - ``` - - Core C/C++ libs (above) must be installed first to use the (non-virtual) interface - - tosdb/_tosdb.py is generated automatically by setup.py - - ##### Java Wrapper (optional) - - Core C/C++ libs (above) must be installed first - - *java/tosdatabridge.jar* should be included (source is in java/src if you want/need to build) - - -### Quick Start -- - - -1. You may need to white-list some of these files (specifically tos-databridge-engine-\[x86|x64].exe) in your Anti-Virus software before proceeding. -2. Start the service: - - ``` - (Admin) C:\> SC start TOSDataBridge - ``` - (consider having the service begin automatically on startup to avoid this step in the future; see #9 in the [Installation Details](README_INSTALL.md).) -3. Log on to your TOS platform - -##### For C/C++: -- Include tos_databridge.h header in your code -- Use the library calls detailed in [C/C++ API](README_API.md) -- Link with *tos-databridge-[version]-[build].dll* -- Build -- Run - -##### For Python: -- import tosdb -- [docs](README_PYTHON.md) -- [tutorial](python/tutorial.md) - -##### For Java: -- add *java/tosdatabridge.jar* to your classpath -- [see docs](README_JAVA.md) for an intro to the API, or the source(java/src) for the details -- Compile -- Run - -### Build (optional) -- - - -- The recommended way to build from source is to use VisualStudio. Open the .sln file in /VisualStudioBuild, go to the appropriate build, and select 'build'. - -- Object code and logs are placed into an intermediate sub-directory of /VisualStudioBuild of form /$(Configuration)/$(Platform). After the object code is linked the binaries are sent to a sub-directory of /bin of form /$(Configuration)/$(Platform) and the symbol files (if applicable) are sent to /symbols/$(Configuration). - -- There are 32 and 64 bit (Win32 and x64) binaries included along with the relevant configurations in the VisualStudio solution. Debug versions have a "_d" suffix to avoid collisions. It's up to the user to choose and use the correct builds for ALL modules. The python library will search for the underlying DLL (-x64 vs. -x86) that matches the build of that version of python. The java wrapper requires a build type that matches the Java Runtime(JRE) version. - - -### Contents -- - - -- **/include** - C/C++ header files; your C/C++ code must include the tos-databridge.h header; C++ code needs to be sure the compiler can find containers.hpp, generic.hpp, exceptions.hpp. - -- **/src** - C/C++ source files; if building from source simply open the .sln file inside /VisualStudioBuild, select the configuration/platform, and build. - -- **/VisualStudioBuild** - The complete Visual Studio solution with properties, pre-processor options, relative links etc. set, ready to be built. - -- **/bin** - Compiled (release only) binaries by build type. **(master branch may, or may not, contain all, or any)** - - - *tos-databridge-serv-[x86|x64].exe* : The service process that spawns and controls the main engine described below. This program is run as a typical windows service with SYSTEM privileges; as such its intended role is very limited. - - - *tos-databridge-engine-[x86|x64].exe* : The main engine - spawned from tos-databridge-serv.exe - that interacts with the TOS platform and our DLL(below). It runs with a lower(ed) integrity level and reduced privileges. - - - *tos-databridge-[version]-[x86|x64].dll* : The library/interface that client code uses to access TOSDB. Review tos-databridge.h, and the sections below, for all the necessary calls, types, and objects. - - - *_tos-databridge-[x86|x64].dll* : A back-end library that provides custom concurrency and IPC objects; logging and utilities; as well as the Topic-String mapping. - - - *tos-databridge-shell-[x86|x64]* : A shell used to interact directly with the library calls. - -- **/Symbols** - Symbol (.pdb) files **(master branch may, or may not, contain all, or any)** - -- **/python** - Files relevant to the python wrapper. - - - **tosdb/** : A python package that serves as a wrapper around *tos-databridge-[version]-[x86|x64].dll*. It provides a more object oriented, simplified means of accessing the core functionality. - - - **tosdb/intervalize/** : Sub-package that provides fixed intervals(e.g OHLC). - - - **tosdb/streaming/** : Sub-package that provides blocking/synchronous mechanisms for accessing streaming data. - - - **tosdb/cli_scripts/** : Sub-package for 'client' scripts built on top of the python wrapper. - -- **/java** - Files relevant to the Java wrapper. - -- **/test** - Test scripts for the various languages and modules. - -- **/sigs** - The detached signature for each binary; sha256 checksums for binaries and signatures; **(master branch may, or may not, contain all, or any)** - -- **/res** - Miscellaneous resources - -- **/log** - All log files and dumps - -### Docs -- - - - -#### [Installation](README_INSTALL.md) - - -#### [Details](README_DETAILS.md) - - -#### [C/C++ API](README_API.md) - - -#### [Python](README_PYTHON.md) - - -#### [Java](README_JAVA.md) - - -#### [Troubleshooting](README_TROUBLESHOOTING.md) - - -#### [FAQ](README_FAQ.md) - - -### Contributions -- - - -This project grew out of personal need and is maintained by a single developer. Contributions - testing, bug fixes, suggestions, extensions, whatever - are always welcome. If you want to contribute something non-trivial it's recommended you communicate the intention first to avoid unnecessary and/or conflicting work. - -Simply reporting bugs or questioning unintuitive interface design can be very helpful. - - -### License & Terms -- - - -*TOSDB is released under the GNU General Public License(GPL); a copy (LICENSE.txt) should be included. If not, see http://www.gnu.org/licenses. The author reserves the right to issue current and/or future versions of TOSDB under other licensing agreements. Any party that wishes to use TOSDB, in whole or in part, in any way not explicitly stipulated by the GPL, is thereby required to obtain a separate license from the author. The author reserves all other rights.* - -*This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.* - -*By choosing to use the software - under the broadest interpretation of the term "use" \- you absolve the author of ANY and ALL responsibility, for ANY and ALL damages incurred; including, but not limited to, damages arising from the accuracy and/or timeliness of data the software does, or does not, provide.* - -*Furthermore, TOSDB is in no way related to TDAmeritrade or affiliated parties; users of TOSDB must abide by their terms of service and are solely responsible for any violations therein.* - -- - - - -*Copyright (C) 2014 Jonathon Ogden* - - diff --git a/README_API.md b/README_API.md deleted file mode 100644 index f8d4e22..0000000 --- a/README_API.md +++ /dev/null @@ -1,754 +0,0 @@ -### C/C++ API -- - - - -#### Types - -name | language | defined by | description --------------------|----------|------------------|------------------------------------------ -TOS_Topics::TOPICS | C++ | tos_databridge.h | enum of fields to be added (e.g. BID, ASK, LAST ) -DateTimeStamp | C / C++ | tos_databridge.h | wrapper around C lib 'tm' struct, adds a micro-second field -UpdateLatency | C / C++ | tos_databridge.h | enum of milliseconds values the client library waits before re-checking buffers -ILSet<> | C++ | containers.hpp | wrapper around std::set<> type that provides additional means of construction / copy / move / assignment -generic_type | C++ | generic.hpp | custom generic type - -#### Typedefs - -typedef | underlying type | language | defined by ---------------------------------|--------------------------------------------------|----------|-------------------------- -LPCSTR | const char* | C / C++ | WINAPI -LPSTR | char* | C / C++ | WINAPI -BOOL | unsigned int | C / C++ | WINAPI -size_type | uint32_t | C / C++ | tos_databridge.h -type_bits_type | uint8_t | C / C++ | tos_databridge.h -str_set_type | ILSet\ | C++ | tos_databridge.h -topic_set_type | ILSet\ | C++ | tos_databridge.h -def_price_type **(DEPRECATED)** | float | C / C++ | tos_databridge.h -ext_price_type **(DEPRECATED)** | double | C / C++ | tos_databridge.h -def_size_type **(DEPRECATED)** | long | C / C++ | tos_databridge.h -ext_size_type **(DEPRECATED)** | long long | C / C++ | tos_databridge.h -pDateTimeStamp | DateTimeStamp* | C / C++ | tos_databridge.h -generic_dts_type | std::pair\ | C++ | tos_databridge.h -generic_vector_type | std::vector\ | C++ | tos_databridge.h -dts_vector_type | std::vector\ | C++ | tos_databridge.h -generic_dts_vectors_type | std::pair\ | C++ | tos_databridge.h -generic_map_type | std::map\ | C++ | tos_databridge.h -generic_dts_map_type | std::map\ | C++ | tos_databridge.h -generic_matrix_type | std::map\ | C++ | tos_databridge.h -generic_dts_matrix_type | std::map\ | C++ | tos_databridge.h - -#### Error Codes - -error | value ----------------------------------|------- -TOSDB_ERROR_BAD_INPUT | -1 -TOSDB_ERROR_BAD_INPUT_BUFFER | -2 -TOSDB_ERROR_NOT_CONNECTED | -3 -TOSDB_ERROR_TIMEOUT | -4 -TOSDB_ERROR_BLOCK_ALREADY_EXISTS | -5 -TOSDB_ERROR_BLOCK_DOESNT_EXIST | -6 -TOSDB_ERROR_BLOCK_CREATION | -7 -TOSDB_ERROR_BLOCK_SIZE | -8 -TOSDB_ERROR_BAD_TOPIC | -9 -TOSDB_ERROR_BAD_ITEM | -10 -TOSDB_ERROR_BAD_SIG | -11 -TOSDB_ERROR_IPC | -12 -TOSDB_ERROR_IPC_MSG | -13 -TOSDB_ERROR_CONCURRENCY | -14 -TOSDB_ERROR_ENGINE_NO_TOPIC | -15 -TOSDB_ERROR_ENGINE_NO_ITEM | -16 -TOSDB_ERROR_SERVICE | -17 -TOSDB_ERROR_GET_DATA | -18 -TOSDB_ERROR_GET_STATE | -19 -TOSDB_ERROR_SET_STATE | -20 -TOSDB_ERROR_DDE_POST | -21 -TOSDB_ERROR_DDE_NO_ACK | -22 -TOSDB_ERROR_SHEM_BUFFER | -23 -TOSDB_ERROR_UNKNOWN | -24 -TOSDB_ERROR_DECREMENT_BASE | <= -25 - -#### C vs C++ Return Conventions - -In most cases an 'int' return value indicates sucess(0) or failure(non-0 error code). Most C calls don't return values directly, they assign a (pointed to) value or populate a buffer/array. C++ versions of the same calls, generally, return a value directly and throw an exception on error. A function returning a pure boolean value is represented by an unsigned int(C) or bool(C++). - -#### Connecting - -**`[C/C++] TOSDB_Connect() -> int`** - -- Connect to the service/engine. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_Disconnect() -> int`** - -- Close connection. (Called automatically when the library is unloaded.) -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_IsConnected() -> unsigned int` [DEPRECATED]** - -- Returns 1 if connected to the service/engine, 0 if not. - -**`[C/C++] TOSDB_IsConnectedToEngine() -> unsigned int`** - -- Returns 1 if connected to the service/engine, 0 if not. - -**`[C/C++] TOSDB_IsConnectedToEngineAndTOS() -> unsigned int`** - -- Returns 1 if connected to the service/engine AND can communicate with TOS platform, 0 if not. - -**`[C/C++] TOSDB_ConnectionState() -> unsigned int`** - -Returns a constant indicating state of connection: -- TOSDB_CONN_NONE: not connected to engine/service or TOS -- TOSDB_CONN_ENGINE: connected to engine/service but not TOS (admin calls only) -- TOSDB_CONN_EGINE_TOS: connected to engine/service and TOS (admin calls AND data from platform) - - -#### Data Blocks - -TOSDB's main organizational unit is the (data) block (struct TOSDBlock in client.hpp). A block contains streams, of a certain size, that hold historical data. - -**`[C/C++] TOSDB_CreateBlock(LPCSTR id, size_type sz, BOOL is_datetime, size_type timeout) -> int`** - -- Create a block. -- 'id' is a unique ID string(<= TOSDB_BLOCK_ID_SZ) that will be used to access it throughout its lifetime. -- 'sz indicates how much historical data is saved in the block's data-streams. -- 'is_datetime' is a flag indicating whether date-time (see DateTimeStamp) should be saved alongside primary data in the streams. -- 'timeout' is milliseconds used for internal waiting/synchronization(see TOSDB_DEF_TIMEOUT, TOSDB_MIN_TIMEOUT). -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_CloseBlock(LPCSTR id) -> int`** - -- Deallocate block's internal resources and remove it. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_CloseBlocks() -> int`** - -- Close all blocks that currently exist in the dll instance. -- Returns 0 on success, error code on failure. - -Within each block is a pointer to a RawDataBlock object created by an internal factory. The factory has a limit (default is 10) which can be adjusted. (The number of blocks should always be the same as the number RawDataBlocks.) - -**`[C/C++] TOSDB_GetBlockLimit() -> size_type`** - -- Returns maximum number of blocks that can exist in the dll instance. - -**`[C/C++] TOSDB_SetBlockLimit(size_type sz) -> size_type`** - -- Set the maximum number of blocks that can exist in the dll instance. -- Returns new limit. - -**`[C/C++] TOSDB_GetBlockCount() -> size_type `** - -- Returns the number of blocks allocated in the dll instance. - -**`[C/C++] TOSDB_GetBlockIDs(LPSTR* dest, size_type array_len, size_type str_len) -> int`** - -- Populates '*dest' with IDs of all blocks that currently exist in the dll instance. -- 'array_len' is the size of the array to be populated and should == TOSDB_GetBlockCount(). -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetBlockIDs() -> str_set_type`** - -- Returns 'str_set_type' with IDs of all blocks that currently exist in the dll instance. -- Throws on failure. - -**`[C/C++] TOSDB_GetBlockSize(LPCSTR id, size_type* pSize) -> int`** - -- Sets '*pSize' to how much historical data can be saved in the block's data-streams. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetBlockSize(std::string id) -> size_type`** - -- Returns how much historical data can be saved in the block's data-streams. - -**`[C/C++] TOSDB_SetBlockSize(LPCSTR id, size_type sz) -> int`** - -- Changes how much historical data can be saved in the block's data-streams. -- 'sz' must be > 1 and <= TOSDB_MAX_BLOCK_SZ. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_IsUsingDateTime(LPCSTR id, unsigned int* is_datetime) -> int`** - -- Set '*is_datetime' to 1 if block is storing DateTimeStamp objects alongside primary data, 0 otherwise. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_IsUsingDateTime(std::string id) -> bool`** - -- Returns if block is storing DateTimeStamp objects alongside primary data. - -The client library extracts data from the engine (tos-databridge-engine[].exe) through a shared memory segment. The library loops through its blocks/streams looking to see what buffers have been updated, reading the buffers if necessary. The 'latency' is the wait time between loops and is represented by the 'UpdateLatency' Enum. The default(Moderate, 300) should be fine for most users. - -**`[C/C++] TOSDB_GetLatency() -> unsigned long`** - -- Returns wait time (in milliseconds) between reads of the shared memory buffer by the client library. - -**`[C/C++] TOSDB_SetLatency(UpdateLatency latency) -> unsigned long`** - -- Set wait time (in milliseconds) between reads of the shared memory buffer by the client library. -- Returns new latency. - - -#### Items / Topics / Streams - -Once a block is created, items and topics are added. Topics are the TOS fields (e.g. LAST, VOLUME, BID ) and items are the individual symbols (e.g. IBM, GE, SPY). C++ uses the TOS_Topics::TOPICS enum. The somewhat unintuitive terms 'item' and 'topic' come from DDE terminology. Data are stored in 'streams', created internally as a result of topics and items being added. - -For example, if we add two topics and three items we have six streams: - -  | SPY | QQQ | GOOG ------------|-----|-----|----- -**LAST** |  X |    X |     X -**VOLUME** |  X |    X |     X - -Items and Topics are **CASE SENSITIVE** and should be entered as all upper-case. Topic strings will be cast to a TOS_Topics::TOPICS enum. Item strings will be checked for invalid characters in the 'Add' calls. In most cases a bad Topic will return TOSDB_ERROR_BAD_TOPIC(C); an invalid Item String will return TOSDB_ERROR_BAD_INPUT(C); or they'll throw an exception(C++). **EXCEPT**. for the 'AddItems', 'AddTopics', and 'Add' calls which take multiple items/topics and will simply remove invalid items/topics from the input array/set. Only if ALL are removed will an error/exception occur. - -**`[C/C++] TOSDB_Add(LPCSTR id, LPCSTR* items, size_type items_len, LPCSTR* topics_str , size_type topics_len) -> int`** -**`[C++] TOSDB_Add(std::string id, str_set_type items, topic_set_type topics_t) -> int`** - -- Add items and topics to the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_AddTopic(LPCSTR id, LPCSTR topic_str) -> int`** -**`[C++] TOSDB_AddTopic(std::string id, TOS_Topics::TOPICS topic_t) -> int`** - -- Add a single topic to the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_AddItem(LPCSTR id, LPCSTR item) -> int`** -**`[C++] TOSDB_AddItem(std::string id, std::string item) -> int`** - -- Add a single item to the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_AddTopics(LPCSTR id, LPCSTR* topics_str, size_type topics_len) -> int`** -**`[C++] TOSDB_AddTopics(std::string id, topic_set_type topics_t) -> int`** - -- Add multiple topics to the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_AddItems(LPCSTR id, LPCSTR* items, size_type items_len) -> int`** -**`[C++] TOSDB_AddItems(std::string id, str_set_type items) -> int`** - -- Add multiple items to the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_RemoveTopic(LPCSTR id, LPCSTR topic_str) -> int`** -**`[C++] TOSDB_RemoveTopic(std::string id, TOS_Topics::TOPICS topic_t) -> int`** - -- Remove a single topic from the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_RemoveItem(LPCSTR id, LPCSTR item) -> int`** -**`[C++] TOSDB_RemoveItem(std::string id, std::string item) -> int`** - -- Remove a single item from the block. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetItemCount(LPCSTR id, size_type* count) -> int`** - -- Sets '*count' to the number of items in the block. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetItemCount(std::string id) -> size_type`** - -- Returns the number of items in the block. -- Throws on failure. - -**`[C/C++] TOSDB_GetTopicCount(LPCSTR id, size_type* count) -> int`** - -- Sets '*count' to the number of topics in the block. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetTopicCount(std::string id) -> size_type`** - -- Returns the number of topics in the block. -- Throws on failure. - -**`[C/C++] TOSDB_GetItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -> int`** - -- Populates 'dest' with item strings currently in the block. -- 'array_len' is the size of the array and should == TOSDB_GetItemCount(). -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetItemNames(std::string id) -> str_set_type`** - -- Returns 'str_set_type' containing all item strings in the block. -- Throws on failure. - -**`[C/C++] TOSDB_GetTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -> int`** - -- Populates 'dest' with topic strings currently in the block. -- 'array_len' is the size of the array and should == TOSDB_GetTopicCount(). -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetTopicNames(std::string id) -> str_set_type`** - -- Returns 'str_set_type' containing all topic strings in the block. -- Throws on failure. - -**`[C++] TOSDB_GetTopicEnums() -> topic_set_type`** - -- Returns 'topic_set_type' containing all topic enums(TOS_Topics::TOPICS) in the block. -- Throws on failure. - -##### Pre-Caching - -Items\[Topics\] added before any topics\[items\] exist in the block will be pre-cached, i.e they will be visible to the back-end but not to the interface until a topic\[item\] is added; likewise if all the items\[topics\] are removed(thereby leaving only topics\[items\]). To view the pre-cache use the following calls: - -**`[C/C++] TOSDB_GetPreCachedItemCount(LPCSTR id, size_type* count) -> int`** - -- Sets '*count' to the number of items in the block's pre-cache. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetPreCachedItemCount(std::string id) -> size_type`** - -- Returns the number of items in the block's pre-cache. -- Throws on failure. - -**`[C/C++] TOSDB_GetPreCachedTopicCount(LPCSTR id, size_type* count) -> int`** - -- Sets '*count' to the number of topics in block's pre-cache. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetPreCachedTopicCount(std::string id) -> size_type ->`** - -- Returns the number of topics in block's pre-cache. -- Throws on failure. - -**`[C/C++] TOSDB_GetPreCachedItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -> int`** - -- Populates 'dest' with item strings currently in the block's pre-cache. -- 'array_len' is the size of the array and should == TOSDB_GetPreCachedItemCount(). -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetPreCachedItemNames(std::string id) -> str_set_type`** - -- Returns 'str_set_type' containing all items in the block's pre-cache. -- Throws on failure. - -**`[C/C++] TOSDB_GetPreCachedTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -> int`** - -- Populates 'dest' with topic strings currently in the block's pre-cache. -- 'array_len' is the size of the array and should == TOSDB_GetPreCachedTopicCount(). -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetPreCachedTopicNames(std::string id) -> str_set_type`** - -- Returns 'str_set_type' containing all topics in the block's pre-cache. -- Throws on failure. - -**`[C++] TOSDB_GetPreCachedTopicEnums() -> topic_set_type`** - -- Returns 'topic_set_type' containing all topic enums(TOS_Topics::TOPICS) in the block's pre-cache. -- Throws on failure. - -##### Determining Type of a Topic - -Different streams store data as different types. You need to unpack the type info from inside the topic enum in order to make the correct 'Get' call(below). - -**`[C++] TOS_Topics::Type::type`** - -- Static object used to get the underlying type of topic enum at compile-time. -- 'E' is a TOS_Topics::TOPICS enum. - -**`[C/C++] TOSDB_GetTypeBits(LPCSTR topic_str, type_bits_type* type_bits) -> int`** - -- Sets '*type_bits' to the type bits of the topic string. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetTypeBits(TOS_Topics::TOPICS topic_t) -> type_bits_type`** - -- Returns the type bits of the topic enum. -- Throws on failure. - -Check the bits with the appropriate constants: - - type_bits_type tbits; - TOSDB_GetTypeBits("BID", &tbits); - - if(tbits == TOSDB_INTGR_BIT) - \\ just the integer bit - \\ data are longs - else if(tbits == TOSDB_INTGR_BIT | TOSDB_QUAD_BIT) - \\ integer bit and the quad bit - \\ data are long longs - else if (tbits == TOSDB_QUAD_BIT) - \\ quad bit, no integer bit - \\ data are doubles - else if (tbits) == 0) - \\ no quad bit, no integer bit - \\ data are floats - else if (tbits == TOSDB_STRING_BIT) - \\ no quad bit, no integer bit, just the string bit - \\ data are strings - else - \\ ERROR - -> **IMPORTANT** Make sure you don't simply check a bit with logical AND when what you really want is to check the entire type_bits_type value. In this example checking for the INTGR_BIT will return true for long AND long long. - -**`[C/C++] TOSDB_GetTypeString(LPCSTR topic_str, LPSTR dest, size_type str_len) -> int`** - -- Populates 'dest' with a (platform dependent) type string of the topic. -- 'str_len' is the size of the buffer and should be >= TOSDB_MAX_STR_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetTypeString(TOS_Topics::TOPICS topic_t) -> std::string`** - -- Returns a (platform dependent) type string of the topic. -- Throws on failure. - - -##### Accessing Engine Directly - -**`[C/C++] TOSDB_DumpSharedBufferStatus() -> int`** - -- Dump engine's buffer info to a file in /log. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_RemoveOrphanedStream(LPCSTR item, LPCSTR topic_str) -> int`** - -- Removes an independent stream from the engine. -- **WARNING** - this should only be used when certain a client lib has failed to close a stream during destruction of the containing block. If that's not the case **YOU CAN CORRUPT THE UNDERLYING BUFFERS FOR ANY OR ALL CLIENT INSTANCE(S)!** -- Returns 0 on success, error code on failure. - - -#### Historical Data - -In order to get data from the block/stream we use simple inclusive indexing. - -- positive indices go from most to least recent(new to old). -- negative indicies go from least to most recent(old to new). -- the beginning('beg') index represents the most recent value of a range. -- the ending('end') index represents the least recent(oldest) value of a range. -- 0 or -block size is the index of the most recent value. -- -1 or block size - 1 is the index of the least recent value. -- C++ calls will throw std::invalid_argument() on index errors. - -Ex. -block size = 10; begin = 3(-7), end = 5(-5), -\[0\] \[1\] \[2\] **\[3\] \[4\] \[5\]** \[6\] \[7\] \[8\] \[9\] - - -##### Occupancy - -The size of a block (and all its streams) is simply the maximum amount of data it can hold. The occupancy tells how much data is actually in each stream. - -**`[C/C++] TOSDB_GetStreamOccupancy(LPCSTR id,LPCSTR item, LPCSTR topic_str, size_type* sz) -> int`** - -- Sets '*sz' to how many data-points are currently in the particular stream. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetStreamOccupancy(std::string id, std::string item, TOS_Topics::TOPICS topic_t) -> size_type`** - -- Returns how many data-points are currently in the particular stream. -- Throws on failure. - - -##### Different Versions - -API offers multiple ways of getting the same historical data: - -1. *type-named* (e.g TOSDB_GetDouble, TOSDB_Get): -figure out the type of the topic for that data-stream (see [Determining Type of a Topic](#determining-type-of-a-topic)) and use the appropriately named call to return the particular type. -2. *string* (e.g TOSDB_GetString, TOSDB_Get): -data will be cast to char* (of length <= TOSDB_STR_DATA_SZ) or std::string if topic is not of string type. **(SLOW)** -3. *generic* (e.g, TOSDB_Get): -returns generic_type object(s) that know their native type(C++ only). **(SLOW)** - -The following functions all have DateTimeStamp versions. In order to use them the block must have been constructed with this option set(is_datetime == TRUE). The C versions need a non-NULL pointer to a DateTimeStamp struct or an array of structs(pass NULL to avoid getting DateTimeStamp(s)). The C++ versions take a boolean template arg instead. - -> **IMPORTANT** DateTimeStamp arrays in C should be the same length as the primary data array. - -> **IMPORTANT** Label arrays (see Frames, below) in C should be the same length as the primary data array and DateTimeStamp array (if used). - - -##### Individual Data-Points - -**`[C/C++] TOSDB_GetDouble(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, double* dest, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetFloat(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, float* dest, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetLongLong(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, long long* dest, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetLong(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, long* dest, pDateTimeStamp datetime) -> int`** - -- Sets '*dest' to the historical value at position 'indx'. -- Sets '*datetime' to DateTime struct if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetString(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, LPSTR dest, size_type str_len, pDateTimeStamp datetime) -> int`** - -- Populates '*dest' with the historical value at position 'indx, as string. -- Populates '*datetime' struct if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'str_len' is the size of the string buffer and should be >= TOSDB_STR_DATA_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> double`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> float`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> long long`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> long`** - -- Returns the historical value at position 'indx'. -- Throws on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::string`** - -- Returns the historical value at position 'indx', as string. -- Throws on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> generic_type`** - -- Returns the historical value at position 'indx', as generic_type. -- Throws on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::pair`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::pair`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::pair`** -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::pair`** - -- Returns an std::pair of the historical value at position 'indx' AND DateTimeStamp if block supports it. -- Throws on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> std::pair`** - -- Returns an std::pair of the historical value at position 'indx', as string, AND DateTimeStamp if block supports it. -- Throws on failure. - -**`[C++] TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx) -> generic_dts_type`** - -- Returns an std::pair of the historical value at position 'indx', as generic_type, AND DateTimeStamp if block supports it. -- Throws on failure. - - -##### Multiple Contiguous Data-Points - -> **IMPLENTATION NOTE** Internally the data-stream tries to limit what is copied by keeping track of the streams occupancy and only returning valid data. C++ calls will therefore return a dynamically sized containter that may be smaller than what you expected. C calls may leave 'tail' elements of the passed array undefined if there is not valid data to fill them. - -> **PERFORMANCE NOTE** Not suprisingly, when n is very large the the non-string C calls are the fastest, with the non-generic, non-string C++ calls just behind. - -**`[C/C++] TOSDB_GetStreamSnapshotDoubles(LPCSTR id,LPCSTR item, LPCSTR topic_str, double* dest, size_type array_len, pDateTimeStamp datetime, long end, long beg) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotFloats(LPCSTR id,LPCSTR item, LPCSTR topic_str, float* dest, size_type array_len, pDateTimeStamp datetime, long end, long beg) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotLongLongs(LPCSTR id,LPCSTR item, LPCSTR topic_str, long long* dest, size_type array_len, pDateTimeStamp datetime, long end, long beg) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotLongs(LPCSTR id,LPCSTR item, LPCSTR topic_str, long* dest, size_type array_len, pDateTimeStamp datetime, long end, long beg) -> int`** - -- Populates '*dest' with historical data between position 'beg' and 'end', inclusively. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetStreamSnapshotStrings(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, pDateTimeStamp datetime, long end, long beg) -> int`** - -- Populates '*dest' with historical data, as strings, between position 'beg' and 'end', inclusively. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_STR_DATA_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::vector`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::vector`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::vector`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::vector`** - -- Returns vector of historical data between position 'beg' and 'end', inclusively. -- Throws on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::vector`** - -- Returns vector of historical data, as strings, between position 'beg' and 'end', inclusively. -- Throws on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> generic_vector_type`** - -- Returns vector of historical data, as generic_types, between position 'beg' and 'end', inclusively. -- Throws on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::pair,dts_vector_type>`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::pair,dts_vector_type>`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::pair,dts_vector_type>`** -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::pair,dts_vector_type>`** - -- Returns pair of vectors of historical data and matching DateTimeStamps between position 'beg' and 'end', inclusively. -- Throws on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> std::pair,dts_vector_type>`** - -- Returns pair of vectors of historical data, as strings, and matching DateTimeStamps between position 'beg' and 'end', inclusively. -- Throws on failure. - -**`[C++] TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg) -> generic_dts_vectors_type`** - -- Returns pair of vectors of historical data, as generic_types, and matching DateTimeStamps between position 'beg' and 'end', inclusively. -- Throws on failure. - - - -##### Multiple Contiguous Data-Points From an 'Atomic' Marker - -It's likely the stream will grow between consecutive calls. These calls guarantee to pick up where the last 'Get', 'GetStreamSnapshot', 'GetStreamSnapshotFromMarker', or 'GetNFromMarker' call ended (under a few assumptions). Internally the stream maintains a 'marker' that tracks the position of the last value pulled; the act of retreiving data and moving the marker can be thought of as a single, 'atomic' operation. - -**`[C/C++] TOSDB_GetStreamSnapshotDoublesFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,double* dest,size_type array_len, pDateTimeStamp datetime, long beg, long *get_size) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotFloatsFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,float* dest,size_type array_len, pDateTimeStamp datetime, long beg, long *get_size) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotLongLongsFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,long long* dest,size_type array_len, pDateTimeStamp datetime, long beg, long *get_size) -> int`** -**`[C/C++] TOSDB_GetStreamSnapshotLongsFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,long* dest,size_type array_len, pDateTimeStamp datetime, long beg, long *get_size) -> int`** - -- Populates '*dest' with historical data between position 'beg' and the marker, inclusively. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- Sets '*get_size' to number of data-points copied. A negative value inidicates buffer is too small or stream is 'dirty'. (see below) -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetStreamSnapshotStringsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, pDateTimeStamp datetime, long beg, long *get_size) -> int`** - -- Populates '*dest' with historical data, as strings, between position 'beg' and the marker, inclusively. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_STR_DATA_SZ. -- Sets '*get_size' to number of data-points copied. A negative value inidicates buffer is too small or stream is 'dirty'. (see below) -- Returns 0 on success, error code on failure. - -**NEW [Oct 2017]** - -**`[C/C++] TOSDB_GetNDoublesFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,double* dest,size_type n, pDateTimeStamp datetime, long *get_size) -> int`** -**`[C/C++] TOSDB_GetNFloatsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, float* dest, size_type n, pDateTimeStamp datetime, long *get_size) -> int`** -**`[C/C++] TOSDB_GetNLongLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long long* dest, size_type n, pDateTimeStamp datetime, long *get_size) -> int`** -**`[C/C++] TOSDB_GetNLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long* dest, size_type n, pDateTimeStamp datetime, long *get_size) -> int`** - -- Populates '*dest' with 'n' historical data points in front of (more recent) the marker. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'n' is the size of the array(s) AND the requested amount of data. -- Sets '*get_size' to number of data-points copied. A negative value inidicates stream is 'dirty'. (see below) -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetNStringsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type n, size_type str_len, pDateTimeStamp datetime, long *get_size) -> int`** - -- Populates '*dest' with 'n' historical data, as strings, in front of (more recent) the marker. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- 'n' is the size of the array(s) AND the requested amount of data. -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_STR_DATA_SZ. -- Sets '*get_size' to number of data-points copied. A negative value inidicates stream is 'dirty'. (see below) -- Returns 0 on success, error code on failure. - -A 'Dirty Stream' results from the marker hiting the back of the stream, indicating all data older than the marker is lost. To avoid this be sure you use a big enough block and/or keep the marker moving foward (by using calls mentioned above). - -**`[C/C++] TOSDB_IsMarkerDirty(LPCSTR id, LPCSTR item, LPCSTR topic_str, unsigned int* is_dirty) -> int`** - -- Sets '*is_dirty' to 1 if the stream is dirty, 0 otherwise. -- There is no guarantee that a 'clean' stream will not become dirty before the next call. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_IsMarkerDirty(std::string id, std::string item, TOS_Topics::TOPICS topic_t) -> bool`** - -- Returns if the stream is dirty. -- There is no guarantee that a 'clean' stream will not become dirty before the next call. -- Throws on failure. - -**`[C/C++] TOSDB_GetMarkerPosition(LPCSTR id,LPCSTR item, LPCSTR topic_str, long long* pos) -> int`** - -- Sets '*pos' with the current position (index) of the marker in the stream. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetMarkerPosition(std::string id, std::string item, TOS_Topics::TOPICS topic_t) -> long long`** - -- Returns the current position (index) of the marker in the stream. -- Throws on failure. - - -#### Frames - -A frame is a collection of the most recent data (position/indx = 0) for ALL topics, ALL items, or both. - -##### Item Frames - -Item Frames are the most recent values of ALL the items and a single topic. (e.g {'LAST','SPY'} {'LAST','QQQ'} {'LAST','IWM'}) - -**`[C/C++] TOSDB_GetItemFrameDoubles(LPCSTR id, LPCSTR topic_str, double* dest, size_type array_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetItemFrameFloats(LPCSTR id, LPCSTR topic_str, float* dest, size_type array_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetItemFrameLongLongs(LPCSTR id, LPCSTR topic_str, long long* dest, size_type array_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** -**`[C/C++] TOSDB_GetItemFrameLongs(LPCSTR id, LPCSTR topic_str, long* dest, size_type array_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** - -- Populates '*dest' with the current 'topic_str' values for each item in the block. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- Populates '*label_dest' with the matching labels(the matching item string) for each topic if NOT NULL. -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- Returns 0 on success, error code on failure. - -**`[C/C++] TOSDB_GetItemFrameStrings(LPCSTR id, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** - -- Populates '*dest' with the current 'topic_str' values, as strings, for each item in the block. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- Populates '*label_dest' with the matching labels(the matching item string) for each item if NOT NULL. -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_STR_DATA_SZ. -- Returns 0 on success, error code on failure. - - -**`[C++] TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t) -> generic_map_type`** - -- Returns std::map of item strings to the maching 'topic_t' value, as generic_type, for each item in the block. -- Throws on error. - -**`[C++] TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t) -> generic_dts_map_type`** - -- Returns std::map of item strings to the std::pairs of the maching 'topic_t' value, as generic_type, and DateTimeStamps, for each item in the block. -- Throws on error. - - -##### Topic Frames - -Topic Frames are the most recent values of ALL the topics and a single item. (e.g {'LAST','SPY'} {'BID','SPY'} {'ASK','SPY'}) - -**`[C/C++] TOSDB_GetTopicFrameStrings(LPCSTR id, LPCSTR item, LPSTR* dest, size_type array_len, size_type str_len, LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime) -> int`** - -- Populates '*dest' with the current 'item' values, as strings, for each topic in the block. -- Populates '*datetime' with the matching DateTime structs if NOT NULL and block supports date-time(see TOSDB_IsUsingDateTime). -- Populates '*label_dest' with the matching labels(the matching topic string) for each topic if NOT NULL. -- 'array_len' is the size of the array(s) and must be big enough to fit the data or it will be truncated. -- 'str_len' is the size of each string buffer in the array and should be >= TOSDB_STR_DATA_SZ. -- Returns 0 on success, error code on failure. - -**`[C++] TOSDB_GetTopicFrame(std::string id, std::string item) -> generic_map_type`** - -- Returns std::map of topic strings to the maching 'item' value, as generic_type, for each topic in the block. -- Throws on error. - -**`[C++] TOSDB_GetTopicFrame(std::string id, std::string item) -> generic_dts_map_type`** - -- Returns std::map of topic strings to the std::pairs of the maching 'item' value, as generic_type, and DateTimeStamps, for each topic in the block. -- Throws on error. - - -##### Total Frames - -Total Frames are the most recent values of ALL the topics and ALL the items. - -**`[C++] TOSDB_GetTotalFrame(std::string id) -> generic_matrix_type`** - -- Returns std::map of item strings to the std::map of topic strings and the maching 'item' value, as generic_type, for each topic in the block. -- Throws on error. - -**`[C++] TOSDB_GetTotalFrame(std::string id) -> generic_dts_matrix_type`** - -- Returns std::map of topic strings to std::map of the std::pairs of the maching 'item' value, as generic_type, and DateTimeStamps, for each topic in the block. -- Throws on error. - - -#### Logging, Exceptions & Stream Overloads - -The library exports some logging functions that dovetail with its use of custom exception classes. Most of the modules use these logging functions internally. The files are sent to appropriately named .log files in /log. Client code is sent to /log/client-log.log by using the following calls: - -**`[C/C++] TOSDB_LogH(LPCSTR tag, LPCSTR desc)` [MACRO]** - -- log high priority message. - -**`[C/C++] TOSDB_Log(LPCSTR tag, LPCSTR desc)` [MACRO]** - -- log low priority message. - -**`[C/C++] TOSDB_LogEx(LPCSTR tag,LPCSTR desc,int error)` [MACRO]** - -- pass additional argument for an error code like one returned from *GetLastError()*. - -The library also defines an exception hierarchy (exceptions.hpp) derived from TOSDB_Error and std::exception. C++ code can catch them and respond accordingly; the base class provides threadID(), processID(), tag(), and info() methods as well as the inherited what() from std::exception. - -There are operator\<\< overloads (client_out.cpp) for most of the custom objects and containers returned by the myriad 'Get...' calls. - diff --git a/README_DETAILS.md b/README_DETAILS.md deleted file mode 100644 index 1327714..0000000 --- a/README_DETAILS.md +++ /dev/null @@ -1,214 +0,0 @@ - -TOSDataBridge consists of three main components: -- TOS Platform -- backend Service -- frontend Library - -![](./res/tosdb_diagram2.png) - -Once the Service and TOS Platform are running, the user loads the library and uses the API to communicate with the Service. -- - - - -#### The Service - -The Service is a background process designed to act as a bridge between the TOS platfrom and (multiple) instances of client code. Upon request the service will continually pull data from the Platform, putting it into circular buffers in shared memory. This modular approach allows: - -- different user-space programs to share raw data, avoiding redundancy -- most of the code to run with lower privileges -- those interested the ability to write their own interfaces/bindings -- easier debugging -- the service to stay running and/or start automatically on system startup - -##### Service Commands: - -A Window's Service is managed differently than a typical program. It can be controlled via 'services.msc' or the command line: - -- ```(Admin) C:\> SC start TOSDataBridge``` -- ```(Admin) C:\> SC query TOSDataBridge``` - Display current status of the service. -- ```(Admin) C:\> SC stop TOSDataBridge``` - Stop the service. All the data collected so far will still exist but the engine will sever its connection to TOS and exit. -- ```(Admin) C:\> SC pause TOSDataBridge``` - Pause the service. All the data collected so far will still exist but the engine will stop recording new data in the buffers. **Pausing the service is not recommended.** -- ```(Admin) C:\> SC continue TOSDataBridge``` - Continue a paused service. All the data collected so far will still exist, the engine will start recording new data into the buffers, but it will have missed any new data while paused. -- ```(Admin) C:\> SC config TOSDataBridge ...``` - Adjust the service's configuration/properties. -- ``` C:\> SC /?``` - Display help for the SC command. - - -##### The Engine - -Once started the service spawns a child process(tos-databridge-engine) with lower/restricted privileges that does all the leg work. Occassionally(debuging, for instance) it can be useful to run the engine directly by calling the service binary with the --noservice switch. The engine binary will then enter a 'detached' state, **requiring the user to manually kill the proces when done.** - - Example 1: (Admin) C:\>TOSDataBridge\bin\Debug\x64\> tos-databridge-serv-x64_d.exe --noservice - Example 2: (Admin) C:\>TOSDataBridge\bin\Release\Win32\> tos-databridge-serv-x86.exe --noservice --admin - -The engine creates a number of kernel objects(mutexs, shared memory segments etc.) that require certain privileges. These privileges are set in SpawnRestrictedProcess() in service.cpp. If you attempt to run the engine binary directly, as a standard user, the creation of these objects will fail, resulting in a fatal uncaught exception. (See the comments near the top of tos_databridge.h, where NO_KGBLNS is defined, for more details.) **Running the engine directly is not recommended.** -- - - - -#### The Library - -The [Library](README_API.md) is a DLL used to communicate with the service. It continually copies the data in shared memory into 'local' streams, which can be returned to the user through the appropriate API call(s). If using the C/C++ API the library is linked and loaded directly. If using [Python](README_PYTHON.md) or [Java](README_JAVA.md) you manually load the library via the init(...) calls. -- - - - -#### Exiting Gracefully - -You want to avoid shutting down the TOS platform and/or the Service while your code is running. If you close TOS without cleaning up from the client side you can corrupt the underlying objects the Service uses to communicate. If you close your program without cleaning up you can cause a resource leak in the Service. - -As a general rule follow this order: -1. Clean up: close blocks, call clean_up etc. -2. Exit your program (optional) -3. Close TOS platform (optional) -4. Stop the Service (optional) -- - - - -#### Objects - -TOSDataBridge uses a mostly object-oriented approach (in concept and in code) to store, manage, and return data to the user. The main object is the ***block***. A user adds ***items*** and ***topics*** to the block for the data they want. Each item-topic pairing represents a ***stream*** that is managed automatically by the block. - -An ***item*** is a string representing the individual instrument(e.g 'SPY', '/ZC:XCBT', '.SPY180119C250', '$DJT', 'EUR/USD'). - -> The underlying C/C++ API is **case-sensitive**; it's up to the client to make sure they are passing connsistenly **upper-case** item strings. The Python and Java APIs are **case-insensitive**; on receiving item and topic strings they are converted to upper-case by default. - -A ***topic*** is a string and/or enum (depending on the language) representing a data field(e.g 'LAST', 'VOLUME', 'BID_SIZE'). The complete list of topics can be found in src/topics.cpp. - -> The implementation can easily handle bad topics passed to it, returning an error code(C) or throwing an exception(C++). Bad item strings on the other hand can be a problem. The DDE server is *supposed* to respond with a negative ACK if the item is invalid but TOS responds with a positive ACK, inserting 'N/A' strings into the stream. **Currently it's up to the user to deal with passing bad Items.** - -If we create one block, add two topics and three items we have six streams: - -  | SPY | QQQ | GOOG ------------|-----|-----|----- -**LAST** |  X |    X |     X -**VOLUME** |  X |    X |     X - -> **The block requires at least one valid topic AND item**, otherwise it can't hold a data-stream. Because of this, if only items(topics) are added they are held in a pre-cache until a valid topic(item) is added. Likewise, if all topics(items) are removed, the remaining items(topics) are moved into the pre-cache. Use the appropriate API calls to see what's in the pre-cache. - -Review the [Data Blocks](README_API.md#data-blocks) and [Items/Topics/Streams](README_API.md#items--topics--streams) sections of the C/C++ API docs for a thorough explanation. -- - - - -#### Getting Data - -There are numerous way to get current and historical data from a block/stream. - -- An (indexed) single historical data-point of a stream -- An (indexed) snapshot of multiple historical data-points of a stream -- An (indexed) snapshot of multiple historical data-points up to an internal, atomic marker -- A snapshot of 'n' historical data-points before an internal, atomic marker -- The most recent value of all streams -- The most recent value of all streams of a particular item -- The most recent value of all streams of a particular topic - -Many of these calls offer type-specific, generic, and string versions. They can also return DateTime for each data-point, if applicable. - -Review the [Historical Data](README_API.md#historical-data) and [Frames](README_API.md#frames) sections of the C/C++ API docs for a thorough explanation. Specific information about the Python and Java API calls can be found in the source documentation: - -- ***python***: python/tosdb/_common.py -- ***java***: java/src/io/github/jeog/tosdatabridge/DataBlock.java -- - - - -#### Custom Topics - -TOS allows users to create custom quote fields that can be added as a column to various widgets/displays. These can be custom calculations, studies, strategy output etc. and can be created using the 'Condition Wizard' of the ThinkScript editor. Once created, and in use on the platform, the output of that field can then be exported via TOSDataBridge, like any other topic. - -To create a CUSTOM quote in TOS: -1. Create a new watch-list or go to MarketWatch->Quotes -2. Right-click on the column header and click 'customize' -3. Select 'Custom Quotes' from the drop-down menu in the upper left. -4. Left-click the little scroll icon to the left of 'CUSTOM' field you want to use. -5. Select the 'Condition Wizard' tab or the ThinkScript editor tab to code your own. -6. When done add it to the list of columns, click Apply. - -To export this CUSTOM quote via TOSDataBridge: -1. ***Add this custom quote to at least one widget/window in TOS.*** -2. ***Add the symbols you want to export to THAT widget/window.*** -3. Add the appropriate topic to your block, i.e topic 'CUSTOM1' for 'CUSTOM 1'. -4. Add the symbols/items you added in #2 to your block. (Symbols added to the block but not to TOS, will only return 'loading'.) - -***Example 1***: Return a comma delimited string containing Fast, Slow, and Full Stochastic values. - -(in the ThinkScript editor for 'CUSTOM 9') - - def sfast = StochasticFast(); - def sslow = StochasticSlow(); - def sfull = StochasticFull(); - AddLabel(Yes, Concat(sfast, Concat(",", Concat(sslow, Concat(",", sfull))))); - -(in TOS create a watch-list that looks something like) - - | LAST | CUSTOM 9 -------|--------|--------- -SPY | 260.10 | 14.37,18.23,17.99 -QQQ | 155.68 | 1.8,1.3,1.29 - -(in python) - - >> block = tosdb.TOSDB_DataBlock(); - >> block.add_items("SPY", "QQQ") - >> block.add_topics("CUSTOM9") - >> val = block.get("SPY", "CUSTOM9", date_time=False) - >> stoch_fast, stoch_slow, stoch_full = map(float, val.split(",")) - - -***Example 2***: Return a boolean value to indicate when the 10 min SMA crosses above the 30. - -(in the ThinkScript editor for 'CUSTOM 9') - - plot data; - if SimpleMovingAvg(close, 10) crosses above SimpleMovingAvg(close, 30) { - data = 1; - }else{ - data = 0; - } - -Follow the steps from Example 1 and check the stream for a change from '0' to '1' -- - - - -#### DDE Data - -It's important to realize that we are at the mercy of the TOS platform, the DDE technology, and TOS's implementation of it. You may notice the streams of data will not match the Time & Sales perfectly. If you take a step back and aggregate the data in more usable forms this really shouldn't be an issue. Another way we are at the mercy of the DDE server is that fields like last price and last size are two different topics. That means they are changing at slightly different times with slightly different time-stamps even if they are the correct pairing. To get around this we can write code like this to simulate a live-stream : - - >>> block = tosdb.TOSDB_DataBlock() - >>> block.add_items('GOOG') - >>> block.add_topics('LAST','VOLUME') - >>> vol = 0 - >>> - >>> print("price".ljust(10), "size") - >>> while(run_flag): - ... v = block.get('GOOG','VOLUME') - ... p = block.get('GOOG','LAST') - ... # if total volume has changed, output the difference - ... if(v != vol): - ... print(str(p).ljust(10), str(v - vol)) - ... vol = v - ... # the less we sleep the more accurate the stream - ... time.sleep(.1) -- - - - -#### Numerical Values for 'Non-Decimal' Instruments - -The TOS DDE server doesn't handle numerical values for 'non-decimal' instruments well. For instance, trying to get a bid/ask/last etc. for a 10-yr note future (/zn) will not return the fractional part of the price. In these cases use the topic versions suffixed with an x (bidx/askx/lastx etc.) which will provide a string you'll need to parse. -- - - - -#### Closing Large Blocks - -Currently Closing/Destroying large blocks(1,000,000+ combined data-stream elements) involves a large number of internal deallocations/destructions and becomes quite CPU intensive. The process is spun-off into its own thread but this may fail, returning to the main thread when the library is being freed, or block the python interpreter regardless of when or how it's called. Use caution when creating/closing large blocks. -- - - - -#### Block Size and Memory - -As mentioned you need to use some sense when creating blocks. As a simple example: let's say you want LAST,BID,ASK for 100 symbols. If you were to create a block of size 1,000,000, WITHOUT DateTime, you would need to allocate over 2.4 GB of memory - not good. As a general rule keep data-streams of similar desired size in the same block, and use new blocks as necessary. In our example if you only need 1,000,000 LAST elems for 10 items and 100 for everything else create 2 blocks: 1) a block of size 1,000,000 with those 10 items and the topic LAST; 2) a block of size 100 with all 100 items and the three topics. Then you would only need a little over 80 MB. (Really you could create three blocks to avoid any/all overlap.) -- - - - -#### Asymmetric Responsibilities & Leaks - -Connection 'probing' only works one way, from master(client/library) to slave(service). If the master disconnects abruptly, without sending the requisite remove signal(s), the slave will continue to maintain the master's resources. Internally, all it does is keep a ref-count to the streams it's been asked to create and write the necessary data into the appropriate shared memory segments. To see the status of the service, and whether there are leaked streams, open up the command shell, connect and use **`DumpBufferStatus`** to dump all the current stream information to /log . -- - - - -#### DateTimeStamp - -***THESE ARE NOT OFFICIAL STAMPS FROM THE EXCHANGE,*** they are manually created once the TOS DDE server returns the data. They use the system clock to assure high_resolution( the micro-seconds field) and therefore there is no guarantee that the clock is accurate or won't change between stamps, as is made by the STL's std::steady_clock. -- - - - -#### SendMessage vs. SendMessageTimeout - - -To initiate a topic with the TOS server we should send out a broadcast message via the SendMessage() system call. This call is built to block to insure the client has had a chance to deal with the ACK message. For some reason, it's deadlocking, so we've been forced to use SendMessageTimeout() with an arbitrary 500 millisecond timeout. Therefore, until this gets fixed adding topics will introduce an amount of latency in milliseconds = 500 x # of topics. -- - - - - diff --git a/README_FAQ.md b/README_FAQ.md deleted file mode 100644 index 7b2bb6a..0000000 --- a/README_FAQ.md +++ /dev/null @@ -1,47 +0,0 @@ -### Frequently Asked Questions -- - - - -##### What does TOSDataBridge do? - -It provides (programmatic) access to real-time streaming data from the ThinkOrSwim(TOS) platform. - -##### What type of data does it provide? - -It attempts to provide all the data that TOS exports via DDE: about 60 fields in total. Some of the more popular are LAST, VOLUME, NET_CHANGE, BID, ASK. It also provides [access to the 19 CUSTOM fields exported by TOS](README_DETAILS.md#custom-topics), allowing users to export custom calculations, studies, strategy triggers etc. from the platform. - -##### What type of securities/instruments does it work with ? - -Generally any symbol that the TOS platform recognizes: equities('AAPL'), ETFs('SPY'), futures('/ZC:XCBT'), options('.SPY180119C250'), indices('$DJT'), forex('EUR/USD'), mutual-funds(VFIAX), and certain indicators($TRIN). If you find an exception please let us know. Different securities are more suited to certain data fields than others. For instance, GAMMA is relevant to options but not much else. - -##### Do I need Windows? - -Yes, only the Windows version of TOS exports DDE data. The python API does provide a virtual layer that allows access from any system as long as it is physically or virtually networked with a Windows system running TOS and the TOSDataBridge Service. - -##### Do I need to have TOS running at all times? - -Yes. - -##### Do I need to know how to program? - -Yes. If you're new to programming, and willing to learn, the Python API is the place to start. Python is an interactive language that is very user friendly and easy to learn. There are plenty of free tutorials online and we provide a tutorial for using the Python API in python/tosdb/tutorial.md. - -##### What programming languages are supported? - -C, C++, Java, and Python interfaces are provided. The Java interface is still in development so it is the least stable. - -##### I'm having trouble connecting to the Service/Engine and/or the TOS platform. What do I do? - -[Review the troubleshooting docs](README_TROUBLESHOOTING.md). - -##### How do I build/debug TOSDataBridge? - -[Review the Build Notes](README.md#build-optional). Release builds include a (.pdb) symbols file for debugging. - -##### How is this different than Ameritrade's API? - -It's almost completely different. All TOSDataBridge provides is the ability to (locally) pull data off the TOS platform. It's not affiliated with Ameritrade. It doesn't (directly) communicate with their servers. You can't execute orders etc. It's purely a data extraction tool. - -##### Why don't the data and associated time-stamps exactly match the Time & Sales data of the platform/exchange? - -TOSDataBridge relies on [TOS's use of the Window's DDE mechanism.](README_DETAILS.md#dde-data). - diff --git a/README_INSTALL.md b/README_INSTALL.md deleted file mode 100644 index 8314f45..0000000 --- a/README_INSTALL.md +++ /dev/null @@ -1,57 +0,0 @@ -### Installation Details -- - - - -1. Move the unzipped tos-databridge folder to its permanent location. If you change it in the future you'll need to redo some of these steps because the Service module relies on an absolute path. - -2. Determine the appropriate build you'll need: 32-bit(x86) or 64-bit(x64). ***Make sure you don't mix and match in future steps. It should match the python or Java Runtime(JRE) version if you plan on using the python or java wrappers, respectively.*** - -3. (***Optional***) Compile the appropriate binaries if [building from source](README.md#build-optional). - -4. Determine if your TOS platform runs under elevated privileges (does it ask for an admin password before starting?) - -5. Determine if you need to run under a custom Session. The tos-databridge-engine-[build].exe binary needs to run in the same session as the ThinkOrSwim platform. **MOST USERS SHOULDN'T WORRY ABOUT THIS** unless they plan to run in a non-standard environment (e.g an EC2 instance). - -6. Open a command shell with Admin Privileges (right-click on cmd.exe and click 'run as administrator'). Navigate to the tos-databridge root directory and run the tosdb-setup.bat setup script with the info from steps #2, #4, and #5: - - ``` - Usage: C:\TOSDataBridge\> tosdb-setup.bat [x86|x64] [admin] [session] - - Example 1: C:\TOSDataBridge\> tosdb-setup.bat x86 - Example 2: C:\TOSDataBridge\> tosdb-setup.bat x64 admin - Example 3: C:\TOSDataBridge\> tosdb-setup.bat x64 admin 2 - ``` - -7. The setup script is going to do a few things: - - make sure you have the appropriate run-time libraries installed; if not the appropriate redist executable will be run to install them. (If this fails you can do this manually by downloading the most recent VC++ Redistributable from Microsoft); - - make sure you have the appropriate -serv and -engine binaries in /bin - - create the Windows Service. - -8. Before we continue, it's a good idea, possibly necessary, to add the tos-databridge binaries, or the whole directory, to your anti-virus's 'white-list'. ***If not tos-databridge-engine-[build].exe may be stopped and/or deleted. If you run into problems connecting to the service/engine module or the TOS platform be sure to check the logs/settings.*** - -9. After the Windows Service has been successfully created run: - - ```(Admin) C:\> SC start TOSDataBridge``` - - Returned text and various system utilities should confirms the service is running(see screen-shot below). Let's also configure the service to begin automatically on system start-up: - - ```(Admin) C:\> SC config TOSDataBridge start= auto``` (notice the space between '=' and 'auto') - - [More information about the service.](README_DETAILS.md#the-service) - - (note: some of the particulars in these (older) screen-shots may be different for newer versions) - ![](./res/SCss1.png) - -10. (***C/C++ ONLY***) Include the tos_databridge.h header in your code. If C++ make sure the compiler can find containers.hpp, generic.hpp, and exceptions.hpp. Adjust the link settings to import the tos-databridge-[version]-[build].lib stub. (If you run into trouble review the VisualStudio settings for tos-databridge-shell-[build].exe as they should resemble what you're trying to do.) - -11. (***C/C++ ONLY***) Add appropriate [C/C++ API calls](README_API.md) to your code. Build. - -12. (***JAVA ONLY***) Add java/tosdatabridge.jar to your classpath. - -13. (***JAVA ONLY***) Add appropriate [Java API calls](README_JAVA.md) to you code. Build. - -14. (***PYTHON ONLY***) [Install the tosdb package.](README_PYTHON.md#install) - -15. Start the TOS platform. - -16. Run your program or use the [python wrapper](README_PYTHON.md) interactively. - diff --git a/README_JAVA.md b/README_JAVA.md deleted file mode 100644 index bb3e3a4..0000000 --- a/README_JAVA.md +++ /dev/null @@ -1,320 +0,0 @@ -### Java Wrapper -- - - - -***WARNING: in early development; not stable; interface subject to change*** - -The java wrapper, like the python wrapper, extends the underlying C/C++ interface. It uses [Java Native Access(JNA)](https://github.com/java-native-access/jna) to load and access the underlying DLL, wrapping these calls in a more user-friendly API. Currently(unlike the python wrapper) there is no virtual layer for use on non-windows systems. - -> **IMPORTANT:** You still must [install the underlying C/C++ modules.](README_INSTALL.md) You also must be sure the build of these modules matches the build of the Java Runtime Environment(JRE) you plan to use. - -> **NOTE:** If the library build is different than your JRE, init() with throw, indicating it can't find the native library in the resource path. - - -#### Getting Started - -1. Add *java/tosdatabridge.jar* to your project's classpath. - -2. Find the path of the appropriate DLL(version and build) to pass to TOSDataBridge.init(...) - -3. Add the requisite API calls and objects found in the *io.github.jeog.tosdatabridge* package(see below). - -4. Compile and run. - -An example using *java/examples/TimeAndSales*: -``` -C:\...\TOSDataBridge\java\examples\TimeAndSales\> javac -classpath "../../tosdatabridge.jar" -d "." src/TimeAndSales.java -C:\...\TOSDataBridge\java\examples\TimeAndSales\> java -classpath "../../tosdatabridge.jar;." TimeAndSales -``` - -An example using *test/java/TOSDataBridgeTest.java*: -``` -C:\...\TOSDataBridge\test\java\> javac -classpath "../../java/tosdatabridge.jar" TOSDataBridgeTest.java -C:\...\TOSDataBridge\test\java\> java -classpath "../../java/tosdatabridge.jar;." TOSDataBridgeTest "../../bin/Release/x64/tos-databridge-0.9-x64.dll" -``` - - -#### API Basics - -As mentioned, the java wrapper mirrors the python wrapper in many important ways. It's recommended you skim the (more thorough) python docs and tutorial - even the C/C++ docs - for an explanation of the core concepts. The source is pretty straight forward: - -- **TOSDataBridge.java**: constants, exceptions, and some of the (high-level) administrative calls. - -- **DataBlock.java**: user-instantiated object used to pull data from the engine/platform (very similar to python's TOSDB_DataBlock) - -- **DataBlockWithDateTime.java**: extends DataBlock, includes a DateTime object with each primary data-point - -- **DateTime.java**: date-time object that wraps the C tm struct with an added millisecond field; returned in DateTimePair<Type> by the 'WithDateTime' methods - -- **Topic.java**: enum that holds all the topics(LAST, BID, VOLUME etc.) that can be added to the block - - -##### Connect to Native Library - - import io.github.jeog.tosdatabridge.TOSDataBridge; - - try{ - // path to the C Lib; tries to load lib and calls connect() - TOSDataBridge.init(".\bin\Release\x64\tos-databridge-0.9-x64.dll"); - - switch( TOSDataBridge.connectionState() ){ - case TOSDataBridge.CONN_NONN: - // Not connected to engine - // TOSDataBridge.connected() == false; - break; - - case TOSDataBridge.CONN_ENGINE: - // Only connected to Engine (only access to admin calls) - // TOSDataBridge.connected() == false; - break; - - case TOSDataBridge.CONN_ENGINE_TOS: - // Connected to Engine AND TOS Platform (can get data from engine/platform) - // TOSDataBridge.connected() == true; - break; - } - }catch(LibraryNotLoaded){ - // init(...) was not successfull or the library was freed - } - - -##### Create DataBlocks - -There are two types of objects used to get real-time data. Both require a size paramater to indicate the maximum number of (historical) data-points the block can hold. - - import io.github.jeog.tosdatabridge.DataBlock; - import io.github.jeog.tosdatabridge.DataBlockWithDateTime; - - int blockSz = 10000; - - try{ - /* only primary data */ - DataBlock block = new DataBlock(blockSz); - - /* include a DateTime object with each primary data-point - extends DataBlock, adds 'WithDateTime' versions of the get methods */ - DataBlockWithDateTime blockDT = new DataBlockWithDateTime(blockSz); - - }catch(LibraryNotLoaded){ - // init(...) was not successfull or the library was freed - }catch(CLibException){ - // an error was thrown from the C Lib (see CError.java) - } - - -##### Add Items/Topics - -Data are stored in 'streams' which are created internally as a result of topics and items being added. - -If we add two topics and three items we have six streams: - -  | SPY | QQQ | GOOG ------------|-----|-----|----- -**LAST** |  X |    X |     X -**VOLUME** |   X |    X |     X - -Clearly we need items AND topics. Items(topics) added before any topics(items) exist in the block will be pre-cached, i.e they won't be visible to the interface until a topic(item) is added; likewise if all the items(topics) are removed, thereby leaving only topics(items). - - import io.github.jeog.tosdatabridge.Topic - - try{ - blockDT.addItem("SPY"); // goes into pre-cache - blockDT.addItem("QQQ"); // goes into pre-cache - - Set myPrecachedItems = blockDT.getItemsPreCached() // {'SPY','QQQ'} - - blockDT.addTopic(Topic.LAST); // pulls items out of pre-cache - - Set myItems = blockDT.getItems(); // {'SPY','QQQ'} - Set myTopics = blockDT.getTopics(); // {Topic.LAST} - myPrecachedItems = blockDT.getItemsPreCached() // { } - - blockDT.removeTopic(Topic.LAST); // items go back into pre-cache - - blockDT.containsItem('SPY'); // false - blockDT.containsItemPreCached('SPY') // true - - }catch(LibraryNotLoaded){ - // init(...) was not successfull or the library was freed - }catch(CLibException){ - // an error was thrown from the C Lib (see CError.java) - }catch(InvalidItemOrTopic){ - // item string was empty, too long, or null; topic enum was null or NULL_TOPIC - } - -##### Get Data-Point from a Stream - -Like the C/C++ interfaces we have type-specific calls. If you call the wrong version the C lib will try to (safely) cast the value for you. If it can't it will return ERROR_GET_DATA and java will throw CLibException. - -Notice below we use the base classes of the exception hierarchy for illustrative purposes. - - import io.github.jeog.tosdatabridge.DateTime.DateTimePair; - - try{ - switch( TOSDataBridge.getTopicType(Topic.LAST) ){ - case TOSDataBridge.TOPIC_IS_LONG: - ... - break; - - case TOSDataBridge.TOPIC_IS_DOUBLE: /* Topic.LAST stores doubles ... */ - - // most recent 'LAST' for 'SPY' - Double d = blockDT.getDouble("SPY", Topic.LAST); - if(d == null){ - // no data in block yet - } - - // 9 data-points older than most recent, with DateTime - DateTimePair dDT = blockDT.getDoubleWithDateTime("SPY", Topic.LAST, 9); - if(dDT == null){ - // no data at this index/position yet - } - break; - - case TOSDataBridge.TOPIC_IS_STRING: - ... - break; - } - }catch(DataBlockException){ /*extends TOSDataBridgeException, so need to catch first*/ - // InvalidItemOrTopic - // DataIndexException - // DirtyMarkerException (see next section) - }catch(TOSDataBridgeException){ /* TOSDataBridgeException is (indirect) base to ALL */ - // LibraryNotLoaded - // CLibException - // DataBlockException - } - - -'WithDateTime' versions return DateTimePair object(s): - - public class DateTime { - //... - public static class DateTimePair extends Pair{ - public final T first; // inherited - public final DateTime second; //inherited - } - //... - } - - - -##### Get (Contiguous) Data-Points from a Stream - -Methods with a plural type in the name(e.g getStreamSnapshotLongs) return a List(ArrayList). - -'StreamSnapshot' calls return an array of data between 'beg' and 'end', with a number of overloads. - -'StreamSnapshotFromMarker' calls provide client code a guarantee that it won't miss data between calls, assuming the 'marker' doesn't hit the back of the stream, becoming 'dirty'. (See python and C/C++ docs for a complete explanation.) - - - try{ - switch( TOSDataBridge.getTopicType(Topic.LAST) ){ - case TOSDataBridge.TOPIC_IS_LONG: - ... - break; - - case TOSDataBridge.TOPIC_IS_DOUBLE: /* Topic.LAST stores doubles ... */ - - // 10 most recent data-points (index 0 to 9, inclusive) - List dd = blockDT.getStreamSnapshotDoubles("SPY", Topic.LAST, 9, 0); - - // ALL valid data-points, with DateTime - List> ddDT = - blockDT.getStreamSnapshotDoublesWithDateTime("SPY", Topic.LAST); - - /* - * use 'FromMarker' methods (below) to guarantee contiguous data BETWEEN calls - * (see python and C/C++ docs for complete explanation) - */ - - // 10 most recent values - THIS MOVES THE MARKER to 'beg' index (0 in this case) - List> oldLasts = - blockDT.getStreamSnapshotDoublesWithDateTime("SPY", Topic.LAST, 9); - - // add some time between the calls so data can come into stream - Thread.sleep(1000); - // as data comes into the stream the 'marker' is moving backwards - - // all the data-points up to the marker - THIS ALSO MOVES THE MARKER (as above) - List> newLasts = - blockDT.getStreamSnapshotDoublesFromMarkerWithDateTime("SPY", Topic.LAST); - - // concatenating the two lists will guarantee we got ALL the data - newLasts.addAll(oldLasts); // most recent first - for(DateTimePair p : newLasts){ - System.out.println(p); - } - - /* - * we can keep using the marker calls in this way; if the 'marker' hits - * the back of the stream(becomes 'dirty') the next 'Marker' call will - * throw DirtyMarkerException; use the 'IgnoreDirty' versions to avoid - * this behavior, allowing it to return as much valid data as possible - */ - break; - - case TOSDataBridge.TOPIC_IS_STRING: - ... - break; - } - }catch(LibraryNotLoaded){ - // init(...) was not successfull or the library was freed - }catch(CLibException){ - // an error was thrown from the C Lib (see CError.java) - }catch(InvalidItemOrTopic){ - /* item string was empty, too long, or null; topic enum was null or NULL_TOPIC -or- - item or topic is currently not in the block -or- - item or topic is currently in the pre-cache */ - }catch(DataIndexException){ - // we tried to access data in a position that isn't possible for that block size - }catch((DirtyMarkerException){ - // we are NOT using an 'IgnoreDirty' version and getStreamSnapshotFromMarker has a 'dirty' marker (see python/C/C++ docs) - } - - -##### Get (Most Recent) Data-Points from Block - -Frame methods are used to get most recent data (as strings) in some 'logical' way: - -1. All the values (getTotalFrame) -2. All the topic values for an item (getTopicFrame) -3. All the item values for a topic (getItemFrame) -``` - import java.util.Map; - - try{ - // get most-recent 'LAST' vals (as strings) for ALL items in the block - Map itemFrame = blockDT.getItemFrame(Topic.LAST) - - // get ALL most-recent topic vals (as strings) for 'SPY' - Map topicFrame = blockDT.getTopicFrame('SPY') - - // get 'matrix' of ALL most-recent topic and item vals (as strings) - Map> totalFrame = blockDT.getTotalFrame(); - - }catch(LibraryNotLoaded){ - // init(...) was not successfull or the library was freed - }catch(CLibException){ - // an error was thrown from the C Lib (see CError.java) - }catch(InvalidItemOrTopic){ - /* item string was empty, too long, or null; topic enum was null or NULL_TOPIC -or- - item or topic is currently not in the block -or- - item or topic is currently in the pre-cache */ - } -``` - -##### Close - - /* - * DataBlock's finalize() method calls close() on destruction but we can't - * guarantee when/if that will happend; IT'S RECOMMENDED you explicitly tell - * the C Lib to close the underlying block when you're done with it. - */ - block.close(); - blockDT.close(); - - - - - - diff --git a/README_PYTHON.md b/README_PYTHON.md deleted file mode 100644 index f1d850a..0000000 --- a/README_PYTHON.md +++ /dev/null @@ -1,68 +0,0 @@ -### Python Wrapper (tosdb/) -- - - - -The python wrapper is a simpler way to get started with the underlying library. The tosdb package will use ctypes.py to load the library and wrap the calls, mostly as tosdb.TOSDB_DataBlock methods. - -> **IMPORTANT:** tosdb was only written to be compatible with python3 - -> **IMPORTANT:** You still must [install the underlying C/C++ modules.](README_INSTALL.md) You also must be sure the build of these modules matches your python build. Open a python shell and look to see if it says 32 bit or 64 bit on the top - 32 bit needs x86 modules; 64 bit needs x64. If they don't match redo the installation. - -#### Install - -`C:\TOSDataBridge\python\> python setup.py install` - -#### Getting Started - -tosdb/ is structured as a package with the bulk of its code in \__init__.py and \_win.py , the latter holding the non-portable parts that \__init__.py will import if it determines it's being run on a windows sytem. -``` -# import the top-level package (1) -import tosdb - -# (or) import an individual module from a sub-package (2) -from tosdb.intervalize import ohlc -``` - -Once imported you'll have to initialize it so it can connect to the underlying C/C++ library. This requires the path of tos-databridge-[].dll or the root directory it's going to search in for the latest version. - -``` -# the dll path directly from the top-level package (1) -tosdb.init(dllpath="C:/TOSDataBridge/bin/Release/x64/tos-databridge-0.9-x64.dll") - -# the relative root(of the dll path) from the top-level package (1) -tosdb.init(root="../bin") - -# (or) from an individual module (2) -ohlc.tosdb.init(root="../bin") -``` - -Please see [python/tutorial.md](./python/tutorial.md) for a walk-through with screen-shots. - -#### Thread Safety - -TOSDB_ThreadSafeDataBlock and VTOSDB_ThreadSafeDataBlock are thread-safe versions of TOSDB_DataBlock and VTOSDB_DataBlock, respectively. - -#### Intervals - -sub-package tosdb/intervalize helps create (fixed) intervals of data; ohlc.py contains objects that create and store open-high-low-close(OHLC), close(C), and user-defined fixed-time interval objects from an underlying data block. - -#### Streaming - -sub-package tosdb/streaming helps manage an active stream of data. streaming.py provides StreamingSession and VStreamingSession(virtual) Context Managers that manage a data block internally, allowing the user to iterate over the session instance to retrieve time-sequential data for a combination of items and topics. - -#### Virtual Layer - -To use TOSDataBridge from a non-windows systems you can use the virtual interface. To do this you'll still need to install the C/C++ modules and the tosdb package on a (physically or virtually) networked windows sytem. - -The virtual interface consists of, more-or-less, the same objects/calls, with a prepended 'v' (i.e VTOSDB_DataBlock, vinit()). See the detailed explanation in tosdb/\_\_init\_\_.\_\_doc\_\_ or the [virtual tutorial](./python/virtualization_tutorial.md) for example usage. - -> **IMPORTANT:** We recently added a (provisional) authentication mechanism to the virtual layer. ***Unless you know what you're doing and can review the code (tosdb/\_\_init\_\_.py and tosdb/\_auth.py) it's prudent to assume it not secure, possibly exploitable for remote code execution.*** (If anyone out there can give any feedback it would be helpful.) Currently it's recommended for internal networks. To use: 1) install the pycrypto package if you don't have it(pip install pycrypto), 2) pass a password to the enable_virtualization call on the server side and (the same password) to the admin_init call and/or the VTOSDB_DataBlock constructor on the client side. - -#### Cleanup - -There is a minor issue with how python uses the underlying library to deallocate shared resources, mostly because of python's use of refcounts. WE STRONGLY RECOMMEND you call clean_up() before exiting to be sure all shared resources have been properly dealt with. If this is not possible - the program terminates abruptly, for instance - there's a chance you've got dangling/orphaned resources. - -You can dump the state of these resources to a file in /log using tos-databridge-shell-[].exe: -``` - [--> Connect - [--> DumpBufferStatus -``` diff --git a/README_TROUBLESHOOTING.md b/README_TROUBLESHOOTING.md deleted file mode 100644 index c5cd9a2..0000000 --- a/README_TROUBLESHOOTING.md +++ /dev/null @@ -1,43 +0,0 @@ -### Troubleshooting -- - - - -#### Connection Problems - -TOSDataBridge needs to 1) connect the client program (e.g. python interpreter) to the Service/Engine backend, and 2) connect the Service/Engine backend to the TOS platform. If either of these steps fails you'll generally be alerted by an error code, exception, or error message, depending on the language. - -If you have trouble with (1) connecting the client to the Service/Engine be sure: - -- you have run the tosdb-setup.bat install script (and it doesn't output any error messages). -- you have started the service ('SC start TOSDataBridge') and it's running. 'SC query TOSDataBridge' will provide the status. -- you are using the correct version of the client library (e.g. branch 0.7 with a 64-bit Service/Engine and 64-bit client should be using tos-databridge-0.7-x64.dll) - -If you can connect to the Service/Engine but have trouble with (2) connecting to the TOS platform be sure: - -- you passed 'admin' to the tosdb-setup.bat script if your TOS runs with elevated privileges -- you are running the Service/Engine in the appropriate session. - 1. open up cmd.exe - 2. enter 'tasklist' - 3. find thinkorswim and tos-databridge-engine on the list - 4. if their Session# fields don't match you need to [re-run the tosdb-setup.bat script](README.md#quick-setup) with a custom session arg that matches the Session# of thinkorswim - -If none of that solves the problem be sure: -- your anti-virus isn't sandboxing/quarantining any of the binaries and/or blocking communication between them -- to check the log files in /log for more information - -If you are still having issues please [file a new issue](https://github.com/jeog/TOSDataBridge/issues/new); you may have run into a serious bug that we'd like to fix. - -#### Bad Futures Data - -Ameritrade has changed the symbols for futures contracts. [The new symbols.](README.md#new-symbols-for-futures-contracts) - -#### Error(s) adding Item and/or Topics - -*TODO* - - -#### Custom Topic Problems - -*TODO* - - - diff --git a/VisualStudioBuild/_tos-databridge.vcxproj b/VisualStudioBuild/_tos-databridge.vcxproj deleted file mode 100644 index 9d72e49..0000000 --- a/VisualStudioBuild/_tos-databridge.vcxproj +++ /dev/null @@ -1,185 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {ADBF916E-83D6-4FD2-8CF6-306B032E428B} - _tosdatabridge - _tos-databridge - - - - DynamicLibrary - true - v110 - MultiByte - - - DynamicLibrary - true - v110 - MultiByte - - - DynamicLibrary - false - v110 - true - MultiByte - - - DynamicLibrary - false - v110 - true - MultiByte - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x86_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x86 - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x64_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x64 - - - - Level3 - Disabled - true - THIS_EXPORTS_IMPLEMENTATION;THIS_DOESNT_IMPORT_INTERFACE - ..\include - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - %(AdditionalDependencies) - - - - - - - Level3 - Disabled - true - ..\include - THIS_EXPORTS_IMPLEMENTATION;THIS_DOESNT_IMPORT_INTERFACE - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - %(AdditionalDependencies) - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_EXPORTS_IMPLEMENTATION;THIS_DOESNT_IMPORT_INTERFACE - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - %(AdditionalDependencies) - - - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_EXPORTS_IMPLEMENTATION;THIS_DOESNT_IMPORT_INTERFACE - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - %(AdditionalDependencies) - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/VisualStudioBuild/_tos-databridge.vcxproj.filters b/VisualStudioBuild/_tos-databridge.vcxproj.filters deleted file mode 100644 index 30d5257..0000000 --- a/VisualStudioBuild/_tos-databridge.vcxproj.filters +++ /dev/null @@ -1,54 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-engine.vcxproj b/VisualStudioBuild/tos-databridge-engine.vcxproj deleted file mode 100644 index cc93b0d..0000000 --- a/VisualStudioBuild/tos-databridge-engine.vcxproj +++ /dev/null @@ -1,174 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - {607451BD-7C91-4990-BC7D-A363B5E70A35} - tosdatabridgeengine - tos-databridge-engine - - - - Application - true - v110 - MultiByte - - - Application - true - v110 - MultiByte - - - Application - false - v110 - true - MultiByte - - - Application - false - v110 - true - MultiByte - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86_d - $(LibraryPath) - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86 - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64 - - - - Level3 - Disabled - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x86_d.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - AsInvoker - - - - - Level3 - Disabled - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x64_d.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x86.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - AsInvoker - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x64.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-engine.vcxproj.filters b/VisualStudioBuild/tos-databridge-engine.vcxproj.filters deleted file mode 100644 index 0467edb..0000000 --- a/VisualStudioBuild/tos-databridge-engine.vcxproj.filters +++ /dev/null @@ -1,22 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-serv.vcxproj b/VisualStudioBuild/tos-databridge-serv.vcxproj deleted file mode 100644 index c7690c5..0000000 --- a/VisualStudioBuild/tos-databridge-serv.vcxproj +++ /dev/null @@ -1,176 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - {21955674-EC3E-4E9A-8822-1D013970BE5E} - tosdatabridgeserv - tos-databridge-serv - - - - Application - true - v110 - MultiByte - - - Application - true - v110 - MultiByte - - - Application - false - v110 - true - MultiByte - - - Application - false - v110 - true - MultiByte - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86_d - $(LibraryPath) - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86 - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64 - - - - Level3 - Disabled - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x86_d.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - RequireAdministrator - - - - - Level3 - Disabled - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x64_d.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - RequireAdministrator - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x86.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - RequireAdministrator - - - - - Level3 - MaxSpeed - true - true - true - ..\include - THIS_DOESNT_IMPORT_INTERFACE;THIS_IMPORTS_IMPLEMENTATION;%(PreprocessorDefinitions) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - _tos-databridge-x64.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - RequireAdministrator - - - - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-serv.vcxproj.filters b/VisualStudioBuild/tos-databridge-serv.vcxproj.filters deleted file mode 100644 index ec9757d..0000000 --- a/VisualStudioBuild/tos-databridge-serv.vcxproj.filters +++ /dev/null @@ -1,22 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-shell.vcxproj b/VisualStudioBuild/tos-databridge-shell.vcxproj deleted file mode 100644 index 590ef9a..0000000 --- a/VisualStudioBuild/tos-databridge-shell.vcxproj +++ /dev/null @@ -1,182 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56} - tosdatabridgeshell - - - - Application - true - v110 - MultiByte - - - Application - true - v110 - MultiByte - - - Application - false - v110 - true - MultiByte - - - Application - false - v110 - true - MultiByte - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x86 - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - $(ProjectName)-x64 - - - - Level3 - Disabled - false - ..\include - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - Default - %(PreprocessorDefinitions) - - - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - tos-databridge-0.9-x86_d.lib;%(AdditionalDependencies) - NotSet - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - - - Level3 - Disabled - false - ..\include - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - Default - %(PreprocessorDefinitions) - - - true - tos-databridge-0.9-x64_d.lib;%(AdditionalDependencies) - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - NotSet - - - - - Level3 - MaxSpeed - true - true - true - ..\include - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - Default - %(PreprocessorDefinitions) - - - true - true - true - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - tos-databridge-0.9-x86.lib;%(AdditionalDependencies) - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - NotSet - - - - - Level3 - MaxSpeed - true - true - true - ..\include - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - Default - %(PreprocessorDefinitions) - - - true - true - true - tos-databridge-0.9-x64.lib;%(AdditionalDependencies) - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - NotSet - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge-shell.vcxproj.filters b/VisualStudioBuild/tos-databridge-shell.vcxproj.filters deleted file mode 100644 index 1b29743..0000000 --- a/VisualStudioBuild/tos-databridge-shell.vcxproj.filters +++ /dev/null @@ -1,42 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Header Files - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge.sln b/VisualStudioBuild/tos-databridge.sln deleted file mode 100644 index 67913ff..0000000 --- a/VisualStudioBuild/tos-databridge.sln +++ /dev/null @@ -1,101 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Express 2012 for Windows Desktop -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tos-databridge-0.9", "tos-databridge.vcxproj", "{2C63A7B0-C01A-408F-A51D-1A92EB6786D0}" - ProjectSection(ProjectDependencies) = postProject - {ADBF916E-83D6-4FD2-8CF6-306B032E428B} = {ADBF916E-83D6-4FD2-8CF6-306B032E428B} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tos-databridge-serv", "tos-databridge-serv.vcxproj", "{21955674-EC3E-4E9A-8822-1D013970BE5E}" - ProjectSection(ProjectDependencies) = postProject - {ADBF916E-83D6-4FD2-8CF6-306B032E428B} = {ADBF916E-83D6-4FD2-8CF6-306B032E428B} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tos-databridge-shell", "tos-databridge-shell.vcxproj", "{F27C1FCD-E305-4CB0-BC99-16187EEA8C56}" - ProjectSection(ProjectDependencies) = postProject - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0} = {2C63A7B0-C01A-408F-A51D-1A92EB6786D0} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_tos-databridge", "_tos-databridge.vcxproj", "{ADBF916E-83D6-4FD2-8CF6-306B032E428B}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tos-databridge-engine", "tos-databridge-engine.vcxproj", "{607451BD-7C91-4990-BC7D-A363B5E70A35}" - ProjectSection(ProjectDependencies) = postProject - {ADBF916E-83D6-4FD2-8CF6-306B032E428B} = {ADBF916E-83D6-4FD2-8CF6-306B032E428B} - EndProjectSection -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Mixed Platforms = Debug|Mixed Platforms - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Mixed Platforms = Release|Mixed Platforms - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|Mixed Platforms.Build.0 = Debug|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|Win32.ActiveCfg = Debug|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|Win32.Build.0 = Debug|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|Win32.Deploy.0 = Debug|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|x64.ActiveCfg = Debug|x64 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Debug|x64.Build.0 = Debug|x64 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|Mixed Platforms.Build.0 = Release|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|Win32.ActiveCfg = Release|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|Win32.Build.0 = Release|Win32 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|x64.ActiveCfg = Release|x64 - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0}.Release|x64.Build.0 = Release|x64 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|Mixed Platforms.Build.0 = Debug|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|Win32.ActiveCfg = Debug|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|Win32.Build.0 = Debug|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|x64.ActiveCfg = Debug|x64 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Debug|x64.Build.0 = Debug|x64 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|Mixed Platforms.Build.0 = Release|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|Win32.ActiveCfg = Release|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|Win32.Build.0 = Release|Win32 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|x64.ActiveCfg = Release|x64 - {21955674-EC3E-4E9A-8822-1D013970BE5E}.Release|x64.Build.0 = Release|x64 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|Mixed Platforms.Build.0 = Debug|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|Win32.ActiveCfg = Debug|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|Win32.Build.0 = Debug|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|x64.ActiveCfg = Debug|x64 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Debug|x64.Build.0 = Debug|x64 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|Mixed Platforms.Build.0 = Release|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|Win32.ActiveCfg = Release|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|Win32.Build.0 = Release|Win32 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|x64.ActiveCfg = Release|x64 - {F27C1FCD-E305-4CB0-BC99-16187EEA8C56}.Release|x64.Build.0 = Release|x64 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|Mixed Platforms.Build.0 = Debug|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|Win32.ActiveCfg = Debug|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|Win32.Build.0 = Debug|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|x64.ActiveCfg = Debug|x64 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Debug|x64.Build.0 = Debug|x64 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|Mixed Platforms.Build.0 = Release|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|Win32.ActiveCfg = Release|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|Win32.Build.0 = Release|Win32 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|x64.ActiveCfg = Release|x64 - {ADBF916E-83D6-4FD2-8CF6-306B032E428B}.Release|x64.Build.0 = Release|x64 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|Mixed Platforms.Build.0 = Debug|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|Win32.ActiveCfg = Debug|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|Win32.Build.0 = Debug|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|x64.ActiveCfg = Debug|x64 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Debug|x64.Build.0 = Debug|x64 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|Mixed Platforms.ActiveCfg = Release|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|Mixed Platforms.Build.0 = Release|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|Win32.ActiveCfg = Release|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|Win32.Build.0 = Release|Win32 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|x64.ActiveCfg = Release|x64 - {607451BD-7C91-4990-BC7D-A363B5E70A35}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/VisualStudioBuild/tos-databridge.vcxproj b/VisualStudioBuild/tos-databridge.vcxproj deleted file mode 100644 index 5d44bd3..0000000 --- a/VisualStudioBuild/tos-databridge.vcxproj +++ /dev/null @@ -1,211 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {2C63A7B0-C01A-408F-A51D-1A92EB6786D0} - tosdatabridge - tos-databridge-0.9 - - - - DynamicLibrary - true - v110 - MultiByte - - - DynamicLibrary - true - v110 - MultiByte - - - DynamicLibrary - false - v110 - true - MultiByte - - - DynamicLibrary - false - v110 - true - MultiByte - - - - - - - - - - - - - - - - - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x86_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x64_d - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x86 - - - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(Configuration)\$(Platform)\ - .dll - $(ProjectName)-x64 - - - - Level3 - Disabled - false - ..\include;%(AdditionalIncludeDirectories) - false - THIS_IMPORTS_IMPLEMENTATION;THIS_EXPORTS_INTERFACE - - - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - _tos-databridge-x86_d.lib - Windows - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - /export:NewStrings /export:DeleteStrings /export:TOSDB_Log_ /export:TOSDB_LogEx_ /export:CheckIDLength /export:CheckStringLength /export:CheckStringLengths /export:IsValidBlockSize /export:IsValidBlockID /export:IsValidItemString - - - - - Level3 - Disabled - false - THIS_IMPORTS_IMPLEMENTATION;THIS_EXPORTS_INTERFACE - ..\include;%(AdditionalIncludeDirectories) - false - - - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - _tos-databridge-x64_d.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - /export:NewStrings /export:DeleteStrings /export:TOSDB_Log_ /export:TOSDB_LogEx_ /export:CheckIDLength /export:CheckStringLength /export:CheckStringLengths /export:IsValidBlockSize /export:IsValidBlockID /export:IsValidItemString - - - - - Level3 - MaxSpeed - true - true - true - ..\src;..\include;%(AdditionalIncludeDirectories) - false - THIS_IMPORTS_IMPLEMENTATION;THIS_EXPORTS_INTERFACE;%(PreprocessorDefinitions) - - - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - _tos-databridge-x86.lib - Windows - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - /export:NewStrings /export:DeleteStrings /export:TOSDB_Log_ /export:TOSDB_LogEx_ /export:CheckIDLength /export:CheckStringLength /export:CheckStringLengths /export:IsValidBlockSize /export:IsValidBlockID /export:IsValidItemString - - - - - Level3 - MaxSpeed - true - true - true - ..\include;%(AdditionalIncludeDirectories) - false - THIS_IMPORTS_IMPLEMENTATION;THIS_EXPORTS_INTERFACE;%(PreprocessorDefinitions) - - - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - - - true - true - true - _tos-databridge-x64.lib;%(AdditionalDependencies) - Windows - $(SolutionDir)..\bin\$(Configuration)\$(Platform)\ - $(SolutionDir)..\symbols\$(Configuration)\$(TargetName).pdb - /export:NewStrings /export:DeleteStrings /export:TOSDB_Log_ /export:TOSDB_LogEx_ /export:CheckIDLength /export:CheckStringLength /export:CheckStringLengths /export:IsValidBlockSize /export:IsValidBlockID /export:IsValidItemString - - - - - - - - - - - - - - - false - true - Document - - - - - false - true - - - - - - \ No newline at end of file diff --git a/VisualStudioBuild/tos-databridge.vcxproj.filters b/VisualStudioBuild/tos-databridge.vcxproj.filters deleted file mode 100644 index 6f58371..0000000 --- a/VisualStudioBuild/tos-databridge.vcxproj.filters +++ /dev/null @@ -1,47 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - \ No newline at end of file diff --git a/bin/Release/Win32/_tos-databridge-x86.dll b/bin/Release/Win32/_tos-databridge-x86.dll deleted file mode 100644 index 4e2c3f9..0000000 Binary files a/bin/Release/Win32/_tos-databridge-x86.dll and /dev/null differ diff --git a/bin/Release/Win32/_tos-databridge-x86.exp b/bin/Release/Win32/_tos-databridge-x86.exp deleted file mode 100644 index 7d7d34c..0000000 Binary files a/bin/Release/Win32/_tos-databridge-x86.exp and /dev/null differ diff --git a/bin/Release/Win32/_tos-databridge-x86.lib b/bin/Release/Win32/_tos-databridge-x86.lib deleted file mode 100644 index 506ef83..0000000 Binary files a/bin/Release/Win32/_tos-databridge-x86.lib and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-0.9-x86.dll b/bin/Release/Win32/tos-databridge-0.9-x86.dll deleted file mode 100644 index 0dfe640..0000000 Binary files a/bin/Release/Win32/tos-databridge-0.9-x86.dll and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-0.9-x86.exp b/bin/Release/Win32/tos-databridge-0.9-x86.exp deleted file mode 100644 index b8539b2..0000000 Binary files a/bin/Release/Win32/tos-databridge-0.9-x86.exp and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-0.9-x86.lib b/bin/Release/Win32/tos-databridge-0.9-x86.lib deleted file mode 100644 index 519df7a..0000000 Binary files a/bin/Release/Win32/tos-databridge-0.9-x86.lib and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-engine-x86.exe b/bin/Release/Win32/tos-databridge-engine-x86.exe deleted file mode 100644 index c46ddbf..0000000 Binary files a/bin/Release/Win32/tos-databridge-engine-x86.exe and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-serv-x86.exe b/bin/Release/Win32/tos-databridge-serv-x86.exe deleted file mode 100644 index a49fadd..0000000 Binary files a/bin/Release/Win32/tos-databridge-serv-x86.exe and /dev/null differ diff --git a/bin/Release/Win32/tos-databridge-shell-x86.exe b/bin/Release/Win32/tos-databridge-shell-x86.exe deleted file mode 100644 index 736581a..0000000 Binary files a/bin/Release/Win32/tos-databridge-shell-x86.exe and /dev/null differ diff --git a/bin/Release/x64/_tos-databridge-x64.dll b/bin/Release/x64/_tos-databridge-x64.dll deleted file mode 100644 index 0258cda..0000000 Binary files a/bin/Release/x64/_tos-databridge-x64.dll and /dev/null differ diff --git a/bin/Release/x64/_tos-databridge-x64.exp b/bin/Release/x64/_tos-databridge-x64.exp deleted file mode 100644 index 2371f03..0000000 Binary files a/bin/Release/x64/_tos-databridge-x64.exp and /dev/null differ diff --git a/bin/Release/x64/_tos-databridge-x64.lib b/bin/Release/x64/_tos-databridge-x64.lib deleted file mode 100644 index 0bc484e..0000000 Binary files a/bin/Release/x64/_tos-databridge-x64.lib and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-0.9-x64.dll b/bin/Release/x64/tos-databridge-0.9-x64.dll deleted file mode 100644 index 3ca707e..0000000 Binary files a/bin/Release/x64/tos-databridge-0.9-x64.dll and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-0.9-x64.exp b/bin/Release/x64/tos-databridge-0.9-x64.exp deleted file mode 100644 index f8aa3a5..0000000 Binary files a/bin/Release/x64/tos-databridge-0.9-x64.exp and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-0.9-x64.lib b/bin/Release/x64/tos-databridge-0.9-x64.lib deleted file mode 100644 index aa94b8c..0000000 Binary files a/bin/Release/x64/tos-databridge-0.9-x64.lib and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-engine-x64.exe b/bin/Release/x64/tos-databridge-engine-x64.exe deleted file mode 100644 index 9a0ffe2..0000000 Binary files a/bin/Release/x64/tos-databridge-engine-x64.exe and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-serv-x64.exe b/bin/Release/x64/tos-databridge-serv-x64.exe deleted file mode 100644 index 1b6e999..0000000 Binary files a/bin/Release/x64/tos-databridge-serv-x64.exe and /dev/null differ diff --git a/bin/Release/x64/tos-databridge-shell-x64.exe b/bin/Release/x64/tos-databridge-shell-x64.exe deleted file mode 100644 index 72824aa..0000000 Binary files a/bin/Release/x64/tos-databridge-shell-x64.exe and /dev/null differ diff --git a/include/client.hpp b/include/client.hpp deleted file mode 100644 index 058377c..0000000 --- a/include/client.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_CLIENT -#define JO_TOSDB_CLIENT - -#include "tos_databridge.h" -#include -#include - -/* sync access to blocks in client_get/client_admin */ -extern std::recursive_mutex global_rmutex; - -#define GLOBAL_RLOCK_GUARD std::lock_guard global_rlock_guard_(global_rmutex) - -/* forward decl - raw_data_block.hpp / raw_data_block.tpp */ -template class RawDataBlock; - -typedef RawDataBlock TOSDB_RawDataBlock; - -typedef struct { -/* TOSDB's main organizational unit - functionally and conceptually. It's - how client_admin and client_get handle 'blocks' that client code creates. - - TOSDB_RawDataBlock manages all the active items/topics/data */ - TOSDB_RawDataBlock* block; - str_set_type item_precache; - topic_set_type topic_precache; - unsigned long timeout; -} TOSDBlock; /* no ptr or const typedefs; force code to state explicitly */ - -TOS_Topics::TOPICS -GetTopicEnum(std::string topic_str, bool log_if_null=true); - -const TOSDBlock* -GetBlockPtr(std::string id, bool log=true); - -const TOSDBlock* -GetBlockOrThrow(std::string id); - -#endif diff --git a/include/concurrency.hpp b/include/concurrency.hpp deleted file mode 100644 index ed21e8a..0000000 --- a/include/concurrency.hpp +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_CONCURRENCY -#define JO_TOSDB_CONCURRENCY - -#include -#include -#include -#include - -#ifdef CPP_COND_VAR - -#include -#include -#include -#include -#include - -class SignalManager { - typedef std::pair _flag_pair_ty; - typedef std::pair _sig_pair_ty; - - std::mutex _mtx; - std::condition_variable _cnd; - - std::multimap _unq_flags; - - SignalManager(const SignalManager&); - SignalManager(SignalManager&&); - SignalManager& operator=(const SignalManager&); - SignalManager& operator=(SignalManager&&); - -public: - SignalManager() - { - } - - void - set_signal_ID(std::string unq_id); - - bool - wait(std::string unq_id); - - bool - wait_for(std::string unq_id, size_t timeout); - - bool - signal(std::string unq_id, bool secondary); -}; - -#else - -#include - -class LightWeightMutex{ - CRITICAL_SECTION _cs; - - LightWeightMutex(const LightWeightMutex&); - LightWeightMutex& operator=(const LightWeightMutex&); - -public: - LightWeightMutex() - { - InitializeCriticalSection(&_cs); - } - - ~LightWeightMutex() - { - DeleteCriticalSection(&_cs); - } - - inline void - lock() - { - EnterCriticalSection(&_cs); - } - - inline bool - try_lock() - { - return TryEnterCriticalSection(&_cs); - } - - inline void - unlock() - { - LeaveCriticalSection(&_cs); - } -}; - - -class WinLockGuard{ - LightWeightMutex& _mtx; - - WinLockGuard(const WinLockGuard&); - WinLockGuard& operator=(const WinLockGuard&); - -public: - WinLockGuard(LightWeightMutex& mutex) - : - _mtx(mutex) - { - _mtx.lock(); - } - - ~WinLockGuard() - { - _mtx.unlock(); - } -}; - - -class IPCNamedMutexClient{ - HANDLE _mtx; - std::string _name; - - IPCNamedMutexClient(const IPCNamedMutexClient&); - IPCNamedMutexClient(IPCNamedMutexClient&&); - IPCNamedMutexClient& operator=(IPCNamedMutexClient& ipm); - -public: - IPCNamedMutexClient(std::string name) - : - _name(name) - { - } - - bool - try_lock(unsigned long timeout, - std::function no_open_cb = nullptr, - std::function timeout_cb = nullptr, - std::function fail_cb = nullptr); - - void - unlock(); - - inline bool - locked() const - { - return (_mtx != NULL); - } - - ~IPCNamedMutexClient() - { - unlock(); - } -}; - - -class NamedMutexLockGuard{ - HANDLE _mtx; - - NamedMutexLockGuard(const NamedMutexLockGuard&); - NamedMutexLockGuard(NamedMutexLockGuard&&); - NamedMutexLockGuard& operator=(NamedMutexLockGuard& ipm); - -public: - NamedMutexLockGuard(std::string name); - ~NamedMutexLockGuard(); -}; - - -class SignalManager { - std::multimap _unq_flags; - LightWeightMutex _mtx; - HANDLE _event; - - SignalManager(const SignalManager&); - SignalManager& operator=(const SignalManager&); - -public: - SignalManager() - : - _event(CreateEvent(NULL, FALSE, FALSE, NULL)) - { - } - - ~SignalManager() - { - CloseHandle(_event); - } - - void - set_signal_ID(std::string unq_id); - - bool - wait(std::string unq_id); - - bool - wait_for(std::string unq_id, size_type timeout); - - bool - signal(std::string unq_id, bool secondary); -}; - -#endif /* CPP_COND_VAR */ - -#endif diff --git a/include/containers.hpp b/include/containers.hpp deleted file mode 100644 index 97406b6..0000000 --- a/include/containers.hpp +++ /dev/null @@ -1,677 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_CONTAINERS -#define JO_TOSDB_CONTAINERS - -#include -#include -#include -#include - -template> -class ILSet - : public std::set { - typedef ILSet _my_ty; - typedef std::set _my_base_ty; - -public: - ILSet() - { - } - - virtual - ~ILSet() - { - } - - ILSet(const T& item) - { - insert(item); - } - - ILSet(T&& item) - { - insert(std::move(item)); // do we need the move ? - } - - /* copy from same/base */ - ILSet(const _my_base_ty& set) - : - _my_base_ty(set) - { - } - - /* move-construct from same - (explicit derived type to avoid partial destruction) */ - ILSet(_my_ty&& set) - : - _my_base_ty(std::move(set)) - { - } - - /*move-construct from same base */ - ILSet(_my_base_ty&& set) - : - _my_base_ty(std::move(set)) - { - } - - /*attempt to construct from like w/ func obj */ - template - ILSet(const ILSet& set, Func func) - { - for(auto & i : set) - insert(func(i)); - } - - /*attempt to construct from like base w/ func obj*/ - template - ILSet(const std::set& set, Func func) - { - for(auto & i : set) - insert(func(i)); - } - - /*attempt to construct from raw-array of like w/ func obj*/ - template - ILSet(const T2* ptr, size_t sz, Func func) - { - while(sz--) - insert(func(ptr[sz])); - } - - /*attempt to construct from array of like w/ func obj*/ - template - ILSet(const T2(&arr)[sz], Func func) - { - for(size_t i = 0; i - ILSet(const T2* ptr, size_t sz) - { - while(sz--) - insert(ptr[sz]); - } - - /*attempt to construct from array of like*/ - template - ILSet(const T2(&arr)[sz]) - { - for(size_t i = 0; i - ILSet(const ILSet& set) - { - for(auto & item : set) - insert(item); - } - - /*attempt to construct from like base*/ - template - ILSet(const std::set& set) - { - for(auto & item : set) - insert(item); - } - - /*move-assign from same - (explicit derived type to avoid partial destruction) */ - _my_ty& - operator=(_my_ty&& set) - { - _my_base_ty::operator=(std::move(set)); - return *this; - } - - /*move-assign from same/base */ - _my_ty& - operator=(_my_base_ty&& set) - { - _my_base_ty::operator=(std::move(set)); - return *this; - } - - /*assign from same/base*/ - _my_ty& - operator=(const _my_base_ty& set) - { - if((_my_base_ty)(*this) == set) - return *this; - _my_base_ty::operator=(set); - return *this; - } - - /*attempt to assign from like base*/ - template - _my_ty& - operator=(const std::set& set) - { - clear(); - for(auto & item : set) - insert(item); - return *this; - } - - /* attemp to assign from like */ - template - _my_ty& - operator=(const ILSet& set) - { - clear(); - for(auto & item : set) - insert(item); - return *this; - } - - /*attempt to assign from array of like*/ - template - _my_ty& - operator=(const T2(&arr)[sz]) - { - clear(); - for(size_t i = 0; i ()(str); - } -}; - -template -class SmartBuffer - : private std::unique_ptr { - typedef SmartBuffer _my_ty; - typedef std::unique_ptr _my_base_ty; - size_t _bytes; - -public: - SmartBuffer() - : - _my_base_ty(nullptr, [](T* p){}) - { - } - - SmartBuffer(size_t bytes) - : - _bytes(bytes), - _my_base_ty((T*)new char[bytes], [](T* p){ delete[] (char*)p; }) - { - memset(get(), 0, bytes); - } - - virtual ~SmartBuffer() - { - } - - using _my_base_ty::get; - - inline size_t - bytes() - { - return _bytes; - } - - _my_ty& - operator=(const SmartBuffer& right) - { - if(*this != right){ - _bytes = right._bytes; - _my_base_ty::reset((T*)new char[_bytes]); //, [](T* p){ delete[] (char*)p;}); - memcpy(get(), right.get(), _bytes); - } - return *this; - } - - SmartBuffer(const SmartBuffer& right) - : - _bytes(right._bytes), - _my_base_ty((T*)new char[_bytes], [](T* p){ delete[] (char*)p; }) - { - memcpy(get(), right.get(), _bytes); - } - - _my_ty& - operator=(SmartBuffer&& right) - { - if(*this != right){ - _bytes = right._bytes; - _my_base_ty::operator=(std::move(right)); - } - return *this; - } - - SmartBuffer(SmartBuffer&& right) - : - _bytes(right._bytes), - _my_base_ty(std::move(right)) - { - } -}; - -template, - typename Hash2 = std::hash, - typename Key1Eq = std::equal_to, - typename Key2Eq = std::equal_to> -class TwoWayHashMap { - static_assert(!std::is_same::value, - "TwoWayHashMap requires different types"); -public: - typedef T1 key_type; - typedef T2 mapped_type; - - typedef std::unordered_map map1_type; - typedef std::unordered_map map2_type; - - /* Dec 8 2016, for initializer_chain compatability */ - typedef typename map1_type::value_type value_type; - - typedef typename map1_type::value_type pair1_type; - typedef typename map2_type::value_type pair2_type; - - typedef typename map1_type::iterator iterator1_type; - typedef typename map2_type::iterator iterator2_type; - - typedef typename map1_type::const_iterator const_iterator1_type; - typedef typename map2_type::const_iterator const_iterator2_type; - - TwoWayHashMap() - { - } - - virtual ~TwoWayHashMap() - { - } - - template - TwoWayHashMap(const pair1_type(&arr)[sz]) - { - for(size_t i = 0; i & map) - { - for(auto & m : map) - _insert(m); - } - - inline void - insert(T1 k, T2 v) - { - _insert(pair1_type(k, v)); - } - - template - void - insert(const pair1_type(&arr)[sz]) - { - for(size_t i = 0; i -class TwoWayHashMap - : public TwoWayHashMap { - /* thread_safe specialization */ - typedef TwoWayHashMap _my_base_ty; - typedef std::lock_guard _my_lock_guard_type; - - std::recursive_mutex _mtx; - -public: - TwoWayHashMap() - : - _my_base_ty() - { - } - - template - TwoWayHashMap(const pair1_type(&arr)[sz]) - : - _my_base_ty(arr) - { - } - - TwoWayHashMap(const map1_type& map) - : - _my_base_ty(map) - { - } - - TwoWayHashMap(const std::map& map) - : - _my_base_ty(map) - { - } - - void - insert(T1 k, T2 v) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::insert(k, v); - } - - void - insert(const pair1_type p) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::insert(p); - } - - /* Dec 8 2016, for initializer_chain compatability */ - void - insert(iterator1_type dummy, const pair1_type p) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::insert(dummy, p); - } - - template - void - insert(const pair1_type(&arr)[sz]) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::insert(arr); - } - - void - remove(T1 k) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::remove(k); - } - - void - remove(T2 k) - { - _my_lock_guard_type lock(_mtx); - _my_base_ty::remove(k); - } - - T2 - operator[](const T1 k) - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::operator[](k); - } - - T1 - operator[](const T2 k) - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::operator[](k); - } - - const_iterator1_type - find(const T1& k) const - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::find(k); - } - - const_iterator2_type - find(const T2& k) const - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::find(k); - } - - iterator1_type - find(const T1& k) - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::find(k); - } - - iterator2_type - find(const T2& k) - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::find(k); - } - - iterator1_type - begin() - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::begin(); - } - - iterator1_type - end() - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::end(); - } - - const_iterator1_type - cbegin() const - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::cbegin(); - } - - const_iterator1_type - cend() const - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::cend(); - } - - size_t - size() const - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::size(); - } - - bool - empty() - { - _my_lock_guard_type lock(_mtx); - return _my_base_ty::empty(); - } - -}; - - -#endif diff --git a/include/data_stream.hpp b/include/data_stream.hpp deleted file mode 100644 index 3e7a694..0000000 --- a/include/data_stream.hpp +++ /dev/null @@ -1,667 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_DATA_STREAM -#define JO_TOSDB_DATA_STREAM - -#ifndef STR_DATA_SZ -#include "tos_databridge.h" -#define STR_DATA_SZ TOSDB_STR_DATA_SZ // ((unsigned long)0xFF) -#endif - -#include -#include -#include -#include - -/* implemented in src/data_stream.tpp */ - -/* interface */ -#define DATASTREAM_INTERFACE_TEMPLATE template -#define DATASTREAM_INTERFACE_CLASS DataStreamInterface - -/* base object */ -#define DATASTREAM_PRIMARY_TEMPLATE \ - template - -#define DATASTREAM_PRIMARY_CLASS DataStream - -/* specialization of base that inherits from base */ -#define DATASTREAM_SECONDARY_TEMPLATE \ - template - -#define DATASTREAM_SECONDARY_CLASS DataStream - -/*forward decl*/ -class DataStreamError; -class DataStreamTypeError; -class DataStreamSizeViolation; -class DataStreamOutOfRange; -class DataStreamInvalidArgument; - -namespace { - -template -void -BuildThrowTypeError(const char* method) -{ - std::ostringstream s; - s << "Invalid argument < " - << (F ? "UNKNOWN" : typeid(InTy).name()) - << " > passed to DataStream." << method - <<" for this instantiation."; - - throw DataStreamTypeError(s.str().c_str()); -} - -}; - -template -class DataStreamInterface { -public: - typedef GenTy generic_ty; - typedef SecTy secondary_ty; - typedef std::pair both_ty; - typedef std::vector generic_vector_ty; - typedef std::vector secondary_vector_ty; - - /* hard-coded 4 BYTE SIGNED MAX to avoid some of the corner cases. */ - static const size_t MAX_BOUND_SIZE = ((65536LL * 65536 / 2) - 1); - -private: - template - size_t - _copy(OutTy *dest, size_t sz, int end, int beg, secondary_ty *sec) const; - - template - long long - _copy_using_atomic_marker(OutTy *dest, size_t sz, int beg, secondary_ty *sec) const; - - template - long long - _ncopy_using_atomic_marker(OutTy *dest, size_t sz, secondary_ty *sec) const; - -protected: - unsigned int _str_push_count; - - DataStreamInterface() - : - _str_push_count(0) - { - } - -public: - virtual - ~DataStreamInterface() - { - } - - virtual size_t - bound_size() const = 0; - - virtual size_t - bound_size(size_t) = 0; - - virtual size_t - size() const = 0; - - virtual bool - empty() const = 0; - - virtual long long - marker_position() const = 0; - - virtual bool - is_marker_dirty() const = 0; - - virtual generic_ty - operator[](int) const = 0; - - virtual both_ty - both(int) const = 0; - - virtual generic_vector_ty - vector(int end = -1, int beg = 0) const = 0; - - virtual secondary_vector_ty - secondary_vector(int end = -1, int beg = 0) const = 0; - - // BUG-FIX: allow frame calls not to move marker Dec 8 2017 - virtual generic_ty - get_leave_marker(int) const = 0; - - // BUG-FIX: allow frame calls not to move marker Dec 8 2017 - virtual both_ty - both_leave_marker(int) const = 0; - - virtual void - push(const generic_ty& gen, secondary_ty sec = secondary_ty()) = 0; - -/* MACROS that help avoid constructing GenTy for push/copy calls if possible - - they create an implicit, safe(hopefully) casting mechanism between the - virtual interface and derived object - - throws DataStreamTypeError if a cast fails */ - -#define VIRTUAL_VOID_PUSH_2ARG_DROP(InTy, OutTy) \ -virtual void \ -push(const InTy v, secondary_ty sec = secondary_ty()) \ -{ \ - this->push((OutTy)v, std::move(sec)); \ -} - -#define VIRTUAL_VOID_PUSH_2ARG_BREAK(InTy) \ -virtual void \ -push(const InTy v, secondary_ty sec = secondary_ty()) \ -{ \ - this->push(std::to_string(v) , std::move(sec)); \ -} - -#define VIRTUAL_VOID_COPY_2ARG_DROP(InTy, OutTy) \ -virtual size_t \ -copy(InTy *dest, size_t sz, int end = -1, int beg = 0, secondary_ty *sec = nullptr) const \ -{ \ - return this->_copy(dest, sz, end, beg, sec); \ -} - -#define VIRTUAL_VOID_COPY_2ARG_BREAK(InTy, DropBool) \ -virtual size_t \ -copy(InTy *dest, size_t sz, int end = -1, int beg = 0, secondary_ty *sec = nullptr) const \ -{ \ - BuildThrowTypeError("copy()"); \ - return 0; \ -} - -#define VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(InTy, OutTy) \ -virtual long long \ -copy_from_marker(InTy *dest, size_t sz, int beg = 0, secondary_ty *sec = nullptr) const \ -{ \ - return this->_copy_using_atomic_marker< OutTy >(dest, sz, beg, sec); \ -} - -#define VIRTUAL_VOID_MARKER_COPY_2ARG_BREAK(InTy, DropBool) \ -virtual long long \ -copy_from_marker(InTy *dest, size_t sz, int beg = 0, secondary_ty *sec = nullptr) const \ -{ \ - BuildThrowTypeError("copy_from_marker()"); \ - return 0; \ -} - -#define VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(InTy, OutTy) \ -virtual long long \ -ncopy_from_marker(InTy *dest, size_t sz, secondary_ty *sec = nullptr) const \ -{ \ - return this->_ncopy_using_atomic_marker< OutTy >(dest, sz, sec); \ -} - -#define VIRTUAL_VOID_MARKER_N_COPY_2ARG_BREAK(InTy, DropBool) \ -virtual long long \ -ncopy_from_marker(InTy *dest, size_t sz, secondary_ty *sec = nullptr) const \ -{ \ - BuildThrowTypeError("ncopy_from_marker()"); \ - return 0; \ -} - - VIRTUAL_VOID_PUSH_2ARG_DROP(float, double) - VIRTUAL_VOID_PUSH_2ARG_BREAK(double) - VIRTUAL_VOID_PUSH_2ARG_DROP(unsigned char, unsigned short) - VIRTUAL_VOID_PUSH_2ARG_DROP(unsigned short, unsigned int) - VIRTUAL_VOID_PUSH_2ARG_DROP(unsigned int, unsigned long) - VIRTUAL_VOID_PUSH_2ARG_DROP(unsigned long, unsigned long long) - VIRTUAL_VOID_PUSH_2ARG_BREAK(unsigned long long) - VIRTUAL_VOID_PUSH_2ARG_DROP(char, short) - VIRTUAL_VOID_PUSH_2ARG_DROP(short, int) - VIRTUAL_VOID_PUSH_2ARG_DROP(int, long) - VIRTUAL_VOID_PUSH_2ARG_DROP(long, long long) - VIRTUAL_VOID_PUSH_2ARG_BREAK(long long) - - virtual void - push(const std::string str, secondary_ty sec = secondary_ty()) - { - if(this->_str_push_count++){ - this->_str_push_count = 0; - BuildThrowTypeError("push()"); - } - this->push(str.c_str(), std::move(sec)); - } - - virtual void - push(const char* str, secondary_ty sec = secondary_ty()) - { - if(this->_str_push_count++){ - this->_str_push_count = 0; - BuildThrowTypeError("push()"); - } - this->push(std::string(str), std::move(sec)); - } - - VIRTUAL_VOID_COPY_2ARG_DROP(long long, long) - VIRTUAL_VOID_COPY_2ARG_DROP(long, int) - VIRTUAL_VOID_COPY_2ARG_DROP(int, short) - VIRTUAL_VOID_COPY_2ARG_DROP(short, char) - VIRTUAL_VOID_COPY_2ARG_BREAK(char, true) - VIRTUAL_VOID_COPY_2ARG_DROP(unsigned long long, unsigned long) - VIRTUAL_VOID_COPY_2ARG_DROP(unsigned long, unsigned int) - VIRTUAL_VOID_COPY_2ARG_DROP(unsigned int, unsigned short) - VIRTUAL_VOID_COPY_2ARG_DROP(unsigned short, unsigned char) - VIRTUAL_VOID_COPY_2ARG_BREAK(unsigned char, true) - VIRTUAL_VOID_COPY_2ARG_DROP(double, float) - VIRTUAL_VOID_COPY_2ARG_BREAK(float, false) - - virtual size_t - copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - virtual size_t - copy(std::string *dest, - size_t sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(long long, long) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(long, int) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(int, short) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(short, char) - VIRTUAL_VOID_MARKER_COPY_2ARG_BREAK(char, true) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(unsigned long long, unsigned long) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(unsigned long, unsigned int) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(unsigned int, unsigned short) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(unsigned short, unsigned char) - VIRTUAL_VOID_MARKER_COPY_2ARG_BREAK(unsigned char, true) - VIRTUAL_VOID_MARKER_COPY_2ARG_DROP(double, float) - VIRTUAL_VOID_MARKER_COPY_2ARG_BREAK(float, false) - - virtual long long - copy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - int beg = 0, - secondary_ty *sec = nullptr) const; - - virtual long long - copy_from_marker(std::string *dest, - size_t sz, - int beg = 0, - secondary_ty *sec = nullptr) const; - - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(long long, long) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(long, int) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(int, short) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(short, char) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_BREAK(char, true) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(unsigned long long, unsigned long) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(unsigned long, unsigned int) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(unsigned int, unsigned short) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(unsigned short, unsigned char) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_BREAK(unsigned char, true) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_DROP(double, float) - VIRTUAL_VOID_MARKER_N_COPY_2ARG_BREAK(float, false) - - virtual long long - ncopy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - secondary_ty *sec = nullptr) const; - - virtual long long - ncopy_from_marker(std::string *dest, - size_t sz, - secondary_ty *sec = nullptr) const; - - virtual void /* SHOULD WE THROW? */ - secondary(secondary_ty *dest, int indx) const - { - dest = nullptr; - } - -}; /* class DataStreamInterface */ - - -template< typename SecTy, typename GenTy > -std::ostream& -operator<<(std::ostream&, const DataStreamInterface&); - - -template> -class DataStream /* CONTAINS A PRIMARY DEQUE */ - : public DataStreamInterface{ - typedef DataStream _my_ty; - typedef DataStreamInterface _my_base_ty; - - class{ - static const bool valid = GenTy::TypeCheck::value; - static_assert(valid, "DataStream: Ty failed GenTy type-check"); - }_my_ty_static_assert; - - _my_ty& - operator=(const _my_ty &); - - void - _push(const Ty v); - -protected: - typedef std::lock_guard _my_lock_guard_type; - - std::deque _deque_primary; - - size_t _qbound; - size_t _qcount; - - long long *const _mark_count; - bool *const _mark_is_dirty; - - volatile bool _push_has_priority; - - std::recursive_mutex *const _mtx; - - inline void - _yld_to_push() const - { - if(!_push_has_priority) - std::this_thread::yield(); - } - - template - bool - _check_adj(int& end, int& beg, const std::deque& d) const; - - void - _incr_internal_counts(); - - template - size_t - _copy_to_ptr(DequeTy& d, - DestTy *dest, - size_t sz, - unsigned int end, - unsigned int beg) const; - -public: - typedef _my_base_ty interface_type; - typedef Ty value_type; - - DataStream(size_t sz); - DataStream(const _my_ty & stream); - DataStream(_my_ty && stream); - - virtual - ~DataStream(); - - inline bool - empty() const - { - return _deque_primary.empty(); - } - - inline size_t - size() const - { - return _qcount; - } - - inline bool - is_marker_dirty() const - { - return *_mark_is_dirty; - } - - inline long long - marker_position() const - { - return *_mark_count; - } - - inline size_t - bound_size() const - { - return _qbound; - } - - size_t - bound_size(size_t sz); - - inline void - push(const Ty v, secondary_ty sec = secondary_ty()) - { - _str_push_count = 0; - _push(v); - } - - inline void - push(const generic_ty& gen, secondary_ty sec = secondary_ty()) - { - _str_push_count = 0; - _push((Ty)gen); - } - - long long - ncopy_from_marker(Ty *dest, - size_t sz, - secondary_ty *sec = nullptr) const; - - long long - ncopy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - secondary_ty *sec = nullptr) const; - - long long - copy_from_marker(Ty *dest, - size_t sz, - int beg = 0, - secondary_ty *sec = nullptr) const; - - long long - copy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - int beg = 0, - secondary_ty *sec = nullptr) const; - - size_t - copy(Ty *dest, - size_t sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - size_t - copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - generic_ty - operator[](int indx) const; - - both_ty - both(int indx) const; - - generic_vector_ty - vector(int end = -1, int beg = 0) const; - - secondary_vector_ty - secondary_vector(int end = -1, int beg = 0) const; - - // BUG-FIX: allow frame calls not to move marker Dec 8 2017 - generic_ty - get_leave_marker(int indx) const; - - // BUG-FIX: allow frame calls not to move marker Dec 8 2017 - both_ty - both_leave_marker(int indx) const; -}; - - -template -class DataStream /* CONTAINS PRIMARY AND SECONDARY DEQUE */ - : public DataStream { - typedef DataStream _my_ty; - typedef DataStream _my_base_ty; - - std::deque _deque_secondary; - - void - _push(const Ty v, const secondary_ty sec); - -public: - typedef Ty value_type; - - DataStream(size_t sz); - DataStream(const _my_ty & stream); - DataStream(_my_ty && stream); - - size_t - bound_size(size_t sz); - - inline void - push(const Ty v, secondary_ty sec = secondary_ty()) - { - _str_push_count = 0; - _push(v, std::move(sec)); - } - - inline void - push(const generic_ty& gen, secondary_ty sec = secondary_ty()) - { - _str_push_count = 0; - _push((Ty)gen, std::move(sec)); - } - - size_t - copy(Ty *dest, - size_t sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - size_t - copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0, - secondary_ty *sec = nullptr) const; - - both_ty - both(int indx) const; - - void - secondary(secondary_ty *dest, int indx) const; - - secondary_vector_ty - secondary_vector(int end = -1, int beg = 0) const; - - // BUG-FIX: allow frame calls not to move marker Dec 8 2017 - both_ty - both_leave_marker(int indx) const; -}; - - -#include "../src/data_stream.tpp" - - -class DataStreamError - : public std::exception{ -public: - DataStreamError(const char* info) - : - std::exception(info) - { - } - - virtual - ~DataStreamError() - { - } -}; - -class DataStreamTypeError - : public DataStreamError{ -public: - DataStreamTypeError(const char* info) - : - DataStreamError(info) - { - } -}; - -class DataStreamSizeViolation - : public DataStreamError{ -public: - const size_t bound_size; - const size_t deque_size; - - DataStreamSizeViolation(const char* msg, size_t bound_size, size_t deque_size) - : - DataStreamError(msg), - bound_size(bound_size), - deque_size(deque_size) - { - } -}; - -class DataStreamOutOfRange - : public DataStreamError{ -public: - const int size; - const int beg; - const int end; - - DataStreamOutOfRange(const char* msg, int sz, int beg, int end) - : - DataStreamError(msg), - size(sz), - beg(beg), - end(end) - { - } -}; - -class DataStreamInvalidArgument - : public DataStreamError { -public: - DataStreamInvalidArgument(const char* msg) - : - DataStreamError(msg) - { - } -}; - -#endif diff --git a/include/exceptions.hpp b/include/exceptions.hpp deleted file mode 100644 index 9af4d0b..0000000 --- a/include/exceptions.hpp +++ /dev/null @@ -1,189 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_EXCEPTIONS -#define JO_TOSDB_EXCEPTIONS - -#include -#include -#include - -class TOSDB_Error - : public std::exception{ - DWORD _threadID; - DWORD _processID; - std::string _tag; - std::string _info; - -public: - TOSDB_Error(std::string info, std::string tag) - : - std::exception(info.c_str()), - _tag(tag), - _info(info), - _threadID(GetCurrentThreadId()), - _processID(GetCurrentProcessId()) - { - } - - TOSDB_Error(const std::exception& e, std::string tag) - : - std::exception(e), - _tag(tag), - _threadID(GetCurrentThreadId()), - _processID(GetCurrentProcessId()) - { - } - - TOSDB_Error(const std::exception& e, std::string info, std::string tag) - : - std::exception(e), - _tag(tag), - _info(info), - _threadID(GetCurrentThreadId()), - _processID(GetCurrentProcessId()) - { - } - - virtual - ~TOSDB_Error() - { - } - - inline DWORD - threadID() const - { - return _threadID; - } - - inline DWORD - processID() const - { - return _processID; - } - - inline std::string - tag() const - { - return _tag; - } - - inline std::string - info() const - { - return _info; - } - - /* do better than this */ - inline std::string - info_and_what() const - { - return "info: " + _info + ", what: " + std::string(what()); - } -}; - - -class TOSDB_BufferError - : public TOSDB_Error{ -public: - TOSDB_BufferError(std::string info, std::string tag = "DataBuffer") - : - TOSDB_Error(info, tag) - { - } -}; - - -class TOSDB_DDE_Error - : public TOSDB_Error{ -public: - TOSDB_DDE_Error(std::string info, std::string tag = "DDE") - : - TOSDB_Error(info, tag) - { - } - - TOSDB_DDE_Error(const std::exception& e, std::string info, std::string tag = "DDE") - : - TOSDB_Error(e, info, tag) - { - } -}; - - -class TOSDB_DataBlockError - : public TOSDB_Error{ -public: - TOSDB_DataBlockError(std::string info, std::string tag = "DataBlock") - : - TOSDB_Error(info, tag) - { - } - - TOSDB_DataBlockError(const std::exception& e, std::string info, std::string tag = "DataBlock") - : - TOSDB_Error(e, info, tag) - { - } -}; - - -class TOSDB_DataBlockDoesntExist - : public TOSDB_DataBlockError{ -public: - TOSDB_DataBlockDoesntExist(std::string name) - : - TOSDB_DataBlockError("block (" + std::string(name) + ") doesn't exist") - { - } -}; - - -class TOSDB_DataBlockLimitError - : public TOSDB_DataBlockError{ -public: - const size_t limit; - - TOSDB_DataBlockLimitError(const size_t limit) - : - TOSDB_DataBlockError("would exceed RawDataBlock limit"), - limit(limit) - { - } -}; - - -/* TOSDB_DataStreamError is the 'external' version of DataStreamError (data_stream.hpp/tpp) - The latter is caught 'internally' and passed to TOSDB_DataStreamError cstr */ -class TOSDB_DataStreamError - : public TOSDB_DataBlockError{ -public: - TOSDB_DataStreamError(std::string info, std::string tag = "DataStream") - : - TOSDB_DataBlockError(info, tag) - { - } - - TOSDB_DataStreamError(const std::exception& e, std::string info, std::string tag = "DataStream") - : - TOSDB_DataBlockError(e, info, tag) - { - } -}; - - -#endif diff --git a/include/generic.hpp b/include/generic.hpp deleted file mode 100644 index 2828971..0000000 --- a/include/generic.hpp +++ /dev/null @@ -1,332 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_JO_GENERIC -#define JO_JO_GENERIC - -#include -#include -#include -#include -#include -#include -#include -#include - -/* why can't we just forward decl in tos_databridge.h ? */ -#if defined(THIS_EXPORTS_INTERFACE) -#define DLL_SPEC_IFACE __declspec (dllexport) -#elif defined(THIS_DOESNT_IMPORT_INTERFACE) -#define DLL_SPEC_IFACE -#else -#define DLL_SPEC_IFACE __declspec (dllimport) /* default == import */ -#endif /* INTERFACE */ - -template /* forward decl */ -inline T -CastGenericFromString(std::string& str); - -/* - TODO: optimize move semantics -*/ - -class DLL_SPEC_IFACE TOSDB_Generic{ - - /* The types we support. Because the biggest object we hold is 8-bytes, we - need to store the type (1-byte), and we are 8-byte aligned we have plenty - of space to expand the supported types and/or cache helper data. Of course, - the internal logic may get a bit cloudier... - - by using sequential ints the compiler *should* build constant-time lookup - tables for the switch statements we use to branch depending on type - - note: we don't hold a 'void' object per-se; when an object holding a string - is 'moved' we set the _type_val to TYVAL_VOID so the destructor knows - not to free the junk _sub then points at */ - - static const int TYVAL_VOID = 0; - static const int TYVAL_LONG = 1; - static const int TYVAL_LONG_LONG = 2; - static const int TYVAL_FLOAT = 3; - static const int TYVAL_DOUBLE = 4; - static const int TYVAL_STRING = 5; - - /* max string length we support is 255 chars (why?) */ - static const int STR_MAX = UCHAR_MAX - 1; - - /* the (max) byte size of our internal representation */ - static const int SUB_SZ = 8; - - /* raw mem holding our 'object' - - an array of byte objects seems the logical way to represent this */ - uint8_t _sub[SUB_SZ]; - - /* the type of our 'object' - - we can expand this to 64 bits because of alignment*/ - unsigned char _type_val; - - /* _val_switch handles casts internally and supports most of the public - interface; should be fast, comprehensive, and safe (in that order, i think) */ - template< typename T > - T - _val_switch() const - { - try{ - switch(_type_val){ - case TYVAL_LONG: - case TYVAL_LONG_LONG: - return (T)(*(long long*)(_sub)); - case TYVAL_FLOAT: - case TYVAL_DOUBLE: - return (T)(*(double*)(_sub)); - case TYVAL_STRING: - return (T)CastGenericFromString(**(std::string**)(_sub)); - default: throw; - } - }catch(...){ - std::ostringstream s; - s << "error casting generic to < " << typeid(T).name() << " >"; -#ifdef __GNUC__ - throw std::bad_cast(); -#else /* GNU doesn't like */ - throw std::bad_cast(s.str().c_str()); -#endif - } - } - -public: - template - struct TypeCheck { - static const bool value = - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value; - }; - - explicit TOSDB_Generic(long val) - : - _type_val( TYVAL_LONG ) - { - *(long long*)_sub = val; - } - - explicit TOSDB_Generic(long long val) - : - _type_val( TYVAL_LONG_LONG ) - { - *(long long*)_sub = val; - } - - explicit TOSDB_Generic(float val) - : - _type_val( TYVAL_FLOAT ) - { - *(double*)_sub = val; - } - - explicit TOSDB_Generic(double val) - : - _type_val( TYVAL_DOUBLE ) - { - *(double*)_sub = val; - } - - explicit TOSDB_Generic(std::string str) - : /* ! must be able to delete (std::string*)_sub at ANY TIME !*/ - _type_val( TYVAL_STRING ) - { /* let string do all the heavy lifting */ - *(std::string**)_sub = new std::string(str.substr(0,STR_MAX)); - } - - TOSDB_Generic(const TOSDB_Generic& gen); - - TOSDB_Generic(TOSDB_Generic&& gen); - - TOSDB_Generic& - operator=(const TOSDB_Generic& gen); - - TOSDB_Generic& - operator=(TOSDB_Generic&& gen); - - ~TOSDB_Generic() - { - if( is_string() ) - delete *(std::string**)_sub; - } - - inline bool /* same type AND value */ - operator==(const TOSDB_Generic& gen) const - { - return (_type_val == gen._type_val ) && ( as_string() == gen.as_string()); - } - - inline bool /* different type OR value */ - operator!=(const TOSDB_Generic& gen) const - { - return !(operator==(gen)); - } - - size_t - size() const; - - inline bool - is_float() const - { - return (_type_val == TYVAL_FLOAT); - } - - inline bool - is_double() const - { - return (_type_val == TYVAL_DOUBLE); - } - - inline bool - is_long() const - { - return (_type_val == TYVAL_LONG); - } - - inline bool - is_long_long() const - { - return (_type_val == TYVAL_LONG_LONG); - } - - inline bool - is_string() const - { - return (_type_val == TYVAL_STRING); - } - - inline bool - is_floating_point() const - { - return (_type_val == TYVAL_FLOAT || _type_val == TYVAL_DOUBLE); - } - - inline bool - is_integer() const - { - return (_type_val == TYVAL_LONG || _type_val == TYVAL_LONG_LONG); - } - - inline long - as_long() const - { - return _val_switch(); - } - - inline long long - as_long_long() const - { - return _val_switch(); - } - - inline float - as_float() const - { - return _val_switch(); - } - - inline double - as_double() const - { - return _val_switch(); - } - - std::string - as_string() const; - - inline operator - long() const - { - return _val_switch(); - } - - inline operator - long long() const - { - return _val_switch(); - } - - inline operator - float() const - { - return _val_switch(); - } - - inline operator - double() const - { - return _val_switch(); - } - - inline operator - std::string() const - { - return as_string(); - } -}; - - -template -inline T -CastGenericFromString(std::string& str) -{ - static_assert(TOSDB_Generic::TypeCheck::value, - "don't know how to cast to this type"); -} - -template<> -inline long long -CastGenericFromString(std::string& str) -{ - return std::stoll(str); -} - -template<> -inline long -CastGenericFromString(std::string& str) -{ - return std::stol(str); -} - -template<> -inline double -CastGenericFromString(std::string& str) -{ - return std::stod(str); -} - -template<> -inline float -CastGenericFromString(std::string& str) -{ - return std::stof(str); -} - -template<> -inline std::string -CastGenericFromString(std::string& str) -{ - return std::string(str); -} - -#endif diff --git a/include/initializer_chain.hpp b/include/initializer_chain.hpp deleted file mode 100644 index 1aaf736..0000000 --- a/include/initializer_chain.hpp +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_INITIALIZER_CHAIN -#define JO_TOSDB_INITIALIZER_CHAIN - -/* - InitializerChain is a 'chaining' mechanism for inline global construction of - certain 'container' objects if initializer lists are not supported. It accepts an - optional custom argument type and functor for transforming/inserting each element. - - If no custom argument type is passed the 'container' MUST expose an - appropriate public 'value_type' typedef to default to. - - Multiple arguments will be converted to std::pair, - std::pair> or std::tuple depending on - # of args and template argument paramater. - - Ex. - - std::map my_map_type; - std::tuple my_entry_type; - - void - MyInserter(std::map& m, my_entry_type e) - { - m[std::get<0>(e)] = std::get<1>(e) * std::get<2>(e); - } - - const my_map_type my_map = - InitializerChain - (my_entry_type("one",1,1)) - (my_entry_type("two",2,2)) - (my_entry_type("three",3,3)); - - This is equivalent to: - - const my_map_type my_map = { {"one",1}, {"two",4}, {"three",9} }; - - - - OR - (in certain cases, like this one) we can use the default version: - - const my_map_type my_map = - InitializerChain - ("one",1*1) - ("two",2*2) - ("three",3*3); - -*/ - -template -void -DefaultInsert(T& t, A a) -{ /* standard insert available to most containers */ - t.insert(t.end(), a); -} - - -template > -class InitializerChain { - T _t; - - /* all the specializations are below */ - template - struct _p_t_branch{ - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(Obj(f,s,t)); - } - }; - -public: - InitializerChain(A arg) - { - operator()(arg); - } - - template - InitializerChain(First f, Second s) - { - operator()(f, s); - } - - template - InitializerChain(First f, Second s, Third t) - { - operator()(f, s, t); - } - - InitializerChain& - operator()(A arg) - { - func(_t, arg); - return *this; - } - - template - InitializerChain& - operator()(First f, Second s) - { - return operator()(std::pair(f,s)); - } - - template - InitializerChain& - operator()(First f, Second s, Third t) - { - return _p_t_branch()(f,s,t,this); - } - - operator T() const - { - return _t; - } - -private: - /* 'general' specialization for pair< obj, pair > arg type */ - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - /* specialization for strings to deal with pair implicit coversion issues */ - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - - template - struct _p_t_branch> > { - template - InitializerChain& - operator()(First f, Second s, Third t, My *self) - { - return self->operator()(f, std::pair(s,t)); - } - }; - -}; - - -#endif JO_TOSDB_INITIALIZER_CHAIN \ No newline at end of file diff --git a/include/ipc.hpp b/include/ipc.hpp deleted file mode 100644 index a0e1f40..0000000 --- a/include/ipc.hpp +++ /dev/null @@ -1,181 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_IPC -#define JO_TOSDB_IPC - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "concurrency.hpp" - - -class IPCBase{ -public: - /* we limit message size to make the implemenation much easier/safer; - a more robust mechanism could manually write/read to/from the pipe server - and loop on the read calls until ERROR_MORE_DATA isn't returned */ - static const int MAX_MESSAGE_SZ = 511; // 2**9 excluding \0 - - /* arbitrary value the 'probe' channel sends/recieves to confirm connection */ -#ifdef BUILD64BIT - static const uint8_t PROBE_BYTE = 64; - static const uint8_t PROBE_BYTE_WRONG_ARCH = 32; -#else - static const uint8_t PROBE_BYTE = 32; - static const uint8_t PROBE_BYTE_WRONG_ARCH = 64; -#endif - - bool - connected(unsigned long timeout); - -protected: - std::string _main_channel_pipe_name; - std::string _probe_channel_pipe_name; - - HANDLE _main_channel_pipe_hndl; - HANDLE _probe_channel_pipe_hndl; - - bool - send(std::string msg) const; - - bool - recv(std::string *msg) const; - - template - bool - _call_pipe( std::string name, - T* in, - DWORD in_sz, - T* out, - DWORD out_sz, - unsigned long timeout, - std::function handle_good_call, - std::function handle_file_not_found = nullptr ); - - IPCBase(std::string name) - : - _main_channel_pipe_name(std::string("\\\\.\\pipe\\") - .append(name).append("_main_channel_pipe")), - _probe_channel_pipe_name(std::string("\\\\.\\pipe\\") - .append(name).append("_probe_channel_pipe")), - _main_channel_pipe_hndl(INVALID_HANDLE_VALUE), - _probe_channel_pipe_hndl(INVALID_HANDLE_VALUE) - { - } - - ~IPCBase() - { - } -}; - - -class IPCSlave - : public IPCBase{ - static const int ACL_SIZE = 144; - - SECURITY_ATTRIBUTES _sec_attr; - SECURITY_DESCRIPTOR _sec_desc; - SmartBuffer _sec_sid; - SmartBuffer _sec_acl; - - std::thread _probe_channel_thread; - bool _probe_channel_run_flag; - - void - _init_security_objects(); - - HANDLE - _create_pipe(std::string name); - - void - _listen_for_probes(); - -public: - IPCSlave(std::string name) - : - IPCBase(name), - _sec_attr(SECURITY_ATTRIBUTES()), - _sec_desc(SECURITY_DESCRIPTOR()), - _sec_sid(SECURITY_MAX_SID_SIZE), - _sec_acl(ACL_SIZE), - _probe_channel_run_flag(true) - { - _init_security_objects(); - - _main_channel_pipe_hndl = _create_pipe(_main_channel_pipe_name); - _probe_channel_pipe_hndl = _create_pipe(_probe_channel_pipe_name); - - /* launch our probe channel so master can asynchronously check connection status*/ - _probe_channel_thread = std::thread( std::bind(&IPCSlave::_listen_for_probes,this) ); - } - - ~IPCSlave() - { - CloseHandle(_main_channel_pipe_hndl); - - _probe_channel_run_flag = false; - connected(TOSDB_DEF_TIMEOUT); /* force evaluation of the run flag */ - - if(_probe_channel_thread.joinable()) - _probe_channel_thread.join(); - - CloseHandle(_probe_channel_pipe_hndl); - } - - bool - wait_for_master(); - - void - drop_master() - { - FlushFileBuffers(_main_channel_pipe_hndl); - DisconnectNamedPipe(_main_channel_pipe_hndl); - } - - using IPCBase::send; - using IPCBase::recv; -}; - - -class IPCMaster - : public IPCBase{ -public: - IPCMaster(std::string name) - : - IPCBase(name) - { - } - - ~IPCMaster() - { - } - - bool - call(std::string *msg, unsigned long timeout); -}; - - - - -#endif diff --git a/include/raw_data_block.hpp b/include/raw_data_block.hpp deleted file mode 100644 index fd0a460..0000000 --- a/include/raw_data_block.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_RAW_DATA_BLOCK -#define JO_TOSDB_RAW_DATA_BLOCK - -#include "data_stream.hpp" -#include "client.hpp" -#include - -/* implemented in src/raw_data_block.tpp */ - -#define RAW_DATA_BLOCK_TEMPLATE template -#define RAW_DATA_BLOCK_CLASS RawDataBlock -#define MAX_BLOCK_COUNT 10 - -template -class RawDataBlock { - static size_type _block_count_; - static size_type _max_block_count_; - - typedef std::map>, - TOS_Topics::top_less> _my_row_ty; - - typedef std::unordered_map> _my_block_ty; - - _my_block_ty _block; - size_type _block_sz; - str_set_type _item_names; - topic_set_type _topic_enums; - bool _datetime; - std::recursive_mutex *const _mtx; - - RawDataBlock(str_set_type items, - topic_set_type topics_t, - const size_type sz, - bool datetime); - - RawDataBlock(const size_type sz, bool datetime); - - RawDataBlock(const RawDataBlock& block) - : - _mtx(new std::recursive_mutex) - { - /* ++_block_count_; */ - } - - RawDataBlock(RawDataBlock&& block) - : - _mtx(new std::recursive_mutex) - { - /* */ - } - - RawDataBlock& - operator=(const RawDataBlock& block); - - RawDataBlock& - operator=(RawDataBlock&& block); - - void - _init(); - - _my_row_ty* - _insert_topic(_my_row_ty*, TOS_Topics::TOPICS topic); - - std::unique_ptr<_my_row_ty> - _populate_tblock(std::unique_ptr<_my_row_ty> tblock); - -public: - typedef GenericTy generic_type; - typedef DateTimeTy datetime_type; - typedef DataStreamInterface stream_type; - typedef const DataStreamInterface* stream_const_ptr_type; - - typedef std::vector vector_type; - typedef std::pair pair_type; - typedef std::map map_type; - typedef std::map matrix_type; - - typedef std::pair, - std::vector> vector_datetime_type; - - typedef std::map> map_datetime_type; - - typedef std::map matrix_datetime_type; - - static RawDataBlock* const - CreateBlock(const str_set_type items, - const topic_set_type topics_t, - const size_type sz, - const bool datetime); - - static RawDataBlock* const - CreateBlock(const size_type sz,const bool datetime); - - static inline size_type - block_count() - { - return _block_count_; - } - - static inline size_type - max_block_count() - { - return _max_block_count_; - } - - static inline size_type - max_block_count(const size_type m) - { - return (m >= _block_count_) ? (_max_block_count_ = m) : _max_block_count_; - } - - size_type - block_size(size_type b); - - inline size_type - block_size() const - { - return _block_sz; - } - - inline size_type - item_count() const - { - return (size_type)(_item_names.size()); - } - - inline size_type - topic_count() const - { - return (size_type)(_topic_enums.size()); - } - - void - add_item(std::string item); - - void - remove_item(std::string item); - - void - add_topic(TOS_Topics::TOPICS topic); - - void - remove_topic(TOS_Topics::TOPICS topic); - - template - void - insert_data(TOS_Topics::TOPICS topic,std::string item,Val val,DT datetime); - - const DataStreamInterface* - raw_stream_ptr(std::string item, TOS_Topics::TOPICS topic) const; - - map_type - map_of_frame_items(TOS_Topics::TOPICS topic) const; - - map_type - map_of_frame_topics(std::string item) const; - - map_datetime_type - pair_map_of_frame_items(TOS_Topics::TOPICS) const; - - map_datetime_type - pair_map_of_frame_topics(std::string item) const; - - matrix_type - matrix_of_frame() const ; - - matrix_datetime_type - pair_matrix_of_frame() const ; - - inline topic_set_type - topics() const - { - return _topic_enums; - } - - inline str_set_type - items() const - { - return _item_names; - } - - inline bool - has_topic(TOS_Topics::TOPICS topic) const - { - return (_topic_enums.find(topic) != _topic_enums.cend()); - } - - inline bool - has_item(const char* item) const - { - return (_item_names.find(item) != _item_names.cend()); - } - - inline bool - uses_dtstamp() const - { - return _datetime; - } - - ~RawDataBlock() - { /* all other deallocs are handled by unique_ptr destructors */ - delete _mtx; - --_block_count_; - } -}; - -#include "../src/raw_data_block.tpp" - -#endif diff --git a/include/shell.hpp b/include/shell.hpp deleted file mode 100644 index 29f9e29..0000000 --- a/include/shell.hpp +++ /dev/null @@ -1,276 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_SHELL -#define JO_TOSDB_SHELL - -#include -#include - -#include "tos_databridge.h" -#include "initializer_chain.hpp" - -#define MAX_DISPLAY_WIDTH 80 -#define LEFT_INDENT_SIZE 10 -#define CMD_OUT_PER_PAGE 10 - -/* forward decl */ -class stream_prompt; - -class stream_prompt_base { -protected: - std::string _prmpt; - - explicit stream_prompt_base(std::string prompt) - : - _prmpt(prompt) - { - } - - virtual - ~stream_prompt_base() - { - } - -public: - /* return the base */ - template - stream_prompt_base& - operator<<(const T& val) - { - std::cout<< val; - return *this; - } - - /* return the base */ - template< typename T > - stream_prompt_base& - operator>>(T& val) - { - std::cout<< _prmpt << ' '; - std::cin>> val; - return *this; - } - - /* return the derived by casting from base - this is safe if done from outside since we protect constructor */ - static stream_prompt& - endl(stream_prompt_base& p) - { - std::cout<< std::endl; - return (stream_prompt&)(p); - } - - inline stream_prompt& - operator<<(decltype(stream_prompt_base::endl) val) - { - return (*val)(*this); - } - - inline stream_prompt& - operator>>(decltype(stream_prompt_base::endl) val) - { - return (*val)(*this); - } -}; - -class stream_prompt - : public stream_prompt_base { -public: - explicit stream_prompt(std::string prompt) - : - stream_prompt_base(prompt) - { - } - - /* RETURN THE BASE */ - template - stream_prompt_base& - operator<<(const T& val) - { - std::cout<< _prmpt << ' ' << val << ' '; - return *this; - } -}; - -class stream_prompt_basic - : public stream_prompt_base { -public: - explicit stream_prompt_basic(std::string prompt) - : - stream_prompt_base(prompt) - { - } - - /* DONT RETURN THE BASE */ - template - stream_prompt_basic& - operator<<(const T& val) - { - std::cout<< _prmpt << ' ' << val << ' '; - return *this; - } -}; - -extern stream_prompt prompt; -extern stream_prompt_basic prompt_b; - -/* fwrd decl */ -struct CommandCtx; - -typedef void(*commands_func_ty)(CommandCtx*); - -struct CommandCtx{ - std::string name; - std::string doc; - commands_func_ty func; - bool exit; -}; - -class CommandsMap - : public std::map { -/* - since we have no initlist, wrap map and provide our own InitChain functor that - derives from a 'chaining' mechanism for inline global construction of a const map - */ -private: - typedef public std::map _my_base_ty; - typedef std::tuple _my_init_arg_ty; - - static void - _insert(_my_base_ty& t, _my_init_arg_ty a) - { - CommandCtx c = {std::get<0>(a), std::get<2>(a), std::get<1>(a), false}; - std::inserter(t, t.end()) = std::make_pair(std::get<0>(a), std::move(c)); - } - - typedef InitializerChain<_my_base_ty, _my_init_arg_ty, _insert> _my_init_base_ty; - -public: - class InitChain - : public _my_init_base_ty { - public: - explicit - InitChain(std::string name, commands_func_ty func, std::string doc="") - : - _my_init_base_ty(_my_init_arg_ty(name,func,doc)) - { - } - - InitChain& - operator()(std::string name, commands_func_ty func, std::string doc="") - { - _my_init_base_ty::operator()(_my_init_arg_ty(name,func,doc)); - return *this; - } - }; - - CommandsMap() - { - } - - CommandsMap(const InitChain& i) - : - _my_base_ty(i) - { - } -}; - -extern CommandsMap commands_admin; -extern CommandsMap commands_get; -extern CommandsMap commands_stream; -extern CommandsMap commands_frame; -extern CommandsMap commands_local; - -/*wrap a reference so we can get around pair restrictions*/ -class CommandsMapRef{ - CommandsMap& _m; -public: - CommandsMapRef(CommandsMap& m) - : - _m(m) - { - } - - CommandsMap::iterator - begin() - { - return _m.begin(); - } - - CommandsMap::iterator - end() - { - return _m.end(); - } -}; - -typedef std::pair command_display_pair; -typedef std::unordered_map commands_map_of_maps_ty; - -extern commands_map_of_maps_ty commands; - -enum class language - : int { - none = 0, - c = 1, - cpp = 2 -}; - -typedef std::unordered_map> language_strings_ty; - -extern language_strings_ty language_strings; - -language -set_default_language(language l); - -language -get_default_language(); - -size_type -get_cstr_entries(std::string label, char ***pstrs, CommandCtx *ctx); - -inline void -prompt_for(std::string sout, std::string* sin, CommandCtx *ctx) -{ - prompt_b<< ctx->name << sout >> *sin; -} - -bool -prompt_for_cpp(CommandCtx *ctx, int recurse=2); - -bool -prompt_for_datetime(std::string block, CommandCtx *ctx); - -void -prompt_for_item_topic(std::string *pitem, std::string *ptopic, CommandCtx *ctx); - -void -prompt_for_block_item_topic(std::string *pblock, - std::string *pitem, - std::string *ptopic, - CommandCtx *ctx); - -void -prompt_for_block_item_topic_index(std::string *pblock, - std::string *pitem, - std::string *ptopic, - std::string *pindex, - CommandCtx *ctx); - - -#endif /* JO_TOSDB_SHELL */ \ No newline at end of file diff --git a/include/tos_databridge.h b/include/tos_databridge.h deleted file mode 100644 index 390fd58..0000000 --- a/include/tos_databridge.h +++ /dev/null @@ -1,1237 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#ifndef JO_TOSDB_DATABRIDGE -#define JO_TOSDB_DATABRIDGE - -#ifndef _WIN32 -#error tos-databridge core libs require Windows ! -#endif - -#ifdef _WIN64 -#define BUILD64BIT -#endif - -/* internal objects of exported impl classes not exported; - look here if link errors on change of_tos-databridge... mods */ -#pragma warning(once : 4251) -/* decorated name length exceeded warning */ -#pragma warning(disable : 4503) -/* chkd iterator warning: we check internally */ -#pragma warning(disable : 4996) -/* BOOL to bool warn */ -#pragma warning(disable : 4800) - -#ifdef __cplusplus -#define EXT_C_SPEC extern "C" -/* make things a bit clearer */ -#define NO_THROW __declspec(nothrow) -#else -#define EXT_C_SPEC -#define NO_THROW -#endif - -/* the following manages what is imported/exported from various modules, - check the preprocessor options for each module to see what is defined*/ - -#if defined(THIS_EXPORTS_INTERFACE) -#define DLL_SPEC_IFACE __declspec (dllexport) -#elif defined(THIS_DOESNT_IMPORT_INTERFACE) -#define DLL_SPEC_IFACE -#else -#define DLL_SPEC_IFACE __declspec (dllimport) /* default == import */ -#endif /* INTERFACE */ - -#if defined(THIS_EXPORTS_IMPLEMENTATION) -#define DLL_SPEC_IMPL __declspec (dllexport) -#elif defined(THIS_IMPORTS_IMPLEMENTATION) -#define DLL_SPEC_IMPL __declspec (dllimport) -#else -#define DLL_SPEC_IMPL /* default == nothing */ -#endif /* IMPLEMENTATION */ - -#define NOMINMAX /* for std::min/max */ - -#ifndef _DEBUG -#define NDEBUG /* for assert */ -#endif - -/*** NO INCLUDES BEFORE HERE ***/ - -#include /* (our) C API uses Window's string typedefs (LPSTR/LPCSTR) */ -#include -#include -#include -#include - -#ifdef __cplusplus - -#include -#include -#include -#include -#include -#include - -#include "containers.hpp"/*custom client-facing containers */ -#include "generic.hpp" /* our 'generic' type */ -#include "exceptions.hpp" /* our global exceptions */ - -#endif /* __cplusplus */ - -#ifdef NO_KERNEL_GLOBAL_NAMESPACE -/* because the service operates in session #0 and the engine/client code - will run in session #1 (or higher) we, in most cases, need to use the - global namespace for mutexes and file-mapping kernel objects - - in rare cases(debugging service in same session) we should - define NO_KERNEL_GLOBAL_NAMESPACE to use the local namespace */ -#define NO_KGBLNS -/***CRASH WARNING *** - - if you try to run tos-databridge-engine directly (not recommended) - without elevated privileges you will probably crash from an uncaught - exception in the IPCSlave constructor caused by an attempt to - create kernel objects in the global namespace w/o the appropriate - privileges. - - 1) you probably should only run the engine via the service - - 2) if for some reason you absolutley have to, recompile with - NO_KERNEL_GLOBAL_NAMESPACE defined or use eleveated privileges - to run the tos-databridge-engine-[].exe - - ***CRASH WARNING ***/ -#endif - -#ifdef CUSTOM_COMM_CHANNEL -/* generally we want a single, static comm channel so client can't accidentaly run - multiple instances of the engine and/or break the underlying IPC mechanism - - If a user tries to create a second IPCSlave (maybe forgeting to close an old - engine before starting a new one) it will crash during construction as it tries - to create kernel objects that already exist, causing a runtime_exception to be - thrown. (This will generate log messages w/ the IPC-Slave tag.) - - Defining CUSTOM_COMM_CHANNEL with the name of the channel to use allows us to - override this behavior. OBVIOUSLY YOU SHOULD BE VERY CAREFUL DOING THIS. */ -#define TOSDB_COMM_CHANNEL CUSTOM_COMM_CHANNEL -#else -#define TOSDB_COMM_CHANNEL "TOSDB_channel_1" -#endif - -/* combine engine and service logs for easier IPC debugging */ -#ifdef _DEBUG -#define LOG_BACKEND_USE_SINGLE_FILE -#define LOG_BACKEND_MUTEX_NAME "Global\\TOSDB_log_mutex_1" -#define LOG_BACKEND_SINGLE_FILE_NAME "engine-service-log.log" -/* redirect stderr to files in /log for the service and engine */ -#define REDIRECT_STDERR_TO_LOG -#endif - -/* the core types implemented by the data engine: engine-core.cpp */ -/* *** DEPRECATED - July 2017 *** */ -typedef long def_size_type; -typedef long long ext_size_type; -typedef float def_price_type; -typedef double ext_price_type; -/* *** DEPRECATED - July 2017 *** */ - -/* guarantees for the Python wrapper */ -typedef uint32_t size_type; -typedef uint8_t type_bits_type; - -struct TypeSizeAsserts_{ - /* sanity checks */ - char ASSERT_def_size_type_atleast_4bytes[sizeof(def_size_type) >= 4 ? 1 : -1]; - char ASSERT_ext_size_type_is_8bytes[sizeof(ext_size_type) == 8 ? 1 : -1]; - char ASSERT_def_price_type_atleast_4bytes[sizeof(def_price_type) >= 4 ? 1 : -1]; - char ASSERT_ext_price_type_is_8bytes[sizeof(ext_price_type) == 8 ? 1 : -1]; - char ASSERT_size_type_is_4bytes[sizeof(size_type) == 4 ? 1 : -1]; - char ASSERT_type_bits_type_is_1byte[sizeof(type_bits_type) == 1 ? 1 : -1]; -}; - -typedef const enum{ /*milliseconds*/ - Fastest = 0, - VeryFast = 3, - Fast = 30, - Moderate = 300, - Slow = 3000, - Glacial = 30000 /* <-- DEBUG ONLY */ -}UpdateLatency; - -/* custom time object */ -typedef struct{ - struct tm ctime_struct; - long micro_second; -} DateTimeStamp, *pDateTimeStamp; - -/* reserve a block name for the implementation */ -#define TOSDB_RESERVED_BLOCK_NAME "___RESERVED_BLOCK_NAME___" - -/* the following #defines contain names/values that are necessary - for the python backage. BE CAREFUL CHANGING NAMES */ - -/* the connected states we can be in; see TOSDB_ConnectedState() */ -#define TOSDB_CONN_NONE 0 -#define TOSDB_CONN_ENGINE 1 -#define TOSDB_CONN_ENGINE_TOS 2 - -#define TOSDB_INTGR_BIT ((type_bits_type)0x80) -#define TOSDB_QUAD_BIT ((type_bits_type)0x40) -#define TOSDB_STRING_BIT ((type_bits_type)0x20) -#define TOSDB_TOPIC_BITMASK ((type_bits_type)0xE0) -#define TOSDB_STR_DATA_SZ ((size_type)40) -#define TOSDB_MAX_STR_SZ ((size_type)0xFF) -#define TOSDB_MAX_NSTRS 1000 -#define TOSDB_DEF_TIMEOUT 2000 -#define TOSDB_DEF_PAUSE 100 -#define TOSDB_PROBE_WAIT ((int)Moderate * 3) -#define TOSDB_MIN_TIMEOUT 1500 -#define TOSDB_SHEM_BUF_SZ 4096 -#define TOSDB_BLOCK_ID_SZ 63 -/* adjust to avoid mem issues with INT_MAX(2**32) */ -#define TOSDB_MAX_BLOCK_SZ 16777216 /* 2**24 */ -#define TOSDB_DEF_LATENCY Moderate - -/* error codes the C API returns */ -#define TOSDB_ERROR_BAD_INPUT -1 -#define TOSDB_ERROR_BAD_INPUT_BUFFER -2 -#define TOSDB_ERROR_NOT_CONNECTED -3 -#define TOSDB_ERROR_TIMEOUT -4 -#define TOSDB_ERROR_BLOCK_ALREADY_EXISTS -5 -#define TOSDB_ERROR_BLOCK_DOESNT_EXIST -6 -#define TOSDB_ERROR_BLOCK_CREATION -7 -#define TOSDB_ERROR_BLOCK_SIZE -8 -#define TOSDB_ERROR_BAD_TOPIC -9 -#define TOSDB_ERROR_BAD_ITEM -10 -#define TOSDB_ERROR_BAD_SIG -11 -#define TOSDB_ERROR_IPC -12 -#define TOSDB_ERROR_IPC_MSG -13 -#define TOSDB_ERROR_CONCURRENCY -14 -#define TOSDB_ERROR_ENGINE_NO_TOPIC -15 -#define TOSDB_ERROR_ENGINE_NO_ITEM -16 -#define TOSDB_ERROR_SERVICE -17 -#define TOSDB_ERROR_GET_DATA -18 -#define TOSDB_ERROR_GET_STATE -19 -#define TOSDB_ERROR_SET_STATE -20 -#define TOSDB_ERROR_DDE_POST -21 -#define TOSDB_ERROR_DDE_NO_ACK -22 -#define TOSDB_ERROR_SHEM_BUFFER -23 -#define TOSDB_ERROR_UNKNOWN -24 -/* this must be our last error because it can be decremented */ -#define TOSDB_ERROR_DECREMENT_BASE -25 - -/* the previous #defines contain names/values that are necessary - for the python backage. BE CAREFUL CHANGING NAMES */ - -/* severity of log events */ -typedef enum{ - low = 0, - high -}Severity; - -/* following block will contain back-end stuff shared by various mods; - to access you should define 'THIS_IMPORTS_IMPLEMENTATION' */ -#if defined(THIS_EXPORTS_IMPLEMENTATION) || defined(THIS_IMPORTS_IMPLEMENTATION) - -#ifdef __cplusplus - -/* CONCURRENCY - concurrency.cpp / concurrency.hpp - define CPP_COND_VAR to use - portable sync objects built on std::condition_variable. (doesn't work - properly in VS2012: throws access violation) */ -#ifdef CPP_COND_VAR -class DLL_SPEC_IMPL SignalManager -#else -class DLL_SPEC_IMPL LightWeightMutex; -class DLL_SPEC_IMPL WinLockGuard; -class DLL_SPEC_IMPL SignalManager; -#endif /* CPP_COND_VAR */ - -class DLL_SPEC_IMPL IPCNamedMutexClient; -class DLL_SPEC_IMPL NamedMutexLockGuard; - -/* IPC - ipc.cpp / ipc.hpp */ -class DLL_SPEC_IMPL IPCBase; -class DLL_SPEC_IMPL IPCMaster; -class DLL_SPEC_IMPL IPCSlave; - -/* - * name of mapping is of form: "TOSDB_Buffer__[topic name]_[item_name]" - * replacing reserved chars w/ ITEM_SYMBOL_BUFFER_MAP strings - */ -DLL_SPEC_IMPL std::string -CreateBufferName(std::string topic_str, std::string item); - -DLL_SPEC_IMPL std::string -BuildLogPath(std::string name); - -DLL_SPEC_IMPL std::string -SysTimeString(bool use_msec=true); - -DLL_SPEC_IMPL void -ParseArgs(std::vector& vec, std::string str); - -DLL_SPEC_IMPL std::string -str_to_lower(std::string str); - -DLL_SPEC_IMPL std::string -str_to_lower(std::string str); - -#endif /*__cplusplus */ - -DLL_SPEC_IMPL void -str_to_lower(char* str, size_t max); - -DLL_SPEC_IMPL void -str_to_upper(char* str, size_t max); - - -/* signals used by the IPC mechanism */ -#define TOSDB_SIG_ADD 1 -#define TOSDB_SIG_REMOVE 2 -#define TOSDB_SIG_PAUSE 3 -#define TOSDB_SIG_CONTINUE 4 -#define TOSDB_SIG_STOP 5 -#define TOSDB_SIG_DUMP 6 -#define TOSDB_SIG_GOOD 7 -#define TOSDB_SIG_BAD 8 -#define TOSDB_SIG_TEST 9 - -/* for securing shared memory buffers */ -typedef const enum{ - SHEM1 = 0, - MUTEX1, -}Securable; - -typedef struct{ - /* - header that will be placed at the front(offset 0) of the mem mapping - logical location: beg_offset + ((raw_size - beg_offset) // elem_size) - */ - volatile unsigned int loop_seq; /* # of times buffer has looped around */ - volatile unsigned int elem_size; /* size of elements in the buffer */ - volatile unsigned int beg_offset; /* logical location (after header) */ - volatile unsigned int end_offset; /* logical location (after header) */ - volatile unsigned int next_offset; /* logical location of next write */ -} BufferHead, *pBufferHead; - -/* we still need to export this for DumpBufferStatus in engine.cpp */ -extern char DLL_SPEC_IMPL TOSDB_LOG_PATH[ MAX_PATH+40 ]; - -/* we'll log to a relative directory - REQUIRES A STABLE DIRECTORY TREE - this will be relative to the back-end lib: _tos-databridge-[].dll */ -#define LOCAL_LOG_PATH "\\..\\..\\..\\log\\" /* needs to be < 40 char */ - -/* LOGGING - logging.cpp - (also, see public Log functions at end of header) */ - -/* 'Raw' versions are implemenation only (they don't sync/block) */ -DLL_SPEC_IMPL void -TOSDB_LogRaw_(Severity, LPCSTR , LPCSTR); - -#define TOSDB_LogRawH(tag,desc) TOSDB_LogRaw_(high, tag, desc) -#define TOSDB_LogRaw(tag,desc) TOSDB_LogRaw_(low, tag, desc) - -EXT_C_SPEC DLL_SPEC_IMPL void -StartLogging(LPCSTR path); - -EXT_C_SPEC DLL_SPEC_IMPL void -StopLogging(); - -EXT_C_SPEC DLL_SPEC_IMPL void -ClearLog(); - -#ifdef _DEBUG -#define TOSDB_LogDebug(desc) TOSDB_LogH("DEBUG", desc) -#else -#define TOSDB_LogDebug(desc) do{}while(0) -#endif - -#endif /* THIS_EXPORTS_IMPLEMENTATION || THIS_IMPORTS_IMPLEMENTATION */ - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetClientLogPath(char* path, size_type sz); - -#ifdef __cplusplus - -typedef std::chrono::steady_clock steady_clock_type; -typedef std::chrono::system_clock system_clock_type; -typedef std::chrono::microseconds micro_sec_type; - -/* Generic STL Types returned by the interface(below) */ -typedef TOSDB_Generic generic_type; -typedef std::pair generic_dts_type; -typedef std::vector generic_vector_type; -typedef std::vector dts_vector_type; -typedef std::pair generic_dts_vectors_type; -typedef std::map generic_map_type; -typedef std::map generic_matrix_type; -typedef std::map generic_dts_map_type; -typedef std::map generic_dts_matrix_type; - -#define TOSDB_BIT_SHIFT_LEFT(T,val) (((T)val)<<((sizeof(T)-sizeof(type_bits_type))*8)) -#define TOSDB_BIT_SHIFT_RIGHT(T,val) (((T)val)>>((sizeof(T)-sizeof(type_bits_type))*8)) - -template -class Topic_Enum_Wrapper { - /* - The TOPICS enum and utility functions/templates inside the wrapper will end - up being defined in each module but trying to export it from both the backend - and the client-side libs creates all kinds of problems. - - The enum-string mapping is stored in the static TwoWayHashMap 'map' and - is used internally. Client code should use the exported MAP() static function - which returns a const reference to 'map'. - */ - static_assert(std::is_integral::value && !std::is_same::value, - "Invalid Topic_Enum_Wrapper Type"); - - Topic_Enum_Wrapper() - { - } - - static const T ADJ_INTGR_BIT = TOSDB_BIT_SHIFT_LEFT(T,TOSDB_INTGR_BIT); - static const T ADJ_QUAD_BIT = TOSDB_BIT_SHIFT_LEFT(T,TOSDB_QUAD_BIT); - static const T ADJ_STRING_BIT = TOSDB_BIT_SHIFT_LEFT(T,TOSDB_STRING_BIT); - static const T ADJ_FULL_MASK = TOSDB_BIT_SHIFT_LEFT(T,TOSDB_TOPIC_BITMASK); - -public: - enum class TOPICS - : T { /* - * pack type info into HO nibble of scoped Enum - * gaps represent the 'reserved' exported TOS/DDE fields - * [ see topics.cpp ] - */ - /* BEGIN TOPICS ksxaw9834hr84hf;esij?>< DO NOT EDIT !!! */ - NULL_TOPIC = 0x00, - HIGH52 = 0x1, - LOW52 = 0x2, - ASK = 0x6 | ADJ_QUAD_BIT, - ASKX = 0x7 | ADJ_STRING_BIT, - ASK_SIZE = 0x8 | ADJ_INTGR_BIT, - AX = 0xa | ADJ_STRING_BIT, - BACK_EX_MOVE = 0x15 | ADJ_QUAD_BIT, - BACK_VOL = 0x16 | ADJ_QUAD_BIT, - BA_SIZE = 0x17 | ADJ_STRING_BIT, - BETA = 0x18 | ADJ_QUAD_BIT, - BID = 0x19 | ADJ_QUAD_BIT, - BIDX = 0x1a | ADJ_STRING_BIT, - BID_SIZE = 0x1b | ADJ_INTGR_BIT, - BX = 0x1c | ADJ_STRING_BIT, - CALL_VOLUME_INDEX = 0x27, - DELTA = 0x46 | ADJ_QUAD_BIT, - DESCRIPTION = 0x48 | ADJ_STRING_BIT, - DIV = 0x4b, - DIV_FREQ = 0x4c | ADJ_STRING_BIT, - EPS = 0x61, - EXCHANGE = 0x62 | ADJ_STRING_BIT, - EXPIRATION = 0x63 | ADJ_STRING_BIT, - EXTRINSIC = 0x64 | ADJ_QUAD_BIT, - EX_DIV_DATE = 0x65 | ADJ_STRING_BIT, - EX_MOVE_DIFF = 0x66 | ADJ_QUAD_BIT, - FRONT_EX_MOVE = 0x73 | ADJ_QUAD_BIT, - FRONT_VOL = 0x74 | ADJ_QUAD_BIT, - FX_PAIR = 0x78 | ADJ_STRING_BIT, - GAMMA = 0x7d | ADJ_QUAD_BIT, - HIGH = 0x7f | ADJ_QUAD_BIT, - HTB_ETB = 0x80 | ADJ_STRING_BIT, - IMPL_VOL = 0x8a | ADJ_QUAD_BIT, - INTRINSIC = 0x8b, - LAST = 0x95 | ADJ_QUAD_BIT, - LASTX = 0x96 | ADJ_STRING_BIT, - LAST_SIZE = 0x97 | ADJ_INTGR_BIT, - LOW = 0x9b | ADJ_QUAD_BIT, - LX = 0x9c | ADJ_STRING_BIT, - MARK = 0xa9 | ADJ_QUAD_BIT, - MARKET_CAP = 0xaa | ADJ_STRING_BIT, - MARK_CHANGE = 0xab | ADJ_QUAD_BIT, - MARK_PERCENT_CHANGE = 0xac | ADJ_QUAD_BIT, - MRKT_MKR_MOVE = 0xaf, - MT_NEWS = 0xb0 | ADJ_STRING_BIT, - NET_CHANGE = 0xc9 | ADJ_QUAD_BIT, - OPEN = 0xcb | ADJ_QUAD_BIT , - OPEN_INT = 0xcc | ADJ_INTGR_BIT, - OPTION_VOLUME_INDEX = 0xcd, - PE = 0xd5, - PERCENT_CHANGE = 0xd6 | ADJ_QUAD_BIT, - PUT_CALL_RATIO = 0xda, - PUT_VOLUME_INDEX = 0xdb, - RHO = 0xed | ADJ_QUAD_BIT, - SHARES = 0xff | ADJ_INTGR_BIT | ADJ_QUAD_BIT, - STRENGTH_METER = 0x102 | ADJ_STRING_BIT, - STRIKE = 0x103, - SYMBOL = 0x104 | ADJ_STRING_BIT, - THETA = 0x11f | ADJ_QUAD_BIT, - VEGA = 0x13c | ADJ_QUAD_BIT, - VOLUME = 0x13d | ADJ_INTGR_BIT | ADJ_QUAD_BIT, - VOL_DIFF = 0x13e | ADJ_QUAD_BIT, - VOL_INDEX = 0x13f | ADJ_QUAD_BIT, - WEIGHTED_BACK_VOL = 0x14b | ADJ_QUAD_BIT, - YIELD = 0x152, - CUSTOM1 = 0x201 | ADJ_STRING_BIT, - CUSTOM2 = 0x202 | ADJ_STRING_BIT, - CUSTOM3 = 0x203 | ADJ_STRING_BIT, - CUSTOM4 = 0x204 | ADJ_STRING_BIT, - CUSTOM5 = 0x205 | ADJ_STRING_BIT, - CUSTOM6 = 0x206 | ADJ_STRING_BIT, - CUSTOM7 = 0x207 | ADJ_STRING_BIT, - CUSTOM8 = 0x208 | ADJ_STRING_BIT, - CUSTOM9 = 0x209 | ADJ_STRING_BIT, - CUSTOM10 = 0x20a | ADJ_STRING_BIT, - CUSTOM11 = 0x20b | ADJ_STRING_BIT, - CUSTOM12 = 0x20c | ADJ_STRING_BIT, - CUSTOM13 = 0x20d | ADJ_STRING_BIT, - CUSTOM14 = 0x20e | ADJ_STRING_BIT, - CUSTOM15 = 0x20f | ADJ_STRING_BIT, - CUSTOM16 = 0x210 | ADJ_STRING_BIT, - CUSTOM17 = 0x211 | ADJ_STRING_BIT, - CUSTOM18 = 0x212 | ADJ_STRING_BIT, - CUSTOM19 = 0x213 | ADJ_STRING_BIT - /* END TOPICS ksxaw9834hr84hf;esij?>< DO NOT EDIT !!! */ - }; - - typedef T enum_value_type; - typedef typename Topic_Enum_Wrapper::TOPICS enum_type; - - /* type at compile-time - (e.g TOS_Topics::Type::type == double) */ - template - struct Type{ - typedef typename std::conditional< - ((T)topic & ADJ_STRING_BIT), std::string, - typename std::conditional< - (T)topic & ADJ_INTGR_BIT, - typename std::conditional< - (T)topic & ADJ_QUAD_BIT, long long, long>::type, - typename std::conditional< - (T)topic & ADJ_QUAD_BIT, - double, float>::type>::type>::type type; - }; - - /* type bits at run-time */ - static inline type_bits_type - TypeBits(enum_type topic) - { - return ((type_bits_type)(TOSDB_BIT_SHIFT_RIGHT(T, (T)topic)) & TOSDB_TOPIC_BITMASK); - } - - /* type size at run-time */ - static size_type - TypeSize(enum_type topic) - { - switch(TypeBits(topic)){ - case TOSDB_STRING_BIT: - return TOSDB_STR_DATA_SZ; - case TOSDB_INTGR_BIT: - return sizeof(long); - case TOSDB_QUAD_BIT: - return sizeof(double); - case TOSDB_INTGR_BIT | TOSDB_QUAD_BIT: - return sizeof(long long); - default : - return sizeof(float); - }; - } - - /* platform-dependent type strings at run-time */ - static std::string - TypeString(enum_type topic) - { - switch(TypeBits(topic)){ - case TOSDB_STRING_BIT: - return typeid(std::string).name(); - case TOSDB_INTGR_BIT: - return typeid(long).name(); - case TOSDB_QUAD_BIT: - return typeid(double).name(); - case TOSDB_INTGR_BIT | TOSDB_QUAD_BIT: - return typeid(long long).name(); - default : - return typeid(float).name(); - }; - } - -#ifdef CUSTOM_HASHER - /* we can't guarantee that the particular STL implementation will - provide a default hasher like VS */ - struct hasher{ - size_t operator()(const enum_type& t) const { - static_assert(std::is_unsigned::value, - "hasher only accepts unsigned integral enum_value_types"); - return (enum_value_type)t; - } - }; - typedef TwoWayHashMap topic_map_type; -#else - typedef TwoWayHashMap topic_map_type; -#endif - - typedef typename topic_map_type::pair1_type topic_map_entry_type; - - /* 'map' is defined in topics.cpp as part of the _tos-databrige[].dll module; - therefore, the client facing lib (tos-databridge[].dll) and the service/engine - need to import it, but the former ALSO needs to export it. - - In this case we define 'map' so the back-end can see it, the client-side lib - can import it, and ALSO export the MAP() wrapper to client code. - - long-story-short, just use MAP() to access the topic-string mapping */ - - /* internally we use map (link w/ _tos-databridge[].dll) */ -#if defined(THIS_IMPORTS_IMPLEMENTATION) || defined(THIS_EXPORTS_IMPLEMENTATION) - static DLL_SPEC_IMPL const topic_map_type map; -#endif - - /* externally we use the MAP() (link w/ tos-databridge-[].dll) */ - static DLL_SPEC_IFACE const topic_map_type& -#if defined(THIS_EXPORTS_INTERFACE) || defined(THIS_DOESNT_IMPORT_INTERFACE) - MAP(){ return map; } -#else - MAP(); -#endif - - struct top_less{ /* back-end should use 'map' but attempts to #define - special versions of top_less won't link w/ client code */ - bool operator()(const enum_type& left, const enum_type& right){ - return (MAP()[left] < MAP()[right]); - } - }; - -}; - -typedef Topic_Enum_Wrapper TOS_Topics; - -typedef ILSet str_set_type; -typedef ILSet topic_set_type; - -#endif - -/* 'Administrative' C API - client_admin.cpp - - NOTE: int return types indicate an error value is returned */ - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_Connect(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_Disconnect(); - -/* DEPRECATED - Jan 10 2017 */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned int -TOSDB_IsConnected(); - -/* ... in favor of... */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned int -TOSDB_IsConnectedToEngine(); - -/* ... and... */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned int -TOSDB_IsConnectedToEngineAndTOS(); - -/* ... and... */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned int -TOSDB_ConnectionState(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_CreateBlock(LPCSTR id, size_type sz, BOOL is_datetime, size_type timeout) ; - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_CloseBlock(LPCSTR id); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_CloseBlocks(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW size_type -TOSDB_GetBlockLimit(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW size_type -TOSDB_SetBlockLimit(size_type sz); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW size_type -TOSDB_GetBlockCount(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetBlockIDs(LPSTR* dest, size_type array_len, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetBlockSize(LPCSTR id, size_type* pSize); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_SetBlockSize(LPCSTR id, size_type sz); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned long -TOSDB_GetLatency(); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW unsigned long -TOSDB_SetLatency(UpdateLatency latency); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_Add(LPCSTR id, LPCSTR* items, size_type items_len, LPCSTR* topics_str , size_type topics_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_AddTopic(LPCSTR id, LPCSTR topic_str); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_AddItem(LPCSTR id, LPCSTR item); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_AddTopics(LPCSTR id, LPCSTR* topics_str, size_type topics_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_AddItems(LPCSTR id, LPCSTR* items, size_type items_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_RemoveTopic(LPCSTR id, LPCSTR topic_str); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_RemoveItem(LPCSTR id, LPCSTR item); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemCount(LPCSTR id, size_type* count); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetTopicCount(LPCSTR id, size_type* count); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetPreCachedItemCount(LPCSTR id, size_type* count); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetPreCachedTopicCount(LPCSTR id, size_type* count); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetPreCachedTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetPreCachedItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetTypeBits(LPCSTR topic_str, type_bits_type* type_bits); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetTypeString(LPCSTR topic_str, LPSTR dest, size_type str_len); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_IsUsingDateTime(LPCSTR id, unsigned int* is_datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamOccupancy(LPCSTR id,LPCSTR item, LPCSTR topic_str, size_type* sz); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetMarkerPosition(LPCSTR id,LPCSTR item, LPCSTR topic_str, long long* pos); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_IsMarkerDirty(LPCSTR id, LPCSTR item, LPCSTR topic_str, unsigned int* is_dirty); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_DumpSharedBufferStatus(); - -/* WARNING - should only be used when you are certain a client lib has failed to - close a stream during destruction of the containing block. If that's not the case - YOU CAN CORRUPT THE UNDERLYING BUFFERS FOR ANY OR ALL CLIENT INSTANCE(S)! */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_RemoveOrphanedStream(LPCSTR item, LPCSTR topic_str); - -#ifdef __cplusplus - -/* (extended) 'Administrative' C++ API - client_admin.cpp - - NOTE: ints indicate error/success (as above) BUT these calls may also throw*/ - -DLL_SPEC_IFACE int -TOSDB_Add(std::string id, str_set_type items, topic_set_type topics_t); - -DLL_SPEC_IFACE int -TOSDB_AddItem(std::string id, std::string item); - -DLL_SPEC_IFACE int -TOSDB_AddTopic(std::string id, TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE int -TOSDB_AddTopics(std::string id, topic_set_type topics_t); - -DLL_SPEC_IFACE int -TOSDB_AddItems(std::string id, str_set_type items); - -DLL_SPEC_IFACE int -TOSDB_RemoveItem(std::string id, std::string item); - -DLL_SPEC_IFACE int -TOSDB_RemoveTopic(std::string id, TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE str_set_type -TOSDB_GetBlockIDs(); - -DLL_SPEC_IFACE topic_set_type -TOSDB_GetTopicEnums(std::string id); - -DLL_SPEC_IFACE str_set_type -TOSDB_GetTopicNames(std::string id); - -DLL_SPEC_IFACE str_set_type -TOSDB_GetItemNames(std::string id); - -DLL_SPEC_IFACE topic_set_type -TOSDB_GetPreCachedTopicEnums(std::string id); - -DLL_SPEC_IFACE str_set_type -TOSDB_GetPreCachedTopicNames(std::string id); - -DLL_SPEC_IFACE str_set_type -TOSDB_GetPreCachedItemNames(std::string id); - -DLL_SPEC_IFACE type_bits_type -TOSDB_GetTypeBits(TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE std::string -TOSDB_GetTypeString(TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE size_type -TOSDB_GetPreCachedItemCount(std::string id); - -DLL_SPEC_IFACE size_type -TOSDB_GetPreCachedTopicCount(std::string id); - -DLL_SPEC_IFACE size_type -TOSDB_GetItemCount(std::string id); - -DLL_SPEC_IFACE size_type -TOSDB_GetTopicCount(std::string id); - -DLL_SPEC_IFACE bool -TOSDB_IsUsingDateTime(std::string id); - -DLL_SPEC_IFACE size_type -TOSDB_GetBlockSize(std::string id); - -DLL_SPEC_IFACE size_type -TOSDB_GetStreamOccupancy(std::string id, std::string item, TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE long long -TOSDB_GetMarkerPosition(std::string id, std::string item, TOS_Topics::TOPICS topic_t); - -DLL_SPEC_IFACE bool -TOSDB_IsMarkerDirty(std::string id, std::string item, TOS_Topics::TOPICS topic_t); - - -/* 'Get' C/C++ API - client_get.cpp - - NOTE: C versions return error code while C++ versions throw; most internal errors - will be wrapped in TOSDB_Error or derived exceptions (exceptions.cpp) but - this IS NOT guaranteed. */ - - -/* individual data points */ - -template -auto -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx = 0) - -> typename std::conditional , T>::type; - -template<> -DLL_SPEC_IFACE generic_type -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template<> -DLL_SPEC_IFACE generic_dts_type -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::string -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE double -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE float -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE long long -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE long -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::pair -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::pair -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::pair -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::pair -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -template DLL_SPEC_IFACE std::pair -TOSDB_Get(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long indx); - -#endif - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetDouble(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, double* dest, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetFloat(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, float* dest, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetLongLong(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, long long* dest, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetLong(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, long* dest, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetString(LPCSTR id, LPCSTR item, LPCSTR topic_str, long indx, LPSTR dest, size_type str_len, pDateTimeStamp datetime); - -#ifdef __cplusplus - -/* get multiple contiguous data points in the stream*/ - -template -auto -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end = -1, long beg = 0) - -> typename std::conditional, dts_vector_type>, std::vector>::type; - -template<> -DLL_SPEC_IFACE generic_vector_type -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template<> -DLL_SPEC_IFACE generic_dts_vectors_type -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -/* 20x faster than generic */ -template DLL_SPEC_IFACE std::vector -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::vector -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::vector -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::vector -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::vector -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - - -template DLL_SPEC_IFACE std::pair,dts_vector_type> -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::pair,dts_vector_type> -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::pair,dts_vector_type> -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::pair,dts_vector_type> -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -template DLL_SPEC_IFACE std::pair, dts_vector_type> -TOSDB_GetStreamSnapshot(std::string id, std::string item, TOS_Topics::TOPICS topic_t, long end, long beg); - -#endif - -/* C calls about 30x faster than generic */ -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotDoubles(LPCSTR id,LPCSTR item, LPCSTR topic_str, double* dest, size_type array_len, - pDateTimeStamp datetime, long end, long beg); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotFloats(LPCSTR id, LPCSTR item, LPCSTR topic_str, float* dest, size_type array_len, - pDateTimeStamp datetime, long end, long beg); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotLongLongs(LPCSTR id, LPCSTR item, LPCSTR topic_str, long long* dest, size_type array_len, - pDateTimeStamp datetime, long end, long beg); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotLongs(LPCSTR id, LPCSTR item, LPCSTR topic_str, long* dest, size_type array_len, - pDateTimeStamp datetime, long end, long beg); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotStrings(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, - pDateTimeStamp datetime, long end, long beg); - -/* 'guaranteed' to be contiguous between calls (Get, GetStreamSnapshot, GetStreamSnapshot) */ - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotDoublesFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,double* dest,size_type array_len, - pDateTimeStamp datetime, long beg, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotFloatsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, float* dest, size_type array_len, - pDateTimeStamp datetime, long beg, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotLongLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long long* dest, size_type array_len, - pDateTimeStamp datetime, long beg, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long* dest, size_type array_len, - pDateTimeStamp datetime, long beg, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetStreamSnapshotStringsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, - pDateTimeStamp datetime, long beg, long *get_size); - -/* get n data-points in front of marker ('guaranteed' to be contiguous a la GetStreamSnapshot) */ - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetNDoublesFromMarker(LPCSTR id,LPCSTR item,LPCSTR topic_str,double* dest,size_type n, pDateTimeStamp datetime, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetNFloatsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, float* dest, size_type n, pDateTimeStamp datetime, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetNLongLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long long* dest, size_type n, pDateTimeStamp datetime, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetNLongsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, long* dest, size_type n, pDateTimeStamp datetime, long *get_size); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetNStringsFromMarker(LPCSTR id, LPCSTR item, LPCSTR topic_str, LPSTR* dest, size_type n, size_type str_len, - pDateTimeStamp datetime, long *get_size); - -#ifdef __cplusplus - -/* get all the most recent item values for a particular topic */ - -template -auto -TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t) - -> typename std::conditional::type; - -template<> -DLL_SPEC_IFACE generic_map_type -TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t); - -template<> -DLL_SPEC_IFACE generic_dts_map_type -TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t); - -#endif - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemFrameDoubles(LPCSTR id, LPCSTR topic_str, double* dest, size_type array_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemFrameFloats(LPCSTR id, LPCSTR topic_str, float* dest, size_type array_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemFrameLongLongs(LPCSTR id, LPCSTR topic_str, long long* dest, size_type array_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemFrameLongs(LPCSTR id, LPCSTR topic_str, long* dest, size_type array_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetItemFrameStrings(LPCSTR id, LPCSTR topic_str, LPSTR* dest, size_type array_len, size_type str_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -#ifdef __cplusplus - -/* get all the most recent topic values for a particular item */ - -template -auto -TOSDB_GetTopicFrame(std::string id, std::string item) - -> typename std::conditional::type; - -template<> -DLL_SPEC_IFACE generic_map_type -TOSDB_GetTopicFrame(std::string id, std::string item); - -template<> -DLL_SPEC_IFACE generic_dts_map_type -TOSDB_GetTopicFrame(std::string id, std::string item); - -#endif - -EXT_C_SPEC DLL_SPEC_IFACE NO_THROW int -TOSDB_GetTopicFrameStrings(LPCSTR id, LPCSTR item, LPSTR* dest, size_type array_len, size_type str_len, - LPSTR* label_dest, size_type label_str_len, pDateTimeStamp datetime); - -#ifdef __cplusplus - -/* get all the most recent item and topic values */ - -template -auto -TOSDB_GetTotalFrame(std::string id) - -> typename std::conditional::type; - -template<> -DLL_SPEC_IFACE generic_matrix_type -TOSDB_GetTotalFrame(std::string id); - -template<> -DLL_SPEC_IFACE generic_dts_matrix_type -TOSDB_GetTotalFrame(std::string id); - - -/* OSTREAM OVERLOADS - client_out.cpp */ - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const DateTimeStamp&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const DateTimeStamp*); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_matrix_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_map_type::value_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_map_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_vector_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const dts_vector_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_dts_matrix_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_dts_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_dts_map_type&); - -DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const generic_dts_vectors_type&); - - -template std::ostream& -operator<<(std::ostream&, const std::pair&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair&); - - -template std::ostream& -operator<<(std::ostream&, const std::vector&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::vector&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::vector&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::vector&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::vector&); - -template DLL_SPEC_IFACE std::ostream& - operator<<(std::ostream&, const std::vector&); - - -template std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -template DLL_SPEC_IFACE std::ostream& -operator<<(std::ostream&, const std::pair,dts_vector_type>&); - -#endif /* __cplusplus */ - -/* when building tos-databridge[].dll these calls need to be both imported( from _tosd-databridge-[].dll) - and exported (from tos-databridge[].dll) - they must use /export:[func name] during link */ - -/* if logging is not enabled high severity events will be sent to std::cerr */ -/* use the macros (below) instead */ -EXT_C_SPEC DLL_SPEC_IMPL void -TOSDB_Log_(Severity, LPCSTR, LPCSTR); - -EXT_C_SPEC DLL_SPEC_IMPL void -TOSDB_LogEx_(Severity, LPCSTR, LPCSTR, int); - -#define TOSDB_LogH(tag,desc) TOSDB_Log_(high, tag, desc) - -#define TOSDB_Log(tag,desc) TOSDB_Log_(low, tag, desc) - -#define TOSDB_LogEx(tag,desc,error) TOSDB_LogEx_(high, tag, desc, error) - -EXT_C_SPEC DLL_SPEC_IMPL char** -NewStrings(size_t num_strs, size_t strs_len); - -EXT_C_SPEC DLL_SPEC_IMPL void -DeleteStrings(char** str_array, size_t num_strs); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -CheckIDLength(LPCSTR id); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -CheckStringLength(LPCSTR str); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -CheckStringLengths(LPCSTR* str, size_type items_len); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -IsValidBlockSize(size_type sz); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -IsValidBlockID(const char* id); - -EXT_C_SPEC DLL_SPEC_IMPL unsigned int -IsValidItemString(const char* item); - - -#endif /* JO_TOSDB_DATABRIDGE */ - diff --git a/index.html b/index.html new file mode 100644 index 0000000..272cef7 --- /dev/null +++ b/index.html @@ -0,0 +1,176 @@ + + + + + + TOSDataBridge v0.8 by jeog + + + + + + + +
+
+

TOSDataBridge v0.8

+

Free, Real-Time Streaming Data from your TOS Platform [C,C++,Java,Python]

+ +

View the Project on GitHub jeog/TOSDataBridge

+ + + +
+
+

+TOSDataBridge

+ +
+ +

TOSDataBridge (TOSDB) is an open-source collection of resources for pulling real-time streaming data off of TDAmeritrade's ThinkOrSwim(TOS) platform, providing C, C++, Java and Python interfaces.

+ +

TOSDB uses TOS's antiquated, yet still useful, DDE feature, directly through the Windows API. The C / C++ interfaces are implemented as a shared library(DLL) that communicates with a back-end Windows Service. The Python and Java interfaces wrap this library in a more object-oriented, user-friendly format.

+ +

The core implementation is not portable, but the python interface does provides a thin virtualization layer over TCP. A user running Windows in a Virtual Machine, for instance, can expose the exact same python interface to a different host system running python3.

+ +

+Requirements

+ +
+ +
    +
  • Windows for the core implementation. The python interface is available to any system running python3.
    • +
  • TDAmeritrade's ThinkOrSwim(TOS) platform that exposes DDE functionality (the Window's verion)
    • +
  • VC++ 2012 Redistributable (included)
    • +
+ +

+Quick Setup

+ +
+ +
    +
  • tosdb-setup.bat will attempt to install the necessary modules/dependencies for you but you should refer to 'Installation Details' for a more detailed explanation
    • +
  • +

    Be sure to know what build you need(x86 vs x64); it should match your system, all the modules you'll need, and your version of Python(if you plan on using the python wrapper), or your version of the Java Runtime(if you plan on using the java wrapper).

    + +
    +Core C/C++ Libraries
    + +
    (Admin) C:\[...TOSDataBridge]\tosdb-setup.bat   [x86|x64]   [admin]   [session]
+
    + +
      +
    • [x86|x64] : the version to build (required)
      • +
    • [admin] : does your TOS platform require elevation? (optional)
      • +
    • [session] : override the service's attempt to determine the session id when exiting from session-0 isolation. The tos-databridge-engine.exe[] binary needs to run in the same session as the ThinkOrSwim platform. Most users shouldn't worry about this unless they plan to run in a non-standard environment (e.g an EC2 instance). An explanation of Sessions, Desktops, and Stations. (optional)
      • +
    + +
    Example 1: C:\TOSDataBridge\> tosdb-setup.bat x86 
+Example 2: C:\TOSDataBridge\> tosdb-setup.bat x64 admin 
+Example 3: C:\TOSDataBridge\> tosdb-setup.bat x64 admin 2 
+
    + +
    +Python Wrapper (optional)
    + +
    C:\[...TOSDataBridge]\python\python setup.py install
+
    + +
      +
    • Core C/C++ libs (above) must be installed first to use the (non-virtual) interface
      • +
    • tosdb/_tosdb.py is generated automatically by setup.py
      • +
    +
    • +
+ + +
+Java Wrapper (optional)
+ +
    +
  • Warning - in development; see docs
    • +
  • Core C/C++ libs (above) must be installed first
    • +
  • java/tosdatabridge.jar should be included (source is in java/src if you want/need to build)
    • +
+ +

+Quick Start

+ +
+ +
    +
  1. You may need to white-list some of these files (specifically tos-databridge-engine-[x86|x64].exe) in your Anti-Virus software before proceeding.
    2. +
  2. +

    Start the service:

    + +
    (Admin) C:\> SC start TOSDataBridge
+
    + +

    (consider having the service begin automatically on startup to avoid this step in the future; see #9 in 'Installation Details' .)

    +
    3. +
  3. Log on to your TOS platform
    4. +
+ +
+For C/C++:
+ +
    +
  • Include tos_databridge.h header in your code
    • +
  • Use the library calls detailed in C/C++ API
    • +
  • Link with tos-databridge-[0.8]-[x86|x64].dll +
    • +
  • Build
    • +
  • Run
    • +
+ +
+For Python:
+ + + +
+For Java:
+ +
    +
  • add java/tosdatabridge.jar to your classpath
    • +
  • see docs for an intro to the API, or the source(java/src) for the details
    • +
  • Compile
    • +
  • Run
    • +
+ +

+License & Terms

+ +
+ +

TOSDB is released under the GNU General Public License(GPL); a copy (LICENSE.txt) should be included. If not, see http://www.gnu.org/licenses. The author reserves the right to issue current and/or future versions of TOSDB under other licensing agreements. Any party that wishes to use TOSDB, in whole or in part, in any way not explicitly stipulated by the GPL, is thereby required to obtain a separate license from the author. The author reserves all other rights.

+ +

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

+ +

By choosing to use the software - under the broadest interpretation of the term "use" - you absolve the author of ANY and ALL responsibility, for ANY and ALL damages incurred; including, but not limited to, damages arising from the accuracy and/or timeliness of data the software does, or does not, provide.

+ +

Furthermore, TOSDB is in no way related to TDAmeritrade or affiliated parties; users of TOSDB must abide by their terms of service and are solely responsible for any violations therein.

+ +
+ +

Copyright (C) 2014 Jonathon Ogden

+
+
+

This project is maintained by jeog

+

Hosted on GitHub Pages — Theme by orderedlist

+
+
+ + + + diff --git a/java/examples/TimeAndSales/src/TimeAndSales.java b/java/examples/TimeAndSales/src/TimeAndSales.java deleted file mode 100644 index b235ac5..0000000 --- a/java/examples/TimeAndSales/src/TimeAndSales.java +++ /dev/null @@ -1,306 +0,0 @@ -import io.github.jeog.tosdatabridge.*; - -import javax.swing.*; -import javax.swing.table.DefaultTableCellRenderer; -import javax.swing.table.DefaultTableModel; -import java.awt.*; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.util.ArrayList; -import java.util.Set; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class TimeAndSales { - public static final boolean IS_32_BIT_ARCH = System.getProperty("os.arch").equals("x86"); - public static final String DLL_BASE_REGEX = "tos-databridge-\\d{1,}\\.\\d{1,}-x"; - public static final String DLL_DIRECTORY_PATH; - static{ - String ourPath = TimeAndSales.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - int indx = ourPath.indexOf("TOSDataBridge/java/examples"); - if( indx != -1 ) { - DLL_DIRECTORY_PATH = ourPath.substring(0, indx) + "TOSDataBridge/bin/Release/" + - (IS_32_BIT_ARCH ? "Win32/" : "x64/"); - }else{ - DLL_DIRECTORY_PATH = ourPath; - } - } - - public static final int DATA_BLOCK_SIZE = 1000; - public static final long LATENCY = 100; - public static final int TIMESALE_TABLE_MAX_ROWS = 300; - public static final int TIMESALE_TABLE_WIDTH = 350; - public static final int TIMESALE_TABLE_HEIGHT = 800; - - private static JFrame myFrame; - private static JPanel myPanel; - private static JToolBar myToolBar; - private static JTextField mySymbolEntryTextField; - private static JButton mySymbolEntryButton; - private static JScrollPane myScrollPane; - private static JTable myTable; - private static JFileChooser myFileChooser; - private static DataBlockWithDateTime myDataBlock; - private static DataRetrievalThread myDataRetrievalThread; - private static List myRowColors = new ArrayList<>(); - - private static class DataRetrievalThread extends Thread{ - private String symbol; - private long volume; - private double previousPrice; - - public - DataRetrievalThread(String symbol){ - this.symbol = symbol; - } - - private void - sleep(){ - try{ - Thread.sleep(LATENCY); - }catch( InterruptedException e ){ - this.interrupt(); - } - } - - @Override - public void - run(){ - while( !interrupted() ){ - final DateTime.DateTimePair last; - final Long volume; - try { - last = myDataBlock.getDoubleWithDateTime(symbol, Topic.LAST); - volume = myDataBlock.getLong(symbol, Topic.VOLUME); - }catch( TOSDataBridge.TOSDataBridgeException e ){ - displayMessage("Error", "Error retrieving data from block: " + e.getMessage(), - JOptionPane.ERROR_MESSAGE); - this.interrupt(); - break; - } - if( last != null && volume != null ) { - final long volumeDiff = volume - this.volume; - if (this.volume != 0 && volumeDiff > 0) { - final Color rowColor; - if (this.previousPrice == 0 || this.previousPrice == last.first) { - rowColor = Color.WHITE; - } else { - rowColor = (last.first > this.previousPrice) ? Color.GREEN : Color.RED; - } - this.previousPrice = last.first; - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - DefaultTableModel tableModel = (DefaultTableModel) myTable.getModel(); - tableModel.insertRow(0, new Object[]{last.second.toString(), - String.valueOf(last.first), String.valueOf(volumeDiff)}); - myRowColors.add(0, rowColor); - int rowCount = tableModel.getRowCount(); - if (rowCount > TIMESALE_TABLE_MAX_ROWS) { - tableModel.removeRow(rowCount - 1); - myRowColors.remove(rowCount - 1); - rowCount--; - } - myFrame.repaint(); - } - }); - } - this.volume = volume; - } - this.sleep(); - } - SwingUtilities.invokeLater( new Runnable() { - @Override - public void run() { - DefaultTableModel tableModel = (DefaultTableModel) myTable.getModel(); - tableModel.setRowCount(0); - myFrame.repaint(); - } - }); - } - } - - - public static void - main(String[] args){ - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - setupGUI(); - myFileChooser = new JFileChooser(DLL_DIRECTORY_PATH); - myFileChooser.setDialogTitle("Load TOSDataBridge DLL"); - while( true ){ - int response = myFileChooser.showOpenDialog(null); - if (response != JFileChooser.APPROVE_OPTION) { - System.exit(0); - } - String dllPath = myFileChooser.getSelectedFile().getPath(); - Pattern dllPattern = Pattern.compile(DLL_BASE_REGEX + "(64|86)\\.dll"); - Matcher dllMatcher = dllPattern.matcher(dllPath); - if( !dllMatcher.find() ){ - displayMessage("Error", "File doesn't appear to be a valid TOSDataBridge DLL", - JOptionPane.ERROR_MESSAGE); - continue; - } - if( !dllMatcher.group(1).equals(IS_32_BIT_ARCH ? "86" : "64") ){ - displayMessage("Error", "JVM is " + (IS_32_BIT_ARCH ? "32" : "64") + " bit. '" + - dllMatcher.group() + "' appears to be the wrong build for this JVM.", - JOptionPane.ERROR_MESSAGE); - continue; - } - if( setupTOSDataBridge(dllPath) ){ - break; - } - } - Runtime.getRuntime().addShutdownHook( new CloseThread() ); - } - }); - } - - private static class CloseThread extends Thread{ - @Override - public void - run() { - if (myDataRetrievalThread != null) { - myDataRetrievalThread.interrupt(); - try { - myDataRetrievalThread.join(3000); - } catch (InterruptedException e) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } - } - if (myDataBlock != null) { - try { - myDataBlock.close(); - } catch (TOSDataBridge.TOSDataBridgeException e) { - e.printStackTrace(); - } - } - } - } - - private static boolean - setupTOSDataBridge(String dllPath){ - try { - TOSDataBridge.init(dllPath); - switch( TOSDataBridge.connectionState() ){ - case TOSDataBridge.CONN_NONE: - displayMessage("Error", "Failed to Connect to Engine (CONN_NONE)", - JOptionPane.ERROR_MESSAGE); - return false; - case TOSDataBridge.CONN_ENGINE: - displayMessage("Error", "Failed to Connect to TOS (CONN_ENGINE)", - JOptionPane.ERROR_MESSAGE); - return false; - case TOSDataBridge.CONN_ENGINE_TOS: - displayMessage("Success", "Connected to Engine and TOS (CONN_ENGINE_TOS)", - JOptionPane.INFORMATION_MESSAGE); - } - myDataBlock = new DataBlockWithDateTime(DATA_BLOCK_SIZE); - myDataBlock.addTopic(Topic.LAST); - myDataBlock.addTopic(Topic.VOLUME); - return true; - }catch( TOSDataBridge.TOSDataBridgeException e ){ - displayMessage("Error", "TOSDataBridge Exception: " + e.getMessage(), JOptionPane.ERROR_MESSAGE); - return false; - } - } - - private static void - setupGUI(){ - myFrame = new JFrame(TimeAndSales.class.getSimpleName()); - myFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); - myPanel = new JPanel(new BorderLayout()); - myToolBar = new JToolBar(); - myTable = new JTable( new MyTableModel() ); - myTable.setDefaultRenderer(Object.class, new MyTableCellRenderer()); - myTable.getColumnModel().getColumn(0).setPreferredWidth(180); - myScrollPane = new JScrollPane(myTable, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, - JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED); - myToolBar.setComponentOrientation(ComponentOrientation.LEFT_TO_RIGHT); - myToolBar.setMargin(new Insets(2,2,2,2)); - mySymbolEntryButton = new JButton("Enter"); - mySymbolEntryButton.addActionListener( new EntryButtonListener() ); - mySymbolEntryTextField = new JTextField("",6); - mySymbolEntryTextField.setMaximumSize(mySymbolEntryTextField.getPreferredSize()); - myPanel.setPreferredSize(new Dimension(TIMESALE_TABLE_WIDTH, TIMESALE_TABLE_HEIGHT)); - mySymbolEntryButton.setPreferredSize(new Dimension(40,30)); - myToolBar.setPreferredSize(new Dimension(TIMESALE_TABLE_WIDTH,30)); - myToolBar.add(mySymbolEntryTextField); - myToolBar.add(mySymbolEntryButton); - myPanel.add(myToolBar, BorderLayout.PAGE_START); - myPanel.add(myScrollPane, BorderLayout.CENTER); - myToolBar.setFloatable(false); - myFrame.setContentPane(myPanel); - myFrame.getRootPane().setDefaultButton(mySymbolEntryButton); - myFrame.pack(); - myFrame.setLocationRelativeTo(null); - myFrame.setVisible(true); - } - - private static class MyTableModel extends DefaultTableModel{ - public - MyTableModel(){ - super( new String[]{"Time", "Price", "Volume"}, 0); - } - @Override - public boolean - isCellEditable(int row, int column){ - return false; - } - } - - private static class MyTableCellRenderer extends DefaultTableCellRenderer{ - @Override - public Component - getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, - int row, int column){ - final Component c = super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); - c.setBackground( myRowColors.get(row) ); - return c; - } - } - - private static class - EntryButtonListener implements ActionListener { - @Override - public void - actionPerformed(ActionEvent actionEvent){ - String symbol = mySymbolEntryTextField.getText(); - try { - if( myDataRetrievalThread != null ){ - myDataRetrievalThread.interrupt(); - myDataRetrievalThread.join(); - - } - Set oldSymbols = myDataBlock.getItems(); - myDataBlock.addItem( symbol ); - for( String item : oldSymbols ){ - myDataBlock.removeItem(item); - } - myDataRetrievalThread = new DataRetrievalThread(symbol); - myDataRetrievalThread.start(); - }catch( TOSDataBridge.TOSDataBridgeException e ){ - displayMessage("Error", "Error adding symbol '" + symbol + "':" + e.getMessage(), - JOptionPane.ERROR_MESSAGE); - }catch( InterruptedException e ){ - Thread.currentThread().interrupt(); - } - } - } - - private static void - displayMessage(final String title, final String message, final int messageType){ - SwingUtilities.invokeLater( - new Runnable() { - @Override - public void run() { - JOptionPane.showMessageDialog(null, message, title, messageType); - } - } - ); - } - -} diff --git a/java/jna.jar b/java/jna.jar deleted file mode 100644 index 713354e..0000000 Binary files a/java/jna.jar and /dev/null differ diff --git a/java/src/io/github/jeog/tosdatabridge/CError.java b/java/src/io/github/jeog/tosdatabridge/CError.java deleted file mode 100644 index fd0beb6..0000000 --- a/java/src/io/github/jeog/tosdatabridge/CError.java +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * Error codes that can be returned by the C Lib. - * - * @author Jonathon Ogden - * @version 0.9 - */ -public final class CError { - public static final int ERROR_BAD_INPUT = -1; - public static final int ERROR_BAD_INPUT_BUFFER = -2; - public static final int ERROR_NOT_CONNECTED = -3; - public static final int ERROR_TIMEOUT = -4; - public static final int ERROR_BLOCK_ALREADY_EXISTS = -5; - public static final int ERROR_BLOCK_DOESNT_EXIST = -6; - public static final int ERROR_BLOCK_CREATION = -7; - public static final int ERROR_BLOCK_SIZE = -8; - public static final int ERROR_BAD_TOPIC = -9; - public static final int ERROR_BAD_ITEM = -10; - public static final int ERROR_BAD_SIG = -11; - public static final int ERROR_IPC = -12; - public static final int ERROR_IPC_MSG = -13; - public static final int ERROR_CONCURRENCY = -14; - public static final int ERROR_ENGINE_NO_TOPIC = -15; - public static final int ERROR_ENGINE_NO_ITEM = -16; - public static final int ERROR_SERVICE = -17; - public static final int ERROR_GET_DATA = -18; - public static final int ERROR_GET_STATE = -19; - public static final int ERROR_SET_STATE = -20; - public static final int ERROR_DDE_POST = -21; - public static final int ERROR_DDE_NO_ACK = -22; - public static final int ERROR_SHEM_BUFFER = -23; - public static final int ERROR_UNKNOWN = -24; - public static final int ERROR_DECREMENT_BASE = -25; - - private static final Map ERROR_STRING_LOOKUP = new HashMap<>(); - static{ - ERROR_STRING_LOOKUP.put(ERROR_BAD_INPUT, "ERROR_BAD_INPUT"); - ERROR_STRING_LOOKUP.put(ERROR_BAD_INPUT_BUFFER, "ERROR_BAD_INPUT_BUFFER"); - ERROR_STRING_LOOKUP.put(ERROR_NOT_CONNECTED, "ERROR_NOT_CONNECTED"); - ERROR_STRING_LOOKUP.put(ERROR_TIMEOUT, "ERROR_TIMEOUT"); - ERROR_STRING_LOOKUP.put(ERROR_BLOCK_ALREADY_EXISTS, "ERROR_BLOCK_ALREADY_EXISTS"); - ERROR_STRING_LOOKUP.put(ERROR_BLOCK_DOESNT_EXIST, "ERROR_BLOCK_DOESNT_EXIST"); - ERROR_STRING_LOOKUP.put(ERROR_BLOCK_CREATION, "ERROR_BLOCK_CREATION"); - ERROR_STRING_LOOKUP.put(ERROR_BLOCK_SIZE, "ERROR_BLOCK_SIZE"); - ERROR_STRING_LOOKUP.put(ERROR_BAD_TOPIC, "ERROR_BAD_TOPIC"); - ERROR_STRING_LOOKUP.put(ERROR_BAD_ITEM, "ERROR_BAD_ITEM"); - ERROR_STRING_LOOKUP.put(ERROR_BAD_SIG, "ERROR_BAD_SIG"); - ERROR_STRING_LOOKUP.put(ERROR_IPC, "ERROR_IPC"); - ERROR_STRING_LOOKUP.put(ERROR_IPC_MSG, "ERROR_IPC_MSG"); - ERROR_STRING_LOOKUP.put(ERROR_CONCURRENCY, "ERROR_CONCURRENCY"); - ERROR_STRING_LOOKUP.put(ERROR_ENGINE_NO_TOPIC, "ERROR_ENGINE_NO_TOPIC"); - ERROR_STRING_LOOKUP.put(ERROR_ENGINE_NO_ITEM, "ERROR_ENGINE_NO_ITEM"); - ERROR_STRING_LOOKUP.put(ERROR_SERVICE, "ERROR_SERVICE"); - ERROR_STRING_LOOKUP.put(ERROR_GET_DATA, "ERROR_GET_DATA"); - ERROR_STRING_LOOKUP.put(ERROR_GET_STATE, "ERROR_GET_STATE"); - ERROR_STRING_LOOKUP.put(ERROR_SET_STATE, "ERROR_SET_STATE"); - ERROR_STRING_LOOKUP.put(ERROR_DDE_POST, "ERROR_DDE_POST"); - ERROR_STRING_LOOKUP.put(ERROR_DDE_NO_ACK, "ERROR_DDE_NO_ACK"); - ERROR_STRING_LOOKUP.put(ERROR_SHEM_BUFFER, "ERROR_SHEM_BUFFER"); - ERROR_STRING_LOOKUP.put(ERROR_UNKNOWN, "ERROR_UNKNOWN"); - ERROR_STRING_LOOKUP.put(ERROR_DECREMENT_BASE, "ERROR_DECREMENT_BASE"); - } - - /** - * Return error string from error code. - * - * @param errorCode error code - * @return error string - */ - public static String - errorLookup(int errorCode){ - String errorStr = ERROR_STRING_LOOKUP.get(errorCode); - if(errorStr != null) { - return errorStr; - } - int baseError = Collections.min(ERROR_STRING_LOOKUP.keySet()); - if(errorCode < baseError) { - return "ERROR_DECREMENT_BASE(" + String.valueOf(errorCode - baseError) + ")"; - }else { - return "***unrecognized error code***"; - } - } -} diff --git a/java/src/io/github/jeog/tosdatabridge/CLib.java b/java/src/io/github/jeog/tosdatabridge/CLib.java deleted file mode 100644 index 7bdebd2..0000000 --- a/java/src/io/github/jeog/tosdatabridge/CLib.java +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import com.sun.jna.Library; -import com.sun.jna.NativeLong; -import com.sun.jna.Pointer; - -/** - * JNA interface used to access the underlying C Lib. - * - * @author Jonathon Ogden - * @version 0.9 - * @see README - C API - */ -public interface CLib extends Library { - int TOSDB_Connect(); - int TOSDB_IsConnectedToEngineAndTOS(); - int TOSDB_ConnectionState(); - int TOSDB_GetBlockLimit(); - int TOSDB_SetBlockLimit(int limit); - int TOSDB_GetBlockCount(); - int TOSDB_GetTypeBits(String topic, byte[] ptrBits); - - int TOSDB_CreateBlock(String name, int size, boolean dateTime, int timeout); - int TOSDB_CloseBlock(String name); - int TOSDB_GetItemCount(String name, int[] ptrCount); - int TOSDB_GetTopicCount(String name, int[] ptrCount); - int TOSDB_GetPreCachedItemCount(String name, int[] ptrCount); - int TOSDB_GetPreCachedTopicCount(String name, int[] ptrCount); - int TOSDB_GetBlockSize(String name, int[] ptrSize); - int TOSDB_SetBlockSize(String name, int size); - int TOSDB_GetStreamOccupancy(String name, String item, String topic, int[] ptrOcc); - - int TOSDB_GetItemNames(String name, Pointer[] arrayStrings, int arraySz, int strSz); - int TOSDB_GetTopicNames(String name, Pointer[] arrayStrings, int arraySz, int strSz); - int TOSDB_GetPreCachedItemNames(String name, Pointer[] arrayStrings, int arraySz, int strSz); - int TOSDB_GetPreCachedTopicNames(String name, Pointer[] arrayStrings, int arraySz, int strSz); - - int TOSDB_AddItem(String name, String item); - int TOSDB_AddTopic(String name, String topic); - int TOSDB_RemoveItem(String name, String item); - int TOSDB_RemoveTopic(String name, String topic); - - int TOSDB_GetString(String name, String item, String topic, int indx, byte[] ptrVal, int strSz, - DateTime ptrDateTime); - - int TOSDB_GetDouble(String name, String item, String topic, int indx, double[] ptrVal, - DateTime ptrDateTime); - - int TOSDB_GetLongLong(String name, String item, String topic, int indx, long[] ptrVal, - DateTime ptrDateTime); - - int TOSDB_GetStreamSnapshotStrings(String name, String item, String topic, Pointer[] arrayVals, - int arraySz, int strSz, DateTime[] arrayDateTime, int end, - int beg); - - int TOSDB_GetStreamSnapshotDoubles(String name, String item, String topic, double[] arrayVals, - int arraySz, DateTime[] arrayDateTime, int end, int beg); - - int TOSDB_GetStreamSnapshotLongLongs(String name, String item, String topic, long[] arrayVals, - int arraySz, DateTime[] arrayDateTime, int end, int beg); - - int TOSDB_IsMarkerDirty(String name, String item, String topic, int[] ptrVal); - int TOSDB_GetMarkerPosition(String name, String item, String topic, long[] ptrVal); - - int TOSDB_GetStreamSnapshotStringsFromMarker(String name, String item, String topic, - Pointer[] arrayVals, int arraySz, int strSz, - DateTime[] arrayDateTime, int beg, - NativeLong[] getSz); - - int TOSDB_GetStreamSnapshotDoublesFromMarker(String name, String item, String topic, - double[] arrayVals, int arraySz, - DateTime[] arrayDateTime, int beg, - NativeLong[] getSz); - - int TOSDB_GetStreamSnapshotLongLongsFromMarker(String name, String item, String topic, - long[] arrayVals, int arraySz, - DateTime[] arrayDateTime, int beg, - NativeLong[] getSz); - - int TOSDB_GetNStringsFromMarker(String name, String item, String topic, Pointer[] arrayVals, - int n, int strSz, DateTime[] arrayDateTime, NativeLong[] getSz); - - int TOSDB_GetNDoublesFromMarker(String name, String item, String topic, double[] arrayVals, - int n, DateTime[] arrayDateTime, NativeLong[] getSz); - - int TOSDB_GetNLongLongsFromMarker(String name, String item, String topic, long[] arrayVals, - int n, DateTime[] arrayDateTime, NativeLong[] getSz); - - int TOSDB_GetItemFrameStrings(String name, String topic, Pointer[] arrayVals, int arraySz, - int strSz, Pointer[] arrayLabels, int strLabelSz, - DateTime[] arrayDateTime); - - int TOSDB_GetTopicFrameStrings(String name, String item, Pointer[] arrayVals, int arraySz, - int strSz, Pointer[] arrayLabels, int strLabelSz, - DateTime[] arrayDateTime); -} diff --git a/java/src/io/github/jeog/tosdatabridge/DataBlock.java b/java/src/io/github/jeog/tosdatabridge/DataBlock.java deleted file mode 100644 index 1bd436c..0000000 --- a/java/src/io/github/jeog/tosdatabridge/DataBlock.java +++ /dev/null @@ -1,1895 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import io.github.jeog.tosdatabridge.TOSDataBridge.*; -import io.github.jeog.tosdatabridge.DateTime.DateTimePair; - -import com.sun.jna.Memory; -import com.sun.jna.Native; -import com.sun.jna.NativeLong; -import com.sun.jna.Pointer; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; - -/** - * DataBlock is used to get real-time financial data from the TOS platform. - * It uses the JNA interface(CLib.java) to access the C API, wrapping the - * underlying (conceptual) block of the C lib with an object-oriented interface. - *

- * IMPORTANT: Before creating/using a DataBlock the underlying library - * must be loaded AND connected to the backend Service/Engine and the TOS platform. - *

    - *
  1. be sure you've installed the C mods (tosdb-setup.bat) - *
  2. start the Service/Engine (SC Start TOSDataBridge) - *
  3. start the TOS Platform - *
  4. call TOSDataBridge.init(...) - *
  5. create DataBlock(s) - *
- *

- * DataBlock works much the same as its python cousin (tosdb._win.TOSDB_DataBlock). - * It's instantiated with a size parameter to indicate how many historical data-points are - * saved. Once created 'topics' (data fields found in the Topic enum) and 'items' (symbols) - * are added to create a matrix of 'streams', each the size of the block. The block can - * be thought of as a 3D object with the following dimensions: - *

- *      (# of topics)   x   (# of items)   x   (block size) - *

- * As data flows into these streams the various 'get' methods are then used to - * 1) examine the underlying state of the block/streams, and 2) pull data from - * the block/streams in myriad ways. - * - * @author Jonathon Ogden - * @version 0.9 - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @see TOSDataBridge - * @see Topic - * @see DateTime - * @see README - * @see README - JAVA API - * @see README - C API - * @see README - DETAILS - */ -public class DataBlock { - /* max size of non-data (e.g topics, labels) strings (excluding \0)*/ - public static final int MAX_STR_SZ = TOSDataBridge.MAX_STR_SZ; - - /* max size of data strings (getString, getStreamSnapshotStrings etc.) */ - public static final int STR_DATA_SZ = TOSDataBridge.STR_DATA_SZ; - - /* how much buffer 'padding' for the marker calls */ - public static final int MARKER_MARGIN_OF_SAFETY = TOSDataBridge.MARKER_MARGIN_OF_SAFETY; - - /* default timeout for communicating with the engine/platform */ - public static final int DEF_TIMEOUT = TOSDataBridge.DEF_TIMEOUT; - - /* an implementation layer (below API, above JNA) we want to share with classes that - may extend DataBlock within the package, but not with client code from outside it */ - private final DataBlockSharedHelper _helper = new DataBlockSharedHelper(); - - /*package-private*/ - final DataBlockSharedHelper - getHelper(){ - return _helper; - } - - private final String _name; - private final int _timeout; - private int _size; - private Set _items = new HashSet<>(); - private Set _topics = new HashSet<>(); - private Set _itemsPreCached = new HashSet<>(); - private Set _topicsPreCached = new HashSet<>(); - - /** - * DataBlock Constructor. - * - * @param size maximum amount of historical data block can hold - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - */ - public DataBlock(int size) throws CLibException, LibraryNotLoaded { - this(size, DEF_TIMEOUT); - } - - /** - * DataBlock Constructor. - * - * @param size maximum amount of historical data block can hold - * @param timeout timeout of internal IPC and DDE mechanisms (milliseconds) - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public DataBlock(int size, int timeout) throws LibraryNotLoaded, CLibException { - this(size, false, timeout); - } - - /** - * INTERNAL CONSTRUCTOR - */ - protected DataBlock(int size, boolean dateTime, int timeout) - throws LibraryNotLoaded, CLibException - { - _size = size; - _timeout = timeout; - _name = UUID.randomUUID().toString(); - int err = TOSDataBridge.getCLibrary() - .TOSDB_CreateBlock(_name, size, dateTime, timeout); - if (err != 0) { - throw new CLibException("TOSDB_CreateBlock", err); - } - } - - /** - * Close the underlying C lib block. - * - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - */ - public void - close() throws CLibException, LibraryNotLoaded { - int err = TOSDataBridge.getCLibrary().TOSDB_CloseBlock(_name); - if (err != 0) { - throw new CLibException("TOSDB_CloseBlock", err); - } - } - - protected void - finalize() throws Throwable { - try { - close(); - } finally { - super.finalize(); - } - } - - /** - * @return name of block (implementation detail). - */ - public String - getName() { - return _name; - } - - /** - * @return if block is storing DateTime alongside primary data - */ - public boolean - isUsingDateTime() { - return false; - } - - /** - * @return timeout the underlying block uses for IPC/DDE (milliseconds) - */ - public int - getTimeout() { - return _timeout; - } - - /** - * @return maximum amount of data that can be stored in the block/streams. - * - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public int - getBlockSize() throws LibraryNotLoaded, CLibException { - int sz = _getBlockSize(); - if (sz != _size) { - throw new IllegalStateException("_getBlockSize() != _size"); - } - return sz; - } - - /** - * Set maximum amount of data that can be stored in the block/streams. - * - * @param size new size of the block - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public void - setBlockSize(int size) throws LibraryNotLoaded, CLibException { - int err = TOSDataBridge.getCLibrary().TOSDB_SetBlockSize(_name, size); - /* need to update regardless */ - _size = _getBlockSize(); - if (err != 0) { - throw new CLibException("TOSDB_SetBlockSize", err); - } - if (size != _size) { - throw new IllegalStateException("size != _size"); - } - } - - /** - * Returns number of data-points currently in the stream. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @return number of data-points currently in the stream - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @see Topic - */ - public int - getStreamOccupancy(String item, Topic topic) throws CLibException, LibraryNotLoaded, - InvalidItemOrTopic { - item = _handleRawItemTopic(item,topic,true); - int occ[] = {0}; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamOccupancy(_name, item, topic.val, occ); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamOccupancy", err); - } - return occ[0]; - } - - /** - * @return items currently in the block - */ - public Set - getItems() { - return new HashSet<>(_items); - } - - /** - * @return topics currently in the block - * - * @see Topic - */ - public Set - getTopics() { - return new HashSet<>(_topics); - } - - /** - * @return items currently in the block's pre-cache - */ - public Set - getItemsPreCached() { - return new HashSet<>(_itemsPreCached); - } - - /** - * @return topics currently in the block's pre-cache - * - * @see Topic - */ - public Set - getTopicsPreCached() { - return new HashSet<>(_topicsPreCached); - } - - /** - * Add item to the block. - * - * @param item item string to be added to the block - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public void - addItem(String item) throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - item = _handleRawItem(item, /*throwIfNotInBlock=*/false); - /* check we are consistent with C lib */ - _goodItemTopicStateOrThrow(); - /* if we already have it */ - if (_items.contains(item) || _itemsPreCached.contains(item)) { - return; - } - int err = TOSDataBridge.getCLibrary().TOSDB_AddItem(_name, item); - /* sync with C block regardless */ - _syncItemTopicState(); - if (err != 0) { - throw new CLibException("TOSDB_AddItem", err); - } - } - - /** - * Add topic to the block. - * - * @param topic topic enum to be added to the block - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public void - addTopic(Topic topic) throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - _handleRawTopic(topic, /*throwIfNotInBlock=*/false); - /* check we are consistent with C lib */ - _goodItemTopicStateOrThrow(); - /* if we already have it */ - if (_topics.contains(topic) || _topicsPreCached.contains(topic)) { - return; - } - int err = TOSDataBridge.getCLibrary().TOSDB_AddTopic(_name, topic.val); - /* sync with C block regardless */ - _syncItemTopicState(); - if (err != 0) { - throw new CLibException("TOSDB_AddTopic", err); - } - } - - /** - * Remove item from the block. - * - * @param item item string to be removed from the block - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public void - removeItem(String item) throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - /* note: doesn't throw if item isn't it block (it might be in pre-cache) */ - item = _handleRawItem(item, /*throwIfNotInBlock=*/false); - /* check we are consistent with C lib */ - _goodItemTopicStateOrThrow(); - int err = TOSDataBridge.getCLibrary().TOSDB_RemoveItem(_name, item); - /* sync with C block regardless */ - _syncItemTopicState(); - if (err != 0) { - throw new CLibException("TOSDB_RemoveItem", err); - } - } - - /** - * Remove topic from the block. - * - * @param topic topic enum to be removed from the block. - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public void - removeTopic(Topic topic) throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - /* note: doesn't throw if topic isn't in block (it might be in pre-cache) */ - _handleRawTopic(topic, /*throwIfNotInBlock=*/false); - /* check we are consistent with C lib */ - _goodItemTopicStateOrThrow(); - int err = TOSDataBridge.getCLibrary().TOSDB_RemoveTopic(_name, topic.val); - /* sync with C block regardless */ - _syncItemTopicState(); - if (err != 0) { - throw new CLibException("TOSDB_RemoveTopic", err); - } - } - - /** - * Returns if item is in the block.(NOT THE PRE-CACHE) - * - * @param item item string to be search for - * @return if item is in the block - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public boolean - containsItem(String item) throws InvalidItemOrTopic { - item = _handleRawItem(item, /*throwIfNotInBlock=*/false); - return _items.contains(item); - } - - /** - * Returns if topic is in the block.(NOT THE PRE-CACHE) - * - * @param topic topic enum to be searched for - * @return if topic is in the block - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public boolean - containsTopic(Topic topic) throws InvalidItemOrTopic { - _handleRawTopic(topic, /*throwIfNotInBlock=*/false); - return _topics.contains(topic); - } - - /** - * Returns if item is in the block's pre-cache. - * - * @param item item string to be searched for - * @return if item is in the block's pre-cache - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public boolean - containsItemPreCached(String item) throws InvalidItemOrTopic { - item = _handleRawItem(item, /*throwIfNotInBlock=*/false); - return _itemsPreCached.contains(item); - } - - /** - * Returns if topic is in the block's pre-cache. - * - * @param topic topic enum to be searched for - * @return if topic is in the block's pre-cache - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public boolean - containsTopicPreCached(Topic topic) throws InvalidItemOrTopic { - _handleRawTopic(topic, /*throwIfNotInBlock=*/false); - return _topicsPreCached.contains(topic); - } - - /** - * Returns most recent data-point of a stream, as Long (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Long - getLong(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return _helper.getMostRecent(item, topic, false, Long.class); - } - - /** - * Returns data-point of a stream, as Long (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Long - getLong(String item, Topic topic, int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return _helper.get(item, topic, indx, false, Long.class); - } - - /** - * Returns most recent data-point of a stream, as Double (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Double - getDouble(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return _helper.getMostRecent(item, topic, false, Double.class); - } - - /** - * Returns data-point of a stream, as Double (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Double - getDouble(String item, Topic topic, int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return _helper.get(item, topic, indx, false, Double.class); - } - - /** - * Returns most recent data-point of a stream, as String (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public String - getString(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return _helper.getMostRecent(item, topic, false, String.class); - } - - /** - * Returns data-point of a stream, as String (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public String - getString(String item, Topic topic, int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return _helper.get(item, topic, indx, false, String.class); - } - - /** - * Returns all data-points of a stream, as List<Long>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongs(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotAll(item, topic, false, Long.class); - } - - /** - * Returns multiple contiguous data-points of a stream, from most recent, - * as List<Long>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongs(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, 0, true, false, Long.class); - } - - /** - * Returns multiple contiguous data-points of a stream, as List<Long>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongs(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, beg, true, false, Long.class); - } - - /** - * Returns all data-points of a stream, as List<Double>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoubles(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotAll(item, topic, false, Double.class); - } - - /** - * Returns multiple contiguous data-points of a stream, from most recent, - * as List<Double>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoubles(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, 0, true, false, Double.class); - } - - /** - * Returns multiple contiguous data-points of a stream, as List<Double>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoubles(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, beg, true, false, Double.class); - } - - /** - * Returns all data-points of a stream, as List<String>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStrings(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotAll(item, topic, false, String.class); - } - - /** - * Returns multiple contiguous data-points of a stream, from most recent, - * as List<String>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStrings(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, 0, true, false, String.class); - } - - /** - * Returns multiple contiguous data-points of a stream, as List<String>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStrings(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshot(item, topic, end, beg, true, false, String.class); - } - - /** - * Returns all data-points up to atomic marker of a stream, as List<Long>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongsFromMarker(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecent(item, topic, false, Long.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<Long>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongsFromMarker(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarker(item, topic, beg, true, false, Long.class); - } - - /** - * Returns all data-points up to atomic marker of a stream, as List<Double>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoublesFromMarker(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecent(item, topic, false, Double.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<Double>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoublesFromMarker(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarker(item, topic, beg, true, false, Double.class); - } - - /** - * Returns all data-points up to atomic marker of a stream, as List<String>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStringsFromMarker(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecent(item, topic, false, String.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<String>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStringsFromMarker(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarker(item, topic, beg, true, false, String.class); - } - - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<Long>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNLongsFromMarker(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getNFromMarker(item, topic, n, true, false, Long.class); - } - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<Double>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNDoublesFromMarker(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getNFromMarker(item, topic, n, true, false, Double.class); - } - - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<String>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNStringsFromMarker(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return _helper.getNFromMarker(item, topic, n, true, false, String.class); - } - - /** - * Returns if marker is currently in a 'dirty' state. NOTE: there is no - * guarantee it won't enter this state before another call is made. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return if stream/marker is dirty - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see "StreamSnapshotFromMarker calls" - * @see Topic - */ - public boolean - isDirty(String item, Topic topic) throws LibraryNotLoaded, CLibException, - InvalidItemOrTopic { - item = _handleRawItemTopic(item,topic,true); - int[] ptrIsDirty = {0}; - int err = TOSDataBridge.getCLibrary() - .TOSDB_IsMarkerDirty(_name, item, topic.val, ptrIsDirty); - if (err != 0) { - throw new CLibException("TOSDB_IsMarkerDirty", err); - } - return ptrIsDirty[0] != 0; - } - - - /** - * Returns all data-points up to atomic marker of a stream, as List<Long>. - * Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongsFromMarkerIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, false, - Long.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<Long>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotLongsFromMarkerIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, false, - Long.class); - } - - /** - * Returns all data-points up to atomic marker of a stream, as List<Double>. - * Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoublesFromMarkerIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, false, - Double.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<Double>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotDoublesFromMarkerIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, false, - Double.class); - } - - /** - * Returns all data-points up to atomic marker of a stream, as List<String>. - * Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStringsFromMarkerIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, false, - String.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<String>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getStreamSnapshotStringsFromMarkerIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, false, - String.class); - } - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<Long>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNLongsFromMarkerIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getNFromMarkerIgnoreDirty(item, topic, n, false, Long.class); - } - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<Double>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNDoublesFromMarkerIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getNFromMarkerIgnoreDirty(item, topic, n, false, Double.class); - } - - /** - * Returns 'n' contiguous data-points in front of atomic marker of a stream, - * as List<String>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List - getNStringsFromMarkerIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return _helper.getNFromMarkerIgnoreDirty(item, topic, n, false, String.class); - } - - /** - * Returns mapping of all item names to most recent item values for a particular topic. - * - * @param topic topic enum of streams - * @return mapping of item names to most recent item values - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Map - getItemFrame(Topic topic) throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return _helper.getFrame(topic, true, false); - } - - /** - * Returns mapping of all topic enums to most recent topic values for a particular item. - * - * @param item item string of streams - * @return mapping of topic enums to most recent topic values - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public Map - getTopicFrame(String item) throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return _helper.getFrame(item, false, false); - } - - /** - * Returns mapping of all item strings to mappings of all topic enums to most recent topic - * values. This returns ALL the most recent data in the block. - * - * @return mapping of item strings to mappings of topic enums to most recent topic values - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public Map> - getTotalFrame() throws LibraryNotLoaded, CLibException { - Map> frame = new HashMap<>(); - for (String item : getItems()) { - Map tf = null; - try { - tf = getTopicFrame(item); - } catch (InvalidItemOrTopic e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("InvalidItemOrTopic caught in getTotalFrame"); - } - frame.put(item, tf); - } - return frame; - } - - /** - * (package-private) inner class that houses a number of 'unsafe' type - * operations that we only want to expose to the implementation of the - * DataBlock hierarchy within the package. It allows for code reuse below - * the API calls but above many of the JNA calls because it still has - * access to DataBlock's private methods/fields. *The* private instance - * is accessed via DataBlock's getHelper() method. - */ - class DataBlockSharedHelper { - - /* only allow DataBlock to instantiate */ - private DataBlockSharedHelper(){ - } - - /* suppress DataIndexException */ - public final T - getMostRecent(String item, Topic topic, boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - try { - return get(item, topic, 0, withDateTime, valType); - } catch (DataIndexException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getMostRecent failed to suppress DataIndexException"); - } - } - - @SuppressWarnings("unchecked") - public final T - get(String item, Topic topic, int indx, boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic - { - item = _handleRawItemTopic(item,topic,true); - if (indx < 0) { - indx += _size; - } - if (indx >= _size) { - throw new DataIndexException("indx >= _size in DataBlock"); - } - if (indx >= getStreamOccupancy(item, topic)) { - return null; - } - return _nativeAccessByType(valType,"_get","", new Class[]{String.class, Topic.class, - int.class, boolean.class}, item, topic, indx, withDateTime); - } - - /* suppress DataIndexException */ - public final List - getStreamSnapshotAll(String item, Topic topic, boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - try { - return getStreamSnapshot(item, topic, -1, 0, true, withDateTime, valType); - } catch (DataIndexException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getStreamSnapshotAll failed to suppress DataIndexException"); - } - } - - @SuppressWarnings("unchecked") - public final List - getStreamSnapshot(String item, Topic topic, int end, int beg, boolean smartSize, - boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, - InvalidItemOrTopic - { - item = _handleRawItemTopic(item,topic,true); - if (end < 0) { - end += _size; - } - if (beg < 0) { - beg += _size; - } - int size = end - beg + 1; - if ((beg < 0) || (end < 0) || (beg >= _size) || (end >= _size) || (size <= 0)) { - throw new DataIndexException("invalid 'beg' and/or 'end' index"); - } - - if (smartSize) { - int occ = getStreamOccupancy(item, topic); - if (occ == 0 || occ <= beg) { //redundant ? - return new ArrayList<>(); - } - end = Math.min(end, occ - 1); - beg = Math.min(beg, occ - 1); - size = end - beg + 1; - } - - return _nativeAccessByType(valType, "_getStreamSnapshot", "s", - new Class[]{String.class, Topic.class, int.class, int.class, - boolean.class, int.class}, item, topic, end, beg, withDateTime, size); - } - - /* suppress DirtyMarkerException */ - public final List - getStreamSnapshotFromMarkerIgnoreDirty(String item, Topic topic, int beg, - boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, - InvalidItemOrTopic { - try { - return getStreamSnapshotFromMarker(item, topic, beg, false, withDateTime, valType); - } catch (DirtyMarkerException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getStreamSnapshotFromMarkerIgnoreDirty " + - "failed to ignore dirty marker"); - } - } - - /* suppress DataIndexException */ - public final List - getStreamSnapshotFromMarkerToMostRecent(String item, Topic topic, boolean withDateTime, - Class valType) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - try { - return getStreamSnapshotFromMarker(item, topic, 0, true, withDateTime, valType); - } catch (DataIndexException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getStreamSnapshotFromMarkerToMostRecent" + - " failed to suppress DataIndexException"); - } - } - - /* suppress DataIndexException and DirtyMarkerException */ - public final List - getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(String item, Topic topic, - boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - try { - return getStreamSnapshotFromMarker(item, topic, 0, false, withDateTime, valType); - } catch (DataIndexException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getStreamSnapshotFromMarkerToMostRecentIgnoreDirty" + - " failed to suppress DataIndexException"); - } catch (DirtyMarkerException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getStreamSnapshotFromMarkerToMostRecentIgnoreDirty" + - " failed to ignore dirty marker"); - } - } - - @SuppressWarnings("unchecked") - public final List - getStreamSnapshotFromMarker(String item, Topic topic, int beg, boolean throwIfDataLost, - boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic - { - item = _handleRawItemTopic(item,topic,true); - if (beg < 0) { - beg += _size; - } - if (beg < 0 || beg >= _size) { - throw new DataIndexException("invalid 'beg' index argument"); - } - if (isDirty(item, topic) && throwIfDataLost) { - throw new DirtyMarkerException(); - } - - long[] markerPosition = {0}; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetMarkerPosition(_name, item, topic.val, markerPosition); - if (err != 0) { - throw new CLibException("TOSDB_GetMarkerPosition", err); - } - long szFromMarker = markerPosition[0] - beg + 1; - if (szFromMarker < 0) { - return new ArrayList<>(); - } - - return _nativeAccessByType(valType, "_getStreamSnapshot", "sFromMarker", - new Class[]{String.class, Topic.class, int.class, boolean.class, - boolean.class, int.class}, item, topic, beg, withDateTime, - throwIfDataLost, (int) szFromMarker + MARKER_MARGIN_OF_SAFETY); - } - - /* suppress DirtyMarkerException */ - public final List - getNFromMarkerIgnoreDirty(String item, Topic topic, int n, boolean withDateTime, - Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, - InvalidItemOrTopic { - try { - return getNFromMarker(item, topic, n, false, withDateTime, valType); - } catch (DirtyMarkerException e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("getNFromMarkerIgnoreDirty " + - "failed to ignore dirty marker"); - } - } - - @SuppressWarnings("unchecked") - public final List - getNFromMarker(String item, Topic topic, int n, boolean throwIfDataLost, - boolean withDateTime, Class valType) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic - { - item = _handleRawItemTopic(item,topic,true); - if (isDirty(item, topic) && throwIfDataLost) { - throw new DirtyMarkerException(); - } - if( n < 1 ){ - throw new DataIndexException("'n' not >= 1"); - } - - return _nativeAccessByType(valType, "_getN", "sFromMarker", - new Class[]{String.class, Topic.class, int.class, boolean.class, - boolean.class}, item, topic, n, withDateTime, throwIfDataLost); - } - - public final Map - getFrame(E e, boolean itemFrame, boolean withDateTime) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic - { - if (itemFrame) { - _handleRawTopic((Topic)e, true); - } else { - _handleRawItem((String)e, true); - } - - int n = itemFrame ? _getItemCount() : _getTopicCount(); - Pointer[] vals = new Pointer[n]; - Pointer[] labels = new Pointer[n]; - for (int i = 0; i < n; ++i) { - vals[i] = new Memory(STR_DATA_SZ + 1); - labels[i] = new Memory(MAX_STR_SZ + 1); - } - DateTime[] dts = (withDateTime ? new DateTime[n] : null); - - int err; - if (itemFrame) { - err = TOSDataBridge.getCLibrary() - .TOSDB_GetItemFrameStrings(_name, ((Topic) e).toString(), vals, n, - STR_DATA_SZ + 1, labels, MAX_STR_SZ + 1, dts); - } else { - err = TOSDataBridge.getCLibrary() - .TOSDB_GetTopicFrameStrings(_name, ((String) e).toUpperCase(), vals, n, - STR_DATA_SZ + 1, labels, MAX_STR_SZ + 1, dts); - } - if (err != 0) { - throw new CLibException("TOSDB_Get" + (itemFrame ? "Item" : "Topic") + "FrameStrings", - err); - } - - Map frame = new HashMap<>(); - for (int i = 0; i < n; ++i) { - @SuppressWarnings("unchecked") - T val = (T) (withDateTime ? new DateTimePair(vals[i].getString(0), dts[i]) - : vals[i].getString(0)); - String label = labels[i].getString(0); - @SuppressWarnings("unchecked") - X lab = (X) (itemFrame ? label : Topic.toEnum(label)); - frame.put(lab, val); - } - return frame; - } - - @SuppressWarnings("unchecked") - private R - _nativeAccessByType(Class valType, String callNamePre, String callNamePost, - Class[] vArgTypes, Object... args) { - String typeName = valType.equals(Long.class) ? "Long" - : (valType.equals(Double.class) ? "Double" : "String"); - String methodName = callNamePre + typeName + callNamePost; - try { - Method m = DataBlock.class.getDeclaredMethod(methodName, vArgTypes); - m.setAccessible(true); - return (R) m.invoke(DataBlock.this, args); - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException("_nativeAccessByType failed to call: " + methodName - + "; " + e.toString()); - } - } - } /* class DataBlockSharedHelper */ - - - private String - _handleRawItem(String item, boolean throwIfNotInBlock) throws InvalidItemOrTopic { - if(item == null) - throw new NullPointerException("null item string"); - item = item.toUpperCase(); - int slen = item.length(); - if(slen < 1 || slen > MAX_STR_SZ){ - throw new InvalidItemOrTopic("invalid item string length: " + slen); - } - if(throwIfNotInBlock && !_items.contains(item)) { - throw new InvalidItemOrTopic("item not in block: " + item); - } - return item; - } - - private void - _handleRawTopic(Topic topic, boolean throwIfNotInBlock) throws InvalidItemOrTopic { - if(topic == null){ - throw new NullPointerException("null topic enum"); - } - if(topic == Topic.NULL_TOPIC){ - throw new InvalidItemOrTopic("'NULL_TOPIC' is not a valid Topic"); - } - if(throwIfNotInBlock && !_topics.contains(topic)){ - throw new InvalidItemOrTopic("topic not in block: " + topic); - } - } - - private String - _handleRawItemTopic(String item, Topic topic, boolean throwIfNotInBlock) - throws InvalidItemOrTopic { - _handleRawTopic(topic, throwIfNotInBlock); - return _handleRawItem(item, throwIfNotInBlock); - } - - private int - _getBlockSize() throws LibraryNotLoaded, CLibException { - int[] size = {0}; - int err = TOSDataBridge.getCLibrary().TOSDB_GetBlockSize(_name, size); - if (err != 0) { - throw new CLibException("TOSDB_GetBlockSize", err); - } - return size[0]; - } - - private int - _getItemCount() throws CLibException, LibraryNotLoaded { - int[] count = {0}; - int err = TOSDataBridge.getCLibrary().TOSDB_GetItemCount(_name, count); - if (err != 0) { - throw new CLibException("TOSDB_GetItemCount", err); - } - return count[0]; - } - - private int - _getTopicCount() throws CLibException, LibraryNotLoaded { - int[] count = {0}; - int err = TOSDataBridge.getCLibrary().TOSDB_GetTopicCount(_name, count); - if (err != 0) { - throw new CLibException("TOSDB_GetTopicCount", err); - } - return count[0]; - } - - private int - _getItemPreCachedCount() throws CLibException, LibraryNotLoaded { - int[] count = {0}; - int err = TOSDataBridge.getCLibrary().TOSDB_GetPreCachedItemCount(_name, count); - if (err != 0) { - throw new CLibException("TOSDB_GetPreCachedItemCount", err); - } - return count[0]; - } - - private int - _getTopicPreCachedCount() throws CLibException, LibraryNotLoaded { - int[] count = {0}; - int err = TOSDataBridge.getCLibrary().TOSDB_GetPreCachedTopicCount(_name, count); - if (err != 0) { - throw new CLibException("TOSDB_GetPreCachedTopicCount", err); - } - return count[0]; - } - - private Set - _getItemTopicNames(boolean useItemVersion, boolean usePreCachedVersion) - throws LibraryNotLoaded, CLibException { - int n; - if (useItemVersion) { - n = usePreCachedVersion ? _getItemPreCachedCount() : _getItemCount(); - } else { - n = usePreCachedVersion ? _getTopicPreCachedCount() : _getTopicCount(); - } - if (n < 1) { - return new HashSet<>(); - } - - Pointer[] vals = new Pointer[n]; - for (int i = 0; i < n; ++i) { - vals[i] = new Memory(MAX_STR_SZ + 1); - } - - int err; - if (useItemVersion) { - err = usePreCachedVersion - ? TOSDataBridge.getCLibrary().TOSDB_GetPreCachedItemNames(_name,vals,n,MAX_STR_SZ) - : TOSDataBridge.getCLibrary().TOSDB_GetItemNames(_name,vals,n,MAX_STR_SZ); - - } else { - err = usePreCachedVersion - ? TOSDataBridge.getCLibrary().TOSDB_GetPreCachedTopicNames(_name,vals,n,MAX_STR_SZ) - : TOSDataBridge.getCLibrary().TOSDB_GetTopicNames(_name,vals,n,MAX_STR_SZ); - } - if (err != 0) { - throw new CLibException("TOSDB_Get" + (usePreCachedVersion ? "PreCached" : "") - + (useItemVersion ? "Item" : "Topic") + "Names", err); - } - - Set names = new HashSet<>(); - for (int i = 0; i < n; ++i) { - String s = vals[i].getString(0); - @SuppressWarnings("unchecked") - T name = (T) (useItemVersion ? s : Topic.toEnum(s)); - names.add(name); - } - return names; - } - - public Set - _getItems() throws LibraryNotLoaded, CLibException { - return _getItemTopicNames(true, false); - } - - public Set - _getTopics() throws LibraryNotLoaded, CLibException { - return _getItemTopicNames(false, false); - } - - public Set - _getItemsPreCached() throws LibraryNotLoaded, CLibException { - return _getItemTopicNames(true, true); - } - - public Set - _getTopicsPreCached() throws LibraryNotLoaded, CLibException { - return _getItemTopicNames(false, true); - } - - private void - _goodItemTopicStateOrThrow() throws CLibException, LibraryNotLoaded { - if (!_topics.equals(_getTopics())) { - throw new IllegalStateException("_topics != _getTopics()"); - } - if (!_topicsPreCached.equals(_getTopicsPreCached())) { - throw new IllegalStateException("_topicsPreCached != _getTopicsPreCached()"); - } - if (!_items.equals(_getItems())) { - throw new IllegalStateException("_items != _getItems()"); - } - if (!_itemsPreCached.equals(_getItemsPreCached())) { - throw new IllegalStateException("_itemsPreCached != _getItemsPreCached()"); - } - } - - private void - _syncItemTopicState() throws CLibException, LibraryNotLoaded { - _topics = _getTopics(); - _items = _getItems(); - _topicsPreCached = _getTopicsPreCached(); - _itemsPreCached = _getItemsPreCached(); - } - @SuppressWarnings("unchecked") - private T - _getLong(String item, Topic topic, int indx, boolean withDateTime) - throws CLibException, LibraryNotLoaded { - DateTime dt = withDateTime ? new DateTime() : null; - long[] val = {0}; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetLongLong(_name, item, topic.val, indx, val, dt); - if (err != 0) { - throw new CLibException("TOSDB_GetLongLong", err); - } - return (T) (withDateTime ? new DateTimePair<>(val[0], dt) : val[0]); - } - - @SuppressWarnings("unchecked") - private T - _getDouble(String item, Topic topic, int indx, boolean withDateTime) - throws CLibException, LibraryNotLoaded { - DateTime dt = withDateTime ? new DateTime() : null; - double[] val = {0}; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetDouble(_name, item, topic.val, indx, val, dt); - if (err != 0) { - throw new CLibException("TOSDB_GetDouble", err); - } - return (T) (withDateTime ? new DateTimePair<>(val[0], dt) : val[0]); - } - - @SuppressWarnings("unchecked") - private T - _getString(String item, Topic topic, int indx, boolean withDateTime) - throws CLibException, LibraryNotLoaded { - DateTime dt = withDateTime ? new DateTime() : null; - byte[] val = new byte[STR_DATA_SZ + 1]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetString(_name, item, topic.val, indx, val, STR_DATA_SZ + 1, dt); - if (err != 0) { - throw new CLibException("TOSDB_GetString", err); - } - return (T) (withDateTime ? new DateTimePair<>(Native.toString(val), dt) - : Native.toString(val)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotLongs(String item, Topic topic, int end, int beg, boolean withDateTime, - int size) - throws LibraryNotLoaded, CLibException, DataIndexException { - DateTime[] dts = (withDateTime ? new DateTime[size] : null); - long[] vals = new long[size]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotLongLongs(_name, item, topic.val, vals, - size, dts, end, beg); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotLongs", err); - } - return (List)(withDateTime ? rawArraysToList(vals, dts, size) - : rawArraysToList(vals, size)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotDoubles(String item, Topic topic, int end, int beg, boolean withDateTime, - int size) - throws LibraryNotLoaded, CLibException, DataIndexException { - DateTime[] dts = (withDateTime ? new DateTime[size] : null); - double[] vals = new double[size]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotDoubles(_name, item, topic.val, vals, - size, dts, end, beg); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotDoubles", err); - } - return (List)(withDateTime ? rawArraysToList(vals, dts, size) - : rawArraysToList(vals, size)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotStrings(String item, Topic topic, int end, int beg, boolean withDateTime, - int size) - throws LibraryNotLoaded, CLibException, DataIndexException { - DateTime[] dts = (withDateTime ? new DateTime[size] : null); - Pointer[] vals = new Pointer[size]; - for (int i = 0; i < size; ++i) { - vals[i] = new Memory(STR_DATA_SZ + 1); - } - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotStrings(_name, item, topic.val, vals, - size, STR_DATA_SZ + 1, dts, end, beg); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotStrings", err); - } - return (List)(withDateTime ? rawArraysToList(vals, dts, size) - : rawArraysToList(vals, size)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotLongsFromMarker(String item, Topic topic, int beg, boolean withDateTime, - boolean throwIfDataLost, int safeSz) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - DateTime[] dts = (withDateTime ? new DateTime[safeSz] : null); - NativeLong[] getSz = {new NativeLong(0)}; - long[] vals = new long[safeSz]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotLongLongsFromMarker(_name, item, topic.val, - vals, safeSz, dts, beg, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotLongsFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotDoublesFromMarker(String item, Topic topic, int beg, boolean withDateTime, - boolean throwIfDataLost, int safeSz) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - DateTime[] dts = (withDateTime ? new DateTime[safeSz] : null); - NativeLong[] getSz = {new NativeLong(0)}; - double[] vals = new double[safeSz]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotDoublesFromMarker(_name, item, topic.val, - vals, safeSz, dts, beg, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotDoublesFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - @SuppressWarnings("unchecked") - private List - _getStreamSnapshotStringsFromMarker(String item, Topic topic, int beg, boolean withDateTime, - boolean throwIfDataLost, int safeSz) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - Pointer[] vals = new Pointer[safeSz]; - DateTime[] dts = (withDateTime ? new DateTime[safeSz] : null); - NativeLong[] getSz = {new NativeLong(0)}; - for (int i = 0; i < safeSz; ++i) { - vals[i] = new Memory(STR_DATA_SZ + 1); - } - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetStreamSnapshotStringsFromMarker(_name, item, topic.val, - vals, safeSz, STR_DATA_SZ + 1, dts, beg, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetStreamSnapshotStringsFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - @SuppressWarnings("unchecked") - private List - _getNLongsFromMarker(String item, Topic topic, int n, boolean withDateTime, - boolean throwIfDataLost) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - DateTime[] dts = (withDateTime ? new DateTime[n] : null); - NativeLong[] getSz = {new NativeLong(0)}; - long[] vals = new long[n]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetNLongLongsFromMarker(_name, item, topic.val, vals, n, dts, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetNLongsFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - @SuppressWarnings("unchecked") - private List - _getNDoublesFromMarker(String item, Topic topic, int n, boolean withDateTime, - boolean throwIfDataLost) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - DateTime[] dts = (withDateTime ? new DateTime[n] : null); - NativeLong[] getSz = {new NativeLong(0)}; - double[] vals = new double[n]; - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetNDoublesFromMarker(_name, item, topic.val, vals, n, dts, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetNDoublesFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - @SuppressWarnings("unchecked") - private List - _getNStringsFromMarker(String item, Topic topic, int n, boolean withDateTime, - boolean throwIfDataLost) - throws CLibException, LibraryNotLoaded, DirtyMarkerException { - Pointer[] vals = new Pointer[n]; - DateTime[] dts = (withDateTime ? new DateTime[n] : null); - NativeLong[] getSz = {new NativeLong(0)}; - for (int i = 0; i < n; ++i) { - vals[i] = new Memory(STR_DATA_SZ + 1); - } - int err = TOSDataBridge.getCLibrary() - .TOSDB_GetNStringsFromMarker(_name, item, topic.val, - vals, n, STR_DATA_SZ + 1, dts, getSz); - if (err != 0) { - throw new CLibException("TOSDB_GetNStringsFromMarker", err); - } - - int szGot = (int) _handleSzGotFromMarker(getSz[0], throwIfDataLost); - return (List)(withDateTime ? rawArraysToList(vals, dts, szGot) - : rawArraysToList(vals, szGot)); - } - - private long - _handleSzGotFromMarker(NativeLong l, boolean throwIfDataLost) throws DirtyMarkerException { - long szGot = l.longValue(); - if (szGot < 0) { - if (throwIfDataLost) { - throw new DirtyMarkerException(); - } else { - szGot *= -1; - } - } - return szGot; - } - - static private List - rawArraysToList(long[] p, int sz) { - final List o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(p[i]); - } - return o; - } - - static private List - rawArraysToList(double[] p, int sz) { - final List o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(p[i]); - } - return o; - } - - static private List - rawArraysToList(Pointer[] p, int sz) { - final List o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(p[i].getString(0)); - } - return o; - } - - static private List> - rawArraysToList(long[] p, DateTime[] dt, int sz) { - final List> o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(new DateTimePair<>(p[i], dt[i])); - } - return o; - } - - static private List> - rawArraysToList(double[] p, DateTime[] dt, int sz) { - final List> o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(new DateTimePair<>(p[i], dt[i])); - } - return o; - } - - static private List> - rawArraysToList(Pointer[] p, DateTime[] dt, int sz) { - final List> o = new ArrayList<>(sz); - for (int i = 0; i < sz; ++i) { - o.add(new DateTimePair<>(p[i].getString(0), dt[i])); - } - return o; - } - -} \ No newline at end of file diff --git a/java/src/io/github/jeog/tosdatabridge/DataBlockWithDateTime.java b/java/src/io/github/jeog/tosdatabridge/DataBlockWithDateTime.java deleted file mode 100644 index c4f8f04..0000000 --- a/java/src/io/github/jeog/tosdatabridge/DataBlockWithDateTime.java +++ /dev/null @@ -1,820 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import io.github.jeog.tosdatabridge.TOSDataBridge.*; -import io.github.jeog.tosdatabridge.DateTime.DateTimePair; - -import java.util.*; - -/** - * DataBlockWithDateTime is used to get real-time financial data from the TOS - * platform. It extends DataBlock, attaching a DateTime object to each data-point. - *

- * IMPORTANT: Before creating/using a DataBlockWithDateTime the - * underlying library must be loaded AND connected to the backend Service/Engine - * and the TOS platform. - *

    - *
  1. be sure you've installed the C mods (tosdb-setup.bat) - *
  2. start the Service/Engine (SC Start TOSDataBridge) - *
  3. start the TOS Platform - *
  4. call TOSDataBridge.init(...) - *
  5. create DataBlockWithDateTime(s) - *
- *

- * DataBlockWithDateTime works much the same as its python cousin - * (tosdb._win.TOSDB_DataBlock). It's instantiated with a size parameter to indicate - * how many historical data-points are saved. Once created 'topics' (data fields found in - * the Topic enum) and 'items' (symbols) are added to create a matrix of 'streams', each - * the size of the block. The block can be thought of as a 3D object with the following - * dimensions: - *

- *      (# of topics)   x   (# of items)   x   (block size) - *

- * As data flows into these streams the various 'get' methods are then used to - * 1) examine the underlying state of the block/streams, and 2) pull data from - * the block/streams in myriad ways. - * - * @author Jonathon Ogden - * @version 0.9 - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @see DataBlock - * @see TOSDataBridge - * @see Topic - * @see DateTime - * @see README - * @see README - JAVA API - * @see README - C API - * @see README - DETAILS - */ -public class DataBlockWithDateTime extends DataBlock { - - /** - * DataBlockWithDateTime Constructor. - * - * @param size maximum amount of historical data block can hold - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - */ - public DataBlockWithDateTime(int size) throws CLibException, LibraryNotLoaded { - super(size, true, DEF_TIMEOUT); // use the protected base constructor - } - - /** - * DataBlockWithDateTime Constructor. - * - * @param size maximum amount of historical data block can hold - * @param timeout timeout of internal IPC and DDE mechanisms (milliseconds) - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public DataBlockWithDateTime(int size, int timeout) throws LibraryNotLoaded, CLibException { - super(size, true, timeout); // use the protected base constructor - } - - /** - * @return if block is storing DateTime alongside primary data - */ - @Override - public boolean - isUsingDateTime() { - return true; - } - - /** - * Returns most recent data-point and DateTime of a stream, as DateTimePair<Long> - * (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getLongWithDateTime(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return getHelper().getMostRecent(item, topic, true, Long.class); - } - - /** - * Returns data-point and DateTime of a stream, as DateTimePair<Long> - * (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getLongWithDateTime(String item, Topic topic,int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return getHelper().get(item,topic, indx, true, Long.class); - } - - /** - * Returns most recent data-point and DateTime of a stream, as DateTimePair<Double> - * (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getDoubleWithDateTime(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return getHelper().getMostRecent(item, topic, true, Double.class); - } - - /** - * Returns data-point and DateTime of a stream, as DateTimePair<Double> - * (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getDoubleWithDateTime(String item, Topic topic,int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return getHelper().get(item,topic, indx, true, Double.class); - } - - /** - * Returns most recent data-point and DateTime of a stream, as DateTimePair<String> - * (or null if no data). - * - * @param item item string of stream - * @param topic topic enum of stream - * @return most recent data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getStringWithDateTime(String item, Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return getHelper().getMostRecent(item, topic, true, String.class); - } - - /** - * Returns data-point and DateTime of a stream, as DateTimePair<String> - * (or null if no data at that position/index). - * - * @param item item string of stream - * @param topic topic enum of stream - * @param indx index/position of data-point - * @return data-point and DateTime of stream (or null) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public DateTimePair - getStringWithDateTime(String item, Topic topic,int indx) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - return getHelper().get(item, topic, indx, true, String.class); - } - - /** - * Returns all data-points and DateTime(s) of a stream, - * as List<DateTimePair<Long>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotAll(item, topic, true, Long.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, from most recent, - * as List<DateTimePair<Long>>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsWithDateTime(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, 0, true, true, Long.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, - * as List<DateTimePair<Long>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsWithDateTime(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, beg, true, true, Long.class); - } - - /** - * Returns all data-points and DateTime(s) of a stream, - * as List<DateTimePair<Double>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotAll(item, topic, true, Double.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, from most recent, - * as List<DateTimePair<Double>>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesWithDateTime(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, 0, true, true, Double.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, - * as List<DateTimePair<Double>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesWithDateTime(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, beg, true, true, Double.class); - } - - /** - * Returns all data-points and DateTime(s) of a stream, - * as List<DateTimePair<String>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotAll(item, topic, true, String.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, from most recent, - * as List<DateTimePair<String>>. - * - * @param item item string of the stream - * @param topic topic enum of the stream - * @param end least recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsWithDateTime(String item, Topic topic, int end) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, 0, true, true, String.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) of a stream, - * as List<DateTimePair<String>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param end least recent index/position from which data are pulled - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsWithDateTime(String item, Topic topic, int end, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshot(item, topic, end, beg, true, true, String.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Long>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsFromMarkerWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecent(item, topic, true, Long.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Long>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsFromMarkerWithDateTime(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarker(item, topic, beg, true, true, Long.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Double>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesFromMarkerWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecent(item, topic, true, Double.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Double>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesFromMarkerWithDateTime(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarker(item, topic, beg, true, true, Double.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<String>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsFromMarkerWithDateTime(String item, Topic topic) - throws LibraryNotLoaded, CLibException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecent(item, topic, true, String.class); - } - - /** - * Returns multiple contiguous data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<String>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsFromMarkerWithDateTime(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarker(item, topic, beg, true, true, String.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<Long>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNLongsFromMarkerWithDateTime(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getNFromMarker(item, topic, n, true, true, Long.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<Double>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNDoublesFromMarkerWithDateTime(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getNFromMarker(item, topic, n, true, true, Double.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<String>>. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws DirtyMarkerException marker is 'dirty' (data lost behind it) - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNStringsFromMarkerWithDateTime(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, DirtyMarkerException, - InvalidItemOrTopic { - return getHelper().getNFromMarker(item, topic, n, true, true, String.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Long>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, - true, Long.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<DateTimePair<Long>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotLongsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, true, - Long.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<Double>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, - true, Double.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<DateTimePair<Double>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotDoublesFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, true, - Double.class); - } - - /** - * Returns all data-points and DateTime(s) up to atomic marker of a stream, - * as List<DateTimePair<String>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @return all data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic) - throws LibraryNotLoaded, CLibException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerToMostRecentIgnoreDirty(item, topic, - true, String.class); - } - - /** - * Returns multiple contiguous data-points up to atomic marker of a stream, - * as List<DateTimePair<String>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param beg most recent index/position from which data are pulled - * @return multiple contiguous data-points and DateTime(s) up to atomic marker of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getStreamSnapshotStringsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int beg) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getStreamSnapshotFromMarkerIgnoreDirty(item, topic, beg, true, - String.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<Long>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNLongsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getNFromMarkerIgnoreDirty(item, topic, n, true, Long.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<Double>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNDoublesFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getNFromMarkerIgnoreDirty(item, topic, n, true, Double.class); - } - - /** - * Returns 'n' contiguous data-points and DateTime(s) in front of atomic marker of a stream, - * as List<DateTimePair<String>>. Ignores 'dirty' marker/stream. - * - * @param item item string of stream - * @param topic topic enum of stream - * @param n number of data-points in front of marker - * @return multiple contiguous data-points and DateTime(s) in front of of stream - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - * @throws DataIndexException invalid index/position value - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public List> - getNStringsFromMarkerWithDateTimeIgnoreDirty(String item, Topic topic, int n) - throws LibraryNotLoaded, CLibException, DataIndexException, InvalidItemOrTopic { - return getHelper().getNFromMarkerIgnoreDirty(item, topic, n, true, String.class); - } - - /** - * Returns mapping of all item names to most recent item values and DateTime(s) - * for a particular topic. - * - * @param topic topic enum of streams - * @return mapping of item names to most recent item values and DateTime(s) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - * @see Topic - */ - public Map> - getItemFrameWithDateTime(Topic topic) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return getHelper().getFrame(topic, true,true); - } - - /** - * Returns mapping of all topic enums to most recent topic values and DateTime(s) - * for a particular item. - * - * @param item item string of streams - * @return mapping of topic enums to most recent topic values and DateTime(s) - * @throws CLibException error code returned by C lib - * @throws LibraryNotLoaded C lib has not been loaded - * @throws InvalidItemOrTopic invalid item or topic argument - */ - public Map> - getTopicFrameWithDateTime(String item) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - return getHelper().getFrame(item, false,true); - } - - /** - * Returns mapping of all item strings to mappings of all topic enums to most recent - * topic values and DateTime(s). This returns ALL the most recent data in the block. - * - * @return mapping of item strings to mappings of topic enums to most recent topic - * values and DateTime(s) - * @throws LibraryNotLoaded C lib has not been loaded - * @throws CLibException error code returned by C lib - */ - public Map>> - getTotalFrameWithDateTime() throws LibraryNotLoaded, CLibException { - Map>> frame = new HashMap<>(); - for(String item : getItems()){ - try { - Map> tf = getTopicFrameWithDateTime(item); - frame.put(item, tf); - } catch (InvalidItemOrTopic e) { - /* SHOULD NEVER GET HERE */ - throw new RuntimeException("InvalidItemOrTopic caught in " + - "getTotalFrameWithDateTime"); - } - } - return frame; - } -} \ No newline at end of file diff --git a/java/src/io/github/jeog/tosdatabridge/DateTime.java b/java/src/io/github/jeog/tosdatabridge/DateTime.java deleted file mode 100644 index 2d04f36..0000000 --- a/java/src/io/github/jeog/tosdatabridge/DateTime.java +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import com.sun.jna.NativeLong; -import com.sun.jna.Structure; - -import java.util.Arrays; -import java.util.List; - -/** - * A JNA Structure used to represent DateTime values. - * It contains two fields: - *

    - *
  • cTime - a java version of C's tm struct, defined as a JNA Structure. - *
  • microSeconds - *
- * @author Jonathon Ogden - * @version 0.9 - */ -public class DateTime extends Structure { - public _CTime cTime; - public NativeLong microSeconds; - { - cTime = new _CTime(); - microSeconds = new NativeLong(0); - } - - public static class _CTime extends Structure { - public int tmSec; - public int tmMin; - public int tmHour; - public int tmMday; - public int tmMon; - public int tmYear; - public int tmWday; - public int tmYday; - public int tmIsDst; - - @Override - protected List - getFieldOrder(){ - return Arrays.asList("tmSec", "tmMin", "tmHour", "tmMday", "tmMon", - "tmYear", "tmWday", "tmYday", "tmIsDst"); - } - } - - @Override - protected List - getFieldOrder(){ - return Arrays.asList("cTime", "microSeconds"); - } - - @Override - public String - toString(){ - return String.valueOf(cTime.tmMon + 1) + "/" + String.valueOf(cTime.tmMday) + "/" - + String.valueOf(1900 + cTime.tmYear) + " " + String.valueOf(cTime.tmHour) + ":" - + String.valueOf(cTime.tmMin) + ":" + String.valueOf(cTime.tmSec) + ":" - + String.valueOf(microSeconds); - } - - /** - * Pair of primary data and DateTime object. - * - * @param type of primary data - */ - public static class DateTimePair extends Pair { - public DateTimePair(T first, DateTime second) { - super(first, second); - } - } - - /** - * Triple(3-tuple) of primary data, DateTime object, and secondary data. - * - * @param type of secondary data - */ - public static class DateTimeTriple extends DateTimePair { - public final T3 third; - - public DateTimeTriple(T first, DateTime second, T3 third) { - super(first, second); - this.third = third; - } - - public DateTimeTriple(DateTimePair dateTimePair, T3 third){ - super(dateTimePair.first, dateTimePair.second); - this.third = third; - } - } - //TODO: implement add/subtract methods -} diff --git a/java/src/io/github/jeog/tosdatabridge/Pair.java b/java/src/io/github/jeog/tosdatabridge/Pair.java deleted file mode 100644 index abb30ec..0000000 --- a/java/src/io/github/jeog/tosdatabridge/Pair.java +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -/** - * Simple object that contains two objects of user-defined type. - * - * @author Jonathon Ogden - * @version 0.9 - */ -public class Pair{ - public final T1 first; - public final T2 second; - - public - Pair(T1 first, T2 second){ - this.first = first; - this.second = second; - } - - public - Pair(){ - this.first = null; - this.second = null; - } - - public String - toString(){ - return "<" + String.valueOf(first) + "," + String.valueOf(second) + ">"; - } -} \ No newline at end of file diff --git a/java/src/io/github/jeog/tosdatabridge/TOSDataBridge.java b/java/src/io/github/jeog/tosdatabridge/TOSDataBridge.java deleted file mode 100644 index 5b35fa0..0000000 --- a/java/src/io/github/jeog/tosdatabridge/TOSDataBridge.java +++ /dev/null @@ -1,283 +0,0 @@ -/* -Copyright (C) 2017 Jonathon Ogden - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -package io.github.jeog.tosdatabridge; - -import com.sun.jna.Native; - -import java.io.File; - -/** - * Core class containing constants, exceptions, a (package-private) getter for the - * JNA C lib interface, and high-level admin calls for the API. - *

- * IMPORTANT: Before using the admin calls the underlying C library - * must be loaded. - *

    - *
  1. be sure you've installed the C mods (tosdb-setup.bat) - *
  2. call init(...) - *
  3. use admin calls - *
- * @author Jonathon Ogden - * @version 0.9 - * @see Topic - * @see README - * @see README - JAVA API - * @see README - C API - * @see README - DETAILS - */ -public final class TOSDataBridge{ - /* hardcode for time being */ - public static final int CONN_NONE = 0; - public static final int CONN_ENGINE = 1; - public static final int CONN_ENGINE_TOS = 2; - - public static final int INTGR_BIT = 0x80; - public static final int QUAD_BIT = 0x40; - public static final int STRING_BIT = 0x20; - - public static final int TOPIC_IS_LONG = 1; - public static final int TOPIC_IS_DOUBLE = 2; - public static final int TOPIC_IS_STRING = 3; - - public static final int MAX_STR_SZ = 0xff; - public static final int STR_DATA_SZ = 40; - - public static final int DEF_TIMEOUT = 2000; - public static final int MARKER_MARGIN_OF_SAFETY = 100; - - public static class TOSDataBridgeException extends Exception{ - TOSDataBridgeException(String info){ - super(info); - } - } - - public static class LibraryNotLoaded extends TOSDataBridgeException{ - LibraryNotLoaded(){ - super("library not loaded"); - } - } - - public static class CLibException extends TOSDataBridgeException{ - CLibException(String callStr, int errorCode){ - super("CLib call [" + callStr + "] failed; " + CError.errorLookup(errorCode)); - } - } - - public static class DataBlockException extends TOSDataBridgeException{ - DataBlockException(String info){ - super(info); - } - } - - public static class InvalidItemOrTopic extends DataBlockException{ - InvalidItemOrTopic(String info) { - super(info); - } - } - - public static class DataIndexException extends DataBlockException{ - DataIndexException(String info) { - super(info); - } - } - - public static class DirtyMarkerException extends DataBlockException{ - DirtyMarkerException() { - super("data was lost behind the marker " - + "(set throwIfDataLost to false avoid this exception"); - } - } - - private static CLib library = null; - - /* package-private */ - static final CLib - getCLibrary() throws LibraryNotLoaded { - if(library == null) { - throw new LibraryNotLoaded(); - } - return library; - } - - /** - * Initialize C lib via JNA. - * - * @param path path of the the tos-databridge DLL. - * @return if library was loaded AND connected to engine - * @throws LibraryNotLoaded if library was not loaded - * @see README - JAVA API - */ - public static boolean - init(String path) throws LibraryNotLoaded { - if(library != null) { - return true; - } - try{ - File f = new File(path); - String n = f.getName(); - String d = f.getParentFile().getPath(); - int pos = n.lastIndexOf("."); - if(pos > 0) { - n = n.substring(0, pos); - } - System.out.println("directory: " + d + ", name: " + n); - System.setProperty("jna.library.path", d); - library = Native.loadLibrary(n, CLib.class); - if( !connect() ){ - System.err.println("TOSDataBridge connect() failed"); - return false; - } - }catch(Throwable t){ - System.err.println("TOSDataBridge.init(..) failed to load library: " + t.toString()); - t.printStackTrace(); - throw new LibraryNotLoaded(); - } - return true; - } - - /** - * If the library is loaded try to connect to the Engine and TOS platform. - * (TOSDataBridge.init(...) should do this automatically.) - * - * @return if library is connected to engine (not necessarily to TOS) - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static boolean - connect() throws LibraryNotLoaded { - return getCLibrary().TOSDB_Connect() == 0; - } - - /** - * @return if library is connected to engine and TOS platform - * - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static boolean - connected() throws LibraryNotLoaded { - return getCLibrary().TOSDB_IsConnectedToEngineAndTOS() != 0; - } - - /** - * @return state of the connection: - *
    - *
  • CONN_NONE: not connected to engine or TOS platform - *
  • CONN_ENGINE: connected to engine but not TOS platform - *
  • CONN_ENGINE_TOS: connected to engine and TOS platform - *
- * - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static int - connectionState() throws LibraryNotLoaded { - return getCLibrary().TOSDB_ConnectionState(); - } - - /** - * @return block limit imposed by the C library. - * - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static int - getBlockLimit() throws LibraryNotLoaded { - return getCLibrary().TOSDB_GetBlockLimit(); - } - - /** - * Set block limit to be imposed by the C library. - * - * @param limit new block limit - * @return new block limit - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static int - setBlockLimit(int limit) throws LibraryNotLoaded { - return getCLibrary().TOSDB_SetBlockLimit(limit); - } - - /** - * @return current block count(according to C lib, not necessarily JRE). - * - * @throws LibraryNotLoaded if library is not loaded - * @see README - JAVA API - * @see README - C API - */ - public static int - getBlockCount() throws LibraryNotLoaded { - return getCLibrary().TOSDB_GetBlockCount(); - } - - /** - * @return type bits (as int) of a particular topic - * - * @param topic topic enum to get type bits of - * @throws LibraryNotLoaded if library is not loaded - * @throws CLibException if the C lib call returns an error - * @see README - JAVA API - * @see README - C API - */ - public static int - getTypeBits(Topic topic) throws LibraryNotLoaded, CLibException { - byte[] bits = {0}; - int err = getCLibrary().TOSDB_GetTypeBits(topic.val, bits); - if(err != 0) { - throw new CLibException("TOSDB_GetTypeBits", err); - } - return bits[0] & 0xFF; - } - - /** - * @return type constant representing the type of a particular topic. - *
    - *
  • TOPIC_IS_LONG
    • - *
  • TOPIC_IS_DOUBLE
    • - *
  • TOPIC_IS_STRING
    • - *
- * - * @param topic topic enum to get type constant of - * @throws LibraryNotLoaded if library is not loaded - * @throws CLibException if the C lib call returns an error - * @see README - */ - public static int - getTopicType(Topic topic) throws CLibException, LibraryNotLoaded { - switch(getTypeBits(topic)){ - case INTGR_BIT | QUAD_BIT: - case INTGR_BIT: - return TOPIC_IS_LONG; - case QUAD_BIT: - case 0: - return TOPIC_IS_DOUBLE; - case STRING_BIT: - return TOPIC_IS_STRING; - default: - throw new RuntimeException("invalid type bits"); - } - } - - -} \ No newline at end of file diff --git a/java/src/io/github/jeog/tosdatabridge/Topic.java b/java/src/io/github/jeog/tosdatabridge/Topic.java deleted file mode 100644 index 9856126..0000000 --- a/java/src/io/github/jeog/tosdatabridge/Topic.java +++ /dev/null @@ -1,112 +0,0 @@ -package io.github.jeog.tosdatabridge; - -/** - * Enumerated object containing all the valid 'Topics' the Engine/TOS accepts. - * - * @author Jonathon Ogden - * @version 0.9 - */ -public enum Topic { - VOL_DIFF ("VOL_DIFF"), - AX ("AX"), - BACK_VOL ("BACK_VOL"), - LOW ("LOW"), - PE ("PE"), - HTB_ETB ("HTB_ETB"), - BIDX ("BIDX"), - CALL_VOLUME_INDEX ("CALL_VOLUME_INDEX"), - NET_CHANGE ("NET_CHANGE"), - EPS ("EPS"), - OPTION_VOLUME_INDEX ("OPTION_VOLUME_INDEX"), - STRIKE ("STRIKE"), - MARKET_CAP ("MARKET_CAP"), - MARK_PERCENT_CHANGE ("MARK_PERCENT_CHANGE"), - FRONT_VOL ("FRONT_VOL"), - YIELD ("YIELD"), - STRENGTH_METER ("STRENGTH_METER"), - ASKX ("ASKX"), - DIV_FREQ ("DIV_FREQ"), - LX ("LX"), - DESCRIPTION ("DESCRIPTION"), - EX_DIV_DATE ("EX_DIV_DATE"), - SHARES ("SHARES"), - BETA ("BETA"), - EXTRINSIC ("EXTRINSIC"), - SYMBOL ("SYMBOL"), - OPEN ("OPEN"), - MARK_CHANGE ("MARK_CHANGE"), - BID_SIZE ("BID_SIZE"), - VOLUME ("VOLUME"), - WEIGHTED_BACK_VOL ("WEIGHTED_BACK_VOL"), - LAST ("LAST"), - LAST_SIZE ("LAST_SIZE"), - BID ("BID"), - MRKT_MKR_MOVE ("MRKT_MKR_MOVE"), - PUT_CALL_RATIO ("PUT_CALL_RATIO"), - RHO ("RHO"), - PERCENT_CHANGE ("PERCENT_CHANGE"), - HIGH52 ("52HIGH"), - MT_NEWS ("MT_NEWS"), - VOL_INDEX ("VOL_INDEX"), - GAMMA ("GAMMA"), - LOW52 ("52LOW"), - PUT_VOLUME_INDEX ("PUT_VOLUME_INDEX"), - DIV ("DIV"), - ASK ("ASK"), - DELTA ("DELTA"), - NULL_TOPIC ("NULL_TOPIC"), // <-- DONT USE - INTRINSIC ("INTRINSIC"), - BACK_EX_MOVE ("BACK_EX_MOVE"), - VEGA ("VEGA"), - BX ("BX"), - EXPIRATION ("EXPIRATION"), - MARK ("MARK"), - LASTX ("LASTX"), - THETA ("THETA"), - FRONT_EX_MOVE ("FRONT_EX_MOVE"), - BA_SIZE ("BA_SIZE"), - FX_PAIR ("FX_PAIR"), - ASK_SIZE ("ASK_SIZE"), - IMPL_VOL ("IMPL_VOL"), - EXCHANGE ("EXCHANGE"), - EX_MOVE_DIFF ("EX_MOVE_DIFF"), - HIGH ("HIGH"), - OPEN_INT ("OPEN_INT"), - CUSTOM1 ("CUSTOM1"), - CUSTOM2 ("CUSTOM2"), - CUSTOM3 ("CUSTOM3"), - CUSTOM4 ("CUSTOM4"), - CUSTOM5 ("CUSTOM5"), - CUSTOM6 ("CUSTOM6"), - CUSTOM7 ("CUSTOM7"), - CUSTOM8 ("CUSTOM8"), - CUSTOM9 ("CUSTOM9"), - CUSTOM10 ("CUSTOM10"), - CUSTOM11 ("CUSTOM11"), - CUSTOM12 ("CUSTOM12"), - CUSTOM13 ("CUSTOM13"), - CUSTOM14 ("CUSTOM14"), - CUSTOM15 ("CUSTOM15"), - CUSTOM16 ("CUSTOM16"), - CUSTOM17 ("CUSTOM17"), - CUSTOM18 ("CUSTOM18"), - CUSTOM19 ("CUSTOM19"); - - public final String val; - Topic(String t){ - this.val = t; - } - - /* package-private */ - static Topic - toEnum(String val){ - /* 2 cases where .name() != .val */ - if(val.equals("52HIGH")) { - return HIGH52; - }else if(val.equals("52LOW")) { - return LOW52; - }else { - return valueOf(val); - } - } -} diff --git a/java/tosdatabridge.jar b/java/tosdatabridge.jar deleted file mode 100644 index 27ab047..0000000 Binary files a/java/tosdatabridge.jar and /dev/null differ diff --git a/javascripts/scale.fix.js b/javascripts/scale.fix.js new file mode 100644 index 0000000..87a40ca --- /dev/null +++ b/javascripts/scale.fix.js @@ -0,0 +1,17 @@ +var metas = document.getElementsByTagName('meta'); +var i; +if (navigator.userAgent.match(/iPhone/i)) { + for (i=0; i SC start TOSDataBridge\r\n ```\r\n (see #9 in the 'Installation Details' section for more info on using the service.)\r\n3. Log on to your TOS platform\r\n\r\n##### For C/C++:\r\n- Include tos_databridge.h header in your code \r\n- Use the library calls detailed in the **C/C++ Interface...** sections below\r\n- Link with *tos-databridge-0.4-[x86|x64].dll*\r\n- Build\r\n- Run\r\n\r\n##### For Python:\r\n- Open a python shell/interpreter or create a script\r\n- import tosdb\r\n\r\n\r\n### License & Terms\r\n- - -\r\n*TOSDB is released under the GNU General Public License(GPL); a copy (LICENSE.txt) should be included. If not, see http://www.gnu.org/licenses. The author reserves the right to issue current and/or future versions of TOSDB under other licensing agreements. Any party that wishes to use TOSDB, in whole or in part, in any way not explicitly stipulated by the GPL, is thereby required to obtain a separate license from the author. The author reserves all other rights.*\r\n\r\n*This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.*\r\n\r\n*By choosing to use the software - under the broadest interpretation of the term \"use\" \\- you absolve the author of ANY and ALL responsibility, for ANY and ALL damages incurred; including, but not limited to, damages arising from the accuracy and/or timeliness of data the software does, or does not, provide.*\r\n\r\n*Furthermore, TOSDB is in no way related to TDAmeritrade or affiliated parties; users of TOSDB must abide by their terms of service and are solely responsible for any violations therein.*\r\n\r\n- - -\r\n*Copyright (C) 2014 Jonathon Ogden*", + "google": "", + "note": "Don't delete this file! It's used internally to help with page regeneration." +} \ No newline at end of file diff --git a/python/setup.py b/python/setup.py deleted file mode 100644 index 47c8088..0000000 --- a/python/setup.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -import sys as _sys - -if _sys.version_info.major < 3: - _sys.stderr.write("fatal: tosdb is built for python3!\n") - exit(1) - -from distutils.core import setup as _setup -from distutils.command.build import build as _Build -from distutils.command.install import install as _Install -from distutils.command.clean import clean as _Clean -from re import match as _match, search as _search -from time import asctime as _asctime -from os.path import join as _path_join, dirname as _dirname, realpath as _realpath -from os import system as _system, getcwd as _getcwd, remove as _remove -from shutil import rmtree as _rmtree - - -NAME = 'tosdb' -VERSION = '0.9' -DESCRIPTION = "Python Front-End / Wrapper for TOSDataBridge" -AUTHOR = "Jonathon Ogden" -AUTHOR_EMAIL = "jeog.dev@gmail.com" -PACKAGES = ['tosdb','tosdb/cli_scripts','tosdb/intervalize', 'tosdb/streaming'] - - -_AUTO_EXT = '_tosdb' -# everything should be relative to the python/setup.py -_OUR_PATH = _dirname(_realpath(__file__)) -_HEADER_NAME = 'tos_databridge.h' -_HEADER_PATH = _path_join(_OUR_PATH, '..', 'include', _HEADER_NAME) -_OUTPUT_PATH = _path_join(_OUR_PATH, NAME, _AUTO_EXT + '.py') - -if _OUR_PATH != _getcwd(): - _sys.stderr.write("fatal: setup.py must be run from its own directory(python/)\n") - exit(1) - - -#string that should bookmark the topics in Topic_Enum_Wrapper::TOPICS -_MAGIC_TOPIC_STR = 'ksxaw9834hr84hf;esij?><' - -#regex for finding our header #define consts -#TODO: adjust so we can pull non-ints -_REGEX_HEADER_CONST = "#define[\s]+([\w]+)[\s]+.*?(-?[\d][\w]*)" - -#adjust for topics we had to permute to form valid enum vars -TOPIC_VAL_REPLACE = {'HIGH52':'52HIGH','LOW52':'52LOW'} - -#exclude certain header consts -HEADER_PREFIX_EXCLUDES = ['TOSDB_SIG_', 'TOSDB_COMM_', 'TOSDB_PROBE_', - 'LOG_BACKEND_MUTEX_NAME', 'LOCAL_LOG_PATH', 'TOSDB_SHEM_BUF_SZ'] - - -class TOSDB_SetupError(Exception): - def __init__(self,*msgs): - super().__init__(*msgs) - - -def _on_exclude_list(h): - for exc in HEADER_PREFIX_EXCLUDES: - if _match(exc,h): - return True - return False - - -def _pull_consts_from_header(verbose=True): - consts = [] - lineno = 0 - with open(_HEADER_PATH,'r') as hfile: - hfile.seek(0) - for hline in hfile: - lineno += 1 - try: - groups = _match(_REGEX_HEADER_CONST,hline).groups() - except AttributeError: - continue - try: - g0, g1 = groups - #custom excluded consts - if _on_exclude_list(g0): - continue - val = str(hex(int(g1,16)) if 'x' in g1 else int(g1)) - consts.append((g0,val)) - if verbose: - print(' ',_HEADER_NAME + ':' + str(lineno)+ ': '+ g0, val) - except ValueError: - raise TOSDB_SetupError("invalid header const value", str(g0)) - except Exception as e: - raise TOSDB_SetupError("couldn't extract const from regex match", e.args) - return consts - - -def _pull_topics_from_header(verbose=True): - read_flag = False - topics = [] - lineno = 0 - with open(_HEADER_PATH,'r') as hfile: - for hline in hfile: - lineno += 1 - if read_flag: - if _MAGIC_TOPIC_STR in hline: - read_flag = False - if verbose: - print(' ', _HEADER_NAME + ':' + str(lineno+1) + ': topic enum END') - break - try: - t = hline.split('=')[0].strip() - except Exception as e: - raise TOSDB_SetupError("failed to parse topic enum line", e.args) - if not _search('[\W]',t): - topics.append(t) - else: - if _MAGIC_TOPIC_STR in hline: - read_flag = True - if verbose: - print(' ', _HEADER_NAME + ':' + str(lineno-1) + ': topic enum BEGIN') - return topics - - -def _build_error_lookup(consts): - errcs = {} - for (k,v) in consts: - if _match("TOSDB_ERROR_.+", k): - errcs[v] = k.lstrip('TOSDB_') - return errcs - - -# build a tosdb/_tosdb.py file from the header extracted vals -def _create__tosdb(consts, topics): - error_consts = _build_error_lookup(consts) - topic_dict = dict(zip(topics,topics)) - for key in topic_dict: - if key in TOPIC_VAL_REPLACE: # don't just .update, check all are valid - topic_dict[key] = TOPIC_VAL_REPLACE[key] - with open(_OUTPUT_PATH,'w') as pfile: - pfile.write('# AUTO-GENERATED BY tosdb/setup.py\n') - pfile.write('# DO NOT EDIT!\n\n') - pfile.write('_BUILD_DATETIME = "' + _asctime() + '"\n') - for k,v in consts: - pfile.write(k.replace('TOSDB_','',1) + ' = ' + v + '\n') - pfile.write('\n') - pfile.write('from tosdb.meta_enum import MetaEnum\n') - pfile.write('class TOPICS(metaclass=MetaEnum):\n') - pfile.write(' fields = ' + str(topic_dict) + '\n') - pfile.write('\n') - pfile.write('ERROR_LOOKUP = { \n') - for k,v in error_consts.items(): - pfile.write( str(int(k)) + ": '" + v + "', \n" ) - pfile.write('} \n') - - - -def _init_build(): - print("pulling constants from " + _HEADER_PATH) - consts = _pull_consts_from_header() - print("pulling topic enum from " + _HEADER_PATH) - topics = _pull_topics_from_header() - print('auto-generating ' + _OUTPUT_PATH) - _create__tosdb(consts, topics) - print(' checking ' + _OUTPUT_PATH) - try: - exec("from " + NAME + " import " + _AUTO_EXT) - except ImportError as ie: - print(' fatal: auto-generated ' + _OUTPUT_PATH + ' could not be imported !') - print(' fatal: ' + ie.args[0]) - exit(1) - print(' success!') - - - -def _check_depends(): #TODO - print("\n*** NOTE: if you plan on using authenticated virtualization " + - "(by passing a password to the appropriate 'virtual' calls) " + - "you need to install the pycrypto package ***") - - -class InstallCommand(_Install): - description = "install tosdb" - def run(self): - super().run() - _check_depends() - - -class BuildCommand(_Build): - description = "build tosdb" - def run(self): - _init_build() - super().run() - - -class CleanCommand(_Clean): - description = "clean tosdb" - def run(self): - try: - print("removing tosdb/_tosdb.py ...") - _remove(_OUTPUT_PATH) - except: - pass - try: - print("removing ./build ...") - _rmtree( _path_join(_OUR_PATH,'build') ) - except: - pass - super().run() - - -_setup( name=NAME, version=VERSION, description=DESCRIPTION, - author=AUTHOR, author_email=AUTHOR_EMAIL, packages=PACKAGES, - cmdclass={'install':InstallCommand, 'build':BuildCommand, 'clean':CleanCommand} ) - - diff --git a/python/tosdb/__init__.py b/python/tosdb/__init__.py deleted file mode 100644 index ae989f4..0000000 --- a/python/tosdb/__init__.py +++ /dev/null @@ -1,1138 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -"""tosdb/: A front-end / wrapper for the TOSDataBridge C library - -Please refer to README.md for an explanation of the C/C++ library, tutorial.md -for a walk-through of the basics, and virtualization_tutorial.md for example -usage on non-Windows systems. - -class TOSDB_DataBlock(WINDOWS ONLY): - very similar to the 'block' approach of the underlying C/C++ library except - the interface is explicitly object-oriented. It abstracts away the underlying - type complexity, raising TOSDB_CLibError on internal library errors. - -class TOSDB_ThreadSafeDataBlock(WINDOWS ONLY): - thread-safe version of TOSDB_DataBlock. - -class VTOSDB_DataBlock: - same interface as TOSDB_DataBlock except it utilizes a thin virtualization - layer over TCP, passing serialized method calls to a windows machine running - the core implemenataion. - -class VTOSDB_ThreadSafeDataBlock: - thread-safe version of VTOSDB_DataBlock. - -class _TOSDB_DataBlock: - abstract base class of TOSDB_DataBlock and VTOSDB_DataBlock - -Init: - context manager that handles initialization and clean-up - -VInit: - version of Init for the virtual layer - -admin_init(): - initializes the vitual library calls (e.g vinit(), vconnect()) - -init()/vinit(): - tell the windows implementation to initialize the underlying C/C++ library - and attempt to connect - -connect()/vconnect(): - tell the windows implementation to connect to the underlying C/C++ library - if init fails to do this automatically - -connected()/vconnected(): - is the windows implementation connected to the engine AND TOS platform? - -connection_state()/vconnection_state() - what connection state is the windows implementation in? - -clean_up()/vclean_up() (IMPORTANT): - de-allocates shared resources of the underlying C lib and engine. - Resources are cleaned-up automatically on exit but there's no guarantee. - HIGHLY RECOMMENDED YOU CALL THIS BEFORE EXITING. - -getblockcount()/vgetblockcount(): - number of instantiated blocks in the C lib - -getblocklimit()/vgetblocklimit(): - get max number of blocks allowed by C lib - -setblocklimit()/vsetblocklimit(): - set max number of blocks allowed by C lib - -* * * - -Ways to initialize via virtual layer: - --------------------------------------------------------------------------------- - [ local(windows) side ] | [ remote side ] --------------------------------------------------------------------------------- - Local with no virtualization support: - | - 1) init() | - 2) connect()** | - 3) use standard calls/objects | - | --------------------------------------------------------------------------------- - Local (interactive) init with virtualization support: - | - 1) init() | - 2) connect()** | - 3) use standard calls/objects... | - 4) enable_virtualization((addr,port)) | - | 5a) use VTOSDB_DataBlock - 5b) use standard calls/objects... | - | 6) admin_init() - | 7) use virtual admin calls --------------------------------------------------------------------------------- - Remote (interactive) init with virtualization support: - | - 1) enable_virtualization((addr,port)) | - | 2) admin_init() - | 3) vinit() - | 4) vconnect()** - | 5a) use virtual calls/objs - 5b) use standard calls/objects... | --------------------------------------------------------------------------------- - Local (daemon/server) Remote (interactive) init with virtualization support: - | - 1) C:\...\python> python tosdb --virtual-server \ | - "ADDR PORT" --root PATH-TO-DLL | - | 2) use VTOSDB_DataBlock - | 3) admin_init() - | 4) use virtual admin calls - ---------------------------------------------------- - | or Remote (client) - | - | 2) :/.../python/$ python tosdb --virtual-client \\ - | "ADDR PORT" - | 3) use virtual calls/objs --------------------------------------------------------------------------------- - - **only needed if init/vinit() fails to call this for us -""" - -from ._common import * -from ._common import _DateTimeStamp, _TOSDB_DataBlock, _type_switch, \ - _recvall_tcp, _recv_tcp, _send_tcp -from ._auth import * -from .doxtend import doxtend as _doxtend - -from collections import namedtuple as _namedtuple -from threading import Thread as _Thread, Lock as _Lock -from functools import partial as _partial -from platform import system as _system -from sys import stderr as _stderr, stdout as _stdout -from re import sub as _sub -from atexit import register as _on_exit -from contextlib import contextmanager as _contextmanager -from time import strftime as _strftime -from inspect import getmembers as _getmembers, isfunction as _isfunction - -import struct as _struct -import socket as _socket -import pickle as _pickle - -_SYS_IS_WIN = _system() in ["Windows","windows","WINDOWS"] - -if _SYS_IS_WIN: - from ._win import * # import the core implementation - -_virtual_hub = None -_virtual_hub_addr = None -_virtual_admin_sock = None # <- what happens when we exit the client side ? - -_vCREATE = 'CREATE' -_vCALL = 'CALL' -_vACK = 'ACK' -_vFAILURE = 'FAILURE' -_vEXCEPTION = 'EXCEPTION' -_vSUCCESS = 'SUCCESS' -_vSUCCESS_NT = 'SUCCESS_NT' -_vCONN_BLOCK = 'CONN_BLOCK' -_vCONN_ADMIN = 'CONN_ADMIN' -_vREQUIRE_AUTH = 'REQUIRE_AUTH' -_vREQUIRE_AUTH_NO = 'REQUIRE_AUTH_NO' - -_vALLOWED_ADMIN = ('init', 'connect', 'connected', 'connection_state', 'clean_up', - 'get_block_limit', 'set_block_limit', 'get_block_count', - 'type_bits', 'type_string') - -# just the name, can't get bound method with ismethod pred (why?) -_vALLOWED_METHS = tuple(m[0] for m in _getmembers(_TOSDB_DataBlock, predicate=_isfunction) \ - if m[0][0] != '_') + ('__init__','__str__') - -## !! _vDELIM MUST NOT HAVE THE SAME VALUE AS _vEEXOR !! ## -_vDELIM = b'\x7E' -_vESC = b'\x7D' -_vDEXOR = chr(ord(_vDELIM) ^ ord(_vESC)).encode() -_vEEXOR = chr(ord(_vESC) ^ ord(_vESC)).encode() # 0 - - -# NOTE: for time being preface virtual calls and objects with 'v' so -# we can load both on the windows side for easier debugging, - -def vinit(dllpath=None, root="C:\\"): - """ Initialize the underlying tos-databridge DLL on host, try to connect. - - Returns True if library was successfully loaded, not necessarily that - it was also able to connect. Details are sent to stdout on Windows side. - - vinit(dllpath=None, root="C:\\") - - dllpath :: str :: exact path (on host) of the DLL -or- - root :: str :: directory (on host) to start walking/searching to find the DLL - - returns -> bool - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - if dllpath is None and root == "C:\\": - if abort_init_after_warn(): - return False - return _admin_call('init', dllpath, root, True) - - -def vconnect(): - """ Attempts to connect to the underlying engine and TOS Platform on host. - - vconnect() - - returns -> bool - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('connect') - - -def vconnected(): - """ Returns True if there is an active connection to the engine AND TOS platform on host. - - vconnected() - - returns -> bool - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('connected') - - -def vconnection_state(): - """ Returns the connection state between the client lib, engine and TOS platform on host. - - vconnection_state() - - if NOT connected to the engine: returns -> CONN_NONE - elif connected to the engine BUT NOT the TOS platform: returns -> CONN_ENGINE - elif connected to the engine AND the TOS platform: returns -> CONN_ENGINE_TOS - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('connection_state') - - -def vclean_up(): - """ Clean up shared resources on host. - - *** CALL BEFORE EXITING INTERPRETER *** - - Attempts to close the underlying block and disconnect gracefully so - streams aren't orphaned/leaked. - - clean_up() - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - _admin_call('clean_up') - - -@_contextmanager -def VInit(address, dllpath=None, root="C:\\", password=None, timeout=DEF_TIMEOUT): - """ Manage a virtual 'session' with a host system's C lib, engine, and TOS platform. - - The context manager handles initialization of the virtual admin interface - and the host C Lib, then tries to connect the latter with the engine and - TOS platform on the host. On exiting the context block it automatically - cleans up the host system and closes down the connection. - - with VInit(...): - # (automatically initializes, connects etc.) - # use virtual admin interface, create data blocks etc. - # (automatically cleans up, closes etc.) - - VInit(address, dllpath=None, root="C:\\", password=None, timeout=DEF_TIMEOUT) - - address :: (str,int) :: (address of the host system, port) - dllpath :: str :: exact path of the DLL -or- - root :: str :: directory to start walking/searching to find the DLL - password :: str :: password for authentication(None for no authentication) - timeout :: int :: timeout used by the underlying socket - - throws TOSDB_VirtualizationError TOSDB_InitError - """ - try: - admin_init(address, password, timeout) - if not vinit(dllpath, root): - raise TOSDB_InitError("failed to initilize library (virtual)") - if not vconnected(): - if not vconnect(): # try again - raise TOSDB_InitError("failed to connect to library (virtual)") - yield - finally: - vclean_up() - admin_close() - - -def vget_block_limit(): - """ Returns the block limit imposed by the C Lib on host. - - vget_block_limit() - - returns -> int - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('get_block_limit') - - -def vset_block_limit(new_limit): - """ Changes the block limit imposed by the C Lib on host. - - vset_block_limit(new_limit) - - new_limit :: int :: new block limit - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - _admin_call('set_block_limit', new_limit) - - -def vget_block_count(): - """ Returns the current count of instantiated blocks (according to C Lib) on host. - - vget_block_count() - - returns -> int - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('get_block_count') - - -def vtype_bits(topic): - """ Returns the type bits for a particular topic from host. - - These type bits can be logical &'d with type bit contstants (ex. QUAD_BIT) - to determine the underlying type of the topic. - - vtype_bits(topic) - - topic :: str :: topic string ('LAST','ASK', etc) - - returns -> int - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('type_bits', topic) - - -def vtype_string(topic): - """ Returns a platform-dependent string of the type of a particular topic from host. - - vtype_string(topic) - - topic :: str :: topic string ('LAST','ASK', etc) - - returns -> str - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - return _admin_call('type_string', topic) - - -# map function objects to names -_vALLOWED_ADMIN_CALLS = dict(zip(_vALLOWED_ADMIN, \ - (eval(('' if _SYS_IS_WIN else 'v') + c) for c in _vALLOWED_ADMIN))) - - -def admin_close(): # do we need to signal the server ? - """ Close admin connection created by admin_init """ - global _virtual_hub_addr, _virtual_admin_sock - if _virtual_admin_sock: - _virtual_admin_sock.close() - _virtual_hub_addr = '' - _virtual_admin_sock = None - - -def admin_init(address, password=None, timeout=DEF_TIMEOUT): - """ Initialize virtual admin calls. - - This establishes a stateful connection with the host system, allowing - the admin calls(e.g vinit(), vconnect()) to communicate with their - siblings(e.g init(), connect()) on the host side. - - admin_init(address, password=None, timeout=DEF_TIMEOUT) - - address :: (str,int) :: (host/address of the windows implementation, port) - password :: str :: password for authentication(None for no authentication) - timeout :: int :: timeout used by the underlying socket - - returns -> bool - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - global _virtual_hub_addr, _virtual_admin_sock - if _virtual_admin_sock: - raise TOSDB_VirtualizationError("virtual admin socket already exists") - if password is not None: - check_password(password) - _virtual_hub_addr = _check_and_resolve_address(address) - _virtual_admin_sock = _socket.socket() - _virtual_admin_sock.settimeout(timeout / 1000) - try: - _virtual_admin_sock.connect(_virtual_hub_addr) - _handle_req_from_server(_virtual_admin_sock,password) - _vcall(_pack_msg(_vCONN_ADMIN), _virtual_admin_sock, _virtual_hub_addr) - except: - admin_close() - raise - return True - - -def _admin_call(method, *arg_buffer): - if not _virtual_admin_sock: - raise TOSDB_VirtualizationError("no virtual admin socket, call admin_init") - if method not in _vALLOWED_ADMIN: - raise TOSDB_VirtualizationError("method '%s' not in _vALLOWED_ADMIN" % method) - a = (method,) - if arg_buffer: - a += (_pickle.dumps(arg_buffer),) - try: - ret_b = _vcall(_pack_msg(*a), _virtual_admin_sock, _virtual_hub_addr) - if ret_b[1]: - return _pickle.loads(ret_b[1]) - except TOSDB_VirtualizationError: - admin_close() - raise - - - - -def _handle_req_from_server(sock,password): - ret = _recv_tcp(sock) - if ret is None: - raise TOSDB_VirtualizationError("no response from server") - _send_tcp(sock, _vACK.encode()) # ack - r = ret.decode() - if r == _vREQUIRE_AUTH: - if password is not None: - try_import_pycrypto() - good_auth = handle_auth_cli(sock,password) - if not good_auth: - raise TOSDB_VirtualizationError("authentication failed") - else: - raise TOSDB_VirtualizationError("server requires authentication") - elif r != _vREQUIRE_AUTH_NO: - raise TOSDB_VirtualizationError("invalid (auth) request from server") - - -class VTOSDB_DataBlock(_TOSDB_DataBlock): - """ The main object for storing TOS data (VIRTUAL) (NOT THREAD SAFE) - - __init__(self, address, password=None, size=1000, date_time=False, - timeout=DEF_TIMEOUT) - - address :: (str,int) :: (host/address of the windows implementation, port) - password :: str :: password for authentication(None for no authentication) - size :: int :: how much historical data can be inserted - date_time :: bool :: should block include date-time with each data-point? - timeout :: int :: how long to wait for responses from engine, - TOS-DDE server, internal IPC/Concurrency mechanisms, - and network communication (milliseconds) - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - def __init__(self, address, password=None, size=1000, date_time=False, - timeout=DEF_TIMEOUT): - self._valid = False - self._hub_addr = _check_and_resolve_address(address) - self._my_sock = _socket.socket() - self._my_sock.settimeout(timeout / 1000) - if password is not None: - check_password(password) - self._my_sock.connect(self._hub_addr) - try: - _handle_req_from_server(self._my_sock,password) - except: - self._my_sock.close() - raise - self._call_LOCK = _Lock() - _vcall(_pack_msg(_vCONN_BLOCK), self._my_sock, self._hub_addr) - # in case __del__ is called during socket op - self._call(_vCREATE, '__init__', size, date_time, timeout) - self._valid = True - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def close(self): - if self._valid: - self._call(_vCALL, 'close') - self._valid = False - if self._my_sock: - self._my_sock.close() - - - def __del__(self): - try: - self.close() - except: - pass - - - def __str__(self): - s = self._call(_vCALL, '__str__') - return s if s else '' - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def is_using_datetime(self): - return self._call(_vCALL, 'is_using_datetime') - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def info(self): - return self._call(_vCALL, 'info') - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def get_block_size(self): - return self._call(_vCALL, 'get_block_size') - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def set_block_size(self, sz): - self._call(_vCALL, 'set_block_size', sz) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_occupancy(self, item, topic): - return self._call(_vCALL, 'stream_occupancy', item, topic) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def items(self, str_max = MAX_STR_SZ): - return self._call(_vCALL, 'items', str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topics(self, str_max = MAX_STR_SZ): - return self._call(_vCALL, 'topics', str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def items_precached(self, str_max = MAX_STR_SZ): - return self._call(_vCALL, 'items_precached', str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topics_precached(self, str_max = MAX_STR_SZ): - return self._call(_vCALL, 'topics_precached', str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_items(self, *items): - self._call(_vCALL, 'add_items', *items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_topics(self, *topics): - self._call(_vCALL, 'add_topics', *topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_items(self, *items): - self._call(_vCALL, 'remove_items', *items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_topics(self, *topics): - self._call(_vCALL, 'remove_topics', *topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def get(self, item, topic, date_time=False, indx = 0, check_indx=True, - data_str_max=STR_DATA_SZ): - - return self._call(_vCALL, 'get', item, topic, date_time, indx, - check_indx, data_str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_snapshot(self, item, topic, date_time=False, end=-1, beg=0, - smart_size=True, data_str_max=STR_DATA_SZ): - - return self._call(_vCALL, 'stream_snapshot', item, topic, date_time, - end, beg, smart_size, data_str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, - margin_of_safety=100, throw_if_data_lost=True, - data_str_max=STR_DATA_SZ): - - return self._call(_vCALL, 'stream_snapshot_from_marker', item, topic, - date_time, beg, margin_of_safety, throw_if_data_lost, - data_str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def n_from_marker(self, item, topic, date_time=False, n=1, - throw_if_data_lost=True, data_str_max=STR_DATA_SZ): - - return self._call(_vCALL, 'n_from_marker', item, topic, date_time, n, - throw_if_data_lost, data_str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def item_frame(self, topic, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ): - - return self._call(_vCALL, 'item_frame', topic, date_time, labels, - data_str_max, label_str_max) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topic_frame(self, item, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ): - - return self._call(_vCALL, 'topic_frame', item, date_time, labels, - data_str_max, label_str_max) - - ## - ## !! CREATE A WAY TO PICKLE AN ITERABLE OF DIFFERENT NAMEDTUPLES !! - ## - ## def total_frame(self, date_time=False, labels=True, - ## data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ): - ## """ Return a matrix of the most recent values: - ## - ## date_time: (True/False) attempt to retrieve a TOSDB_DateTime object - ## labels: (True/False) pull the item and topic labels with the values - ## data_str_max: the maximum length of string data returned - ## label_str_max: the maximum length of label strings returned - ## - ## if labels and date_time are True: returns-> dict of namedtuple of 2tuple - ## if labels is True: returns -> dict of namedtuple - ## if date_time is True: returns -> list of 2tuple - ## else returns-> list - ## """ - ## return self._call(_vCALL, 'total_frame', date_time, labels, - ## data_str_max, label_str_max) - - def _call(self, virt_type, method='', *arg_buffer): - if method not in _vALLOWED_METHS: - raise TOSDB_VirtualizationError("method '%s' not in _vALLOWED_METHS" % method) - if virt_type == _vCREATE: - req_b = _pack_msg(_vCREATE, _pickle.dumps(arg_buffer)) - elif virt_type == _vCALL: - a = (_vCALL, method) + ((_pickle.dumps(arg_buffer),) if arg_buffer else ()) - req_b = _pack_msg(*a) - else: - raise TOSDB_VirtualizationError("invalid virt_type") - with self._call_LOCK: - ret_b = _vcall(req_b, self._my_sock, self._hub_addr) - if virt_type == _vCREATE: - return True - elif virt_type == _vCALL and ret_b[1]: - if ret_b[0].decode() == _vSUCCESS_NT: - return _loadnamedtuple(ret_b[1]) - else: - return _pickle.loads(ret_b[1]) - - -@make_block_thread_safe('__str__') -class VTOSDB_ThreadSafeDataBlock(VTOSDB_DataBlock): - """ The main object for storing TOS data (VIRTUAL) (THREAD SAFE) - - __init__(self, address, password=None, size=1000, date_time=False, - timeout=DEF_TIMEOUT) - - address :: (str,int) :: (host/address of the windows implementation, port) - password :: str :: password for authentication(None for no authentication) - size :: int :: how much historical data can be inserted - date_time :: bool :: should block include date-time with each data-point? - timeout :: int :: how long to wait for responses from engine, - TOS-DDE server, internal IPC/Concurrency mechanisms, - and network communication (milliseconds) - - throws TOSDB_VirtualizationError TOSDB_ImplErrorWrapper - """ - def __init__(self, address, password=None, size=1000, date_time=False, - timeout=DEF_TIMEOUT): - super().__init__(address, password, size, date_time, timeout) - - -def enable_virtualization(address, password=None, timeout=DEF_TIMEOUT, verbose=True): - """ enable virtualization, making *this* the host system. - - This will start a background thread (_VTOS_Hub) to listen for virtual - 'clients' looking to use the admin calls (via Thread(s) _VTOS_AdminServer) - or data blocks (via Thread(s) _VTOS_BlockServer). - - The virtual client call will serlialize it's arguments, sending them - to the host. On success the local/host version of the call will be - executed by its respective Server. Finally, a return value/exception - will be serialized and sent back to the client. - - enable_virtualization(address, password=None, timeout=DEF_TIMEOUT, verbose=True) - - address :: (str,int) :: (address of the host system, port) - password :: str :: password for authentication(None for no authentication) - timeout :: int :: timeout(s) used by the underlying socket(s) - verbose :: bool :: extra information to stdout about client connection(s) - - throws TOSDB_VirtualizationError - """ - global _virtual_hub - - if password is not None: - try_import_pycrypto() - - try: - if _virtual_hub is None: - if password is not None: - check_password(password) - _virtual_hub = _VTOS_Hub(address, password, timeout, verbose) - _virtual_hub.start() - except Exception as e: - raise TOSDB_VirtualizationError("enable virtualization error", e) - - -def disable_virtualization(): - """ disable virtualization on host system. - - disable_virtualization() - - throws TOSDB_VirtualizationError - """ - global _virtual_hub - try: - if _virtual_hub is not None: - _virtual_hub.stop() - _virtual_hub = None - except Exception as e: - raise TOSDB_VirtualizationError("disable virtualization error", e) - - -def log_conn(msg, addr, file=_stdout, **info): - t = _strftime("%m/%d/%Y %H:%M:%S") - msg = msg[:40].ljust(40) - s = "+" if file == _stdout else "-" - print(s, t, " " + s + " ", msg, str(addr), file=file) - for k in info: - print(' ', k + ":", str(info[k]), file=file) - - -class _VTOS_BlockServer(_Thread): - def __init__(self, conn, timeout, stop_callback, verbose): - super().__init__(daemon=True) - self._my_sock = conn[0] - self._cli_addr = conn[1] - self._timeout = timeout - self._my_sock.settimeout(timeout / 1000) - self._blk = None - self._rflag = False - self._stop_callback = stop_callback - self._verbose = verbose - if self._verbose: - log_conn("OPEN BLOCK SERVER",self._cli_addr) - - def __del__(self): - if self._verbose: - log_conn("CLOSE BLOCK SERVER",self._cli_addr) - if self._my_sock: - self._my_sock.close() - - def stop(self): - self._rflag = False - - def run(self): - self._rflag = True - while self._rflag: - try: - dat = _recv_tcp(self._my_sock) - if not dat: - break - ret, kill = self._handle_msg(dat) - try: - _send_tcp(self._my_sock, ret) - finally: - if kill: - self.stop() - except _socket.timeout: - pass - except ConnectionResetError: - log_conn("CONNECTION RESET IN BLOCK SERVER",self._cli_addr, file=_stderr) - break - except Exception as e: - log_conn("FATAL: UNHANDLED EXCEPTION IN BLOCKSERVER", self._cli_addr, - file=_stderr, exception=e) - self._close() - raise - self._close() - - def _handle_msg(self,dat): - r = None - kill = False - args = _unpack_msg(dat) - msg_t = args[0].decode() - try: - if msg_t == _vCREATE: - uargs = _pickle.loads(args[1]) - self._blk = TOSDB_DataBlock(*uargs) - if self._verbose: - log_conn("BLOCK CREATED", self._cli_addr, name=self._blk._name.decode(), - block_size=self._blk._block_size, date_time=self._blk._date_time, - timeout=self._blk._timeout) - r = _pack_msg(_vSUCCESS) - if not r: - kill = True - elif msg_t == _vCALL: - r = self._handle_call(args) - else: - raise TOSDB_ValueError("invalid msg type") - except Exception as e: - kill = True - r = _pack_msg(_vFAILURE, _vEXCEPTION, repr(e)) - return (r, kill) - - def _handle_call(self,args): - try: - m = args[1].decode() - if m not in _vALLOWED_METHS: - raise TOSDB_VirtualizationError("method '%s' not in _vALLOWED_METHS" % m) - meth = getattr(self._blk, m) - uargs = _pickle.loads(args[2]) if len(args) > 2 else () - ret = meth(*uargs) - if ret is None: # None is still a success - return _pack_msg(_vSUCCESS) - elif hasattr(ret,NTUP_TAG_ATTR): #special namedtuple tag - return _pack_msg(_vSUCCESS_NT, _dumpnamedtuple(ret)) - else: - return _pack_msg(_vSUCCESS, _pickle.dumps(ret)) - except Exception as e: - return _pack_msg(_vFAILURE, _vEXCEPTION, repr(e)) - - def _close(self): - del self._blk - self._rflag = False - self._my_sock.close() - self._stop_callback(self) - - -class _VTOS_AdminServer(_Thread): - def __init__(self, conn, timeout, stop_callback, verbose): - super().__init__(daemon=True) - self._my_sock = conn[0] - self._cli_addr = conn[1] - self._timeout = timeout - self._my_sock.settimeout(timeout / 1000) - self._rflag = False - self._stop_callback = stop_callback - self._verbose = verbose - if self._verbose: - log_conn("OPEN ADMIN SERVER",self._cli_addr) - - def __del__(self): - if self._verbose: - log_conn("CLOSE ADMIN SERVER",self._cli_addr) - if self._my_sock: - self._my_sock.close() - - def stop(self): - self._rflag = False - - def run(self): - self._rflag = True - while self._rflag: - try: - dat = _recv_tcp(self._my_sock) - if not dat: - break - ret = self._handle_call(dat) - _send_tcp(self._my_sock, ret) - except _socket.timeout: - pass - except ConnectionResetError: - log_conn("CONNECTION RESET IN ADMIN SERVER",self._cli_addr, file=_stderr) - break - except Exception as e: - log_conn("FATAL: UNHANDLED EXCEPTION IN ADMIN SERVER", self._cli_addr, - file=_stderr, exception=e) - self._close() - raise - self._close() - - def _handle_call(self,dat): - args = _unpack_msg(dat) - rmsg = _pack_msg(_vFAILURE) - try: - m = args[0].decode() - if m not in _vALLOWED_ADMIN: - raise TOSDB_VirtualizationError("method '%s' not in _vALLOWED_ADMIN" % m) - meth = _vALLOWED_ADMIN_CALLS[m] - uargs = _pickle.loads(args[1]) if len(args) > 1 else () - r = meth(*uargs) - msg = (_vSUCCESS,) if (r is None) else (_vSUCCESS,_pickle.dumps(r)) - rmsg = _pack_msg(*msg) - except Exception as e: - rmsg = _pack_msg(_vFAILURE, _vEXCEPTION, repr(e)) - return rmsg - - def _close(self): - self._rflag = False - self._my_sock.close() - self._stop_callback(self) - - -class _VTOS_Hub(_Thread): - def __init__(self, address, password, timeout, verbose): - super().__init__(daemon=True) - self._my_addr = _check_and_resolve_address(address) - self._rflag = False - self._password = password - self._timeout = timeout - self._my_sock = _socket.socket() - self._my_sock.settimeout(timeout / 1000) - self._my_sock.bind(self._my_addr) - self._my_sock.listen(0) - self._virtual_block_servers = set() - self._virtual_admin_servers = set() - self._verbose = verbose - - def stop(self): - self._rflag = False - - def run(self): - self._rflag = True - while self._rflag: - try: - conn = None - try: - conn = self._my_sock.accept() - except _socket.timeout: - continue - conn[0].settimeout(self._timeout / 1000) - # indicate the response/handshake auth protocol required - if not self._ack_auth_protocol(conn): - conn[0].close() - continue - # use auth mechanims if we got a password arg - if self._password is not None and not self._authenticate(conn): - conn[0].close() - continue - if self._verbose: - log_conn("CLIENT CONNECTED",conn[1]) - # get the type of server to start(_vCONN_BLOCK, _vCONN_ADMIN) - dat = self._recv_msg(conn) - if not dat: - conn[0].close() - continue - try: - self._handle_msg(dat, conn) # (DONT CLOSE CONN AFTER THIS POINT) - rmsg = _pack_msg(_vSUCCESS) - except TOSDB_VirtualizationError as e: - log_conn("FAILED TO HANDLE MESSAGE",conn[1], file=_stderr, exception=e) - rmsg = _pack_msg(_vFAILURE, _vEXCEPTION, repr(e)) - continue - except Exception as e: - log_conn("FAILED TO HANDLE MESSAGE",conn[1], file=_stderr, exception=e) - rmsg = _pack_msg(_vFAILURE, _vEXCEPTION, repr(e)) - raise - finally: - _send_tcp(conn[0], rmsg) - except ConnectionResetError: - log_conn("CONNECTION RESET IN HUB SERVER", conn[1], file=_stderr) - continue - except Exception as e: # anything else... shutdown the hub - addr = conn[1] if conn else "(no client address)" - log_conn("FATAL: UNHANDLED EXCEPTION IN HUB SERVER", addr, file=_stderr, exception=e) - self._shutdown_servers() - raise - self._shutdown_servers() - - def _recv_msg(self, conn): - dat = None - try: - dat = _recv_tcp(conn[0]) - if not dat: - log_conn("DID NOT RECEIVE VALID MESSAGE", conn[1], file=_stderr) - except _socket.timeout: - log_conn("TIMED OUT WAITING FOR MESSAGE", conn[1], file=_stderr) - return dat - - def _ack_auth_protocol(self, conn): - amsg = _vREQUIRE_AUTH_NO - if self._password: - amsg = _vREQUIRE_AUTH - if self._verbose: - log_conn("TELL CLIENT TO AUTHENTICATE", conn[1]) - try: - _send_tcp(conn[0], amsg.encode()) - except _socket.timeout: - log_conn("TIMED OUT SENDING PROTOCOL MESSAGE", conn[1], file=_stderr) - return False - # get an ack or timeout - try: - r = _recv_tcp(conn[0]) - if r != _vACK.encode(): - log_conn("BAD ACK TOKEN RECEIVED", conn[1], file=_stderr) - return False - except _socket.timeout: - log_conn("TIMED OUT WAITING FOR ACK TOKEN", conn[1], file=_stderr) - return False - return True - - #TODO: add delay/throttle mechanism - def _authenticate(self, conn): - try: - good_auth = handle_auth_serv(conn,self._password) - if good_auth: - if self._verbose: - log_conn("CLIENT AUTHENTICATION SUCCEEDED", conn[1]) - return True - log_conn("CLIENT AUTHENTICATION FAILED", conn[1], file=_stderr) - except TOSDB_VirtualizationError as e: - log_conn("HANDSHAKE FAILED", conn[1], file=_stderr, exception=e) - except Exception as e: - log_conn("HANDSHAKE FAILED", conn[1], file=_stderr, exception=e) - raise - return False - - def _handle_msg(self,dat,conn): - dat = _unpack_msg(dat)[0].decode() - if dat == _vCONN_BLOCK: - vserv = _VTOS_BlockServer(conn, self._timeout, - self._virtual_block_servers.discard, self._verbose) - self._virtual_block_servers.add(vserv) - vserv.start() - elif dat == _vCONN_ADMIN: - vserv = _VTOS_AdminServer(conn, self._timeout, - self._virtual_admin_servers.discard, self._verbose) - self._virtual_admin_servers.add(vserv) - vserv.start() - else: - conn[0].close() - raise TOSDB_VirtualizationError("init message not _vCONN_BLOCK or _vCONN_ADMIN") - - def _shutdown_servers(self): - if self._verbose: - log_conn("SHUTDOWN BLOCK & ADMIN SERVERS", "") - while self._virtual_block_servers: - self._virtual_block_servers.pop().stop() - while self._virtual_admin_servers: - self._virtual_admin_servers.pop().stop() - - -def _vcall(msg, my_sock, hub_addr, rcnt=3): - try: - #clear any stale data in the stream(e.g our last call timed-out midway) - old_timeout = my_sock.gettimeout() - my_sock.settimeout(0) #set to non-blocking - try: - while True: #could we get stuck in this loop ? - my_sock.recv(4096) - except BlockingIOError: - pass - my_sock.settimeout(old_timeout) - #initiate new call - _send_tcp(my_sock, msg) - ret_b = _recv_tcp(my_sock) - args = _unpack_msg(ret_b) - status = None - try: - status = args[0].decode() - except Exception as e: - raise TOSDB_VirtualizationError("server failed to return (valid) message", - "_vcall", str(e)) - if status == _vFAILURE: - desc = args[2].decode() - if args[1].decode() == _vEXCEPTION: - raise wrap_impl_error(eval(desc)) - else: - raise TOSDB_VirtualizationError("failure status returned", desc) - else: - return (args[0],args[1]) if len(args) > 1 else (args[0],None) - except _socket.timeout as e: - raise TOSDB_VirtualizationError("socket timed out", "_vcall") - except ConnectionResetError: - try: - if rcnt > 0: # attemp rcnt retries via recursion - my_sock.connect(hub_addr) - return _vcall(msg, my_sock, hub_addr, rcnt-1) - else: - raise TOSDB_VirtualizationError("_vcall recursion limit hit") - except: - raise TOSDB_VirtualizationError("failed to reconnect to hub","_vcall") - - -def _dumpnamedtuple(nt): - n = type(nt).__name__ - od = nt._asdict() # __dict__ remove in 3.5.1 - return _pickle.dumps((n,tuple(od.keys()),tuple(od.values()))) - - -def _loadnamedtuple(nt): - name,keys,vals = _pickle.loads(nt) - ty = _namedtuple(name, keys) - return ty(*vals) - - -def _pack_msg(*parts): - def _escape_part(part): - enc = part.encode() if type(part) is not bytes else part - esc1 = _sub(_vESC, _vESC + _vEEXOR, enc) #escape the escape FIRST - return _sub(_vDELIM, _vESC + _vDEXOR, esc1) #escape the delim SECOND - return _vDELIM.join([_escape_part(p) for p in parts]) - - -def _unpack_msg(msg): - def _unescape_part(part): - unesc1 = _sub(_vESC + _vDEXOR, _vDELIM, part) #unescape the delim FIRST - return _sub(_vESC + _vEEXOR, _vESC, unesc1) #unescape the escape SECOND - if not msg: - return msg - return [_unescape_part(p) for p in msg.strip().split(_vDELIM)] - - -def _check_and_resolve_address(addr): - if type(addr) is tuple: - if len(addr) == 2: - if type(addr[0]) is str: - if type(addr[1]) is int: - return (_socket.gethostbyname(addr[0]),addr[1]) - raise TOSDB_TypeError("address must be of type (str,int)") - - - - - - - - - diff --git a/python/tosdb/__main__.py b/python/tosdb/__main__.py deleted file mode 100644 index 3c830dc..0000000 --- a/python/tosdb/__main__.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -from tosdb import * -from tosdb import _SYS_IS_WIN, _Thread - -from argparse import ArgumentParser as _ArgumentParser -from sys import stderr as _stderr -from time import sleep as _sleep - -args=None - -def _main_init(): - global args - parser = _ArgumentParser() - parser.add_argument('--virtual-client', type=str, - help='initialize virtual client "address port [timeout]"') - parser.add_argument('--virtual-server', type=str, - help='start virtual server "address port [timeout]"') - parser.add_argument('--root', help='root directory to search for the library') - parser.add_argument('--path', help='the exact path of the library') - parser.add_argument('--auth', help='password to use for authentication') - parser.add_argument('-v', '--verbose', action="store_true", - help='extra info to stdout about client connection(s)') - args = parser.parse_args() - - #remove quotes - if args.auth: - args.auth = args.auth.strip("'").strip('"') - - if args.virtual_server and _SYS_IS_WIN: - raw_args = args.virtual_server.split(' ') - try: - vs_args = ((raw_args[0],int(raw_args[1])),) - except IndexError: - print("usage: tosdb --virtual-server 'address port [timeout]'",file=_stderr) - exit(1) - vs_args += (args.auth,) - vs_args += (int(raw_args[2]),) if len(raw_args) > 2 else (DEF_TIMEOUT,) - vs_args += (args.verbose,) - #spin off so we don't block on exit - _Thread(target=enable_virtualization,args=vs_args).start() - - if args.virtual_client: - raw_args = args.virtual_client.split(' ') - try: - vc_args = ((raw_args[0],int(raw_args[1])),) - except IndexError: - print("usage: tosdb --virtual-client 'address port [timeout]'",file=_stderr) - exit(1) - vc_args += (args.auth,) - vc_args += (int(raw_args[2]),) if len(raw_args) > 2 else (DEF_TIMEOUT,) - admin_init(*vc_args) - - if args.virtual_client: - if args.path: - vinit(dllpath=args.path) - elif args.root: - vinit(root=args.root) - else: - if args.path: - init(dllpath=args.path) - elif args.root: - init(root=args.root) - - -_main_init() -globals().pop('_main_init') - -if args and args.virtual_server and _SYS_IS_WIN: - exit_commands = ('q','Q','quit','QUIT','Quit','Exit','EXIT','exit') - print('\n', str(exit_commands) + " to terminate\n") - while input() not in exit_commands: - pass -else: - import code - code.interact(local=locals()) - - diff --git a/python/tosdb/_auth.py b/python/tosdb/_auth.py deleted file mode 100644 index 5cdf5a0..0000000 --- a/python/tosdb/_auth.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -from ._common import _recvall_tcp, _recv_tcp, _send_tcp, \ - TOSDB_VirtualizationError, TOSDB_Error -import socket as _socket -from os import urandom as _urandom -from importlib import find_loader as _find_loader - -MAX_PASSWORD_SZ = 128 -MIN_PASSWORD_SZ = 12 -RAND_SEQ_SZ = 512 - -_vAUTH_SUCCESS = 'AUTH_SUCCESS' -_vAUTH_FAILURE = 'AUTH_FAILURE' - -_AES=None -_SHA256=None -# hold of on importing pycrypto as late as possible -# don't force dependency outside the std lib unless it's needed -def try_import_pycrypto(): - """try to import pycrypto library (if needed) - - throws TOSDB_VirtualizationError - """ - global _AES, _SHA256 - try: - from Crypto.Hash import SHA256 - from Crypto.Cipher import AES - _AES = AES - _SHA256 = SHA256 - except ImportError: - raise TOSDB_VirtualizationError( - "could not import Crypto (be sure pycrypto package is installed)") - - -def do_i_have_pycrypto(): - """check to see if pycrypto is installed and can be loaded""" - return bool( _find_loader('Crypto') ) - - -def check_password(password): - """check the password string is valid - - check_password(password) - - password :: str :: password string - - throws ValueError - """ - if len(password) < MIN_PASSWORD_SZ: - raise ValueError("password must be at least %i characters" % MIN_PASSWORD_SZ) - if len(password) > MAX_PASSWORD_SZ: - raise ValueError("password must be at most %i characters" % MAX_PASSWORD_SZ) - - -def handle_auth_cli(my_sock,password): - """handle authentication for the client side of the virtual connection - - handle_auth_cli(my_sock,password) - - my_sock :: socket :: socket that is making the authentication attempt - password :: str :: password used for authentication - - returns -> bool - - throws TOSDB_VirtualizationError - """ - - #hash/overwite password - password = _hash_password(password) - - # recv random sequence from server - try: - rand_seq_iv = _recv_tcp(my_sock) #raw (no need to unpack) - iv = rand_seq_iv[:_AES.block_size] - rand_seq = rand_seq_iv[_AES.block_size:] - except _socket.timeout as e: - raise TOSDB_VirtualizationError("socket timed out receiving random sequence") - - if len(rand_seq) != RAND_SEQ_SZ: - raise TOSDB_VirtualizationError("invalid size of random sequence") - - # encrypt random sequence using hashed password, attached iv - crypt_rand_seq = _AES.new(password, _AES.MODE_CFB, iv).encrypt(rand_seq) - - # send that back to server - _send_tcp(my_sock, crypt_rand_seq) #raw (no need to pack) - - # see if it worked - try: - ret = _recv_tcp(my_sock) - ret = ret.decode() - except _socket.timeout as e: - raise TOSDB_VirtualizationError("socket timed out receiving status") - - return (ret == _vAUTH_SUCCESS) - - -def handle_auth_serv(my_conn,password): - """handle authentication for the server side of the virtual connection - - handle_auth_serv(my_conn,password) - - my_conn :: socket :: connected socket that is recieving the authentication attempt - password :: str :: password used for authentication - - returns -> bool - - throws TOSDB_VirtualizationError - """ - - my_sock, my_addr = my_conn - - #hash/overwite password - password = _hash_password(password) - - # generate random sequence and iv - rseq = _urandom(RAND_SEQ_SZ) - iv = _urandom(_AES.block_size) - - # send to client - _send_tcp(my_sock, iv+rseq) - - # get back the encrypted version - try: - crypt_seq_cli = _recv_tcp(my_sock) #raw (no need to unpack) - except ConnectionAbortedError as e: - raise TOSDB_VirtualizationError("client aborted connection", e) - except _socket.timeout: - raise TOSDB_VirtualizationError("socket timed out receive encrypted random sequence") - - if crypt_seq_cli is None: - raise TOSDB_VirtualizationError("client failed to return encrypted sequence") - - # decrypt, compare to original, signal client - seq_cli = _AES.new(password, _AES.MODE_CFB, iv).decrypt(crypt_seq_cli) - seq_match = (rseq == seq_cli) - try: - rmsg = _vAUTH_SUCCESS if seq_match else _vAUTH_FAILURE - _send_tcp(my_sock, rmsg.encode()) - except BaseException as e: - raise TOSDB_VirtualizationError("failed to send success/failure to client", e) - - return seq_match - - -def _hash_password(password): - """hash password string via SHA256 to generate a 256 bit key for our AES Cipher""" - hasher = _SHA256.new() - hasher.update(password.encode()) - hp = hasher.digest() - return hp diff --git a/python/tosdb/_common.py b/python/tosdb/_common.py deleted file mode 100644 index f5f2c8d..0000000 --- a/python/tosdb/_common.py +++ /dev/null @@ -1,815 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - - -from ._tosdb import * -from .meta_enum import MetaEnum -from .doxtend import doxtend as _doxtend - -import sys as _sys -import struct as _struct -from collections import namedtuple as _namedtuple -from abc import ABCMeta as _ABCMeta, abstractmethod as _abstractmethod -from inspect import getmembers as _getmembers, isfunction as _isfunction -from types import MethodType as _MethodType -from threading import RLock as _RLock - -from time import mktime as _mktime, struct_time as _struct_time, \ - asctime as _asctime, localtime as _localtime, \ - strftime as _strftime - -from ctypes import Structure as _Structure, \ - c_long as _long_, \ - c_int as _int_, \ - c_double as _double_, \ - c_float as _float_, \ - c_longlong as _longlong_, \ - c_char_p as _str_ - -BASE_YR = 1900 -NTUP_TAG_ATTR = "_dont_worry_about_why_this_attribute_has_a_weird_name_" - -class _TOSDB_DataBlock(metaclass=_ABCMeta): - """ The DataBlock interface """ - @classmethod - def __subclasshook__(cls, C): - if cls is _TOSDB_DataBlock: - for ameth in cls.__abstractmethods__: - if not any(ameth in b.__dict__ for b in C.__mro__): - return False - return True - return NotImplemented - - @classmethod - def is_thread_safe(self): - """ Is the block thread-safe """ - return False - - @_abstractmethod - def close(): - """ Close the underlying block - - NOTE: once called the block will become unusable. It should be be the last - method used on the block. - """ - pass - - @_abstractmethod - def __str__(): - pass - - @_abstractmethod - def info(): - """ Returns a more readable dict of info about the underlying block """ - pass - - @_abstractmethod - def is_using_datetime(): - """ Is the blocking including date-time with each data-point """ - pass - - @_abstractmethod - def get_block_size(): - """ Returns the amount of historical data that can be stored in the block - - get_block_size(self) - - returns -> int - - throws TOSDB_CLibError - """ - pass - - @_abstractmethod - def set_block_size(): - """ Changes the amount of historical data that can be stored in the block - - set_block_size(self, sz) - - sz :: int :: new size - - throws TOSDB_CLibError - """ - - pass - - @_abstractmethod - def stream_occupancy(): - """ Returns the current number of data-points inserted into the data-stream - - stream_occupancy(self, item, topic) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - - returns -> int - - throws TOSDB_CLibError, TOSDB_TypeError - """ - pass - - @_abstractmethod - def items(): - """ Returns the items currently in the block (and not pre-cached). - - items(self, str_max=MAX_STR_SZ) - - str_max :: int :: maximum length of item strings returned - - returns -> list of str - - throws TOSDB_CLibError - """ - pass - - @_abstractmethod - def topics(): - """ Returns the topics currently in the block (and not pre-cached). - - topics(self, str_max=MAX_STR_SZ) - - str_max :: int :: maximum length of topic strings returned - - returns -> list of str - - throws TOSDB_CLibError - """ - pass - - @_abstractmethod - def items_precached(): - """ Returns the items currently in the block's pre-cache. - - items_precached(self, str_max=MAX_STR_SZ) - - str_max :: int :: maximum length of item strings returned - - returns -> list of str - - throws TOSDB_CLibError - """ - pass - - @_abstractmethod - def topics_precached(): - """ Returns the topics currently in the block's pre-cache. - - topics_precached(self, str_max=MAX_STR_SZ) - - str_max :: int :: maximum length of topic strings returned - - returns -> list of str - - throws TOSDB_CLibError - """ - pass - - @_abstractmethod - def add_items(): - """ Add items (ex. 'IBM', 'SPY') to the block. - - NOTE: if there are no topics currently in the block, these items will - be pre-cached and appear not to exist, until a valid topic is added. - - add_items(self, *items) - - *items :: *str :: any numer of item strings - - throws TOSDB_Error - """ - pass - - @_abstractmethod - def add_topics(): - """ Add topics (ex. 'LAST', 'ASK') to the block. - - NOTE: if there are no items currently in the block, these topics will - be pre-cached and appear not to exist, until a valid item is added. - - add_topics(self, *topics) - - *topics :: *str :: any numer of topic strings - - throws TOSDB_Error - """ - pass - - @_abstractmethod - def remove_items(): - """ Remove items (ex. 'IBM', 'SPY') from the block. - - NOTE: if this call removes all items from the block the remaining topics - will be pre-cached and appear not to exist, until a valid item is re-added. - - remove_items(self, *items) - - *items :: *str :: any numer of item strings in the block - - throws TOSDB_Error - """ - pass - - @_abstractmethod - def remove_topics(): - """ Remove topics (ex. 'LAST', 'ASK') from the block. - - NOTE: if this call removes all topics from the block the remaining items - will be pre-cached and appear not to exist, until a valid topic is re-added. - - remove_topics(self, *topics) - - *topics :: *str :: any numer of topic strings in the block - - throws TOSDB_Error - """ - pass - - @_abstractmethod - def get(): - """ Return a single data-point from the data-stream - - get(self, item, topic, date_time=False, indx=0, check_indx=True, - data_str_max=STR_DATA_SZ) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - date_time :: bool :: include a TOSDB_DateTime object - indx :: int :: index/position of data-point to get - [0 to block_size-1] or [-block_size to -1] - check_indx :: bool :: throw if datum doesn't exist at that index - data_str_max :: int :: maximum size of string data returned - - if date_time == True: returns -> 2-tuple** - else: returns -> int/float/str** - - **data are of type int, float, or str (depending on the topic) - - throws TOSDB_DataTimeError, TOSDB_IndexError, TOSDB_DataError, - TOSDB_CLibError, TOSDB_TypeError, TOSDB_ValueError - """ - pass - - @_abstractmethod - def stream_snapshot(): - """ Return multiple data-points(a snapshot) from the data-stream - - stream_snapshot(self, item, topic, date_time=False, end=-1, beg=0, - smart_size=True, data_str_max=STR_DATA_SZ) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - date_time :: bool :: include TOSDB_DateTime objects - end :: int :: index of least recent data-point - beg :: int :: index of most recent data-point - smart_size :: bool :: limit amount of returned data by occupancy - data_str_max :: int :: maximum length of string data returned - - if date_time == True: returns -> list of 2-tuple** - else: returns -> list** - - **data are of type int, float, or str (depending on the topic) - - throws TOSDB_DataTimeError, TOSDB_IndexError, TOSDB_CLibError, - TOSDB_TypeError, TOSDB_ValueError - """ - pass - - @_abstractmethod - def stream_snapshot_from_marker(): - """ Return multiple data-points(a snapshot) from an 'atomic' marker - - It's likely the stream will grow between consecutive calls. This call - guarantees to pick up where the last get(), stream_snapshot(), or - stream_snapshot_from_marker() call ended (under a few assumptions, see - below). - - Internally the stream maintains a 'marker' that tracks the position of - the last value pulled; the act of retreiving data and moving the - marker can be thought of as a single, 'atomic' operation. - - There are three states to be aware of: - 1) a 'beg' value that is greater than the marker (even if beg = 0) - 2) a marker that moves through the entire stream and hits the bound - 3) passing a buffer that is too small for the whole range - - State (1) can be caused by passing in a beginning index that is past - the current marker, or by passing in 0 when the marker has yet to - move. 'None' will be returned. - - State (2) occurs when the marker doesn't get reset before it hits the - end of stream (block_size); as the oldest data is popped off the back of - the stream it is lost (the marker can't grow past the end of the stream). - - State (3) occurs when an inadequately large buffer is used. The call - handles buffer sizing for you by calling down to get the marker index, - adjusting by 'beg' and 'margin_of_safety'. The latter helps assure the - marker doesn't outgrow the buffer by the time the low-level retrieval - operation completes. The default value indicates that over 100 push - operations would have to take place during this call(highly unlikely). - - If throw_if_data_lost is True, (2) or (3) will raise TOSDB_DataError. - - * * * - - stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, - margin_of_safety=100, throw_if_data_lost=True, - data_str_max=STR_DATA_SZ) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - date_time :: bool :: include TOSDB_DateTime objects - beg :: int :: index of most recent data-point - margin_of_safety :: int :: error margin for async stream growth (see above) - throw_if_data_loss :: bool :: how to handle error states (see above) - data_str_max :: int :: maximum length of string data returned - - if beg > internal marker position: returns -> None - elif date_time == True: returns -> list of 2-tuple** - else: returns -> list** - - **data are of type int, float, or str (depending on the topic) - - throws TOSDB_DataTimeError, TOSDB_IndexError, TOSDB_ValueError, - TOSDB_DataError, TOSDB_CLibError, TOSDB_TypeError - """ - pass - - @_abstractmethod - def n_from_marker(): - """ Return n data-points from an 'atomic' marker - - It's likely the stream will grow between consecutive calls. This call - guarantees to pick up where the last get(), stream_snapshot(), - stream_snapshot_from_marker(), or n_from_marker() call ended. - - Internally the stream maintains a 'marker' that tracks the position of - the last value pulled; the act of retreiving 'n' data-points IN FRONT OF - the marker(more recent) and moving the marker can be thought of as a - single, 'atomic' operation. - - If the marker doesn't get reset(moved) before it hits the end of stream - (block_size) the stream becomes 'dirty' and will raise TOSDB_DataError - if throw_if_data_lost == True. As the oldest data is popped off the back - of the stream it is lost(the marker can't grow past the end of the stream). - - * * * - - n_from_marker(self, item, topic, date_time=False, n=1, - throw_if_data_lost=True, data_str_max=STR_DATA_SZ) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - date_time :: bool :: include TOSDB_DateTime objects - n :: int :: number of data-points in front of marker - throw_if_data_loss :: bool :: how to handle error states (see above) - data_str_max :: int :: maximum length of string data returned - - if beg > internal marker position: returns -> None - elif date_time == True: returns -> list of 2-tuple** - else: returns -> list** - - **data are of type int, float, or str (depending on the topic) - - throws TOSDB_DataTimeError, TOSDB_IndexError, TOSDB_ValueError, - TOSDB_DataError, TOSDB_CLibError, TOSDB_TypeError - """ - pass - - @_abstractmethod - def item_frame(): - """ Return all the most recent item values for a particular topic. - - item_frame(self, topic, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ) - - topic :: str :: any topic string in the block - date_time :: bool :: include TOSDB_DateTime objects - labels :: bool :: include item labels - data_str_max :: int :: maximum length of string data returned - label_str_max :: int :: maximum length of item label strings returned - - if labels == date_time == True: returns -> namedtuple of 2-tuple** - elif labels == True: returns -> namedtuple** - elif date_time == True: returns -> list of 2-tuple** - else: returns -> list** - - **data are of type int, float, or str (depending on the topic) - - throws TOSDB_DataTimeError, TOSDB_CLibError, TOSDB_TypeError, - TOSDB_ValueError - """ - pass - - @_abstractmethod - def topic_frame(): - """ Return all the most recent topic values for a particular item: - - topic_frame(self, item, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ) - - item :: str :: any item string in the block - date_time :: bool :: include TOSDB_DateTime objects - labels :: bool :: include topic labels - data_str_max :: int :: maximum length of string data returned - label_str_max :: int :: maximum length of topic label strings returned - - if labels == date_time == True: returns -> namedtuple of 2-tuple** - elif labels == True: returns -> namedtuple** - elif date_time == True: returns -> list of 2-tuple** - else: returns -> list** - - **data are ALL of type str (frame can contain topics of differnt type) - - throws TOSDB_DataTimeError, TOSDB_CLibError, TOSDB_TypeError, - TOSDB_ValueError - """ - pass - - -def make_block_thread_safe(*non_public_methods): - class FunctionObject: - def __init__(self, func): - self._func = func - def __call__(self, instance, *args, **kargs): - with instance._rlock: - return self._func(instance, *args, **kargs) - def __get__(self, instance, parent): - if instance is None: - return self - return _MethodType(self, instance) - def _make_block_thread_safe(cls): - # have __init__ create a recursive lock - __init_old__ = getattr(cls, '__init__') - def __init_new__(self, *args, **kargs): - __init_old__(self, *args, **kargs) - self._rlock = _RLock() - setattr(cls, '__init__', __init_new__) - # protect each public method (or 'private' methods in the - # 'non_public_methods' arg) with the recursive lock via FunctionObject - for m,f in [i for i in _getmembers(cls, predicate=_isfunction) \ - if i[0][0] != '_' or i[0] in non_public_methods]: - c = FunctionObject(f) - c.__doc__ = f.__doc__ - c.__name__ = repr(c) - setattr(cls, m, c) - # override is_thread_safe - @_doxtend(_TOSDB_DataBlock) - def is_thread_safe(cls): - return True - setattr(cls, 'is_thread_safe', is_thread_safe) - return cls - return _make_block_thread_safe - - -class TOSDB_Error(Exception): - """ Base exception for tosdb """ - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_InitError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_PlatformError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_CLibError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_DateTimeError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_DataError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_ValueError(TOSDB_Error, ValueError): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_TypeError(TOSDB_Error, TypeError): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_IndexError(TOSDB_Error, IndexError): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_TimeoutError(TOSDB_Error, TimeoutError): - def __init__(self, *messages): - super().__init__(*messages) - - -class TOSDB_VirtualizationError(TOSDB_Error): - def __init__(self, *messages): - super().__init__(*messages) - - -def wrap_impl_error(clss): - if not isinstance(clss, Exception): - raise TypeError("clss must be instance of Exception") - our_def = """\ -class TOSDB_ImplErrorWrapper({clss}): - def __init__(self, error): - {clss}(error) -""".format(clss=(type(clss).__name__)) - exec(our_def) - our_obj = eval("TOSDB_ImplErrorWrapper")(clss) - try: - our_obj.__module__ = _sys._getframe(1).f_globals.get('__name__') - except: - pass - return our_obj - - -def _recv_tcp(sock): - packedlen = _recvall_tcp(sock, 8) - if not packedlen: - return None - dlen = _struct.unpack('Q', packedlen)[0] - return _recvall_tcp(sock, dlen) - - -def _recvall_tcp(sock, n): - data = b'' - while len(data) < n: - p = sock.recv(n - len(data)) - if not p: - return None - data += p - return data - - -def _send_tcp(sock, data): - dl = len(data) - msg = _struct.pack('Q',len(data)) + data - return sock.sendall(msg) - - -class _DateTimeStamp(_Structure): - """ 'private' C stdlib date-time struct w/ added micro-second field """ - class _CTime(_Structure): - """ C stdlib tm struct """ - _fields_ = [("tm_sec", _int_), - ("tm_min", _int_), - ("tm_hour", _int_), - ("tm_mday", _int_), - ("tm_mon", _int_), - ("tm_year", _int_), - ("tm_wday", _int_), - ("tm_yday", _int_), - ("tm_isdst", _int_)] - - _fields_ = [("ctime_struct", _CTime), ("micro_second", _long_)] - - -class TOSDB_DateTime( _namedtuple("DateTime",["micro","sec","min", - "hour", "day","month","year"]) ): - """ The object used for handling DateTime values. - - TOSDB_DateTime is built on top of a simplified named tuple. It can be - constructed from _DateTimeStamp, _struct_time from the time.py library, or - itself (copy construction). It can be pickled (serialized); allows for basic - addition and subtraction returning a new object or a DateTimeDiff tuple, - respectively; overloads comparison operators; and provides static utility - functions to convert betwen DateTimeDiff objects and microseconds. - - __init__(self, obj, micro_second=0) - - obj :: _DateTimeStamp -or- time._struct_time -or- TOSDB_DateTime object - micro_second :: int microsecond value - - throws TOSDB_DataTimeError - """ - dtd_tuple = _namedtuple("DateTimeDiff",["micro","sec","min","hour","day","sign"]) - - _mktime = None # cache total sec from epoch to make ops easier - - def __getnewargs__(self): - return ((self.sec,self.min,self.hour,self.day,self.month,self.year),self.micro) - - def __getstate__(self): - return self.__dict__ - - def __new__(cls, obj, micro_second = 0): - if micro_second != 0: - micro_second %= 1000000 - _stime = None - if isinstance(obj, _DateTimeStamp): - _stime = cls._to_struct_time(obj) - micro_second = obj.micro_second - elif isinstance(obj, _struct_time): - _stime = obj - elif isinstance(obj, TOSDB_DateTime): - return super(TOSDB_DateTime, cls).__new__(cls, obj.micro, obj.sec, - obj.min, obj.hour, obj.day, - obj.month, obj.year) - elif isinstance(obj, tuple): - return super(TOSDB_DateTime, cls).__new__(cls, micro_second, *obj) - else: - raise TOSDB_DateTimeError("invalid 'object' passed to __new__") - - return super(TOSDB_DateTime, cls).__new__(cls, micro_second, _stime.tm_sec, - _stime.tm_min, _stime.tm_hour, - _stime.tm_mday, _stime.tm_mon, - _stime.tm_year) - - - def __init__(self, obj, micro_second=0): - if isinstance(obj, _DateTimeStamp): - try: - self._mktime = _mktime(TOSDB_DateTime._to_struct_time(obj)) - except: - self._mktime = 0 - elif isinstance(obj, _struct_time): - self._mktime = _mktime(obj) - elif isinstance(obj, TOSDB_DateTime): - self._mktime = obj._mktime - else: - raise TOSDB_DateTimeError("invalid 'object' passed to __init__") - - - def __str__(self): - ftime = _strftime("%m/%d/%y %H:%M:%S", _localtime(self._mktime)) - return ftime + " " + str(self.micro) - - - def __add__(self, micro_seconds): - if not isinstance(micro_seconds, int): - raise TOSDB_DateTimeError("micro_seconds must be integer") - incr_sec = micro_seconds // 1000000 - ms_new = self.micro + (micro_seconds % 1000000 ) - if ms_new >= 1000000: - incr_sec += 1 - ms_new -= 1000000 - # reduce to seconds, increment, return to _struct_time - mk_time = self._mktime - mk_time += incr_sec - new_time = _localtime(mk_time) - # return a new TOSDB_DateTime - return TOSDB_DateTime(new_time, micro_second = ms_new) - - - def __sub__(self, other_or_micro_sec): - other = other_or_micro_sec - if isinstance(other , int): # subtracting an integer - if other < 0: - return self.__add__(other * -1) - other_sec = other // 1000000 - ms_diff = self.micro - (other % 1000000) - if ms_diff < 0: # take a second, if needed - other_sec += 1 - ms_diff += 1000000 - # reduce to seconds, decrement, return to stuct_time - mk_time = self._mktime - mk_time -= other_sec - new_time = _localtime(mk_time) - return TOSDB_DateTime(new_time, micro_second = ms_diff) - elif isinstance(other, TOSDB_DateTime): # subtracting another TOSDB_DateTime - try: # try to get other's time in seconds - sec_diff = self._mktime - other._mktime - ms_diff = self.micro - other.micro - # convert the diff in micro_seconds to the diff-tuple - return TOSDB_DateTime.micro_to_dtd(sec_diff * 1000000 + ms_diff) - except Exception as e: - raise TOSDB_DateTimeError("invalid TOSDB_DateTime object", e) - else: - raise TOSDB_DateTimeError("other_or_micro_sec not TOSDB_DateTime or int") - - - def __lt__(self, other): - return self._compare(other) < 0 - - def __ge__(self, other): - return not self.__lt__(other) - - def __gt__(self, other): - return self._compare(other) > 0 - - def __le__(self, other): - return not self.__gt__(other) - - def __eq__(self, other): - return not self._compare(other) - - def __ne__(self, other): - return self._compare(other) - - - def _compare(self, other): - if not isinstance(other, TOSDB_DateTime): - raise TOSDB_DateTimeError("unable to compare; unorderable types") - diff = self._mktime - other._mktime - if diff == 0: - diff = self.micro - other.micro - return (-1 if diff < 0 else 1) if diff else 0 - - - @property - def mktime(self): - return self._mktime - - @property - def mktime_micro(self): - return int(self._mktime * 1000000) + self.micro - - @staticmethod - def _to_struct_time(obj): - return _struct_time([obj.ctime_struct.tm_year + BASE_YR, - obj.ctime_struct.tm_mon +1, - obj.ctime_struct.tm_mday, - obj.ctime_struct.tm_hour, - obj.ctime_struct.tm_min, - obj.ctime_struct.tm_sec, - obj.ctime_struct.tm_wday + 1, - obj.ctime_struct.tm_yday + 1, - obj.ctime_struct.tm_isdst ]) - - - @staticmethod - def micro_to_dtd(micro_seconds): - """ Converts micro_seconds to DateTimeDiff """ - pos = True - if micro_seconds < 0: - pos = False - micro_seconds = abs(micro_seconds) - new_ms = int(micro_seconds % 1000000) - micro_seconds //= 1000000 - new_sec = int(micro_seconds % 60) - micro_seconds //= 60 - new_min = int(micro_seconds % 60) - micro_seconds //= 60 - new_hour = int(micro_seconds % 24) - new_day = int(micro_seconds // 24) - - return TOSDB_DateTime.dtd_tuple(new_ms, new_sec, new_min, new_hour, - new_day, ('+' if pos else '-')) - - - @staticmethod - def dtd_to_micro(dtd): - """ Converts DateTimeDiff to micro_seconds """ - tmp = dtd.day * 24 + dtd.hour - tmp = tmp*60 + dtd.min - tmp = tmp*60 + dtd.sec - tmp *= 1000000 - tmp += dtd.micro - if dtd.sign == '+': - return tmp - elif dtd.sign == '-': - return tmp * -1 - else: - raise TOSDB_DateTimeError("invalid 'sign' field in DateTimeDiff") - - -def abort_init_after_warn(): - print("***WARNING***") - print("By not supplying 'dllpath' or 'root' args to init/vinit you are opting for " - + "a default search root of 'C:\\'. This will attempt to search the ENTIRE " - + "disk/drive for the tos-databridge library. IT'S RECOMMENDED YOU PROVIDE " - + "AN EXACT PATH OR A MORE SPECIFIC DIRECTORY ROOT.") - print("***WARNING***") - if input("Do you want to continue anyway?") in ['y','Y','YES','yes','Yes']: - return False - else: - print("- init(root='C:\\') aborted") - return True - - -# convert type_bits to string and ctypes type -def _type_switch(type_bits): - if type_bits == INTGR_BIT | QUAD_BIT: - return ("LongLong", _longlong_) - elif type_bits == INTGR_BIT: - return ("Long", _long_) - elif type_bits == QUAD_BIT: - return ("Double", _double_) - elif type_bits == 0: - return ("Float", _float_) - else: # default to string - return("String", _str_) diff --git a/python/tosdb/_win.py b/python/tosdb/_win.py deleted file mode 100644 index bfa955a..0000000 --- a/python/tosdb/_win.py +++ /dev/null @@ -1,1212 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -"""_win.py: The non-portable bits of the window's tosdb implementation - -Please refer to the tosdb (tosdb/__init__.py) docstring for detailed information. -""" - -from ._common import * -from ._common import _DateTimeStamp, _TOSDB_DataBlock, _type_switch -from .doxtend import doxtend as _doxtend - -from io import StringIO as _StringIO -from uuid import uuid4 as _uuid4 -from functools import partial as _partial -from itertools import product as _product -from math import log as _log, ceil as _ceil -from sys import maxsize as _maxsize, stderr as _stderr -from atexit import register as _on_exit -from collections import namedtuple as _namedtuple -from time import asctime as _asctime, localtime as _localtime, sleep as _sleep, \ - perf_counter as _perf_counter -from platform import system as _system -from contextlib import contextmanager as _contextmanager - -from os import walk as _walk, stat as _stat, curdir as _curdir, \ - listdir as _listdir, sep as _sep, path as _path, \ - getpid as _getpid - -from re import compile as _compile, search as _search, match as _match, \ - split as _split - -if _system() not in ["Windows","windows","WINDOWS"]: - print("error: tosdb/_win.py is for windows only !", file=_stderr) - exit(1) - -from ctypes import CDLL as _CDLL, \ - cast as _cast, \ - pointer as _pointer, \ - create_string_buffer as _BUF_, \ - POINTER as _PTR_, \ - c_double as _double_, \ - c_float as _float_, \ - c_ulong as _ulong_, \ - c_long as _long_, \ - c_longlong as _longlong_, \ - c_char_p as _str_, \ - c_char as _char_, \ - c_ubyte as _uchar_, \ - c_int as _int_, \ - c_void_p as _pvoid_, \ - c_uint as _uint_, \ - c_uint32 as _uint32_, \ - c_uint8 as _uint8_ - - -_pchar_ = _PTR_(_char_) -_ppchar_ = _PTR_(_pchar_) -_cast_cstr = lambda x: _cast(x,_str_).value.decode() - -_gen_str_buffers = lambda sz, n: [_BUF_(sz) for _ in range(n)] -_gen_str_buffers_ptrs = lambda bufs: (_pchar_ * len(bufs))(*[_cast(b,_pchar_) for b in bufs]) - -_map_cstr = _partial(map,_cast_cstr) -_map_dt = _partial(map, TOSDB_DateTime) -_zip_cstr_dt = lambda cstr, dt: zip(_map_cstr(cstr),_map_dt(dt)) - -DLL_BASE_NAME = "tos-databridge" -DLL_DEPENDS1_NAME = "_tos-databridge" -SYS_ARCH_TYPE = "x64" if (_log(_maxsize * 2, 2) > 33) else "x86" -MIN_MARGIN_OF_SAFETY = 10 - -_REGEX_NON_ALNUM = _compile("[\W+]") -_REGEX_LETTER = _compile("[a-zA-Z]") -_VER_SFFX = '[\d]{1,2}.[\d]{1,2}' -_REGEX_VER_SFFX = _compile('-' + _VER_SFFX + '-') - -_REGEX_DLL_NAME = _compile('^(' + DLL_BASE_NAME + '-)' \ - + _VER_SFFX + '-' \ - + SYS_ARCH_TYPE +'(.dll)$') - -_REGEX_DBG_DLL_PATH = _compile('^.+(' + DLL_BASE_NAME + '-)' \ - + _VER_SFFX + '-' \ - + SYS_ARCH_TYPE + '_d(.dll)$') - -_dll = None -_dll_depend1 = None - - -def init(dllpath=None, root="C:\\", bypass_check=False): - """ Initialize the underlying tos-databridge DLL and try to connect. - - Returns True if library was successfully loaded, not necessarily that - it was also able to connect. Details are sent to stdout. - - init(dllpath=None, root="C:\\", bypass_check=False) - - dllpath :: str :: exact path of the DLL -or- - root :: str :: directory to start walking/searching to find the DLL - bypass_check :: bool :: used by virtual layer implemenation (DO NOT SET) - - returns -> bool - - throws TOSDB_InitError TOSDB_Error - """ - global _dll, _dll_depend1 - rel = set() - if not bypass_check and dllpath is None and root == "C:\\": - if abort_init_after_warn(): - return False - - def _remove_older_versions(): - nonlocal rel - getver = lambda x: _search(_REGEX_VER_SFFX,x).group().strip('-') - vers = tuple(zip(map(getver, rel), rel)) - vers_max = max(vers)[0].split('.')[0] - mtup = tuple(( x[0].split('.')[1],x[1]) \ - for x in vers if x[0].split('.')[0] == vers_max ) - mtup_max = max(mtup)[0] - rel = set(x[1] for x in mtup if x[0] == mtup_max) - - def _get_depends1_dll_path(dllpath): - d = _path.dirname(dllpath) - dbg = _match(_REGEX_DBG_DLL_PATH, dllpath) - base = d + "/" + DLL_DEPENDS1_NAME + "-" + SYS_ARCH_TYPE - path = base + ("_d.dll" if dbg else ".dll") - return path - - try: - if dllpath is None: - matcher = _partial(_match, _REGEX_DLL_NAME) - for nfile in map(matcher, _listdir(_curdir)): - if nfile: # try the current dir first - rel.add(_curdir+ _sep + nfile.string) - if not rel: # no luck, walk the dir tree - for root, dirs, files in _walk(root): - for file in map(matcher, files): - if file: - rel.add(root + _sep + file.string) - if not rel: # if still nothing throw - raise TOSDB_InitError(" could not locate DLL") - if len(rel) > 1: # only use the most recent version(s) - _remove_older_versions() - # most recently updated - d = dict(zip(map(lambda x: _stat(x).st_mtime, rel), rel)) - rec = max(d) - dllpath = d[rec] - - dllpath_depends1 = _get_depends1_dll_path(dllpath) - _dll_depend1 = _CDLL(dllpath_depends1) - _dll = _CDLL(dllpath) - print("+ Using Module(s) ", dllpath) - print(" ", dllpath_depends1) - print("+ Last Update:", _asctime(_localtime(_stat(dllpath).st_mtime))) - print("+ Process ID:", str(_getpid())) - if connect(): - print("+ Succesfully Connected to Service\Engine") - if connected(): - print("+ Succesfully Connected to TOS") - else: - print("- Failed to Connect to TOS") - else: - print("- Failed to Connect to Service\Engine") - print("- Failed to Connect to TOS") - return True # indicate the lib was loaded (but not if connect succeeded) - except Exception as e: - raise TOSDB_InitError("unable to initialize library:", e) - - -def connect(): - """ Attempts to connect to the engine and TOS platform - - connect() - - returns -> bool - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - ret = _lib_call("TOSDB_Connect", error_check=False) - return (ret == 0) # 0 on success (error code) - - -def connected(): - """ Returns True if there is an active connection to the engine AND TOS platform - - connected() - - returns -> bool - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - ret = _lib_call("TOSDB_IsConnectedToEngineAndTOS", ret_type=_uint_, error_check=False) - return bool(ret) # 1 on success (bool value) - - -def connection_state(): - """ Returns the connection state between the C lib, engine and TOS platform - - connection_state() - - if NOT connected to the engine: returns -> CONN_NONE - elif connected to the engine BUT NOT the TOS platform: returns -> CONN_ENGINE - elif connected to the engine AND the TOS platform: returns -> CONN_ENGINE_TOS - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - return _lib_call("TOSDB_ConnectionState", ret_type=_uint_, error_check=False) - - -def clean_up(): - """ Clean up shared resources. - - *** CALL BEFORE EXITING INTERPRETER *** - - Attempts to close the underlying block and disconnect gracefully so - streams aren't orphaned/leaked. - - clean_up() - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - global _dll, _dll_depend1 - if _dll is not None: - err = _lib_call("TOSDB_CloseBlocks", error_check=False) - print("+ Closing Blocks" if not err else "- Error Closing Blocks") - _lib_call("TOSDB_Disconnect", error_check=False) - print("+ Disconnecting From Service \ Engine") - print("+ Closing Module(s) ", _dll._name) - print(" ", _dll_depend1._name) - _dll = None - _dll_depend1 = None - - -_on_exit(clean_up) # try automatically on exit (NO GUARANTEE) - - -@_contextmanager -def Init(dllpath=None, root="C:\\"): - """ Manage a 'session' with the C lib, engine, and TOS platform. - - The context manager handles initialization of the library and tries to - connect to the engine and TOS platform. On exiting the context block it - automatically cleans up. - - with Init(...): - # (automatically initializes, connects etc.) - # use admin interface, create data blocks etc. - # (automatically cleans up, closes etc.) - - def Init(dllpath=None, root="C:\\") - - dllpath :: str :: exact path of the DLL -or- - root :: str :: directory to start walking/searching to find the DLL - - throws TOSDB_InitError - """ - try: - if not init(dllpath, root): - raise TOSDB_InitError("failed to initilize library") - if not connected(): - if not connect(): # try again - raise TOSDB_InitError("failed to connect to library") - yield - finally: - clean_up() - - -def get_block_limit(): - """ Returns the block limit imposed by the C Lib - - get_block_limit() - - returns -> int - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - return _lib_call("TOSDB_GetBlockLimit", ret_type=_uint32_, error_check=False) - - -def set_block_limit(new_limit): - """ Changes the block limit imposed by the C Lib - - set_block_limit(new_limit) - - new_limit :: int :: new block limit - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - _lib_call("TOSDB_SetBlockLimit", - new_limit, - ret_type=_uint32_, - arg_types=(_uint32_,), - error_check=False) - - -def get_block_count(): - """ Returns the current count of instantiated blocks (according to C Lib) - - get_block_count() - - returns -> int - - throws TOSDB_CLibError - **does not raise an exception if lib call returns error code - """ - return _lib_call("TOSDB_GetBlockCount", ret_type=_uint32_, error_check=False) - - -def type_bits(topic): - """ Returns the type bits for a particular topic - - These type bits can be logical &'d with type bit contstants (ex. QUAD_BIT) - to determine the underlying type of the topic. - - type_bits(topic) - - topic :: str :: topic string ('LAST','ASK', etc) - - returns -> int - - throws TOSDB_CLibError - """ - b = _uint8_() - _lib_call("TOSDB_GetTypeBits", - topic.upper().encode("ascii"), - _pointer(b), - arg_types=(_str_,_PTR_(_uint8_)) ) - - return b.value - - -def type_string(topic): - """ Returns a platform-dependent string of the type of a particular topic - - type_string(topic) - - topic :: str :: topic string ('LAST','ASK', etc) - - returns -> str - - throws TOSDB_CLibError - """ - s = _BUF_(MAX_STR_SZ + 1) - _lib_call("TOSDB_GetTypeString", - topic.upper().encode("ascii"), - s, - (MAX_STR_SZ + 1)) - - return s.value.decode() - - -class TOSDB_DataBlock(_TOSDB_DataBlock): - """ The base object for storing TOS data (NOT THREAD SAFE) - - __init__(self, size=1000, date_time=False, timeout=DEF_TIMEOUT) - - size :: int :: how much historical data can be inserted - date_time :: bool :: should block include date-time with each data-point? - timeout :: int :: how long to wait for responses from engine, TOS-DDE server, - and/or internal IPC/Concurrency mechanisms (milliseconds) - - throws TOSDB_CLibError - """ - def __init__(self, size=1000, date_time=False, timeout=DEF_TIMEOUT): - self._name = (_uuid4().hex).encode("ascii") - self._block_size = size - self._timeout = timeout - self._date_time = date_time - self._items = [] - self._topics = [] - self._items_precached = [] - self._topics_precached = [] - self._valid = False - _lib_call("TOSDB_CreateBlock", - self._name, - size, - date_time, - timeout, - arg_types=(_str_,_uint32_,_int_,_uint32_)) - self._valid= True - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def close(self): - # need to be sure block creation was actually successful - and - # close() hasn't already been called - using the _valid flag - # (__del__ can be called if __init__ fails/throws) - if self._valid: - _lib_call("TOSDB_CloseBlock", self._name) - self._valid = False - - - # for convenience, no guarantee - def __del__(self): - # cleaning up can be problematic if we exit python abruptly: - # 1) need to be sure _lib_call is still around to call the C Lib with - # 2) need to be sure the DLL object is still around - if _lib_call is not None and _dll is not None: - try: - self.close() - except Exception as e: - print("WARN: block[" + self._name.decode() + "] __del__ failed " - "to call TOSDB_CloseBlock - leak possible") - print(' ', str(e), file=_stderr) - - - def __str__(self): - sio = _StringIO() # ouput buffer - tf = self.total_frame() # get the frame - count = 0 - mdict = {"col0":0} - for k in tf: # first, find the min column sizes not to truncate - val = tf[k] - if count == 0: - mdict.update({(k,len(k)) for k in val._fields}) - if len(k) > mdict["col0"]: - mdict["col0"] = len(k) - for f in val._fields: - l = len(str(getattr(val,f))) - if l > mdict[f]: - mdict[f] = l - count += 1 - count = 0 - for k in tf: # next, 'print' the formatted frame data - val = tf[k] - if count == 0: - print(" " * mdict["col0"], end=' ', file=sio) - for f in val._fields: - print(f.ljust(mdict[f]),end=' ', file=sio) - print('',file=sio) - print(k.ljust(mdict["col0"]),end=' ', file=sio) - for f in val._fields: - print(str(getattr(val,f)).ljust(mdict[f]), end=' ', file=sio) - print('',file=sio) - count += 1 - sio.seek(0) - return sio.read() - - - def _get_item_or_topic_count(self, fname): - c = _uint32_() - _lib_call("TOSDB_Get" + fname + "Count", - self._name, - _pointer(c), - arg_types=(_str_,_PTR_(_uint32_))) - return c.value - - - def _get_items_or_topics(self, fname, str_max=MAX_STR_SZ): - size = self._get_item_or_topic_count(fname) - strs = _gen_str_buffers(str_max+1, size) - pstrs = _gen_str_buffers_ptrs(strs) - - _lib_call("TOSDB_Get" + fname + "Names", - self._name, - pstrs, - size, - str_max + 1, - arg_types=(_str_, _ppchar_, _uint32_, _uint32_)) - - return list(map(_cast_cstr,pstrs)) - - - def _sync_items_topics(self): - self._items = self._get_items_or_topics("Item") - self._topics = self._get_items_or_topics("Topic") - self._items_precached = self._get_items_or_topics("PreCachedItem") - self._topics_precached = self._get_items_or_topics("PreCachedTopic") - - - def _items_topics_are_synced(self): - return self._items == self._get_items_or_topics("Item") \ - and self._topics == self._get_items_or_topics("Topic") \ - and self._items_precached == self._get_items_or_topics("PreCachedItem") \ - and self._topics_precached == self._get_items_or_topics("PreCachedTopic") - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def info(self): - return {"Name": self._name.decode('ascii'), - "Items": list(self._items), - "Topics": list(self._topics), - "ItemsPreCached": list(self._items_precached), - "TopicsPreCached": list(self._topics_precached), - "Size": self._block_size, - "DateTime": "Enabled" if self._date_time else "Disabled", - "Timeout": self._timeout} - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def is_using_datetime(self): - return self._date_time - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def get_block_size(self): - b = _uint32_() - _lib_call("TOSDB_GetBlockSize", - self._name, - _pointer(b), - arg_types=(_str_,_PTR_(_uint32_))) - return b.value - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def set_block_size(self, sz): - try: - _lib_call("TOSDB_SetBlockSize", - self._name, - sz, - arg_types=(_str_,_uint32_)) - finally: - try: - self._block_size = self.get_block_size() - except Exception as e: - raise TOSDB_Error("unable to sync block size: " + str(e)) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def items(self, str_max=MAX_STR_SZ): - return list(self._items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topics(self, str_max=MAX_STR_SZ): - return list(self._topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def items_precached(self, str_max=MAX_STR_SZ): - return list(self._items_precached) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topics_precached(self, str_max=MAX_STR_SZ): - return list(self._topics_precached) - - - def _in_block_or_precache(self, is_item, e): - return e in (self._items + self._items_precached) if is_item else \ - e in (self._topics + self._topics_precached) - - - def _add_remove(self, cname, hfunc, elems): - if not self._items_topics_are_synced(): - raise TOSDB_Error("item/topics not synced with C lib") - remove = 'Remove' in cname - fails = {} - for elem in elems: - try: - el = hfunc(elem, False) - if remove and not self._in_block_or_precache('Item' in cname, el): - fails[str(elem)] = "item not in block or precache: " + el - continue - _lib_call(cname, - self._name, - el.encode("ascii"), - arg_types=(_str_,_str_)) - except Exception as e: - fails[str(elem)] = e.args[0] - self._sync_items_topics() - if fails: - raise TOSDB_Error("error(s) adding/removing items/topics", str(fails)) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_items(self, *items): - self._add_remove('TOSDB_AddItem', self._handle_raw_item, items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_topics(self, *topics): - self._add_remove('TOSDB_AddTopic', self._handle_raw_topic, topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_items(self, *items): - self._add_remove('TOSDB_RemoveItem', self._handle_raw_item, items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_topics(self, *topics): - self._add_remove('TOSDB_RemoveTopic', self._handle_raw_topic, topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_occupancy(self, item, topic): - item = self._handle_raw_item(item) - topic = self._handle_raw_topic(topic) - occ = _uint32_() - - _lib_call("TOSDB_GetStreamOccupancy", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - _pointer(occ), - arg_types=(_str_, _str_, _str_, _PTR_(_uint32_))) - - return occ.value - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def get(self, item, topic, date_time=False, indx=0, check_indx=True, - data_str_max=STR_DATA_SZ): - - item = self._handle_raw_item(item) - topic = self._handle_raw_topic(topic) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - if indx < 0: - indx += self._block_size - if indx >= self._block_size: - raise TOSDB_IndexError("invalid index value passed to get()") - if check_indx and indx >= self.stream_occupancy(item, topic): - raise TOSDB_DataError("data not available at this index yet " + - "(disable check_indx to avoid this error)") - - tytup = _type_switch( type_bits(topic) ) - if tytup[0] == "String": - return self._get_string(item, topic, date_time, indx, data_str_max) - else: - return self._get_number(tytup, item, topic, date_time, indx) - - - def _get_string(self, item, topic, date_time, indx, data_str_max): - dt = _DateTimeStamp() - s = _BUF_(data_str_max + 1) - - _lib_call("TOSDB_GetString", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - indx, - s, - data_str_max + 1, - _pointer(dt) if date_time else _PTR_(_DateTimeStamp)(), - arg_types=(_str_, _str_, _str_, _long_, _pchar_, - _uint32_, _PTR_(_DateTimeStamp))) - - s = s.value.decode() - return (s, TOSDB_DateTime(dt)) if date_time else s - - - def _get_number(self, tytup, item, topic, date_time, indx): - dt = _DateTimeStamp() - n = tytup[1]() - - _lib_call("TOSDB_Get"+tytup[0], - self._name, - item.encode("ascii"), - topic.encode("ascii"), - indx, - _pointer(n), - _pointer(dt) if date_time else _PTR_(_DateTimeStamp)(), - arg_types=(_str_, _str_, _str_, _long_, _PTR_(tytup[1]), - _PTR_(_DateTimeStamp))) - - return (n.value, TOSDB_DateTime(dt)) if date_time else n.value - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_snapshot(self, item, topic, date_time=False, end=-1, beg=0, - smart_size=True, data_str_max=STR_DATA_SZ): - - item = self._handle_raw_item(item) - topic = self._handle_raw_topic(topic) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - if end < 0: - end += self._block_size - if beg < 0: - beg += self._block_size - size = (end - beg) + 1 - if beg < 0 \ - or end < 0 \ - or beg >= self._block_size \ - or end >= self._block_size \ - or size <= 0: - raise TOSDB_IndexError("invalid 'beg' and/or 'end' index value(s)") - - if smart_size: - so = self.stream_occupancy(item, topic) - if so == 0 or so <= beg: - return [] - end = min(end, so - 1) - beg = min(beg, so - 1) - size = (end - beg) + 1 - - tytup = _type_switch( type_bits(topic) ) - if tytup[0] == "String": - return self._stream_snapshot_strings(item, topic, date_time, end, - beg, size, data_str_max) - else: - return self._stream_snapshot_numbers(tytup, item, topic, date_time, - end, beg, size) - - - def _stream_snapshot_strings(self, item, topic, date_time, end, beg, size, - data_str_max): - strs = _gen_str_buffers(data_str_max+1, size) - pstrs = _gen_str_buffers_ptrs(strs) - dts = (_DateTimeStamp * size)() - - _lib_call("TOSDB_GetStreamSnapshotStrings", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - pstrs, - size, - data_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - end, - beg, - arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, - _PTR_(_DateTimeStamp), _long_, _long_)) - - return list(_zip_cstr_dt(pstrs,dts) if date_time else _map_cstr(pstrs)) - - - def _stream_snapshot_numbers(self, tytup, item, topic, date_time, end, beg, size): - nums = (tytup[1] * size)() - dts = (_DateTimeStamp * size)() - - _lib_call("TOSDB_GetStreamSnapshot"+tytup[0]+"s", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - nums, - size, - dts if date_time else _PTR_(_DateTimeStamp)(), - end, - beg, - arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, - _PTR_(_DateTimeStamp), _long_, _long_)) - - return list(zip(nums,_map_dt(dts)) if date_time else nums) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, - margin_of_safety=100, throw_if_data_lost=True, - data_str_max=STR_DATA_SZ): - - item = self._handle_raw_item(item) - topic = self._handle_raw_topic(topic) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - if beg < 0: - beg += self._block_size - if beg < 0 or beg >= self._block_size: - raise TOSDB_IndexError("invalid 'beg' index value") - - if margin_of_safety < MIN_MARGIN_OF_SAFETY: - raise TOSDB_ValueError("margin_of_safety < MIN_MARGIN_OF_SAFETY") - - if throw_if_data_lost: - # check this first so we don't move marker if dirty - is_dirty = _uint_() - _lib_call("TOSDB_IsMarkerDirty", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - _pointer(is_dirty), - arg_types=(_str_, _str_, _str_, _PTR_(_uint_))) - if is_dirty: - raise TOSDB_DataError("marker is already dirty") - - mpos = _longlong_() - _lib_call("TOSDB_GetMarkerPosition", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - _pointer(mpos), - arg_types=(_str_, _str_, _str_, _PTR_(_longlong_))) - - cur_sz = mpos.value - beg + 1 - if cur_sz < 0: - return None - - tytup = _type_switch( type_bits(topic) ) - if tytup[0] == "String": - return self._stream_snapshot_from_marker_strings(item, topic, - date_time, beg, cur_sz + margin_of_safety, - throw_if_data_lost, data_str_max) - else: - return self._stream_snapshot_from_marker_numbers(tytup, item, - topic, date_time, beg, cur_sz + margin_of_safety, - throw_if_data_lost) - - - def _stream_snapshot_from_marker_strings(self, item, topic, date_time, beg, - size, throw_if_data_lost, data_str_max): - strs = _gen_str_buffers(data_str_max+1, size) - pstrs = _gen_str_buffers_ptrs(strs) - dts = (_DateTimeStamp * size)() - g = _long_() - - _lib_call("TOSDB_GetStreamSnapshotStringsFromMarker", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - pstrs, - size, - data_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - beg, - _pointer(g), - arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, - _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) - - g = g.value - if g == 0: - return None - elif g < 0: - if throw_if_data_lost: - raise TOSDB_DataError("data lost behind the 'marker'") - else: - g *= -1 - - return list(_zip_cstr_dt(pstrs[:g],dts[:g]) if date_time else _map_cstr(pstrs[:g])) - - - def _stream_snapshot_from_marker_numbers(self, tytup, item, topic, date_time, beg, - size, throw_if_data_lost): - nums = (tytup[1] * size)() - dts = (_DateTimeStamp * size)() - g = _long_() - - _lib_call("TOSDB_GetStreamSnapshot" + tytup[0] + "sFromMarker", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - nums, - size, - dts if date_time else _PTR_(_DateTimeStamp)(), - beg, - _pointer(g), - arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, - _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) - - g = g.value - if g == 0: - return None - elif g < 0: - if throw_if_data_lost: - raise TOSDB_DataError("data lost behind the 'marker'") - else: - g *= -1 - - return list(zip(nums[:g], _map_dt(dts[:g])) if date_time else nums[:g]) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def n_from_marker(self, item, topic, date_time=False, n=1, - throw_if_data_lost=True, data_str_max=STR_DATA_SZ): - - item = self._handle_raw_item(item) - topic = self._handle_raw_topic(topic) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - if throw_if_data_lost: - # check this first so we don't move marker if dirty - is_dirty = _uint_() - _lib_call("TOSDB_IsMarkerDirty", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - _pointer(is_dirty), - arg_types=(_str_, _str_, _str_, _PTR_(_uint_))) - if is_dirty: - raise TOSDB_DataError("marker is already dirty") - - tytup = _type_switch( type_bits(topic) ) - if tytup[0] == "String": - return self._n_from_marker_strings(item, topic, date_time, n, - throw_if_data_lost, data_str_max) - else: - return self._n_from_marker_numbers(tytup, item, topic, date_time, - n, throw_if_data_lost) - - - def _n_from_marker_strings(self, item, topic, date_time, n, throw_if_data_lost, - data_str_max): - strs = _gen_str_buffers(data_str_max+1, n) - pstrs = _gen_str_buffers_ptrs(strs) - dts = (_DateTimeStamp * n)() - g = _long_() - - _lib_call("TOSDB_GetNStringsFromMarker", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - pstrs, - n, - data_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - _pointer(g), - arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, - _PTR_(_DateTimeStamp), _PTR_(_long_))) - - g = g.value - if g == 0: - return None - elif g < 0: - if throw_if_data_lost: - raise TOSDB_DataError("data lost behind the 'marker'") - else: - g *= -1 - - return list(_zip_cstr_dt(pstrs[:g],dts[:g]) if date_time else _map_cstr(pstrs[:g])) - - - def _n_from_marker_numbers(self, tytup, item, topic, date_time, n, - throw_if_data_lost): - nums = (tytup[1] * n)() - dts = (_DateTimeStamp * n)() - g = _long_() - - _lib_call("TOSDB_GetN" + tytup[0] + "sFromMarker", - self._name, - item.encode("ascii"), - topic.encode("ascii"), - nums, - n, - dts if date_time else _PTR_(_DateTimeStamp)(), - _pointer(g), - arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, - _PTR_(_DateTimeStamp), _PTR_(_long_))) - - g = g.value - if g == 0: - return None - elif g < 0: - if throw_if_data_lost: - raise TOSDB_DataError("data lost behind the 'marker'") - else: - g *= -1 - - return list(zip(nums[:g], _map_dt(dts[:g])) if date_time else nums[:g]) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def item_frame(self, topic, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ): - - topic = self._handle_raw_topic(topic) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - size = self._get_item_or_topic_count("Item") - tytup = _type_switch( type_bits(topic) ) - if tytup[0] is "String": - return self._item_frame_strings(topic, date_time, labels, size, - data_str_max, label_str_max) - else: - return self._item_frame_numbers(tytup, topic, date_time, labels, - size, label_str_max) - - - def _item_frame_strings(self, topic, date_time, labels, size, data_str_max, - label_str_max): - strs = _gen_str_buffers(data_str_max+1, size) - labs = _gen_str_buffers(label_str_max+1, size) - pstrs = _gen_str_buffers_ptrs(strs) - plabs = _gen_str_buffers_ptrs(labs) - dts = (_DateTimeStamp * size)() - - _lib_call("TOSDB_GetItemFrameStrings", - self._name, - topic.encode("ascii"), - pstrs, - size, - data_str_max + 1, - plabs if labels else _ppchar_(), - label_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - arg_types=(_str_, _str_, _ppchar_, _uint32_, _uint32_, - _ppchar_, _uint32_, _PTR_(_DateTimeStamp))) - - dat = list(_zip_cstr_dt(pstrs, dts) if date_time else _map_cstr(pstrs)) - if labels: - nt = _gen_namedtuple(_str_clean(topic)[0], _str_clean(*_map_cstr(plabs))) - return nt(*dat) - else: - return dat - - - def _item_frame_numbers(self, tytup, topic, date_time, labels, size, label_str_max): - nums = (tytup[1] * size)() - dts = (_DateTimeStamp * size)() - labs = _gen_str_buffers(label_str_max+1, size) - plabs = _gen_str_buffers_ptrs(labs) - - _lib_call("TOSDB_GetItemFrame"+tytup[0]+"s", - self._name, - topic.encode("ascii"), - nums, - size, - plabs if labels else _ppchar_(), - label_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - arg_types=(_str_, _str_, _PTR_(tytup[1]), _uint32_, _ppchar_, - _uint32_, _PTR_(_DateTimeStamp))) - - dat = list(zip(nums,_map_dt(dts)) if date_time else nums) - if labels: - nt = _gen_namedtuple(_str_clean(topic)[0], _str_clean(*_map_cstr(plabs))) - return nt(*dat) - else: - return dat - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def topic_frame(self, item, date_time=False, labels=True, - data_str_max=STR_DATA_SZ, label_str_max=MAX_STR_SZ): - - item = self._handle_raw_item(item) - - if date_time and not self._date_time: - raise TOSDB_DateTimeError("date_time not available for this block") - - size = self._get_item_or_topic_count("Topic") - strs = _gen_str_buffers(data_str_max+1, size) - labs = _gen_str_buffers(label_str_max+1, size) - pstrs = _gen_str_buffers_ptrs(strs) - plabs = _gen_str_buffers_ptrs(labs) - dts = (_DateTimeStamp * size)() - - _lib_call("TOSDB_GetTopicFrameStrings", - self._name, - item.encode("ascii"), - pstrs, - size, - data_str_max + 1, - plabs if labels else _ppchar_(), - label_str_max + 1, - dts if date_time else _PTR_(_DateTimeStamp)(), - arg_types=(_str_, _str_, _ppchar_, _uint32_, _uint32_, - _ppchar_, _uint32_, _PTR_(_DateTimeStamp))) - - dat = list(_zip_cstr_dt(pstrs,dts) if date_time else _map_cstr(pstrs)) - if labels: - nt = _gen_namedtuple(_str_clean(item)[0], _str_clean(*_map_cstr(plabs))) - return nt(*dat) - else: - return dat - - - # total_frame not party of abstract base, so include doc string - def total_frame(self, date_time=False, labels=True, data_str_max=STR_DATA_SZ, - label_str_max=MAX_STR_SZ): - """ Return ALL of the block's most recent values: - - total_frame(self, date_time=False, labels=True, data_str_max=STR_DATA_SZ, - label_str_max=MAX_STR_SZ) - - date_time :: bool :: include TOSDB_DateTime objects - labels :: bool :: include item and topic labels - data_str_max :: int :: maximum length of string data returned - label_str_max :: int :: maximum length of label strings returned - - if labels == date_time == True: returns -> dict of namedtuple of 2-tuple** - elif labels == True: returns -> dict of namedtuple** - elif date_time == True: returns -> list of list of 2-tuple** - else: returns -> list of list** - - **data are ALL of type str (frame can contain topics of different type) - - throws TOSDB_DataTimeError, TOSDB_CLibError - """ - p = _partial(self.topic_frame, date_time=date_time, labels=labels, - data_str_max=data_str_max, label_str_max=label_str_max) - if labels: - return {x : p(x) for x in self._items} - else: - return list(map(p,self._items)) - - - def _handle_raw(self, s): - if len(s) < 1 or len(s) > MAX_STR_SZ: - raise TOSDB_ValueError("invalid str len: " + str(len(s))) - return s.upper() - - - def _handle_raw_item(self, item, throw_if_not_in_block=True): - if type(item) is not str: - raise TOSDB_TypeError("item must be str") - - item = self._handle_raw(item) - if throw_if_not_in_block and item not in self._items: - raise TOSDB_ValueError("item '" + str(item) + "' not in block") - - return item - - - def _handle_raw_topic(self, topic, throw_if_not_in_block=True): - if topic in TOPICS: - topic = topic.val - elif type(topic) is not str: - raise TOSDB_TypeError("topic must be TOPICS enum or str") - - topic = self._handle_raw(topic) - if topic not in TOPICS.val_dict: - raise TOSDB_ValueError("invalid topic: " + topic) - - if throw_if_not_in_block and topic not in self._topics: - raise TOSDB_ValueError("topic '" + str(topic) + "' not in block") - - return topic - - -@make_block_thread_safe('__str__') -class TOSDB_ThreadSafeDataBlock(TOSDB_DataBlock): - """ The main object for storing TOS data (THREAD SAFE) - - __init__(self, size=1000, date_time=False, timeout=DEF_TIMEOUT) - - size :: int :: how much historical data can be inserted - date_time :: bool :: should block include date-time with each data-point? - timeout :: int :: how long to wait for responses from engine, TOS-DDE server, - and/or internal IPC/Concurrency mechanisms (milliseconds) - - throws TOSDB_CLibError - """ - def __init__(self, size=1000, date_time=False, timeout=DEF_TIMEOUT): - super().__init__(size, date_time, timeout) - - -### HOW WE ACCESS THE UNDERLYING C CALLS ### -def _lib_call(f, *fargs, ret_type=_int_, arg_types=None, error_check=True): - if not _dll: - raise TOSDB_CLibError("tos-databridge DLL is currently not loaded") - return # in case exc gets ignored on clean_up - ret = None - err = None - try: - attr = getattr(_dll, str(f)) - if ret_type: - attr.restype = ret_type - if isinstance(arg_types, tuple): - attr.argtypes = arg_types - elif arg_types is not None: - raise ValueError('arg_types should be tuple or None') - ret = attr(*fargs) # <- THE ACTUAL LIBRRY CALL - except BaseException as e: - err = e - - if err: - raise TOSDB_CLibError("unable to execute library function [%s]" % f, err) - elif error_check and ret: - raise TOSDB_CLibError("library function [%s] returned error code [%i,%s]" \ - % (f, ret, _lookup_error_name(ret))) - - return ret - - -def _lookup_error_name(e): - try: - return ERROR_LOOKUP[e] - except: - dbase = min(ERROR_LOOKUP.keys()) - if e < dbase: - return "ERROR_DECREMENT_BASE(" + str(e - dbase) + ")" - return '***unrecognized error code***' - - -# create a custom namedtuple with an i.d tag for special pickling -def _gen_namedtuple(name, attrs): - nt = _namedtuple(name, attrs) - setattr(nt, NTUP_TAG_ATTR, True) - return nt - - -# clean strings for namedtuple fields -def _str_clean(*strings): - fin = [] - for s in strings: - tmp = '' - if not _match(_REGEX_LETTER, s): - s = 'X_' + s - for sub in _split(_REGEX_NON_ALNUM, s): - tmp += sub - fin.append(tmp) - return fin diff --git a/python/tosdb/cli_scripts/__init__.py b/python/tosdb/cli_scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/tosdb/cli_scripts/_ohlcv_callbacks.py b/python/tosdb/cli_scripts/_ohlcv_callbacks.py deleted file mode 100644 index 8cff82e..0000000 --- a/python/tosdb/cli_scripts/_ohlcv_callbacks.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - - -TTOGGLE = {"LAST":"VOLUME", "VOLUME":"LAST"} - - -class _Basic: - def __init__(self, to_str_func, write_func): - self._to_str_func = to_str_func - self._write_func = write_func - - def callback(self, item, topic, iobj): - self._write_func(item, iobj.asctime().ljust(50)) - s = self._to_str_func(iobj) if not iobj.is_null() else 'N/A' - self._write_func(item, s + '\n') - - -class _Matcher: - def __init__(self, attrs, write_func): - self._other_interval = dict() - self._props = attrs - self._write_func = write_func - - def callback(self, item, topic, iobj): - if item not in self._other_interval: - self._other_interval[item] = {"LAST":dict(), "VOLUME":dict()} - ise = iobj.intervals_since_epoch - otopic = TTOGGLE[topic] - if ise in self._other_interval[item][otopic]: - self._write_func(item, iobj.asctime().ljust(50)) - if not iobj.is_null(): - m = self._other_interval[item][otopic][ise] - if topic == 'VOLUME': - d = tuple((getattr(m, v) for v in self._props)) + (iobj.c,) - else: - d = tuple((getattr(iobj, v) for v in self._props)) + (m.c,) - self._write_func(item, str(d) + '\n') - else: - self._write_func(item, 'N/A \n') - self._other_interval[item][otopic].pop(ise) - else: - self._other_interval[item][topic][ise] = iobj - - diff --git a/python/tosdb/cli_scripts/daemon.py b/python/tosdb/cli_scripts/daemon.py deleted file mode 100644 index 571dd66..0000000 --- a/python/tosdb/cli_scripts/daemon.py +++ /dev/null @@ -1,132 +0,0 @@ -# -# daemon.py by Sander Marechal : -# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ -# -# made a few tweaks for python3 and tab/spacing size - Jonathon Ogden - -import sys, os, time, atexit -from signal import SIGTERM - -class Daemon: - """ - A generic daemon class. - - Usage: subclass the Daemon class and override the run() method - """ - def __init__( self, pidfile, stdin='/dev/null', stdout='/dev/null', - stderr='/dev/null'): - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - self.pidfile = pidfile - - def daemonize(self): - """ - do the UNIX double-fork magic, see Stevens' "Advanced - Programming in the UNIX Environment" for details (ISBN 0201563177) - http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 - """ - try: - pid = os.fork() - if pid > 0: - # exit first parent - sys.exit(0) - except OSError as e: - message = "fork #1 failed: %d (%s)\n" - print( message % (e.errno, e.strerror), file=sys.stderr) - sys.exit(1) - - # decouple from parent environment - os.chdir("/") - os.setsid() - os.umask(0) - - # do second fork - try: - pid = os.fork() - if pid > 0: - # exit from second parent - sys.exit(0) - except OSError as e: - print( "fork #2 failed: %d (%s)\n" % (e.errno, e.strerror), - file=sys.stderr ) - sys.exit(1) - - #redirect standard file descriptors - sys.stdout.flush() - sys.stderr.flush() - si = open(self.stdin, 'r') - so = open(self.stdout, 'a+') - se = open(self.stderr, 'a+', 1) - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) - - # write pidfile - atexit.register(self.delpid) - pid = str(os.getpid()) - open(self.pidfile,'w+').write("%s\n" % pid) - - def delpid(self): - os.remove(self.pidfile) - - def start(self): - """ - Start the daemon - """ - # Check for a pidfile to see if the daemon already runs - try: - pf = open(self.pidfile,'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if pid: - message = "pidfile %s already exist. Daemon already running?\n" - print(message % str(self.pidfile), file=sys.stderr) - sys.exit(1) - - # Start the daemon - self.daemonize() - self.run() - - def stop(self): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pf = open(self.pidfile,'r') - pid = int(pf.read().strip()) - pf.close() - except IOError: - pid = None - - if not pid: - message = "pidfile %s does not exist. Daemon not running?\n" - print(message % str(self.pidfile), file=sys.stderr) - return # not an error in a restart - - # Try killing the daemon process - try: - while 1: - os.kill(pid, SIGTERM) - time.sleep(0.1) - except OSError as e: - e = str(e) - if "No such process" in e: - if os.path.exists(self.pidfile): - os.remove(self.pidfile) - else: - print(e) - sys.exit(1) - - def restart(self): - """ - Restart the daemon - """ - self.stop() - self.start() - - diff --git a/python/tosdb/cli_scripts/ohlcv_daemon.py b/python/tosdb/cli_scripts/ohlcv_daemon.py deleted file mode 100644 index 115f2b3..0000000 --- a/python/tosdb/cli_scripts/ohlcv_daemon.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - - -import tosdb -from tosdb.intervalize.ohlc import TOSDB_OpenHighLowCloseIntervals as OHLCIntervals, \ - TOSDB_CloseIntervals as CIntervals - -from tosdb.cli_scripts.daemon import Daemon as _Daemon -from tosdb.cli_scripts import _ohlcv_callbacks - -from argparse import ArgumentParser as _ArgumentParser -from time import localtime as _localtime, strftime as _strftime, sleep as _sleep -from os.path import realpath as _path -from sys import stderr as _stderr - -AINIT_TIMEOUT = 5000 -BLOCK_SIZE = 1000 - - -class MyDaemon(_Daemon): - def __init__(self, addr, auth, outdir, pidfile, errorfile, interval, itype, symbols): - _Daemon.__init__(self, pidfile, stderr = errorfile) - self._addr = addr - self._auth = auth - self._outdir = _path(outdir) - self._interval = interval - self._has_vol = 'V' in itype - self._is_ohlc = 'OHLC' in itype - self._symbols = symbols - self._iobj = None - # date prefix for filename - dprfx = _strftime("%Y%m%d", _localtime()) - # generate paths from filenames - self._paths = {s.upper() : (_path(self._outdir) + '/' + dprfx + '_' \ - + s.replace('/','-S-').replace('$','-D-').replace('.','-P-') \ - + '_' + itype + '_' + str(self._interval) + 'sec.tosdb') \ - for s in self._symbols} - # create callback object - if self._has_vol: - self._callback = _ohlcv_callbacks._Matcher('ohlc' if self._is_ohlc else 'c', self._write) - else: - l = (lambda o: str((o.o, o.h, o.l, o.c))) if self._is_ohlc else (lambda o: str(o.c)) - self._callback = _ohlcv_callbacks._Basic(l, self._write) - - - def run(self): - # create block - blk = tosdb.VTOSDB_ThreadSafeDataBlock(self._addr, self._auth, BLOCK_SIZE, date_time=True) - blk.add_items(*(self._symbols)) - blk.add_topics('last') - if self._has_vol: - blk.add_topics('volume') - - # create interval object - IObj = OHLCIntervals if self._is_ohlc else CIntervals - self._iobj = IObj(blk, self._interval, interval_cb=self._callback.callback) - try: - while self._iobj.running(): - _sleep(1) - except: - self._iobj.stop() - finally: - blk.close() - - - def _write(self, item, s): - with open(self._paths[item], 'a') as f: - f.write(s) - - -if __name__ == '__main__': - parser = _ArgumentParser() - parser.add_argument('addr', type=str, - help = 'address of the host system "address port"') - parser.add_argument('--root', help='root directory to search for the library (on host)') - parser.add_argument('--path', help='the exact path of the library (on host)') - parser.add_argument('--auth', help='password to use if authentication required') - parser.add_argument('outdir', type=str, help = 'directory to output data to') - parser.add_argument('pidfile', type=str, help = 'path of pid file') - parser.add_argument('errorfile', type=str, help = 'path of error file') - parser.add_argument('interval', type=int, help="interval size in seconds") - parser.add_argument('--ohlc', action="store_true", - help="use open/high/low/close instead of close") - parser.add_argument('--vol', action="store_true", help="use volume") - parser.add_argument('symbols', nargs='*', help="symbols to pull") - args = parser.parse_args() - - addr_parts = args.addr.split(' ') - addr = (addr_parts[0],int(addr_parts[1])) - - if args.ohlc: - itype = 'OHLCV' if args.vol else 'OHLC' - else: - itype = 'CV' if args.vol else 'C' - - if not args.path and not args.root: - print("need --root or --path argument", file=_stderr) - exit(1) - - # connect - tosdb.admin_init(addr, password=args.auth, timeout=AINIT_TIMEOUT) - tosdb.vinit(args.path, args.root) - - MyDaemon(addr, args.auth, args.outdir, args.pidfile, args.errorfile, - args.interval, itype, args.symbols).start() - - - - diff --git a/python/tosdb/cli_scripts/ohlcv_wintask.py b/python/tosdb/cli_scripts/ohlcv_wintask.py deleted file mode 100644 index 342cc96..0000000 --- a/python/tosdb/cli_scripts/ohlcv_wintask.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -import tosdb -from tosdb.intervalize.ohlc import TOSDB_OpenHighLowCloseIntervals as OHLCIntervals, \ - TOSDB_CloseIntervals as CIntervals - -from tosdb.cli_scripts import _ohlcv_callbacks - -from argparse import ArgumentParser as _ArgumentParser -from time import localtime as _localtime, strftime as _strftime, sleep as _sleep -from os.path import realpath as _path -from sys import stderr as _stderr - -BLOCK_SIZE = 1000 -_paths = dict() - -def spawn(outdir, interval, is_ohlc, has_vol, *symbols): - global _paths - - itype = ("OHLC" if is_ohlc else "C") + ("V" if has_vol else "") - # date prefix for filename - dprfx = _strftime("%Y%m%d", _localtime()) - # generate paths from filenames - _paths = {s.upper() : (_path(outdir) + '/' + dprfx + '_' \ - + s.replace('/','-S-').replace('$','-D-').replace('.','-P-') \ - + '_' + itype + '_' + str(interval) + 'sec.tosdb') - for s in symbols} - - # create callback object - if has_vol: - callback = _ohlcv_callbacks._Matcher('ohlc' if is_ohlc else 'c', _write) - else: - l = (lambda o: str((o.o, o.h, o.l, o.c))) if is_ohlc else (lambda o: str(o.c)) - callback = _ohlcv_callbacks._Basic(l, _write) - - # create block - blk = tosdb.TOSDB_ThreadSafeDataBlock(BLOCK_SIZE, date_time=True) - blk.add_items(*(symbols)) - blk.add_topics('last') - if has_vol: - blk.add_topics('volume') - - # create interval object - IObj = OHLCIntervals if is_ohlc else CIntervals - iobj = IObj(blk, interval, interval_cb=callback.callback) - try: - while iobj.running(): - _sleep(1) - except: - iobj.stop() - finally: - blk.close() - - -def _write(item, s): - with open(_paths[item], 'a') as f: - f.write(s) - - -if __name__ == '__main__': - parser = _ArgumentParser() - parser.add_argument('--root', help='root directory to search for the library') - parser.add_argument('--path', help='the exact path of the library') - parser.add_argument('outdir', type=str, help='directory to output data to') - parser.add_argument('interval', type=int, help="interval size in seconds") - parser.add_argument('--ohlc', action="store_true", - help="use open/high/low/close instead of close") - parser.add_argument('--vol', action="store_true", help="use volume") - parser.add_argument('symbols', nargs='*', help="symbols to pull") - args = parser.parse_args() - - if not args.path and not args.root: - print("need --root or --path argument", file=_stderr) - exit(1) - - # connect - tosdb.init(args.path,args.root) - spawn(args.outdir, args.interval, bool(args.ohlc), bool(args.vol), *args.symbols) - - - - - - diff --git a/python/tosdb/doxtend.py b/python/tosdb/doxtend.py deleted file mode 100644 index 7004fba..0000000 --- a/python/tosdb/doxtend.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -from inspect import isfunction as _isfunction - -class DoxtendError(Exception): - def __init__(self, *msg): - super().__init__(*msg) - -def doxtend(*bases, func_name=None, sep='\n\n'): - """ Inherit doc-string(s) from other functions/methods. - - bases: base objects to search in, 'None' looks in globals() - func_name: alternative name to search for(defaults to name of decorated function) - sep: str to place between inherited docstring - - Attempts to order doc-strings in a cummulative/hierarchical fashion - moving - left to right through 'bases' - excluding the same string(s) from different - objects(except for the final (non)derived doc-string which is guaranteed to - be included last)" - """ - def doxtend_decorator(func): - nonlocal bases, func_name - uniq_doc_strs = [] - if func_name is None: - func_name = func.__name__ - for bc in bases: - try: - f = globals()[func_name] if bc is None else getattr(bc,func_name) - except (KeyError, AttributeError): - raise DoxtendError("couldn't find function '" + func_name + "'") - if f.__doc__ not in uniq_doc_strs: - uniq_doc_strs.append(f.__doc__) - if uniq_doc_strs: - if func.__doc__: - uniq_doc_strs.append(func.__doc__) - func.__doc__ = sep.join(uniq_doc_strs) - return func - ### doxtend_decorator() ### - - if bases: - if _isfunction(bases[0]): - f = bases[0] - bases = (None,) - return doxtend_decorator(f) - else: - return doxtend_decorator - else: - bases = (None,) - return doxtend_decorator - - - -def _test(): - def f(): - """ (1, 5) global f() docstring """ - pass - - globals()['f'] = f - - class B: - def f(self): - """ B.f() docstring """ - pass - def f2(self): - """ (3) B.f2() docstring """ - pass - - class Z(B): - @doxtend(B,func_name='f2',sep="\n seperator between B.f2() and Z.f() \n") - def f(self): - """ (4) Z.f() docstring """ - pass - - class D(B): - @doxtend # <- looking in global by default - def f(self): - """ (2) D.f() docstring """ - pass - - class DD(D): - @doxtend(D,Z,None,sep="\n seperator between D.f() / Z.f() / f() and DD.f() \n") - def f(self): - """ (6) DD.f() docstring """ - pass - - print('DD.f.__doc__:\n\n') - print(DD.f.__doc__) - globals().pop('f') diff --git a/python/tosdb/intervalize/__init__.py b/python/tosdb/intervalize/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/tosdb/intervalize/constant_value.py b/python/tosdb/intervalize/constant_value.py deleted file mode 100644 index 69e6dc0..0000000 --- a/python/tosdb/intervalize/constant_value.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -"""intervalize/constant_value.py: structure streaming tosdb data in constant-value intervals - -*** IN DEVELOPMENT *** - -Constant value intervals break data up based on some constant value being reached during -the interval. For instance, constant volume intervals create a new interval each time -the underlying volume of the item reaches a certain fixed amount for that interval: - -tradional fixed-time interval: - -last: 22 23 21 22 23 24 27 28 30 -bid: 21 23 21 21 22 23 26 28 29 -volume: 700 1300 900 200 400 500 0 100 2000 -index: 1 2 3 4 5 6 7 8 9 - -2000 share constant-volume interval: - -last: 23 24 30 -bid: 23 23 29 -volume: 2000 2000 2000 -index: 1 2 3 - -*** IN DEVELOPMENT *** -""" diff --git a/python/tosdb/intervalize/ohlc.py b/python/tosdb/intervalize/ohlc.py deleted file mode 100644 index b850321..0000000 --- a/python/tosdb/intervalize/ohlc.py +++ /dev/null @@ -1,758 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -"""intervalize/ohlc.py: structure streaming tosdb data in fixed-time intervals - -*** IN DEVELOPMENT *** - -TOSDB_FixedTimeIntervals: - base class that pulls, transforms and stores numerical data of a certain - fixed-time(e.g 30 seconds) interval from a thread-safe DataBlock - -TOSDB_OpenHighLowCloseIntervals: - extends TOSDB_FixedTimeIntervals using 'OHLC' fixed-time interval objects - -TOSDB_CloseIntervals: - extends TOSDB_FixedTimeIntervals using 'C' fixed-time interval objects - -OHLC: - object that stores open, high, low, close of a fixed-time interval along - with datetime info - -C: - object that stores close of a fixed-time interval along with datetime info - -NULL: - base object that stores no data of a fixed-time interval but does include - datetime info -""" - -import tosdb -from tosdb import TOPICS, TOSDB_Error -from tosdb.doxtend import doxtend as _doxtend -from tosdb._common import _TOSDB_DataBlock - -from threading import Thread as _Thread, Lock as _Lock -from types import MethodType as _MethodType -from collections import deque as _Deque -from concurrent import futures as _futures -from time import time as _time, sleep as _sleep, localtime as _localtime, \ - asctime as _asctime, struct_time as _struct_time, mktime as _mktime, \ - strftime as _strftime, perf_counter as _perf_counter -from itertools import groupby as _groupby -from inspect import signature as _signature - -import traceback as _traceback -import time - -try: - type_bits = tosdb.type_bits -except AttributeError: - type_bits = tosdb.vtype_bits - - -class NULL: - DATETIME_FIELDS = ('second','minute','hour','day','month','year') - - __slots__ = ('_intervals_since_epoch','_isec','_tfunc') - - def __init__(self, intervals_since_epoch, interval_seconds, - time_func=time.localtime): - self._intervals_since_epoch = intervals_since_epoch - self._isec = interval_seconds - if not callable(time_func): - raise TOSDB_Error("time_func not callable") - self._tfunc = time_func - - @classmethod - def is_null(cls): - return cls is NULL - - second = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_sec) - minute = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_min) - hour = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_hour) - day = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_mday) - month = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_mon) - year = property(lambda s: s._tfunc(s._intervals_since_epoch * s._isec).tm_year) - datetime = property(lambda s: (s.second, s.minute, s.hour, s.day, s.month, s.year)) - intervals_since_epoch = property(lambda s: int(s._intervals_since_epoch)) - interval_seconds = property(lambda s: int(s._isec)) - - def strftime(self, frmt): - return _strftime(frmt,self._tfunc(self._intervals_since_epoch * self._isec)) - - def asctime(self): - return _asctime(self._tfunc(self._intervals_since_epoch * self._isec)) - - def struct_time(self): - return _struct_time(self._tfunc(self._intervals_since_epoch * self._isec)) - - def as_dict(self): - return dict( zip(OHLC.DATETIME_FIELDS, self.as_tuple()) ) - - def as_tuple(self): - return self.datetime - - def __str__(self): - labels = OHLC.DATETIME_FIELDS - return str(tuple(a + "=" + str(v) for a,v in zip(labels, self.as_tuple()))) - - -class C(NULL): - __slots__ = ('_c','_ticks') - - def __init__(self, dat, intervals_since_epoch, interval_seconds, - time_func=time.localtime): - self._c = dat[-1] - self._ticks = len(dat) - super().__init__(intervals_since_epoch, interval_seconds, time_func) - - c = property(lambda s: s._c) - ticks = property(lambda s: s._ticks) - - def update(self, dat): - self._c = dat[-1] - self._ticks += len(dat) - - def as_dict(self): - d = {'c':self._c, 'ticks':self._ticks} - d.update( super().as_dict() ) - return d - - def as_tuple(self): - return (self._c,) + super().as_tuple() + (self._ticks,) - - def __str__(self): - labels = ('c',) + OHLC.DATETIME_FIELDS + ('ticks',) - return str(tuple(a + "=" + str(v) for a,v in zip(labels, self.as_tuple()))) - - -class OHLC(C): - __slots__ = ('_o', '_h', '_l') - - def __init__(self, dat, intervals_since_epoch, interval_seconds, - time_func=time.localtime): - super().__init__(dat, intervals_since_epoch, interval_seconds, time_func) - self._o = dat[0] - self._h = max(dat) - self._l = min(dat) - - o = property(lambda s: s._o) - h = property(lambda s: s._h) - l = property(lambda s: s._l) - - def update(self, dat): - self._h = max(self._h, max(dat)) - self._l = min(self._l, min(dat)) - super().update(dat) - - def as_dict(self): - d = {'o':self._o, 'h':self._h, 'l':self._l} - d.update( super().as_dict() ) - return d - - def as_tuple(self): - return (self._o, self._h, self._l) + super().as_tuple() - - def __str__(self): - labels = ('o','h','l','c') + self.DATETIME_FIELDS + ('ticks',) - return str(tuple(a + "=" + str(v) for a,v in zip(labels, self.as_tuple()))) - - -class TOSDB_FixedTimeIntervals: - """Base object that pulls data from a thread-safe DataBlock in fixed-time intervals - - This class takes streaming data from a 'DataBlock' object (_win.py/__init__.py), - breaks it up along fixed-time intervals, transforms the data into instances of a - custom object, and stores those instance in internal buffers that can be accessed - via get(...) and stream_snapshot(...) methods. - - 1) Create a thread-safe 'DataBlock' (e.g tosdb.TOSDB_ThreadSafeDataBlock) - - 2) Add 'items' and 'topics' directly through the 'DataBlock' interface, or - use the methods provided. 'topics' that return string data will be ignored - (e.g DESCRIPTION, LASTX) - - 3) Define a custom class that extends 'NULL' to represent data in each interval - (see OHLC and C above). - i) its constructor should have the following prototype: - def __init__(self, data, intervals_since_epoch, interval_seconds, time_func): - super().__init__(self, intervals_since_epoch, interval_seconds, time_func) - # handle 'data' - ii) it should define an 'update' method to handle new data coming in: - def update(self, data): - # handle 'data' - - 4) Create a TOSDB_FixedTimeIntervals object with: - i) the block we created in #1 - ii) the class we created in #3 (the actuall class object, not an instance) - iii) the length of the interval in seconds (>= 10, <= 14,400) - iv) (optional) how often each retrieval/collection operations should occur - (i.e interval_sec=30, poll_sec=1 means 30 operations per interval) - v) (optional) custom time function to translate time data stored in - interval_object from seconds-from-epoch to time.struct_time - vi) (optional) callback function that will pass the item, topic, and - last completed interval as its arguments - - 5) Use methods to access stored data: - get(...) - get_by_datetime(...) - stream_snapshot(...) - stream_snapshot_between_datetimes(...) - - 6) WHEN DONE: call stop() method. The 'DataBlock' object you passed into the - constructor is still valid and in the same state. - - __init__(self, block, interval_obj, interval_sec, poll_sec=1, - interval_cb=None, time_func=time.localtime) - - block :: object :: object that exposes certain _TOSDB_DataBlock methods - interval_obj :: class :: class object used to stored data over a fixed-time interval - interval_sec :: int :: size of fixed-time interval in seconds - poll_sec :: int :: time in seconds of each data retrieval/collection operation - interval_cb :: func :: callback for each 'completed' interval - (def func(item, topic, interval_obj instance)) - time_func :: func :: custom time function that returns time.struct_time - - throws TOSDB_Error - """ - - MIN_INTERVAL_SEC = 10 - MAX_INTERVAL_SEC = 60 * 60 * 4 # 4 hours - MIN_POLL_SEC = .10 - BLOCK_SIZE_PER_PSEC = 10 - WAIT_ADJ_DOWN = .999 - WAIT_ADJ_THRESHOLD = .01 - WAIT_ADJ_THRESHOLD_FATAL = .10 - BLOCK_ATTR = ['is_thread_safe', 'stream_snapshot_from_marker', 'topics', 'items', - 'add_items', 'add_topics', 'remove_items', 'remove_topics'] - - def __init__(self, block, interval_obj, interval_sec, poll_sec=1, - interval_cb=None, time_func=time.localtime): - self._rflag = False - self._check_params(interval_obj, interval_sec, poll_sec, interval_cb, time_func) - self._iobj = interval_obj - self._isec = interval_sec - self._psec = round(interval_sec / round(interval_sec/poll_sec), 5) - self._min_block_size = self.BLOCK_SIZE_PER_PSEC * poll_sec - self._check_block(block) - self._block = block - self._texcludes = set() - self._ssfunc = block.stream_snapshot_from_marker - self._restrict_block_resize(block, poll_sec) - self._tfunc = time_func - self._interval_cb = interval_cb - self._wait_adj_down_exp = 0 - self._buffers = dict() - self._buffers_lock = _Lock() - self._bthread = _Thread(target=self._background_worker) - self._bthread.start() - - - def __del__(self): - if self._rflag: - self.stop() - - - def running(self): - return self._rflag - - - def stop(self): - """ Stop retrieving data from the underlying 'DataBlock' """ - self._rflag = False - self._bthread.join() - if hasattr(self._block, '_old_set_block_size'): - self._block.set_block_size = self._block._old_set_block_size - delattr(self._block, '_old_set_block_size') - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_items(*items): - with self._buffers_lock: - self._block.add_items(*items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def add_topics(*topics): - for t in topics: - if type_bits(t) & tosdb.STRING_BIT: - raise TOSDB_Error("topic %s returns string data" % t) - with self._buffers_lock: - self._block.add_topics(*topics) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_items(*items): - with self._buffers_lock: - self._block.remove_items(*items) - - - @_doxtend(_TOSDB_DataBlock) # __doc__ from ABC _TOSDB_DataBlock - def remove_topics(*topics): - with self._buffers_lock: - self._block.remove_topics(*topics) - - - def occupancy(self, item, topic): - """ Returns the current number of interval objects being held - - stream_occupancy(self, item, topic) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - - returns -> int - """ - with self._buffers_lock: - return len(self._get_buffer_deque(topic,item)) - - - def get(self, item, topic, indx=0): - """ Return a single interval object - - get(self, item, topic, indx=0) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - indx :: int :: index/position of interval to get - """ - with self._buffers_lock: - return self._get_buffer_deque(topic,item)[indx] - - - def get_by_datetime(self, item, topic, datetime): - """ Return a single interval object by its datetime tuple - - get(self, item, topic, datetime) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - datetime :: int :: datetime tuple of the interval object - (second, minute, hour, day, month, year) - - throws TOSDB_Error if interval object doesn't exist at that datetime - """ - with self._buffers_lock: - b = self._get_buffer_deque(topic,item) - i = self._intervals_since_epoch(*datetime) - off = b[0].intervals_since_epoch - i - if off < 0: - raise TOSDB_Error("OHLC does not exist at this datetime") - try: - return b[off] - except IndexError: - raise TOSDB_Error("OHLC does not exist at this datetime") - - - def stream_snapshot(self, item, topic, end=-1, beg=0): - """ Return multiple interval objects - - stream_snapshot(self, item, topic, end=-1, beg=0) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - end :: int :: index/position of least recent interval to get - beg :: int :: index/position of most recent interval to get - """ - with self._buffers_lock: - return list(self._get_buffer_deque(topic,item))[beg:end] - - - # INCLUSIVE INDEXING ! - def stream_snapshot_between_datetimes(self, item, topic, end_datetime=tuple(), - beg_datetime=tuple()): - """ Return multiple interval objects between datetime tuples - - stream_snapshot(self, item, topic, end_datetime=tuple(), beg_datetime=tuple()) - - item :: str :: any item string in the block - topic :: str :: any topic string in the block - end_datetime :: int :: datetime tuple of least recent interval to get - (second, minute, hour, day, month, year) - beg_datetime :: int :: datetime tuple of most recent interval to get - (second, minute, hour, day, month, year) - - throws TOSDB_Error if interval objects don't exist between the datetime tuples - """ - with self._buffers_lock: - b = self._get_buffer_deque(topic,item) - if beg_datetime: - i_beg = self._intervals_since_epoch(*beg_datetime) - off_beg = b[0].intervals_since_epoch - i_beg - if off_beg < 0: - raise TOSDB_Error("'beg' OHLC does not exist at this datetime") - else: - off_beg = 0 - if end_datetime: - i_end = self._intervals_since_epoch(*end_datetime) - off_end = b[0].intervals_since_epoch - i_end - if off_end < 0: - raise TOSDB_Error("'end' OHLC does not exist at this datetime") - off_end += 1 # make range incluseive - else: - off_end=-1 - try: - return list(b)[off_beg:off_end] - except IndexError: - raise TOSDB_Error("OHLCs do not exist between these datetimes") - - class _Buffer: - def __init__(self): - self.deque = _Deque() - self.reset() - def incr(self): - self.count += 1 - def reset(self): - self.count = 0 - - def _get_buffer_object(self, topic, item): - return self._buffers[topic.upper(),item.upper()] - - def _get_buffer_deque(self, topic, item): - return self._get_buffer_object(topic,item).deque - - def _callback(self, item, topic, d, i=1): - if self._interval_cb: - self._interval_cb(item, topic, d[i]) - - def _background_worker(self): - try: - ni = self._isec / self._psec - itbeg = _perf_counter() - count = 0 - self._rflag = True - while self._rflag: - tbeg = _perf_counter() - with self._buffers_lock: - self._manage_buffers() - for (t,i), b in self._buffers.items(): - b.incr() - dat = self._ssfunc(i, t, date_time=True, throw_if_data_lost=False) - if dat: - self._parse_data(t,i,dat) - if b.count == 0: - continue - if (b.count % ni) == 0: - #print("b.count mod interval: %i" % b.count) - self._handle_null_interval(t, i, b) - count += 1 - tend = _perf_counter() - trem = self._psec - (tend - tbeg) - if trem < 0: - ## TODO :: this will create problems handling nulls as we wont be able - ## to speed up by using WAIT_ADJ_DOWN (below) - ## considering adjusting _psec - print("WARN: _background_worker taking longer than _psec (%i) seconds" - % self._psec) - _sleep( max(trem,0) * (self.WAIT_ADJ_DOWN ** self._wait_adj_down_exp) ) - if (count % ni) == 0: - self._tune_background_worker(count,ni,itbeg) - finally: - self._rflag = False - - - def _tune_background_worker(self, count, ni, itbeg): - tnow = _perf_counter() - adjbeg = itbeg + (count // ni) * self._isec - terr = tnow - adjbeg - #print("DEBUG RUNNING ERROR SECONDS: ", str(terr)) - if terr < 0: - #print("DEBUG RUNNING ERROR SLEEP: ", str(terr)) - if self._wait_adj_down_exp > 0: - self._wait_adj_down_exp -= 1 - _sleep(abs(terr)) - elif abs(terr) > (self._isec * self.WAIT_ADJ_THRESHOLD_FATAL): - TOSDB_Error("_background worker entered fatal wait/sync states: %f, %i" \ - % (terr, self._wait_adj_down_exp)) - elif terr > (self._isec * self.WAIT_ADJ_THRESHOLD): - self._wait_adj_down_exp += 1 - #print("DEBUG RUNNING ERROR SPEED UP: %f, %i" % (terr,self._wait_adj_down_exp)) - - - def _handle_null_interval(self, t, i, b): - ei = b.deque[0].intervals_since_epoch + 1 - obj = NULL(ei, self._isec, self._tfunc) - b.deque.appendleft(obj) - self._callback(i,t,b.deque) # callback with penultimate interval - #print("DEBUG WORK NULL: %s %i" % (str((t,i)),ei)) - - - def _parse_data(self, topic, item, dat): - b = self._get_buffer_object(topic,item) - ei_old = int(dat[-1][1].mktime // self._isec) - ei_new = ei_old + 1 # making big assumption here - fi = 0 - for d in dat: - ei_last = int(d[1].mktime // self._isec) - if ei_last == ei_old: - break - fi += 1 - vodat = [d[0] for d in dat[fi:]] - vndat = [d[0] for d in dat[:fi]] - if vodat: - self._insert_data(ei_old, vodat, b, True, topic, item) - if vndat: - self._insert_data(ei_new, vndat, b, False, topic, item) - - - def _insert_data(self, ei, vdat, b, old, topic, item): - ei_current = b.deque[0].intervals_since_epoch - tstr = "old" if old else "new" - if ei > ei_current: - #print("VDAT (%s > current) %i (%i) %i" % (tstr, ei, len(vdat), ei_current)) - obj = self._iobj(vdat, ei, self._isec, self._tfunc) - b.deque.appendleft(obj) - b.reset() - self._callback(item, topic, b.deque) # callback with penultimate interval - else: - #print("VDAT (%s <= current) %i (%i) %i" % (tstr, ei, len(vdat), ei_current)) - for i in range(len(b.deque)): - if b.deque[i].intervals_since_epoch == ei: - if type(b.deque[i]) is NULL: - b.deque[i] = self._iobj(vdat, ei, self._isec, self._tfunc) - else: - b.deque[i].update(vdat) - return - self._warn_on_stranded_data(ei, b.deque, tstr, topic, item) - - - def _warn_on_stranded_data(self, ei, d, tstr, topic, item): - dbeg = d[0].intervals_since_epoch if (len(d) > 0) else 0 - dend = d[-1].intervals_since_epoch if (len(d) > 1) else 0 - print("WARN: can not find place for %s data: %s, %s, %i, (%i, %i)" \ - % (tstr, item, topic, ei, dbeg, dend) ) - - - def _manage_buffers(self): - old_buffers = dict(self._buffers) - new_buffer_ids = [] - self._buffers.clear() - items = self._block.items() - for t in self._block.topics(): - if t in self._texcludes: - continue - elif type_bits(t) & tosdb.STRING_BIT: - print("WARN: topic %s returns string data - excluding" % t) - self._texcludes.add(t) - continue - for i in items: - if (t,i) not in old_buffers: - new_buffer_ids.append((t,i)) - self._buffers[(t,i)] = old_buffers.get((t,i),self._Buffer()) - if new_buffer_ids: - self._init_buffers_async(new_buffer_ids) - - - def _init_buffers_async(self, new_buffer_ids): - with _futures.ThreadPoolExecutor(max_workers = len(new_buffer_ids)) as e: - futs = {e.submit(self._init_buffer,t,i):(t,i) for t,i in new_buffer_ids} - for f in _futures.as_completed(futs): - t, i = futs[f] - dat = f.result() - laste = 0 - b = self._get_buffer_deque(t,i) - cb_count = 0 - # does groupby guarantee sorted ?? - for e, grp in _groupby(dat, lambda t: int(t[1].mktime // self._isec)): - # fill in gaps - while laste and (e - laste > 1): - obj = NULL(laste + 1, self._isec, self._tfunc) - b.append(obj) - cb_count += 1 - laste += 1 - d = [i[0] for i in grp] - obj = self._iobj(d, e, self._isec, self._tfunc) - b.append(obj) - cb_count += 1 - laste = e - # we sort new to old so need do all callbacks after appends - for n in range(cb_count-1,0,-1): - self._callback(i,t,b,n) - - - def _init_buffer(self, topic, item): - for _ in range(int(self._isec / self._psec)): - dat = self._ssfunc(item, topic, date_time=True, throw_if_data_lost=False) - if dat: - return dat - _sleep(self._psec) - raise TOSDB_Error("failed to get any data for first interval: %s, %s" % (topic,item)) - - - def _intervals_since_epoch(self, second, minute, hour, day, month, year): - is_dst = self._tfunc().tm_isdst - sfe = _mktime(_struct_time([year,month,day,hour,minute,second,0,0,is_dst])) - if sfe % self._isec: - raise TOSDB_Error("invalid interval constructed") - return int(sfe // self._isec) - - - def _restrict_block_resize(self, block, poll_sec): - block._old_set_block_size = block.set_block_size - @_doxtend(type(block), func_name='set_block_size') - def _inj_set_block_size(instance, sz): - if sz < self._min_block_size: - raise TOSDB_Error("setting a block size < %i has been restricted " - "by TOSDB_FixedTimeIntervals" % self._min_block_size) - block._old_set_block_size(sz) - block.set_block_size = _MethodType(_inj_set_block_size, block) - - - def _check_block(self, block): - for a in TOSDB_FixedTimeIntervals.BLOCK_ATTR: - if not hasattr(block, a): - raise TOSDB_Error("'block' does not have attr '%s'" % a) - if not block.is_thread_safe(): - raise TOSDB_Error("'block' is not thread safe") - if not block.is_using_datetime(): - raise TOSDB_Error("'block' is not using datetime") - if block.get_block_size() < self._min_block_size: - print("WARN: block is too small, resize to %i" % self._min_block_size) - block.set_block_size(self._min_block_size) - - - @classmethod - def _check_params(cls, interval_obj, interval_sec, poll_sec, interval_cb, time_func): - if not hasattr(interval_obj, 'update'): - raise TOSDB_Error("'interval obj' has no 'update' function/attribute") - try: - test = interval_obj([0], 30, 1) - except Exception as e: - raise TOSDB_Error("could not instantiate 'interval_obj': %s" % str(e)) - if interval_sec < cls.MIN_INTERVAL_SEC: - raise TOSDB_Error("interval_sec(%i) < MIN_INTERVAL_SEC(%i)" \ - % (interval_sec, cls.MIN_INTERVAL_SEC) ) - if interval_sec > cls.MAX_INTERVAL_SEC: - raise TOSDB_Error("interval_sec(%i) > MAX_INTERVAL_SEC(%i)" \ - % (interval_sec, cls.MAX_INTERVAL_SEC) ) - if interval_sec < (2*poll_sec): - raise TOSDB_Error("interval_sec(%i) < 2 * poll_sec(%i)" \ - % (interval_sec, poll_sec)) - if poll_sec < cls.MIN_POLL_SEC: - raise TOSDB_Error("poll_sec(%i) < MIN_POLL_SEC(%i)" \ - % (poll_sec, cls.MIN_POLL_SEC) ) - if interval_cb is not None: - if not callable(interval_cb): - raise TOSDB_Error("'interval_cb' not callable") - if len(_signature(interval_cb).parameters) != 3: - raise TOSDB_Error("'interval_cb' signature requires 3 paramaters") - if not callable(time_func): - raise TOSDB_Error("'time_func' not callable") - - -class TOSDB_OpenHighLowCloseIntervals(TOSDB_FixedTimeIntervals): - """Pulls data from a thread-safe DataBlock in fixed-time ohlc.OHLC intervals - - This class takes streaming data from a 'DataBlock' object (_win.py/__init__.py), - breaks them up along fixed-time intervals, transforms the data into an 'OHLC' object, - and stores those objects in internal buffers that can be accessed via get(...) - and stream_snapshot(...) methods. - - 1) Create a thread-safe 'DataBlock' (e.g tosdb.TOSDB_ThreadSafeDataBlock) - - 2) Add 'items' and 'topics' directly through the 'DataBlock' interface, or - use the methods provided - i) 'topics' that return string data will be ignored (e.g DESCRIPTION, LASTX) - - 3) Create a TOSDB_OpenHighLowCloseIntervals object with: - i) the block we created in #1 - ii) the length of the interval in seconds (>= 10, <= 14,400) - iii) (optional) how often each retrieval/collection operations should occur - (i.e interval_sec=30, poll_sec=1 means 30 operations per interval) - iv) (optional) custom time function to translate time data stored in - interval_object from seconds-from-epoch to time.struct_time - vi) (optional) callback function that will pass the item, topic, and - last completed interval as its arguments - - 5) Use methods to access stored data: - get(...) - get_by_datetime(...) - stream_snapshot(...) - stream_snapshot_between_datetimes(...) - - 6) WHEN DONE: call stop() method. The 'DataBlock' object you passed into the - constructor is still valid and in the same state. - - __init__(self, block, interval_sec, poll_sec=1, interval_cb=None, - time_func=time.localtime) - - block :: object :: object that exposes certain _TOSDB_DataBlock methods - interval_sec :: int :: size of fixed-time interval in seconds - poll_sec :: int :: time in seconds of each data retrieval/collection operation - interval_cb :: func :: callback for each 'completed' interval - (def func(item, topic, interval_obj instance)) - time_func :: func :: custom time function that returns time.struct_time - - throws TOSDB_Error - """ - def __init__(self, block, interval_sec, poll_sec=1, interval_cb=None, - time_func=time.localtime): - super().__init__(block, OHLC, interval_sec, poll_sec, interval_cb, time_func) - - -class TOSDB_CloseIntervals(TOSDB_FixedTimeIntervals): - """Pulls data from a thread-safe DataBlock in fixed-time ohlc.C intervals - - This class takes streaming data from a 'DataBlock' object (_win.py/__init__.py), - breaks them up along fixed-time intervals, transforms the data into a 'C' object, - and stores those objects in internal buffers that can be accessed via get(...) - and stream_snapshot(...) methods. - - 1) Create a thread-safe 'DataBlock' (e.g tosdb.TOSDB_ThreadSafeDataBlock) - - 2) Add 'items' and 'topics' directly through the 'DataBlock' interface, or - use the methods provided - i) 'topics' that return string data will be ignored (e.g DESCRIPTION, LASTX) - - 3) Create a TOSDB_CloseIntervals object with: - i) the block we created in #1 - ii) the length of the interval in seconds (>= 10, <= 14,400) - iii) (optional) how often each retrieval/collection operations should occur - (i.e interval_sec=30, poll_sec=1 means 30 operations per interval) - iv) (optional) custom time function to translate time data stored in - interval_object from seconds-from-epoch to time.struct_time - vi) (optional) callback function that will pass the item, topic, and - last completed interval as its arguments - - 5) Use methods to access stored data: - get(...) - get_by_datetime(...) - stream_snapshot(...) - stream_snapshot_between_datetimes(...) - - 6) WHEN DONE: call stop() method. The 'DataBlock' object you passed into the - constructor is still valid and in the same state. - - __init__(self, block, interval_sec, poll_sec=1, interval_cb=None, - time_func=time.localtime) - - block :: object :: object that exposes certain _TOSDB_DataBlock methods - interval_sec :: int :: size of fixed-time interval in seconds - poll_sec :: int :: time in seconds of each data retrieval/collection operation - interval_cb :: func :: callback for each 'completed' interval - (def func(item, topic, interval_obj instance)) - time_func :: func :: custom time function that returns time.struct_time - - throws TOSDB_Error - """ - def __init__(self, block, interval_sec, poll_sec=1, interval_cb=None, - time_func=time.localtime): - super().__init__(block, C, interval_sec, poll_sec, interval_cb, time_func) - - - - diff --git a/python/tosdb/meta_enum.py b/python/tosdb/meta_enum.py deleted file mode 100644 index 1ce0f90..0000000 --- a/python/tosdb/meta_enum.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -class MetaEnum(type): - """ metaclass that creates an enum object w/ user-defined values - - class Color(metaclass=MetaEnum): - fields = ( 'Red', 'Blue', 'Green' ) - - class Speed(metaclass=MetaEnum): - fields = { 'fast':300, 'moderate':200, 'slow':100, 'default':300 } - - >>> Speed.fast.val - 300 - >>> Speed.fast.name - 'fast' - >>> str( Speed.fast ) - '300' - >>> Speed.fast in Speed: # check if we belong to Speed - True - >>> Speed.fast == Speed.default: # compare values (only if we have them) - True - >>> Speed.fast is Speed.default: # same values but different objects - False - - raises MetaEnum.EnumError(Exception) - """ - - class EnumError(Exception): - def __init__(self,*msgs): - Exception.__init__(self,*msgs) - - - def __new__(cls,name,bases,d): - from collections import Mapping - - # restrict subclassing - for b in bases: - if isinstance(b,MetaEnum): - raise MetaEnum.EnumError(b.__name__ + ' can not be subclassed') - - # cls needs a fields attribute that defines '__iter__' - if 'fields' not in d: - raise MetaEnum.EnumError('class using metaclass Enum must define fields') - if not hasattr(d['fields'], '__iter__'): - raise MetaEnum.EnumError( str(cls) + '.fields must define __iter__' ) - - def _field_eq(self, other): - try: - return self._enum_class == other._enum_class \ - and self._val is not None \ - and self._val == other._val - except: - return False - - #our field type - our_field_dict = { - '__str__': lambda s: str(s._val) if s._val is not None else '', - '__eq__': _field_eq, # <-- compare if of same enum and with vals - '_enum_class': None - } - our_field_clss = type('EnumField',(), our_field_dict) - - def _field_prop_no_set(self,obj,value): - raise MetaEnum.EnumError("can't set constant enum field") - - # our 'property' descriptor to protect the field - our_field_prop_dict = { - '__init__': lambda self, obj: setattr(self,'_obj',obj), - '__get__': lambda self, obj, objtype: self._obj, - '__set__': _field_prop_no_set, - '_set_enum_class': lambda self, t: setattr(self._obj,'_enum_class',t) - } - our_field_prop_clss = type('EnumFieldProperty',(), our_field_prop_dict) - - # remove fields - fields = d.pop('fields') - - #convert zipped object - if isinstance(fields,zip): - fields = dict(fields) - - for n in fields: - if not isinstance(n,str): - raise MetaEnum.EnumError( 'Enum names must be strings' ) - # create an EnumField instance for each name - obj = our_field_clss() - obj._name = n - if isinstance( fields, Mapping): - # attach a val if fields provides a mapping to one - obj._val = fields[n] - if not hasattr( obj._val, '__str__'): - raise MetaEnum.EnumError( 'Enum value must define __str__') - else: - obj._val = None - # bind stateful descriptors of same object in metaclass and class - setattr(cls,n, our_field_prop_clss( obj ) ) - d[n] = our_field_prop_clss( obj ) - - # create name/val properties for each field - our_field_clss.name = property( lambda self : self._name ) - our_field_clss.val = property( lambda self : self._val ) - - # no need to instantiate the class - def _no_init(self,*val): - raise MetaEnum.EnumError('Enum ' + name + ' can not be instantiated') - d['__init__'] = _no_init - - # return an iter of the EnumFields to support 'in' built-in - def _iter(cls): - return iter([getattr(cls,f) for f in dir(cls) - if isinstance(getattr(cls,f), our_field_clss)]) - - d['_iter'] = classmethod(_iter) - - # cleanup metaclass namespace if we can - def _del(c): - for f in dir(c): - if isinstance(getattr(c,f), our_field_clss): - delattr(cls,f) - d['_del'] = classmethod(_del) - - if isinstance(fields, Mapping): - # reattach a reverse lookup for convenience - d['val_dict'] = { fields[k]:k for k in fields } - - # the finished class object - clss = super(MetaEnum,cls).__new__(cls,name,bases,d) - - # bind a reference to it, in each EnumField, for compare ops - for n in fields: - d[n]._set_enum_class(clss) - getattr(cls,n)._enum_class = clss - - return clss - - - def __iter__(self): # iterate over the enum_fields - return self._iter() - - - def __del__(self): # try to clean up our namespace - self._del() - - diff --git a/python/tosdb/streaming/__init__.py b/python/tosdb/streaming/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/tosdb/streaming/streaming.py b/python/tosdb/streaming/streaming.py deleted file mode 100644 index a52b240..0000000 --- a/python/tosdb/streaming/streaming.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -""" streaming/streaming.py: Streaming functionality for tosdb *** NEW *** """ - -import tosdb - -from sys import stderr as _stderr -from time import sleep as _sleep, perf_counter as _perf_counter -from platform import system as _system -from itertools import product as _product - -DEFAULT_LATENCY = 100 -DEFAULT_TIMEOUT = 0 -DEFAULT_INIT_TIMEOUT = 5000 - -INT_MAX = 2**63 - 1 - -class _StreamingSession(): - INIT_TIMEOUT = DEFAULT_INIT_TIMEOUT - - __doc__ = """ Manage a {{session_type}} of streaming data *** NEW *** - - An iterable Context Manager used to get streaming data for items and topics. - On entry a {{block_type}} is created; and items and topics are added. The - user then iterates over the instance to get available data. Each iteration - will block until data is ready, or timeout. - - {{session_class}} provides two types of blocking/timeout behaviour: - - 1) guarantee=True : returned data will all be sequential, by time. To - insure this(because of the C implementation) each iteration will block - until new data FROM ALL INDIVIDUAL STREAMS is available. If, for - example, you add two topics and two items(four streams) and one of - the items is invalid, returning no data, the whole session will block - - indefinitely if timeout == 0.** - - 2) guarantee=False : most all returned data will be sequential, by time, - but occassionaly not. Each iteration blocks until new data FROM ANY - STREAM is available. The session will be slightly more responsive and - less likely to timeout. - - **To avoid this we force a timeout of {{session_class}}.INIT_TIMEOUT - msec({init_timeout}) if NO data comes into the stream INITIALLY. Change - this field as neeeded. - - Available data is returned as lists of 4-tuples, newest first, e.g.: - [ ('spy', 'LAST', 250.00, TOSDB_DataTime), # float - ('qqq', 'VOLUME', 1380000, TOSDB_DataTime), # int - ('qqq', 'LASTX', '145.57 B', TOSDB_DateTime), # str - ('spy', 'VOLUME', 24562387, TOSDBDateTime) ... ] - - * * * - - {{constructor}} - - throws TOSDB_DataTimeError, TOSDB_IndexError, TOSDB_ValueError, - TOSDB_DataError, TOSDB_CLibError, TOSDB_TypeError, - TOSDB_TimeoutError - - * * * - - try: - with {{session_class}}({{example_args}}) as ss: - d = next(ss) - print("First Streaming Quote: ", str(d[-1]) ) - do_something_with_quotes(d) - for d in ss: - if not run_flag: - break - do_something_with_quotes(d) - print("Last Streaming Quote: ", str(d[0]) ) - except TOSDB_TimeoutError as e: - print("TIMEOUT") - """.format(init_timeout=INIT_TIMEOUT) - - def __init__(self, items, topics, guarantee, timeout, latency): - if not items: - raise tosdb.TOSDB_ValueError("no items to stream") - if not topics: - raise tosdb.TOSDB_ValueError("no topics to stream") - self._items = items - self._topics = topics - self._guarantee = guarantee - self._timeout = timeout / 1000 - self._latency = latency / 1000 - self._run_flag = True - - def __enter__(self): - try: - self._block.add_items(*self._items) - self._block.add_topics(*self._topics) - self._buffers = {k:[] for k in _product(self._items, self._topics)} - self._good_init = False - self._stream_time_start = _perf_counter() - self._cutoff_mktime = 0 - except: - self._block.close() - raise - return self - - def __exit__(self, *args): - self._block.close() - - def __iter__(self): - return self - - def __next__(self): - good = [] - time_start = _perf_counter() - while True: - # look for data - for item in self._items: - for topic in self._topics: - dat = self._block.stream_snapshot_from_marker( - item, topic, date_time=True, throw_if_data_lost=False) - if dat is not None: - dat.reverse() - self._buffers[item, topic].extend(dat) - - # check for empty buffers - bad_buffers = [k for k,v in self._buffers.items() if not v] - need_to_sleep = bool(bad_buffers) if self._guarantee else \ - len(bad_buffers) == len(self._buffers) - # sleep and check for timeout if necessary - if need_to_sleep: - self._sleep_or_timeout(time_start, bad_buffers) - continue - else: - self._good_init = True - - # of newest datum in each stream, find the oldest - oldest_new_mktime = INT_MAX - for v in self._buffers.values(): - if v and v[-1][1].mktime_micro < oldest_new_mktime: - oldest_new_mktime = v[-1][1].mktime_micro - self._cutoff_mktime = oldest_new_mktime if oldest_new_mktime != INT_MAX else 0 - - # remove all data older or equal to the aforementioned datum - for k, v in self._buffers.items(): - ngood = 0 - for vv in v: - if vv[1].mktime_micro <= self._cutoff_mktime: - good.append( (k[0], k[1], vv[0], vv[1]) ) - ngood += 1 - self._buffers[k] = v[ngood:] - assert good, "'good' list is empty (PLEASE REPORT THIS BUG)" - - # sort all that and return it - return sorted(good, key=lambda v: v[3].mktime_micro, reverse=True) - - - def _sleep_or_timeout(self, time_start, bad_buffers): - time_now = _perf_counter() - if not self._good_init: - time_left = (self.INIT_TIMEOUT/1000) - (time_now - self._stream_time_start) - if time_left > 0: - _sleep( min(self._latency, time_left) ) - return - raise tosdb.TOSDB_TimeoutError("StreamingSession timed out trying to INIT -" + - " BAD STREAMS " + str(bad_buffers)) - elif self._timeout: - time_left = self._timeout - (time_now - time_start) - if time_left <= 0: - raise tosdb.TOSDB_TimeoutError("StreamingSession timed out -" + - " BAD STREAMS " + str(bad_buffers)) - _sleep( min(self._latency, time_left) ) - else: - _sleep(self._latency) - - -class StreamingSession(_StreamingSession): - __doc__ = _StreamingSession.__doc__.format( - session_type="session", - block_type="TOSDB_DataBlock", - session_class="StreamingSession", - example_args="['spy','qqq'], ['LAST', 'VOLUME'], True, 10000", - constructor=\ - """__init__(self, items, topics, guarantee=True, timeout=DEFAULT_TIMEOUT, - latency=DEFAULT_LATENCY): - - items :: list(str) :: item strings to stream - topics :: list(str) :: topics to stream - guarantee :: bool :: streaming data guaranteed to be in time-order - timeout :: int :: milliseconds to wait for data before - blocking iteration times out (0 == no timeout) - latency :: int :: internal wait milliseconds if no data""") - - - def __init__(self, items, topics, guarantee=True, timeout=DEFAULT_TIMEOUT, - latency=DEFAULT_LATENCY): - super().__init__(items, topics, guarantee, timeout, latency) - - def __enter__(self): - self._block = tosdb.TOSDB_DataBlock(1000, date_time=True) - return super().__enter__() - - -if _system() not in ["Windows","windows","WINDOWS"]: - globals().pop(StreamingSession.__name__) - - -class VStreamingSession(_StreamingSession): - __doc__ = _StreamingSession.__doc__.format( - session_type="virtual session", - block_type="VTOSDB_DataBlock", - session_class="VStreamingSession", - example_args="""['spy','qqq'], ['LAST', 'VOLUME'], ('10.0.0.1',55555), - 'password', True, 10000""", - constructor=\ - """__init__(self, items, topics, address, password=None, guarantee=True, - timeout=DEFAULT_TIMEOUT, latency=DEFAULT_LATENCY): - - items :: list(str) :: item strings to stream - topics :: list(str) :: topics to stream - address :: (str,int) :: (host/address of the windows implementation, port) - password :: str :: password for authentication(None for no authentication) - guarantee :: bool :: streaming data guaranteed to be in time-order - timeout :: int :: milliseconds to wait for data before - blocking iteration times out (0 == no timeout) - latency :: int :: internal wait milliseconds if no data""") - - def __init__(self, items, topics, address, password=None, guarantee=True, - timeout=DEFAULT_TIMEOUT, latency=DEFAULT_LATENCY): - super().__init__(items, topics, guarantee, timeout, latency) - self._address = address - self._password = password - - def __enter__(self): - self._block = tosdb.VTOSDB_DataBlock(self._address, self._password, 1000, True) - return super().__enter__() - - - diff --git a/python/tutorial.md b/python/tutorial.md deleted file mode 100644 index 013bca9..0000000 --- a/python/tutorial.md +++ /dev/null @@ -1,50 +0,0 @@ -### tosdb/ tutorial ---- - -This tutorial attempts to show example usage of the core tosdb package. Please refer to *\_win.py*, *\_common.py* and *\__init__.py* for more details on the core package; *\__init__.\__doc__* for an explanation of the virtual layer; *\__main__.py* for running the package outside of the interactive interpreter; *intervalize/ohlc.py* for fixed-time intervals(e.g OHLC); and *cli_scripts/* for code built on top of the core package. - -Comments inside the screen-shots help explain what we are doing and why. - -**IMPORTANT:** Before using tosdb/ be sure to: - -1. build/install the backend C/C++ modules with the appropriate build(x86 vs. x64) for your version of python, -2. start the TOSDataBridge Service(SC Start TOSDataBridge), -3. have the TOS platform running, with the correct privileges (i.e if you run as admin you should have passed 'admin' to the tosdb-setup.bat script), -4. install the tosdb/ package (python setup.py install) - ---- - -**Initialize the library with the init() call:** - -This loads the shared library into python and tries to automatically connect to the engine. Be sure to use the correct version of the tos-databridge DLL. If you are using branch/version v0.7 and 64-bit python, for instance, you should be using tos-databridge-0.7-x64.dll. -![](./../res/tosdb_tut1.png) - -**Topics and Items:** -![](./../res/tosdb_tut2.png) - -**TOSDB_DataBlock:** -![](./../res/tosdb_tut3.png) - -**Total Frame:** -![](./../res/tosdb_tut4.png) - -**Topic / Item Frames:** -![](./../res/tosdb_tut5.png) - -**Historical Data:** -![](./../res/tosdb_tut6.png) - -**Historical Data (2):** -![](./../res/tosdb_tut7.png) - -**Historical Data (3):** -![](./../res/tosdb_tut8.png) -![](./../res/tosdb_tut9.png) - -**Miscellany & Clean-Up:** -![](./../res/tosdb_tut10.png) - - - - - diff --git a/python/virtualization_tutorial.md b/python/virtualization_tutorial.md deleted file mode 100644 index 25b6ae3..0000000 --- a/python/virtualization_tutorial.md +++ /dev/null @@ -1,108 +0,0 @@ -### tosdb virtualization tutorial ---- - -This tutorial attempts to show simple example usage of the tosdb virtual layer. Please refer to *\__init__.\__doc__* for an explanation of the virtual layer and the myriad ways to initialize and use it not show in this tutorial. - -For this tutorial we are using TOS in a Windows virtual machine. We've set up a host-only network adapter and assigned an address of 192.168.56.101 to this guest machine, and 192.168.56.1 to the host(linux) machine. We've also added a firewall rule allowing TCP IN to port 55555 from our host address. The screen shots are of the 'local'/guest/windows side; the code blocks are of the 'remote'/host/linux side. - -**IMPORTANT:** We recently added a (provisional) authentication mechanism to the virtual layer. ***Unless you know what you're doing and can review the code (tosdb/\_\_init\_\_.py and tosdb/\_auth.py) it's prudent to assume it not secure, possibly exploitable for remote code execution.*** (If anyone out there can give any feedback it would be helpful.) Currently it's recommended for internal networks. To use: 1) install the pycrypto package if you don't have it(pip install pycrypto), 2) pass a password to the enable_virtualization call on the server side and (the same password) to the admin_init call and/or the VTOSDB_DataBlock constructor on the client side. - ---- - -#### Local Interactive Init - -From an interactive interpreter we are going to initialize tosdb as normal, followed by a call to enable_virtualization with an address tuple indicating what (local address, port) to bind to. - -![](./../res/tosdb_virtual_tutorial_1a.png) - -If sucessfull we can, from the remote machine, create a VTOSDB_DataBlock or use admin_init to access the admin calls. When done we delete the block remotely and use admin_close to close the connections. - - -``` -jon@jdeb:~/dev/TOSDataBridge$ python3 ->>> import tosdb ->>> vblock = tosdb.VTOSDB_DataBlock(('192.168.56.101',55555)) ->>> tosdb.admin_init(('192.168.56.101',55555)) ->>> tosdb.vconnected() -True ->>> del vblock ->>> tosdb.admin_close() ->>> exit() -``` - -We can use clean_up locally or vclean_up remotely(before admin_close), if necessary. If we want to stop virtuatlization and keep using the local machine we can call disable_virtualization. - -![](./../res/tosdb_virtual_tutorial_1b.png) - - -#### Remote Interactive Init - -Similar to the previous approach except we handle all the initialization remotely after we call enable_virtualization locally(as above). - -![](./../res/tosdb_virtual_tutorial_2a.png) - -To initialize we first call admin_init(from the remote machine) in order to access vinit: the remote version of init that takes the sames arguments. (Remember we are passing the 'local' windows path of the underlying library to vinit.) - - -``` -jon@jdeb:~/dev/TOSDataBridge$ python3 ->>> import tosdb ->>> tosdb.admin_init(('192.168.56.101',55555)) ->>> tosdb.vinit(root="C:/TOSDataBridge") -True -``` - -![](./../res/tosdb_virtual_tutorial_2b.png) - - -Use an admin call, create a block, delete the block, clean up and close the admin interface - all from the remote machine. - -``` ->>> tosdb.vconnected() -True ->>> vblock = tosdb.VTOSDB_DataBlock(('192.168.56.101',55555)) ->>> del vblock ->>> tosdb.vclean_up() ->>> tosdb.admin_close() ->>> exit() -``` - -![](./../res/tosdb_virtual_tutorial_2c.png) - - -#### Local Server Init - -This enables virtualization and handles initialization for you, returning an interactive shell that only accepts a quit command to close. - -\**notice the preceding space in the --virtual-server arg tuple " 55555". This indicates an address of 'all available interfaces'* - -![](./../res/tosdb_virtual_tutorial_3a.png) - -From the remote side we can make the necessary calls(as above). - - -``` -jon@jdeb:~/dev/TOSDataBridge$ python3 ->>> import tosdb ->>> vblock = tosdb.VTOSDB_DataBlock(('192.168.56.101',55555)) ->>> tosdb.admin_init(('192.168.56.101',55555)) ->>> tosdb.vconnected() -True ->>> del vblock ->>> tosdb.admin_close() ->>> exit() -``` - -Or, from the command-line use the --virtual-client switch which will do it for us and drop us into an interactive interpreter. - -``` -jon@jdeb:~/dev/TOSDataBridge$ python3 python/tosdb --virtual-client "192.168.56.101 55555" -(InteractiveConsole) ->>> vconnected() -True ->>> exit() -``` - -Close the server when done. - -![](./../res/tosdb_virtual_tutorial_3b.png) diff --git a/res/SCss1.png b/res/SCss1.png deleted file mode 100644 index 8a34ffc..0000000 Binary files a/res/SCss1.png and /dev/null differ diff --git a/res/main_diagram.png b/res/main_diagram.png deleted file mode 100644 index db65ac6..0000000 Binary files a/res/main_diagram.png and /dev/null differ diff --git a/res/tosdb_diagram2.png b/res/tosdb_diagram2.png deleted file mode 100644 index 8e72eaa..0000000 Binary files a/res/tosdb_diagram2.png and /dev/null differ diff --git a/res/tosdb_tut1.png b/res/tosdb_tut1.png deleted file mode 100644 index 4bdca8c..0000000 Binary files a/res/tosdb_tut1.png and /dev/null differ diff --git a/res/tosdb_tut10.png b/res/tosdb_tut10.png deleted file mode 100644 index 9cf9de8..0000000 Binary files a/res/tosdb_tut10.png and /dev/null differ diff --git a/res/tosdb_tut2.png b/res/tosdb_tut2.png deleted file mode 100644 index ad9ac47..0000000 Binary files a/res/tosdb_tut2.png and /dev/null differ diff --git a/res/tosdb_tut3.png b/res/tosdb_tut3.png deleted file mode 100644 index 26e6979..0000000 Binary files a/res/tosdb_tut3.png and /dev/null differ diff --git a/res/tosdb_tut4.png b/res/tosdb_tut4.png deleted file mode 100644 index 23da9ab..0000000 Binary files a/res/tosdb_tut4.png and /dev/null differ diff --git a/res/tosdb_tut5.png b/res/tosdb_tut5.png deleted file mode 100644 index b00eee7..0000000 Binary files a/res/tosdb_tut5.png and /dev/null differ diff --git a/res/tosdb_tut6.png b/res/tosdb_tut6.png deleted file mode 100644 index 6840c43..0000000 Binary files a/res/tosdb_tut6.png and /dev/null differ diff --git a/res/tosdb_tut7.png b/res/tosdb_tut7.png deleted file mode 100644 index 838f97e..0000000 Binary files a/res/tosdb_tut7.png and /dev/null differ diff --git a/res/tosdb_tut8.png b/res/tosdb_tut8.png deleted file mode 100644 index 9d5409b..0000000 Binary files a/res/tosdb_tut8.png and /dev/null differ diff --git a/res/tosdb_tut9.png b/res/tosdb_tut9.png deleted file mode 100644 index 932b22d..0000000 Binary files a/res/tosdb_tut9.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_1a.png b/res/tosdb_virtual_tutorial_1a.png deleted file mode 100644 index bee703f..0000000 Binary files a/res/tosdb_virtual_tutorial_1a.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_1b.png b/res/tosdb_virtual_tutorial_1b.png deleted file mode 100644 index 090f53d..0000000 Binary files a/res/tosdb_virtual_tutorial_1b.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_2a.png b/res/tosdb_virtual_tutorial_2a.png deleted file mode 100644 index 1104842..0000000 Binary files a/res/tosdb_virtual_tutorial_2a.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_2b.png b/res/tosdb_virtual_tutorial_2b.png deleted file mode 100644 index 03beeaf..0000000 Binary files a/res/tosdb_virtual_tutorial_2b.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_2c.png b/res/tosdb_virtual_tutorial_2c.png deleted file mode 100644 index cf1541d..0000000 Binary files a/res/tosdb_virtual_tutorial_2c.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_3a.png b/res/tosdb_virtual_tutorial_3a.png deleted file mode 100644 index e3bd3d7..0000000 Binary files a/res/tosdb_virtual_tutorial_3a.png and /dev/null differ diff --git a/res/tosdb_virtual_tutorial_3b.png b/res/tosdb_virtual_tutorial_3b.png deleted file mode 100644 index 66d1fd1..0000000 Binary files a/res/tosdb_virtual_tutorial_3b.png and /dev/null differ diff --git a/sigs/_checksum.sh b/sigs/_checksum.sh deleted file mode 100755 index 8032080..0000000 --- a/sigs/_checksum.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash - -SIGS_DIR=$( cd "$( dirname "$0" )" && pwd ) -KEY_FILE=jeog.dev.key -CHECK_PATH=$SIGS_DIR/notes_checksums.txt - -X86_BIN_DIR=$SIGS_DIR/../bin/Release/Win32 -X64_BIN_DIR=$SIGS_DIR/../bin/Release/x64 - - -echo -e "* All the binaries are signed with the jeog.dev.key (jeog.dev@gmail.com) -public key. It should be available on public key servers. \n" > $CHECK_PATH - -echo -e "* Being in a seprate 'sigs' folder you'll have to point the verifier to the -corresponding binary file for each signature, or use verify.sh on linux. \n" >> $CHECK_PATH - -echo -e "SHA256 checksums:" >> $CHECK_PATH - -echo -e "\n(THE SIGNATURE FILES)" >> $CHECK_PATH -sha256sum $(ls | egrep '\.sig$') >> $CHECK_PATH - -echo -e "\n(THE ACTUAL BINARIES)" >> $CHECK_PATH -cd $X86_BIN_DIR -sha256sum $(ls) >> $CHECK_PATH -cd $X64_BIN_DIR -sha256sum $(ls) >> $CHECK_PATH - -cd $SIGS_DIR - - - - - - diff --git a/sigs/_sign.sh b/sigs/_sign.sh deleted file mode 100755 index 464c991..0000000 --- a/sigs/_sign.sh +++ /dev/null @@ -1,28 +0,0 @@ -#! /bin/bash - -SIGS_DIR=$( cd "$( dirname "$0" )" && pwd ) - -X86_BIN_DIR=$SIGS_DIR/../bin/Release/Win32 -X64_BIN_DIR=$SIGS_DIR/../bin/Release/x64 - -cd $X86_BIN_DIR - -X86_BIN_FILES=($(ls)) -for f in ${X86_BIN_FILES[*]}; do - gpg -sb --local-user jeog.dev $f; -done - -mv *.sig $SIGS_DIR - -cd $X64_BIN_DIR - -X64_BIN_FILES=($(ls)) -for f in ${X64_BIN_FILES[*]}; do - gpg -sb --local-user jeog.dev $f; -done - -mv *.sig $SIGS_DIR - -cd $SIGS_DIR - - diff --git a/sigs/jeog.dev.key b/sigs/jeog.dev.key deleted file mode 100644 index c10db60..0000000 --- a/sigs/jeog.dev.key +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mQENBFVdRMUBCAC5OaMiaUj/mJOKgiNsd1lVAQhNShFiXd5+Ri6KIH4sqwvHx1uC -1J4akpX4BqcGCFyo9Wr3Xd/ckhyYMXWc+TI6kzZKORGcVzsvbBecf14KWHuYd/Rl -XFbtvVvJyfgvnAMCTEfHdTd0zgGgMREJDMEf5fLqnUuJ0XDCadmlMKN1NzBiRuXy -PKNLsc6UeuhtY4cRycWyv+4gomvIDhTDJSUBlkWjcqZlFxnEz2IOYv4Xh7UzgNGd -SMmfx/EorBxH4bm9SJN3LHZqFt4rSTgnyfpoTql40kz/N0/yUVknQyFJ/qKMZx5q -rZkaPN4fmb7XepQVHqkWidc9+BRQqncowfnFABEBAAG0HWplb2cuZGV2IDxqZW9n -LmRldkBnbWFpbC5jb20+iQE4BBMBAgAiAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIe -AQIXgAUCWBn8AgAKCRAI5DUFUY+fWgq5B/9uQb5cCVnIJAJPOeSpf/Qj3+1Qy2FZ -tHCqPeE+ryV84iPc4qf8Pu6JIWzBMSt/F+dAr7YT3aKs3eReLs181QcRHOx04FC9 -xGDlNCdjieiI+d56KHtgMY43JZUrwtjh6b5Skku4RUHsqPhiZZhBNw0uIKRs2W65 -5wtkqcLkzBoi5uLxiSwVETHgPtkpHLV7JoQAUYefGVDVUhjM+iCy6zNqU/FnWXPL -x7RXdtz/dXWuiX3l0gi3M3W/NpddMrDSObcQuuQaDzL/0QUOlMD0lPZ/fKZP8n90 -DpzMV0NMVxA/Nzr3CBhnWe6PfyzCfhbbXjGRa3N9TLzBLCNfzggykWbGuQENBFVd -RMUBCAD4J00qCWDuwx1+YefLt0P6DhZJ/2KlFWsgTJ77y+brEmNmk0Z3eKRw3+Wj -TlMHMmWOPOqWNTDiKgFA+eA6k9T1ncKdf8iA0Z1dZt0aN+4mcHSlvBmXv3LlgMHy -hlxUSLNF8PJOl5WhXPoVCld+UFStjEc8PfjE9saoMIdwNqCN4GJQA3c+2DDzOR1p -MEdGb4fgmNpS2s7R9vHSTOb7MXH0YwzEVUYvYDJ+maemXlKPDwugu//lDDCX44eG -ORQNhrRF8hdV6dzWdCmw2ZVw28+oWAxSwIiij6/8EWxvWjjkujrS+Z4Ks+Mrtzpd -udEJ8dgLyGA4Il0R75cTcJR+hJpZABEBAAGJAR8EGAECAAkCGwwFAlgaxusACgkQ -COQ1BVGPn1qzzwf7BYaHyaM5VC4JMoU92B6bFxhODiIrI+BBlmFxlTqXd3bvBwCT -DM+w5lYSce1FSmTQLnrDivEn10UhmWEURWsXOyrKljL5mceRSyiqteulWgpjI/0h -D3K48uFJGWwnjCME1gIC/SjrOYpIvzpMZKLaPfjcuKOZEoVGyh5pMiU/bdcXbMnL -iphc2tRBW5NGY1t6DsHoXGeyPjSUqcgDfq2iJT6zTw7LDfwyu5oaguopCNfxwe6h -mawz/fIuIiauIVw7JoGTZxWOAZnWCjEapSx2hWD4ZXfiG0kletVN71uRDo5iNiVX -UX/2MILV29BR235cdQ2yWck+F0a1Pc3mol7nKw== -=b69Q ------END PGP PUBLIC KEY BLOCK----- diff --git a/sigs/jeog.dev.key.old b/sigs/jeog.dev.key.old deleted file mode 100644 index 0154f2a..0000000 --- a/sigs/jeog.dev.key.old +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mQENBFVdRMUBCAC5OaMiaUj/mJOKgiNsd1lVAQhNShFiXd5+Ri6KIH4sqwvHx1uC -1J4akpX4BqcGCFyo9Wr3Xd/ckhyYMXWc+TI6kzZKORGcVzsvbBecf14KWHuYd/Rl -XFbtvVvJyfgvnAMCTEfHdTd0zgGgMREJDMEf5fLqnUuJ0XDCadmlMKN1NzBiRuXy -PKNLsc6UeuhtY4cRycWyv+4gomvIDhTDJSUBlkWjcqZlFxnEz2IOYv4Xh7UzgNGd -SMmfx/EorBxH4bm9SJN3LHZqFt4rSTgnyfpoTql40kz/N0/yUVknQyFJ/qKMZx5q -rZkaPN4fmb7XepQVHqkWidc9+BRQqncowfnFABEBAAG0HWplb2cuZGV2IDxqZW9n -LmRldkBnbWFpbC5jb20+iQE+BBMBAgAoBQJVXUTFAhsDBQkB3+IABgsJCAcDAgYV -CAIJCgsEFgIDAQIeAQIXgAAKCRAI5DUFUY+fWrBGB/0eKBhdsVfwGoSWb2pY4288 -nYlBk9TUN6Kb2xhgg1C8w8/O1SAon/IE8oSfllf36P6Z3njJz9Hd0urFP/athBGg -DrL1NgpI30u7hEdA6MSe2il3QI0o7M+GCGIhnxJW46LMGMmF+iKSP6z2F5pfFyQh -BXSE40gOuNW/YI+MvYMklihRl9Vl5pkU2/wgJJ1gz++7sgsfxo1Hl9GYF0cB5mqz -rWwqskp89y/L44Q0vJ57roTDY25oTvq7j7VgLoIbJWKDfl29F3na57oM9yG1QcrV -DzUj9CBtvzHC8rJrMhcSCfKmb7M2qriIfpOlRtufRbGVqZ0caxAz1K9Zo8qn+nnn -uQENBFVdRMUBCAD4J00qCWDuwx1+YefLt0P6DhZJ/2KlFWsgTJ77y+brEmNmk0Z3 -eKRw3+WjTlMHMmWOPOqWNTDiKgFA+eA6k9T1ncKdf8iA0Z1dZt0aN+4mcHSlvBmX -v3LlgMHyhlxUSLNF8PJOl5WhXPoVCld+UFStjEc8PfjE9saoMIdwNqCN4GJQA3c+ -2DDzOR1pMEdGb4fgmNpS2s7R9vHSTOb7MXH0YwzEVUYvYDJ+maemXlKPDwugu//l -DDCX44eGORQNhrRF8hdV6dzWdCmw2ZVw28+oWAxSwIiij6/8EWxvWjjkujrS+Z4K -s+MrtzpdudEJ8dgLyGA4Il0R75cTcJR+hJpZABEBAAGJASUEGAECAA8FAlVdRMUC -GwwFCQHf4gAACgkQCOQ1BVGPn1pYdgf/eRdRKpIA5KaXRtleeWknC3MIx5jDZKR4 -vPoA4OZjji4oUl8bn7g+s4nw5owzUKtMMr3b8aJjbpwBnqrscuEF/OgW/yauLhqW -vu7CPWdikHxtUQq5KfBCJWy7zfstzxl43ZR0FO9vJoofWWQ18AyEjlwydOsbuRcI -jTFK53FZJ7QEdvqTW3XBW+PnN06DIZkyXD3upIfUgLs1WC25m3rAJ2y724VFpeQw -jGYAF8iZ8vmYYyZZSUqIopp45bibDmfr95zQrIwYFNM17l0TU0mqk2kQniTazpCL -0wgt2gYRPpeEcZfLYg+QYcTA8CyrhSA9GDqSmf7gMuYxcYCRMDa8uw== -=arWh ------END PGP PUBLIC KEY BLOCK----- diff --git a/sigs/notes_checksums.txt b/sigs/notes_checksums.txt deleted file mode 100644 index d0e746b..0000000 --- a/sigs/notes_checksums.txt +++ /dev/null @@ -1,17 +0,0 @@ -* All the binaries are signed with the jeog.dev.key.asc (jeog.dev@gmail.com) -public key. It should be available on public key servers. - -* Being in a seprate 'sigs' folder you'll have to point the verifier to the -corresponding binary file for each signature, or use verify.sh on linux. - -* NOTE: MASTER BRANCH MAY NOT CONTAIN ALL (OR ANY) SIGS/CHECKSUMS - - -SHA256 checksums: - -(THE SIGNATURE FILES) - - -(THE ACTUAL BINARIES) - - diff --git a/sigs/verify.sh b/sigs/verify.sh deleted file mode 100755 index 00060a3..0000000 --- a/sigs/verify.sh +++ /dev/null @@ -1,119 +0,0 @@ -#! /bin/bash - -echo "verify.sh assumes: - 1) you have a working gpg in your path - 2) you've imported jeog.dev.key -" - -sigs_dir=$( cd "$( dirname "$0" )" && pwd ) - -cd $sigs_dir - -x86_sig_files=($(ls | egrep '.*tos-databridge.+x86\..+sig')) -x64_sig_files=($(ls | egrep '.*tos-databridge.+x64\..+sig')) - -check_return=0 - -cmp_checksums() -{ - echo ' ' $1 'sha256 checksums' - off=$(cat notes_checksums.txt | grep ' '$1'$' | cut -d' ' -f1) - calc=$(sha256sum $2 | cut -d' ' -f1) - echo ' ' $off - echo ' ' $calc - [[ $off == $calc ]] - check_return=$? -} - -run_checks() -{ - files=("${!1}") - for s in ${files[*]}; do - echo "::: ${s%.sig} :::" - - # signature checksums - cmp_checksums $s $s - if (( $check_return )); then - vbad_checksum_sig=$(($vbad_checksum_sig+1)) - else - vgood_checksum_sig=$(($vgood_checksum_sig+1)) - fi - - # binary checksums - bin="${s%.sig}" - binpath=$2$bin - cmp_checksums $bin $binpath - if (( $check_return )); then - vbad_checksum_bin=$(($vbad_checksum_bin+1)) - else - vgood_checksum_bin=$(($vgood_checksum_bin+1)) - fi - - # verify - gpg --verify $s $binpath; - if (( $? )); then - vbad=$(($vbad+1)) - else - vgood=$(($vgood+1)) - fi - echo "" - done -} - - -vgood=0 -vbad=0 -vgood_checksum_sig=0 -vbad_checksum_sig=0 -vgood_checksum_bin=0 -vbad_checksum_bin=0 - - -run_checks x86_sig_files[@] "../bin/Release/Win32/" - -x86_summary="x86 binaries: $vgood good, $vbad bad" -x86_summary_checksum_sig="x86 binaries: $vgood_checksum_sig good, $vbad_checksum_sig bad" -x86_summary_checksum_bin="x86 binaries: $vgood_checksum_bin good, $vbad_checksum_bin bad" - -bad_total=$(($vbad + $vbad_checksum_sig + $vbad_checksum_bin)) - - -vgood=0 -vbad=0 -vgood_checksum_sig=0 -vbad_checksum_sig=0 -vgood_checksum_bin=0 -vbad_checksum_bin=0 - -run_checks x64_sig_files[@] "../bin/Release/x64/" - -x64_summary="x64 binaries: $vgood good, $vbad bad" -x64_summary_checksum_sig="x64 binaries: $vgood_checksum_sig good, $vbad_checksum_sig bad" -x64_summary_checksum_bin="x64 binaries: $vgood_checksum_bin good, $vbad_checksum_bin bad" - - -bad_total=$(($bad_total + $vbad + $vbad_checksum_sig + $vbad_checksum_bin)) - -echo -if (( $bad_total )); then - echo "*** $bad_total ERRORS ***" -else - echo "*** SUCCESS ***" -fi -echo - -echo -echo "::: summary (sig checksums) :::" -echo "$x86_summary_checksum_sig" -echo "$x64_summary_checksum_sig" -echo -echo "::: summary (bin checksums) :::" -echo "$x86_summary_checksum_bin" -echo "$x64_summary_checksum_bin" -echo -echo "::: summary (signatures) :::" -echo "$x86_summary" -echo "$x64_summary" -echo - - diff --git a/src/client/client_admin.cpp b/src/client/client_admin.cpp deleted file mode 100644 index eb4ee42..0000000 --- a/src/client/client_admin.cpp +++ /dev/null @@ -1,1203 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "tos_databridge.h" -#include "client.hpp" -#include "raw_data_block.hpp" -#include "ipc.hpp" - -std::recursive_mutex global_rmutex; - -namespace { - -typedef std::tuple, HANDLE, HANDLE> buffer_info_ty; - -typedef std::map, buffer_info_ty> buffers_ty; - -LPCSTR LOG_NAME = "client-log.log"; - -/* item/topic to test connection */ -const TOS_Topics::TOPICS TEST_TOPIC = TOS_Topics::TOPICS::LAST; -const std::string TEST_ITEM = "SPY"; - -/* blocks in *this* instance of the dll */ -std::map dde_blocks; - -/* buffers in shared mem */ -buffers_ty buffers; -std::mutex buffers_mtx; - -/* !!! 'buffers_lock_guard_' is reserved inside this namespace !!! */ -#define LOCAL_BUFFERS_LOCK_GUARD std::lock_guard buffers_lock_guard_(buffers_mtx) - -/* for 'scheduling' buffer reads */ -steady_clock_type steady_clock; - -unsigned long buffer_latency = TOSDB_DEF_LATENCY; -HANDLE buffer_thread = NULL; -DWORD buffer_thread_id = 0; - -/* our IPC mechanism */ -IPCMaster master(TOSDB_COMM_CHANNEL); - -/* atomic flag that supports IPC connectivity */ -std::atomic aware_of_connection(false); - - -/* get our block (or NULL) to modify internally */ -TOSDBlock* -_getBlockPtr(std::string id) -{ - try{ - return dde_blocks.at(id); - }catch(...){ - return NULL; - } -} - - -bool -_connected(bool log_if_not_connected=false) -{ - if(!aware_of_connection.load()){ - if(log_if_not_connected){ - TOSDB_LogH("IPC", "not connected to slave (!aware_of_connection)"); - } - return false; - } - - if(!master.connected(TOSDB_DEF_TIMEOUT)){ - if(log_if_not_connected){ - TOSDB_LogH("IPC", "not connected to slave (!master.connected)"); - } - return false; - } - - return true; -} - - -long -_requestStreamOP(TOS_Topics::TOPICS topic_t, - std::string item, - unsigned long timeout, - unsigned int opcode) -{ /* needs exclusivity but can't block; CALLING CODE MUST LOCK - returns 0 on sucess, TOSDB_ERROR... on error */ - - /* build ipc msg early so we can log it on error */ - std::string msg = std::to_string(opcode) + ' ' + TOS_Topics::MAP()[topic_t] + ' ' - + item + ' ' + std::to_string(timeout); - - switch(opcode){ - case TOSDB_SIG_ADD: - case TOSDB_SIG_REMOVE: - case TOSDB_SIG_TEST: - break; - default: - TOSDB_LogRawH("IPC", ("_requestStreamOP received bad opcode, msg:" + msg).c_str()); - return TOSDB_ERROR_BAD_SIG; - } - - if( !_connected() ){ - TOSDB_LogRawH("IPC", ("_requestStreamOP failed, not connected, msg:" + msg).c_str()); - return TOSDB_ERROR_NOT_CONNECTED; - } - - if( !master.call(&msg,timeout) ){ - TOSDB_LogRawH("IPC",("master.call failled in _requestStreamOP, msg:" + msg).c_str()); - return TOSDB_ERROR_IPC; - } - - try{ - long r = std::stol(msg); - if(r) - TOSDB_LogRawH("ENGINE", ("error code returned from engine: " + std::to_string(r)).c_str()); - return r; - }catch(...){ - TOSDB_LogRawH("IPC", ("failed to convert return message to long, msg:" + msg).c_str()); - return TOSDB_ERROR_IPC; - } -} - - -void -_captureBuffer(TOS_Topics::TOPICS topic_t, - std::string item, - const TOSDBlock* db) -{ - void *fm_hndl; - void *mem_addr; - void *mtx_hndl; - DWORD err; - std::string err_msg; - buffers_ty::key_type buf_key(topic_t, item); - - LOCAL_BUFFERS_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - buffers_ty::iterator b_iter = buffers.find(buf_key); - if( b_iter != buffers.end() ){ - std::get<2>(b_iter->second).insert(db); - }else{ - std::string buf_name = CreateBufferName(TOS_Topics::map[topic_t], item); - std::string mtx_name = std::string(buf_name).append("_mtx"); - - fm_hndl = OpenFileMapping(FILE_MAP_READ, 0, buf_name.c_str()); - if( !fm_hndl ){ - err = GetLastError(); - err_msg = "failed to open file mapping: " + buf_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - throw TOSDB_BufferError(err_msg); - } - - mem_addr = MapViewOfFile(fm_hndl,FILE_MAP_READ,0,0,0); - if( !mem_addr ){ - err = GetLastError(); - err_msg = "failed to map shared memory: " + buf_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - CloseHandle(fm_hndl); - throw TOSDB_BufferError(err_msg); - } - - CloseHandle(fm_hndl); - - mtx_hndl = OpenMutex(SYNCHRONIZE,FALSE,mtx_name.c_str()); - if( !mtx_hndl ){ - err = GetLastError(); - err_msg = "failed to open MUTEX handle: " + mtx_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - UnmapViewOfFile(mem_addr); - throw TOSDB_BufferError(err_msg); - } - - std::set db_set; - db_set.insert(db); - - auto binfo = std::make_tuple(0,0,std::move(db_set),mem_addr,mtx_hndl); - buffers.insert( buffers_ty::value_type(std::move(buf_key),std::move(binfo)) ); - } - /* --- CRITICAL SECTION --- */ -} - - -void -_releaseBuffer(TOS_Topics::TOPICS topic_t, - std::string item, - const TOSDBlock* db) -{ - buffers_ty::key_type buf_key(topic_t, item); - - LOCAL_BUFFERS_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - buffers_ty::iterator b_iter = buffers.find(buf_key); - if(b_iter != buffers.end()){ - std::get<2>(b_iter->second).erase(db); - if(std::get<2>(b_iter->second).empty()) - { - UnmapViewOfFile(std::get<3>(b_iter->second)); - CloseHandle(std::get<4>(b_iter->second)); - buffers.erase(b_iter); - } - } - /* --- CRITICAL SECTION --- */ -} - - -template -inline T -_castToVal(char* val) -{ - return *(T*)val; -} - -template<> -inline std::string -_castToVal(char* val) -{ - return std::string(val); -} - - -template -void -_extractFromBuffer(TOS_Topics::TOPICS topic, - std::string item, - buffer_info_ty& buf_info) -{ /* some unecessary comp in here; cache some of the buffer params */ - long npos; - long long loop_diff, nelems; - unsigned int dlen; - char* spot; - - pBufferHead head = (pBufferHead)std::get<3>(buf_info); - - if( head->next_offset - head->beg_offset == std::get<0>(buf_info) - && head->loop_seq == std::get<1>(buf_info) - || WaitForSingleObject(std::get<4>(buf_info), 0) ) - { - /* attempt to bail early if chance buffer hasn't changed; - dont wait for the mutex, move on to the next */ - return; - } - - /* get the effective size of the buffer */ - dlen = head->end_offset - head->beg_offset; - - /* relative position since last read (begin offset needs to be 0) */ - npos = head->next_offset - head->beg_offset - std::get<0>(buf_info); - - if((loop_diff = (head->loop_seq - std::get<1>(buf_info))) < 0) - {/* how many times has the buffer looped around, if any */ - loop_diff+=(((long long)UINT_MAX+1)); /* in case of num limit */ - } - - /* how many elems have been written, 'borrow' a dlen if necesary */ - nelems = (npos + (loop_diff * dlen)) / head->elem_size; - if(!nelems){ /* if no new elems we're done */ - ReleaseMutex(std::get<4>(buf_info)); - return; - }else if(nelems < 0){ /* if something very bad happened */ - ReleaseMutex(std::get<4>(buf_info)); - throw TOSDB_BufferError("numElems < 0"); - }else{ /* extract */ - - if((unsigned long long)nelems >= (dlen / head->elem_size)){ - /* make sure we don't insert more than the max elems */ - nelems = dlen / head->elem_size; - } - - do{ /* go through each elem, last first */ - spot = (char*)head + - (((head->next_offset - (nelems * head->elem_size)) + dlen) % dlen); - - for(const TOSDBlock* block : std::get<2>(buf_info)){ - /* insert those elements into each block's raw data block */ - block-> - block-> - insert_data( - topic, - item, - _castToVal(spot), - *(pDateTimeStamp)(spot + ((head->elem_size) - sizeof(DateTimeStamp))) - ); - } - }while(--nelems); - } - /* adjust our buffer info to the present values */ - std::get<0>(buf_info) = head->next_offset - head->beg_offset; - std::get<1>(buf_info) = head->loop_seq; - - ReleaseMutex(std::get<4>(buf_info)); -} - - -DWORD WINAPI -_cleanupBlock(LPVOID lParam) -{ - TOSDB_RawDataBlock* cln = (TOSDB_RawDataBlock*)lParam; - if(cln) - delete cln; - - return 0; -} - - -DWORD WINAPI -_threadedExtractLoop(LPVOID lParam) -{ - using namespace std::chrono; - - steady_clock_type::time_point tbeg; - steady_clock_type::time_point tend; - long tdiff; - long probe_waiting; - - if( master.connected(TOSDB_DEF_TIMEOUT) ) - aware_of_connection.store(true); - - while( _connected() ){ - /* [jan 2017] we should be more careful about how often we call _connected - (i.e every 30 msec is no good) */ - probe_waiting = 0; - /* after waiting for (atleast) TOSDB_PROBE_WAIT msec break to check for connection */ - while(probe_waiting < TOSDB_PROBE_WAIT && aware_of_connection.load()){ - /* the concurrent read loop errs on the side of greedyness */ - tbeg = steady_clock.now(); /* include time waiting for lock */ - { - LOCAL_BUFFERS_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - for(buffers_ty::value_type & buf : buffers) - { - switch(TOS_Topics::TypeBits(buf.first.first)){ - case TOSDB_STRING_BIT : - _extractFromBuffer(buf.first.first, buf.first.second, buf.second); - break; - case TOSDB_INTGR_BIT : - _extractFromBuffer(buf.first.first, buf.first.second, buf.second); - break; - case TOSDB_QUAD_BIT : - _extractFromBuffer(buf.first.first, buf.first.second, buf.second); - break; - case TOSDB_INTGR_BIT | TOSDB_QUAD_BIT : - _extractFromBuffer(buf.first.first, buf.first.second, buf.second); - break; - default : - _extractFromBuffer(buf.first.first, buf.first.second, buf.second); - }; - } - /* --- CRITICAL SECTION --- */ - } /* make sure we give up this lock each time through the buffers */ - tend = steady_clock.now(); - tdiff = duration_cast>(tend - tbeg).count(); - /* 0 <= (buffer_latency - tdiff) <= Glacial */ - Sleep(std::min(std::max((buffer_latency - tdiff),0),Glacial)); - probe_waiting += buffer_latency; - } - } - aware_of_connection.store(false); - buffer_thread = NULL; - buffer_thread_id = 0; - return 0; -} - - -int -_createBlock(LPCSTR id, - size_type sz, - BOOL is_datetime, - size_type timeout) -{ - TOSDBlock* db; - - if( !IsValidBlockSize(sz) ) - return TOSDB_ERROR_BLOCK_SIZE; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - if( GetBlockPtr(id,false) ){ - TOSDB_LogH("BLOCK", ("block (" + std::string(id) + ") already exists").c_str()); - return TOSDB_ERROR_BLOCK_ALREADY_EXISTS; - } - - db = new TOSDBlock; - db->timeout = std::max(timeout,TOSDB_MIN_TIMEOUT); - db->block = nullptr; - - try{ - db->block = TOSDB_RawDataBlock::CreateBlock(sz, is_datetime); - }catch(const TOSDB_DataBlockLimitError){ - TOSDB_LogH("BLOCK", "attempt to exceed block limit"); - }catch(const std::exception& e){ - TOSDB_LogH("BLOCK", e.what()); - } - - if(!db->block){ - TOSDB_LogH("BLOCK", "failed to create block"); - delete db; - return TOSDB_ERROR_BLOCK_CREATION; - } - - dde_blocks.insert( std::pair(id, db) ); - return 0; - /* --- CRITICAL SECTION --- */ -} - -}; /* namespace */ - - -BOOL WINAPI -DllMain(HANDLE mod, DWORD why, LPVOID res) /* ENTRY POINT */ -{ - /* NOTE: using HANDLE vs HINSTANCE for some reason; - no current need for mod so just leave for now to avoid - breaking something (Nov 10 2016) - */ - switch(why){ - case DLL_PROCESS_ATTACH: - /* ! NO AUTO-CONNECT ! need blocking ops deep into ->try_for_slave() */ - { - std::string p = BuildLogPath(LOG_NAME); - StartLogging(p.c_str()); - } - break; - case DLL_THREAD_ATTACH: - break; - case DLL_THREAD_DETACH: - break; - case DLL_PROCESS_DETACH: - { - for(const auto & buffer : buffers) - {/* signal the service and close the handles */ - _requestStreamOP(buffer.first.first, buffer.first.second, - TOSDB_DEF_TIMEOUT, TOSDB_SIG_REMOVE); - UnmapViewOfFile(std::get<3>(buffer.second)); - CloseHandle(std::get<4>(buffer.second)); - } - /* needs to come after close ops or _requestStreamOP will fail on _connected() */ - aware_of_connection.store(false); - StopLogging(); - } - break; - default: - break; - } - - return TRUE; -} - - -int -TOSDB_Connect() -{ - /* We should be able to block in here as long as this is not called from DllMain */ - if(_connected()) - return 0; - - if(buffer_thread) - return 0; - - char mod_name[MAX_PATH+1]; - memset(mod_name, 0, MAX_PATH+1); - GetModuleFileName(NULL, mod_name, MAX_PATH+1); - std::string mod_name_str(mod_name); - - buffer_thread = CreateThread(0, 0, _threadedExtractLoop, 0, 0, &buffer_thread_id); - if(!buffer_thread){ - TOSDB_LogH("THREAD", "error initializing _threadedExtractLoop"); - TOSDB_Log("IPC", ("NOT connected to engine, client: " + mod_name_str).c_str()); - return TOSDB_ERROR_CONCURRENCY; - } - - for(int msec = TOSDB_DEF_PAUSE; msec <= TOSDB_DEF_TIMEOUT; msec += TOSDB_DEF_PAUSE){ - /* we need a timed wait on aware_of_connection to avoid situations - where a lib call is made before _threadedExtractLoop sets it to true */ - if(aware_of_connection.load()){ - TOSDB_Log("IPC", ("connected to engine, client: " + mod_name_str).c_str()); - return 0; - } - Sleep(TOSDB_DEF_PAUSE); - } - - TOSDB_LogH("IPC", "timed out waiting for aware_of_connection"); - TOSDB_Log("IPC", ("NOT connected to engine, client: " + mod_name_str).c_str()); - return TOSDB_ERROR_TIMEOUT; -} - - -int -TOSDB_Disconnect() -{ - aware_of_connection.store(false); - return 0; -} - - -/* DEPRECATED - Jan 10 2017 */ -unsigned int -TOSDB_IsConnected() -{ - return TOSDB_IsConnectedToEngine(); -} - - -/* ... in favor of... */ -unsigned int -TOSDB_IsConnectedToEngine() -{ - return (unsigned int)_connected(); -} - - -/* ... and... */ -unsigned int -TOSDB_IsConnectedToEngineAndTOS() -{ - if(!_connected()) - return 0; - - return _requestStreamOP(TEST_TOPIC, TEST_ITEM, TOSDB_MIN_TIMEOUT, TOSDB_SIG_TEST) ? 0 : 1; -} - - -/* ... and... */ -unsigned int -TOSDB_ConnectionState() -{ - if(!_connected()) - return TOSDB_CONN_NONE; - - if( _requestStreamOP(TEST_TOPIC, TEST_ITEM, TOSDB_MIN_TIMEOUT, TOSDB_SIG_TEST) ) - return TOSDB_CONN_ENGINE; - - /* TODO: develop a heuristic for inicating: - a) we're connected to engine AND TOS - b) we're not getting good/normal data */ - - return TOSDB_CONN_ENGINE_TOS; - -} - - -/* note: if we want to do anything else with the Reserved Block(s) we'll - need to introduce private versions of the API that don't call - IsValidBlockID (i.e TOSDB_CreateBlock / _createBlock) */ -int -TOSDB_CreateBlock(LPCSTR id, - size_type sz, - BOOL is_datetime, - size_type timeout) -{ - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - if( !IsValidBlockID(id) ) - return TOSDB_ERROR_BAD_INPUT; - - return _createBlock(id,sz,is_datetime,timeout); -} - - -int -TOSDB_Add(std::string id, str_set_type items, topic_set_type topics_t) -/* adding sets of topics and strings, dealing with pre-cache; - all Add_ methods end up in here */ -{ - topic_set_type tdiff; - topic_set_type old_topics; - topic_set_type tot_topics; - str_set_type idiff; - str_set_type old_items; - str_set_type tot_items; - str_set_type iunion; - bool is_empty; - TOSDBlock *db; - - HWND hndl = NULL; - int err = TOSDB_ERROR_DECREMENT_BASE; - - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - /* log and remove NULL topics */ - old_topics = topics_t; - topics_t.clear(); - for( auto& t : old_topics ){ - if( t == TOS_Topics::TOPICS::NULL_TOPIC ) - TOSDB_LogH("INPUT", "NULL topic removed from TOSDB_Add"); - else - topics_t.insert(t); - } - old_topics.clear(); - - /* log and remove INVALID items */ - old_items = items; - items.clear(); - for( auto& s : old_items ){ - if( !IsValidItemString(s.c_str()) ) - TOSDB_LogH("INPUT", ("Invalid item string removed from TOSDB_Add: " + s).c_str() ); - else - items.insert(s); - } - old_items.clear(); - - /* fail if both are empty - note: this could result from NULL topics and/or invalid items*/ - if( items.empty() && topics_t.empty() ) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - db = _getBlockPtr(id); - if(!db){ - TOSDB_LogH("BLOCK", ("block (" + id + ") doesn't exist").c_str()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - } - - old_topics = db->block->topics(); - old_items = db->block->items(); - - if( !db->item_precache.empty() ) - /* if we have pre-cached items, include them */ - std::set_union(items.cbegin(), items.cend(), - db->item_precache.cbegin(), db->item_precache.cend(), - std::insert_iterator(tot_items,tot_items.begin())); - else - tot_items = items; /* 'copy', keep items available for pre-caching */ - - - if( !db->topic_precache.empty() ) - /* if we have pre-cached topics, include them */ - std::set_union(topics_t.cbegin(), topics_t.cend(), - db->topic_precache.cbegin(), db->topic_precache.cend(), - std::insert_iterator(tot_topics,tot_topics.begin()), - TOS_Topics::top_less()); - else - tot_topics = std::move(topics_t); /* move, don't need topics_t anymore */ - - - /* find new items and topics to add */ - std::set_difference(tot_topics.cbegin(), tot_topics.cend(), - old_topics.cbegin(), old_topics.cend(), - std::insert_iterator(tdiff,tdiff.begin()), - TOS_Topics::top_less()); - - std::set_difference(tot_items.cbegin(), tot_items.cend(), - old_items.cbegin(), old_items.cend(), - std::insert_iterator(idiff, idiff.begin())); - - - if( !tdiff.empty() ){ - /* if new topics, atleast one item, add them to the block - add ALL the items (new and old) for each */ - std::set_union(tot_items.cbegin(), tot_items.cend(), - old_items.cbegin(), old_items.cend(), - std::insert_iterator(iunion, iunion.begin())); - - is_empty = iunion.empty(); - - for(auto & topic : tdiff){ - if(is_empty) - db->topic_precache.insert(topic); - - for(auto & item : iunion){ - /* TRY TO ADD TO BLOCK */ - if(_requestStreamOP(topic, item, db->timeout, TOSDB_SIG_ADD) == 0){ - db->block->add_topic(topic); - db->block->add_item(item); - db->item_precache.clear(); - db->topic_precache.clear(); - _captureBuffer(topic, item, db); - }else - --err; - } - } - }else if(old_topics.empty()){ /* don't ignore items if no topics yet.. */ - for(auto & i : items) - db->item_precache.insert(i); /* ...pre-cache them */ - } - - - /* add new items to the old topics */ - for(auto & topic : old_topics){ - for(auto & item : idiff){ - /* TRY TO ADD TO BLOCK */ - if(_requestStreamOP(topic, item, db->timeout, TOSDB_SIG_ADD) == 0){ - db->block->add_item(item); - _captureBuffer(topic, item, db); - }else - --err; - } - } - - /* if we didn't decr err return success */ - return (err == TOSDB_ERROR_DECREMENT_BASE) ? 0 : err; - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_AddTopic(std::string id, TOS_Topics::TOPICS topic_t) -{ - if( !IsValidBlockID(id.c_str()) ) - return TOSDB_ERROR_BAD_INPUT; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC){ - TOSDB_LogH("INPUT", "NULL topic passed to TOSDB_AddTopic"); - return TOSDB_ERROR_BAD_TOPIC; - } - - return TOSDB_Add(id, str_set_type(), topic_t); -} - - -int -TOSDB_AddItem(std::string id, std::string item) -{ - if( !IsValidBlockID(id.c_str()) || !IsValidItemString(item.c_str()) ) - return TOSDB_ERROR_BAD_INPUT; - - return TOSDB_Add(id, item, topic_set_type()); -} - - -int -TOSDB_AddTopic(LPCSTR id, LPCSTR topic_str) -{ - if( !IsValidBlockID(id) || !CheckStringLength(topic_str) ) - return TOSDB_ERROR_BAD_INPUT; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_AddTopic(id, t); -} - - -int -TOSDB_AddTopics(std::string id, topic_set_type topics_t) -{ - if( !IsValidBlockID(id.c_str()) ) - return TOSDB_ERROR_BAD_INPUT; - - return TOSDB_Add(id, str_set_type(), topics_t); -} - - -int -TOSDB_AddTopics(LPCSTR id, LPCSTR* topics_str, size_type topics_len) -{ - if( !IsValidBlockID(id) || !CheckStringLengths(topics_str, topics_len) ) - return TOSDB_ERROR_BAD_INPUT; - - auto f = [=](LPCSTR str){ return GetTopicEnum(str); }; - topic_set_type tset(topics_str, topics_len, f); - - return TOSDB_Add(id, str_set_type(), std::move(tset)); -} - - -int -TOSDB_AddItem(LPCSTR id, LPCSTR item) -{ - if( !IsValidBlockID(id) || !IsValidItemString(item) ) - return TOSDB_ERROR_BAD_INPUT; - - return TOSDB_Add(id, std::string(item), topic_set_type()); -} - - -int -TOSDB_AddItems(std::string id, str_set_type items) -{ - if( !IsValidBlockID(id.c_str()) ) - return TOSDB_ERROR_BAD_INPUT; - - return TOSDB_Add(id, items, topic_set_type()); -} - - -int -TOSDB_AddItems(LPCSTR id, LPCSTR* items, size_type items_len) -{ - if( !IsValidBlockID(id) || !CheckStringLengths(items, items_len) ) - return TOSDB_ERROR_BAD_INPUT; - - return TOSDB_Add(id, str_set_type(items,items_len), topic_set_type()); -} - - -int -TOSDB_Add(LPCSTR id, - LPCSTR* items, - size_type items_len, - LPCSTR* topics_str, - size_type topics_len) -{ - if( !IsValidBlockID(id) - || !CheckStringLengths(items, items_len) - || !CheckStringLengths(topics_str, topics_len) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - auto f = [=](LPCSTR str){ return GetTopicEnum(str); }; - topic_set_type tset(topics_str, topics_len, f); - - return TOSDB_Add(id, str_set_type(items,items_len), std::move(tset)); -} - - -int -TOSDB_RemoveTopic(LPCSTR id, LPCSTR topic_str) -{ - if( !IsValidBlockID(id) || !CheckStringLength(topic_str) ) - return TOSDB_ERROR_BAD_INPUT; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC){ - return TOSDB_ERROR_BAD_TOPIC; - } - - return TOSDB_RemoveTopic(id, t); -} - - -int -TOSDB_RemoveTopic(std::string id, TOS_Topics::TOPICS topic_t) -{ - TOSDBlock* db; - int err = TOSDB_ERROR_DECREMENT_BASE; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC){ - TOSDB_Log("INPUT", "NULL topic passed to TOSDB_RemoveTopic"); - return TOSDB_ERROR_BAD_TOPIC; - } - - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - db = _getBlockPtr(id); - if(!db){ - TOSDB_LogH("BLOCK", ("block (" + id + ") doesn't exist").c_str()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - } - - if( db->block->has_topic(topic_t) ){ - for(auto & item : db->block->items()) - { - _releaseBuffer(topic_t, item, db); - if(_requestStreamOP(topic_t, item, db->timeout, TOSDB_SIG_REMOVE) != 0){ - --err; - TOSDB_LogH("IPC","_requestStreamOP(REMOVE) failed, stream leaked"); - } - } - db->block->remove_topic(topic_t); - if( db->block->topics().empty() ){ - for(const std::string & item : db->block->items()){ - db->item_precache.insert(item); - db->block->remove_item(item); - } - } - }else if(db->topic_precache.find(topic_t) == db->topic_precache.end()){ - return TOSDB_ERROR_BAD_TOPIC; - } - - db->topic_precache.erase(topic_t); - /* if we didn't decr err return success */ - return (err == TOSDB_ERROR_DECREMENT_BASE) ? 0 : err; - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_RemoveItem(std::string id, std::string item) -{ - return TOSDB_RemoveItem(id.c_str(), item.c_str()); -} - - -int -TOSDB_RemoveItem(LPCSTR id, LPCSTR item) -{ - TOSDBlock* db; - int err = TOSDB_ERROR_DECREMENT_BASE; - - if( !CheckStringLength(item) || !IsValidBlockID(id) ) - return TOSDB_ERROR_BAD_INPUT; - - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - db = _getBlockPtr(id); - if(!db){ - TOSDB_LogH("BLOCK", ("block (" + std::string(id) + ") doesn't exist").c_str()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - } - - if( db->block->has_item(item) ){ - for(auto topic : db->block->topics()) - { - _releaseBuffer(topic, item, db); - if(_requestStreamOP(topic, item, db->timeout, TOSDB_SIG_REMOVE) != 0){ - --err; - TOSDB_LogH("IPC","_requestStreamOP(REMOVE) failed, stream leaked"); - } - } - db->block->remove_item(item); - if( db->block->items().empty() ){ - for(const TOS_Topics::TOPICS topic : db->block->topics()){ - db->topic_precache.insert (topic); - db->block->remove_topic(topic); - } - } - }else if(db->item_precache.find(item) == db->item_precache.end()){ - return TOSDB_ERROR_BAD_ITEM; - } - - db->item_precache.erase(item); - /* if we didn't decr err return success */ - return (err == TOSDB_ERROR_DECREMENT_BASE) ? 0 : err; - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_CloseBlock(LPCSTR id) -{ - TOSDBlock* db; - HANDLE del_thrd_hndl; - DWORD del_thrd_id; - - int err = TOSDB_ERROR_DECREMENT_BASE; - - if( !IsValidBlockID(id) ) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - db = _getBlockPtr(id); - if(!db){ - TOSDB_LogH("BLOCK", ("block (" + std::string(id) + ") doesn't exist").c_str()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - } - - for(auto & item : db->block->items()){ - for(auto topic : db->block->topics()) - { - _releaseBuffer(topic, item, db); - if(_requestStreamOP(topic, item, db->timeout, TOSDB_SIG_REMOVE) != 0){ - --err; - TOSDB_LogH("IPC", "_requestStreamOP(REMOVE) failed, stream leaked"); - } - } - } - - dde_blocks.erase(id); - - /* spin-off block destruction to its own thread so we don't block main */ - del_thrd_hndl = CreateThread(NULL, 0, _cleanupBlock, (LPVOID)db->block, 0, &(del_thrd_id)); - if(!del_thrd_hndl){ - TOSDB_LogH("THREAD", "using main thread to clean-up(THIS MAY BLOCK)"); - _cleanupBlock((LPVOID)db->block); - } - - delete db; - - /* if we didn't decr err return success */ - return (err == TOSDB_ERROR_DECREMENT_BASE) ? 0 : err; - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_CloseBlocks() -{ - std::map bcopy; - int err = TOSDB_ERROR_DECREMENT_BASE; - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - /* need a copy, _CloseBlock removes from original */ - std::insert_iterator> i(bcopy,bcopy.begin()); - std::copy(dde_blocks.begin(), dde_blocks.end(), i); - for(auto & b: bcopy){ - if( TOSDB_CloseBlock(b.first.c_str()) ) - --err; - } - /* --- CRITICAL SECTION --- */ - }catch(...){ - return TOSDB_ERROR_UNKNOWN; // really ? - } - - /* if we didn't decr err return success */ - return (err == TOSDB_ERROR_DECREMENT_BASE) ? 0 : err; -} - - -int -TOSDB_DumpSharedBufferStatus() -{ - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - - std::string msg = std::to_string(TOSDB_SIG_DUMP); - - if( !master.call(&msg, TOSDB_DEF_TIMEOUT) ){ - TOSDB_LogH("IPC",("master.call failled, msg:" + msg).c_str()); - return TOSDB_ERROR_IPC; - } - - try{ - return (std::stol(msg) == TOSDB_SIG_GOOD) ? 0 : TOSDB_ERROR_IPC; - }catch(...){ - TOSDB_LogH("IPC", "failed to convert return message to long"); - return TOSDB_ERROR_IPC; - } - /* --- CRITICAL SECTION --- */ -} - - -/* WARNING - removes individual stream from the engine. It should only - be used when you are certain a client lib has failed to close a stream - during destruction of the containing block (e.g the program crashes - before DLL_PROCESS_DETACH in DllMain is hit.) - - YOU NEED TO BE SURE THIS IS THE CASE OR YOU CAN CORRUPT THE UNDERLYING - BUFFERS FOR ANY OR ALL CLIENT INSTANCE(S)! */ -int -TOSDB_RemoveOrphanedStream(LPCSTR item, LPCSTR topic_str) -{ - if( !CheckStringLength(item) || !CheckStringLength(topic_str) ) - return TOSDB_ERROR_BAD_INPUT; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC){ - return TOSDB_ERROR_BAD_TOPIC; - } - - if( !_connected(true) ) - return TOSDB_ERROR_NOT_CONNECTED; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return _requestStreamOP(t, item, TOSDB_DEF_TIMEOUT, TOSDB_SIG_REMOVE); - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_GetBlockIDs(LPSTR* dest, size_type array_len, size_type str_len) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - if(array_len < dde_blocks.size()) - return TOSDB_ERROR_BAD_INPUT; - - int i = 0; - - for(auto & name : dde_blocks){ - if( strcpy_s(dest[i++], str_len, name.first.c_str()) ) - return TOSDB_ERROR_BAD_INPUT; - } - - return 0; - /* --- CRITICAL SECTION --- */ -} - -str_set_type -TOSDB_GetBlockIDs() -{ - str_set_type tmp; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - for(auto & name : dde_blocks) - tmp.insert(name.first); - - return tmp; - /* --- CRITICAL SECTION --- */ -} - - -unsigned long -TOSDB_GetLatency() -{ - return buffer_latency; -} - - -int -TOSDB_GetClientLogPath(char* path, size_type sz) -{ - std::string p = BuildLogPath(LOG_NAME); - /* downcast OK because we limit length of log path */ - size_type psz = p.size() + 1; /* for /n */ - - if(sz < psz) - return psz; - - return strcpy_s(path,psz,p.c_str()) ? -1 : 0; -} - - -unsigned long -TOSDB_SetLatency(UpdateLatency latency) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - unsigned long tmp = buffer_latency; - - switch(latency){ - case Fastest: - case VeryFast: - case Fast: - case Moderate: - case Slow: - case Glacial: - buffer_latency = latency; - break; - default: - buffer_latency = Moderate; - } - - return tmp; - /* --- CRITICAL SECTION --- */ -} - - -TOS_Topics::TOPICS -GetTopicEnum(std::string topic_str, bool log_if_null) -{ - /* operator[] uses .at(), catches out_of_range, returns NULL_TOPIC */ - TOS_Topics::TOPICS t = TOS_Topics::map[topic_str]; - if(log_if_null && t == TOS_Topics::TOPICS::NULL_TOPIC) - TOSDB_LogH("INPUT", ("bad topic string: " + std::string(topic_str)).c_str()); - - return t; -} - - -const TOSDBlock* -GetBlockPtr(const std::string id, bool log) -{ - try{ - return dde_blocks.at(id); - }catch(...){ - if(log){ - std::string msg = "block (" + id + ") doesn't exist"; - TOSDB_LogH("BLOCK", msg.c_str()); - } - return NULL; - } -} - - -const TOSDBlock* -GetBlockOrThrow(std::string id) -{ - const TOSDBlock* db; - - db = GetBlockPtr(id,false); - if(!db) - throw TOSDB_DataBlockDoesntExist(id.c_str()); - - return db; -} - - diff --git a/src/client/client_get.cpp b/src/client/client_get.cpp deleted file mode 100644 index 5cc6ea4..0000000 --- a/src/client/client_get.cpp +++ /dev/null @@ -1,2108 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "raw_data_block.hpp" -#include "client.hpp" -#include -#include -#include -#include - -size_type -TOSDB_GetBlockLimit() -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return TOSDB_RawDataBlock::max_block_count(); - /* --- CRITICAL SECTION --- */ -} - -size_type -TOSDB_SetBlockLimit(size_type sz) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return TOSDB_RawDataBlock::max_block_count(sz); - /* --- CRITICAL SECTION --- */ -} - -size_type -TOSDB_GetBlockCount() -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return TOSDB_RawDataBlock::block_count(); - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetBlockSize(LPCSTR id, size_type* pSize) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *pSize = GetBlockOrThrow(id)->block->block_size(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetgetBlockSize", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -int -TOSDB_SetBlockSize(LPCSTR id, size_type sz) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - if(!IsValidBlockSize(sz)) - return TOSDB_ERROR_BLOCK_SIZE; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - GetBlockOrThrow(id)->block->block_size(sz); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_SET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetSetBlockSize", e.what()); - return TOSDB_ERROR_SET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -int -TOSDB_GetItemCount(LPCSTR id, size_type* count) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *count = GetBlockOrThrow(id)->block->item_count(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetPreItemCount", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -int -TOSDB_GetTopicCount(LPCSTR id, size_type* count) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *count = GetBlockOrThrow(id)->block->topic_count(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetTopicCount", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - - -int -TOSDB_GetTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -{ - const TOSDBlock *db; - topic_set_type topics; - int i; - - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockPtr(id); - if(!db) - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - topics = db->block->topics(); - if (array_len < topics.size()) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - i = 0; - for(auto & t : topics){ - if( strcpy_s(dest[i++], str_len, TOS_Topics::map[t].c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - - return 0; - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -{ - const TOSDBlock *db; - str_set_type items; - int i; - - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockPtr(id); - if (!db) - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - items = db->block->items(); - if (array_len < items.size()) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - i = 0; - for(auto & item : items){ - if( strcpy_s(dest[i++], str_len, item.c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - - return 0; - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_GetPreCachedItemCount(LPCSTR id, size_type* count) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *count = GetBlockOrThrow(id)->item_precache.size(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetPreCachedItemCount", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -int -TOSDB_GetPreCachedTopicCount(LPCSTR id, size_type* count) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *count = GetBlockOrThrow(id)->topic_precache.size(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetPreCachedTopicCount", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - - -int -TOSDB_GetPreCachedTopicNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -{ - const TOSDBlock *db; - int i; - - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockPtr(id); - if (!db) - return TOSDB_ERROR_BLOCK_DOESNT_EXIST;; - - if (array_len < db->topic_precache.size()) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - i = 0; - for(auto & t : db->topic_precache){ - if( strcpy_s(dest[i++], str_len,TOS_Topics::map[t].c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - - return 0; - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetPreCachedItemNames(LPCSTR id, LPSTR* dest, size_type array_len, size_type str_len) -{ - const TOSDBlock *db; - int i; - - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockPtr(id); - if (!db) - return TOSDB_ERROR_BLOCK_DOESNT_EXIST;; - - if (array_len < db->item_precache.size()) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - i = 0; - for(auto & item : db->item_precache){ - if( strcpy_s(dest[i++], str_len, item.c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - - return 0; - /* --- CRITICAL SECTION --- */ -} - - -int -TOSDB_GetTypeBits(LPCSTR topic_str, type_bits_type* type_bits) -{ - TOS_Topics::TOPICS t; - - if(!CheckStringLength(topic_str)) - return TOSDB_ERROR_BAD_INPUT; - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - *type_bits = TOSDB_GetTypeBits(t); - return 0; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetTypeBits", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -int -TOSDB_GetTypeString(LPCSTR topic_str, LPSTR dest, size_type str_len) -{ - TOS_Topics::TOPICS t; - - if(!CheckStringLength(topic_str)) - return TOSDB_ERROR_BAD_INPUT; - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - if( strcpy_s(dest, str_len, TOS_Topics::TypeString(t).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - return 0; -} - -int -TOSDB_IsUsingDateTime(LPCSTR id, unsigned int* is_datetime) -{ - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - *is_datetime = GetBlockOrThrow(id)->block->uses_dtstamp(); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_IsUsingDateTime", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -topic_set_type -TOSDB_GetTopicEnums(std::string id) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->topics(); - /* --- CRITICAL SECTION --- */ -} - -str_set_type -TOSDB_GetTopicNames(std::string id) -{ - const TOSDBlock *db; - auto f = [&](TOS_Topics::TOPICS t){ return TOS_Topics::map[t]; }; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return str_set_type(db->block->topics(), f); - /* --- CRITICAL SECTION --- */ -} - -str_set_type -TOSDB_GetItemNames(std::string id) -{ - const TOSDBlock* db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->items(); - /* --- CRITICAL SECTION --- */ -} - -topic_set_type -TOSDB_GetPreCachedTopicEnums(std::string id) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->topic_precache; - /* --- CRITICAL SECTION --- */ -} - -str_set_type -TOSDB_GetPreCachedTopicNames(std::string id) -{ - const TOSDBlock *db; - auto f = [=](TOS_Topics::TOPICS top){ return TOS_Topics::map[top]; }; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return str_set_type(db->topic_precache, f); - /* --- CRITICAL SECTION --- */ -} - -str_set_type -TOSDB_GetPreCachedItemNames(std::string id) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->item_precache; - /* --- CRITICAL SECTION --- */ -} - -type_bits_type -TOSDB_GetTypeBits(TOS_Topics::TOPICS topic_t) -{ - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - return TOS_Topics::TypeBits(topic_t); -} - -std::string -TOSDB_GetTypeString(TOS_Topics::TOPICS topic_t) -{ - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - return TOS_Topics::TypeString(topic_t); -} - -size_type -TOSDB_GetItemCount(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->block->item_count(); - /* --- CRITICAL SECTION --- */ -} - -size_type -TOSDB_GetTopicCount(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->block->topic_count(); - /* --- CRITICAL SECTION --- */ -} - - -size_type -TOSDB_GetPreCachedItemCount(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->item_precache.size(); - /* --- CRITICAL SECTION --- */ -} - -size_type -TOSDB_GetPreCachedTopicCount(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->topic_precache.size(); - /* --- CRITICAL SECTION --- */ -} - - -void -TOSDB_SetBlockSize(std::string id, size_type sz) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - GetBlockOrThrow(id)->block->block_size(sz); - /* --- CRITICAL SECTION --- */ -} - -size_type -TOSDB_GetBlockSize(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->block->block_size(); - /* --- CRITICAL SECTION --- */ -} - -bool -TOSDB_IsUsingDateTime(std::string id) -{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - return GetBlockOrThrow(id)->block->uses_dtstamp(); - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetStreamOccupancy(LPCSTR id, LPCSTR item, LPCSTR topic_str, size_type* sz) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, t); - *sz = (size_type)(dat->size()); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetStreamOccupancy", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -size_type -TOSDB_GetStreamOccupancy(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - - try{ - return (size_type)(dat->size()); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_GetStreamOccupancy"); - } - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetMarkerPosition(LPCSTR id, LPCSTR item, LPCSTR topic_str, long long* pos) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, t); - *pos = (dat->marker_position()); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetMarkerPosition", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -long long -TOSDB_GetMarkerPosition(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return dat->marker_position(); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_GetMarkerPosition"); - } - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_IsMarkerDirty(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - unsigned int* is_dirty) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, t); - *is_dirty = (unsigned int)(dat->is_marker_dirty()); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_STATE; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_IsMarkerDirty", e.what()); - return TOSDB_ERROR_GET_STATE; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -bool -TOSDB_IsMarkerDirty(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return dat->is_marker_dirty(); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_IsMarkerDirty"); - } - /* --- CRITICAL SECTION --- */ -} - -template<> -generic_type -TOSDB_Get(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long indx) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return dat->operator[](indx); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_Get"); - } - /* --- CRITICAL SECTION --- */ -} - -template<> -generic_dts_type -TOSDB_Get(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long indx) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return dat->both(indx); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_Get"); - } - /* --- CRITICAL SECTION --- */ -} - -/* use functors of partially specialized object to overload by explicit call */ -template -struct GRetType; - -template -struct GRetType{ - std::pair - operator()(T val, DateTimeStamp dts) - { - return std::pair(val, std::move(dts)); - } -}; - -template -struct GRetType{ - T - operator()(T val, DateTimeStamp dts) - { - return val; - } -}; - -template -auto -TOSDB_Get(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long indx) -> typename std::conditional, T>::type -{ - T tmp; - DateTimeStamp datetime; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - if(TOSDB_Get_(id, item, topic_t, indx, &tmp, &datetime)) - throw TOSDB_DataStreamError("TOSDB_Get_"); - - return GRetType()(tmp, std::move(datetime)); -} - -template -int -TOSDB_Get_(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long indx, - T* dest, - pDateTimeStamp datetime) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - dat->copy(dest, 1,indx, indx, datetime); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_Get", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template -int -TOSDB_Get_(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - T* dest, - pDateTimeStamp datetime) -{ - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_Get_(id, item, t, indx, dest, datetime); -} - -int -TOSDB_GetDouble(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - double* dest, - pDateTimeStamp datetime) -{ - return TOSDB_Get_(id, item, topic_str , indx, dest, datetime); -} - -int -TOSDB_GetFloat(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - float* dest, - pDateTimeStamp datetime) -{ - return TOSDB_Get_(id, item, topic_str , indx, dest, datetime); -} - -int -TOSDB_GetLongLong(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - long long* dest, - pDateTimeStamp datetime) -{ - return TOSDB_Get_(id, item, topic_str , indx, dest, datetime); -} - -int -TOSDB_GetLong(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - long* dest, - pDateTimeStamp datetime) -{ - return TOSDB_Get_(id, item, topic_str , indx, dest, datetime); -} - -int -TOSDB_GetString(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long indx, - LPSTR dest, - size_type str_len, - pDateTimeStamp datetime) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, t); - dat->copy(&dest, 1, str_len, indx,indx,datetime); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetString", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template<> -generic_vector_type -TOSDB_GetStreamSnapshot(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long end, - long beg) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return dat->vector(end, beg); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e,"TOSDB_GetStreamSnapshot"); - } - /* --- CRITICAL SECTION --- */ -} - -template<> -std::pair,std::vector> -TOSDB_GetStreamSnapshot(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long end, - long beg) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - try{ - return std::pair, - std::vector>( - dat->vector(end, beg), - dat->secondary_vector(end,beg) - ); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_GetStreamSnapshot"); - } - /* --- CRITICAL SECTION --- */ -} - -/* - use functors of partially specialized object to overload by explicit call - note: data_stream takes raw pointers so we need to pass the addr of the - dereferenced first element of the vector - - ARE WE RELYING ON VECTOR IMPLEMENTATION ?? -*/ -template -struct GSSRetType; - -template -struct GSSRetType{ - std::pair, std::vector> - operator()(TOSDB_RawDataBlock::stream_const_ptr_type dat, - long end, - long beg, - size_t diff) - { - std::vector v(diff); /* adjust for [) -> [ ] */ - std::vector dtsv(diff); /* adjust for [) -> [ ] */ - if(diff > 0) - dat->copy(&(*(v.begin())), diff, end, beg, &(*(dtsv.begin()))); - - return std::pair,std::vector>(v, dtsv); - } -}; - -template -struct GSSRetType{ - std::vector - operator()(TOSDB_RawDataBlock::stream_const_ptr_type dat, - long end, - long beg, - size_t diff) - { - std::vector v(diff); /* adjust for [) -> [ ] */ - if(diff > 0) - dat->copy(&(*(v.begin())), diff, end, beg, nullptr); - - return v; - } -}; - -template -auto -TOSDB_GetStreamSnapshot(std::string id, - std::string item, - TOS_Topics::TOPICS topic_t, - long end, - long beg) - -> typename std::conditional, std::vector>, - std::vector>::type -{ - size_type sz; - long diff; - long long mindiff; - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); /* get stream size */ - sz = (size_type)(dat->bound_size()); - - if(end < 0) - end += sz; - if(beg < 0) - beg += sz; - diff = end - beg; - if(diff < 0) - throw std::invalid_argument("end < beg"); - - mindiff = (long long)std::min((size_t)end + 1, dat->size()) - beg; - if(mindiff < 0) - mindiff = 0; - else if(mindiff > TOSDB_MAX_BLOCK_SZ) /* protect the size_t cast(below) */ - throw std::invalid_argument("mindiff > TOSDB_MAX_BLOCK_SZ"); - - try{ - return GSSRetType()(dat, end, beg, (size_t)mindiff); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "TOSDB_GetStreamSnapshot"); - } - /* --- CRITICAL SECTION --- */ -} - -template -int -TOSDB_GetStreamSnapshot_(LPCSTR id, - LPCSTR item, - TOS_Topics::TOPICS topic_t, - T* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(!IsValidBlockID(id) || !CheckStringLength(item)) - { - return TOSDB_ERROR_BAD_INPUT; - } - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - dat->copy(dest,array_len,end,beg,datetime); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("GetStreamSnapshot", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template -int -TOSDB_GetStreamSnapshot_(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - T* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - if(!CheckStringLength(topic_str)) /* let this go thru std::string ? */ - return TOSDB_ERROR_BAD_INPUT; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_GetStreamSnapshot_(id, item, t, dest, array_len, datetime, end, beg); -} - -int -TOSDB_GetStreamSnapshotDoubles(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - double* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - return TOSDB_GetStreamSnapshot_(id, item, topic_str, dest, array_len, - datetime, end, beg); -} - -int -TOSDB_GetStreamSnapshotFloats(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - float* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - return TOSDB_GetStreamSnapshot_(id, item, topic_str, dest, array_len, - datetime, end, beg); -} - -int -TOSDB_GetStreamSnapshotLongLongs(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long long* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - return TOSDB_GetStreamSnapshot_(id, item, topic_str, dest, array_len, - datetime, end, beg); -} - -int -TOSDB_GetStreamSnapshotLongs(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long* dest, - size_type array_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - return TOSDB_GetStreamSnapshot_(id, item, topic_str, dest, array_len, - datetime, end, beg); -} - -int -TOSDB_GetStreamSnapshotStrings(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - LPSTR* dest, - size_type array_len, - size_type str_len, - pDateTimeStamp datetime, - long end, - long beg) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS topic_t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - topic_t = GetTopicEnum(topic_str); - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - dat->copy(dest,array_len,str_len,end,beg,datetime); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetStreamSnapshotStrings", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template -int -TOSDB_GetStreamSnapshotFromMarker_(LPCSTR id, - LPCSTR item, - TOS_Topics::TOPICS topic_t, - T* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(!IsValidBlockID(id) || !CheckStringLength(item)) - { - return TOSDB_ERROR_BAD_INPUT; - } - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - /* O.K. as long as data_stream::MAX_BOUND_SIZE == INT_MAX */ - *get_size = (long)(dat->copy_from_marker(dest,array_len,beg,datetime)); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("GetStreamSnapshotFromMarker", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template -int -TOSDB_GetStreamSnapshotFromMarker_(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - T* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - if(!CheckStringLength(topic_str)) /* let this go thru std::string ? */ - return 0; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_GetStreamSnapshotFromMarker_(id, item, t, dest, array_len, datetime, beg, get_size); -} - -int -TOSDB_GetStreamSnapshotDoublesFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - double* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - return TOSDB_GetStreamSnapshotFromMarker_(id, item, topic_str, dest, array_len, - datetime, beg, get_size); -} - -int -TOSDB_GetStreamSnapshotFloatsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - float* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - return TOSDB_GetStreamSnapshotFromMarker_(id, item, topic_str, dest, array_len, - datetime, beg, get_size); -} - -int -TOSDB_GetStreamSnapshotLongLongsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long long* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - return TOSDB_GetStreamSnapshotFromMarker_(id, item, topic_str, dest, array_len, - datetime, beg, get_size); -} - -int -TOSDB_GetStreamSnapshotLongsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long* dest, - size_type array_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - return TOSDB_GetStreamSnapshotFromMarker_(id, item, topic_str, dest, array_len, - datetime, beg, get_size); -} - -int -TOSDB_GetStreamSnapshotStringsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - LPSTR* dest, - size_type array_len, - size_type str_len, - pDateTimeStamp datetime, - long beg, - long *get_size) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS topic_t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - topic_t = GetTopicEnum(topic_str); - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - /* O.K. as long as data_stream::MAX_BOUND_SIZE == INT_MAX */ - *get_size = (long)(dat->copy_from_marker(dest, array_len, str_len, beg, datetime)); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetStreamSnapshotStringsFromMarker", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - - -template -int -TOSDB_GetNFromMarker_(LPCSTR id, - LPCSTR item, - TOS_Topics::TOPICS topic_t, - T* dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - - if(!IsValidBlockID(id) || !CheckStringLength(item)) - { - return TOSDB_ERROR_BAD_INPUT; - } - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - /* O.K. as long as data_stream::MAX_BOUND_SIZE == INT_MAX */ - *get_size = (long)(dat->ncopy_from_marker(dest,n,datetime)); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetNFromMarker", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - -template -int -TOSDB_GetNFromMarker_(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - T* dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - if(!CheckStringLength(topic_str)) /* let this go thru std::string ? */ - return 0; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_GetNFromMarker_(id, item, t, dest, n, datetime, get_size); -} - -int -TOSDB_GetNDoublesFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - double *dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - return TOSDB_GetNFromMarker_(id, item, topic_str, dest, n, datetime, get_size); -} - -int -TOSDB_GetNFloatsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - float *dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - return TOSDB_GetNFromMarker_(id, item, topic_str, dest, n, datetime, get_size); -} - -int -TOSDB_GetNLongLongsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long long *dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - return TOSDB_GetNFromMarker_(id, item, topic_str, dest, n, datetime, get_size); -} - -int -TOSDB_GetNLongsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - long *dest, - size_type n, - pDateTimeStamp datetime, - long *get_size) -{ - return TOSDB_GetNFromMarker_(id, item, topic_str, dest, n, datetime, get_size); -} - -int -TOSDB_GetNStringsFromMarker(LPCSTR id, - LPCSTR item, - LPCSTR topic_str, - LPSTR* dest, - size_type n, - size_type str_len, - pDateTimeStamp datetime, - long *get_size) -{ - const TOSDBlock *db; - TOSDB_RawDataBlock::stream_const_ptr_type dat; - TOS_Topics::TOPICS topic_t; - - if( !IsValidBlockID(id) - || !CheckStringLength(item) - || !CheckStringLength(topic_str) ) - { - return TOSDB_ERROR_BAD_INPUT; - } - - topic_t = GetTopicEnum(topic_str); - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - dat = db->block->raw_stream_ptr(item, topic_t); - /* O.K. as long as data_stream::MAX_BOUND_SIZE == INT_MAX */ - *get_size = (long)(dat->ncopy_from_marker(dest, n, str_len, datetime)); - return 0; - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetNStringsFromMarker", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } -} - - -template<> -generic_map_type -TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t) -{ - const TOSDBlock *db; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->map_of_frame_items(topic_t); - /* --- CRITICAL SECTION --- */ -} - -template<> -generic_dts_map_type -TOSDB_GetItemFrame(std::string id, TOS_Topics::TOPICS topic_t) -{ - const TOSDBlock *db; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - throw std::invalid_argument("NULL TOPIC"); - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->pair_map_of_frame_items(topic_t); - /* --- CRITICAL SECTION --- */ -} - -template -int -TOSDB_GetItemFrame_(LPCSTR id, - TOS_Topics::TOPICS topic_t, - T* dest, - size_type array_len, - LPSTR* dest2, - size_type str_len2, - pDateTimeStamp datetime) -{ - const TOSDBlock *db; - - if(!IsValidBlockID(id)) - return TOSDB_ERROR_BAD_INPUT; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC){ - TOSDB_LogH("INPUT", "NULL topic passed to TOSDB_GetItemFrame_"); - return TOSDB_ERROR_BAD_TOPIC; - } - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - - if(datetime){ - auto dtsm = db->block->pair_map_of_frame_items(topic_t); - auto b_iter = dtsm.cbegin(); - auto e_iter = dtsm.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i ) - { - dest[i] = (T)b_iter->second.first; - datetime[i] = b_iter->second.second; - if(dest2){ - if( strcpy_s(dest2[i], str_len2, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - }else{ - auto m = db->block->map_of_frame_items(topic_t); - auto b_iter = m.cbegin(); - auto e_iter = m.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i ) - { - dest[i] = (T)b_iter->second; - if(dest2){ - if( strcpy_s(dest2[i], str_len2, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - } - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("GetItemFrame", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } - - return 0; -} - -template -int -TOSDB_GetItemFrame_(LPCSTR id, - LPCSTR topic_str, - T* dest, - size_type array_len, - LPSTR* dest2, - size_type str_len2, - pDateTimeStamp datetime) -{ - if(!CheckStringLength(topic_str)) - return TOSDB_ERROR_BAD_INPUT; - - TOS_Topics::TOPICS t = GetTopicEnum(topic_str); - if(t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - return TOSDB_GetItemFrame_(id, t, dest, array_len, dest2, str_len2, datetime); -} - -int -TOSDB_GetItemFrameDoubles(LPCSTR id, - LPCSTR topic_str, - double* dest, - size_type array_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - return TOSDB_GetItemFrame_(id, topic_str, dest, array_len, label_dest, - label_str_len, datetime); -} - -int -TOSDB_GetItemFrameFloats(LPCSTR id, - LPCSTR topic_str, - float* dest, - size_type array_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - return TOSDB_GetItemFrame_(id, topic_str, dest, array_len, label_dest, - label_str_len, datetime); -} - -int -TOSDB_GetItemFrameLongLongs(LPCSTR id, - LPCSTR topic_str, - long long* dest, - size_type array_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - return TOSDB_GetItemFrame_(id, topic_str, dest, array_len, label_dest, - label_str_len, datetime); -} - -int -TOSDB_GetItemFrameLongs(LPCSTR id, - LPCSTR topic_str, - long* dest, - size_type array_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - return TOSDB_GetItemFrame_(id, topic_str, dest, array_len, label_dest, - label_str_len, datetime); -} - -int -TOSDB_GetItemFrameStrings(LPCSTR id, - LPCSTR topic_str, - LPSTR* dest, - size_type array_len, - size_type str_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - const TOSDBlock *db; - TOS_Topics::TOPICS topic_t; - - if(!IsValidBlockID(id) || !CheckStringLength(topic_str)) - { - return TOSDB_ERROR_BAD_INPUT; - } - - topic_t = GetTopicEnum(topic_str); - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC) - return TOSDB_ERROR_BAD_TOPIC; - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - - if(datetime){ - auto dtsm = db->block->pair_map_of_frame_items(topic_t); - auto b_iter = dtsm.cbegin(); - auto e_iter = dtsm.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i) - { - datetime[i] = b_iter->second.second; - - if( strcpy_s(dest[i], str_len, (b_iter->second.first).as_string().c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - if(label_dest){ - if( strcpy_s(label_dest[i], label_str_len, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - }else{ - auto m = db->block->map_of_frame_items(topic_t); - auto b_iter = m.cbegin(); - auto e_iter = m.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i ) - { - if( strcpy_s(dest[i], str_len, (b_iter->second).as_string().c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - if(label_dest){ - if( strcpy_s(label_dest[i], label_str_len, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - } - /* --- CRITICAL SECTION --- */ - - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetItemFrameStrings", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } - - return 0; -} - -template<> -generic_map_type -TOSDB_GetTopicFrame(std::string id, std::string item) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->map_of_frame_topics(item); - /* --- CRITICAL SECTION --- */ -} - -template<> -generic_dts_map_type -TOSDB_GetTopicFrame(std::string id,std::string item) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->pair_map_of_frame_topics(item); - /* --- CRITICAL SECTION --- */ -} - -int -TOSDB_GetTopicFrameStrings(LPCSTR id, - LPCSTR item, - LPSTR* dest, - size_type array_len, - size_type str_len, - LPSTR* label_dest, - size_type label_str_len, - pDateTimeStamp datetime) -{ - const TOSDBlock *db; - int err = 0; - - if(!IsValidBlockID(id) || !CheckStringLength(item)) - { - return TOSDB_ERROR_BAD_INPUT; - } - - try{ - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - - if(datetime){ - auto dtsm = db->block->pair_map_of_frame_topics(item); - auto b_iter = dtsm.cbegin(); - auto e_iter = dtsm.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i) - { - datetime[i] = b_iter->second.second; - - if( strcpy_s(dest[i], str_len, (b_iter->second.first).as_string().c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - if(label_dest){ - if( strcpy_s(label_dest[i], label_str_len, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - }else{ - auto m = db->block->map_of_frame_topics(item); - auto b_iter = m.cbegin(); - auto e_iter = m.cend(); - - for( size_type i = 0; - (i < array_len) && (b_iter != e_iter); - ++b_iter, ++i) - { - if( strcpy_s(dest[i], str_len, (b_iter->second).as_string().c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - - if(label_dest){ - if( strcpy_s(label_dest[i], label_str_len, (b_iter->first).c_str()) ) - return TOSDB_ERROR_BAD_INPUT_BUFFER; - } - } - } - /* --- CRITICAL SECTION --- */ - }catch(const TOSDB_DataBlockDoesntExist& e){ - TOSDB_LogH("BLOCK", e.what()); - return TOSDB_ERROR_BLOCK_DOESNT_EXIST; - - }catch(const TOSDB_Error& e){ - TOSDB_LogH(e.tag().c_str(), e.info_and_what().c_str()); - return TOSDB_ERROR_GET_DATA; - - }catch(const std::exception& e){ - TOSDB_LogH("TOSDB_GetTopicFrameStrings", e.what()); - return TOSDB_ERROR_GET_DATA; - - }catch(...){ - return TOSDB_ERROR_UNKNOWN; - } - - return 0; -} - -template<> -generic_matrix_type -TOSDB_GetTotalFrame(std::string id) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->matrix_of_frame(); - /* --- CRITICAL SECTION --- */ -} - -template<> -generic_dts_matrix_type -TOSDB_GetTotalFrame(std::string id) -{ - const TOSDBlock *db; - - GLOBAL_RLOCK_GUARD; - /* --- CRITICAL SECTION --- */ - db = GetBlockOrThrow(id); - return db->block->pair_matrix_of_frame(); - /* --- CRITICAL SECTION --- */ -} - - - - - - - diff --git a/src/client/client_out.cpp b/src/client/client_out.cpp deleted file mode 100644 index edc26f9..0000000 --- a/src/client/client_out.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" - -/* THIS COULD USE SOME WORK */ - -std::ostream& -operator<<(std::ostream& out, const generic_type& gen) -{ - out<< gen.as_string(); - return out; -} - - -std::ostream& -operator<<(std::ostream& out, const DateTimeStamp& dts) -{ - out << dts.ctime_struct.tm_mon+1 << '/' - << dts.ctime_struct.tm_mday << '/' - << dts.ctime_struct.tm_year+1900 <<' ' - << dts.ctime_struct.tm_hour<<':' - << dts.ctime_struct.tm_min<<':' - << dts.ctime_struct.tm_sec<<':' - << dts.micro_second; - - return out; -} - - -std::ostream& -operator<<(std::ostream& out, const DateTimeStamp* dts) -{ - if(dts){ - out << dts->ctime_struct.tm_mon+1 << '/' - << dts->ctime_struct.tm_mday << '/' - << dts->ctime_struct.tm_year+1900 <<' ' - << dts->ctime_struct.tm_hour<<':' - << dts->ctime_struct.tm_min<<':' - << dts->ctime_struct.tm_sec<<':' - << dts->micro_second; - } - - return out; -} - -std::ostream& -operator<<(std::ostream& out, const generic_matrix_type& mat) -{ - for(auto & item : mat){ - std::cout<< item.first << " ::: "; - for(auto & topic : item.second) - std::cout<< topic.first << "[" << topic.second<< "] "; - out< -std::ostream& -operator<<(std::ostream& out, const std::pair& pair) -{ - out<< pair.first <<' '<< pair.second; - return out; -} - - -template -std::ostream& -operator<<(std::ostream& out, const std::vector& vec) -{ - for(const auto & v : vec) - out<< v << ' '; - return out; -} - - -template -std::ostream& -operator<<(std::ostream& out, const std::pair,dts_vector_type>& vecs) -{ - auto f_iter = vecs.first.cbegin(); - auto s_iter = vecs.second.cbegin(); - - for( ; - f_iter != vecs.first.cend() && s_iter != vecs.second.cend(); - f_iter++, s_iter++) - { - out<< generic_type(*f_iter) << ' ' << *s_iter << std::endl; - } - - return out; -} diff --git a/src/concurrency.cpp b/src/concurrency.cpp deleted file mode 100644 index 9561979..0000000 --- a/src/concurrency.cpp +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" -#include "concurrency.hpp" - -#ifdef CPP_COND_VAR - -bool -SignalManager::wait(std::string unq_id) -{ - std::unique_lock lck(_mtx); - auto iter = _unq_flags.find(unq_id); - - if(iter == _unq_flags.end()) - return false; - - _cnd.wait(lck, [=]{ return iter->second.first; }); - _unq_flags.erase(iter); - - return iter->second.second; -} - -bool -SignalManager::wait_for(std::string unq_id, size_t timeout) -{ - bool wait_res; - { - std::unique_lock lck(_mtx); - auto iter = _unq_flags.find(unq_id); - - if(iter == _unq_flags.end()) - return false; - - wait_res = _cnd.wait_for(lck, std::chrono::milliseconds(timeout), - [=]{ return iter->second.first; }); - - wait_res = wait_res && iter->second.second; - _unq_flags.erase(iter); - } - return wait_res; -} - -void -SignalManager::set_signal_ID(std::string unq_id) -{ - std::lock_guard lck(_mtx); - /* --- CRITICAL SECTION --- */ - _unq_flags.insert(_sig_pair_ty(unq_id,_flag_pair_ty(false,true))); - /* --- CRITICAL SECTION --- */ -} - -bool -SignalManager::signal(std::string unq_id, bool secondary) -{ - { - std::lock_guard lck(_mtx); - /* --- CRITICAL SECTION --- */ - auto iter = _unq_flags.find(unq_id); - - if(iter == _unq_flags.end()) - return false; - - iter->second.first = true; - iter->second.second = secondary; - /* --- CRITICAL SECTION --- */ - } - _cnd.notify_one(); - return true; -} - -#else - -void -SignalManager::set_signal_ID(std::string unq_id) -{ - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - _unq_flags.insert(std::pair(unq_id, true)); - /* --- CRITICAL SECTION --- */ -} - -bool -SignalManager::wait(std::string unq_id) -{ - std::map::iterator iter; - { - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - iter = _unq_flags.find(unq_id); - if(iter == _unq_flags.end()) - return false; - /* --- CRITICAL SECTION --- */ - } - WaitForSingleObject(_event, INFINITE); - - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - _unq_flags.erase(iter); - return iter->second; - /* --- CRITICAL SECTION --- */ -} - -bool -SignalManager::wait_for(std::string unq_id, size_type timeout) -{ - std::map::iterator iter; - DWORD wait_res; - bool b_res; - { - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - iter = _unq_flags.find(unq_id); - if(iter == _unq_flags.end()) - return false; - /* --- CRITICAL SECTION --- */ - } - wait_res = WaitForSingleObject(_event, timeout); - - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - b_res = iter->second; - _unq_flags.erase(iter); - return (wait_res == WAIT_TIMEOUT) ? false : b_res; - /* --- CRITICAL SECTION --- */ -} - -bool -SignalManager::signal(std::string unq_id, bool secondary) -{ - { - WinLockGuard lock(_mtx); - /* --- CRITICAL SECTION --- */ - std::map::iterator iter = _unq_flags.find(unq_id); - if(iter == _unq_flags.end()) - return false; - iter->second = secondary; - /* --- CRITICAL SECTION --- */ - } - SetEvent(_event); - return true; -} - - -bool -IPCNamedMutexClient::try_lock(unsigned long timeout, - std::function no_open_cb, - std::function timeout_cb, - std::function fail_cb) -{ - if(_mtx) - return true; - - /* open late to be sure slave has already created */ - HANDLE tmp = OpenMutex(SYNCHRONIZE, FALSE, _name.c_str()); - if(!tmp){ - if(no_open_cb) - no_open_cb( GetLastError() ); - return false; - } - - /* we use 'tmp' because technically the mutex has yet to be - locked but locked() simply checks that _mtx != NULL */ - DWORD wstate = WaitForSingleObject(tmp, timeout); - errno_t e = GetLastError(); - - switch(wstate){ - case WAIT_TIMEOUT: - CloseHandle(tmp); - if(timeout_cb) - timeout_cb(); - return false; - case WAIT_FAILED: - CloseHandle(tmp); - if(fail_cb) - fail_cb(e); - return false; - } - - /* on success we pass the handle to _mtx */ - _mtx = tmp; - return true; -} - - -void -IPCNamedMutexClient::unlock() -{ - if(locked()){ - ReleaseMutex(_mtx); - CloseHandle(_mtx); - _mtx = NULL; - } -} - - -#define HANDLE_MUTEX_ERROR(msg, name, e) do{ \ - std::stringstream m; \ - m << msg << ": " << name << " (" << e << ")"; \ - /* can't block here, use ...Raw */ \ - TOSDB_LogRawH("MUTEX", m.str().c_str()); \ - throw std::runtime_error(m.str()); \ -}while(0) - -NamedMutexLockGuard::NamedMutexLockGuard(std::string name) -{ - errno_t e; - /* no guarantee the service/engine won't create the mutex, leaving us w/o - adequate privileges; if so we try to open the mutex; if the mutex was - closed AFTER our call to CreateMutex failed we loop back and try again */ - do{ - _mtx = CreateMutex(NULL, FALSE, name.c_str()); - if(_mtx) - break; - e = GetLastError(); - if(e == ERROR_ACCESS_DENIED){ - /* if we dont have privileges to create, just try to open */ - _mtx = OpenMutex(NULL, FALSE, name.c_str()); - if(_mtx) - break; - e = GetLastError(); - if(e == ERROR_FILE_NOT_FOUND) /* if it was closed, try again */ - continue; - HANDLE_MUTEX_ERROR("failed to open mutex", name, e); - } - HANDLE_MUTEX_ERROR("failed to create mutex", name, e); - }while(1); - - /* lock mutex */ - if( WaitForSingleObject(_mtx, INFINITE) == WAIT_FAILED ) - { - errno_t e = GetLastError(); - HANDLE_MUTEX_ERROR("failed to lock mutex", name, e); - } -} - -#undef HANDLE_MUTEX_ERROR - - -NamedMutexLockGuard::~NamedMutexLockGuard() -{ - if(_mtx){ - ReleaseMutex(_mtx); - CloseHandle(_mtx); - } -} - -#endif diff --git a/src/data_stream.tpp b/src/data_stream.tpp deleted file mode 100644 index 265f884..0000000 --- a/src/data_stream.tpp +++ /dev/null @@ -1,828 +0,0 @@ -#include "data_stream.hpp" - -DATASTREAM_INTERFACE_TEMPLATE -template -size_t -DATASTREAM_INTERFACE_CLASS::_copy(OutTy *dest, - size_t sz, - int end, - int beg, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec) const -{ - size_t ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - std::unique_ptr tmp(new InTy[sz], [](InTy *p){ delete[] p; }); - - ret = copy(tmp.get(), sz, end, beg, sec); - for(size_t i = 0; i < sz; ++i) - dest[i] = (OutTy)tmp.get()[i]; - - return ret; -} - -DATASTREAM_INTERFACE_TEMPLATE -template -long long -DATASTREAM_INTERFACE_CLASS::_copy_using_atomic_marker(OutTy *dest, - size_t sz, - int beg, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec) const -{ - long long ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - std::unique_ptr tmp(new InTy[sz], [](InTy *p){ delete[] p; }); - - ret = copy_from_marker(tmp.get(), sz, beg, sec); - for(size_t i = 0; i < sz; ++i) - dest[i] = (OutTy)tmp.get()[i]; - - return ret; -} - -DATASTREAM_INTERFACE_TEMPLATE -template -long long -DATASTREAM_INTERFACE_CLASS::_ncopy_using_atomic_marker(OutTy *dest, - size_t sz, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec) const -{ - long long ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - std::unique_ptr tmp(new InTy[sz], [](InTy *p){ delete[] p; }); - - ret = ncopy_from_marker(tmp.get(), sz, sec); - for(size_t i = 0; i < sz; ++i) - dest[i] = (OutTy)tmp.get()[i]; - - return ret; -} - -DATASTREAM_INTERFACE_TEMPLATE -size_t -DATASTREAM_INTERFACE_CLASS::copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0 , - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - BuildThrowTypeError("copy()"); - return 0; -} - -DATASTREAM_INTERFACE_TEMPLATE -size_t -DATASTREAM_INTERFACE_CLASS::copy(std::string *dest, - size_t sz, - int end = -1, - int beg = 0, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - size_t ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - auto dstr = [sz](char **p){ DeleteStrings(p, sz); }; - std::unique_ptr sptr(NewStrings(sz,STR_DATA_SZ), dstr); - - ret = copy(sptr.get(), sz, STR_DATA_SZ , end, beg, sec); - std::copy_n(sptr.get(), sz, dest); - - return ret; -} - -DATASTREAM_INTERFACE_TEMPLATE -long long -DATASTREAM_INTERFACE_CLASS::copy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - int beg = 0, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - BuildThrowTypeError("copy_from_marker()"); - return 0; -} - -DATASTREAM_INTERFACE_TEMPLATE -long long -DATASTREAM_INTERFACE_CLASS::copy_from_marker(std::string *dest, - size_t sz, - int beg = 0, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - long long ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - auto dstr = [sz](char **pptr){ DeleteStrings(pptr, sz); }; - std::unique_ptr sptr(NewStrings(sz,STR_DATA_SZ), dstr); - - ret = copy_from_marker(sptr.get(), sz, STR_DATA_SZ, beg, sec); - std::copy_n(sptr.get(), sz, dest); - - return ret; -} - -DATASTREAM_INTERFACE_TEMPLATE -long long -DATASTREAM_INTERFACE_CLASS::ncopy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - BuildThrowTypeError("ncopy_from_marker()"); - return 0; -} - -DATASTREAM_INTERFACE_TEMPLATE -long long -DATASTREAM_INTERFACE_CLASS::ncopy_from_marker(std::string *dest, - size_t sz, - typename DATASTREAM_INTERFACE_CLASS::secondary_ty *sec = nullptr) const -{ - long long ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - auto dstr = [sz](char **pptr){ DeleteStrings(pptr, sz); }; - std::unique_ptr sptr(NewStrings(sz,STR_DATA_SZ), dstr); - - ret = ncopy_from_marker(sptr.get(), sz, STR_DATA_SZ, sec); - std::copy_n(sptr.get(), sz, dest); - - return ret; -} - -DATASTREAM_PRIMARY_TEMPLATE -void -DATASTREAM_PRIMARY_CLASS::_push(const Ty v) -{ /* - * if can't obtain lock indicate other threads should yield to us - */ - _push_has_priority = _mtx->try_lock(); /*O.K. push/pop doesn't throw*/ - if(!_push_has_priority) - _mtx->lock(); /* block regardless */ - /* --- CRITICAL SECTION --- */ - _deque_primary.push_front(v); - _deque_primary.pop_back(); - _incr_internal_counts(); - /* --- CRITICAL SECTION --- */ - _mtx->unlock(); -} - -DATASTREAM_PRIMARY_TEMPLATE -template -bool -DATASTREAM_PRIMARY_CLASS::_check_adj(int& end, int& beg, const std::deque& d) const -{ - int sz = (int)d.size(); /* O.K. sz can't be > INT_MAX */ - if(_qbound != sz) - throw DataStreamSizeViolation("internal size/bounds violation", _qbound, sz); - - if(end < 0) - end += sz; - - if(beg < 0) - beg += sz; - - if(beg >= sz || end >= sz || beg < 0 || end < 0) - throw DataStreamOutOfRange("adj index value out of range", sz, beg, end); - else if(beg > end) - throw DataStreamInvalidArgument("adjusted beg index > end index"); - - return true; -} - -DATASTREAM_PRIMARY_TEMPLATE -void -DATASTREAM_PRIMARY_CLASS::_incr_internal_counts() -{ - long long penult = (long long)(_qbound) -1; - - if(_qcount < _qbound) - ++_qcount; - - if(*_mark_count == penult) - *_mark_is_dirty = true; - else if(*_mark_count < penult) - ++(*_mark_count); - else{/* - * WE CANT THROW so attempt to get _mark_count back in line; - * also set dirty flag to alert caller of possible data issue - */ - --(*_mark_count); - *_mark_is_dirty = true; - } -} - -DATASTREAM_PRIMARY_TEMPLATE -template -size_t -DATASTREAM_PRIMARY_CLASS::_copy_to_ptr(DequeTy& d, - DestTy *dest, - size_t sz, - unsigned int end, - unsigned int beg) const -{ - auto b_iter = d.cbegin() + beg; - auto e_iter = d.cbegin() + std::min(sz+beg, std::min(++end, _qcount)); - - return (b_iter < e_iter) ? (std::copy(b_iter, e_iter, dest) - dest) : 0; -} - -DATASTREAM_PRIMARY_TEMPLATE -DATASTREAM_PRIMARY_CLASS::DataStream(size_t sz) - : - _deque_primary(std::max(std::min(sz,MAX_BOUND_SIZE),1)), - _qbound(std::max(std::min(sz,MAX_BOUND_SIZE),1)), - _qcount(0), - _mark_count(new long long(-1)), - _mark_is_dirty(new bool(false)), - _push_has_priority(true), - _mtx(new std::recursive_mutex) - { - } - -DATASTREAM_PRIMARY_TEMPLATE -DATASTREAM_PRIMARY_CLASS::DataStream(const typename DATASTREAM_PRIMARY_CLASS::_my_ty & stream) - : - _deque_primary(stream._deque_primary), - _qbound(stream._qbound), - _qcount(stream._qcount), - _mark_count(new long long(*(stream._mark_count))), - _mark_is_dirty(new bool(*(stream._mark_is_dirty))), - _push_has_priority(true), - _mtx(new std::recursive_mutex) - { - } - -DATASTREAM_PRIMARY_TEMPLATE -DATASTREAM_PRIMARY_CLASS::DataStream(typename DATASTREAM_PRIMARY_CLASS::_my_ty && stream) - : - _deque_primary(std::move(stream._deque_primary)), - _qbound(stream._qbound), - _qcount(stream._qcount), - _mark_count(stream._mark_count), - _mark_is_dirty(stream._mark_is_dirty), - _push_has_priority(true), - _mtx(stream._mtx) // ?? - { - stream._mark_count = nullptr; - stream._mark_is_dirty = nullptr; - stream._mtx = nullptr; - } - -DATASTREAM_PRIMARY_TEMPLATE -DATASTREAM_PRIMARY_CLASS::~DataStream() -{ - if(_mtx) - delete _mtx; - - if(_mark_count) - delete _mark_count; - - if(_mark_is_dirty) - delete _mark_is_dirty; - } - - -DATASTREAM_PRIMARY_TEMPLATE -size_t -DATASTREAM_PRIMARY_CLASS::bound_size(size_t sz) -{ - sz = std::max(std::min(sz,MAX_BOUND_SIZE),1); - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _deque_primary.resize(sz); - - if(sz < _qbound){ - /* IF bound is 'clipped' from the left(end) */ - _deque_primary.shrink_to_fit(); - - if( (long long)sz <= *_mark_count ){ - /* IF marker is 'clipped' from the left(end) */ - *_mark_count = (long long)sz -1; - *_mark_is_dirty = true; - } - } - - _qbound = sz; - if(sz < _qcount) /* IF count is 'clipped' from the left(end) */ - _qcount = sz; - - return _qbound; - /* --- CRITICAL SECTION --- */ -} - -/// - -DATASTREAM_PRIMARY_TEMPLATE -long long -DATASTREAM_PRIMARY_CLASS::ncopy_from_marker(Ty *dest, - size_t sz, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - int beg = std::max((int)(*_mark_count - sz + 1), 0); - return copy_from_marker(dest, sz, beg, sec); - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -long long -DATASTREAM_PRIMARY_CLASS::ncopy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - int beg = std::max((int)(*_mark_count - dest_sz + 1), 0); - return copy_from_marker(dest, dest_sz, str_sz, beg, sec); - /* --- CRITICAL SECTION --- */ -} - -/// - -DATASTREAM_PRIMARY_TEMPLATE -long long -DATASTREAM_PRIMARY_CLASS::copy_from_marker(Ty *dest, - size_t sz, - int beg = 0, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ - /* 1) we need to cache mark vals before copy changes state - 2) adjust beg here; _check_adj requires a ref that we can't pass - 3) casts to long long O.K as long as MAX_BOUND_SIZE == INT_MAX */ - - long long copy_sz, req_sz; - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - bool was_dirty = *_mark_is_dirty; - - if(beg < 0) - beg += (int)size(); - - req_sz = *_mark_count - (long long)beg + 1; - if(beg < 0 || req_sz < 1) - /* if beg is still invalid or > marker ... CALLER'S PROBLEM - req_sz needs to account for inclusive range by adding 1 */ - return 0; - - /* CAREFUL: we cant have a negative *_mark_count past this point */ - copy_sz = (long long)copy(dest, sz, (int)(*_mark_count), beg, sec); - - if(was_dirty || copy_sz < req_sz) - /* IF mark is dirty (i.e hits back of stream) or we - don't copy enough(sz is too small) return negative size */ - copy_sz *= -1; - - return copy_sz; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -long long -DATASTREAM_PRIMARY_CLASS::copy_from_marker(char **dest, - size_t dest_sz, - size_t str_sz, - int beg = 0, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ - /* 1) we need to cache mark vals before copy changes state - 2) adjust beg here; _check_adj requires a ref that we can't pass - 3) casts to long long O.K as long as MAX_BOUND_SIZE == INT_MAX */ - - long long copy_sz, req_sz; - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - bool was_dirty = *_mark_is_dirty; - - if(beg < 0) - beg += (int)size(); - - req_sz = *_mark_count - (long long)beg + 1; - if(beg < 0 || req_sz < 1) - /* if beg is still invalid or > marker ... CALLER'S PROBLEM - req_sz needs to account for inclusive range by adding 1 */ - return 0; - - /* CAREFUL: we cant have a negative *_mark_count past this point */ - copy_sz = (long long)copy(dest, dest_sz, str_sz, (int)(*_mark_count), beg, sec); - - if(was_dirty || copy_sz < req_sz) - /* IF mark is dirty (i.e hits back of stream) or we - don't copy enough(sz is too small) return negative size */ - copy_sz *= -1; - - return copy_sz; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -size_t -DATASTREAM_PRIMARY_CLASS::copy(Ty *dest, - size_t sz, - int end = -1, - int beg = 0, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ - size_t ret; - - static_assert(!std::is_same::value, "copy doesn't accept char*"); - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _check_adj(end, beg, _deque_primary); - - if(end == beg){ - *dest = beg ? _deque_primary.at(beg) : _deque_primary.front(); - ret = 1; - }else - ret = _copy_to_ptr(_deque_primary, dest, sz, end, beg); - - *_mark_count = beg - 1; - *_mark_is_dirty = false; - - return ret; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -size_t -DATASTREAM_PRIMARY_CLASS::copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0, - typename DATASTREAM_PRIMARY_CLASS::secondary_ty *sec = nullptr) const -{ /* - * slow(er), has to go thru generic_ty to get strings - * note: if sz <= gstr.length() the string is truncated - */ - size_t i; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _check_adj(end, beg, _deque_primary); - - auto b_iter = _deque_primary.cbegin() + beg; - auto e_iter = _deque_primary.cbegin() + std::min(++end, _qcount); - - for( i = 0; - (i < dest_sz) && (b_iter < e_iter); - ++b_iter, ++i ) - { - std::string gstr = generic_ty(*b_iter).as_string(); - strncpy_s(dest[i], str_sz, gstr.c_str(), std::min(str_sz-1, gstr.length())); - } - - *_mark_count = beg - 1; - *_mark_is_dirty = false; - - return i; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::generic_ty -DATASTREAM_PRIMARY_CLASS::operator[](int indx) const -{ - int dummy = 0; - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if(!indx){ - /* optimize for indx == 0 */ - *_mark_count = -1; - *_mark_is_dirty = false; - return generic_ty(_deque_primary.front()); - } - - _check_adj(indx, dummy, _deque_primary); - - *_mark_count = indx - 1; - *_mark_is_dirty = false; - - return generic_ty(_deque_primary.at(indx)); - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::both_ty -DATASTREAM_PRIMARY_CLASS::both(int indx) const -{ - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - return both_ty(operator[](indx), secondary_ty()); - /* --- CRITICAL SECTION --- */ -} - -// BUG-FIX: allow frame calls not to move marker Dec 8 2017 -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::generic_ty -DATASTREAM_PRIMARY_CLASS::get_leave_marker(int indx) const -{ - int dummy = 0; - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if(!indx){ - return generic_ty(_deque_primary.front()); - } - - _check_adj(indx, dummy, _deque_primary); - - return generic_ty(_deque_primary.at(indx)); - /* --- CRITICAL SECTION --- */ -} - -// BUG-FIX: allow frame calls not to move marker Dec 8 2017 -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::both_ty -DATASTREAM_PRIMARY_CLASS::both_leave_marker(int indx) const -{ - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - return both_ty(get_leave_marker(indx), secondary_ty()); - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::generic_vector_ty -DATASTREAM_PRIMARY_CLASS::vector(int end = -1, int beg = 0) const -{ - generic_vector_ty tmp; - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _check_adj(end, beg, _deque_primary); - - auto b_iter = _deque_primary.cbegin() + beg; - auto e_iter = _deque_primary.cbegin() + std::min(++end, _qcount); - - if(b_iter < e_iter){ - std::transform( - b_iter, - e_iter, - /* have to use slower insert_iterator approach, - generic_ty doesn't allow default construction */ - std::insert_iterator(tmp, tmp.begin()), - [](Ty x){ return generic_ty(x); } - ); - } - - *_mark_count = beg - 1; - *_mark_is_dirty = false; - - return tmp; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_PRIMARY_TEMPLATE -typename DATASTREAM_PRIMARY_CLASS::secondary_vector_ty -DATASTREAM_PRIMARY_CLASS::secondary_vector(int end = -1, int beg = 0) const -{ - _check_adj(end, beg, _deque_primary); - return secondary_vector_ty(std::min< size_t >(++end - beg, _qcount)); -} - - -DATASTREAM_SECONDARY_TEMPLATE -void -DATASTREAM_SECONDARY_CLASS::_push(const Ty v, const typename DATASTREAM_SECONDARY_CLASS::secondary_ty sec) -{ - _push_has_priority = _mtx->try_lock(); - if(!_push_has_priority) - _mtx->lock(); - /* --- CRITICAL SECTION --- */ - _my_base_ty::_deque_primary.push_front(v); - _my_base_ty::_deque_primary.pop_back(); - _deque_secondary.push_front(std::move(sec)); - _deque_secondary.pop_back(); - _incr_internal_counts(); - /* --- CRITICAL SECTION --- */ - _mtx->unlock(); -} - -DATASTREAM_SECONDARY_TEMPLATE -DATASTREAM_SECONDARY_CLASS::DataStream(size_t sz) - : - _deque_secondary(std::max(std::min(sz,MAX_BOUND_SIZE),1)), - _my_base_ty(std::max(std::min(sz,MAX_BOUND_SIZE),1)) - { - } - -DATASTREAM_SECONDARY_TEMPLATE -DATASTREAM_SECONDARY_CLASS::DataStream(const typename DATASTREAM_SECONDARY_CLASS::_my_ty & stream) - : - _deque_secondary(stream._deque_secondary), - _my_base_ty(stream) - { - } - -DATASTREAM_SECONDARY_TEMPLATE -DATASTREAM_SECONDARY_CLASS::DataStream(typename DATASTREAM_SECONDARY_CLASS::_my_ty && stream) - : - _deque_secondary(std::move(stream._deque_secondary)), - _my_base_ty(std::move(stream)) - { - } - - -DATASTREAM_SECONDARY_TEMPLATE -size_t -DATASTREAM_SECONDARY_CLASS::bound_size(size_t sz) -{ - sz = std::max(std::min(sz,MAX_BOUND_SIZE),1); - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _deque_secondary.resize(sz); - if (sz < _qcount) - _deque_secondary.shrink_to_fit(); - - return _my_base_ty::bound_size(sz); - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_SECONDARY_TEMPLATE -size_t -DATASTREAM_SECONDARY_CLASS::copy(Ty *dest, - size_t sz, - int end = -1, - int beg = 0, - typename DATASTREAM_SECONDARY_CLASS::secondary_ty *sec = nullptr) const -{ - size_t ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - ret = _my_base_ty::copy(dest, sz, end, beg); /*_mark_count reset by _my_base_ty*/ - - if(!sec) - return ret; - - _check_adj(end, beg, _deque_secondary); /*repeat to update index vals */ - - if(end == beg){ - *sec = beg ? _deque_secondary.at(beg) : _deque_secondary.front(); - ret = 1; - }else - ret = _copy_to_ptr(_deque_secondary, sec, sz, end, beg); - - /* check ret vs. the return value of _my_base_ty::copy for consistency ? */ - return ret; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_SECONDARY_TEMPLATE -size_t -DATASTREAM_SECONDARY_CLASS::copy(char **dest, - size_t dest_sz, - size_t str_sz, - int end = -1, - int beg = 0, - typename DATASTREAM_SECONDARY_CLASS::secondary_ty *sec = nullptr) const -{ - size_t ret; - - if(!dest) - throw DataStreamInvalidArgument("NULL dest argument"); - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - ret = _my_base_ty::copy(dest, dest_sz, str_sz, end, beg); - - if(!sec) - return ret; - - _check_adj(end, beg, _deque_secondary); /*repeat to update index vals*/ - - if(end == beg){ - *sec = beg ? _deque_secondary.at(beg) : _deque_secondary.front(); - ret = 1; - }else - ret = _copy_to_ptr(_deque_secondary, sec, dest_sz, end, beg); - - /* check ret vs. the return value of _my_base_ty::copy for consistency ? */ - /* --- CRITICAL SECTION --- */ - return ret; -} - -DATASTREAM_SECONDARY_TEMPLATE -typename DATASTREAM_SECONDARY_CLASS::both_ty -DATASTREAM_SECONDARY_CLASS::both(int indx) const -{ - int dummy = 0; - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - generic_ty gen = operator[](indx); /* _mark_count reset by _my_base_ty */ - if(!indx) - return both_ty(gen, _deque_secondary.front()); - - _check_adj(indx, dummy, _deque_secondary); - - return both_ty(gen, _deque_secondary.at(indx)); - /* --- CRITICAL SECTION --- */ -} - -// BUG-FIX: allow frame calls not to move marker Dec 8 2017 -DATASTREAM_SECONDARY_TEMPLATE -typename DATASTREAM_SECONDARY_CLASS::both_ty -DATASTREAM_SECONDARY_CLASS::both_leave_marker(int indx) const -{ - int dummy = 0; - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - generic_ty gen = get_leave_marker(indx); - if(!indx) - return both_ty(gen, _deque_secondary.front()); - - _check_adj(indx, dummy, _deque_secondary); - - return both_ty(gen, _deque_secondary.at(indx)); - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_SECONDARY_TEMPLATE -void -DATASTREAM_SECONDARY_CLASS::secondary(typename DATASTREAM_SECONDARY_CLASS::secondary_ty *dest, int indx) const -{ - int dummy = 0; - - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _check_adj(indx, dummy, _deque_secondary); - - if(!indx) - *dest = _deque_secondary.front(); - else - *dest = _deque_secondary.at(indx); - - *_mark_count = indx - 1; /* _mark_count NOT reset by _my_base_ty */ - *_mark_is_dirty = false; - /* --- CRITICAL SECTION --- */ -} - -DATASTREAM_SECONDARY_TEMPLATE -typename DATASTREAM_SECONDARY_CLASS::secondary_vector_ty -DATASTREAM_SECONDARY_CLASS::secondary_vector(int end = -1, int beg = 0) const -{ - secondary_vector_ty tmp; - - _yld_to_push(); - _my_lock_guard_type lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _check_adj(end, beg, _deque_secondary); - - auto b_iter = _deque_secondary.cbegin() + beg; - auto e_iter = _deque_secondary.cbegin() + std::min(++end, _qcount); - auto ndiff = e_iter - b_iter; - - if(ndiff > 0){ - /* do this manually; insert iterators too slow */ - tmp.resize(ndiff); - std::copy(b_iter, e_iter, tmp.begin()); - } - - *_mark_count = beg - 1; /* _mark_count NOT reset by _my_base_ty */ - *_mark_is_dirty = false; - - return tmp; - /* --- CRITICAL SECTION --- */ -} diff --git a/src/generic.cpp b/src/generic.cpp deleted file mode 100644 index 197b065..0000000 --- a/src/generic.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "generic.hpp" - - -/* COPY CONSTRUCT */ -TOSDB_Generic::TOSDB_Generic(const TOSDB_Generic& gen) - : - _type_val( gen._type_val ) - { - if(gen.is_string()) - *(std::string**)_sub = new std::string(**(std::string**)(gen._sub)); - else - memcpy(_sub, gen._sub, SUB_SZ); - } - - -/* MOVE CONSTRUCT */ -TOSDB_Generic::TOSDB_Generic(TOSDB_Generic&& gen) - : - _type_val( gen._type_val ) - { - /* we are stealing the scalar value OR the string pointer */ - memcpy(_sub, gen._sub, SUB_SZ); - - /* leave the pointer but set _type_val to VOID to avoid bad free; ptr is an - implementation detail of the moved object that we can safely ignore */ - gen._type_val = TYVAL_VOID; - } - - -/* ASSIGNMENT */ -TOSDB_Generic& -TOSDB_Generic::operator=(const TOSDB_Generic& gen) -{ - if(*this == gen) - return *this; - - if(gen.is_string()){ - if(is_string()) /* if we're also a string simply assign */ - **(std::string**)_sub = **(std::string**)(gen._sub); - else /* otherwise allocate new */ - *(std::string**)_sub = new std::string(**(std::string**)(gen._sub)); - }else{ - if(is_string()) - delete *(std::string**)_sub; - - memcpy(_sub, gen._sub, SUB_SZ); - } - - _type_val = gen._type_val; - return *this; -} - - -/* MOVE ASSIGNMENT */ -TOSDB_Generic& -TOSDB_Generic::operator=(TOSDB_Generic&& gen) -{ - /* not sure if this is optimal or its better to branch(as above) and move - the string object in, allowing gen's destructor to delete - - although, this way is a lot cleaner than 2 nested conditionals*/ - - if(_type_val == TYVAL_STRING) - delete *(std::string**)_sub; - - _type_val = gen._type_val; - - /* we are stealing the scalar value OR the string pointer */ - memcpy(_sub, gen._sub, SUB_SZ); - - /* leave the pointer but set _type_val to VOID to avoid bad free; ptr is an - implementation detail of the moved object that we can safely ignore */ - gen._type_val = TYVAL_VOID; - - return *this; -} - - -std::string -TOSDB_Generic::as_string() const -{ - switch(_type_val){ - case(TYVAL_LONG): - case(TYVAL_LONG_LONG): - return std::to_string(*(long long*)_sub); - case(TYVAL_FLOAT): - case(TYVAL_DOUBLE): - return std::to_string(*(double*)_sub); - case(TYVAL_STRING): - return **(std::string**)_sub; - default: - return std::string(); - }; -} - -size_t -TOSDB_Generic::size() const -{ - switch(_type_val){ - case(TYVAL_LONG): - return sizeof(long); - case(TYVAL_LONG_LONG): - return sizeof(long long); - case(TYVAL_FLOAT): - return sizeof(float); - case(TYVAL_DOUBLE): - return sizeof(double); - case(TYVAL_STRING): - return (*(std::string**)_sub)->size(); - default: - return 0; - }; -} - - diff --git a/src/ipc.cpp b/src/ipc.cpp deleted file mode 100644 index a34dd87..0000000 --- a/src/ipc.cpp +++ /dev/null @@ -1,313 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" -#include "concurrency.hpp" -#include "ipc.hpp" - -#include -#include -#include - - -bool -IPCBase::send(std::string msg) const -{ - DWORD d; - BOOL ret; - - if(msg.length() > MAX_MESSAGE_SZ){ - TOSDB_LogH("IPC", "IPCBase::send() :: msg length > MAX_MESSAGE_SZ"); - throw std::invalid_argument("IPCBase::send() :: msg length > MAX_MESSAGE_SZ"); - } - - /* size_t to DWORD cast ok because we check msg length above */ - ret = WriteFile(_main_channel_pipe_hndl, (void*)msg.c_str(), msg.length() + 1, &d, NULL); - if(!ret){ - TOSDB_LogEx("IPC", "WriteFile failed in _recv()", GetLastError()); - return false; - } - - return true; -} - - -bool -IPCBase::recv(std::string *msg) const -{ - DWORD d; - BOOL ret; - - msg->clear(); - msg->resize(MAX_MESSAGE_SZ); - - ret = ReadFile(_main_channel_pipe_hndl, (void*)msg->c_str(), MAX_MESSAGE_SZ + 1, &d, NULL); - if(!ret || d == 0){ - TOSDB_LogEx("IPC", "ReadFile failed in _recv()", GetLastError()); - return false; - } - - return true; -} - -// needs to be non-blocking, called from ::call() which is called from DLLMain -template -bool -IPCBase::_call_pipe( std::string name, - T* in, - DWORD in_sz, - T* out, - DWORD out_sz, - unsigned long timeout, - std::function handle_good_call, - std::function handle_file_not_found ) -{ - errno_t e; - DWORD r = 0; - - if( CallNamedPipe(name.c_str(), (void*)in, in_sz, (void*)out, out_sz, &r, timeout) ) - { - return handle_good_call(out, r); - } - - e = GetLastError(); - if(e == ERROR_FILE_NOT_FOUND){ - if(handle_file_not_found){ - handle_file_not_found(); - } - }else{ - TOSDB_LogRawH("IPC", ("CallNamedPipe failed for pipe: " + name + - ", ERROR# " + std::to_string(e)).c_str() ); - } - - return false; -} - -template bool /* connected() / _probe_channel_pipe */ -IPCBase::_call_pipe(std::string, uint8_t*, DWORD, uint8_t*, DWORD, unsigned long, - std::function, std::function); - -template bool /* call() / _main_channel_pipe */ -IPCBase::_call_pipe(std::string, const char*, DWORD, const char*, DWORD, unsigned long, - std::function, std::function); - - -bool -IPCBase::connected(unsigned long timeout) -{ - uint8_t i = PROBE_BYTE; - uint8_t o = 0; - - /* check for a single byte that == PROBE_BYTE */ - std::function cb = - [](uint8_t *out, DWORD r){ - if (r != sizeof(PROBE_BYTE) ){ - TOSDB_LogRawH("IPC", ("bad probe size returned: " + std::to_string(r)).c_str()); - return false; - } - if( *out != PROBE_BYTE ){ - TOSDB_LogRawH("IPC", ("bad probe value returned: " + std::to_string(*out)).c_str()); - if( *out == PROBE_BYTE_WRONG_ARCH ){ - TOSDB_LogRawH("IPC", "build mismatch between engine and library(x86 vs x64)"); - } - return false; - } - return true; - }; - - return _call_pipe(_probe_channel_pipe_name, &i, sizeof(i), &o, sizeof(o), timeout, cb); -} - - -// needs to be non-blocking, called from DLLMain -bool -IPCMaster::call(std::string *msg, unsigned long timeout) -{ - /* buffer for returned string */ - std::string recv(MAX_MESSAGE_SZ, '\0'); - - /* check buffer length */ - if(msg->length() > MAX_MESSAGE_SZ){ - TOSDB_LogRawH("IPC", "IPCMaster::call() :: msg length > MAX_MESSAGE_SZ"); - throw std::invalid_argument("IPCMaster::call() :: msg length > MAX_MESSAGE_SZ"); - } - - /* assign the receive buffer*/ - std::function cb_good = - [msg,&recv](const char* d1, DWORD d2){ *msg = recv; return true; }; - - /* if we get here the main pipe *should* be available; log if not */ - std::function cb_no_file = - [](){ TOSDB_LogRawH("IPC", "main pipe not found (slave not available)"); }; - - /* size_t to DWORD cast ok because we check msg length above */ - return _call_pipe( _main_channel_pipe_name, - msg->c_str(), msg->length() + 1, - recv.c_str(), recv.length() + 1, - timeout, cb_good, cb_no_file ); -} - - -HANDLE -IPCSlave::_create_pipe(std::string name) -{ - HANDLE h = CreateNamedPipe(name.c_str(), PIPE_ACCESS_DUPLEX, - PIPE_TYPE_MESSAGE | PIPE_READMODE_BYTE | PIPE_WAIT, - 1, 0, 0, NMPWAIT_NOWAIT, &_sec_attr); - - if(h == INVALID_HANDLE_VALUE){ - errno_t e = GetLastError(); - std::string msg = "IPCSlave failed to create pipe: " + name + " (" + std::to_string(e) + ")"; - TOSDB_LogEx("IPC-Slave", msg.c_str(), e); - throw std::runtime_error(msg); - } - - return h; -} - - -bool -IPCSlave::wait_for_master() -{ - if( !ConnectNamedPipe(_main_channel_pipe_hndl, NULL) ) - { - TOSDB_LogEx("IPC-Slave", "ConnectNamedPipe failed in wait_for_master", GetLastError()); - return false; - } - return true; -} - - -void -IPCSlave::_listen_for_probes() -{ - BOOL ret; - DWORD r; - uint8_t b; - errno_t e; - - while(_probe_channel_run_flag){ - /* Main run loop for the probe channel: a very simple named pipe server that - allows clients (master AND slave objects) to share a single pipe instance - in order to send a PROBE_BYTE. If the server receives a valid PROBE_BYTE it - returns it, disconnects the pipe instance and waits for another */ - - ret = ConnectNamedPipe(_probe_channel_pipe_hndl, NULL); - if(!ret){ - e = GetLastError(); - if(e != ERROR_PIPE_CONNECTED){ /* in case client connects first */ - TOSDB_LogEx("IPC", "ConnectNamedPipe failed in probe_channel", e); - continue; - } - } - - b = 0; - r = 0; - ret = ReadFile(_probe_channel_pipe_hndl, (void*)&b, sizeof(b), &r, NULL); - if(!ret){ - e = GetLastError(); - if(e == ERROR_BROKEN_PIPE) - TOSDB_LogDebug("***IPC*** PROBE RECV - (BROKEN_PIPE)"); - else - TOSDB_LogEx("IPC", "ReadFile failed in _listen_for_probes()", e); - } - - if (r != sizeof(PROBE_BYTE)){ - TOSDB_LogH("IPC", ("bad probe size received: " + std::to_string(r)).c_str()); - b = 0; - }else if(b != PROBE_BYTE){ - TOSDB_LogH("IPC", ("bad probe value received: " + std::to_string(b)).c_str()); - if (b == PROBE_BYTE_WRONG_ARCH){ - TOSDB_LogH("IPC", "build mismatch between engine and library(x86 vs x64)"); - b = PROBE_BYTE; - }else{ - b = 0; - } - } - - r = 0; - ret = WriteFile(_probe_channel_pipe_hndl, (void*)&b, sizeof(b), &r, NULL); - if(!ret || r != sizeof(PROBE_BYTE)) - TOSDB_LogEx("IPC", "WriteFile failed in _listen_for_probes()", GetLastError()); - - FlushFileBuffers(_probe_channel_pipe_hndl); - DisconnectNamedPipe(_probe_channel_pipe_hndl); - } -} - - -void -IPCSlave::_init_security_objects() -{ - std::string call_str; - - SID_NAME_USE dummy; - DWORD dom_sz = 128; - DWORD sid_sz = SECURITY_MAX_SID_SIZE; - - SmartBuffer dom_buf(dom_sz); - - if( !LookupAccountName(NULL, "Everyone", _sec_sid.get(), &sid_sz, dom_buf.get(), &dom_sz, &dummy) ){ - call_str = "LookupAccountName"; - goto handle_error; - } - - _sec_attr.nLength = sizeof(SECURITY_ATTRIBUTES); - _sec_attr.bInheritHandle = FALSE; - _sec_attr.lpSecurityDescriptor = &_sec_desc; - - if( !InitializeSecurityDescriptor(&_sec_desc, SECURITY_DESCRIPTOR_REVISION) ){ - call_str = "LookupAccountName"; - goto handle_error; - } - - if( !SetSecurityDescriptorGroup(&_sec_desc, _sec_sid.get(), FALSE) ){ - call_str = "SetSecurityDescriptorGroup"; - goto handle_error; - } - - if( !InitializeAcl(_sec_acl.get(), ACL_SIZE, ACL_REVISION) ){ - call_str = "InitializeAcl"; - goto handle_error; - } - - if( !AddAccessAllowedAce(_sec_acl.get(), ACL_REVISION, FILE_GENERIC_WRITE, _sec_sid.get()) ){ - call_str = "AddAccessAllowedAce(... FILE_GENERIC_WRITE ...)"; - goto handle_error; - } - - if( !AddAccessAllowedAce(_sec_acl.get(), ACL_REVISION, FILE_GENERIC_READ, _sec_sid.get()) ){ - call_str = "AddAccessAllowedAce(... FILE_GENERIC_READ ...)";; - goto handle_error; - } - - if( !SetSecurityDescriptorDacl(&_sec_desc, TRUE, _sec_acl.get(), FALSE) ){ - call_str = "SetSecurityDescriptorDacl"; - goto handle_error; - } - - return; - - handle_error: - { - errno_t e = GetLastError(); - call_str.append(" failed in _init_security_objects()"); - TOSDB_LogEx("IPC-Slave", call_str.c_str(), e); - throw std::runtime_error(call_str); - } - -} diff --git a/src/logging.cpp b/src/logging.cpp deleted file mode 100644 index 015112f..0000000 --- a/src/logging.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" -#include "concurrency.hpp" - -#include -#include -#include - -/* [Jan 16 2017] in order to synchronize 'atomic' write operations we use - a named mutex via NamedMutexLockGuard (if LOG_BACKEND_USE_SINGLE_FILE is - #defined and we need inter-process sync) or std::lock_guard - - (ideally we should use an async producer/consumer queue for better performance) */ - -namespace { - -const system_clock_type SYSTEM_CLOCK; -const size_type COL_WIDTHS[5] = {36, 12, 12, 20, 12}; -const LPCSTR SEVERITY[2] = {"LOW","HIGH"}; - -#ifndef LOG_BACKEND_USE_SINGLE_FILE -std::mutex log_file_mtx; -#endif - -std::ofstream log_file; - -void -_write_log(std::ofstream& fout, - std::string now, - Severity sevr, - std::string tag, - std::string desc) -{ - fout << std::setw(COL_WIDTHS[0]) << std::left << now.substr(0,COL_WIDTHS[0]-1) - << std::setw(COL_WIDTHS[1]) << std::left << GetCurrentProcessId() - << std::setw(COL_WIDTHS[2]) << std::left << GetCurrentThreadId() - << std::setw(COL_WIDTHS[3]) << std::left << std::string(tag).substr(0,COL_WIDTHS[3]-1) - << std::setw(COL_WIDTHS[4]) << std::left << SEVERITY[sevr] - << std::left << desc << std::endl; -} - -void -_write_err(std::string now, std::string tag, std::string desc) -{ - std::cerr << std::setw(COL_WIDTHS[0]) << std::left << now.substr(0,COL_WIDTHS[0]-1) - << std::setw(COL_WIDTHS[1]) << std::left << GetCurrentProcessId() - << std::setw(COL_WIDTHS[2]) << std::left << GetCurrentThreadId() - << std::setw(COL_WIDTHS[3]) << std::left << std::string(tag).substr(0,COL_WIDTHS[3]-1) - << std::left << desc << std::endl; -} - -}; - - -void -StartLogging(LPCSTR path) -{ -/* need to sync admin ops if we are sharing log between processes */ -#ifdef LOG_BACKEND_USE_SINGLE_FILE - NamedMutexLockGuard lock(LOG_BACKEND_MUTEX_NAME); -#endif - - if(log_file.is_open()) - return; - - log_file.open(path , std::ios::out | std::ios::app); - - auto pos = log_file.seekp(0,std::ios::end).tellp(); - if(pos == std::ios::pos_type(0)){ - /* write header if necessary */ - log_file << std::setw(COL_WIDTHS[0]) << std::left << "DATE / TIME" - << std::setw(COL_WIDTHS[1]) << std::left << "Process ID" - << std::setw(COL_WIDTHS[2]) << std::left << "Thread ID" - << std::setw(COL_WIDTHS[3]) << std::left << "Log TAG" - << std::setw(COL_WIDTHS[4]) << std::left << "Severity" - << std::left << "Description" - << std::endl << std::endl; - } -} - - -void -StopLogging() -{ -/* need to sync admin ops if we are sharing log between processes */ -#ifdef LOG_BACKEND_USE_SINGLE_FILE - NamedMutexLockGuard lock(LOG_BACKEND_MUTEX_NAME); -#endif - - log_file.close(); -} - - -void -ClearLog() -{ -/* need to sync admin ops if we are sharing log between processes */ -#ifdef LOG_BACKEND_USE_SINGLE_FILE - NamedMutexLockGuard lock(LOG_BACKEND_MUTEX_NAME); -#endif - - if(!log_file.is_open()) - return; - - log_file.clear(); -} - - -void -TOSDB_Log_(Severity sevr, LPCSTR tag, LPCSTR description) -{ - -#ifdef LOG_BACKEND_USE_SINGLE_FILE - /* if we share back end logs we need to sync across processes */ - NamedMutexLockGuard lock(LOG_BACKEND_MUTEX_NAME); -#else - /* if not just use a typical mutex to sync across intra-process threads */ - std::lock_guard lock(log_file_mtx); -#endif - - /* --- CRITICAL SECTION --- */ - std::string now = SysTimeString(); - - if(log_file.is_open()) - _write_log(log_file, now, sevr, tag, description); - else if(sevr > 0) - _write_err(now, tag, description); - /* --- CRITICAL SECTION --- */ -} - - -void -TOSDB_LogEx_(Severity sevr, LPCSTR tag, LPCSTR description, int error) -{ - std::string desc = std::string(description) + ", ERROR# " + std::to_string(error); - TOSDB_Log_(sevr, tag, desc.c_str()); -} - - -void -TOSDB_LogRaw_(Severity sevr, LPCSTR tag, LPCSTR description) -{ - std::string now = SysTimeString(); - - if(log_file.is_open()) - _write_log(log_file, now, sevr, tag, description); - else if(sevr > 0) - _write_err(now, tag, description); -} - - -std::string -SysTimeString(bool use_msec) -{ - using namespace std::chrono; - - time_t now; - long long ms_rem = 0; - int madj = use_msec ? 10 : 0; // 2 + 6 + 1 + 1(pad) = 10 char - - SmartBuffer buf(COL_WIDTHS[0] - madj); - - auto tp = SYSTEM_CLOCK.now(); - - if(use_msec){ - auto ms = duration_cast(tp.time_since_epoch()); - ms_rem = ms.count() % 1000000; - /* remove the msec so we don't accidentally round up */ - now = SYSTEM_CLOCK.to_time_t(tp - microseconds(ms_rem)); - }else{ - now = SYSTEM_CLOCK.to_time_t(tp); - } - - ctime_s(buf.get(), COL_WIDTHS[0] - madj, &now); - - std::string tmp(buf.get()); - tmp.pop_back(); - - if(use_msec) - tmp += " [" + std::to_string(ms_rem) + ']'; - - return tmp; -} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index 44da9cf..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,347 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" -#include "initializer_chain.hpp" - -#include -#include -#include -#include -#include - -/* this will be defined/exported for ALL (implemenation) modules; */ -/* extern */ char TOSDB_LOG_PATH[MAX_PATH+40]; - -namespace{ - -const std::set ITEM_VALID_START_CHARS = - InitializerChain> - ('-') - ('+') - ('.') - ('$') - ('/') - ('#'); - -const std::set ITEM_VALID_MID_CHARS = - InitializerChain> - ('-') - ('.') - ('*') - ('+') - (':') - ('$') - ('/') - ('#'); - -const std::set ITEM_VALID_END_CHARS = - InitializerChain> - ('.') - ('$') - ('#'); - -const std::unordered_map ITEM_SYMBOL_BUFFER_MAP = - InitializerChain> - ('-', "(1)") - ('.', "(2)") - ('$', "(3)") - ('/', "(4)") - ('*', "(5)") - ('+', "(6)") - ('#', "(7)") - (':', "(8)"); - -bool -_isValidItemChar(const char c, const std::set& valid_other_chars=std::set()) -{ - return (isalnum(c) || (valid_other_chars.find(c) != valid_other_chars.end())); -} - -}; /* namespace */ - - -BOOL WINAPI -DllMain(HINSTANCE mod, DWORD why, LPVOID res) -{ - /* NOTE: change HANDLE to HINSTANCE to avoid passing NULL to - GetModuleFileName which returns path of calling module - instead of *this* module, screwing up the relative log path. - - ***IF SOMETHING FUNDAMENTAL BREAKS LOOK HERE*** (Nov 10 2016) - */ - switch(why){ - case DLL_PROCESS_ATTACH: - { - /* - either use a relative(local) log path or create a log folder in - appdata; define TOSDB_LOG_PATH for all other implemenation modules - */ -#ifdef LOCAL_LOG_PATH /* add error checks */ - GetModuleFileName(mod, TOSDB_LOG_PATH, MAX_PATH); - std::string lp(TOSDB_LOG_PATH); - std::string nlp(lp.begin(),lp.begin() + lp.find_last_of('\\')); - nlp.append(LOCAL_LOG_PATH); - memset(TOSDB_LOG_PATH,0,(MAX_PATH+40)* sizeof(*TOSDB_LOG_PATH)); - strcpy(TOSDB_LOG_PATH, nlp.c_str()); -#else - GetEnvironmentVariable("APPDATA", TOSDB_LOG_PATH, MAX_PATH); - strcat_s(TOSDB_LOG_PATH, "\\tos-databridge\\"); /* needs to be < 40 char */ -#endif - CreateDirectory(TOSDB_LOG_PATH, NULL); - break; - } - case DLL_THREAD_ATTACH: /* no break */ - case DLL_THREAD_DETACH: /* no break */ - case DLL_PROCESS_DETACH:/* no break */ - default: break; - } - - return TRUE; -} - -/* IMPLEMENTATION ONLY */ - -std::string -CreateBufferName(std::string topic_str, std::string item) -{ /* - * name of mapping is of form: "TOSDB_Buffer__[topic name]_[item_name]" - * replacing reserved chars w/ ITEM_SYMBOL_BUFFER_MAP strings - */ - std::stringstream bname; - std::string str = "TOSDB_Buffer__" + topic_str + "_" + item; - - for( char c : str ){ - auto f = ITEM_SYMBOL_BUFFER_MAP.find(c); - bname << ((f == ITEM_SYMBOL_BUFFER_MAP.end()) ? std::string(1,c) : f->second); - } - -#ifdef NO_KGBLNS - return bname.str(); -#else - return std::string("Global\\").append(bname.str()); -#endif -} - -std::string -BuildLogPath(std::string name) -{ - return std::string(TOSDB_LOG_PATH) + name; -} - - -void -ParseArgs(std::vector& vec, std::string str) -{ - std::string::size_type i = str.find_first_of(' '); - - if( str.empty() ){ /* done */ - return; - }else if(i == std::string::npos){ /* only 1 str */ - vec.push_back(str); - return; - }else if(i == 0){ /* trim initial space(s) */ - ParseArgs(vec, str.substr(1,-1)); - return; - }else{ /* atleast 2 strings */ - vec.push_back(str.substr(0,i)); - ParseArgs(vec, str.substr(i+1,str.size())); - } -} - - -void /* in place 'safe'*/ -str_to_lower(char* str, size_t max) -{ - while(str && max--){ - str[0] = tolower(str[0]); - ++str; - } -} - - -void /* in place 'safe' */ -str_to_upper(char* str, size_t max) -{ - while(str && max--){ - str[0] = toupper(str[0]); - ++str; - } -} - -std::string -str_to_lower(std::string str) -{ - std::transform( str.begin(), str.end(), str.begin(), - [](unsigned char c){ return tolower(c); } ); - - return str; -} - - -std::string -str_to_upper(std::string str) -{ - std::transform( str.begin(), str.end(), str.begin(), - [](unsigned char c){ return toupper(c); } ); - - return str; -} - - -/* IMPLEMENTATION AND INTERFACE */ - -char** -NewStrings(size_t num_strs, size_t strs_len) -{ - char** strs = new char*[num_strs]; - - for(size_t i = 0; i < num_strs; ++i) - strs[i] = new char[strs_len + 1]; - - return strs; -} - -void -DeleteStrings(char** str_array, size_t num_strs) -{ - if(!str_array) - return; - - while(num_strs--){ - if(str_array[num_strs]) - delete[] str_array[num_strs]; - } - - delete[] str_array; -} - - -unsigned int -CheckStringLength(LPCSTR str) -{ - size_t slen = strnlen_s(str, TOSDB_MAX_STR_SZ+1); - if( slen == (TOSDB_MAX_STR_SZ+1) ){ - TOSDB_LogH("INPUT", "string length > TOSDB_MAX_STR_SZ"); - return 0; - } - - return 1; -} - - -unsigned int -CheckStringLengths(LPCSTR* str, size_type items_len) -{ - size_t slen; - - if(items_len > TOSDB_MAX_NSTRS){ - TOSDB_LogH("INPUT", "# of strings > TOSDB_MAX_NSTRS"); - return 0; - } - - while(items_len--){ - slen = strnlen_s(str[items_len], TOSDB_MAX_STR_SZ + 1); - if( slen == (TOSDB_MAX_STR_SZ+1) ){ - TOSDB_LogH("INPUT", "string length > TOSDB_MAX_STR_SZ"); - return 0; - } - } - - return 1; -} - - -unsigned int -CheckIDLength(LPCSTR id) -{ - size_t slen = strnlen_s(id, TOSDB_BLOCK_ID_SZ + 1); - if( slen == (TOSDB_BLOCK_ID_SZ + 1) ){ - TOSDB_LogH("INPUT", "block name length > TOSDB_BLOCK_ID_SZ"); - return 0; - } - - return 1; -} - - -unsigned int -IsValidBlockSize(size_type sz) -{ - if(sz < 1){ - TOSDB_LogH("BLOCK", "block size < 1"); - return 0; - } - - if(sz > TOSDB_MAX_BLOCK_SZ){ - TOSDB_LogH("BLOCK", ("block size (" + std::to_string(sz) + ") > TOSDB_MAX_BLOCK_SZ (" - + std::to_string(TOSDB_MAX_BLOCK_SZ) + ")").c_str()); - return 0; - } - - return 1; -} - - -unsigned int -IsValidBlockID(const char* id) -{ - /* check length */ - if( !CheckIDLength(id) ) - return 0; - - /* check isn't reserved */ - if(std::string(id) == std::string(TOSDB_RESERVED_BLOCK_NAME)){ - TOSDB_LogH("BLOCK", ("block name '" + std::string(id) + "' is reserved").c_str()); - return 0; - } - - return 1; -} - - -unsigned int -IsValidItemString(const char* item) -{ - if( !item || !CheckStringLength(item) ) - return 0; - - size_t slen = strlen(item); - - /* first char */ - if( !_isValidItemChar(item[0], ITEM_VALID_START_CHARS) ){ - TOSDB_LogH("INPUT", ("invalid item string: " + std::string(item)).c_str() ); - return 0; - } - - /* middle chars */ - for(int i = 1; i < slen -1; ++i){ - if( !_isValidItemChar(item[i], ITEM_VALID_MID_CHARS) ){ - TOSDB_LogH("INPUT", ("invalid item string: " + std::string(item)).c_str() ); - return 0; - } - } - - /* last char*/ - if( !_isValidItemChar(item[slen-1], ITEM_VALID_END_CHARS) ){ - TOSDB_LogH("INPUT", ("invalid item string: " + std::string(item)).c_str() ); - return 0; - } - - return 1; -} - diff --git a/src/raw_data_block.tpp b/src/raw_data_block.tpp deleted file mode 100644 index ab26873..0000000 --- a/src/raw_data_block.tpp +++ /dev/null @@ -1,498 +0,0 @@ - - -RAW_DATA_BLOCK_TEMPLATE -size_type RAW_DATA_BLOCK_CLASS::_block_count_ = 0; - -RAW_DATA_BLOCK_TEMPLATE -size_type RAW_DATA_BLOCK_CLASS::_max_block_count_ = MAX_BLOCK_COUNT; - - -RAW_DATA_BLOCK_TEMPLATE -RAW_DATA_BLOCK_CLASS::RawDataBlock(str_set_type items, - topic_set_type topics_t, - const size_type sz, - bool datetime) - : - _item_names(items), - _topic_enums(topics_t), - _block_sz(sz), - _datetime(datetime), - _mtx(new std::recursive_mutex) - { - _init(); - ++_block_count_; - } - -RAW_DATA_BLOCK_TEMPLATE -RAW_DATA_BLOCK_CLASS::RawDataBlock(const size_type sz, bool datetime) - : - _item_names(), - _topic_enums(), - _block_sz(sz), - _datetime(datetime), - _mtx(new std::recursive_mutex) - { - ++_block_count_; - } - -RAW_DATA_BLOCK_TEMPLATE -void -RAW_DATA_BLOCK_CLASS::_init() -{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - for(auto & i : _item_names) { - auto tmp = _populate_tblock( std::unique_ptr<_my_row_ty>(new _my_row_ty) ); - _block.insert( _my_block_ty::value_type(i, std::move(tmp)) ); - } - /* --- CRITICAL SECTION --- */ -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::_my_row_ty* -RAW_DATA_BLOCK_CLASS::_insert_topic( typename RAW_DATA_BLOCK_CLASS::_my_row_ty* row, - TOS_Topics::TOPICS topic ) -{ - DataStreamInterface *stream; - - switch(TOS_Topics::TypeBits(topic)){ - case TOSDB_STRING_BIT : - stream = _datetime - ? new DataStream(_block_sz) - : new DataStream(_block_sz); - break; - case TOSDB_INTGR_BIT : - stream = _datetime - ? new DataStream(_block_sz) - : new DataStream(_block_sz); - break; - case TOSDB_QUAD_BIT : - stream = _datetime - ? new DataStream(_block_sz) - : new DataStream(_block_sz); - break; - case TOSDB_INTGR_BIT | TOSDB_QUAD_BIT : - stream = _datetime - ? new DataStream(_block_sz) - : new DataStream(_block_sz); - break; - default : - stream = _datetime - ? new DataStream(_block_sz) - : new DataStream(_block_sz); - } - - try{ - row->erase(topic); - row->insert( - _my_row_ty::value_type( - topic, - std::unique_ptr>(stream) - ) - ); - }catch(...){ - TOSDB_LogH("RawDataBlock","problem inserting t-block"); - if(stream) - delete stream; - } - - return row; -} - -RAW_DATA_BLOCK_TEMPLATE -std::unique_ptr -RAW_DATA_BLOCK_CLASS::_populate_tblock(std::unique_ptr tblock) -{ - for(auto elem : _topic_enums) - _insert_topic(tblock.get(), elem); - - return tblock; -} - -RAW_DATA_BLOCK_TEMPLATE -RAW_DATA_BLOCK_CLASS* const -RAW_DATA_BLOCK_CLASS::CreateBlock(const str_set_type items, - const topic_set_type topics_t, - const size_type sz, - const bool datetime) -{ - if(items.empty()) - throw TOSDB_DataBlockError("items empty"); - - if(topics_t.empty()) - throw TOSDB_DataBlockError("topics_t empty"); - - if (_block_count_ >= _max_block_count_) - throw TOSDB_DataBlockLimitError(_max_block_count_); - - return new RawDataBlock(items, topics_t, sz, datetime) ; -} - -RAW_DATA_BLOCK_TEMPLATE -RAW_DATA_BLOCK_CLASS* const -RAW_DATA_BLOCK_CLASS::CreateBlock(const size_type sz,const bool datetime) -{ - if (_block_count_ >= _max_block_count_) - throw TOSDB_DataBlockLimitError(_max_block_count_); - - return new RawDataBlock(sz, datetime); -} - -RAW_DATA_BLOCK_TEMPLATE -size_type -RAW_DATA_BLOCK_CLASS::block_size(size_type b) -{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if(b > TOSDB_MAX_BLOCK_SZ) - b = TOSDB_MAX_BLOCK_SZ; - - for(auto& col : _block){ - for(auto& row : *(col.second)) - row.second->bound_size(b); - } - - return (_block_sz = b); - /* --- CRITICAL SECTION --- */ -} - - -RAW_DATA_BLOCK_TEMPLATE -template -void -RAW_DATA_BLOCK_CLASS::insert_data(TOS_Topics::TOPICS topic, - std::string item, - ValTy val, - DtTy datetime) -{ - DataStreamInterface *stream; - _my_row_ty *topics; - - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - try{ - topics = _block.at(item).get(); - if (!topics) - throw std::out_of_range("item not in block"); - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", e.what()); - throw TOSDB_DataBlockError(e.what()); - } - - try{ - stream = (topics->at(topic)).get(); - if (!stream) - throw std::out_of_range("topic not in block"); - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", e.what()); - throw TOSDB_DataBlockError(e.what()); - } - - try{ - stream->push(val, std::move(datetime)); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "insert_data"); - } - /* --- CRITICAL SECTION --- */ -} - -RAW_DATA_BLOCK_TEMPLATE -void -RAW_DATA_BLOCK_CLASS::add_item(std::string item) -{ - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if( !(_item_names.insert(item).second) ) - return; - - _block.erase(item); - - auto tmp = _populate_tblock( std::unique_ptr<_my_row_ty>(new _my_row_ty) ); - - _block.insert( _my_block_ty::value_type(item,std::move(tmp)) ); - /* --- CRITICAL SECTION --- */ - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "add_item"); - } -} - -RAW_DATA_BLOCK_TEMPLATE -void -RAW_DATA_BLOCK_CLASS::remove_item(std::string item) -{ - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if( !(_item_names.erase(item)) ) - return; - - _my_row_ty *row = _block.at(item).get(); - if(!row) - return; - - for(auto & elem : *row) - elem.second.reset(); - - _block.at(item).reset(); - _block.erase(item); - - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "remove_item out_of_range exception"); - throw TOSDB_DataBlockError(e, "remove_item"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "remove_item"); - } -} - -RAW_DATA_BLOCK_TEMPLATE -void -RAW_DATA_BLOCK_CLASS::add_topic(TOS_Topics::TOPICS topic) -{ - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if( !(_topic_enums.insert(topic).second) ) - return; - - for(auto & elem : _block) - _insert_topic(elem.second.get(), topic); - /* --- CRITICAL SECTION --- */ - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "add_topic"); - } -} - -RAW_DATA_BLOCK_TEMPLATE -void -RAW_DATA_BLOCK_CLASS::remove_topic(TOS_Topics::TOPICS topic) -{ - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - if( !(_topic_enums.erase(topic)) ) - return; - - for(auto & elem : _block){ - _my_row_ty *row = elem.second.get(); - if(row){ - (row->at(topic)).reset(); - row->erase(topic); - } - } - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "remove_topic out_of_range exception"); - throw TOSDB_DataBlockError(e, "remove_topic"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "remove_topic"); - } -} - -template -const DataStreamInterface* -RAW_DATA_BLOCK_CLASS::raw_stream_ptr(std::string item, - TOS_Topics::TOPICS topic) const -{ - const DataStreamInterface * stream = nullptr; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _my_row_ty* row = _block.at(item).get(); - if(row) - stream = (row->at(topic)).get(); - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "raw_stream_ptr out_of_range exception"); - throw TOSDB_DataBlockError(e, "raw_stream_ptr"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "raw_stream_ptr"); - } - - if(!stream) - throw TOSDB_DataBlockError("stream does not exist in block"); - - return stream; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::map_type -RAW_DATA_BLOCK_CLASS::map_of_frame_topics(std::string item) const -{ - map_type map; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _my_row_ty * row = _block.at(item).get(); - if(!row) - return map; - - for(auto & elem : *row){ - map.insert( - pair_type( - TOS_Topics::map[elem.first], - elem.second->get_leave_marker(0) - ) - ); - } - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "map_of_frame_topics out_of_range exception"); - throw TOSDB_DataBlockError(e, "map_of_frame_topics"); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "map_of_frame_topics"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "map_of_frame_topics"); - } - - return map; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::map_type -RAW_DATA_BLOCK_CLASS::map_of_frame_items(TOS_Topics::TOPICS topic) const -{ - map_type map; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - for(auto & elem : _block){ - map.insert( - pair_type(elem.first, elem.second->at(topic)->get_leave_marker(0)) - ); - } - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "map_of_frame_items out_of_range exception"); - throw TOSDB_DataBlockError(e, "map_of_frame_items"); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "map_of_frame_items"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "map_of_frame_items"); - } - - return map; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::map_datetime_type -RAW_DATA_BLOCK_CLASS::pair_map_of_frame_topics(std::string item) const -{ - map_datetime_type map; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - _my_row_ty * row = _block.at(item).get(); - if(!row) - return map; - - for(auto & elem : *row){ - map.insert( - map_datetime_type::value_type( - TOS_Topics::map[elem.first], - elem.second->both_leave_marker(0) - ) - ); - } - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "pair_map_of_frame_topics out_of_range exception"); - throw TOSDB_DataBlockError(e, "pair_map_of_frame_topics"); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "pair_map_of_frame_topics"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "pair_map_of_frame_topics"); - } - - return map; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::map_datetime_type -RAW_DATA_BLOCK_CLASS::pair_map_of_frame_items(TOS_Topics::TOPICS topic) const -{ - map_datetime_type map; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - for(auto & elem : _block){ - map.insert( - map_datetime_type::value_type( - elem.first, - elem.second->at(topic)->both_leave_marker(0) - ) - ); - } - /* --- CRITICAL SECTION --- */ - }catch(const std::out_of_range& e){ - TOSDB_LogH("RawDataBlock", "pair_map_of_frame_items out_of_range exception"); - throw TOSDB_DataBlockError(e, "pair_map_of_frame_items"); - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "pair_map_of_frame_items"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "pair_map_of_frame_items"); - } - - return map; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::matrix_type -RAW_DATA_BLOCK_CLASS::matrix_of_frame() const -{ - matrix_type matrix; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - for(auto & items : _block){ - map_type map; - for(auto & tops : *items.second){ - map.insert( - map_type::value_type( - TOS_Topics::map[tops.first], - tops.second->get_leave_marker(0) - ) - ); - } - matrix.insert(matrix_type::value_type(items.first, std::move(map))); - } - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "matrix_of_frame"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "matrix_of_frame"); - } - - return matrix; -} - -RAW_DATA_BLOCK_TEMPLATE -typename RAW_DATA_BLOCK_CLASS::matrix_datetime_type -RAW_DATA_BLOCK_CLASS::pair_matrix_of_frame() const -{ - matrix_datetime_type matrix; - try{ - std::lock_guard lock(*_mtx); - /* --- CRITICAL SECTION --- */ - for(auto & items : _block){ - map_datetime_type map; - for(auto & tops : *items.second){ - map.insert( - map_datetime_type::value_type( - TOS_Topics::map[tops.first], - tops.second->both_leave_marker(0) - ) - ); - } - matrix.insert( - matrix_datetime_type::value_type(items.first, std::move(map))); - } - /* --- CRITICAL SECTION --- */ - }catch(const DataStreamError& e){ - throw TOSDB_DataStreamError(e, "pair_matrix_of_frame"); - }catch(const std::exception & e){ - throw TOSDB_DataBlockError(e, "pair_matrix_of_frame"); - } - - return matrix; -} diff --git a/src/service/engine.cpp b/src/service/engine.cpp deleted file mode 100644 index 4e62475..0000000 --- a/src/service/engine.cpp +++ /dev/null @@ -1,1516 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include -#include /* debug */ -#include -#include -#include -#include -#include -#include -#include - -#include "tos_databridge.h" -#include "ipc.hpp" -#include "concurrency.hpp" - -namespace { - -typedef struct{ - void* hfile; /* handle to mapping */ - void* raw_addr; /* physical location in our process space */ - unsigned int raw_sz; /* physical size of the buffer */ - void* hmtx; -} StreamBuffer, *pStreamBuffer; - -typedef std::map item_refcounts_ty; -typedef std::pair buffer_id_ty; - -typedef TwoWayHashMap, std::hash, - std::equal_to, std::equal_to> convos_ty; - -LPCSTR CLASS_NAME = "DDE_CLIENT_WINDOW"; -LPCSTR APP_NAME = "TOS"; - -#ifdef LOG_BACKEND_USE_SINGLE_FILE -LPCSTR LOG_NAME = LOG_BACKEND_SINGLE_FILE_NAME; -#else -LPCSTR LOG_NAME = "engine-log.log"; -#endif - -#ifdef REDIRECT_STDERR_TO_LOG -LPCSTR ERR_LOG_NAME = "engine-stderr.log"; -#endif - -const system_clock_type system_clock; - -const unsigned int ACL_SIZE = 96; -const int NSECURABLE = 2; - -/* our 'private' messages; OK between 0x0400 and 0x7fff */ -const unsigned int LINK_DDE_ITEM = 0x0500; -const unsigned int REQUEST_DDE_ITEM = 0x0501; -const unsigned int DELINK_DDE_ITEM = 0x0502; -const unsigned int CLOSE_CONVERSATION = 0x0503; - -HINSTANCE hinstance = NULL; -SYSTEM_INFO sys_info; -SECURITY_ATTRIBUTES sec_attr[2]; -SECURITY_DESCRIPTOR sec_desc[2]; - -SmartBuffer sids[] = { - SmartBuffer(SECURITY_MAX_SID_SIZE), - SmartBuffer(SECURITY_MAX_SID_SIZE) -}; - -SmartBuffer acls[] = { - SmartBuffer(ACL_SIZE), - SmartBuffer(ACL_SIZE) -}; - -std::map buffers; -std::map topic_refcounts; - -convos_ty convos; - -LightWeightMutex topic_mtx; -LightWeightMutex buffer_mtx; -SignalManager ack_signals; - -/* !!! 'buffer_lock_guard_' is reserved inside this namespace !!! */ -#define BUFFER_LOCK_GUARD WinLockGuard buffer_lock_guard_(buffer_mtx) - -HANDLE init_event = NULL; -HANDLE msg_thrd = NULL; -HWND msg_window = NULL; -DWORD msg_thrd_id = 0; -LPCSTR msg_window_name = "TOSDB_ENGINE_MSG_WNDW"; - -volatile bool pause_flag = false; -volatile bool shutdown_flag = false; - -/* forward decl */ -template -class DDE_Data; - -bool -CheckExecType(LPSTR cmd); - -DWORD WINAPI -ThreadedWinInit(LPVOID lParam); - -bool -RunMainCommLoop(IPCSlave *pslave); - -bool -ParseIPCMessage(std::string msg, - unsigned int *op, - TOS_Topics::TOPICS *topic, - std::string *item, - unsigned long *timeout); - -int -HandleGoodIPCMessage(unsigned int op, - TOS_Topics::TOPICS topic, - std::string item, - unsigned long timeout); - -int -CleanUpMain(int ret_code); - -int -AddStream(TOS_Topics::TOPICS topic_t,std::string item, unsigned long timeout, bool log=true); - -int -RemoveStream(TOS_Topics::TOPICS topic_t,std::string item, unsigned long timeout); - -void -RemoveAllStreams(unsigned long timeout); - -int -TestStream(TOS_Topics::TOPICS topic_t,std::string item, unsigned long timeout); - -int -CreateTopic(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout); - -int -CreateItem(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout); - -void -CloseTopic(TOS_Topics::TOPICS topic_t, unsigned long timeout); - -int -CloseItem(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout); - -bool -PostItem(std::string item,TOS_Topics::TOPICS topic_t, unsigned long timeout); - -bool -PostCloseItem(std::string item,TOS_Topics::TOPICS topic_t, unsigned long timeout); - -bool -CreateBuffer(TOS_Topics::TOPICS topic_t, - std::string item, - unsigned int buffer_sz = TOSDB_SHEM_BUF_SZ); - -bool -DestroyBuffer(TOS_Topics::TOPICS topic_t, std::string item); - -template -void -RouteToBuffer(DDE_Data data); - -LRESULT CALLBACK -WndProc(HWND, UINT, WPARAM, LPARAM); - -void -HandleData(UINT msg, WPARAM wparam, LPARAM lparam); - -void -DumpBufferStatus(); - -bool -SetSecurityPolicy(); - -}; - - -int WINAPI -WinMain(HINSTANCE hInst, HINSTANCE hPrevInst, LPSTR lpCmdLn, int nShowCmd) -{ - std::string logpath(TOSDB_LOG_PATH); - -#ifdef REDIRECT_STDERR_TO_LOG - freopen( (logpath + ERR_LOG_NAME).c_str(), "a", stderr); -#endif - - /* start logging */ - logpath.append(std::string(LOG_NAME)); - StartLogging( logpath.c_str() ); - - /* check/log type of execution (service v. pure, spawned v. direct) */ - if( !CheckExecType(lpCmdLn) ){ - return 0; // only exit if user agrees (no error state) - } - - /* the other side of our IPC channel (see client_admin.cpp) */ - IPCSlave slave(TOSDB_COMM_CHANNEL); - - /* initialize our security objects for IPC */ - if( !SetSecurityPolicy() ){ - TOSDB_LogH("STARTUP", "engine failed to initialize security objects"); - return TOSDB_ERROR_IPC; - } - - /* setup our windows class that will run the engine */ - WNDCLASS clss = {}; - hinstance = GetModuleHandle(NULL); - clss.hInstance = hinstance; - clss.lpfnWndProc = WndProc; - clss.lpszClassName = CLASS_NAME; - RegisterClass(&clss); - - /* spin-off the windows msg loop that we'll communicate w/ directly via - private messages in the main communication loop below; it will also - respond to return DDE messages from the TOS platform */ - msg_thrd = CreateThread(NULL, 0, ThreadedWinInit, NULL, 0, &msg_thrd_id); - if(!msg_thrd) - return TOSDB_ERROR_CONCURRENCY; - - init_event = CreateEvent(NULL, FALSE, FALSE, NULL); - - /* block until its ready */ - switch( WaitForSingleObject(init_event,TOSDB_DEF_TIMEOUT) ){ - case WAIT_TIMEOUT: - TOSDB_LogH("STARTUP","main thread timed out waiting for signal from msg thread"); - return CleanUpMain(TOSDB_ERROR_CONCURRENCY); - case WAIT_FAILED: - TOSDB_LogH("STARTUP","main thread failed to receive signal from msg thread"); - return CleanUpMain(TOSDB_ERROR_CONCURRENCY); - } - - GetSystemInfo(&sys_info); - - /* Start the main communciation loop that client code and service will - use to communicate with the back-end; this will block until: - 1) the slave's wait_for_master call returns false(IPC ERROR), OR - 2) TOSDB_SIG_STOP signal is received from client/service */ - int err = RunMainCommLoop(&slave) ? 0 : TOSDB_ERROR_IPC; - - TOSDB_LogH("CONTROL","out of run loop (remove all streams, clean up)"); - RemoveAllStreams(TOSDB_DEF_TIMEOUT); - err = CleanUpMain(err); - - StopLogging(); - -#ifdef REDIRECT_STDERR_TO_LOG - fclose(stderr); -#endif - - return err; -} - - -namespace { - -bool -CheckExecType(LPSTR cmd) -{ - bool is_service = false; - bool is_spawned = false; - - std::vector args; - ParseArgs(args, cmd); - - /* check whether we were spawned by service... */ - if(!args.empty() && args[0] == "--spawned") - is_spawned=true; - - /* ...if not, warn */ - if(!is_spawned){ - std::string warn_msg("Running tos-databridge-engine directly is not recommended. " - "Inadequate privileges can cause connection issues with TOS or fatal errors. \n\n" - "Using 'tos-databridge-serv --noservice' to spawn the engine is recommended.\n\n"); - -#ifndef NO_KERNEL_GLOBAL_NAMESPACE - warn_msg.append( - "NO_KERNEL_GLOBAL_NAMESPACE is not defined and you may not have the " - "necessary privileges to create global kernel objects. This may result " - " in a CRASH.\n\n" - ); - - TOSDB_LogH("STARTUP", "engine being run directly without NO_KERNEL_GLOBAL_NAMESPACE"); -#endif - if(MessageBox(NULL,warn_msg.c_str(),"Warning",MB_OKCANCEL | MB_ICONWARNING) == IDCANCEL) - { - TOSDB_LogH("STARTUP", "Warning Box - Cancel; aborting startup"); - return false; - } - }; - - std::stringstream ss_args; - ss_args<< "lpCmdLn args: "; - for(auto & a : args){ - ss_args<< a << ' '; - /* if we get the service arg run as service, pure executable otherwise*/ - if(a == "--service") - is_service = true; - } - - if(is_service && !is_spawned){ - TOSDB_LogH("STARTUP", "can not run as 'unspawned' service; default to --noservice"); - is_service = false; - } - - TOSDB_Log("STARTUP", (is_service ? "is_service == true" : "is_service == false")); - TOSDB_Log("STARTUP", ss_args.str().c_str()); - - return true; -} - - -DWORD WINAPI -ThreadedWinInit(LPVOID lParam) -{ - if(!hinstance) - hinstance = GetModuleHandle(NULL); - - msg_window = CreateWindow(CLASS_NAME, msg_window_name, WS_OVERLAPPEDWINDOW, - 0, 0, 0, 0, NULL, NULL, hinstance, NULL); - - if(!msg_window) - return 1; - - SetEvent(init_event); - - MSG msg = {}; - while( GetMessage(&msg, NULL, 0, 0) ) - { - TranslateMessage(&msg); - DispatchMessage(&msg); - } - - return 0; -} - - -bool -RunMainCommLoop(IPCSlave *pslave) -{ - TOS_Topics::TOPICS cli_topic; - std::string cli_item; - std::string ipc_msg; - unsigned long cli_timeout; - unsigned int cli_op; - bool good_msg; - int resp; - - TOSDB_Log("STARTUP", "entering RunMainCommLoop"); - while(!shutdown_flag){ - - /* BLOCK until master is ready */ - if( !pslave->wait_for_master() ){ - shutdown_flag = true; - TOSDB_LogH("IPC", "wait_for_master failed"); - return false; - } - - /* BLOCK until we get a message */ - if( !pslave->recv(&ipc_msg) ){ - pslave->drop_master(); - TOSDB_LogH("IPC", "recv failed"); - continue; - } - - /* parse the received msg */ - good_msg = ParseIPCMessage(ipc_msg, &cli_op, &cli_topic, &cli_item, &cli_timeout); - if(good_msg){ - resp = HandleGoodIPCMessage(cli_op, cli_topic, cli_item, cli_timeout); - }else{ - resp = TOSDB_ERROR_IPC_MSG; - TOSDB_LogH("IPC", ("failed to parse message: " + ipc_msg).c_str()); - } - - /* reply to MASTER */ - if( !pslave->send(std::to_string(resp)) ){ - TOSDB_LogH("IPC", "send/reply failed in main comm loop"); - } - - pslave->drop_master(); - } - - TOSDB_Log("SHUTDOWN", "exiting MainCommLoop"); - return true; -} - - -#define STREAM_CHECK_LOG_ERROR(e,call,topic,item,tout) do{ \ -if(e){ \ - std::stringstream err_s; \ - err_s << "Error (" << std::to_string(e) << ") in " << call << " (" \ - << TOS_Topics::map[topic] << ',' << item << ',' \ - << std::to_string(tout) << ')'; \ - TOSDB_LogH("STREAM-OP", err_s.str().c_str()); \ -} \ -}while(0) - - -int -HandleGoodIPCMessage( unsigned int op, - TOS_Topics::TOPICS topic, - std::string item, - unsigned long timeout ) -{ - int ret = TOSDB_SIG_BAD; - - switch(op){ - case TOSDB_SIG_ADD: - ret = AddStream(topic, item, timeout); - STREAM_CHECK_LOG_ERROR(ret, "AddStream", topic, item, timeout); - break; - - case TOSDB_SIG_REMOVE: - ret = RemoveStream(topic, item, timeout); - STREAM_CHECK_LOG_ERROR(ret, "RemoveStream", topic, item, timeout); - break; - - case TOSDB_SIG_TEST: - ret = TestStream(topic, item, timeout); - break; - - case TOSDB_SIG_PAUSE: - TOSDB_Log("SERVICE-MSG", "TOSDB_SIG_PAUSE message received"); - pause_flag = true; - ret = TOSDB_SIG_GOOD; - break; - - case TOSDB_SIG_CONTINUE: - TOSDB_Log("SERVICE-MSG", "TOSDB_SIG_CONTINUE message received"); - pause_flag = false; - ret = TOSDB_SIG_GOOD; - break; - - case TOSDB_SIG_STOP: - TOSDB_Log("SERVICE-MSG", "TOSDB_SIG_STOP message received"); - shutdown_flag = true; - ret = TOSDB_SIG_GOOD; - break; - - case TOSDB_SIG_DUMP: - TOSDB_Log("SERVICE-MSG", "TOSDB_SIG_DUMP message received"); - DumpBufferStatus(); - ret = TOSDB_SIG_GOOD; - break; - - default: - TOSDB_LogH("IPC", ("invalid opcode: " + std::to_string(op)).c_str()); - } - - return ret; -} - -#undef STREAM_CHECK_LOG_ERROR - - -bool -ParseIPCMessage( std::string msg, - unsigned int *op, - TOS_Topics::TOPICS *topic, - std::string *item, - unsigned long *timeout ) -{ - size_t mlen = msg.length(); - if(mlen > IPCBase::MAX_MESSAGE_SZ) - { - TOSDB_LogH("IPC", ("message length (" + std::to_string(mlen) + ") > MAX_MESSAGE_SZ (" - + std::to_string(IPCBase::MAX_MESSAGE_SZ) + ')').c_str()); - return false; - } - - std::vector args; - ParseArgs(args, msg); - - size_t nargs = args.size(); - if(nargs == 0){ - TOSDB_LogH("IPC", ("ParseArgs returned 0 args, msg: " + msg).c_str()); - return false; - } - - try{ /* first, get the opcode */ - *op = (unsigned int)std::stoul(args[0]); - }catch(...){ - TOSDB_LogH("IPC", ("failed to get 'op' arg from msg, args[0]: " + args[0]).c_str()); - return false; - } - - switch(*op){ /* break out if we just need an opcode */ - case TOSDB_SIG_PAUSE: - case TOSDB_SIG_CONTINUE: - case TOSDB_SIG_STOP: - case TOSDB_SIG_DUMP: - return true; - }; - - if(nargs != 4){ /* if we we have a stream op check we have 4 args */ - TOSDB_LogH("IPC", ("ParseArgs didn't return 4 args (" - + std::to_string(nargs) + "), msg: " + msg).c_str()); - return false; - } - - try{ /* second, get the topic */ - *topic = TOS_Topics::MAP()[args[1]]; - }catch(...){ - TOSDB_LogH("IPC", ("failed to get 'topic' arg from msg, args[1]: " + args[1]).c_str()); - return false; - } - - /* third, the item */ - *item = args[2]; - - try{ /* fourth, the timeout */ - *timeout = std::stoul(args[3]); - }catch(...){ - TOSDB_LogH("IPC", ("failed to get 'timeout' arg from msg, args[3]: " + args[3]).c_str()); - return false; - } - - return true; -} - - -int -CleanUpMain(int ret_code) -{ - if(ret_code) - TOSDB_LogEx("SHUTDOWN", "engine exiting because of error", ret_code); - else - TOSDB_LogH("SHUTDOWN", "engine exiting"); - - if(msg_window) - DestroyWindow(msg_window); - - if(!hinstance) - hinstance = GetModuleHandle(NULL); - - UnregisterClass(CLASS_NAME, hinstance); - return ret_code; -} - - -int -AddStream( TOS_Topics::TOPICS topic_t, - std::string item, - unsigned long timeout, - bool log ) -{ - int err = 0; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC){ - return TOSDB_ERROR_BAD_TOPIC; - } - - auto topic_iter = topic_refcounts.find(topic_t); - if(topic_iter == topic_refcounts.end()) /* if topic isn't in our global mapping */ - { - err = CreateTopic(topic_t, item, timeout); - if(err && log){ - TOSDB_LogEx("STREAM", "error creating new topic stream", err); - } - } - else /* if it already is */ - { - auto item_iter = topic_iter->second.find(item); - if(item_iter == topic_iter->second.end()) /* and it doesn't have that item yet */ - { - err = CreateItem(topic_t, item, timeout); - if(err && log){ - TOSDB_LogEx("STREAM", "error creating new item stream", err); - } - } - else /* if it does, simply increment the ref-count */ - { - ++(item_iter->second); - } - } - - /* unwind if it fails during creation */ - switch(err){ - case TOSDB_ERROR_SHEM_BUFFER: - PostCloseItem(item, topic_t, timeout); - topic_refcounts[topic_t].erase(item); - /* no break */ - case TOSDB_ERROR_DDE_POST: - if( !topic_refcounts[topic_t].empty() ) - break; - /* else - no break */ - case TOSDB_ERROR_DDE_NO_ACK: - CloseTopic(topic_t, timeout); - } - - return err; -} - - -int -RemoveStream( TOS_Topics::TOPICS topic_t, - std::string item, - unsigned long timeout ) -{ - int err = 0; - - if(topic_t == TOS_Topics::TOPICS::NULL_TOPIC){ - return TOSDB_ERROR_BAD_TOPIC; - } - - auto topic_iter = topic_refcounts.find(topic_t); - /* if topic is not in our global mapping */ - if(topic_iter == topic_refcounts.end()) - return TOSDB_ERROR_ENGINE_NO_TOPIC; - - auto item_iter = topic_iter->second.find(item); - /* if it has that item */ - if(item_iter != topic_iter->second.end()){ - /* decr the ref count */ - --(item_iter->second); - /* if ref-count hits zero post close msg and destroy the buffer*/ - if(item_iter->second == 0){ - err = CloseItem(topic_t, item, timeout); - topic_iter->second.erase(item_iter); - } - }else{ - err = TOSDB_ERROR_ENGINE_NO_ITEM; - } - - /* if no items close the convo */ - if(topic_iter->second.empty()) - CloseTopic(topic_t, timeout); - - return err; -} - - -void -RemoveAllStreams(unsigned long timeout) -{ /* need to iterate through copies */ - std::map t_copy; - item_refcounts_ty rc_copy; - - auto tii = std::inserter(t_copy, t_copy.begin()); - std::copy(topic_refcounts.begin(), topic_refcounts.end(), tii); - - for(const auto& topic: t_copy){ - auto rii = std::inserter(rc_copy,rc_copy.begin()); - std::copy(topic.second.begin(), topic.second.end(), rii); - for(const auto& item : rc_copy){ - RemoveStream(topic.first, item.first, timeout); - } - rc_copy.clear(); - } -} - - -int -TestStream(TOS_Topics::TOPICS topic_t,std::string item, unsigned long timeout) -{ - int a = AddStream(topic_t, item, timeout, false); - if(a) - return a; - - int r = RemoveStream(topic_t, item, timeout); - if(r){ - TOSDB_LogEx("STREAM-OP", "TestStream failed to remove stream (stream leaked)", r); - return r; - } - - return 0; -} - - -int -CreateItem(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout) -{ - /* consider creating buffer before posting to avoid getting data messages - before we have a data buffer to put them in */ - - if( !PostItem(item, topic_t, timeout) ){ - return TOSDB_ERROR_DDE_POST; - } - - topic_refcounts[topic_t][item] = 1; - - if( !CreateBuffer(topic_t, item) ){ - return TOSDB_ERROR_SHEM_BUFFER; - } - - return 0; -} - - -int -CreateTopic(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout) -{ - std::string topic_str; - ATOM topic_atom; - ATOM app_atom; - bool ret; - - topic_str = TOS_Topics::map[topic_t]; - topic_atom = GlobalAddAtom(topic_str.c_str()); - app_atom = GlobalAddAtom(APP_NAME); - - ack_signals.set_signal_ID(TOS_Topics::map[topic_t]); - - if(topic_atom){ - SendMessageTimeout( (HWND)HWND_BROADCAST, - WM_DDE_INITIATE, - (WPARAM)msg_window, - MAKELONG(app_atom,topic_atom), - SMTO_NORMAL, 500, NULL ); - } - - if(app_atom) - GlobalDeleteAtom(app_atom); - - if(topic_atom) - GlobalDeleteAtom(topic_atom); - - /* wait for ack from DDE server */ - ret = ack_signals.wait_for(TOS_Topics::map[topic_t], timeout); - if(!ret){ /* are we sure about this? error unwind will call CloseTopic - - whats the purpose if we never got the 'ack'? (maybe a late ack) - - deadlock or corrupt 'convos' on sending WM_DDE_TERMINATE in this state?*/ - return TOSDB_ERROR_DDE_NO_ACK; - } - - /* once we get our ack set up/initialize our refcounts - we do it here because CloseTopic will remove if we have error below */ - topic_refcounts[topic_t] = item_refcounts_ty(); - - ret = PostItem(item, topic_t, timeout); - if(!ret) - return TOSDB_ERROR_DDE_POST; - - /* because of how AddStream unwinds errors order is important; - this MUST come after PostItem, before CreateBuffer */ - topic_refcounts[topic_t][item] = 1; - - ret = CreateBuffer(topic_t, item); - if(!ret) - return TOSDB_ERROR_SHEM_BUFFER; - - return 0; -} - - -int -CloseItem(TOS_Topics::TOPICS topic_t, std::string item, unsigned long timeout) -{ - int err = 0; - - /* if we return error, continue with remove but log it */ - if( !PostCloseItem(item, topic_t, timeout) ) - { - err = TOSDB_ERROR_DDE_POST; - TOSDB_LogH("ENGINE", "PostCloseItem failed, continue with CloseItem"); - } - - /* previous err code takes priority */ - if( !DestroyBuffer(topic_t, item) ){ - err = (err ? err : TOSDB_ERROR_SHEM_BUFFER); - } - - return err; -} - - -void -CloseTopic(TOS_Topics::TOPICS topic_t, unsigned long timeout) -{ - if( !PostMessage(msg_window, CLOSE_CONVERSATION, (WPARAM)convos[topic_t], NULL) ) - { - TOSDB_LogEx("ENGINE", "CloseTopic::PostMessage::CLOSE_CONVERSATION failed", GetLastError()); - } - topic_refcounts.erase(topic_t); - convos.remove(topic_t); -} - - -bool -PostItem(std::string item, - TOS_Topics::TOPICS topic_t, - unsigned long timeout) -{ - HWND convo = convos[topic_t]; - std::string sid_id = std::to_string((size_t)convo) + item; - - ack_signals.set_signal_ID(sid_id); - if( !PostMessage(msg_window, REQUEST_DDE_ITEM, (WPARAM)convo, (LPARAM)(item.c_str())) ) - { - TOSDB_LogEx("ENGINE", "PostItem::PostMessage::REQUEST_DDE_ITEM failed", GetLastError()); - } - - /* for whatever reason a bad item gets a posive ack from an attempt - to link it, so that message must post second to give the request - a chance to preempt it */ - - if( !PostMessage(msg_window, LINK_DDE_ITEM, (WPARAM)convo, (LPARAM)(item.c_str())) ) - { - TOSDB_LogEx("ENGINE", "PostItem::PostMessage::LINK_DDE_ITEM failed", GetLastError()); - } - - return ack_signals.wait_for(sid_id , timeout); -} - - -bool -PostCloseItem(std::string item, - TOS_Topics::TOPICS topic_t, - unsigned long timeout) -{ - HWND convo = convos[topic_t]; - std::string sid_id = std::to_string((size_t)convo) + item; - - ack_signals.set_signal_ID(sid_id); - if( !PostMessage(msg_window, DELINK_DDE_ITEM, (WPARAM)convo, (LPARAM)(item.c_str())) ) - { - TOSDB_LogEx("ENGINE", "PostCloseItem::PostMessage::DELINK_DDE_ITEM failed", GetLastError()); - } - - return ack_signals.wait_for(sid_id, timeout); -} - - -bool -CreateBuffer(TOS_Topics::TOPICS topic_t, - std::string item, - unsigned int buffer_sz) -{ - StreamBuffer buf; - std::string err_msg; - DWORD err; - - buffer_id_ty id(item, topic_t); - - {/* Feb-15-2017 - protect the buffers map; write thread may try to access */ - BUFFER_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - if(buffers.find(id) != buffers.end()) - return false; - } - - std::string buf_name = CreateBufferName(TOS_Topics::map[topic_t], item); - std::string mtx_name = std::string(buf_name).append("_mtx"); - - buf.raw_sz = (buffer_sz < sys_info.dwPageSize) ? sys_info.dwPageSize : buffer_sz; - - buf.hfile = CreateFileMapping( INVALID_HANDLE_VALUE, - &sec_attr[SHEM1], - PAGE_READWRITE, 0, - buf.raw_sz, - buf_name.c_str() ); - if(!buf.hfile){ - err = GetLastError(); - err_msg = "failed to create file mapping: " + buf_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - return false; - } - - buf.raw_addr = MapViewOfFile(buf.hfile, FILE_MAP_ALL_ACCESS, 0, 0, 0); - if(!buf.raw_addr){ - err = GetLastError(); - err_msg = "failed to map shared memory: " + buf_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - CloseHandle(buf.hfile); - return false; - } - - // should we close the handle to the file mapping ?? - - buf.hmtx = CreateMutex(&sec_attr[MUTEX1], FALSE, mtx_name.c_str()); - if(!buf.hmtx){ - err = GetLastError(); - err_msg = "failed to open MUTEX handle: " + mtx_name; - TOSDB_LogEx("DATA BUFFER", err_msg.c_str(), err); - CloseHandle(buf.hfile); - UnmapViewOfFile(buf.raw_addr); - return false; - } - - /* cast mem-map to our header and fill values */ - pBufferHead ptmp = (pBufferHead)(buf.raw_addr); - ptmp->loop_seq = 0; - ptmp->next_offset = ptmp->beg_offset = sizeof(BufferHead); - ptmp->elem_size = TOS_Topics::TypeSize(topic_t) + sizeof(DateTimeStamp); - ptmp->end_offset = ptmp->beg_offset - + ((buf.raw_sz - ptmp->beg_offset) / ptmp->elem_size) - * ptmp->elem_size ; - - /* Feb-15-2017 - protect the buffers map; write thread may try to access */ - BUFFER_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - buffers.insert( std::make_pair(id,buf) ); - return true; - /* --- CRITICAL SECTION --- */ -} - - -bool -DestroyBuffer(TOS_Topics::TOPICS topic_t, std::string item) -{ - bool b = true; - - /* don't allow write attempt while buffer is being destroyed */ - BUFFER_LOCK_GUARD; - /* ---CRITICAL SECTION --- */ - - auto buf_iter = buffers.find( buffer_id_ty(item,topic_t) ); - if(buf_iter == buffers.end()) - return false; - - if( !UnmapViewOfFile(buf_iter->second.raw_addr) ){ - TOSDB_LogEx("BUFFER", "UnmapViewOfFile(raw_addr) failed", GetLastError()); - b = false; - } - - if( !CloseHandle(buf_iter->second.hfile) ){ - TOSDB_LogEx("BUFFER", "CloseHandle(hfile) failed", GetLastError()); - b = false; - } - - if( !CloseHandle(buf_iter->second.hmtx) ){ - TOSDB_LogEx("BUFFER", "CloseHanlde(hmtx) failed", GetLastError()); - b = false; - } - - buffers.erase(buf_iter); - return b; - /* ---CRITICAL SECTION --- */ -} - - -template -inline void -ValToBuf(void* pos, T val) -{ - *(T*)pos = val; -} - -template<> -inline void -ValToBuf(void* pos, std::string val) /* copy the string, truncate if necessary */ -{ - strncpy_s((char*)pos, TOSDB_STR_DATA_SZ, val.c_str(), TOSDB_STR_DATA_SZ-1); -} - -template -void -RouteToBuffer(DDE_Data data) -{ - pBufferHead head; - - BUFFER_LOCK_GUARD; - /* ---(INTRA-PROCESS) CRITICAL SECTION --- */ - - auto buf_iter = buffers.find(buffer_id_ty(data.item,data.topic)); - if(buf_iter == buffers.end()){ - /* simply aborting avoids the need for sync between the thread that - creates/destroys buffers and the thread (this one) that writes to them */ - return; - } - - head = (pBufferHead)(buf_iter->second.raw_addr); - - WaitForSingleObject(buf_iter->second.hmtx, INFINITE); - /* ---(INTER-PROCESS) CRITICAL SECTION --- */ - ValToBuf((void*)((char*)head + head->next_offset), data.data); - - *(pDateTimeStamp)((char*)head + head->next_offset - + (head->elem_size - sizeof(DateTimeStamp))) = *data.time; - - if((head->next_offset + head->elem_size) >= head->end_offset){ - head->next_offset = head->beg_offset; - ++(head->loop_seq); - }else{ - head->next_offset += head->elem_size; - } - - /* ---(INTER-PROCESS) CRITICAL SECTION --- */ - ReleaseMutex(buf_iter->second.hmtx); - /* ---(INTRA-PROCESS) CRITICAL SECTION --- */ -} - - -LRESULT CALLBACK -WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) -{ - switch (message){ - case WM_DDE_DATA: - { /* TODO: de-link it all and store state, then re-init on continue */ - if(!pause_flag) - HandleData(message, wParam, lParam); - break; - } - case LINK_DDE_ITEM: - { - DDEADVISE FAR* lp_options; - LPARAM lp; - ATOM item; - - HGLOBAL hoptions = GlobalAlloc(GMEM_MOVEABLE, sizeof(DDEADVISE)); - if (!hoptions){ - TOSDB_LogEx("DDE", "LINK_DDE_ITEM::GlobalAlloc failed", GetLastError()); - break; - } - - lp_options = (DDEADVISE FAR*)GlobalLock(hoptions); - if (!lp_options){ - TOSDB_LogEx("DDE", "LINK_DDE_ITEM::GlobalLock failed", GetLastError()); - GlobalFree(hoptions); - break; - } - lp_options->cfFormat = CF_TEXT; - lp_options->fAckReq = FALSE; - lp_options->fDeferUpd = FALSE; - - GlobalUnlock(hoptions); - - item = GlobalAddAtom((LPCSTR)lParam); - if(!item){ - TOSDB_LogEx("DDE", "LINK_DDE_ITEM::GlobalAddAtom failed", GetLastError()); - break; - } - - lp = PackDDElParam(WM_DDE_ADVISE, (UINT_PTR)hoptions, item); - - if( !PostMessage((HWND)wParam, WM_DDE_ADVISE, (WPARAM)msg_window, lp) ) - { - TOSDB_LogEx("DDE", "LINK_DDE_ITEM::PostMessage failed", GetLastError()); - GlobalDeleteAtom(item); - GlobalFree(hoptions); - FreeDDElParam(WM_DDE_ADVISE, lp); - } - break; - } - case REQUEST_DDE_ITEM: - { - LPARAM lp; - - ATOM item = GlobalAddAtom((LPCSTR)lParam); - if(!item){ - TOSDB_LogEx("DDE", "REQUEST_DDE_ITEM::GlobalAddAtom failed", GetLastError()); - break; - } - - lp = PackDDElParam(WM_DDE_REQUEST, CF_TEXT, item); - - if( !PostMessage((HWND)wParam, WM_DDE_REQUEST, (WPARAM)(msg_window), lp) ) - { - TOSDB_LogEx("DDE", "REQUEST_DDE_ITEM::PostMessage failed", GetLastError()); - GlobalDeleteAtom(item); - } - - break; - } - case DELINK_DDE_ITEM: - { - LPARAM lp; - - ATOM item = GlobalAddAtom((LPCSTR)lParam); - if(!item){ - TOSDB_LogEx("DDE", "DELINK_DDE_ITEM::GlobalAddAtom failed", GetLastError()); - break; - } - - lp = PackDDElParam(WM_DDE_UNADVISE, 0, item); - - if( !PostMessage((HWND)wParam, WM_DDE_UNADVISE, (WPARAM)(msg_window), lp) ) - { - TOSDB_LogEx("DDE", "DELINK_DDE_ITEM::PostMessage failed", GetLastError()); - GlobalDeleteAtom(item); - FreeDDElParam(WM_DDE_UNADVISE, lp); - } - - break; - } - case CLOSE_CONVERSATION: - { - if( !PostMessage((HWND)wParam, WM_DDE_TERMINATE, (WPARAM)msg_window, NULL) ) - { - TOSDB_LogEx("DDE", "CLOSE_CONVERSATION::PostMessage failed", GetLastError()); - } - break; - } - case WM_DESTROY: - { - PostQuitMessage(0); - break; - } - case WM_DDE_ACK: - { - char topic_atom[TOSDB_MAX_STR_SZ + 1]; - char app_atom[TOSDB_MAX_STR_SZ + 1]; - char app_str_lower[TOSDB_MAX_STR_SZ + 1]; - char item_atom[TOSDB_MAX_STR_SZ + 1]; - - UINT_PTR pho = 0; - UINT_PTR plo = 0; - - TOSDB_LogDebug( ("ACK RECEIVED - lParam: " + std::to_string(lParam) + - ", wParam: " + std::to_string(wParam)).c_str() ); - - if (lParam <= 0){ - pho = HIWORD(lParam); /*topic*/ - plo = LOWORD(lParam); /*app*/ - - GlobalGetAtomName((ATOM)(pho), topic_atom, (TOSDB_MAX_STR_SZ + 1)); - GlobalGetAtomName((ATOM)(plo), app_atom, (TOSDB_MAX_STR_SZ + 1)); - - strcpy_s(app_str_lower, APP_NAME); - str_to_lower(app_atom, TOSDB_MAX_STR_SZ); - str_to_lower(app_str_lower, TOSDB_MAX_STR_SZ); - - TOSDB_LogDebug( ("lParam <= 0 - app_atom: " + std::string(app_atom) + - ", topic_atom: " + std::string(topic_atom)).c_str() ); - - if( strcmp(app_atom, app_str_lower) ){ - TOSDB_LogH("DDE", (app_atom + std::string(" != ") + app_str_lower).c_str() ); - break; - } - - convos_ty::pair1_type cp(TOS_Topics::map[topic_atom],(HWND)wParam); - convos.insert(std::move(cp)); - - ack_signals.signal(topic_atom, true); - }else{ - /* IF SOMETHING FUNDAMENTAL SEEMS BROKEN LOOK HERE ... - - The logic in here is a bit... tenuous. The DDE behavior - doesn't exactly follow the MSDN docs. We had to more-or-less - go trial-and-error to see how to handle communication - with the server. Whether or not this relies on impl details - is left to be seen. - */ - UnpackDDElParam(message,lParam, (PUINT_PTR)&plo, (PUINT_PTR)&pho); - GlobalGetAtomName((ATOM)(pho), item_atom, (TOSDB_MAX_STR_SZ + 1)); - - TOSDB_LogDebug( ("lParam > 0 - plo: " + std::to_string(plo) + ", pho: " + - std::to_string(pho) + ", item_atom: " + - std::string(item_atom)).c_str() ); - - if(plo == 0x0000){ - std::string sarg(std::to_string((size_t)(HWND)wParam)); - ack_signals.signal(sarg + std::string(item_atom), false); - TOSDB_LogH("DDE", ("NEG ACK for item: " + std::string(item_atom)).c_str()); - }else if(plo == 0x8000){ - std::string sarg(std::to_string((size_t)(HWND)wParam)); - ack_signals.signal(sarg + std::string(item_atom), true); - } - } - break; - } - default: - return DefWindowProc(hWnd, message, wParam, lParam); - } - - return 0; -} - - -template -class DDE_Data { - static const system_clock_type::time_point EPOCH_TP; - - DDE_Data(const DDE_Data&); - DDE_Data& operator=(const DDE_Data&); - - void - _init_datetime(); - -public: - TOS_Topics::TOPICS topic; - pDateTimeStamp time; - bool valid_datetime; - std::string item; - T data; - - DDE_Data(const TOS_Topics::TOPICS topic, - std::string item, - const T d, - bool datetime) - : - topic(topic), - item(item), - data(d), - time(new DateTimeStamp), - valid_datetime(datetime) - { - if(datetime) - _init_datetime(); - } - - ~DDE_Data() - { - if(time) - delete time; - } - - DDE_Data(DDE_Data&& d) - : - topic(d.topic), - item(d.item), - data(d.data), - time(d.time), - valid_datetime(d.valid_datetime) - { - d.time = nullptr; - } - - DDE_Data& - operator=(DDE_Data&& d) - { - topic = d.topic; - item = d.item; - data = d.data; - valid_datetime = d.valid_datetime; - time = d.time; - d.time = nullptr; - - return *this; - } -}; - -template -const system_clock_type::time_point -DDE_Data::EPOCH_TP; - -template -void -DDE_Data::_init_datetime() -{ - system_clock_type::time_point now; - std::chrono::seconds sec; - micro_sec_type ms; - time_t t; - - /* current timepoint*/ - now = system_clock.now(); - /* number of ms since epoch */ - ms = std::chrono::duration_cast(now - EPOCH_TP); - /* this is necessary to avoid issues w/ conversions to C time */ - sec = std::chrono::duration_cast< std::chrono::seconds >(ms); - /* ms since last second */ - time->micro_second = (long)((ms % micro_sec_type::period::den).count()); - /* get the ctime by adjusting epoch by seconds since */ - t = system_clock.to_time_t(EPOCH_TP + sec); - localtime_s(&time->ctime_struct, &t); -} - - -void -HandleData(UINT msg, WPARAM wparam, LPARAM lparam) -{ - DDEDATA FAR *dde_data; - BOOL clnt_rel; - PVOID data; - UINT_PTR atom; - LPARAM lpneg; - int cpret; - - char cp_data[TOSDB_STR_DATA_SZ+1]; /* include CR LF, exclude added \0 */ - char item_atom[TOSDB_MAX_STR_SZ + 1]; - - UnpackDDElParam(msg, lparam, (PUINT_PTR)&data, &atom); - - dde_data = (DDEDATA FAR*)GlobalLock(data); - /* if we can't lock the data or its not expected frmt */ - if(!dde_data || dde_data->cfFormat != CF_TEXT) - { - /* SEND NEG ACK TO SERVER - convo already destroyed if NULL */ - lpneg = PackDDElParam(WM_DDE_ACK, 0, (UINT_PTR)atom); - PostMessage((HWND)wparam, WM_DDE_ACK, (WPARAM)msg_window, lpneg); - - /* free resources */ - GlobalDeleteAtom((WORD)atom); - FreeDDElParam(WM_DDE_ACK, lpneg); - return; - } - - cpret = strncpy_s(cp_data, (LPCSTR)(dde_data->Value), TOSDB_STR_DATA_SZ); - if(cpret) - TOSDB_LogH("DDE", "error copying data->Value string"); - - if(dde_data->fAckReq){ - /* SEND POS ACK TO SERVER - convo already destroyed if NULL */ - PostMessage((HWND)wparam, WM_DDE_ACK, (WPARAM)msg_window, - PackDDElParam(WM_DDE_ACK, 0x8000, (UINT_PTR)atom)); - } - - /* extract before we unlock and free */ - GlobalGetAtomName((WORD)atom, item_atom, TOSDB_MAX_STR_SZ + 1); - clnt_rel = dde_data->fRelease; - GlobalUnlock(data); - - if(clnt_rel) - GlobalFree(data); - - GlobalDeleteAtom((WORD)atom); - - /* need to free lParam, as well, or we leak (not documented well on MSDN) */ - FreeDDElParam(WM_DDE_DATA, lparam); - - /* extract the topic from wparam */ - TOS_Topics::TOPICS topic_t = convos[(HWND)(wparam)]; - - try{ - std::string str(cp_data); - - switch(TOS_Topics::TypeBits(topic_t)){ - case TOSDB_STRING_BIT : /* STRING */ - { - /* clean up problem chars */ - auto r = std::remove_if(str.begin(), str.end(), [](char c){return c < 32;}); - str.erase(r, str.end()); - RouteToBuffer( DDE_Data(topic_t, item_atom, str, true) ); - break; - } - case TOSDB_INTGR_BIT : /* LONG */ - { - /* remove commas */ - auto r = std::remove_if(str.begin(), str.end(), [](char c){return std::isdigit(c) == 0;}); - str.erase(r,str.end()); - RouteToBuffer( DDE_Data(topic_t, item_atom, std::stol(str), true) ); - break; - } - case TOSDB_QUAD_BIT : /* DOUBLE */ - { - RouteToBuffer( DDE_Data(topic_t, item_atom, std::stod(str), true) ); - break; - } - case TOSDB_INTGR_BIT | TOSDB_QUAD_BIT :/* LONG LONG */ - { - /* remove commas */ - auto r = std::remove_if(str.begin(), str.end(), [](char c){return std::isdigit(c) == 0;}); - str.erase(r,str.end()); - RouteToBuffer( DDE_Data(topic_t, item_atom, std::stoll(str), true) ); - break; - } - case 0 : /* FLOAT */ - { - RouteToBuffer( DDE_Data(topic_t, item_atom, std::stof(str), true) ); - break; - } - }; - - }catch(const std::out_of_range& e){ - TOSDB_LogH("DDE", e.what()); - - }catch(const std::invalid_argument&){ - /* Dec 20 2016 - comment out, cluttering log file */ - /* - std::string serr(e.what()); - TOSDB_Log("DDE", serr.append(" Value:: ").append(cp_data).c_str()); - */ - - }catch(const std::exception& e){ - TOSDB_LogH("DDE", e.what()); - throw TOSDB_DDE_Error(e, "error handling dde data"); - } - - return /* 0 */; -} - - -bool -SetSecurityPolicy() -{ - SID_NAME_USE dummy; - BOOL ret; /* int */ - - DWORD dom_sz = 128; - DWORD sid_sz = SECURITY_MAX_SID_SIZE; - - SmartBuffer dom_buf(dom_sz); - SmartBuffer sid(sid_sz); - - ret = LookupAccountName(NULL, "Everyone", sid.get(), &sid_sz, dom_buf.get(), &dom_sz, &dummy); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "LookupAccountName failed", GetLastError()); - return false; - } - - for(int i = 0; i < NSECURABLE; ++i){ - sec_attr[(Securable)i].nLength = sizeof(SECURITY_ATTRIBUTES); - sec_attr[(Securable)i].bInheritHandle = FALSE; - sec_attr[(Securable)i].lpSecurityDescriptor = &sec_desc[(Securable)i]; - - /* memcpy 'TRUE' is error */ - ret = memcpy_s( sids[(Securable)i].get(), SECURITY_MAX_SID_SIZE, - sid.get(), SECURITY_MAX_SID_SIZE ); - if(ret){ - TOSDB_LogEx("ENGINE-SEC", "memcpy_s(sid) failed", GetLastError()); - return false; - } - - ret = InitializeSecurityDescriptor(&sec_desc[(Securable)i], SECURITY_DESCRIPTOR_REVISION); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "InitializeSecurityDescriptor failed", GetLastError()); - return false; - } - - ret = SetSecurityDescriptorGroup(&sec_desc[(Securable)i], sids[(Securable)i].get(), FALSE); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "SetSecurityDescriptorGroup failed", GetLastError()); - return false; - } - - ret = InitializeAcl(acls[(Securable)i].get(), ACL_SIZE, ACL_REVISION); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "InitializeAcl failed", GetLastError()); - return false; - } - } - - /* add ACEs individually */ - - ret = AddAccessDeniedAce(acls[SHEM1].get(), ACL_REVISION, FILE_MAP_WRITE, sids[SHEM1].get()); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "AddAccessDeniedAce failed", GetLastError()); - return false; - } - - ret = AddAccessAllowedAce(acls[SHEM1].get(), ACL_REVISION, FILE_MAP_READ, sids[SHEM1].get()); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "AddAccessAllowedAce(SHEM1) failed", GetLastError()); - return false; - } - - ret = AddAccessAllowedAce(acls[MUTEX1].get(), ACL_REVISION, SYNCHRONIZE, sids[MUTEX1].get()); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "AddAccessAllowedAce(MUTEX1) failed", GetLastError()); - return false; - } - - for(int i = 0; i < NSECURABLE; ++i){ - ret = SetSecurityDescriptorDacl( &sec_desc[(Securable)i], TRUE, acls[(Securable)i].get(), FALSE); - if(!ret){ - TOSDB_LogEx("ENGINE-SEC", "SetSecurityDescriptorDacl failed", GetLastError()); - return false; - } - } - - return true; -} - - -void DumpBufferStatus() -{ - const size_t log_col_width[5] = { 30, 30, 10, 60, 16}; - - std::string time_now(SysTimeString()); - std::string lpath(TOSDB_LOG_PATH); - std::string name("buffer-status-"); - std::ofstream lout; - - auto f = [](char x){ return std::isalnum(x) == 0; }; - std::replace_if(time_now.begin(), time_now.end(), f, '-'); - - lpath.append(name).append(time_now).append(".log"); - lout.open(lpath, std::ios::out | std::ios::app); - - lout <<" --- TOPIC INFO --- " << std::endl; - lout << std::setw(log_col_width[0]) << std::left << "Topic" - << std::setw(log_col_width[1]) << std::left << "Item" - << std::setw(log_col_width[2]) << std::left << "Ref-Count" - << std::endl; - - for(const auto & t : topic_refcounts){ - for(const auto & i : t.second){ - lout << std::setw(log_col_width[0]) << std::left - << TOS_Topics::map[t.first] - << std::setw(log_col_width[1]) << std::left << i.first - << std::setw(log_col_width[2]) << std::left << i.second - << std::endl; - } - } - - lout <<" --- BUFFER INFO --- " << std::endl; - lout << std::setw(log_col_width[3])<< std::left << "BufferName" - << std::setw(log_col_width[4])<< std::left << "Handle" << std::endl; - - { - BUFFER_LOCK_GUARD; - /* --- CRITICAL SECTION --- */ - for(const auto & b : buffers){ - lout << std::setw(log_col_width[3]) << std::left - << CreateBufferName(TOS_Topics::map[b.first.second], b.first.first) - << std::setw(log_col_width[4]) << std::left - << (size_t)b.second.hfile << std::endl; - } - /* --- CRITICAL SECTION --- */ - } - - lout<< " --- END END END --- "< - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include -#include //dbg -#include -#include -#include -#include - -#include "tos_databridge.h" -#include "ipc.hpp" -#include "concurrency.hpp" -#include "initializer_chain.hpp" - -namespace { - -const unsigned int COMM_BUFFER_SIZE = 4; -const unsigned int ACL_SIZE = 96; -const unsigned int UPDATE_PERIOD = 2000; -const unsigned int MAX_ARG_SIZE = 20; - -const std::map STATE_STRINGS = - InitializerChain> - (SERVICE_START_PENDING, "SERVICE_START_PENDING") - (SERVICE_STOP_PENDING, "SERVICE_STOP_PENDING") - (SERVICE_STOPPED, "SERVICE_STOPPED") - (SERVICE_RUNNING, "SERVICE_RUNNING") - (SERVICE_CONTINUE_PENDING, "SERVICE_CONTINUE_PENDING") - (SERVICE_PAUSE_PENDING, "SERVICE_PAUSE_PENDING") - (SERVICE_PAUSED, "SERVICE_PAUSED"); - -LPSTR SERVICE_NAME = "TOSDataBridge"; -LPCSTR ENGINE_BASE_NAME = "tos-databridge-engine"; - -#ifdef LOG_BACKEND_USE_SINGLE_FILE -LPCSTR LOG_NAME = LOG_BACKEND_SINGLE_FILE_NAME; -#else -LPCSTR LOG_NAME = "service-log.log"; -#endif - -#ifdef REDIRECT_STDERR_TO_LOG -LPCSTR ERR_LOG_NAME = "service-stderr.log"; -#endif - -std::string engine_path; -std::string integrity_level; - -std::unique_ptr master; - -PROCESS_INFORMATION engine_pinfo; -SYSTEM_INFO sys_info; -SERVICE_STATUS service_status; -SERVICE_STATUS_HANDLE service_status_hndl; - -HINSTANCE hinstance = NULL; - -volatile bool shutdown_flag = false; -volatile bool pause_flag = false; -volatile bool is_service = true; - -int custom_session = -1; - - -void -LogState(int state, std::string msg="") -{ - if(state == -1) - return; - - try{ - TOSDB_Log("STATE", (STATE_STRINGS.at(state) + ' ' + msg).c_str()); - }catch(const std::out_of_range){ - std::string msg = "invalid state (" + std::to_string(state) + ") passed to LogState"; - TOSDB_LogH("LOG", msg.c_str()); - } -} - -void -UpdateStatus(int status, int check_point) -{ - BOOL ret; - - if(check_point < 0) - ++service_status.dwCheckPoint; - else - service_status.dwCheckPoint = check_point; - - if(status >= 0) - service_status.dwCurrentState = status; - - ret = SetServiceStatus(service_status_hndl, &service_status); - if(!ret){ - TOSDB_LogH("ADMIN","error setting service status"); - service_status.dwCurrentState = SERVICE_STOPPED; - service_status.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - service_status.dwServiceSpecificExitCode = 2; - ++service_status.dwCheckPoint; - - TOSDB_LogH("SHUTDOWN","UpdateStatus terminating engine"); - shutdown_flag = true; - TerminateProcess(engine_pinfo.hProcess, EXIT_FAILURE); - UpdateStatus(SERVICE_STOPPED, -1); - } - - LogState(status); -} - - -bool -SendMsgWaitForResponse(long msg) -{ - long ret; - - std::string ipc_msg = std::to_string(msg); - - TOSDB_LogDebug("***IPC*** SERVICE - CHECK CONNECTED"); - if( !master->connected(TOSDB_DEF_TIMEOUT) ){ - TOSDB_LogH("IPC", "Service's IPCMaster is not connected"); - return false; - } - - TOSDB_LogDebug("***IPC*** SERVICE - CALL"); - if( !master->call(&ipc_msg, TOSDB_DEF_TIMEOUT) ){ - TOSDB_LogH("IPC",("master.call failed in SendMsgWaitForResponse, msg:" + ipc_msg).c_str()); - return false; - } - - try{ - ret = std::stol(ipc_msg); - return (ret == TOSDB_SIG_GOOD); - }catch(...){ - TOSDB_LogH("IPC", "failed to convert return message to long in SendMsgWaitForResponse"); - return false; - } -} - - -VOID WINAPI -ServiceController(DWORD cntrl) -{ - switch(cntrl){ - case SERVICE_CONTROL_SHUTDOWN: - case SERVICE_CONTROL_STOP: - - shutdown_flag = true; - UpdateStatus(SERVICE_STOP_PENDING, -1); - - if(pause_flag){ - /* if we're paused... get it to continue silently */ - if(!SendMsgWaitForResponse(TOSDB_SIG_CONTINUE)) - TOSDB_Log("ADMIN", "error resuming paused service stop it"); - } - - if(!SendMsgWaitForResponse(TOSDB_SIG_STOP)){ - TOSDB_Log("ADMIN","failed to send stop signal to engine"); - TerminateProcess(engine_pinfo.hProcess, EXIT_FAILURE); - } - - break; - - case SERVICE_CONTROL_PAUSE: - - if(pause_flag) - break; - - UpdateStatus(SERVICE_PAUSE_PENDING, -1); - - if(SendMsgWaitForResponse(TOSDB_SIG_PAUSE)){ - UpdateStatus(SERVICE_PAUSED, -1); - pause_flag = true; - }else{ - TOSDB_LogH("ADMIN", "failed to send pause signal to engine"); - } - - break; - - case SERVICE_CONTROL_CONTINUE: - - if(!pause_flag) - break; - - UpdateStatus(SERVICE_CONTINUE_PENDING, -1); - - if(SendMsgWaitForResponse(TOSDB_SIG_CONTINUE)){ - UpdateStatus(SERVICE_RUNNING, -1); - pause_flag = false; - }else{ - TOSDB_Log("ADMIN","failed to send continue signal to engine"); - } - - break; - } -} - - -#define SPAWN_ERROR_CHECK(ret, msg) do{ \ - if(!ret){ \ - TOSDB_LogEx("SPAWN", msg, GetLastError()); \ - goto cleanup_and_exit; \ - } \ -}while(0) - -bool -SpawnRestrictedProcess(std::string engine_cmd, int session = -1) -{ - TOKEN_MANDATORY_LABEL tml; - STARTUPINFO startup_info; - SID_NAME_USE dummy; - PTOKEN_PRIVILEGES tpriv; - LUID cg_id; - BOOL ret; - - HANDLE tkn_hndl = NULL; - HANDLE ctkn_hndl = NULL; - DWORD tpriv_len = 0; - DWORD session_id = 0; - DWORD sid_sz = SECURITY_MAX_SID_SIZE; - DWORD dom_sz = 256; - bool success = false; - - SmartBuffer sid_buf(sid_sz); - SmartBuffer dom_buf(dom_sz); - SmartBuffer cmd_buf(engine_path.size() + engine_cmd.size() + 3); - SmartBuffer tpriv_buf; - - ret = OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS, &tkn_hndl); - SPAWN_ERROR_CHECK(ret,"(1) failed to open token handle"); - - ret = DuplicateTokenEx(tkn_hndl, 0, NULL, SecurityImpersonation, TokenPrimary, &ctkn_hndl); - SPAWN_ERROR_CHECK(ret,"(2) failed to duplicate token"); - - /* try to drop our integrity level from System */ - ret = LookupAccountName(NULL, integrity_level.c_str(), sid_buf.get(), - &sid_sz, dom_buf.get(), &dom_sz, &dummy); - SPAWN_ERROR_CHECK(ret,"(3a) failed to lookup Mandatory Level group"); - - tml.Label.Attributes = SE_GROUP_INTEGRITY; - tml.Label.Sid = sid_buf.get(); - - ret = SetTokenInformation(ctkn_hndl, TokenIntegrityLevel, &tml, - sizeof(TOKEN_MANDATORY_LABEL) + GetLengthSid(sid_buf.get())); - SPAWN_ERROR_CHECK(ret,"(3b) failed to set Integrity Level"); - - session_id = (session < 0) ? WTSGetActiveConsoleSessionId() : session; - - /* try to get out of Session 0 isolation if we need to */ - if(is_service){ - std::string sess_id_msg = std::string("(4) set session id: ") - + std::to_string(session_id) - + std::string(" (this may may fail during (6) 'CreateProcessAsUser')"); - TOSDB_Log("SPAWN", sess_id_msg.c_str()); - - ret = SetTokenInformation(ctkn_hndl,TokenSessionId,&session_id,sizeof(DWORD)); - SPAWN_ERROR_CHECK(ret,"(4) failed to set session ID"); - } - - /* LUID of the privilege we need to retain for creating global kernel objects*/ - ret = LookupPrivilegeValue(NULL, SE_CREATE_GLOBAL_NAME, &cg_id); - SPAWN_ERROR_CHECK(ret,"(5a) failed to lookup privilege value: SE_CREATE_GLOBAL_NAME"); - - /* get size needed for privileges buffer - NO ERROR CHECK (this fails on success)*/ - GetTokenInformation(ctkn_hndl, TokenPrivileges, NULL, 0, &tpriv_len); - - /* fill buffer with privileges info */ - tpriv_buf = SmartBuffer(tpriv_len); - ret = GetTokenInformation(ctkn_hndl, TokenPrivileges, tpriv_buf.get(), tpriv_len, &tpriv_len); - SPAWN_ERROR_CHECK(ret,"(5b) failed to get TokenPrivileges information"); - - tpriv = tpriv_buf.get(); - for(DWORD i = 0; i < tpriv->PrivilegeCount; ++i){ - /* set the removed attribute on all but the one(s) to retain */ - if( tpriv->Privileges[i].Luid.LowPart != cg_id.LowPart - || tpriv->Privileges[i].Luid.HighPart != cg_id.HighPart) - { - tpriv->Privileges[i].Attributes = SE_PRIVILEGE_REMOVED; - } - } - - /* remove the privileges from the token */ - ret = AdjustTokenPrivileges(ctkn_hndl, FALSE, tpriv_buf.get(),0, NULL, NULL); - SPAWN_ERROR_CHECK(ret,"(5c) failed to adjust token privileges"); - - GetStartupInfo(&startup_info); - - /*prepend path to command string*/ - strcpy(cmd_buf.get(), (engine_path + " " + engine_cmd).c_str()); - - /* try to create the process with the new token */ - ret = CreateProcessAsUser(ctkn_hndl, engine_path.c_str(), cmd_buf.get(), NULL, NULL, - FALSE, 0, NULL, NULL, &startup_info, &engine_pinfo); - SPAWN_ERROR_CHECK(ret,"(6) failed to create engine process with new token"); - - success = true; - - cleanup_and_exit: - if(tkn_hndl) - CloseHandle(tkn_hndl); - if(ctkn_hndl) - CloseHandle(ctkn_hndl); - - return success; -} - -#undef SPAWN_ERROR_CHECK - -}; - - -void WINAPI -ServiceMain(DWORD argc, LPSTR argv[]) -{ - bool good_engine; - - service_status.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - service_status.dwCurrentState = SERVICE_START_PENDING; - service_status.dwControlsAccepted = SERVICE_ACCEPT_STOP - | SERVICE_ACCEPT_PAUSE_CONTINUE - | SERVICE_ACCEPT_SHUTDOWN; - service_status.dwWin32ExitCode = NO_ERROR; - service_status.dwServiceSpecificExitCode = 0; - service_status.dwCheckPoint = 0; - service_status.dwWaitHint = (DWORD)(2.5 * UPDATE_PERIOD); - - service_status_hndl = RegisterServiceCtrlHandler(SERVICE_NAME, ServiceController); - if(!service_status_hndl){ - TOSDB_LogH("ADMIN","failed to register control handler, exiting"); - service_status.dwCurrentState = SERVICE_STOPPED; - service_status.dwWin32ExitCode = ERROR_SERVICE_SPECIFIC_ERROR; - service_status.dwServiceSpecificExitCode = 1; - UpdateStatus(SERVICE_STOPPED, -1); - return; - } - TOSDB_Log("ADMIN","successfully registered control handler"); - - SetServiceStatus(service_status_hndl, &service_status); - LogState(SERVICE_START_PENDING); - UpdateStatus(-1, -1); - - TOSDB_Log("STARTUP", "start tos-databridge-engine.exe (AS A SERVICE)"); - good_engine = SpawnRestrictedProcess("--spawned --service", custom_session); - if(!good_engine){ - TOSDB_LogH("STARTUP", ("failed to spawn " + engine_path + " --spawned --service").c_str()); - UpdateStatus(SERVICE_STOPPED, -1); - return; - } - - /* create 'master' to communicate with engine */ - master = std::unique_ptr( new IPCMaster(TOSDB_COMM_CHANNEL) ); - - UpdateStatus(SERVICE_RUNNING, -1); - - /* MAIN UPDATE LOOP */ - do{ - /* first check if engine is running */ - if( WaitForSingleObject(engine_pinfo.hProcess, 0) != WAIT_TIMEOUT){ - TOSDB_LogH("SHUTDOWN", "service believes engine closed unexpectedly"); - shutdown_flag = true; - break; - } - Sleep(UPDATE_PERIOD); - UpdateStatus(-1, -1); - }while(!shutdown_flag); - - /* wait on engine process to exit (if necessary), or timeout and force exit */ - if( WaitForSingleObject(engine_pinfo.hProcess, TOSDB_DEF_TIMEOUT * 2) == WAIT_TIMEOUT){ - /* forcefully close the engine */ - TOSDB_LogH("SHUTDOWN", "engine took too long to shutdown, terminated"); - TerminateProcess(engine_pinfo.hProcess, EXIT_FAILURE); - } - - TOSDB_LogH("SHUTDOWN", "service believes engine has shutdown"); - - /*** IF WE GET HERE WE WILL SHUTDOWN ***/ - UpdateStatus(SERVICE_STOPPED, 0); - /*** NO GUARANTEE ANYTHING AFTER THIS WILL STILL BE VALID ***/ -} - - -int WINAPI -WinMain(HINSTANCE hInst, HINSTANCE hPrevInst, LPSTR lpCmdLn, int nShowCmd) -{ - bool good_engine; - std::vector args; - - std::string cmd_str(lpCmdLn); - std::string logpath(TOSDB_LOG_PATH); - -#ifdef REDIRECT_STDERR_TO_LOG - freopen( (logpath + ERR_LOG_NAME).c_str(), "a", stderr); -#endif - - /* start logging */ - logpath.append(LOG_NAME); - StartLogging(logpath.c_str()); - - /* get 'our' module name */ - SmartBuffer module_buf(MAX_PATH); - GetModuleFileName(NULL, module_buf.get(), MAX_PATH); - - /* add the appropriate engine name to the module path */ - std::string path(module_buf.get()); - std::string serv_ext(path.begin() + path.find_last_of("-"), path.end()); - -#ifdef _DEBUG - if(serv_ext != "-x86_d.exe" && serv_ext != "-x64_d.exe"){ -#else - if(serv_ext != "-x86.exe" && serv_ext != "-x64.exe"){ -#endif - TOSDB_LogH("STARTUP", "service path doesn't provide proper extension"); - return 1; - } - - path.erase(path.find_last_of("\\")); - engine_path = path; - engine_path.append("\\").append(ENGINE_BASE_NAME).append(serv_ext); - - GetSystemInfo(&sys_info); - - ParseArgs(args,cmd_str); - - size_t argc = args.size(); - int admin_pos = 0; - int no_service_pos = 0; - - /* look for --admin and/or --noservice args */ - if(argc > 0 && args[0] == "--admin") - admin_pos = 1; - else if(argc > 1 && args[1] == "--admin") - admin_pos = 2; - - if(argc > 0 && args[0] == "--noservice") - no_service_pos = 1; - else if(argc > 1 && args[1] == "--noservice") - no_service_pos = 2; - - switch(argc){ /* look for custom_session arg */ - case 0: - break; - case 1: - if(admin_pos == 0 && no_service_pos == 0) - custom_session = std::stoi(args[0]); - break; - case 2: - if( (admin_pos == 1 && no_service_pos != 2) - || (no_service_pos == 1 && admin_pos != 2)) - { - custom_session = std::stoi(args[1]); - } - break; - case 3: - if(admin_pos > 0 && no_service_pos > 0) - custom_session = std::stoi(args[2]); - break; - default: - std::string serr("invalid # of args: "); - serr.append(std::to_string(argc)); - TOSDB_LogH("STARTUP",serr.c_str()); - return 1; - } - - std::stringstream ss_args; - ss_args << "argc: " << std::to_string(argc) - << " custom_session: " << std::to_string(custom_session) - << " admin_pos: " << std::to_string(admin_pos) - << " no_service_pos: " << std::to_string(no_service_pos); - - TOSDB_Log("STARTUP", std::string("lpCmdLn: ").append(cmd_str).c_str() ); - TOSDB_Log("STARTUP", ss_args.str().c_str() ); - - integrity_level = admin_pos > 0 ? "High Mandatory Level" : "Medium Mandatory Level"; - - /* populate the engine command and if --noservice is passed jump right into - the engine via SpawnRestrictedProcess; otherwise Start the service which - will handle that for us */ - - if(no_service_pos > 0) - { - is_service = false; - TOSDB_Log("STARTUP", "starting tos-databridge-engine.exe directly(NOT A SERVICE)"); - /* prepend '--spawned' so engine knows *we* called it; - if someone else passes '--spawned' they deserve what they get */ - good_engine = SpawnRestrictedProcess("--spawned --noservice", custom_session); - if(good_engine){ - TOSDB_Log("STARTUP", ("SUCCESS spawning: " + engine_path + " --spawned --noservice").c_str()); - }else{ - TOSDB_LogH("STARTUP", ("FAILURE spawning: " + engine_path + " --spawned --noservice").c_str()); - } - } - else - { - SERVICE_TABLE_ENTRY dtable[] = {{SERVICE_NAME,ServiceMain}, {NULL,NULL}}; - /* BEGIN SERVICE */ - if( !StartServiceCtrlDispatcher(dtable) ) - { - TOSDB_LogH("STARTUP", "StartServiceCtrlDispatcher() failed. " - "(Be sure to use an appropriate Window's tool to start the service " - "(e.g SC.exe, Services.msc) or pass '--noservice' to run directly.)"); - } - /* END SERVICE */ - } - - TOSDB_Log("SHUTDOWN","service exiting"); - StopLogging(); - -#ifdef REDIRECT_STDERR_TO_LOG - fclose(stderr); -#endif - - return 0; -} - diff --git a/src/shell/commands_admin.cpp b/src/shell/commands_admin.cpp deleted file mode 100644 index a5f550b..0000000 --- a/src/shell/commands_admin.cpp +++ /dev/null @@ -1,1047 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "shell.hpp" - -namespace{ - -void Connect(CommandCtx *ctx); -void Disconnect(CommandCtx *ctx); -void IsConnected(CommandCtx *ctx); -void IsConnectedToEngine(CommandCtx *ctx); -void IsConnectedToEngineAndTOS(CommandCtx *ctx); -void ConnectionState(CommandCtx *ctx); -void CreateBlock(CommandCtx *ctx); -void CloseBlock(CommandCtx *ctx); -void CloseBlocks(CommandCtx *ctx); -void GetBlockLimit(CommandCtx *ctx); -void SetBlockLimit(CommandCtx *ctx); -void GetBlockCount(CommandCtx *ctx); -void GetBlockIDs(CommandCtx *ctx); -void GetBlockSize(CommandCtx *ctx); -void SetBlockSize(CommandCtx *ctx); -void GetLatency(CommandCtx *ctx); -void SetLatency(CommandCtx *ctx); -void Add(CommandCtx *ctx); -void AddTopic(CommandCtx *ctx); -void AddItem(CommandCtx *ctx); -void AddTopics(CommandCtx *ctx); -void AddItems(CommandCtx *ctx); -void RemoveTopic(CommandCtx *ctx); -void RemoveItem(CommandCtx *ctx); -void GetItemCount(CommandCtx *ctx); -void GetTopicCount(CommandCtx *ctx); -void GetTopicNames(CommandCtx *ctx); -void GetItemNames(CommandCtx *ctx); -void GetTopicEnums(CommandCtx *ctx); -void GetPreCachedTopicEnums(CommandCtx *ctx); -void GetPreCachedItemCount(CommandCtx *ctx); -void GetPreCachedTopicCount(CommandCtx *ctx); -void GetPreCachedItemNames(CommandCtx *ctx); -void GetPreCachedTopicNames(CommandCtx *ctx); -void GetTypeBits(CommandCtx *ctx); -void GetTypeString(CommandCtx *ctx); -void IsUsingDateTime(CommandCtx *ctx); -void GetStreamOccupancy(CommandCtx *ctx); -void GetMarkerPosition(CommandCtx *ctx); -void IsMarkerDirty(CommandCtx *ctx); -void DumpBufferStatus(CommandCtx *ctx); -void RemoveOrphanedStream(CommandCtx *ctx); - -}; /* namespace */ - -CommandsMap commands_admin( - CommandsMap::InitChain("Connect",Connect,"connect to the library") - ("Disconnect",Disconnect) - ("IsConnected",IsConnected) - ("IsConnectedToEngine",IsConnectedToEngine) - ("IsConnectedToEngineAndTOS",IsConnectedToEngineAndTOS) - ("ConnectionState",ConnectionState) - ("CreateBlock",CreateBlock) - ("CloseBlock",CloseBlock) - ("CloseBlocks",CloseBlocks) - ("GetBlockLimit",GetBlockLimit) - ("SetBlockLimit",SetBlockLimit) - ("GetBlockCount",GetBlockCount) - ("GetBlockIDs",GetBlockIDs) - ("GetBlockSize",GetBlockSize) - ("SetBlockSize",SetBlockSize) - ("GetLatency",GetLatency) - ("SetLatency",SetLatency) - ("Add",Add) - ("AddTopic",AddTopic) - ("AddItem",AddItem) - ("AddTopics",AddTopics) - ("AddItems",AddItems) - ("RemoveTopic",RemoveTopic) - ("RemoveItem",RemoveItem) - ("GetItemCount",GetItemCount) - ("GetTopicCount",GetTopicCount) - ("GetTopicNames",GetTopicNames) - ("GetItemNames",GetItemNames) - ("GetTopicEnums",GetTopicEnums) - ("GetPreCachedTopicEnums",GetPreCachedTopicEnums) - ("GetPreCachedItemCount",GetPreCachedItemCount) - ("GetPreCachedTopicCount",GetPreCachedTopicCount) - ("GetPreCachedItemNames",GetPreCachedItemNames) - ("GetPreCachedTopicNames",GetPreCachedTopicNames) - ("GetTypeBits",GetTypeBits) - ("GetTypeString",GetTypeString) - ("IsUsingDateTime",IsUsingDateTime) - ("GetStreamOccupancy",GetStreamOccupancy) - ("GetMarkerPosition",GetMarkerPosition) - ("IsMarkerDirty",IsMarkerDirty) - ("DumpBufferStatus",DumpBufferStatus) - ("RemoveOrphanedStream", RemoveOrphanedStream) -); - - -namespace { - -void -_check_display_ret(int r); - -template -void -_check_display_ret(int r, T v); - -template<> -void -_check_display_ret(int r, bool v); - -template -void -_check_display_ret(int r, T *v, size_type n); - - -inline size_type -_get_cstr_items(char ***p, CommandCtx *ctx) -{ - return get_cstr_entries("item", p, ctx); -} - - -inline size_type -_get_cstr_topics(char ***p, CommandCtx *ctx) -{ - return get_cstr_entries("topic", p, ctx); -} - - -inline void -_del_cstr_items(char **items, size_type nitems) -{ - DeleteStrings(items, nitems); -} - - -inline void -_del_cstr_topics(char **topics, size_type ntopics) -{ - DeleteStrings(topics, ntopics); -} - - -void -Connect(CommandCtx *ctx) -{ - int ret = TOSDB_Connect(); - _check_display_ret(ret); -} - - -void -Disconnect(CommandCtx *ctx) -{ - int ret = TOSDB_Disconnect(); - _check_display_ret(ret); -} - - -void -IsConnected(CommandCtx *ctx) -{ - unsigned int ret = TOSDB_IsConnected(); - std::cout<< std::endl << "*** DEPRECATED (jan 2017) ***" << std::endl - << std::endl << std::boolalpha << (ret == 1) << std::endl << std::endl; -} - - -void -IsConnectedToEngine(CommandCtx *ctx) -{ - unsigned int ret = TOSDB_IsConnectedToEngine(); - std::cout<< std::endl << std::boolalpha << (ret == 1) << std::endl << std::endl; -} - - -void -IsConnectedToEngineAndTOS(CommandCtx *ctx) -{ - unsigned int ret = TOSDB_IsConnectedToEngineAndTOS(); - std::cout<< std::endl << std::boolalpha << (ret == 1) << std::endl << std::endl; -} - - -void -ConnectionState(CommandCtx *ctx) -{ - unsigned int ret = TOSDB_ConnectionState(); - - std::cout<< std::endl; - - switch(ret){ - case TOSDB_CONN_NONE: - std::cout<< "TOSDB_CONN_NONE"; - break; - case TOSDB_CONN_ENGINE: - std::cout<< "TOSDB_CONN_ENGINE"; - break; - case TOSDB_CONN_ENGINE_TOS: - std::cout<< "TOSDB_CONN_ENGINE_TOS"; - break; - default: - std::cout<< "Invalid connection state returned."; - break; - } - - std::cout<< std::endl << std::endl; -} - - -void -CreateBlock(CommandCtx *ctx) -{ - std::string timeout; - std::string block; - std::string size; - std::string dts_y_or_n; - unsigned long timeout_num; - int ret; - - prompt_for("block id", &block, ctx); - prompt_for("block size", &size, ctx); - prompt_for("use datetime stamp?(y/n)", &dts_y_or_n, ctx); - - if(dts_y_or_n != "y" && dts_y_or_n != "n") - std::cerr<< std::endl << "INVALID - default to 'n'" << std::endl << std::endl; - - prompt_for("timeout(milliseconds) [positive number or defaults]", &timeout, ctx); - - try{ - timeout_num = std::stoul(timeout); - if(timeout_num <= 0) - throw std::exception("dummy"); - - if(timeout_num < TOSDB_MIN_TIMEOUT){ - std::cout<< std::endl << "WARNING - using a timeout < " << TOSDB_MIN_TIMEOUT - << " milliseconds is NOT recommended" << std::endl; - } - }catch(...){ - timeout_num = TOSDB_DEF_TIMEOUT; - std::cout<< std::endl << "Using default timeout of " << timeout_num - << " milliseconds" << std::endl; - } - - ret = TOSDB_CreateBlock(block.c_str(), std::stoul(size) , (dts_y_or_n == "y"), timeout_num); - _check_display_ret(ret); -} - - -void -CloseBlock(CommandCtx *ctx) -{ - int ret; - std::string block; - - prompt_for("block id", &block, ctx); - - ret = TOSDB_CloseBlock(block.c_str()); - _check_display_ret(ret); -} - - -void -CloseBlocks(CommandCtx *ctx) -{ - int ret = TOSDB_CloseBlocks(); - _check_display_ret(ret); -} - - -void -GetBlockLimit(CommandCtx *ctx) -{ - std::cout<< std::endl << TOSDB_GetBlockLimit() << std::endl << std::endl; -} - - -void -SetBlockLimit(CommandCtx *ctx) -{ - std::string size; - - prompt_for("block limit", &size, ctx); - - std::cout<< std::endl << TOSDB_SetBlockLimit(std::stoul(size)) << std::endl << std::endl; -} - -void -GetBlockCount(CommandCtx *ctx) -{ - std::cout<< std::endl << TOSDB_GetBlockCount() << std::endl << std::endl; -} - - -void -GetBlockIDs(CommandCtx *ctx) -{ - if(prompt_for_cpp(ctx)){ - std::cout<< std::endl; - for(auto & str : TOSDB_GetBlockIDs()) - std::cout<< str << std::endl; - std::cout<< std::endl; - }else{ - char** strs; - size_type nblocks = TOSDB_GetBlockCount(); - try{ - int ret; - - strs = NewStrings(nblocks, TOSDB_BLOCK_ID_SZ); - - ret = TOSDB_GetBlockIDs(strs, nblocks, TOSDB_BLOCK_ID_SZ); - _check_display_ret(ret,strs,nblocks); - - DeleteStrings(strs, nblocks); - }catch(...){ - DeleteStrings(strs, nblocks); - throw; - } - } -} - - -void -GetBlockSize(CommandCtx *ctx) -{ - int ret; - std::string block; - - size_type sz = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - std::cout<< std::endl << TOSDB_GetBlockSize(block) << std::endl << std::endl; - }else{ - ret = TOSDB_GetBlockSize(block.c_str(), &sz); - _check_display_ret(ret, sz); - } -} - - -void -SetBlockSize(CommandCtx *ctx) -{ - int ret; - std::string block; - std::string size; - - prompt_for("block id", &block, ctx); - prompt_for("block size", &size, ctx); - - ret = TOSDB_SetBlockSize(block.c_str(), std::stoul(size)); - _check_display_ret(ret); -} - - -void -GetLatency(CommandCtx *ctx) -{ - std::cout<< std::endl << TOSDB_GetLatency() << std::endl << std::endl; -} - - -void -SetLatency(CommandCtx *ctx) -{ - std::string lat; - - prompt_for("latency value(milleseconds)", &lat, ctx); - - TOSDB_SetLatency((UpdateLatency)std::stoul(lat)); -} - - -void -Add(CommandCtx *ctx) -{ - std::string block; - size_type nitems; - size_type ntopics; - int ret; - - char **items_raw = nullptr; - char **topics_raw = nullptr; - - prompt_for("block id", &block, ctx); - - try{ - nitems = _get_cstr_items(&items_raw, ctx); - ntopics = _get_cstr_topics(&topics_raw, ctx); - - if(prompt_for_cpp(ctx)){ - auto topics = topic_set_type( - topics_raw, - ntopics, - [=](LPCSTR str){ return TOS_Topics::MAP()[str]; } - ); - ret = TOSDB_Add(block, str_set_type(items_raw, nitems), topics); - }else{ - ret = TOSDB_Add(block.c_str(), (const char**)items_raw, nitems, - (const char**)topics_raw, ntopics); - } - - _check_display_ret(ret); - - _del_cstr_items(items_raw, nitems); - _del_cstr_topics(topics_raw, ntopics); - }catch(...){ - _del_cstr_items(items_raw, nitems); - _del_cstr_topics(topics_raw, ntopics); - throw; - } -} - - -void -AddTopic(CommandCtx *ctx) -{ - std::string block; - std::string topic; - int ret; - - prompt_for("block id", &block, ctx); - prompt_for("topic", &topic, ctx); - - ret = prompt_for_cpp(ctx) - ? TOSDB_AddTopic(block, TOS_Topics::MAP()[topic]) - : TOSDB_AddTopic(block.c_str(), topic.c_str()); - - _check_display_ret(ret); -} - - -void -AddItem(CommandCtx *ctx) -{ - std::string block; - std::string item; - int ret; - - prompt_for("block id", &block, ctx); - prompt_for("item", &item, ctx); - - ret = prompt_for_cpp(ctx) - ? TOSDB_AddItem(block, item) - : TOSDB_AddItem(block.c_str(), item.c_str()); - - _check_display_ret(ret); -} - - -void -AddTopics(CommandCtx *ctx) -{ - std::string block; - size_type ntopics; - int ret; - - char **topics_raw = nullptr; - - prompt_for("block id", &block, ctx); - - try{ - ntopics = _get_cstr_topics(&topics_raw, ctx); - - if(prompt_for_cpp(ctx)){ - auto topics = topic_set_type( - topics_raw, - ntopics, - [=](LPCSTR str){ return TOS_Topics::MAP()[str]; } - ); - ret = TOSDB_AddTopics(block, topics); - }else{ - ret = TOSDB_AddTopics(block.c_str(), (const char**)topics_raw, ntopics); - } - _check_display_ret(ret); - - _del_cstr_topics(topics_raw, ntopics); - }catch(...){ - _del_cstr_topics(topics_raw, ntopics); - throw; - } -} - - -void -AddItems(CommandCtx *ctx) -{ - std::string block; - size_type nitems; - int ret; - - char **items_raw = nullptr; - - prompt_for("block id", &block, ctx); - - try{ - nitems = _get_cstr_items(&items_raw, ctx); - - ret = prompt_for_cpp(ctx) - ? TOSDB_AddItems(block, str_set_type(items_raw, nitems)) - : TOSDB_AddItems(block.c_str(), (const char**)items_raw, nitems); - - _check_display_ret(ret); - _del_cstr_items(items_raw, nitems); - }catch(...){ - _del_cstr_items(items_raw, nitems); - throw; - } -} - - -void -RemoveTopic(CommandCtx *ctx) -{ - std::string block; - std::string topic; - int ret; - - prompt_for("block id", &block, ctx); - prompt_for("topic", &topic, ctx); - - ret = prompt_for_cpp(ctx) - ? TOSDB_RemoveTopic(block, TOS_Topics::MAP()[topic]) - : TOSDB_RemoveTopic(block.c_str(), topic.c_str()); - - _check_display_ret(ret); -} - - -void -RemoveItem(CommandCtx *ctx) -{ - std::string block; - std::string item; - int ret; - - prompt_for("block id", &block, ctx); - prompt_for("item: ", &item, ctx); - - ret = prompt_for_cpp(ctx) - ? TOSDB_RemoveItem(block, item) - : TOSDB_RemoveItem(block.c_str(), item.c_str()); - - _check_display_ret(ret); -} - - -void -GetItemCount(CommandCtx *ctx) -{ - std::string block; - int ret; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl << TOSDB_GetItemCount(block) << std::endl << std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetItemCount(block.c_str(), &count); - _check_display_ret(ret, count); - } -} - - -void -GetTopicCount(CommandCtx *ctx) -{ - std::string block; - int ret; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl << TOSDB_GetTopicCount(block) << std::endl << std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetTopicCount(block.c_str(), &count); - _check_display_ret(ret, count); - } -} - - -void -GetTopicNames(CommandCtx *ctx) -{ - std::string block; - int ret; - char **strs; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl; - for(auto & t : TOSDB_GetTopicNames(block)) - std::cout<< t << std::endl; - std::cout<< std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - TOSDB_GetTopicCount(block.c_str(), &count); - try{ - strs = NewStrings(count, TOSDB_MAX_STR_SZ); - - ret = TOSDB_GetTopicNames(block.c_str(), strs, count, TOSDB_MAX_STR_SZ); - _check_display_ret(ret, strs, count); - - DeleteStrings(strs, count); - }catch(...){ - DeleteStrings(strs, count); - throw; - } - } -} - - -void -GetItemNames(CommandCtx *ctx) -{ - int ret; - char **strs; - std::string block; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl; - for(auto & i : TOSDB_GetItemNames(block)) - std::cout<< i << std::endl; - std::cout<< std::endl; - }catch(const std::exception& e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - TOSDB_GetItemCount(block.c_str(), &count); - try{ - strs = NewStrings(count, TOSDB_MAX_STR_SZ); - - ret = TOSDB_GetItemNames(block.c_str(), strs, count, TOSDB_MAX_STR_SZ); - _check_display_ret(ret, strs, count); - - DeleteStrings(strs, count); - }catch(...){ - DeleteStrings(strs, count); - throw; - } - } -} - - -void -GetTopicEnums(CommandCtx *ctx) -{ - std::string block; - - prompt_for("block id", &block, ctx); - - std::cout<< std::endl; - for(auto & t : TOSDB_GetTopicEnums(block)) - std::cout<< (TOS_Topics::enum_value_type)t <<' ' - << TOS_Topics::MAP()[t] << std::endl; - std::cout<< std::endl; -} - -void -GetPreCachedTopicNames(CommandCtx *ctx) -{ - char **strs; - int ret; - std::string block; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl; - for(auto & t : TOSDB_GetPreCachedTopicNames(block)) - std::cout<< t << std::endl; - std::cout<< std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - TOSDB_GetPreCachedTopicCount(block.c_str(), &count); - try{ - strs = NewStrings(count, TOSDB_MAX_STR_SZ); - - ret = TOSDB_GetPreCachedTopicNames(block.c_str(), strs, count, TOSDB_MAX_STR_SZ); - _check_display_ret(ret, strs, count); - - DeleteStrings(strs, count); - }catch(...){ - DeleteStrings(strs, count); - throw; - } - } -} - - -void -GetPreCachedItemNames(CommandCtx *ctx) -{ - char **strs; - int ret; - std::string block; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl; - for(auto & i : TOSDB_GetPreCachedItemNames(block)) - std::cout<< i << std::endl; - std::cout<< std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - TOSDB_GetPreCachedItemCount(block.c_str(), &count); - try{ - strs = NewStrings(count, TOSDB_MAX_STR_SZ); - - ret = TOSDB_GetPreCachedItemNames(block.c_str(), strs, count, TOSDB_MAX_STR_SZ); - _check_display_ret(ret, strs, count); - - DeleteStrings(strs, count); - }catch(...){ - DeleteStrings(strs, count); - throw; - } - } -} - -void -GetPreCachedTopicEnums(CommandCtx *ctx) -{ - std::string block; - - prompt_for("block id", &block, ctx); - - std::cout<< std::endl; - for(auto & t : TOSDB_GetPreCachedTopicEnums(block)) - std::cout<< (TOS_Topics::enum_value_type)t << ' ' - << TOS_Topics::MAP()[t] << std::endl; - std::cout<< std::endl; -} - - -void -GetPreCachedItemCount(CommandCtx *ctx) -{ - int ret; - std::string block; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl - << TOSDB_GetPreCachedItemCount(block) - << std::endl << std::endl; - }catch(const std::exception & e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetPreCachedItemCount(block.c_str(), &count); - _check_display_ret(ret, count); - } -} - - -void -GetPreCachedTopicCount(CommandCtx *ctx) -{ - int ret; - std::string block; - - size_type count = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl - << TOSDB_GetPreCachedTopicCount(block) - << std::endl << std::endl; - }catch(const std::exception& e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetPreCachedTopicCount(block.c_str(), &count); - _check_display_ret(ret, count); - } -} - - -void -GetTypeBits(CommandCtx *ctx) -{ - int ret; - std::string topic; - - type_bits_type bits = 0; - - prompt_for("topic", &topic, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl << std::hex - << (int)TOSDB_GetTypeBits(TOS_Topics::MAP()[topic]) - << std::endl << std::endl; - }catch(const std::exception& e){ - std::cout<< std::endl <<"error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetTypeBits(topic.c_str(), &bits); - if(ret) - std::cout<< std::endl << "error: "<< ret << std::endl << std::endl; - else - std::cout<< std::endl << std::hex << (int)bits << std::endl << std::endl; - } -} - - -void -GetTypeString(CommandCtx *ctx) -{ - int ret; - char tstring[256]; - std::string topic; - - prompt_for("topic", &topic, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl - << TOSDB_GetTypeString(TOS_Topics::MAP()[topic]) - << std::endl << std::endl; - }catch(const std::exception& e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_GetTypeString(topic.c_str(), tstring, 256); - _check_display_ret(ret, tstring); - } -} - - -void -IsUsingDateTime(CommandCtx *ctx) -{ - int ret; - std::string block; - - unsigned int using_dt = 0; - - prompt_for("block id", &block, ctx); - - if(prompt_for_cpp(ctx)){ - try{ - std::cout<< std::endl << std::boolalpha - << TOSDB_IsUsingDateTime(block) - << std::endl << std::endl; - }catch(const std::exception& e){ - std::cout<< std::endl << "error: " << e.what() << std::endl << std::endl; - } - }else{ - ret = TOSDB_IsUsingDateTime(block.c_str(), &using_dt); - _check_display_ret(ret, (using_dt==1)); - } -} - - -void -GetStreamOccupancy(CommandCtx *ctx) -{ - int ret; - std::string block; - std::string item; - std::string topic; - - size_type occ = 0; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - - if(prompt_for_cpp(ctx)){ - std::cout<< std::endl - << TOSDB_GetStreamOccupancy(block, item, TOS_Topics::MAP()[topic]) - << std::endl << std::endl; - }else{ - ret = TOSDB_GetStreamOccupancy(block.c_str(), item.c_str(), topic.c_str(), &occ); - _check_display_ret(ret, occ); - } -} - - -void -GetMarkerPosition(CommandCtx *ctx) -{ - int ret; - std::string block; - std::string item; - std::string topic; - - long long pos = 0; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - - if(prompt_for_cpp(ctx)){ - std::cout<< std::endl - << TOSDB_GetMarkerPosition(block, item, TOS_Topics::MAP()[topic]) - << std::endl << std::endl; - }else{ - ret = TOSDB_GetMarkerPosition(block.c_str(), item.c_str(), topic.c_str(), &pos); - _check_display_ret(ret, pos); - } -} - - -void -IsMarkerDirty(CommandCtx *ctx) -{ - int ret; - std::string block; - std::string item; - std::string topic; - - unsigned int is_dirty = false; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - - if(prompt_for_cpp(ctx)){ - std::cout<< std::endl << std::boolalpha - << TOSDB_IsMarkerDirty(block, item,TOS_Topics::MAP()[topic]) - << std::endl << std::endl; - }else{ - ret = TOSDB_IsMarkerDirty(block.c_str(), item.c_str(),topic.c_str(), &is_dirty); - _check_display_ret(ret, (is_dirty==1)); - } -} - - -void -DumpBufferStatus(CommandCtx *ctx) -{ - _check_display_ret( TOSDB_DumpSharedBufferStatus() ); -} - - -void -RemoveOrphanedStream(CommandCtx *ctx) -{ - std::string item; - std::string topic; - - prompt_for_item_topic(&item, &topic, ctx); - - _check_display_ret( TOSDB_RemoveOrphanedStream(item.c_str(), topic.c_str()) ); -} - - -template -void -_check_display_ret(int r, T v) -{ - if(r) - std::cout<< std::endl << "error: "<< r << std::endl << std::endl; - else - std::cout<< std::endl << v << std::endl << std::endl; -} - - -void -_check_display_ret(int r) -{ - if(r) - std::cout<< std::endl << "error: "<< r << std::endl << std::endl; - else - std::cout<< std::endl << "SUCCESS" << std::endl << std::endl; -} - - -template<> -void -_check_display_ret(int r, bool v) -{ - if(r) - std::cout<< std::endl << "error: "<< r << std::endl << std::endl; - else - std::cout<< std::endl << std::boolalpha << v << std::endl << std::endl; -} - - -template -void -_check_display_ret(int r, T *v, size_type n) -{ - if(r) - std::cout<< std::endl << "error: "<< r << std::endl << std::endl; - else{ - std::cout<< std::endl; - for(size_type i = 0; i < n; ++i) - std::cout<< v[i] << std::endl; - std::cout<< std::endl; - } -} - -}; /* namespace */ \ No newline at end of file diff --git a/src/shell/commands_frame.cpp b/src/shell/commands_frame.cpp deleted file mode 100644 index 9f16ff4..0000000 --- a/src/shell/commands_frame.cpp +++ /dev/null @@ -1,307 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - - -#include "shell.hpp" - -namespace{ - -void GetItemFrame(CommandCtx *ctx); -void GetItemFrameStrings(CommandCtx *ctx); -void GetItemFrameDoubles(CommandCtx *ctx); -void GetItemFrameFloats(CommandCtx *ctx); -void GetItemFrameLongLongs(CommandCtx *ctx); -void GetItemFrameLongs(CommandCtx *ctx); -void GetTopicFrame(CommandCtx *ctx); -void GetTopicFrameStrings(CommandCtx *ctx); -void GetTotalFrame(CommandCtx *ctx); - - -}; /* namespace */ - -CommandsMap commands_frame( - CommandsMap::InitChain("GetItemFrame",GetItemFrame) - ("GetItemFrameStrings",GetItemFrameStrings) - ("GetItemFrameDoubles",GetItemFrameDoubles) - ("GetItemFrameFloats",GetItemFrameFloats) - ("GetItemFrameLongLongs",GetItemFrameLongLongs) - ("GetItemFrameLongs",GetItemFrameLongs) - ("GetTopicFrame",GetTopicFrame) - ("GetTopicFrameStrings",GetTopicFrameStrings) - ("GetTotalFrame",GetTotalFrame) -); - - -namespace{ - -template -void -_get_item_frame( int(*func)(LPCSTR, LPCSTR, T*, size_type, - LPSTR*,size_type,pDateTimeStamp), - CommandCtx *ctx ); - -template -void -_check_display_ret(int r, T *v, char **l, size_type n, pDateTimeStamp d); - - -void -GetItemFrame(CommandCtx *ctx) -{ - std::string block; - std::string topic; - bool get_dts; - - prompt_for("block id", &block, ctx); - prompt_for("topic", &topic, ctx); - - get_dts = prompt_for_datetime(block, ctx); - - if(get_dts) - std::cout<< std::endl << TOSDB_GetItemFrame(block,TOS_Topics::MAP()[topic]); - else - std::cout<< std::endl << TOSDB_GetItemFrame(block,TOS_Topics::MAP()[topic]); - - std::cout<< std::endl; -} - - -void -GetItemFrameDoubles(CommandCtx *ctx) -{ - _get_item_frame(TOSDB_GetItemFrameDoubles, ctx); -} - - -void -GetItemFrameFloats(CommandCtx *ctx) -{ - _get_item_frame(TOSDB_GetItemFrameFloats, ctx); -} - - -void -GetItemFrameLongLongs(CommandCtx *ctx) -{ - _get_item_frame(TOSDB_GetItemFrameLongLongs, ctx); -} - - -void -GetItemFrameLongs(CommandCtx *ctx) -{ - _get_item_frame(TOSDB_GetItemFrameLongs, ctx); -} - - -void -GetItemFrameStrings(CommandCtx *ctx) -{ - size_type nitems; - std::string block; - std::string topic; - int ret; - bool get_dts; - - char **dat = nullptr; - char **lab = nullptr; - pDateTimeStamp dts = nullptr; - - prompt_for("block id", &block, ctx); - prompt_for("topic", &topic, ctx); - - get_dts = prompt_for_datetime(block,ctx); - - TOSDB_GetItemCount(block.c_str(), &nitems); - - try{ - dat = NewStrings(nitems, TOSDB_STR_DATA_SZ - 1); - lab = NewStrings(nitems, TOSDB_STR_DATA_SZ - 1); - - if(get_dts) - dts = new DateTimeStamp[ nitems ]; - - ret = TOSDB_GetItemFrameStrings(block.c_str(), topic.c_str(), dat, nitems, - TOSDB_STR_DATA_SZ, lab, TOSDB_STR_DATA_SZ, dts); - - _check_display_ret(ret, dat, lab, nitems, dts); - - DeleteStrings(dat, nitems); - DeleteStrings(lab, nitems); - if(dts) - delete dts; - }catch(...){ - DeleteStrings(dat, nitems); - DeleteStrings(lab, nitems); - if(dts) - delete dts; - throw; - } -} - - -void -GetTopicFrameStrings(CommandCtx *ctx) -{ - size_type ntopics; - std::string block; - std::string item; - int ret; - bool get_dts; - - char** dat = nullptr; - char** lab = nullptr; - pDateTimeStamp dts = nullptr; - - prompt_for("block id", &block, ctx); - prompt_for("item", &item, ctx); - - get_dts = prompt_for_datetime(block, ctx); - - TOSDB_GetTopicCount(block.c_str(), &ntopics); - - try{ - dat = NewStrings(ntopics, TOSDB_STR_DATA_SZ -1); - lab = NewStrings(ntopics, TOSDB_STR_DATA_SZ - 1); - - if(get_dts) - dts = new DateTimeStamp[ntopics]; - - ret = TOSDB_GetTopicFrameStrings(block.c_str(), item.c_str(), dat, ntopics, - TOSDB_STR_DATA_SZ, lab, TOSDB_STR_DATA_SZ, dts); - - _check_display_ret(ret, dat, lab, ntopics, dts); - - DeleteStrings(dat, ntopics); - DeleteStrings(lab, ntopics); - if(dts) - delete dts; - - }catch(...){ - DeleteStrings(dat, ntopics); - DeleteStrings(lab, ntopics); - if(dts) - delete dts; - } -} - - -void -GetTopicFrame(CommandCtx *ctx) -{ - std::string block; - std::string item; - bool get_dts; - - prompt_for("block id", &block, ctx); - prompt_for("item", &item, ctx); - - get_dts = prompt_for_datetime(block, ctx); - - if(get_dts) - std::cout<< std::endl << TOSDB_GetTopicFrame(block,item); - else - std::cout<< std::endl << TOSDB_GetTopicFrame(block,item); - - std::cout<< std::endl; -} - - -void -GetTotalFrame(CommandCtx *ctx) -{ - std::string block; - bool get_dts; - - prompt_for("block id", &block, ctx); - - get_dts = prompt_for_datetime(block, ctx); - - if(get_dts) - std::cout<< std::endl << TOSDB_GetTotalFrame(block); - else - std::cout<< std::endl << TOSDB_GetTotalFrame(block); - - std::cout<< std::endl; -} - - -template -void -_get_item_frame( int(*func)(LPCSTR, LPCSTR, T*, size_type, - LPSTR*, size_type, pDateTimeStamp), - CommandCtx *ctx ) -{ - std::string block; - std::string topic; - size_type nitems; - int ret; - bool get_dts; - - pDateTimeStamp dts= nullptr; - T *dat = nullptr; - char** lab = nullptr; - - prompt_for("block id", &block, ctx); - prompt_for("topic", &topic, ctx); - - get_dts = prompt_for_datetime(block,ctx); - - TOSDB_GetItemCount(block.c_str(), &nitems); - - try{ - dat = new T[nitems]; - lab = NewStrings(nitems, TOSDB_MAX_STR_SZ); - - if(get_dts) - dts = new DateTimeStamp[nitems]; - - ret = func(block.c_str(), topic.c_str() ,dat , nitems, lab, TOSDB_MAX_STR_SZ, dts); - - _check_display_ret(ret, dat, lab, nitems, dts); - - DeleteStrings(lab, nitems); - if(dat) - delete[] dat; - if(dts) - delete[] dts; - }catch(...){ - DeleteStrings(lab, nitems); - if(dat) - delete[] dat; - if(dts) - delete[] dts; - } - -} - - -template -void -_check_display_ret(int r, T *v, char **l, size_type n, pDateTimeStamp d) -{ - if(r) - std::cout<< std::endl << "error: " << r << std::endl << std::endl; - else{ - std::cout<< std::endl; - for(size_type i = 0; i < n; ++i) - std::cout<< l[i] << ' ' << v[i] << ' ' << (d ? (d + i) : d) << std::endl; - std::cout<< std::endl; - } -} - -}; /* namespace */ \ No newline at end of file diff --git a/src/shell/commands_get.cpp b/src/shell/commands_get.cpp deleted file mode 100644 index 7d3cbb6..0000000 --- a/src/shell/commands_get.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "shell.hpp" - -namespace{ - -void GetDouble(CommandCtx *ctx); -void GetFloat(CommandCtx *ctx); -void GetLongLong(CommandCtx *ctx); -void GetLong(CommandCtx *ctx); -void GetString(CommandCtx *ctx); -void GetGeneric(CommandCtx *ctx); - -}; /* namespace */ - -CommandsMap commands_get( - CommandsMap::InitChain("GetDouble",GetDouble) - ("GetFloat",GetFloat) - ("GetLongLong",GetLongLong) - ("GetLong",GetLong) - ("GetString",GetString) - ("GetGeneric",GetGeneric) -); - -namespace { - -template -void -_get(CommandCtx *ctx); - -template -void -_get( int(*func)(LPCSTR, LPCSTR, LPCSTR, long, T*, pDateTimeStamp), CommandCtx *ctx ); - -template -void -_check_display_ret(int r, T v, pDateTimeStamp d); - - -void -GetDouble(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) ? _get(ctx) : _get(TOSDB_GetDouble, ctx); -} - - -void -GetFloat(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) ? _get(ctx) : _get(TOSDB_GetFloat, ctx); -} - - -void -GetLongLong(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) ? _get(ctx) : _get(TOSDB_GetLongLong, ctx); -} - - -void -GetLong(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) ? _get(ctx) : _get(TOSDB_GetLong, ctx); -} - - -void -GetString(CommandCtx *ctx) -{ - if(prompt_for_cpp(ctx)){ - _get(ctx); - }else{ - char s[TOSDB_STR_DATA_SZ]; - DateTimeStamp dts; - std::string block; - std::string item; - std::string topic; - std::string index_s; - int ret; - bool get_dts; - - prompt_for_block_item_topic_index(&block, &item, &topic, &index_s, ctx); - - long index = 0; - try{ - index = std::stol(index_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - ret = TOSDB_GetString(block.c_str(), item.c_str(), topic.c_str(), index, - s, TOSDB_STR_DATA_SZ, (get_dts ? &dts : nullptr)); - - _check_display_ret(ret, s, (get_dts ? &dts : nullptr)); - } -} - - -void -GetGeneric(CommandCtx *ctx) -{ - _get(ctx); -} - - -template -void -_get(CommandCtx *ctx) -{ - std::string block; - std::string item; - std::string topic; - std::string index_s; - bool get_dts; - - prompt_for_block_item_topic_index(&block, &item, &topic, &index_s, ctx); - - long index = 0; - try{ - index = std::stol(index_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - if(get_dts) - std::cout<< std::endl - << TOSDB_Get(block, item, TOS_Topics::MAP()[topic], index) - << std::endl << std::endl; - else - std::cout<< std::endl - << TOSDB_Get(block,item,TOS_Topics::MAP()[topic], index) - < -void -_get( int(*func)(LPCSTR, LPCSTR, LPCSTR, long, T*, pDateTimeStamp), CommandCtx *ctx ) -{ - T d; - DateTimeStamp dts; - std::string block; - std::string item; - std::string topic; - std::string index_s; - int ret; - bool get_dts; - - prompt_for_block_item_topic_index(&block, &item, &topic, &index_s, ctx); - - long index = 0; - try{ - index = std::stol(index_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - ret = func(block.c_str(), item.c_str(), topic.c_str(), index, - &d, (get_dts ? &dts : nullptr)); - _check_display_ret(ret, d, (get_dts ? &dts : nullptr)); -} - - -template -void -_check_display_ret(int r, T v, pDateTimeStamp d) -{ - if(r) - std::cout<< std::endl << "error: "<< r << std::endl << std::endl; - else - std::cout<< std::endl << v << ' ' << d << std::endl << std::endl; -} - -}; /* namespace */ \ No newline at end of file diff --git a/src/shell/commands_local.cpp b/src/shell/commands_local.cpp deleted file mode 100644 index bec8c54..0000000 --- a/src/shell/commands_local.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include -#include - -#include "shell.hpp" - -namespace{ - -void local_topics(CommandCtx *ctx); -void local_commands(CommandCtx *ctx); -void local_exit(CommandCtx *ctx); -void local_language(CommandCtx *ctx); - -}; /* namespace */ - -CommandsMap commands_local( - CommandsMap::InitChain("topics",local_topics,"list TOPICS that TOS accepts") - ("commands",local_commands, "list commands for this shell") - ("language",local_language, "set default language(C/C++/none)") - ("exit",local_exit, "exit this shell") -); - - -namespace{ - -bool -_decr_page_latch_and_wait(int *n, std::function cb_on_wait ); - -void -_display_default_language() -{ - auto l = language_strings[get_default_language()]; - - std::cout<< std::endl << "Current default language: " << l.first << " (" << l.second << ")" - << std::endl << std::endl; -} - -void -local_exit(CommandCtx *ctx) -{ - ctx->exit = true; -} - -void -local_language(CommandCtx *ctx) -{ - std::string input; - - _display_default_language(); - - std::cout<< "Would you like to change default language?(y/n)"; - prompt>> input << stream_prompt::endl; - - if(input == "n") - return; - else if(input != "y"){ - std::cerr<< "INVALID - did not change default language" << std::endl << std::endl; - return; - } - - std::cout<< "Please select new default language:" << std::endl; - for(auto & s : language_strings) - std::cout<< " - " << std::setw(8) << s.second.first - << " - " << s.second.second << std::endl; - std::cout<< std::endl; - prompt>> input; - - for(auto & s : language_strings){ - if(input == s.second.first){ - set_default_language(s.first); - _display_default_language(); - return; - } - } - - std::cerr<< std::endl << "INVALID (check case) - did not change default language" - << std::endl << std::endl; -} - -void -local_topics(CommandCtx *ctx) -{ - const int SPC = 3; - const std::string TOPIC_HEAD("TOPICS: "); - - int wc; - - std::vector topics; - - std::transform( - TOS_Topics::MAP().cbegin(), - TOS_Topics::MAP().cend(), - std::insert_iterator>(topics,topics.begin()), - [&](const TOS_Topics::topic_map_entry_type& e){ return e.second; } - ); - - std::sort(topics.begin(), topics.end(), std::less()); - - std::cout<< std::endl << TOPIC_HEAD; - wc = TOPIC_HEAD.size(); - - for(auto & t : topics){ - if(t == " ") /* exclude NULL Topic */ - continue; - if(wc + t.size() >= MAX_DISPLAY_WIDTH){ /* stay within maxw chars */ - std::cout<< std::endl << std::string(TOPIC_HEAD.size(),' '); - wc = TOPIC_HEAD.size(); - } - std::cout<< t; - wc += t.size(); - if(wc + SPC < MAX_DISPLAY_WIDTH){ - std::cout<< std::string(SPC,' '); - wc += SPC; - } - } - std::cout<< std::endl << std::endl; -} - - -void -local_commands(CommandCtx *ctx) -{ - std::string cmd; - size_t maxl; - - int count = CMD_OUT_PER_PAGE; - auto reset_count = [&count](){ count = CMD_OUT_PER_PAGE; }; - - std::cout<< std::endl << "What type of command?" << std::endl << std::endl; - for(auto & p : commands){ - std::cout<< "- " << p.first << std::endl; - } - std::cout<< std::endl << "- back" << std::endl << std::endl; - - while(1){ - prompt_b<< "commands" >> cmd; - - if(cmd == "exit"){ - ctx->exit = true; - return; - } - - if(cmd == "back") - return; - - for(auto & group : commands){ - if(cmd == group.first){ - std::cout<< std::endl << group.second.first << " COMMANDS:" - << std::endl << std::endl; - maxl = 0; - for(auto & c : group.second.second) - maxl = std::max(maxl, c.first.size()); - for(auto & c : group.second.second){ - if( !_decr_page_latch_and_wait(&count, reset_count) ) - break; - std::cout<<" - "<< std::setw(maxl + 2) << c.first - << (c.second.doc.empty() ? "" : " - ") << c.second.doc << std::endl; - } - std::cout<< std::endl; - return; - } - } - - std::cout<< std::endl << "BAD COMMAND" << std::endl << std::endl ; - } -} - -bool -_decr_page_latch_and_wait(int *n, std::function cb_on_wait ) -{ - std::string more_y_or_n; - - if(*n > 0){ - --(*n); - }else{ - std::cout<< std::endl << " More? (y/n) "; - prompt>> more_y_or_n; - cb_on_wait(); - if(more_y_or_n != "y") /* stop on everythin but 'y' */ - return false; - std::cout<< std::endl; - } - return true; -} - -}; /* namespace */ \ No newline at end of file diff --git a/src/shell/commands_stream.cpp b/src/shell/commands_stream.cpp deleted file mode 100644 index 0906a07..0000000 --- a/src/shell/commands_stream.cpp +++ /dev/null @@ -1,656 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include - -#include "shell.hpp" - -namespace{ - -void GetStreamSnapshotDoubles(CommandCtx *ctx); -void GetStreamSnapshotFloats(CommandCtx *ctx); -void GetStreamSnapshotLongLongs(CommandCtx *ctx); -void GetStreamSnapshotLongs(CommandCtx *ctx); -void GetStreamSnapshotStrings(CommandCtx *ctx); -void GetStreamSnapshotGenerics(CommandCtx *ctx); -void GetStreamSnapshotDoublesFromMarker(CommandCtx *ctx); -void GetStreamSnapshotFloatsFromMarker(CommandCtx *ctx); -void GetStreamSnapshotLongLongsFromMarker(CommandCtx *ctx); -void GetStreamSnapshotLongsFromMarker(CommandCtx *ctx); -void GetStreamSnapshotStringsFromMarker(CommandCtx *ctx); -void GetNDoublesFromMarker(CommandCtx *ctx); -void GetNFloatsFromMarker(CommandCtx *ctx); -void GetNLongLongsFromMarker(CommandCtx *ctx); -void GetNLongsFromMarker(CommandCtx *ctx); -void GetNStringsFromMarker(CommandCtx *ctx); - -}; /* namespace */ - -CommandsMap commands_stream( - CommandsMap::InitChain("GetStreamSnapshotDoubles",GetStreamSnapshotDoubles) - ("GetStreamSnapshotFloats",GetStreamSnapshotFloats) - ("GetStreamSnapshotLongLongs",GetStreamSnapshotLongLongs) - ("GetStreamSnapshotLongs",GetStreamSnapshotLongs) - ("GetStreamSnapshotStrings",GetStreamSnapshotStrings) - ("GetStreamSnapshotGenerics",GetStreamSnapshotGenerics) - ("GetStreamSnapshotDoublesFromMarker",GetStreamSnapshotDoublesFromMarker) - ("GetStreamSnapshotFloatsFromMarker",GetStreamSnapshotFloatsFromMarker) - ("GetStreamSnapshotLongLongsFromMarker",GetStreamSnapshotLongLongsFromMarker) - ("GetStreamSnapshotLongsFromMarker",GetStreamSnapshotLongsFromMarker) - ("GetStreamSnapshotStringsFromMarker",GetStreamSnapshotStringsFromMarker) - ("GetNDoublesFromMarker",GetNDoublesFromMarker) - ("GetNFloatsFromMarker",GetNFloatsFromMarker) - ("GetNLongLongsFromMarker",GetNLongLongsFromMarker) - ("GetNLongsFromMarker",GetNLongsFromMarker) - ("GetNStringsFromMarker",GetNStringsFromMarker) -); - - -namespace{ - -template -void -_get_stream_snapshot( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long, long), - CommandCtx *ctx ); - -template -void _get_stream_snapshot(CommandCtx *ctx); - -template -void -_get_stream_snapshot_from_marker( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long, long*), - CommandCtx *ctx ); - - -template -void -_get_n_from_marker( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long*), - CommandCtx *ctx ); - -template -void -_check_display_ret(int r, T *v, size_type n, pDateTimeStamp d); - -template -void -_display_stream_data(size_type len, T* dat, pDateTimeStamp dts); - -size_type -_min_stream_len(std::string block, long beg, long end, size_type len); - - -void -GetStreamSnapshotDoubles(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) - ? _get_stream_snapshot(ctx) - : _get_stream_snapshot(TOSDB_GetStreamSnapshotDoubles, ctx); -} - - -void -GetStreamSnapshotFloats(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) - ? _get_stream_snapshot(ctx) - : _get_stream_snapshot(TOSDB_GetStreamSnapshotFloats, ctx); -} - - -void -GetStreamSnapshotLongLongs(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) - ? _get_stream_snapshot(ctx) - : _get_stream_snapshot(TOSDB_GetStreamSnapshotLongLongs, ctx); -} - - -void -GetStreamSnapshotLongs(CommandCtx *ctx) -{ - prompt_for_cpp(ctx) - ? _get_stream_snapshot(ctx) - : _get_stream_snapshot(TOSDB_GetStreamSnapshotLongs, ctx); -} - - -void -GetStreamSnapshotStrings(CommandCtx *ctx) -{ - if(prompt_for_cpp(ctx)){ - _get_stream_snapshot(ctx); - }else{ - size_type display_len; - int ret; - std::string block; - std::string item; - std::string topic; - std::string len_s; - std::string beg_s; - std::string end_s; - bool get_dts; - - char **dat = nullptr; - pDateTimeStamp dts = nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("length of array", &len_s, ctx); - prompt_for("beginning datastream index", &beg_s, ctx); - prompt_for("ending datastream index", &end_s, ctx); - - size_type len = 0; - long beg = 0; - long end = 0; - try{ - len = std::stoul(len_s); - beg = std::stol(beg_s); - end = std::stol(end_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - try{ - dat = NewStrings(len, TOSDB_STR_DATA_SZ - 1); - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = TOSDB_GetStreamSnapshotStrings(block.c_str(), item.c_str(), topic.c_str(), - dat, len, TOSDB_STR_DATA_SZ, dts, end, beg); - - display_len = _min_stream_len(block,beg,end,len); - - _check_display_ret(ret, dat, display_len, dts); - - DeleteStrings(dat, len); - if(dts) - delete dts; - }catch(...){ - DeleteStrings(dat, len); - if(dts) - delete dts; - throw; - } - } -} - - -void -GetStreamSnapshotGenerics(CommandCtx *ctx) -{ - _get_stream_snapshot(ctx); -} - - -void -GetStreamSnapshotDoublesFromMarker(CommandCtx *ctx) -{ - _get_stream_snapshot_from_marker(TOSDB_GetStreamSnapshotDoublesFromMarker, ctx); -} - - -void -GetStreamSnapshotFloatsFromMarker(CommandCtx *ctx) -{ - _get_stream_snapshot_from_marker(TOSDB_GetStreamSnapshotFloatsFromMarker, ctx); -} - - -void -GetStreamSnapshotLongLongsFromMarker(CommandCtx *ctx) -{ - _get_stream_snapshot_from_marker(TOSDB_GetStreamSnapshotLongLongsFromMarker, ctx); -} - - -void -GetStreamSnapshotLongsFromMarker(CommandCtx *ctx) -{ - _get_stream_snapshot_from_marker(TOSDB_GetStreamSnapshotLongsFromMarker, ctx); -} - - -void -GetStreamSnapshotStringsFromMarker(CommandCtx *ctx) -{ - long get_size; - int ret; - std::string block; - std::string item; - std::string topic; - std::string len_s; - std::string beg_s; - bool get_dts; - - char **dat = nullptr; - pDateTimeStamp dts= nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - - prompt_for("length of array", &len_s, ctx); - prompt_for("beginning datastream index", &beg_s, ctx); - - size_type len = 0; - long beg = 0; - try{ - len = std::stoul(len_s); - beg = std::stol(beg_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - try{ - dat = NewStrings(len, TOSDB_STR_DATA_SZ - 1); - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = TOSDB_GetStreamSnapshotStringsFromMarker(block.c_str(), item.c_str(), topic.c_str(), - dat, len, TOSDB_STR_DATA_SZ, - dts, beg, &get_size); - - _check_display_ret(ret, dat, abs(get_size), dts); - - DeleteStrings(dat, len); - if(dts) - delete dts; - }catch(...){ - DeleteStrings(dat, len); - if(dts) - delete dts; - } -} - - -void -GetNDoublesFromMarker(CommandCtx *ctx) -{ - _get_n_from_marker(TOSDB_GetNDoublesFromMarker, ctx); -} - - -void -GetNFloatsFromMarker(CommandCtx *ctx) -{ - _get_n_from_marker(TOSDB_GetNFloatsFromMarker, ctx); -} - - -void -GetNLongLongsFromMarker(CommandCtx *ctx) -{ - _get_n_from_marker(TOSDB_GetNLongLongsFromMarker, ctx); -} - - -void -GetNLongsFromMarker(CommandCtx *ctx) -{ - _get_n_from_marker(TOSDB_GetNLongsFromMarker, ctx); -} - - -void -GetNStringsFromMarker(CommandCtx *ctx) -{ - long get_size; - int ret; - std::string block; - std::string item; - std::string topic; - std::string len_s; - bool get_dts; - - char **dat = nullptr; - pDateTimeStamp dts= nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("n", &len_s, ctx); - - size_type len = 0; - try{ - len = std::stoul(len_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - try{ - dat = NewStrings(len, TOSDB_STR_DATA_SZ - 1); - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = TOSDB_GetNStringsFromMarker(block.c_str(), item.c_str(), topic.c_str(), - dat, len, TOSDB_STR_DATA_SZ, - dts, &get_size); - - _check_display_ret(ret, dat, abs(get_size), dts); - - DeleteStrings(dat, len); - if(dts) - delete dts; - }catch(...){ - DeleteStrings(dat, len); - if(dts) - delete dts; - } -} - - -template -void -_get_stream_snapshot(CommandCtx *ctx) -{ - std::string block; - std::string item; - std::string topic; - std::string beg_s; - std::string end_s; - bool get_dts; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("beginning datastream index", &beg_s, ctx); - prompt_for("ending datastream index", &end_s, ctx); - - long beg = 0; - long end = 0; - try{ - beg = std::stol(beg_s); - end = std::stol(end_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - if(get_dts) - std::cout<< std::endl - << TOSDB_GetStreamSnapshot(block.c_str(), item.c_str(), - TOS_Topics::MAP()[topic], end, beg); - else - std::cout<< std::endl - << TOSDB_GetStreamSnapshot(block.c_str(), item.c_str(), - TOS_Topics::MAP()[topic], end, beg) - << std::endl; - - std::cout<< std::endl; -} - - -template -void -_get_stream_snapshot( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long, long), - CommandCtx *ctx ) -{ - size_type display_len; - int ret; - std::string block; - std::string item; - std::string topic; - std::string len_s; - std::string beg_s; - std::string end_s; - bool get_dts; - - pDateTimeStamp dts = nullptr; - T *dat = nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("length of array", &len_s, ctx); - prompt_for("beginning datastream index", &beg_s, ctx); - prompt_for("ending datastream index", &end_s, ctx); - - size_type len = 0; - long beg = 0; - long end = 0; - try{ - len = std::stoul(len_s); - beg = std::stol(beg_s); - end = std::stol(end_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - try{ - dat = new T[len]; - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = func(block.c_str(), item.c_str(), topic.c_str(), dat, len, dts, end, beg); - - display_len = _min_stream_len(block,beg,end,len); - - _check_display_ret(ret, dat, display_len, dts); - - if(dat) - delete[] dat; - if(dts) - delete[] dts; - }catch(...){ - if(dat) - delete[] dat; - if(dts) - delete[] dts; - } -} - - -template -void -_get_stream_snapshot_from_marker( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long, long*), - CommandCtx *ctx ) -{ - long get_size; - int ret; - std::string block; - std::string item; - std::string topic; - std::string beg_s; - std::string len_s; - bool get_dts; - - pDateTimeStamp dts = nullptr; - T *dat = nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("length of array", &len_s, ctx); - prompt_for("beginning datastream index", &beg_s, ctx); - - size_type len = 0; - long beg = 0; - try{ - len = std::stoul(len_s); - beg = std::stol(beg_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - try{ - dat = new T[len]; - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = func(block.c_str(), item.c_str(), topic.c_str(), dat, len, dts, beg, &get_size); - - _check_display_ret(ret, dat, abs(get_size), dts); - - if(dat) - delete[] dat; - if(dts) - delete[] dts; - }catch(...){ - if(dat) - delete[] dat; - if(dts) - delete[] dts; - } -} - - -template -void -_get_n_from_marker( int(*func)(LPCSTR, LPCSTR, LPCSTR, T*, - size_type, pDateTimeStamp, long*), - CommandCtx *ctx ) -{ - long get_size; - int ret; - std::string block; - std::string item; - std::string topic; - std::string beg_s; - std::string len_s; - bool get_dts; - - pDateTimeStamp dts = nullptr; - T *dat = nullptr; - - prompt_for_block_item_topic(&block, &item, &topic, ctx); - prompt_for("n", &len_s, ctx); - - size_type len = 0; - try{ - len = std::stoul(len_s); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - return; - } - - get_dts = prompt_for_datetime(block, ctx); - - try{ - dat = new T[len]; - - if(get_dts) - dts = new DateTimeStamp[len]; - - ret = func(block.c_str(), item.c_str(), topic.c_str(), dat, len, dts, &get_size); - - _check_display_ret(ret, dat, abs(get_size), dts); - - if(dat) - delete[] dat; - if(dts) - delete[] dts; - }catch(...){ - if(dat) - delete[] dat; - if(dts) - delete[] dts; - } -} - -template -void -_check_display_ret(int r, T *v, size_type n, pDateTimeStamp d) -{ - if(r) - std::cout<< std::endl << "error: " << r << std::endl << std::endl; - else - _display_stream_data(n, v, d); -} - - -template -void -_display_stream_data(size_type len, T* dat, pDateTimeStamp dts) -{ - std::string index; - - do{ - std::cout<< std::endl; - std::cout<< "Show data from what index value?('all' to show all, 'none' to quit) "; - prompt>> index; - - if(index == "none") - break; - - if(index == "all"){ - std::cout<< std::endl; - for(size_type i = 0; i < len; ++i) - std::cout<< dat[i] << ' ' << (dts ? &dts[i] : nullptr) << std::endl; - break; - } - - try{ - long long i = std::stoll(index); - if(i >= len){ - std::cerr<< std::endl - << "BAD INPUT - index must be < min(array length, abs(end-beg))" - << std::endl; - continue; - } - std::cout<< std::endl << dat[i] << ' ' << (dts ? &dts[i] : nullptr) << std::endl; - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - continue; - } - - }while(1); - - std::cout<< std::endl; -} - - -size_type -_min_stream_len(std::string block, long beg, long end, size_type len) -{ - size_type block_size = TOSDB_GetBlockSize(block); - size_type min_size = 0; - - if(end < 0) - end += block_size; - if(beg < 0) - beg += block_size; - - min_size = std::min((size_type)(end-beg+1), len); - - if(beg < 0 || end < 0) - throw TOSDB_Error("STREAM DATA", "negative 'beg' or 'end' index values"); - - if( (size_type)beg >= block_size - || (size_type)end >= block_size - || (size_type)min_size <= 0) - throw TOSDB_Error("STREAM DATA", "invalid beg or end index values"); - - return min_size; -} - -}; /* namespace */ \ No newline at end of file diff --git a/src/shell/shell.cpp b/src/shell/shell.cpp deleted file mode 100644 index dafb70c..0000000 --- a/src/shell/shell.cpp +++ /dev/null @@ -1,312 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include -#include - -#include "shell.hpp" - -stream_prompt prompt("[-->"); -stream_prompt_basic prompt_b("[-->"); - -/* to avoid compile-time ordinality issues we pass the commands groups by - reference via CommandsMapRef */ -commands_map_of_maps_ty -commands = InitializerChain - ( "admin", "ADMINISTRATIVE", CommandsMapRef(commands_admin) ) - ( "get", "GET", CommandsMapRef(commands_get) ) - ( "stream", "STREAM-SNAPSHOT", CommandsMapRef(commands_stream) ) - ( "frame", "FRAME", CommandsMapRef(commands_frame) ) - ( "local", "LOCAL", CommandsMapRef(commands_local) ); - -language_strings_ty -language_strings = InitializerChain - ( language::none, "NONE", "no default; ask user during call") - ( language::c, "C", "use C version of call(if available)") - ( language::cpp, "C++", "use C++ version of call(if available)"); - - -namespace{ - -template -void -_display_header_line(std::string pre, std::string post, std::string text); - -template -void -_display_header(std::string hpre, std::string hpost); - -}; /* namespace */ - -int main(int argc, char* argv[]) -{ - std::string cmd; - - _display_header("[--", "--]"); - - while(1){ - new_command: - try{ - prompt>> cmd; - - for(auto & group : commands){ - for(auto & c : group.second.second){ - if(c.first != cmd) - continue; - c.second.exit = false; - c.second.func(&c.second); - if(c.second.exit) - goto exit_prompt; - goto new_command; - - } - } - - /* default to here */ - std::cout<< std::endl << "BAD COMMAND" << std::endl << std::endl; - - }catch(const TOSDB_Error& e){ - std::cerr<< std::endl << "*** TOSDB_Error caught by shell **" << std::endl - << std::setw(15) << " Process ID: "<< e.processID() << std::endl - << std::setw(15) << " Thread ID: "<< e.threadID() << std::endl - << std::setw(15) << " tag: " << e.tag() << std::endl - << std::setw(15) << " info: " << e.info() << std::endl - << std::setw(15) << " what: " << e.what() << std::endl << std::endl; - goto exit_prompt; - } - - continue; - - /* use goto so we can cleanly break out of main while loop */ - exit_prompt: - { - std::string cont; - std::cout<< std::endl << "Exit shell? (y/n) "; - prompt>> cont; - std::cout<< std::endl; - if(cont == "y") - break; - else if(cont != "n"){ - std::cout<< "INVALID - enter 'y' or 'n'" << std::endl; - goto exit_prompt; - } - } - } - return 0; -} - - -namespace{ - -language _default_language = language::none; - -}; /* namespace */ - - -language -set_default_language(language l) -{ - language oldl = _default_language; - _default_language = l; - return oldl; -} - - -language -get_default_language() -{ - return _default_language; -} - - -size_type -get_cstr_entries(std::string label, char ***pstrs, CommandCtx *ctx) -{ - std::string nentry; - std::string item; - - prompt_for("# " + label + "s", &nentry, ctx); - - unsigned long n = 0; - try{ - n = std::stoul(nentry); - }catch(...){ - std::cerr<< std::endl << "INVALID INPUT" << std::endl << std::endl; - *pstrs = nullptr; - return 0; - } - - try{ - *pstrs = NewStrings(n, TOSDB_MAX_STR_SZ); - - for(size_type i = 0; i < n; ++i){ - prompt_b<< ctx->name; - prompt<< label << "(" << (i+1) << ") " >> item; - strcpy_s((*pstrs)[i], item.length() + 1, item.c_str()); - } - }catch(...){ - DeleteStrings(*pstrs, n); - *pstrs = nullptr; - return 0; - } - - return n; -} - - -bool -prompt_for_cpp(CommandCtx *ctx, int recurse) /*recurse=2*/ -{ - std::string cpp_y_or_n; - - language l = get_default_language(); - - if(l == language::c) - return false; - - if(l == language::cpp) - return true; - - prompt_for("use C++ version? (y/n)", &cpp_y_or_n, ctx); - - if(cpp_y_or_n == "y") - return true; - else if(cpp_y_or_n == "n") - return false; - else if(recurse > 0){ - std::cout<< "INVALID - enter 'y' or 'n'" << std::endl; - return prompt_for_cpp(ctx, recurse-1); - }else{ - std::cout<< "INVALID - defaulting to C version" << std::endl; - return false; - } -} - - -bool -prompt_for_datetime(std::string block, CommandCtx *ctx) -{ - std::string dts_y_or_n; - - if( !TOSDB_IsUsingDateTime(block) ) - return false; - - prompt_for("get datetime stamp? (y/n)", &dts_y_or_n, ctx); - - if(dts_y_or_n == "y") - return true; - else if(dts_y_or_n != "n") - std::cout<< "INVALID - default to NO datetime" << std::endl; - - return false; -} - - -void -prompt_for_item_topic(std::string *pitem, std::string *ptopic, CommandCtx *ctx) -{ - prompt_for("item", pitem, ctx); - prompt_for("topic", ptopic, ctx); -} - -void -prompt_for_block_item_topic(std::string *pblock, - std::string *pitem, - std::string *ptopic, - CommandCtx *ctx) -{ - prompt_for("block", pblock, ctx); - prompt_for_item_topic(pitem, ptopic, ctx); -} - - -void -prompt_for_block_item_topic_index(std::string *pblock, - std::string *pitem, - std::string *ptopic, - std::string *pindex, - CommandCtx *ctx) -{ - prompt_for_block_item_topic(pblock, pitem, ptopic, ctx); - prompt_for("index", pindex, ctx); -} - - -namespace{ - -template -void -_display_header_line(std::string pre, std::string post, std::string text) -{ - int f = W - (pre.size() + post.size() + text.size()) - 1; // account for newline - if(f < 0) - throw std::exception("can't fit header line"); - - std::cout<< pre << std::string(int(f/2), FILL) - << text << std::string(f - int(f/2), FILL) - << post << std::endl; -} - - -template -void -_display_header(std::string hpre, std::string hpost) -{ - char lpath[MAX_PATH+40+40]; - TOSDB_GetClientLogPath(lpath,MAX_PATH+40+40); - - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, "Welcome to the TOS-DataBridge Command Shell"); - _display_header_line(hpre,hpost, "Copyright (C) 2014 Jonathon Ogden"); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, "This program is distributed WITHOUT ANY WARRANTY."); - _display_header_line(hpre,hpost, "See the GNU General Public License for more details."); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, "Use 'Connect' command to connect to the Service."); - _display_header_line(hpre,hpost, "Use 'commands' for a list of commands by category."); - _display_header_line(hpre,hpost, "Use 'topics' for a list of topics(fields) TOS accepts."); - _display_header_line(hpre,hpost, "Use 'exit' to exit."); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, "NOTE: Topics/Items are case sensitive; use upper-case"); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - _display_header_line(hpre,hpost, ""); - - int wc = INDENT; - - std::cout<< std::endl << std::setw(INDENT) << std::left << "LOG: "; - for(auto s : std::string(lpath)){ - if(++wc >= MAX_DISPLAY_WIDTH){ - std::cout<< std::endl << std::string(INDENT,' ') ; - wc = INDENT; - } - std::cout<< s; - } - - std::cout<< std::endl << std::endl << std::setw(INDENT) << std::left - << "PID: " << GetCurrentProcessId() << std::endl; - - commands_local.at("topics").func(nullptr); -} - -}; /* namespace */ - diff --git a/src/topics.cpp b/src/topics.cpp deleted file mode 100644 index 5c0eb93..0000000 --- a/src/topics.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -#include "tos_databridge.h" -#include "initializer_chain.hpp" - -template<> const TOS_Topics::topic_map_type TOS_Topics::map = - InitializerChain - (TOS_Topics::TOPICS::NULL_TOPIC, " ") - (TOS_Topics::TOPICS::HIGH52, "52HIGH") - (TOS_Topics::TOPICS::LOW52, "52LOW") - (TOS_Topics::TOPICS::ASK, "ASK") - (TOS_Topics::TOPICS::ASKX, "ASKX") - (TOS_Topics::TOPICS::ASK_SIZE, "ASK_SIZE") - (TOS_Topics::TOPICS::AX, "AX") - (TOS_Topics::TOPICS::BACK_EX_MOVE, "BACK_EX_MOVE") - (TOS_Topics::TOPICS::BACK_VOL, "BACK_VOL") - (TOS_Topics::TOPICS::BA_SIZE, "BA_SIZE") - (TOS_Topics::TOPICS::BETA, "BETA") - (TOS_Topics::TOPICS::BID, "BID") - (TOS_Topics::TOPICS::BIDX, "BIDX") - (TOS_Topics::TOPICS::BID_SIZE, "BID_SIZE") - (TOS_Topics::TOPICS::BX, "BX") - (TOS_Topics::TOPICS::CALL_VOLUME_INDEX, "CALL_VOLUME_INDEX") - (TOS_Topics::TOPICS::DELTA, "DELTA") - (TOS_Topics::TOPICS::DESCRIPTION, "DESCRIPTION") - (TOS_Topics::TOPICS::DIV, "DIV") - (TOS_Topics::TOPICS::DIV_FREQ, "DIV_FREQ") - (TOS_Topics::TOPICS::EPS, "EPS") - (TOS_Topics::TOPICS::EXCHANGE, "EXCHANGE") - (TOS_Topics::TOPICS::EXPIRATION, "EXPIRATION") - (TOS_Topics::TOPICS::EXTRINSIC, "EXTRINSIC") - (TOS_Topics::TOPICS::EX_DIV_DATE, "EX_DIV_DATE") - (TOS_Topics::TOPICS::EX_MOVE_DIFF, "EX_MOVE_DIFF") - (TOS_Topics::TOPICS::FRONT_EX_MOVE, "FRONT_EX_MOVE") - (TOS_Topics::TOPICS::FRONT_VOL, "FRONT_VOL") - (TOS_Topics::TOPICS::FX_PAIR, "FX_PAIR") - (TOS_Topics::TOPICS::GAMMA, "GAMMA") - (TOS_Topics::TOPICS::HIGH, "HIGH") - (TOS_Topics::TOPICS::HTB_ETB, "HTB_ETB") - (TOS_Topics::TOPICS::IMPL_VOL, "IMPL_VOL") - (TOS_Topics::TOPICS::INTRINSIC, "INTRINSIC") - (TOS_Topics::TOPICS::LAST, "LAST") - (TOS_Topics::TOPICS::LASTX, "LASTX") - (TOS_Topics::TOPICS::LAST_SIZE, "LAST_SIZE") - (TOS_Topics::TOPICS::LOW, "LOW") - (TOS_Topics::TOPICS::LX, "LX") - (TOS_Topics::TOPICS::MARK, "MARK") - (TOS_Topics::TOPICS::MARKET_CAP, "MARKET_CAP") - (TOS_Topics::TOPICS::MARK_CHANGE, "MARK_CHANGE") - (TOS_Topics::TOPICS::MARK_PERCENT_CHANGE, "MARK_PERCENT_CHANGE") - (TOS_Topics::TOPICS::MRKT_MKR_MOVE, "MRKT_MKR_MOVE") - (TOS_Topics::TOPICS::MT_NEWS, "MT_NEWS") - (TOS_Topics::TOPICS::NET_CHANGE, "NET_CHANGE") - (TOS_Topics::TOPICS::OPEN, "OPEN") - (TOS_Topics::TOPICS::OPEN_INT, "OPEN_INT") - (TOS_Topics::TOPICS::OPTION_VOLUME_INDEX, "OPTION_VOLUME_INDEX") - (TOS_Topics::TOPICS::PE, "PE") - (TOS_Topics::TOPICS::PERCENT_CHANGE, "PERCENT_CHANGE") - (TOS_Topics::TOPICS::PUT_CALL_RATIO, "PUT_CALL_RATIO") - (TOS_Topics::TOPICS::PUT_VOLUME_INDEX, "PUT_VOLUME_INDEX") - (TOS_Topics::TOPICS::RHO, "RHO") - (TOS_Topics::TOPICS::SHARES, "SHARES") - (TOS_Topics::TOPICS::STRENGTH_METER, "STRENGTH_METER") - (TOS_Topics::TOPICS::STRIKE, "STRIKE") - (TOS_Topics::TOPICS::SYMBOL, "SYMBOL") - (TOS_Topics::TOPICS::THETA, "THETA") - (TOS_Topics::TOPICS::VEGA, "VEGA") - (TOS_Topics::TOPICS::VOLUME, "VOLUME") - (TOS_Topics::TOPICS::VOL_DIFF, "VOL_DIFF") - (TOS_Topics::TOPICS::VOL_INDEX, "VOL_INDEX") - (TOS_Topics::TOPICS::WEIGHTED_BACK_VOL, "WEIGHTED_BACK_VOL") - (TOS_Topics::TOPICS::YIELD, "YIELD") - (TOS_Topics::TOPICS::CUSTOM1, "CUSTOM1") - (TOS_Topics::TOPICS::CUSTOM2, "CUSTOM2") - (TOS_Topics::TOPICS::CUSTOM3, "CUSTOM3") - (TOS_Topics::TOPICS::CUSTOM4, "CUSTOM4") - (TOS_Topics::TOPICS::CUSTOM5, "CUSTOM5") - (TOS_Topics::TOPICS::CUSTOM6, "CUSTOM6") - (TOS_Topics::TOPICS::CUSTOM7, "CUSTOM7") - (TOS_Topics::TOPICS::CUSTOM8, "CUSTOM8") - (TOS_Topics::TOPICS::CUSTOM9, "CUSTOM9") - (TOS_Topics::TOPICS::CUSTOM10, "CUSTOM10") - (TOS_Topics::TOPICS::CUSTOM11, "CUSTOM11") - (TOS_Topics::TOPICS::CUSTOM12, "CUSTOM12") - (TOS_Topics::TOPICS::CUSTOM13, "CUSTOM13") - (TOS_Topics::TOPICS::CUSTOM14, "CUSTOM14") - (TOS_Topics::TOPICS::CUSTOM15, "CUSTOM15") - (TOS_Topics::TOPICS::CUSTOM16, "CUSTOM16") - (TOS_Topics::TOPICS::CUSTOM17, "CUSTOM17") - (TOS_Topics::TOPICS::CUSTOM18, "CUSTOM18") - (TOS_Topics::TOPICS::CUSTOM19, "CUSTOM19"); - - diff --git a/stylesheets/github-light.css b/stylesheets/github-light.css new file mode 100644 index 0000000..d64ba5d --- /dev/null +++ b/stylesheets/github-light.css @@ -0,0 +1,124 @@ +/* +The MIT License (MIT) + +Copyright (c) 2015 GitHub, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +*/ + +.pl-c /* comment */ { + color: #969896; +} + +.pl-c1 /* constant, markup.raw, meta.diff.header, meta.module-reference, meta.property-name, support, support.constant, support.variable, variable.other.constant */, +.pl-s .pl-v /* string variable */ { + color: #0086b3; +} + +.pl-e /* entity */, +.pl-en /* entity.name */ { + color: #795da3; +} + +.pl-s .pl-s1 /* string source */, +.pl-smi /* storage.modifier.import, storage.modifier.package, storage.type.java, variable.other, variable.parameter.function */ { + color: #333; +} + +.pl-ent /* entity.name.tag */ { + color: #63a35c; +} + +.pl-k /* keyword, storage, storage.type */ { + color: #a71d5d; +} + +.pl-pds /* punctuation.definition.string, string.regexp.character-class */, +.pl-s /* string */, +.pl-s .pl-pse .pl-s1 /* string punctuation.section.embedded source */, +.pl-sr /* string.regexp */, +.pl-sr .pl-cce /* string.regexp constant.character.escape */, +.pl-sr .pl-sra /* string.regexp string.regexp.arbitrary-repitition */, +.pl-sr .pl-sre /* string.regexp source.ruby.embedded */ { + color: #183691; +} + +.pl-v /* variable */ { + color: #ed6a43; +} + +.pl-id /* invalid.deprecated */ { + color: #b52a1d; +} + +.pl-ii /* invalid.illegal */ { + background-color: #b52a1d; + color: #f8f8f8; +} + +.pl-sr .pl-cce /* string.regexp constant.character.escape */ { + color: #63a35c; + font-weight: bold; +} + +.pl-ml /* markup.list */ { + color: #693a17; +} + +.pl-mh /* markup.heading */, +.pl-mh .pl-en /* markup.heading entity.name */, +.pl-ms /* meta.separator */ { + color: #1d3e81; + font-weight: bold; +} + +.pl-mq /* markup.quote */ { + color: #008080; +} + +.pl-mi /* markup.italic */ { + color: #333; + font-style: italic; +} + +.pl-mb /* markup.bold */ { + color: #333; + font-weight: bold; +} + +.pl-md /* markup.deleted, meta.diff.header.from-file */ { + background-color: #ffecec; + color: #bd2c00; +} + +.pl-mi1 /* markup.inserted, meta.diff.header.to-file */ { + background-color: #eaffea; + color: #55a532; +} + +.pl-mdr /* meta.diff.range */ { + color: #795da3; + font-weight: bold; +} + +.pl-mo /* meta.output */ { + color: #1d3e81; +} + diff --git a/stylesheets/pygment_trac.css b/stylesheets/pygment_trac.css new file mode 100644 index 0000000..c6a6452 --- /dev/null +++ b/stylesheets/pygment_trac.css @@ -0,0 +1,69 @@ +.highlight { background: #ffffff; } +.highlight .c { color: #999988; font-style: italic } /* Comment */ +.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ +.highlight .k { font-weight: bold } /* Keyword */ +.highlight .o { font-weight: bold } /* Operator */ +.highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */ +.highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ +.highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .gr { color: #aa0000 } /* Generic.Error */ +.highlight .gh { color: #999999 } /* Generic.Heading */ +.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ +.highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */ +.highlight .go { color: #888888 } /* Generic.Output */ +.highlight .gp { color: #555555 } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #800080; font-weight: bold; } /* Generic.Subheading */ +.highlight .gt { color: #aa0000 } /* Generic.Traceback */ +.highlight .kc { font-weight: bold } /* Keyword.Constant */ +.highlight .kd { font-weight: bold } /* Keyword.Declaration */ +.highlight .kn { font-weight: bold } /* Keyword.Namespace */ +.highlight .kp { font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */ +.highlight .m { color: #009999 } /* Literal.Number */ +.highlight .s { color: #d14 } /* Literal.String */ +.highlight .na { color: #008080 } /* Name.Attribute */ +.highlight .nb { color: #0086B3 } /* Name.Builtin */ +.highlight .nc { color: #445588; font-weight: bold } /* Name.Class */ +.highlight .no { color: #008080 } /* Name.Constant */ +.highlight .ni { color: #800080 } /* Name.Entity */ +.highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #990000; font-weight: bold } /* Name.Function */ +.highlight .nn { color: #555555 } /* Name.Namespace */ +.highlight .nt { color: #000080 } /* Name.Tag */ +.highlight .nv { color: #008080 } /* Name.Variable */ +.highlight .ow { font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mf { color: #009999 } /* Literal.Number.Float */ +.highlight .mh { color: #009999 } /* Literal.Number.Hex */ +.highlight .mi { color: #009999 } /* Literal.Number.Integer */ +.highlight .mo { color: #009999 } /* Literal.Number.Oct */ +.highlight .sb { color: #d14 } /* Literal.String.Backtick */ +.highlight .sc { color: #d14 } /* Literal.String.Char */ +.highlight .sd { color: #d14 } /* Literal.String.Doc */ +.highlight .s2 { color: #d14 } /* Literal.String.Double */ +.highlight .se { color: #d14 } /* Literal.String.Escape */ +.highlight .sh { color: #d14 } /* Literal.String.Heredoc */ +.highlight .si { color: #d14 } /* Literal.String.Interpol */ +.highlight .sx { color: #d14 } /* Literal.String.Other */ +.highlight .sr { color: #009926 } /* Literal.String.Regex */ +.highlight .s1 { color: #d14 } /* Literal.String.Single */ +.highlight .ss { color: #990073 } /* Literal.String.Symbol */ +.highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */ +.highlight .vc { color: #008080 } /* Name.Variable.Class */ +.highlight .vg { color: #008080 } /* Name.Variable.Global */ +.highlight .vi { color: #008080 } /* Name.Variable.Instance */ +.highlight .il { color: #009999 } /* Literal.Number.Integer.Long */ + +.type-csharp .highlight .k { color: #0000FF } +.type-csharp .highlight .kt { color: #0000FF } +.type-csharp .highlight .nf { color: #000000; font-weight: normal } +.type-csharp .highlight .nc { color: #2B91AF } +.type-csharp .highlight .nn { color: #000000 } +.type-csharp .highlight .s { color: #A31515 } +.type-csharp .highlight .sc { color: #A31515 } diff --git a/stylesheets/styles.css b/stylesheets/styles.css new file mode 100644 index 0000000..2e1768e --- /dev/null +++ b/stylesheets/styles.css @@ -0,0 +1,324 @@ +@font-face { + font-family: 'Noto Sans'; + font-weight: 400; + font-style: normal; + src: url('../fonts/Noto-Sans-regular/Noto-Sans-regular.eot'); + src: url('../fonts/Noto-Sans-regular/Noto-Sans-regular.eot?#iefix') format('embedded-opentype'), + local('Noto Sans'), + local('Noto-Sans-regular'), + url('../fonts/Noto-Sans-regular/Noto-Sans-regular.woff2') format('woff2'), + url('../fonts/Noto-Sans-regular/Noto-Sans-regular.woff') format('woff'), + url('../fonts/Noto-Sans-regular/Noto-Sans-regular.ttf') format('truetype'), + url('../fonts/Noto-Sans-regular/Noto-Sans-regular.svg#NotoSans') format('svg'); +} + +@font-face { + font-family: 'Noto Sans'; + font-weight: 700; + font-style: normal; + src: url('../fonts/Noto-Sans-700/Noto-Sans-700.eot'); + src: url('../fonts/Noto-Sans-700/Noto-Sans-700.eot?#iefix') format('embedded-opentype'), + local('Noto Sans Bold'), + local('Noto-Sans-700'), + url('../fonts/Noto-Sans-700/Noto-Sans-700.woff2') format('woff2'), + url('../fonts/Noto-Sans-700/Noto-Sans-700.woff') format('woff'), + url('../fonts/Noto-Sans-700/Noto-Sans-700.ttf') format('truetype'), + url('../fonts/Noto-Sans-700/Noto-Sans-700.svg#NotoSans') format('svg'); +} + +@font-face { + font-family: 'Noto Sans'; + font-weight: 400; + font-style: italic; + src: url('../fonts/Noto-Sans-italic/Noto-Sans-italic.eot'); + src: url('../fonts/Noto-Sans-italic/Noto-Sans-italic.eot?#iefix') format('embedded-opentype'), + local('Noto Sans Italic'), + local('Noto-Sans-italic'), + url('../fonts/Noto-Sans-italic/Noto-Sans-italic.woff2') format('woff2'), + url('../fonts/Noto-Sans-italic/Noto-Sans-italic.woff') format('woff'), + url('../fonts/Noto-Sans-italic/Noto-Sans-italic.ttf') format('truetype'), + url('../fonts/Noto-Sans-italic/Noto-Sans-italic.svg#NotoSans') format('svg'); +} + +@font-face { + font-family: 'Noto Sans'; + font-weight: 700; + font-style: italic; + src: url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.eot'); + src: url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.eot?#iefix') format('embedded-opentype'), + local('Noto Sans Bold Italic'), + local('Noto-Sans-700italic'), + url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.woff2') format('woff2'), + url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.woff') format('woff'), + url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.ttf') format('truetype'), + url('../fonts/Noto-Sans-700italic/Noto-Sans-700italic.svg#NotoSans') format('svg'); +} + +body { + background-color: #fff; + padding:50px; + font: 14px/1.5 "Noto Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; + color:#727272; + font-weight:400; +} + +h1, h2, h3, h4, h5, h6 { + color:#222; + margin:0 0 20px; +} + +p, ul, ol, table, pre, dl { + margin:0 0 20px; +} + +h1, h2, h3 { + line-height:1.1; +} + +h1 { + font-size:28px; +} + +h2 { + color:#393939; +} + +h3, h4, h5, h6 { + color:#494949; +} + +a { + color:#39c; + text-decoration:none; +} + +a:hover { + color:#069; +} + +a small { + font-size:11px; + color:#777; + margin-top:-0.3em; + display:block; +} + +a:hover small { + color:#777; +} + +.wrapper { + width:860px; + margin:0 auto; +} + +blockquote { + border-left:1px solid #e5e5e5; + margin:0; + padding:0 0 0 20px; + font-style:italic; +} + +code, pre { + font-family:Monaco, Bitstream Vera Sans Mono, Lucida Console, Terminal, Consolas, Liberation Mono, DejaVu Sans Mono, Courier New, monospace; + color:#333; + font-size:12px; +} + +pre { + padding:8px 15px; + background: #f8f8f8; + border-radius:5px; + border:1px solid #e5e5e5; + overflow-x: auto; +} + +table { + width:100%; + border-collapse:collapse; +} + +th, td { + text-align:left; + padding:5px 10px; + border-bottom:1px solid #e5e5e5; +} + +dt { + color:#444; + font-weight:700; +} + +th { + color:#444; +} + +img { + max-width:100%; +} + +header { + width:270px; + float:left; + position:fixed; + -webkit-font-smoothing:subpixel-antialiased; +} + +header ul { + list-style:none; + height:40px; + padding:0; + background: #f4f4f4; + border-radius:5px; + border:1px solid #e0e0e0; + width:270px; +} + +header li { + width:89px; + float:left; + border-right:1px solid #e0e0e0; + height:40px; +} + +header li:first-child a { + border-radius:5px 0 0 5px; +} + +header li:last-child a { + border-radius:0 5px 5px 0; +} + +header ul a { + line-height:1; + font-size:11px; + color:#999; + display:block; + text-align:center; + padding-top:6px; + height:34px; +} + +header ul a:hover { + color:#999; +} + +header ul a:active { + background-color:#f0f0f0; +} + +strong { + color:#222; + font-weight:700; +} + +header ul li + li + li { + border-right:none; + width:89px; +} + +header ul a strong { + font-size:14px; + display:block; + color:#222; +} + +section { + width:500px; + float:right; + padding-bottom:50px; +} + +small { + font-size:11px; +} + +hr { + border:0; + background:#e5e5e5; + height:1px; + margin:0 0 20px; +} + +footer { + width:270px; + float:left; + position:fixed; + bottom:50px; + -webkit-font-smoothing:subpixel-antialiased; +} + +@media print, screen and (max-width: 960px) { + + div.wrapper { + width:auto; + margin:0; + } + + header, section, footer { + float:none; + position:static; + width:auto; + } + + header { + padding-right:320px; + } + + section { + border:1px solid #e5e5e5; + border-width:1px 0; + padding:20px 0; + margin:0 0 20px; + } + + header a small { + display:inline; + } + + header ul { + position:absolute; + right:50px; + top:52px; + } +} + +@media print, screen and (max-width: 720px) { + body { + word-wrap:break-word; + } + + header { + padding:0; + } + + header ul, header p.view { + position:static; + } + + pre, code { + word-wrap:normal; + } +} + +@media print, screen and (max-width: 480px) { + body { + padding:15px; + } + + header ul { + width:99%; + } + + header li, header ul li + li + li { + width:33%; + } +} + +@media print { + body { + padding:0.4in; + font-size:12pt; + color:#444; + } +} diff --git a/symbols/Release/_tos-databridge-x64.pdb b/symbols/Release/_tos-databridge-x64.pdb deleted file mode 100644 index f0c66e8..0000000 Binary files a/symbols/Release/_tos-databridge-x64.pdb and /dev/null differ diff --git a/symbols/Release/_tos-databridge-x86.pdb b/symbols/Release/_tos-databridge-x86.pdb deleted file mode 100644 index 6b54603..0000000 Binary files a/symbols/Release/_tos-databridge-x86.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-0.9-x64.pdb b/symbols/Release/tos-databridge-0.9-x64.pdb deleted file mode 100644 index d180619..0000000 Binary files a/symbols/Release/tos-databridge-0.9-x64.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-0.9-x86.pdb b/symbols/Release/tos-databridge-0.9-x86.pdb deleted file mode 100644 index 66ef070..0000000 Binary files a/symbols/Release/tos-databridge-0.9-x86.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-engine-x64.pdb b/symbols/Release/tos-databridge-engine-x64.pdb deleted file mode 100644 index ff7ba7f..0000000 Binary files a/symbols/Release/tos-databridge-engine-x64.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-engine-x86.pdb b/symbols/Release/tos-databridge-engine-x86.pdb deleted file mode 100644 index 7d21edc..0000000 Binary files a/symbols/Release/tos-databridge-engine-x86.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-serv-x64.pdb b/symbols/Release/tos-databridge-serv-x64.pdb deleted file mode 100644 index 56de45a..0000000 Binary files a/symbols/Release/tos-databridge-serv-x64.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-serv-x86.pdb b/symbols/Release/tos-databridge-serv-x86.pdb deleted file mode 100644 index 6397306..0000000 Binary files a/symbols/Release/tos-databridge-serv-x86.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-shell-x64.pdb b/symbols/Release/tos-databridge-shell-x64.pdb deleted file mode 100644 index 6e45391..0000000 Binary files a/symbols/Release/tos-databridge-shell-x64.pdb and /dev/null differ diff --git a/symbols/Release/tos-databridge-shell-x86.pdb b/symbols/Release/tos-databridge-shell-x86.pdb deleted file mode 100644 index 6bac2d5..0000000 Binary files a/symbols/Release/tos-databridge-shell-x86.pdb and /dev/null differ diff --git a/test/c_cpp/TestBuild.bat b/test/c_cpp/TestBuild.bat deleted file mode 100644 index a9f7543..0000000 --- a/test/c_cpp/TestBuild.bat +++ /dev/null @@ -1,191 +0,0 @@ -@echo off - -setlocal enableDelayedExpansion - -set "OURsrc=test.c" -set "RELdir=..\..\bin\Release" -set "INCLdir=..\..\include" -set "BINbase=TestBuild" -set "THISbin=%0%" - -set "VERSION=0.9" - -IF [%3] == [] ( - CALL :usage_warn - EXIT /B 1 -) else ( - set "VScpath=!VS%3%COMNTOOLS!..\..\VC" - if NOT EXIST !VScpath! ( - echo fatal: can't find VS%3%COMNTOOLS; or it contains an invalid path - EXIT /B 1 - ) -) - -IF "%1"=="x86" ( - set "CMDpath=\bin" - set "OURlibdir=!RELdir!\Win32" - set "OURlibstub=!RELdir!\Win32\tos-databridge-!VERSION!-x86.lib" - set "OURdllname=tos-databridge-!VERSION!-x86.dll" - set "OURdll2name=_tos-databridge-x86.dll" - set "OURdllpath=!RELdir!\Win32\!OURdllname!" - set "OURdll2path=!RELdir!\Win32\!OURdll2name!" - set "VCvars=bin\VCvars32.bat" - set "MACHarg=X86" - set "OURobj=!BINbase!32.obj" - set "OURexec=!BINbase!32.exe" - set "OURsyms=!BINbase!32.pdb" -) else ( - IF "%1"=="x64" ( - set "CMDpath=\bin\x86_amd64" - set "OURlibdir=!RELdir!\x64" - set "OURlibstub=!RELdir!\x64\tos-databridge-!VERSION!-x64.lib" - set "OURdllname=tos-databridge-!VERSION!-x64.dll" - set "OURdll2name=_tos-databridge-x64.dll" - set "OURdllpath=!RELdir!\x64\!OURdllname!" - set "OURdll2path=!RELdir!\x64\!OURdll2name!" - set "VCvars=bin\x86_amd64\VCvarsx86_amd64.bat" - set "MACHarg=X64" - set "OURobj=!BINbase!64.obj" - set "OURexec=!BINbase!64.exe" - set "OURsyms=!BINbase!64.pdb" - ) else ( - CALL :usage_warn - EXIT /B 1 - ) -) - -setlocal disableDelayedExpansion - -IF "%2"=="C" ( - set "LANGflag=/TC" - set "LANG=C" -) else ( - IF "%2"=="CPP" ( - set "LANGflag=/TP" - set "LANG=C++" - ) else ( - CALL :usage_warn - EXIT /B 1 - ) -) - -echo -------------------------------------------------------------------------- -echo MACHarg: %MACHarg% -echo LANG: %LANG% -echo VCvars: %VCvars% -echo VScpath: %VScpath% -echo CMDpath: %CMDpath% -echo OURsrc: %OURsrc% -echo OURobj: %OURobj% -echo OURexec: %OURexec% -echo OURsyms: %OURsyms% -echo INCLdir: %INCLdir% -echo OURlibdir: %OURlibdir% -echo OURlibstub: %OURlibstub% -echo OURdllname: %OURdllname% -echo OURdllpath: %OURdllpath% -echo OURdll2name: %OURdll2name% -echo OURdll2path: %OURdll2path% -echo ------------------------------------------------------------------------------- - - -if NOT EXIST "%VSINSTALLDIR%" ( - echo. - echo Setting up environment... - CALL "%VScpath%\%VCvars%" - IF %ERRORLEVEL% NEQ 0 ( - echo fatal: error setting up environment - EXIT /B 1 - ) -) - -IF NOT EXIST "%VSINSTALLDIR%" ( - echo fatal: can't find 'VSINSTALLDIR' - EXIT /B 1 -) - -echo. -echo Clearing Files... -del *.exe *.obj *.dll *.lib *.pdb *.suo 2>NUL - -echo. -echo Compiling... -@echo on - -"%VSINSTALLDIR%\VC\%CMDpath%\CL.exe" /c /Fo"%OURobj%" /IC:%INCLdir% /Zi /nologo ^ -/W3 /WX- /sdl /O2 /Oi /GL /D _MBCS /Gm- /EHsc /MD /GS /Gy /fp:precise /Zc:wchar_t ^ - /Zc:forScope /Gd %LANGflag% /errorReport:prompt %OURsrc% - -@echo off - -IF %ERRORLEVEL% NEQ 0 ( - echo fatal: compilation error - EXIT /B 1 -) - -echo. -echo Linking... -@echo on - -"%VSINSTALLDIR%\VC\%CMDpath%\link.exe" /ERRORREPORT:PROMPT /OUT:%OURexec% /NOLOGO ^ -/LIBPATH:"%OURlibdir%" "%OURlibstub%" kernel32.lib user32.lib gdi32.lib ^ -winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib ^ -odbc32.lib odbccp32.lib /OPT:REF /OPT:ICF /LTCG /TLBID:1 /DYNAMICBASE /NXCOMPAT ^ -/MACHINE:%MACHarg% %OURobj% - -@echo off - -IF %ERRORLEVEL% NEQ 0 ( - echo fatal: link error - EXIT /B 1 -) - -echo. -echo Checking Dependencies... -IF NOT EXIST %OURdllname% ( - COPY %OURdllpath% . -) -IF %ERRORLEVEL% NEQ 0 ( - echo fatal: error copying %OURdllname% - EXIT /B 1 -) - -IF NOT EXIST %OURdll2name% ( - COPY %OURdll2path% . -) -IF %ERRORLEVEL% NEQ 0 ( - echo fatal: error copying %OURdll2name% - EXIT /B 1 -) - -echo. -echo Running %OURexec%... -@echo on -"%OURexec%" -@echo off - -IF %ERRORLEVEL% NEQ 0 ( - echo - Fatal: error running %OURexec% - echo. - echo Clearing Files... - del *.exe *.obj *.dll *.lib *.pdb *.suo 2>NUL - EXIT /B 1 -) else ( - echo + Success! - echo. - echo Clearing Files... - del *.exe *.obj *.dll *.lib *.pdb *.suo 2>NUL -) - - -EXIT /B %ERRORLEVEL% - - -:usage_warn -echo. -echo Usage: %THISbin% [x86^|x64] [C^|CPP] [VS_TOOLS_NUM] -echo. -echo [x86^|x64] - build -echo [C^|CPP] - language -echo [VS_TOOLS_NUM] - integer part of VS...COMNTOOLS variable (e.g VS110COMNTOOLS -^> 110) -EXIT /B 0 \ No newline at end of file diff --git a/test/c_cpp/test.c b/test/c_cpp/test.c deleted file mode 100644 index 8eedc96..0000000 --- a/test/c_cpp/test.c +++ /dev/null @@ -1,289 +0,0 @@ -#include -#include "tos_databridge.h" - -void StaticAdminTests(); -int DynamicAdminTests(); -void InvalidItemTests(); -void GetTests(); -void StreamSnapshotTests(); -void FromMarkerTests(); -void FrameTests(); -void CloseTests(); - -const char* block1_id = "test_block_1"; -static size_type block1_sz = 1000; -static int block1_use_dt = 1; -static size_type block1_timeout = 3000; - -int -main(int argc, char* argv[]) -{ - char lpath[2048]; - - printf("\n*** BEGIN %s BEGIN ***\n\n", argv[0]); - TOSDB_GetClientLogPath(lpath,2048); - printf("LOG: %s\n\n", lpath); - -#ifdef __cplusplus - try{ -#endif - StaticAdminTests(); - - Sleep(500); - if( DynamicAdminTests() ) - return 1; - - Sleep(500); - GetTests(); - - Sleep(500); - StreamSnapshotTests(); - - Sleep(500); - CloseTests(); -#ifdef __cplusplus - }catch(TOSDB_Error & t){ - printf("\n*** TOSDB ERROR ***\n"); - printf("- %s\n", t.info_and_what().c_str() ); - return 1; - }catch(std::exception & e){ - printf("\n*** EXCEPTION ***\n"); - printf("- %s\n", e.what() ); - return 1; - }catch(...){ - printf("\n*** UNKNOWN ERROR ***\n"); - return 1; - } -#endif - printf("\n*** END %s END ***\n\n", argv[0]); - return 0; -} - -void -StaticAdminTests() -{ - printf("+ TOSDB_GetBlockLimit() :: %u \n", TOSDB_GetBlockLimit()); - printf("+ TOSDB_GetBlockCount() :: %u \n", TOSDB_GetBlockCount()); - printf("+ TOSDB_GetLatency() :: %u \n", TOSDB_GetLatency()); - -#ifdef __cplusplus - printf("+ TOSDB_GetTypeBits(TOS_Topics::TOPICS::LAST) :: %u \n", - TOSDB_GetTypeBits(TOS_Topics::TOPICS::LAST)); - printf("+ TOSDB_GetTypeString(TOS_Topics::TOPICS::LAST) :: %s \n", - TOSDB_GetTypeString(TOS_Topics::TOPICS::LAST).c_str()); -#endif -} - - -void -InvalidItemTests() -{ -#ifdef __cplusplus - int ret_c, ret_c_array, ret_c_both, ret_cpp, ret_cpp_set, ret_cpp_both; - - std::vector> sfuncs; - - sfuncs.push_back( [](char i)->std::string{ return std::string(1,i) + "test"; } ); - sfuncs.push_back( [](char i)->std::string{ return "test" + std::string(1,i) + "test"; }); - sfuncs.push_back( [](char i)->std::string{ return "test" + std::string(1,i); }); - - for(char i = 32; i < 127; ++i){ - std::vector catches; - for(auto& f : sfuncs){ - std::string item = f(i); - const char* items[] = {item.c_str()}; - ret_c = TOSDB_AddItem(block1_id, items[0]); - ret_c_array = TOSDB_AddItems(block1_id, items, 1); - ret_c_both = TOSDB_Add(block1_id, items, 1, NULL, 0); - ret_cpp = TOSDB_AddItem(std::string(block1_id), item); - ret_cpp_set = TOSDB_AddItems(std::string(block1_id), str_set_type(item) ); - ret_cpp_both = TOSDB_Add(std::string(block1_id), str_set_type(item), topic_set_type() ); - if( !(ret_c == ret_c_array && ret_c_array == ret_c_both && ret_c_both == ret_cpp - && ret_cpp == ret_cpp_set && ret_cpp_set == ret_cpp_both) ){ - printf(" - Error codes failed to match (%i, %i, %i, %i, %i, %i) \n", - ret_c, ret_c_array, ret_c_both, ret_cpp, ret_cpp_set, ret_cpp_both); - }else if( ret_c != 0 ){ - catches.push_back( items[0] ); - } - } - if( catches.size() > 0 ){ - std::cout<<" + Successfully caught bad input for " << i <<": "; - for(std::string s : catches) - std::cout<< s << " "; - std::cout<(block1_id,"SPY",TOS_Topics::TOPICS::LAST, 0); - printf("+ TOSDB_Get(): %s, %s, %d, %f \n", "SPY", "LAST", 0, p); - - std::pair sp = - TOSDB_Get(block1_id,"QQQ", TOS_Topics::TOPICS::VOLUME, 0); - printf("+ TOSDB_Get(): %s, %s, %d, %lld, %d:%d:%d \n", "QQQ", "VOLUME", - 0, sp.first, sp.second.ctime_struct.tm_hour, sp.second.ctime_struct.tm_min, - sp.second.ctime_struct.tm_sec); - - auto g = TOSDB_Get(block1_id,"SPY",TOS_Topics::TOPICS::LAST, 0); - printf("+ TOSDB_Get(): %s, %s, %d, %f \n", "SPY", "LAST", 0, g.as_double()); - std::cout << " Check Generic Type: \n"; - std::cout << " size: " << g.size() << std::endl; - std::cout << " is_float: " << std::boolalpha<< g.is_float() << std::endl; - std::cout << " is_double: " << std::boolalpha<< g.is_double() << std::endl; - std::cout << " is_long: " << std::boolalpha<< g.is_long() << std::endl; - std::cout << " is_long_long: " << std::boolalpha << g.is_long_long() << std::endl; - std::cout << " is_string: " << std::boolalpha << g.is_string() << std::endl; - std::cout << " is_floating_point: " << std::boolalpha<< g.is_floating_point() << std::endl; - std::cout << " is_integer: " << std::boolalpha<< g.is_integer() << std::endl; - std::cout << " as_float: " << g.as_float() << std::endl; - std::cout << " as_double: " << g.as_double() << std::endl; - std::cout << " as_long: " << g.as_long() << std::endl; - std::cout << " as_long_long: " << g.as_long_long() << std::endl; - std::cout << " as_string: " << g.as_string() << std::endl; - -#endif - -} - - -void -StreamSnapshotTests() -{ - -#ifdef __cplusplus - auto gvec = TOSDB_GetStreamSnapshot(block1_id,"SPY",TOS_Topics::TOPICS::LAST); - std::cout<< "TOSDB_GetStreamSnapshot(): SPY, LAST" << std::endl; - std::cout<< " "; - for(auto& i : gvec.first) - std::cout<< i.as_string() << ' '; - std::cout<(block1_id,"SPY",TOS_Topics::TOPICS::CUSTOM1); - std::cout<< "TOSDB_GetStreamSnapshot(): SPY, CUSTOM1" << std::endl; - std::cout<< " "; - for(auto& i : gvec2) - std::cout<< i << ' '; - std::cout< - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses. -*/ - -import io.github.jeog.tosdatabridge.TOSDataBridge; -import io.github.jeog.tosdatabridge.TOSDataBridge.*; -import io.github.jeog.tosdatabridge.DataBlock; -import io.github.jeog.tosdatabridge.DataBlockWithDateTime; -import io.github.jeog.tosdatabridge.DateTime.DateTimePair; -import io.github.jeog.tosdatabridge.Topic; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; - -/** - * TOSDataBirdgeTest.java - * - * Simple test for TOSDataBridge Java Wrapper/API (tosdatabridge.jar) - * - * 1) load native library and try to connect - * 2) check connection state - * 3) test admin calls - * 4) create DataBlocks - * 5) add/remove items/topics - * 6) use different versions of 'get' methods - * 7) use 'frame methods' - * - * Attempt to check what it can statically and print/throw on error but - * most of the methods that return data must be run dynamically and the - * output reviewed manually, in real-time. - * - * @author Jonathon Ogden - * @version 0.9 - */ -public class TOSDataBridgeTest { - private static final int SLEEP_PERIOD = 1000; - - public static void - main(String[] args) - { - if(args.length == 0 || args[0] == null){ - System.err.println("TOSDataBridge library path must be passed as arg0"); - return; - } - - if( !(new File(args[0])).exists() ){ - System.err.println("Arg0 (" + args[0] + ") is not a valid file."); - throw new IllegalArgumentException("Arg0 is not a valid file"); - } - - try{ - if( !TOSDataBridge.init(args[0]) ){ - throw new RuntimeException("library failed to connect to Engine/TOS"); - } - testConnection(); - testAdminCalls(); - testBlock1(); - testBlock2(); - System.out.println(); - System.out.println("*** DONE (SUCCESS) ***"); - return; - } catch (LibraryNotLoaded e) { - System.out.println("EXCEPTION: LibraryNotLoaded"); - System.out.println(e.toString()); - e.printStackTrace(); - } catch (CLibException e) { - System.out.println("EXCEPTION: CLibException"); - System.out.println(e.toString()); - e.printStackTrace(); - } catch (InvalidItemOrTopic e) { - System.out.println("EXCEPTION: InvalidItemOrTopic"); - System.out.println(e.toString()); - e.printStackTrace(); - } catch (InterruptedException e) { - System.out.println("EXCEPTION: InterruptedException"); - System.out.println(e.toString()); - e.printStackTrace(); - } catch (Throwable t) { - System.out.println("EXCEPTION: ***"); - System.out.println(t.toString()); - t.printStackTrace(); - // catch anything else so we can print fail message - } - - System.out.println(); - System.err.println("*** DONE (FAILURE) ***"); - } - - private static void - testBlock1() throws LibraryNotLoaded, CLibException, NoSuchMethodException, - IllegalAccessException, InvocationTargetException, InterruptedException, - DataIndexException, InvalidItemOrTopic { - int sz = 100000; - System.out.println(); - System.out.println("CREATE BLOCK 1..."); - DataBlock block = new DataBlock(sz); - - if( !testBlockState(block, sz, false, TOSDataBridge.DEF_TIMEOUT) ) { - // just break out of the try-block - throw new IllegalStateException("testBlockState(...) failed"); - }else { - System.out.println("Successfully created block: " + block.getName()); - } - - Set items = new HashSet<>(Arrays.asList("/ES:XCME","/ZN:XCBT")); - Set topics = new HashSet<>(Arrays.asList(Topic.LASTX, Topic.LAST, Topic.LAST_SIZE, Topic.CUSTOM1)); - - System.out.println("Add items: " + items.toString()); - for( String s : items ) { - block.addItem(s); - } - printBlockItemsTopics(block); - - // check pre-cache - for( String i : items ){ - if(!block.getItemsPreCached().contains(i)) { - System.err.println("Item pre-cache test failed: " + i); - throw new RuntimeException("Item pre-cache test failed"); - } - } - System.out.println("Item pre-cache looks good: " + block.getItemsPreCached().toString()); - - // add bad topic - try{ - block.addTopic(Topic.NULL_TOPIC); - throw new RuntimeException("Add NULL_TOPIC/CLIbException test failed"); - }catch(TOSDataBridge.InvalidItemOrTopic e){ - System.out.println("Successfully caught exception from adding NULL_TOPIC: " + e.toString() ); - } - - // add bad topic - try{ - block.addTopic(null); - throw new RuntimeException("Add null topic test failed"); - }catch(NullPointerException e){ - System.out.println("Successfully caught exception from adding null topic: " + e.toString() ); - } - - // add bad item - try{ - block.addItem(""); - throw new RuntimeException("Add empty item string test failed"); - }catch(TOSDataBridge.InvalidItemOrTopic e){ - System.out.println("Successfully caught exception from adding empty item string: " + e.toString() ); - } - - // add bad item - try{ - char[] s = new char[TOSDataBridge.MAX_STR_SZ + 1]; - Arrays.fill(s,'x'); - block.addItem(new String(s)); - throw new RuntimeException("Add too large item string failed"); - }catch(TOSDataBridge.InvalidItemOrTopic e){ - System.out.println("Successfully caught exception from adding too large item string: " + e.toString() ); - } - - //add good topics - System.out.println("Add Topics: " + topics.toString()); - for( Topic t : topics ) { - block.addTopic(t); - } - printBlockItemsTopics(block); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - //NEED TO TEST DIRTY MARKER BEFORE ANY OTHER GET CALLS - System.out.println("TEST DIRTY MARKER EXCEPTION: " + block.getName()); - testDirtyMarkerExceptions(block, false); - System.out.println(); - - System.out.println("TEST DATA INDEX EXCEPTION: " + block.getName()); - testDataIndexExceptions(block, false); - System.out.println(); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST GET CALLS, BLOCK: " + block.getName()); - testGetCalls(block,false); - System.out.println(); - - System.out.println("TEST GET CALLS NULL/EXCEPTION BEHAVIOR, BLOCK: " + block.getName()); - testGetCallsExcNullBehavior(block,false); - System.out.println(); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT CALLS, BLOCK: " + block.getName()); - testStreamSnapshotCalls(block,5,false); - System.out.println(); - - System.out.println(); - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT FROM MARKER CALLS, BLOCK: " + block.getName()); - testStreamSnapshotFromMarkerCalls(block,3,SLEEP_PERIOD,false,false); - System.out.println(); - - System.out.println("TEST N FROM MARKER CALLS, BLOCK: " + block.getName()); - testNFromMarkerCalls(block,3,SLEEP_PERIOD,false,false); - System.out.println(); - - System.out.println("TEST TOTAL FRAME CALLS: " + block.getName()); - testTotalFrameCalls(block, false); - System.out.println(); - - System.out.println("TEST ITEM FRAME CALLS: " + block.getName()); - testItemFrameCalls(block, false); - System.out.println(); - - System.out.println("TEST TOPIC FRAME CALLS: " + block.getName()); - testTopicFrameCalls(block, false); - System.out.println(); - - } - - private static void - testBlock2() throws LibraryNotLoaded, CLibException, InterruptedException, DataIndexException, - NoSuchMethodException, IllegalAccessException, InvocationTargetException, - InvalidItemOrTopic { - int sz = 1000; - int timeout = 3000; - System.out.println(); - System.out.println("CREATE BLOCK 2..."); - DataBlockWithDateTime block = new DataBlockWithDateTime(sz, timeout); - - if( !testBlockState(block, sz, true, timeout) ) { - // just break out of the try-block - throw new IllegalStateException("testBlockState(...) failed"); - }else { - System.out.println("Successfully created block: " + block.getName()); - } - System.out.println(); - System.out.println("Double block size..."); - block.setBlockSize(block.getBlockSize() * 2); - if(block.getBlockSize() != 2 * sz){ - System.err.println("failed to double block size"); - throw new IllegalStateException(); - } - - String item1 = "SPY"; - String item2 = "QQQ"; - Topic topic1 = Topic.LAST; // double - Topic topic2 = Topic.VOLUME; // long - Topic topic3 = Topic.STRENGTH_METER; // string - - System.out.println("Add item: " + item1); - block.addItem(item1); - printBlockItemsTopics(block); - - System.out.println("Add item: " + item2); - block.addItem(item2); - printBlockItemsTopics(block); - - System.out.println("Remove item: " + item2); - block.removeItem(item2); - printBlockItemsTopics(block); - - System.out.println("Add item: " + item2); - block.addItem(item2); - printBlockItemsTopics(block); - - System.out.print("check containsItemPreCached: "); - if(!block.containsItemPreCached(item1) || !block.containsItemPreCached(item2)){ - System.out.println("FAILURE"); - throw new IllegalStateException("containsItem failed"); - } - System.out.println("SUCCESS"); - System.out.println(); - - System.out.println("Add topic: " + topic1); - block.addTopic(topic1); - printBlockItemsTopics(block); - - System.out.println("Remove ALL items"); - block.removeItem(item1); - block.removeItem(item2); - printBlockItemsTopics(block); - - System.out.print("check containsTopicPreCached: "); - if(!block.containsTopicPreCached(topic1)){ - System.out.println("FAILURE"); - throw new IllegalStateException("containsTopic failed"); - } - System.out.println("SUCCESS"); - System.out.println(); - - System.out.println("Remove Topic: " + topic1); - block.removeTopic(topic1); - printBlockItemsTopics(block); - - System.out.println("Add ALL topics"); - block.addTopic(topic1); - block.addTopic(topic2); - block.addTopic(topic3); - printBlockItemsTopics(block); - - System.out.println("Remove Topic: " + topic2); - block.removeTopic(topic2); - printBlockItemsTopics(block); - - System.out.println("Add ALL items"); - block.addItem(item1); - block.addItem(item2); - printBlockItemsTopics(block); - - System.out.println("Add topic: " + topic2); - block.addTopic(topic2); - printBlockItemsTopics(block); - - System.out.print("check containsTopic: "); - if(!block.containsTopic(topic1) || !block.containsTopic(topic2) - || !block.containsTopic(topic3)){ - System.out.println("FAILURE"); - throw new IllegalStateException("containsTopic failed"); - } - System.out.println("SUCCESS"); - System.out.println(); - - System.out.print("check containsItem: "); - if(!block.containsItem(item1) || !block.containsItem(item2)){ - System.out.println("FAILURE"); - throw new IllegalStateException("containsItem failed"); - } - System.out.println("SUCCESS"); - System.out.println(); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - //NEED TO TEST DIRTY MARKER BEFORE ANY OTHER GET CALLS - System.out.println("TEST DIRTY MARKER EXCEPTION (WITH DATETIME): " + block.getName()); - testDirtyMarkerExceptions(block, true); - System.out.println(); - - System.out.println("TEST DATA INDEX EXCEPTION (WITH DATETIME): " + block.getName()); - testDataIndexExceptions(block, true); - System.out.println(); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST GET CALLS, BLOCK: " + block.getName()); - testGetCalls(block,false); - System.out.println(); - - System.out.println("TEST GET CALLS (WITH DATETIME), BLOCK: " + block.getName()); - testGetCalls(block,true); - System.out.println(); - - System.out.println("TEST GET CALLS NULL/EXCEPTION BEHAVIOR, BLOCK: " + block.getName()); - testGetCallsExcNullBehavior(block,false); - System.out.println(); - - System.out.println("TEST GET CALLS NULL/EXCEPTION BEHAVIOR (WITH DATETIME), BLOCK: " + block.getName()); - testGetCallsExcNullBehavior(block,true); - System.out.println(); - - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT CALLS, BLOCK: " + block.getName()); - testStreamSnapshotCalls(block,5,false); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT CALLS (WITH DATETIME), BLOCK: " + block.getName()); - testStreamSnapshotCalls(block,3,true); - System.out.println(); - - System.out.println(); - System.out.println("***SLEEP FOR " + String.valueOf(SLEEP_PERIOD) + " MILLISECONDS***"); - Thread.sleep(SLEEP_PERIOD); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT FROM MARKER CALLS, BLOCK: " + block.getName()); - testStreamSnapshotFromMarkerCalls(block,3,SLEEP_PERIOD,false,false); - System.out.println(); - - System.out.println("TEST STREAM SNAPSHOT FROM MARKER (WITH DATETIME) CALLS, BLOCK: " + block.getName()); - testStreamSnapshotFromMarkerCalls(block,3,SLEEP_PERIOD,true,true); - System.out.println(); - - System.out.println("TEST N FROM MARKER CALLS, BLOCK: " + block.getName()); - testNFromMarkerCalls(block,3,SLEEP_PERIOD,false,false); - System.out.println(); - - System.out.println("TEST N FROM MARKER (WITH DATETIME) CALLS, BLOCK: " + block.getName()); - testNFromMarkerCalls(block,3,SLEEP_PERIOD,true,true); - System.out.println(); - - System.out.println("TEST TOTAL FRAME CALLS: " + block.getName()); - testTotalFrameCalls(block, false); - System.out.println(); - - System.out.println("TEST TOTAL FRAME (WITH DATETIME) CALLS, BLOCK: " + block.getName()); - testTotalFrameCalls(block, true); - System.out.println(); - - System.out.println("TEST ITEM FRAME CALLS: " + block.getName()); - testItemFrameCalls(block, false); - System.out.println(); - - System.out.println("TEST ITEM FRAME (WITH DATETIME) CALLS, BLOCK: " + block.getName()); - testItemFrameCalls(block, true); - System.out.println(); - - System.out.println("TEST TOPIC FRAME CALLS: " + block.getName()); - testTopicFrameCalls(block, false); - System.out.println(); - - System.out.println("TEST TOPIC FRAME (WITH DATETIME) CALLS, BLOCK: " + block.getName()); - testTopicFrameCalls(block, true); - System.out.println(); - - } - - - private static void - testConnection() throws LibraryNotLoaded { - // connect() (should already be connected) - TOSDataBridge.connect(); - // connected() - if(!TOSDataBridge.connected()){ - System.out.println("NOT CONNECTED"); - return; - } - // connection_state() - int connState = TOSDataBridge.connectionState(); - System.out.println("CONNECTION STATE: " + String.valueOf(connState)); - if(connState != TOSDataBridge.CONN_ENGINE_TOS){ - System.out.println("INVALID CONNECTION STATE"); - } - } - - private static boolean - testAdminCalls() throws LibraryNotLoaded, CLibException { - // get_block_limit() - int l = TOSDataBridge.getBlockLimit(); - System.out.println("block limit: " + String.valueOf(l)); - - // set_block_limit() - System.out.println("double block limit"); - TOSDataBridge.setBlockLimit(l*2); - - // get_block_limit() - int ll = TOSDataBridge.getBlockLimit(); - if( ll != l * 2 ){ - System.err.println("failed to double block limit"); - return false; - } - System.out.println("block limit: " + String.valueOf(ll)); - - // get_block_count() - int c = TOSDataBridge.getBlockCount(); - if(c != 0){ - System.err.println("initial block count != 0"); - return false; - } - System.out.println("block count: " + String.valueOf(c)); - - // type_bits() - if(!testTypeBits(Topic.LAST, TOSDataBridge.QUAD_BIT, "QUAD_BIT", - TOSDataBridge.TOPIC_IS_DOUBLE, "TOPIC_IS_DOUBLE")) { - return false; - } - if(!testTypeBits(Topic.EPS, 0, "", TOSDataBridge.TOPIC_IS_DOUBLE, "TOPIC_IS_DOUBLE")) { - return false; - } - if(!testTypeBits(Topic.VOLUME, TOSDataBridge.QUAD_BIT | TOSDataBridge.INTGR_BIT, "QUAD_BIT | INTGR_BIT", - TOSDataBridge.TOPIC_IS_LONG, "TOPIC_IS_LONG")){ - return false; - } - if(!testTypeBits(Topic.LAST_SIZE, TOSDataBridge.INTGR_BIT, "INTGR_BIT", - TOSDataBridge.TOPIC_IS_LONG, "TOPIC_IS_LONG")){ - return false; - } - if(!testTypeBits(Topic.SYMBOL, TOSDataBridge.STRING_BIT, "STRING_BIT", - TOSDataBridge.TOPIC_IS_STRING, "TOPIC_IS_STRING")) { - return false; - } - return true; - } - - private static boolean - testTypeBits(Topic topic, int cBits, String cName, int jTypeId, String jName) - throws CLibException, LibraryNotLoaded { - if( TOSDataBridge.getTypeBits(topic) != cBits) { - System.err.println("Type Bits for '" + topic + "' != " + cName + "(" + String.valueOf(cBits) + ")"); - return false; - } - if( TOSDataBridge.getTopicType(topic) != jTypeId) { - System.err.println("Topic type for '" + topic + "' != " + jName + "(" + String.valueOf(jTypeId) + ")"); - return false; - } - return true; - } - - private static boolean - testBlockState(DataBlock block, int blockSize, boolean withDateTime, int timeout) - throws CLibException, LibraryNotLoaded { - if(block.getBlockSize() != blockSize){ - System.out.println("invalid block size: " - + String.valueOf(block.getBlockSize()) + ", " - + String.valueOf(blockSize)); - return false; - } - if(block.isUsingDateTime() != withDateTime){ - System.out.println("invalid block DateTime: " - + String.valueOf(block.isUsingDateTime()) + ", " - + String.valueOf(withDateTime)); - return false; - } - if(block.getTimeout() != timeout){ - System.out.println("invalid block timeout: " - + String.valueOf(block.getTimeout()) + ", " - + String.valueOf(timeout)); - return false; - } - System.out.println("Block: " + block.getName()); - System.out.println("using datetime: " + String.valueOf(block.isUsingDateTime())); - System.out.println("timeout: " + String.valueOf(block.getTimeout())); - System.out.println("block size: " + String.valueOf(block.getBlockSize())); - return true; - } - - private static void - testGetCalls(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded, DataIndexException, InvalidItemOrTopic { - Random rand = new Random(Double.doubleToLongBits(Math.random())); - String dtSuffix = withDateTime ? "WithDateTime" : ""; - Set items = block.getItems(); - Set topics = block.getTopics(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - int occ = block.getStreamOccupancy(item,topic); - List iParams = new ArrayList<>(); - switch(occ){ - case 0: - System.out.println("testGetCalls (" + item + "," + topic + ") occ == 0"); - continue; - case 1: - iParams.add(0); //first - break; - case 2: - iParams.add(0); //first - iParams.add(occ-1); //last - break; - default: - iParams.add(0); //first - iParams.add(occ-1); //last - iParams.add(1 + rand.nextInt(occ-1)); // something in between - } - for(int i : iParams) { - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - printGet(block, item, topic, i, "getLong" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - printGet(block, item, topic, i, "getDouble" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_STRING: - printGet(block, item, topic, i, "getString" + dtSuffix); - break; - } - } - } - } - - } - - private static void - testGetExcNull(DataBlock block, String item, Topic topic, String fname) - throws CLibException, LibraryNotLoaded, NoSuchMethodException, - IllegalAccessException, InvocationTargetException { - String outBndl = "(" + fname + "," + item + "," + topic + ")"; - try { - //test null behavior of get calls - T l = genericGet(block,item,topic,-1,fname); - if(l != null){ - throw new RuntimeException("get from indx -1 failed to return null: " + outBndl ); - }else{ - System.out.println("get from indx -1 successfully returned null: " + outBndl); - } - //test exc behavior of get calls - try { - l = genericGet(block, item, topic, block.getBlockSize() + 1, fname); - }catch(InvocationTargetException e){ - Throwable t = e.getCause(); - if (t.getClass().equals(DataIndexException.class)) { - throw (DataIndexException)t; - } - } - throw new RuntimeException("DataIndexException test failed for get failed: " + outBndl); - }catch( DataIndexException e){ - System.out.println("Successfully caught DataIndexException for get: " + outBndl ); - } - } - - private static void - testGetCallsExcNullBehavior(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded, NoSuchMethodException, IllegalAccessException, - InvocationTargetException - { - String dtSuffix = withDateTime ? "WithDateTime" : ""; - Set items = block.getItems(); - Set topics = block.getTopics(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - testGetExcNull(block,item,topic,"getLong" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - testGetExcNull(block,item,topic,"getDouble" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_STRING: - testGetExcNull(block,item,topic,"getString" + dtSuffix); - break; - } - } - } - - } - - private static void - testStreamSnapshotCalls(DataBlock block, int sz, boolean withDateTime) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - if(sz < 1) { - throw new IllegalArgumentException("sz < 1"); - } - String dtSuffix = withDateTime ? "WithDateTime" : ""; - Set items = block.getItems(); - Set topics = block.getTopics(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - int occ = block.getStreamOccupancy(item,topic); - if(occ < sz){ - System.out.println("testStreamSnapshotCalls (" + item + "," + topic - + ") occ < sz "); - continue; - } - for(int[] ii : new int[][]{{0,sz-1}, {occ-sz,-1}}) { - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - printGetStreamSnapshot(block,item,topic,ii[1],ii[0], - "getStreamSnapshotLongs" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - printGetStreamSnapshot(block,item,topic,ii[1],ii[0], - "getStreamSnapshotDoubles" + dtSuffix); - break; - case TOSDataBridge.TOPIC_IS_STRING: - printGetStreamSnapshot(block,item,topic,ii[1],ii[0], - "getStreamSnapshotStrings" + dtSuffix); - break; - } - } - } - } - - } - - private static void - testStreamSnapshotFromMarkerCalls(DataBlock block, int passes, int wait, boolean withDateTime, - boolean ignoreDirty) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - if(passes < 1) { - throw new IllegalArgumentException("passes < 1"); - } - String callSuffix = withDateTime ? "WithDateTime" : ""; - callSuffix = callSuffix + (ignoreDirty ? "IgnoreDirty" : ""); - Set items = block.getItems(); - Set topics = block.getTopics(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - int occ = block.getStreamOccupancy(item,topic); - for(int i = 0; i < passes; ++i) { - System.out.print("PASS #" + String.valueOf(i+1) + " :: "); - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - printGetStreamSnapshotFromMarker(block,item,topic,0, - "getStreamSnapshotLongsFromMarker" + callSuffix); - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - printGetStreamSnapshotFromMarker(block,item,topic,0, - "getStreamSnapshotDoublesFromMarker" + callSuffix); - break; - case TOSDataBridge.TOPIC_IS_STRING: - printGetStreamSnapshotFromMarker(block,item,topic,0, - "getStreamSnapshotStringsFromMarker" + callSuffix); - break; - } - try { - Thread.sleep(wait); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - } - - private static void - testNFromMarkerCalls(DataBlock block, int passes, int wait, boolean withDateTime, - boolean ignoreDirty) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - if(passes < 1) { - throw new IllegalArgumentException("passes < 1"); - } - Random rand = new Random(Double.doubleToLongBits(Math.random())); - String callSuffix = withDateTime ? "WithDateTime" : ""; - callSuffix = callSuffix + (ignoreDirty ? "IgnoreDirty" : ""); - Set items = block.getItems(); - Set topics = block.getTopics(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - for(int i = 0; i < passes; ++i) { - int n = 1 + rand.nextInt(99); - System.out.print("PASS #" + String.valueOf(i+1) + " :: n = " + String.valueOf(n) + " :: "); - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - printGetStreamSnapshotFromMarker(block,item,topic,n, - "getNLongsFromMarker" + callSuffix); - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - printGetStreamSnapshotFromMarker(block,item,topic,n, - "getNDoublesFromMarker" + callSuffix); - break; - case TOSDataBridge.TOPIC_IS_STRING: - printGetStreamSnapshotFromMarker(block,item,topic,n, - "getNStringsFromMarker" + callSuffix); - break; - } - try { - Thread.sleep(wait); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - } - - private static void - testDirtyMarkerExceptions(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - Set items = block.getItems(); - Set topics = block.getTopics(); - int sz = block.getBlockSize(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - int occ = block.getStreamOccupancy(item,topic); - if(occ < 2) { - System.err.println("occ < 2, can't test DirtyMarkerException: " + item + "," + topic); - continue; - } - block.setBlockSize(occ); //force a dirty marker by shrinking block - if( !block.isDirty(item,topic) ){ - try { - Thread.sleep(SLEEP_PERIOD); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if( !block.isDirty(item,topic) ) { - System.err.println("stream did not become dirty after wait: " + item + "," + topic); - continue; - } - } - try { - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotLongsFromMarkerWithDateTime(item, topic); - }else { - block.getStreamSnapshotLongsFromMarker(item, topic); - } - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotDoublesFromMarkerWithDateTime(item, topic); - }else { - block.getStreamSnapshotDoublesFromMarker(item, topic); - } - break; - case TOSDataBridge.TOPIC_IS_STRING: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotStringsFromMarkerWithDateTime(item, topic); - }else { - block.getStreamSnapshotStringsFromMarker(item, topic); - } - break; - } - System.out.println("Failed to throw/catch DirtyMarkerExeption: " + item + "," + topic); - }catch(DirtyMarkerException e) { - System.out.println("Successfully caught DirtyMarkerException: " + item + "," + topic); - } - } - } - block.setBlockSize(sz); - } - - private static void - testDataIndexExceptions(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded, InvalidItemOrTopic { - Set items = block.getItems(); - Set topics = block.getTopics(); - int sz = block.getBlockSize(); - for(Topic topic : topics){ - int tType = TOSDataBridge.getTopicType(topic); - for(String item : items) { - try { - switch (tType) { - case TOSDataBridge.TOPIC_IS_LONG: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotLongsWithDateTime(item, topic,sz); - }else { - block.getStreamSnapshotLongs(item, topic, sz-1, -sz -1); - } - break; - case TOSDataBridge.TOPIC_IS_DOUBLE: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotDoublesWithDateTime(item, topic, sz-1, -sz -1); - }else { - block.getStreamSnapshotDoubles(item, topic,sz); - } - break; - case TOSDataBridge.TOPIC_IS_STRING: - if(withDateTime) { - ((DataBlockWithDateTime)block) - .getStreamSnapshotStringsWithDateTime(item, topic,sz); - }else { - block.getStreamSnapshotStrings(item, topic, sz-1, -sz -1); - } - break; - } - System.out.println("Failed to throw/catch DataIndexException: " + item + "," + topic); - }catch(DataIndexException e) { - System.out.println("Successfully caught DataIndexException: " + item + "," + topic); - } - } - } - } - - @SuppressWarnings("unchecked") - private static void - testTotalFrameCalls(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded - { - Method m; - try { - m = block.getClass().getMethod((withDateTime ? "getTotalFrameWithDateTime" : "getTotalFrame")); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - return; - } - Map> frame; - try{ - frame = (Map>)m.invoke(block); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - return; - } - for(String i : frame.keySet()){ - System.out.print(String.format("%-12s ::: ", i)); - Map row = frame.get(i); - for(Topic t : row.keySet()){ - if(withDateTime){ - @SuppressWarnings("unchecked") - DateTimePair p = (DateTimePair)row.get(t); - System.out.print(String.format("%s %s %s ", t, p.first, p.second.toString())); - }else { - System.out.print(String.format("%s %s ", t, row.get(t))); - } - } - System.out.println(); - } - } - - @SuppressWarnings("unchecked") - private static void - testItemFrameCalls(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded - { - Method m; - try { - m = block.getClass().getMethod((withDateTime ? "getItemFrameWithDateTime" : "getItemFrame"), - Topic.class); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - return; - } - - Set topics = block.getTopics(); - for(Topic topic : topics) { - Map frame; - try{ - frame = (Map)m.invoke(block, topic); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - return; - } - System.out.print(String.format("%-12s ::: ", topic)); - for(String item : frame.keySet()){ - if(withDateTime){ - @SuppressWarnings("unchecked") - DateTimePair p = (DateTimePair)frame.get(item); - System.out.print(String.format("%s %s %s ", item, p.first, - p.second.toString())); - }else { - System.out.print(String.format("%s %s ", item, frame.get(item))); - } - } - System.out.println(); - } - } - - @SuppressWarnings("unchecked") - private static void - testTopicFrameCalls(DataBlock block, boolean withDateTime) - throws CLibException, LibraryNotLoaded - { - Method m; - try { - m = block.getClass().getMethod((withDateTime ? "getTopicFrameWithDateTime" : "getTopicFrame"), - String.class); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - return; - } - - Set items = block.getItems(); - for(String item : items) { - Map frame; - try{ - frame = (Map)m.invoke(block, item); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - return; - } - System.out.print(String.format("%-12s ::: ", item)); - for(Topic topic: frame.keySet()){ - if(withDateTime){ - DateTimePair p = (DateTimePair)frame.get(topic); - System.out.print(String.format("%s %s %s ", topic, p.first, - p.second.toString())); - }else { - System.out.print(String.format("%s %s ", topic, frame.get(topic))); - } - } - System.out.println(); - } - } - - @SuppressWarnings("unchecked") - private static T - genericGet(DataBlock block, String item, Topic topic, int indx, String mname) - throws CLibException, LibraryNotLoaded, NoSuchMethodException, - InvocationTargetException, IllegalAccessException, DataIndexException - { - Method m = block.getClass().getMethod(mname,String.class,Topic.class,int.class); - return (T)m.invoke(block,item,topic,indx); - } - - private static void - printGet(DataBlock block, String item, Topic topic, int indx, String mname) - throws CLibException, LibraryNotLoaded, DataIndexException { - try { - T r1 = genericGet(block,item,topic,indx,mname); - System.out.println(mname + "(" + item + "," + topic + "," + String.valueOf(indx) - + "): " + r1.toString()); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - } - } - - private static void - printGetStreamSnapshot(DataBlock block, String item, Topic topic, int end, int beg, String mname) - throws CLibException, LibraryNotLoaded - { - Method m; - try { - m = block.getClass().getMethod(mname,String.class,Topic.class,int.class,int.class); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - return; - } - try { - @SuppressWarnings("unchecked") - List ret = (List)m.invoke(block,item,topic,end,beg); - System.out.println(mname + "(" + item + "," + topic + "," + String.valueOf(beg) - + " to " + String.valueOf(end) + "): "); - Collections.reverse(ret); - for (T r : ret){ - System.out.println( r.toString()); - } - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - } - - private static void - printGetStreamSnapshotFromMarker(DataBlock block, String item, Topic topic, int beg, String mname) - throws CLibException, LibraryNotLoaded - { - Method m; - try { - m = block.getClass().getMethod(mname,String.class,Topic.class,int.class); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - return; - } - try { - @SuppressWarnings("unchecked") - List ret = (List)m.invoke(block,item,topic,beg); - System.out.println(mname + "(" + item + "," + topic + "," + String.valueOf(beg) + "): "); - Collections.reverse(ret); - for (T r : ret){ - System.out.println( r.toString()); - } - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - } - - private static void - printBlockItemsTopics(DataBlock block) throws CLibException, LibraryNotLoaded { - System.out.println("Block: " + block.getName()); - for(String item : block.getItems()) { - System.out.println("item: " + item); - } - for(Topic topic : block.getTopics()) { - System.out.println("topic: " + topic); - } - for(String item : block.getItemsPreCached()) { - System.out.println("item(pre-cache): " + item); - } - for(Topic topic : block.getTopicsPreCached()) { - System.out.println("topic(pre-cache): " + topic); - } - System.out.println(); - } - -} diff --git a/test/python/engine_test.py b/test/python/engine_test.py deleted file mode 100644 index 269a3d6..0000000 --- a/test/python/engine_test.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -import tosdb - -from random import choice as _choice, randint as _randint -from time import sleep as _sleep, gmtime as _gmtime -from argparse import ArgumentParser as _ArgumentParser -from platform import system as _system - -NEXT_YEAR = _gmtime().tm_year + 1 - -#expires 01/19/2018 -SPY_CALL = '.SPY180119C250' - -T_ITEMS = ['SPY','GOOG','QQQ','XLU','XLI','XLE','XLF','XLB','XLP','XLK', - 'MSFT','IWM',SPY_CALL] - -T_TOPICS = [t.val for t in tosdb.TOPICS] -T_TOPICS.remove('NULL_TOPIC') - -BSIZE = 100 -BTIMEOUT = 3000 -MAX_ADD = 7 - -args=None - -def init(): - global vaddr - - print() - if args.path: - print('+ lib path:', args.path) - elif args.root: - print('+ lib root:', args.root) - else: - print('error: tosdb_test.py requires a --root or --path arg for library') - return False - - if _system() not in ["Windows","windows","WINDOWS"]: - print("error: can only run '--virtual' on non Windows systems") - return False - try: - tosdb.init(dllpath=args.path, root=args.root) - print('+ init successful') - except Exception as e: - print('- init failed:',str(e)) - return False - - print() - return True - -def choice_without_replace(i, n, o=[]): - i = list(i) - o = list(o) - if len(i) == 0 or len(o) >= n: - return o - else: - v = _choice(i) - i.remove(v) - o.append(v) - return choice_without_replace(i,n,o) - - -def test(n,pause): - is_conn = tosdb.connected() - if not is_conn: - print("*** COULD NOT CONNECT... exiting ***\n") - exit(1) - - print("-- CREATE BLOCK --\n") - b1 = tosdb.TOSDB_DataBlock(BSIZE,True,BTIMEOUT) - - for _ in range(n): - nt = _randint(1,MAX_ADD) - ni = _randint(1,MAX_ADD) - topics = choice_without_replace(T_TOPICS, nt) - items = choice_without_replace(T_ITEMS, ni) - print('ADD ITEMS: ', str(items)) - b1.add_items(*items) - print('ADD TOPICS: ', str(topics)) - b1.add_topics(*topics) - - _sleep(pause) - print() - print(b1) - _sleep(pause) - - items = b1.items() - ni = len(items) - if ni > 1: - ni = _randint(1,ni-1) - items = choice_without_replace(items, ni) - print('REMOVE ITEMS: ', str(items)) - b1.remove_items(*items) - - topics = b1.topics() - nt = len(topics) - if nt > 1: - nt = _randint(1, nt-1) - topics = choice_without_replace(topics, nt) - print('REMOVE TOPICS: ', str(topics)) - b1.remove_topics(*topics) - - _sleep(pause) - print() - print(b1) - _sleep(pause) - - - -if __name__ == '__main__': - parser = _ArgumentParser() - parser.add_argument('--virtual', type=str, - help='test virtual interface via "address port [timeout]"') - parser.add_argument('num', help='number of runs',type=int, default=100) - parser.add_argument('pause', help='seconds to pause',choices=['.5','1','3','5']) - parser.add_argument('--root', help='root directory to search for the library') - parser.add_argument('--path', help='the exact path of the library') - args = parser.parse_args() - if init(): - test(args.num, float(args.pause)) - else: - exit(1) - - diff --git a/test/python/marker_test.py b/test/python/marker_test.py deleted file mode 100644 index eb3cf3c..0000000 --- a/test/python/marker_test.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -import tosdb -from sys import stderr as _stderr, maxsize as _maxsize -from math import log as _log -from random import choice as _choice, shuffle as _shuffle, randint as _randint -from time import sleep as _sleep -from argparse import ArgumentParser as _ArgumentParser -from platform import system as _system - -N=1000 -MAX_SEC_PER_TEST=30 -TOPICS=['LAST','VOLUME','LAST_SIZE','LASTX'] -ITEMS=['SPY','/ES:XCME'] - - -def test(func_name, block, args, passes, wait): - mdata = [] - for i in range(passes): - print('+ ', func_name, args[0], args[1], str(i)) - _sleep(wait) - d = getattr(block,func_name)(*args) - if d: - d.reverse() - mdata.extend(d) - else: - print('- ', 'NONE') - sdata = block.stream_snapshot(args[0], args[1], date_time=args[2]) - sdata.reverse() - fout_name = 'out/' + func_name + '-' + args[0].replace('/','+') + '-' \ - + args[1] + '-' + str(args[2]) + '-' + str(passes) + '-' \ - + str(wait) + '.test' - with open(fout_name, 'w') as fout: - for i in range(max(len(mdata),len(sdata))): - _write_line(fout, mdata, sdata, i, args[2]) - - -def test_marker_func(fname, *args): - block = tosdb.TOSDB_DataBlock(N,True) - block.add_items(*ITEMS) - block.add_topics(*TOPICS) - topics = list(TOPICS) - _shuffle(topics) - for topic in topics: - item = _choice(ITEMS) - date_time = _choice([True,False]) - passes = _randint(3,10) - wait = int(MAX_SEC_PER_TEST/passes) - print("+ TEST", fname, str((item,topic,date_time,passes,wait))) - test(fname, block, (item, topic, date_time) + args, passes, wait) - block.close() - - -def _write_line(fout, mdata, sdata, i, date_time): - def _try_write(lst, i, w, *ii): - try: - v = lst[i][ii[0]] if ii else lst[i] - fout.write( str(v).ljust(w) ) - except IndexError: - fout.write( "".ljust(w) ) - if date_time: - _try_write(mdata,i,10,0) - fout.write(' ') - _try_write(sdata,i,10,0) - fout.write('\t') - _try_write(mdata,i,10,1) - fout.write(' ') - _try_write(sdata,i,10,1) - else: - _try_write(mdata,i,50) - fout.write('\t') - _try_write(sdata,i,50) - fout.write('\n') - - -if __name__ == '__main__': - parser = _ArgumentParser() - parser.add_argument('--root', help='root directory to search for the library') - parser.add_argument('--path', help='the exact path of the library') - args = parser.parse_args() - if not args.path and not args.root: - print('error: --root or --path arg requried') - exit(1) - try: - res = tosdb.init(dllpath=args.path, root=args.root) - except Exception as e: - print('error: exception during init:', str(e)) - exit(1) - print("+ BEGIN") - test_marker_func('stream_snapshot_from_marker') - test_marker_func('n_from_marker', N) - tosdb.clean_up() - print("+ END") - - - diff --git a/test/python/tosdb_test.py b/test/python/tosdb_test.py deleted file mode 100644 index d7471a4..0000000 --- a/test/python/tosdb_test.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright (C) 2014 Jonathon Ogden < jeog.dev@gmail.com > -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License, -# 'LICENSE.txt', along with this program. If not, see -# . - -import tosdb -from sys import stderr as _stderr, maxsize as _maxsize -from math import log as _log -from random import choice as _rchoice -from time import sleep as _sleep -from argparse import ArgumentParser as _ArgumentParser -from platform import system as _system - -T_ITEMS = ['SPY','GOOG','QQQ'] -T_TOPICS = ['LAST','LASTX','VOLUME','LAST_SIZE','CUSTOM9','PUT_CALL_RATIO'] - -bsize = 100 -args=None -vaddr=None - -_ladj = lambda s : s.ljust(20) - -BASE_ADMIN_CALLS = ['connected','get_block_limit','set_block_limit','type_bits','type_string'] - - -def init(): - global vaddr - - print() - if args.path: - print('+ lib path:', args.path) - elif args.root: - print('+ lib root:', args.root) - else: - print('error: tosdb_test.py requires a --root or --path arg for library') - return False - - if args.virtual: - raw = args.virtual.split(' ') - try: - v = ((raw[0],int(raw[1])),) - except IndexError: - print("usage: tosdb_test --virtual 'address port [timeout]'",file=_stderr) - return False - vaddr = v[0] - v += (int(raw[2]),) if len(raw) > 2 else () - v = v[:1] + (args.auth,) + v[1:] - try: - tosdb.admin_init(*v) - print('+ admin_init successful') - except Exception as e: - print('- admin_init failed:', str(e)) - return False - try: - res = tosdb.vinit(dllpath=args.path, root=args.root) - print('+ vinit successful') - except Exception as e: - print('- vinit failed:',str(e)) - return False - else: - if _system() not in ["Windows","windows","WINDOWS"]: - print("error: can only run '--virtual' on non Windows systems") - return False - try: - res = tosdb.init(dllpath=args.path, root=args.root) - print('+ init successful') - except Exception as e: - print('- init failed:',str(e)) - return False - - print() - return True - - -def test_admin(): - global bsize - if args.virtual: - admin_calls = {k:getattr(tosdb,'v'+k) for k in BASE_ADMIN_CALLS} - else: - admin_calls = {k:getattr(tosdb,k) for k in BASE_ADMIN_CALLS} - - is_conn = admin_calls['connected']() - if not is_conn: - print("*** COULD NOT CONNECT... exiting ***\n") - exit(1) - - print(_ladj("CONNECTED:"), admin_calls['connected']() ) - - blim = admin_calls['get_block_limit']() - if blim < 2: # in case someone else lowered this - admin_calls['set_block_limit'](2) # (need to be able to half it w/o hitting 0) - blim = admin_calls['get_block_limit']() - print(_ladj("GET BLOCK LIMIT:"), blim ) - - print(_ladj("HALVE BLOCK LIMIT:")) - admin_calls['set_block_limit'](int(blim/2)) - print(_ladj("GET BLOCK LIMIT:"), admin_calls['get_block_limit']() ) - - print() - print(_ladj("-- TOPIC BITS/STRINGS --")) - for t in T_TOPICS: - print(' ', _ladj(t), str(admin_calls['type_bits'](t)).ljust(4), - _ladj(admin_calls['type_string'](t))) - - -def test_block(b1): - - for i,v in b1.info().items(): - print(' ',_ladj(i), v) - - print() - bsize = b1.get_block_size() - print(_ladj("GET BLOCK SIZE"), bsize) - - print(_ladj("DOUBLE BLOCK SIZE")) - bsize *= 2 - b1.set_block_size(bsize) - print(_ladj("GET BLOCK SIZE"), b1.get_block_size()) - - print() - print(_ladj("-- ADD ITEMS --")) - b1.add_items(*T_ITEMS) - print(' ',_ladj("ITEMS:"), b1.items()) - print(' ',_ladj("ITEMS_PRECACHED:"), b1.items_precached()) - print(' ',_ladj("TOPICS:"), b1.topics()) - print(' ',_ladj("TOPICS_PRECACHED:"), b1.topics_precached()) - - print() - print(_ladj("-- ADD TOPICS --")) - b1.add_topics(*T_TOPICS) - print(' ',_ladj("ITEMS:"), b1.items()) - print(' ',_ladj("ITEMS_PRECACHED:"), b1.items_precached()) - print(' ',_ladj("TOPICS:"), b1.topics()) - print(' ',_ladj("TOPICS_PRECACHED:"), b1.topics_precached()) - - if sorted(b1.items()) != sorted(T_ITEMS): - print("!! ITEMS DON'T MATCH !!", file=_stderr) - if sorted(b1.topics()) != sorted(T_TOPICS): - print("!! TOPICS DON'T MATCH !!", file=_stderr) - - print() - rt = T_TOPICS.pop() - print("-- REMOVE TOPIC: " + rt + " --") - b1.remove_topics(rt) - - if sorted(b1.topics()) != sorted(T_TOPICS): - print("!! TOPICS DON'T MATCH !!", file=_stderr) - else: - print(' ',_ladj("ITEMS:"), b1.items()) - print(' ',_ladj("TOPICS:"), b1.topics()) - - print() - ri = T_ITEMS.pop() - print("-- REMOVE ITEM: " + ri + " --") - b1.remove_items(ri) - - if sorted(b1.items()) != sorted(T_ITEMS): - print("!! ITEMS DON'T MATCH !!", file=_stderr) - else: - print(' ',_ladj("ITEMS:"), b1.items()) - print(' ',_ladj("TOPICS:"), b1.topics()) - - print() - print("Sleep 5 seconds for historical data...",flush=1,end='') - for _ in range(10): - _sleep(.5) - print('...',flush=1,end='') - print() - - print() - print("-- GET (RANDOM) --") - for i in range(0,10,2): - ii = _rchoice(T_ITEMS) - tt = _rchoice(T_TOPICS) - dd = _rchoice([1,0]) - print(' ',ii.ljust(6),_ladj(tt),str(bool(dd)).ljust(6),end=' ') - try: - v = b1.get(ii,tt,dd,i) - print(str(v)) - except tosdb._common.TOSDB_DataError: - print("no data yet") - - ii = T_ITEMS[0] # SPY - tt = "LAST" - - print() - print("-- STREAM SNAPSHOT --", ii,tt,'date_time=True','end=3') - vdt = b1.stream_snapshot(ii,tt,True,3) - for k in vdt: - print(' ',str(k[1]),str(k[0])) - - print() - print("-- STREAM SNAPSHOT --", ii,tt,'date_time=False','end=10') - vdt = b1.stream_snapshot(ii,tt,False,10) - print(' ',str(vdt)) - - print() - print("-- STREAM SNAPSHOT FROM MARKER --",ii,tt, 'date_time=True','beg = 0') - for i in range(3): - print(' ',"(sleep 2 second(s))") - _sleep(2) - ssfm = b1.stream_snapshot_from_marker(ii,tt,True, 0) - if(ssfm): - ssfm.reverse() - for k in ssfm: - print(' ',str(k[1]), str(k[0])) - else: - print(' ...') - - print() - print("-- STREAM SNAPSHOT FROM MARKER --",ii,tt, 'date_time=False','beg = 1') - for i in range(3): - print(' ',"(sleep 2 second(s))") - _sleep(2) - ssfm = b1.stream_snapshot_from_marker(ii,tt,False, 1) - if(ssfm): - ssfm.reverse() - for k in ssfm: - print(' ',str(k)) - else: - print(' ...') - - print() - print("-- ITEM FRAME --",tt,'date_time=False, labels=True') - print(' ',b1.item_frame(tt,False,True)) - - print() - print("-- ITEM FRAME --",tt,'date_time=True, labels=True') - print(' ',b1.item_frame(tt,True,True)) - - print() - print("-- ITEM FRAME --",tt,'date_time=False, labels=False') - print(' ',b1.item_frame(tt,False,False)) - - print() - print("-- ITEM FRAME --",tt,'date_time=True, labels=False') - print(' ',b1.item_frame(tt,True,False)) - - - print() - print("-- TOPIC FRAME --",ii,'date_time=False, labels=True') - print(' ',b1.topic_frame(ii,False,True)) - - print() - print("-- TOPIC FRAME --",ii,'date_time=True, labels=True') - print(' ',b1.topic_frame(ii,True,True)) - - print() - print("-- TOPIC FRAME --",ii,'date_time=False, labels=False') - print(' ',b1.topic_frame(ii,False,False)) - - print() - print("-- TOPIC FRAME --",ii,'date_time=True, labels=False') - print(' ',b1.topic_frame(ii,True,False)) - - if not args.virtual: - print() - print("-- TOTAL FRAME --",ii,'date_time=False, labels=True') - print(' ',b1.total_frame(False,True)) - - print() - print("-- TOTAL FRAME --",ii,'date_time=True, labels=True') - print(' ',b1.total_frame(True,True)) - - print() - print("-- TOTAL FRAME --",ii,'date_time=False, labels=False') - print(' ',b1.total_frame(False,False)) - - print() - print("-- TOTAL FRAME --",ii,'date_time=True, labels=False') - print(' ',b1.total_frame(True,False)) - print() - - - -if __name__ == '__main__': - parser = _ArgumentParser() - parser.add_argument('--virtual', type=str, - help='test virtual interface via "address port [timeout]"') - parser.add_argument('--root', help='root directory to search for the library') - parser.add_argument('--path', help='the exact path of the library') - parser.add_argument('--auth', help='password to use for authentication') - args = parser.parse_args() - - if init(): - test_admin() - if args.virtual: - print() - print("-- CREATE (VIRTUAL) BLOCK --") - b1 = tosdb.VTOSDB_DataBlock(vaddr,args.auth,bsize,True,2000) - test_block(b1) - print() - print("-- CREATE (VIRTUAL, THREAD SAFE) BLOCK --") - b2 = tosdb.VTOSDB_ThreadSafeDataBlock(vaddr,args.auth,bsize,True,2000) - test_block(b2) - else: - print() - print("-- CREATE BLOCK --") - b1 = tosdb.TOSDB_DataBlock(bsize,True,2000) - test_block(b1) - print() - print("-- CREATE (THREAD SAFE) BLOCK --") - b2 = tosdb.TOSDB_ThreadSafeDataBlock(bsize,True,2000) - test_block(b2) - else: - exit(1) - - diff --git a/tosdb-setup-debug.bat b/tosdb-setup-debug.bat deleted file mode 100644 index 54ff18d..0000000 --- a/tosdb-setup-debug.bat +++ /dev/null @@ -1,104 +0,0 @@ -@echo off - -NET FILE 1>NUL 2>NUL & - -IF %ERRORLEVEL% NEQ 0 ( - echo tosdb-setup-debug.bat must be run as administrator. - EXIT /B 1 -) - -set bCRTfiles=false - -IF EXIST C:/Windows/System32/msvcr110d.dll ( - IF EXIST C:/Windows/System32/msvcp110d.dll ( - set bCRTfiles=true - ) -) - - -IF /I "%2"=="admin" ( - set servCmd= --admin %3 -) else ( - set servCmd= %2 -) - -IF "%1"=="x64" ( - set vcRedist=vcredist_x64.exe - set servBin=%cd%\\bin\\Debug\\x64\\tos-databridge-serv-x64_d.exe - set engBin=%cd%\\bin\\Debug\\x64\\tos-databridge-engine-x64_d.exe - set createCmd=%cd%\\bin\\Debug\\x64\\tos-databridge-serv-x64_d.exe%servCmd% -) else ( - IF "%1"=="x86" ( - IF EXIST C:/Windows/SysWOW64 set bCRTfiles=false - IF EXIST C:/Windows/SysWOW64/msvcr110d.dll ( - IF EXIST C:/Windows/SysWOW64/msvcp110d.dll ( - set bCRTfiles=true - ) - ) - set vcRedist=vcredist_x86.exe - set servBin=%cd%\\bin\\Debug\\Win32\\tos-databridge-serv-x86_d.exe - set engBin=%cd%\\bin\\Debug\\Win32\\tos-databridge-engine-x86_d.exe - set createCmd=%cd%\bin\Debug\Win32\tos-databridge-serv-x86_d.exe%servCmd% - ) else ( - echo - Invalid Command Line Argument. Use 'x86' or 'x64' to signify which build to setup. - EXIT /B 1 - ) -) - - -echo + Checking for VC++ Redistributable Files ... -IF %bCRTfiles%==false ( - echo ++ Not Found. Installing VC++ Redistributable ... - "%vcRedist%" - IF ERRORLEVEL 1 ( - echo - Installation failed, exiting setup... - EXIT /B 1 - ) -) - -echo + Checking for binaries... -IF NOT EXIST %servBin% ( - echo - Can't find "%servBin%", exiting setup... - EXIT /B 1 -) -IF NOT EXIST %engBin% ( - echo - Can't find "%engBin%", exiting setup... - EXIT /B 1 -) - -echo + stop old TOSDataBridge Service ... -SC stop TOSDataBridge 1>NUL 2>NUL -timeout /t 3 /nobreak -echo. - -echo + delete old TOSDataBridge Service ... -SC delete TOSDataBridge 1>NUL 2>NUL -timeout /t 3 /nobreak -echo. - -echo + Creating TOSDataBridge Service ... -echo + %createCmd% -SC create TOSDataBridge binPath= "%createCmd%" - -IF ERRORLEVEL 1 ( - echo - SC create failed, exiting setup... - echo - Be sure 'Process Explorer' is not running and you are not accessing the old service. - echo - Try running this setup script again. If that fails you may need to manually stop/delete old service: - echo - SC stop TOSDataBridge - echo - SC delete TOSDataBridge - echo - Then run the setup script again. - echo - (If that fails please report the issue @ github.com/jeog/TOSDataBridge/issues) - EXIT /B 1 -) - -EXIT /B 0 - - - - - - - - - - diff --git a/tosdb-setup.bat b/tosdb-setup.bat deleted file mode 100644 index 5071555..0000000 --- a/tosdb-setup.bat +++ /dev/null @@ -1,104 +0,0 @@ -@echo off - -NET FILE 1>NUL 2>NUL & - -IF %ERRORLEVEL% NEQ 0 ( - echo tosdb-setup.bat must be run as administrator. - EXIT /B 1 -) - -set bCRTfiles=false - -IF EXIST C:/Windows/System32/msvcr110.dll ( - IF EXIST C:/Windows/System32/msvcp110.dll ( - set bCRTfiles=true - ) -) - - -IF /I "%2"=="admin" ( - set servCmd= --admin %3 -) else ( - set servCmd= %2 -) - -IF "%1"=="x64" ( - set vcRedist=vcredist_x64.exe - set servBin=%cd%\\bin\\Release\\x64\\tos-databridge-serv-x64.exe - set engBin=%cd%\\bin\\Release\\x64\\tos-databridge-engine-x64.exe - set createCmd=%cd%\\bin\\Release\\x64\\tos-databridge-serv-x64.exe%servCmd% -) else ( - IF "%1"=="x86" ( - IF EXIST C:/Windows/SysWOW64 set bCRTfiles=false - IF EXIST C:/Windows/SysWOW64/msvcr110.dll ( - IF EXIST C:/Windows/SysWOW64/msvcp110.dll ( - set bCRTfiles=true - ) - ) - set vcRedist=vcredist_x86.exe - set servBin=%cd%\\bin\\Release\\Win32\\tos-databridge-serv-x86.exe - set engBin=%cd%\\bin\\Release\\Win32\\tos-databridge-engine-x86.exe - set createCmd=%cd%\bin\Release\Win32\tos-databridge-serv-x86.exe%servCmd% - ) else ( - echo - Invalid Command Line Argument. Use 'x86' or 'x64' to signify which build to setup. - EXIT /B 1 - ) -) - - -echo + Checking for VC++ Redistributable Files ... -IF %bCRTfiles%==false ( - echo ++ Not Found. Installing VC++ Redistributable ... - "%vcRedist%" - IF ERRORLEVEL 1 ( - echo - Installation failed, exiting setup... - EXIT /B 1 - ) -) - -echo + Checking for binaries... -IF NOT EXIST %servBin% ( - echo - Can't find "%servBin%", exiting setup... - EXIT /B 1 -) -IF NOT EXIST %engBin% ( - echo - Can't find "%engBin%", exiting setup... - EXIT /B 1 -) - -echo + stop old TOSDataBridge Service ... -SC stop TOSDataBridge 1>NUL 2>NUL -timeout /t 3 /nobreak -echo. - -echo + delete old TOSDataBridge Service ... -SC delete TOSDataBridge 1>NUL 2>NUL -timeout /t 3 /nobreak -echo. - -echo + Creating TOSDataBridge Service ... -echo + %createCmd% -SC create TOSDataBridge binPath= "%createCmd%" - -IF ERRORLEVEL 1 ( - echo - SC create failed, exiting setup... - echo - Be sure 'Process Explorer' is not running and you are not accessing the old service. - echo - Try running this setup script again. If that fails you may need to manually stop/delete old service: - echo - SC stop TOSDataBridge - echo - SC delete TOSDataBridge - echo - Then run the setup script again. - echo - (If that fails please report the issue @ github.com/jeog/TOSDataBridge/issues) - EXIT /B 1 -) - -EXIT /B 0 - - - - - - - - - - diff --git a/vcredist_x64.exe b/vcredist_x64.exe deleted file mode 100644 index f830586..0000000 Binary files a/vcredist_x64.exe and /dev/null differ diff --git a/vcredist_x86.exe b/vcredist_x86.exe deleted file mode 100644 index 4d8fbbf..0000000 Binary files a/vcredist_x86.exe and /dev/null differ